feat(web): web server and basic dashboard

This commit is contained in:
Mahdi Dibaiee 2019-04-22 09:49:31 +04:30
parent 8d4010b5dc
commit e18fc7692b
14 changed files with 254 additions and 31 deletions

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
maps
logs
temp_logs
checkpoints
geodata
*.p

View File

@ -135,10 +135,10 @@ class Model():
def predict(self, a):
return np.argmax(self.model.predict(a), axis=1)
def prepare_for_use(self, df=None, batch_size=DEFAULT_BUFFER_SIZE, layers=DEFAULT_LAYERS, out_activation=DEFAULT_OUT_ACTIVATION, loss=DEFAULT_LOSS, optimizer=DEFAULT_OPTIMIZER):
def prepare_for_use(self, df=None, batch_size=DEFAULT_BUFFER_SIZE, layers=DEFAULT_LAYERS, out_activation=DEFAULT_OUT_ACTIVATION, loss=DEFAULT_LOSS, optimizer=DEFAULT_OPTIMIZER, dataset_fn=dataframe_to_dataset_biomes):
if df is None:
df = pd.read_pickle('data.p')
self.prepare_dataset(df, dataframe_to_dataset_biomes, batch_size=batch_size)
self.prepare_dataset(df, dataset_fn, batch_size=batch_size)
self.create_model(layers=layers, out_activation=out_activation)
self.compile(loss=loss, optimizer=optimizer)

90
biomes/train_temp.py Normal file
View File

@ -0,0 +1,90 @@
import fire
import ray
import pandas as pd
import tensorflow as tf
import numpy as np
from tensorflow import keras
from utils import *
from model import Model
from constants import *
CHECKPOINT = 'checkpoints/temp.h5'
SEED = 1
np.random.seed(SEED)
df = pd.read_pickle('data.p')
dataset_size, x_columns, y_columns, dataset = dataframe_to_dataset_temp_precip(df)
batch_size = 5
epochs = 500
def baseline_model():
model = keras.models.Sequential()
params = {
'kernel_initializer': 'lecun_uniform',
'bias_initializer': 'zeros',
}
model.add(keras.layers.Dense(x_columns, input_dim=x_columns, **params, activation='elu'))
model.add(keras.layers.Dense(6, **params, activation='relu'))
model.add(keras.layers.Dense(y_columns, **params))
model.compile(loss='mse', optimizer='adam', metrics=['mae'])
return model
model = baseline_model()
model.summary()
dataset = dataset.shuffle(500)
TRAIN_SIZE = int(dataset_size * 0.85)
TEST_SIZE = dataset_size - TRAIN_SIZE
(training, test) = (dataset.take(TRAIN_SIZE),
dataset.skip(TRAIN_SIZE))
training_batched = training.batch(batch_size).repeat()
test_batched = test.batch(batch_size).repeat()
logger.debug('Model dataset info: size=%s, train=%s, test=%s', dataset_size, TRAIN_SIZE, TEST_SIZE)
# model.load_weights(CHECKPOINT)
def predict():
columns = INPUTS
YEAR = 2000
print(columns)
print(df[0:batch_size])
inputs = df[columns].to_numpy()
inputs = normalize_ndarray(inputs, df[columns].to_numpy())
print(inputs[0:batch_size])
out_columns = []
for season in SEASONS:
out_columns += ['temp_{}_{}'.format(season, YEAR), 'precip_{}_{}'.format(season, YEAR)]
print(out_columns)
out = model.predict(inputs)
print(out)
print(df[out_columns][0:batch_size])
print(denormalize(out, df[out_columns].to_numpy()))
def train():
tfb_callback = tf.keras.callbacks.TensorBoard(batch_size=batch_size, log_dir='temp_logs')
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=CHECKPOINT, monitor='val_loss')
model.fit(training_batched,
batch_size=batch_size,
epochs=epochs,
steps_per_epoch=int(TRAIN_SIZE / batch_size),
validation_data=test_batched,
validation_steps=int(TEST_SIZE / batch_size),
callbacks=[tfb_callback, checkpoint_callback],
verbose=1)
model.save_weights(CHECKPOINT)
# train()
if __name__ == "__main__":
fire.Fire({ 'predict': predict, 'train': train })

View File

@ -16,7 +16,13 @@ def normalize(v, o=None):
o = v
return (v - np.mean(o)) / np.std(o)
def normalize_ndarray(ar, o=None):
def denormalize(v, o=None):
if o is None:
o = v
return (v * np.std(o) + np.mean(o))
def on_ndarray(ar, o=None, fn=None):
if o is None:
o = ar
@ -24,11 +30,17 @@ def normalize_ndarray(ar, o=None):
tr = np.transpose(ar)
to = np.transpose(o)
for i in range(tr.shape[0]):
tr[i] = normalize(tr[i], to[i])
tr[i] = fn(tr[i], to[i])
# transpose back
return np.transpose(tr)
def normalize_ndarray(ar, o=None):
return on_ndarray(ar, o=o, fn=normalize)
def denormalize_ndarray(ar, o=None):
return on_ndarray(ar, o=o, fn=denormalize)
def dataframe_to_dataset_biomes(df):
rows = df.shape[0]
@ -67,24 +79,25 @@ def dataframe_to_dataset_biomes(df):
def dataframe_to_dataset_temp_precip(df):
rows = df.shape[0]
# elevation, distance_to_water, latitude
# season, year
# elevation, distance_to_water, latitude, mean_temp, mean_precip
input_columns = 5
num_classes = 2
# (temp, precip) * 4 seasons
num_classes = 8
tf_inputs = np.empty((0, input_columns))
tf_output = np.empty((0, num_classes))
for year in range(MIN_YEAR, MAX_YEAR + 1):
local_inputs = list(INPUTS)
for idx, season in enumerate(SEASONS):
season_index = idx / len(season)
local_df = df[local_inputs]
local_df.loc[:, 'season'] = pd.Series(np.repeat(season_index, rows), index=local_df.index)
local_df.loc[:, 'year'] = pd.Series(np.repeat(year, rows), index=local_df.index)
all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS]
all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS]
local_df.loc[:, 'mean_temp'] = np.mean(df[all_temps].values)
local_df.loc[:, 'mean_precip'] = np.mean(df[all_precips].values)
output = []
output = all_temps + all_precips
output = ['temp_{}_{}'.format(season, year), 'precip_{}_{}'.format(season, year)]
tf_inputs = np.concatenate((tf_inputs, local_df.values), axis=0)
tf_output = np.concatenate((tf_output, df[output].values), axis=0)

View File

@ -4,6 +4,8 @@ from matplotlib import colors, cm
import scipy.interpolate as interpolate
from scipy import ndimage
import math
from io import BytesIO
import base64
WIDTH = 900
HEIGHT = 450
@ -27,21 +29,8 @@ GROUND_PROPORTION = 1 - WATER_PROPORTION
DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)]
def s(x):
# return x
return -2 * x**3 + 3 * x**2
# class Point(object):
# def __init__(self, x, y, z):
# self.x = x
# self.y = y
# self.z = z
# def is_ground(self):
# return self.z > WATER_LEVEL
# def is_water(self):
# return not(self.is_ground())
def is_ground(value):
return value > WATER_LEVEL
@ -156,7 +145,7 @@ def constant_filter(a):
return max(1, a[0])
return 0
def generate_map(width, height, continents=4):
def generate_map(width=WIDTH, height=HEIGHT, continents=4):
ground = np.zeros((width, height))
ground_size = width * height * GROUND_PROPORTION
@ -181,12 +170,16 @@ def generate_map(width, height, continents=4):
ground = ndimage.gaussian_filter(ground, sigma=4)
ground = ndimage.generic_filter(ground, constant_filter, size=1)
print(np.min(ground), np.max(ground), MAX_ELEVATION)
print(np.unique(ground))
plt.imshow(ground.T, cmap=greys, norm=norm)
plt.show()
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0)
return figfile
if __name__ == "__main__":
generate_map(WIDTH, HEIGHT)
plt.show()

View File

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

7
map-generator/static/bootstrap.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,18 @@
const mapSettings = document.getElementById('map-settings');
//const board = document.getElementById('board');
const map = document.getElementById('map');
const spinner = document.getElementById('spinner');
mapSettings.addEventListener('submit', (e) => {
console.log('form submit');
e.preventDefault();
spinner.classList.remove('d-none');
map.src = '/map?q=' + (new Date()).getTime();
map.classList.add('d-none');
});
map.addEventListener('load', () => {
spinner.classList.add('d-none');
map.classList.remove('d-none');
});

View File

@ -0,0 +1,14 @@
* {
box-sizing: border-box;
}
html, body {
margin: 0;
}
aside {
text-align: center;
width: 300px;
height: 100vh;
}

View File

@ -0,0 +1,65 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>World Map Generator</title>
<link rel='stylesheet' href="{{ url_for('static', filename='style.css') }}">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
</head>
<body class='container-fluid'>
<div class='row'>
<main class='col d-flex justify-content-center align-items-center'>
<!-- <canvas id='board'></canvas> -->
<img src='/map' id='map'>
<div class='spinner-border text-primary' role='status' id='spinner'>
<span class='sr-only'>Loading...</span>
</div>
</main>
<aside class='col-3 px-4 bg-dark text-light text-center py-3'>
<h3>World Map Generator</h3>
<div class='panel px-4'>
<form class='mt-5' id='map-settings'>
<div class='form-group'>
<label name='preset'>Preset</label>
<select id='preset' name='preset' class='form-control'>
<option value='desert'>Desert</option>
<option value='forest'>Forest</option>
</select>
</div>
<div class='form-group'>
<label name='mean-temperature'>Mean Temperature</label>
<input id='mean-temperature' name='mean-temperature' type='range' min='-50' max='50' class='form-control'>
</div>
<div class='form-group'>
<label name='mean-precipitation'>Mean Precipitation</label>
<input id='mean-precipitation' name='mean-precipitation' type='range' min='-50' max='50' class='form-control'>
</div>
<div class='form-group'>
<label name='min-elevation'>Minimum Elevation</label>
<input id='min-elevation' name='min-elevation' type='range' min='-100' max='100' class='form-control'>
</div>
<div class='form-group'>
<label name='max-elevation'>Maximum Elevation</label>
<input id='max-elevation' name='max-elevation' type='range' min='-100' max='100' class='form-control'>
</div>
<button type="submit" class="btn btn-primary">Generate</button>
</form>
</div>
</aside>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
<script src="{{ url_for('static', filename='script.js') }}"></script>
</body>
</html>

13
map-generator/web.py Normal file
View File

@ -0,0 +1,13 @@
from flask import Flask, render_template, make_response, send_file
from index import generate_map
app = Flask(__name__)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/map')
def get_map():
res = send_file(generate_map(), mimetype='image/png')
return res