feat(web): web server and basic dashboard

This commit is contained in:
Mahdi Dibaiee 2019-04-22 09:49:31 +04:30
parent 8d4010b5dc
commit e18fc7692b
14 changed files with 254 additions and 31 deletions

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
maps maps
logs logs
temp_logs
checkpoints checkpoints
geodata geodata
*.p *.p

View File

@ -135,10 +135,10 @@ class Model():
def predict(self, a): def predict(self, a):
return np.argmax(self.model.predict(a), axis=1) return np.argmax(self.model.predict(a), axis=1)
def prepare_for_use(self, df=None, batch_size=DEFAULT_BUFFER_SIZE, layers=DEFAULT_LAYERS, out_activation=DEFAULT_OUT_ACTIVATION, loss=DEFAULT_LOSS, optimizer=DEFAULT_OPTIMIZER): def prepare_for_use(self, df=None, batch_size=DEFAULT_BUFFER_SIZE, layers=DEFAULT_LAYERS, out_activation=DEFAULT_OUT_ACTIVATION, loss=DEFAULT_LOSS, optimizer=DEFAULT_OPTIMIZER, dataset_fn=dataframe_to_dataset_biomes):
if df is None: if df is None:
df = pd.read_pickle('data.p') df = pd.read_pickle('data.p')
self.prepare_dataset(df, dataframe_to_dataset_biomes, batch_size=batch_size) self.prepare_dataset(df, dataset_fn, batch_size=batch_size)
self.create_model(layers=layers, out_activation=out_activation) self.create_model(layers=layers, out_activation=out_activation)
self.compile(loss=loss, optimizer=optimizer) self.compile(loss=loss, optimizer=optimizer)

90
biomes/train_temp.py Normal file
View File

@ -0,0 +1,90 @@
import fire
import ray
import pandas as pd
import tensorflow as tf
import numpy as np
from tensorflow import keras
from utils import *
from model import Model
from constants import *
CHECKPOINT = 'checkpoints/temp.h5'
SEED = 1
np.random.seed(SEED)
df = pd.read_pickle('data.p')
dataset_size, x_columns, y_columns, dataset = dataframe_to_dataset_temp_precip(df)
batch_size = 5
epochs = 500
def baseline_model():
model = keras.models.Sequential()
params = {
'kernel_initializer': 'lecun_uniform',
'bias_initializer': 'zeros',
}
model.add(keras.layers.Dense(x_columns, input_dim=x_columns, **params, activation='elu'))
model.add(keras.layers.Dense(6, **params, activation='relu'))
model.add(keras.layers.Dense(y_columns, **params))
model.compile(loss='mse', optimizer='adam', metrics=['mae'])
return model
model = baseline_model()
model.summary()
dataset = dataset.shuffle(500)
TRAIN_SIZE = int(dataset_size * 0.85)
TEST_SIZE = dataset_size - TRAIN_SIZE
(training, test) = (dataset.take(TRAIN_SIZE),
dataset.skip(TRAIN_SIZE))
training_batched = training.batch(batch_size).repeat()
test_batched = test.batch(batch_size).repeat()
logger.debug('Model dataset info: size=%s, train=%s, test=%s', dataset_size, TRAIN_SIZE, TEST_SIZE)
# model.load_weights(CHECKPOINT)
def predict():
columns = INPUTS
YEAR = 2000
print(columns)
print(df[0:batch_size])
inputs = df[columns].to_numpy()
inputs = normalize_ndarray(inputs, df[columns].to_numpy())
print(inputs[0:batch_size])
out_columns = []
for season in SEASONS:
out_columns += ['temp_{}_{}'.format(season, YEAR), 'precip_{}_{}'.format(season, YEAR)]
print(out_columns)
out = model.predict(inputs)
print(out)
print(df[out_columns][0:batch_size])
print(denormalize(out, df[out_columns].to_numpy()))
def train():
tfb_callback = tf.keras.callbacks.TensorBoard(batch_size=batch_size, log_dir='temp_logs')
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=CHECKPOINT, monitor='val_loss')
model.fit(training_batched,
batch_size=batch_size,
epochs=epochs,
steps_per_epoch=int(TRAIN_SIZE / batch_size),
validation_data=test_batched,
validation_steps=int(TEST_SIZE / batch_size),
callbacks=[tfb_callback, checkpoint_callback],
verbose=1)
model.save_weights(CHECKPOINT)
# train()
if __name__ == "__main__":
fire.Fire({ 'predict': predict, 'train': train })

View File

@ -16,7 +16,13 @@ def normalize(v, o=None):
o = v o = v
return (v - np.mean(o)) / np.std(o) return (v - np.mean(o)) / np.std(o)
def normalize_ndarray(ar, o=None): def denormalize(v, o=None):
if o is None:
o = v
return (v * np.std(o) + np.mean(o))
def on_ndarray(ar, o=None, fn=None):
if o is None: if o is None:
o = ar o = ar
@ -24,11 +30,17 @@ def normalize_ndarray(ar, o=None):
tr = np.transpose(ar) tr = np.transpose(ar)
to = np.transpose(o) to = np.transpose(o)
for i in range(tr.shape[0]): for i in range(tr.shape[0]):
tr[i] = normalize(tr[i], to[i]) tr[i] = fn(tr[i], to[i])
# transpose back # transpose back
return np.transpose(tr) return np.transpose(tr)
def normalize_ndarray(ar, o=None):
return on_ndarray(ar, o=o, fn=normalize)
def denormalize_ndarray(ar, o=None):
return on_ndarray(ar, o=o, fn=denormalize)
def dataframe_to_dataset_biomes(df): def dataframe_to_dataset_biomes(df):
rows = df.shape[0] rows = df.shape[0]
@ -67,26 +79,27 @@ def dataframe_to_dataset_biomes(df):
def dataframe_to_dataset_temp_precip(df): def dataframe_to_dataset_temp_precip(df):
rows = df.shape[0] rows = df.shape[0]
# elevation, distance_to_water, latitude # elevation, distance_to_water, latitude, mean_temp, mean_precip
# season, year
input_columns = 5 input_columns = 5
num_classes = 2 # (temp, precip) * 4 seasons
num_classes = 8
tf_inputs = np.empty((0, input_columns)) tf_inputs = np.empty((0, input_columns))
tf_output = np.empty((0, num_classes)) tf_output = np.empty((0, num_classes))
for year in range(MIN_YEAR, MAX_YEAR + 1): for year in range(MIN_YEAR, MAX_YEAR + 1):
local_inputs = list(INPUTS) local_inputs = list(INPUTS)
local_df = df[local_inputs]
all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS]
all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS]
local_df.loc[:, 'mean_temp'] = np.mean(df[all_temps].values)
local_df.loc[:, 'mean_precip'] = np.mean(df[all_precips].values)
output = []
for idx, season in enumerate(SEASONS): output = all_temps + all_precips
season_index = idx / len(season)
local_df = df[local_inputs]
local_df.loc[:, 'season'] = pd.Series(np.repeat(season_index, rows), index=local_df.index)
local_df.loc[:, 'year'] = pd.Series(np.repeat(year, rows), index=local_df.index)
output = ['temp_{}_{}'.format(season, year), 'precip_{}_{}'.format(season, year)] tf_inputs = np.concatenate((tf_inputs, local_df.values), axis=0)
tf_inputs = np.concatenate((tf_inputs, local_df.values), axis=0) tf_output = np.concatenate((tf_output, df[output].values), axis=0)
tf_output = np.concatenate((tf_output, df[output].values), axis=0)
tf_inputs = tf.cast(normalize_ndarray(tf_inputs), tf.float32) tf_inputs = tf.cast(normalize_ndarray(tf_inputs), tf.float32)
tf_output = tf.cast(tf_output, tf.float32) tf_output = tf.cast(tf_output, tf.float32)

View File

@ -4,6 +4,8 @@ from matplotlib import colors, cm
import scipy.interpolate as interpolate import scipy.interpolate as interpolate
from scipy import ndimage from scipy import ndimage
import math import math
from io import BytesIO
import base64
WIDTH = 900 WIDTH = 900
HEIGHT = 450 HEIGHT = 450
@ -27,21 +29,8 @@ GROUND_PROPORTION = 1 - WATER_PROPORTION
DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)] DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)]
def s(x): def s(x):
# return x
return -2 * x**3 + 3 * x**2 return -2 * x**3 + 3 * x**2
# class Point(object):
# def __init__(self, x, y, z):
# self.x = x
# self.y = y
# self.z = z
# def is_ground(self):
# return self.z > WATER_LEVEL
# def is_water(self):
# return not(self.is_ground())
def is_ground(value): def is_ground(value):
return value > WATER_LEVEL return value > WATER_LEVEL
@ -156,7 +145,7 @@ def constant_filter(a):
return max(1, a[0]) return max(1, a[0])
return 0 return 0
def generate_map(width, height, continents=4): def generate_map(width=WIDTH, height=HEIGHT, continents=4):
ground = np.zeros((width, height)) ground = np.zeros((width, height))
ground_size = width * height * GROUND_PROPORTION ground_size = width * height * GROUND_PROPORTION
@ -181,12 +170,16 @@ def generate_map(width, height, continents=4):
ground = ndimage.gaussian_filter(ground, sigma=4) ground = ndimage.gaussian_filter(ground, sigma=4)
ground = ndimage.generic_filter(ground, constant_filter, size=1) ground = ndimage.generic_filter(ground, constant_filter, size=1)
print(np.min(ground), np.max(ground), MAX_ELEVATION) print(np.min(ground), np.max(ground), MAX_ELEVATION)
print(np.unique(ground)) print(np.unique(ground))
plt.imshow(ground.T, cmap=greys, norm=norm) plt.imshow(ground.T, cmap=greys, norm=norm)
plt.show()
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0)
return figfile
if __name__ == "__main__": if __name__ == "__main__":
generate_map(WIDTH, HEIGHT) generate_map(WIDTH, HEIGHT)
plt.show()

View File

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

7
map-generator/static/bootstrap.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,18 @@
const mapSettings = document.getElementById('map-settings');
//const board = document.getElementById('board');
const map = document.getElementById('map');
const spinner = document.getElementById('spinner');
mapSettings.addEventListener('submit', (e) => {
console.log('form submit');
e.preventDefault();
spinner.classList.remove('d-none');
map.src = '/map?q=' + (new Date()).getTime();
map.classList.add('d-none');
});
map.addEventListener('load', () => {
spinner.classList.add('d-none');
map.classList.remove('d-none');
});

View File

@ -0,0 +1,14 @@
* {
box-sizing: border-box;
}
html, body {
margin: 0;
}
aside {
text-align: center;
width: 300px;
height: 100vh;
}

View File

@ -0,0 +1,65 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>World Map Generator</title>
<link rel='stylesheet' href="{{ url_for('static', filename='style.css') }}">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
</head>
<body class='container-fluid'>
<div class='row'>
<main class='col d-flex justify-content-center align-items-center'>
<!-- <canvas id='board'></canvas> -->
<img src='/map' id='map'>
<div class='spinner-border text-primary' role='status' id='spinner'>
<span class='sr-only'>Loading...</span>
</div>
</main>
<aside class='col-3 px-4 bg-dark text-light text-center py-3'>
<h3>World Map Generator</h3>
<div class='panel px-4'>
<form class='mt-5' id='map-settings'>
<div class='form-group'>
<label name='preset'>Preset</label>
<select id='preset' name='preset' class='form-control'>
<option value='desert'>Desert</option>
<option value='forest'>Forest</option>
</select>
</div>
<div class='form-group'>
<label name='mean-temperature'>Mean Temperature</label>
<input id='mean-temperature' name='mean-temperature' type='range' min='-50' max='50' class='form-control'>
</div>
<div class='form-group'>
<label name='mean-precipitation'>Mean Precipitation</label>
<input id='mean-precipitation' name='mean-precipitation' type='range' min='-50' max='50' class='form-control'>
</div>
<div class='form-group'>
<label name='min-elevation'>Minimum Elevation</label>
<input id='min-elevation' name='min-elevation' type='range' min='-100' max='100' class='form-control'>
</div>
<div class='form-group'>
<label name='max-elevation'>Maximum Elevation</label>
<input id='max-elevation' name='max-elevation' type='range' min='-100' max='100' class='form-control'>
</div>
<button type="submit" class="btn btn-primary">Generate</button>
</form>
</div>
</aside>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
<script src="{{ url_for('static', filename='script.js') }}"></script>
</body>
</html>

13
map-generator/web.py Normal file
View File

@ -0,0 +1,13 @@
from flask import Flask, render_template, make_response, send_file
from index import generate_map
app = Flask(__name__)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/map')
def get_map():
res = send_file(generate_map(), mimetype='image/png')
return res