feat(web): web server and basic dashboard

.gitignore

@@ -1,5 +1,6 @@
 maps
 logs
+temp_logs
 checkpoints
 geodata
 *.p

biomes/model.py

@@ -135,10 +135,10 @@ class Model():
     def predict(self, a):
         return np.argmax(self.model.predict(a), axis=1)
 
-    def prepare_for_use(self, df=None, batch_size=DEFAULT_BUFFER_SIZE, layers=DEFAULT_LAYERS, out_activation=DEFAULT_OUT_ACTIVATION, loss=DEFAULT_LOSS, optimizer=DEFAULT_OPTIMIZER):
+    def prepare_for_use(self, df=None, batch_size=DEFAULT_BUFFER_SIZE, layers=DEFAULT_LAYERS, out_activation=DEFAULT_OUT_ACTIVATION, loss=DEFAULT_LOSS, optimizer=DEFAULT_OPTIMIZER, dataset_fn=dataframe_to_dataset_biomes):
         if df is None:
             df = pd.read_pickle('data.p')
-        self.prepare_dataset(df, dataframe_to_dataset_biomes, batch_size=batch_size)
+        self.prepare_dataset(df, dataset_fn, batch_size=batch_size)
         self.create_model(layers=layers, out_activation=out_activation)
         self.compile(loss=loss, optimizer=optimizer)
 

biomes/train_temp.py

@@ -0,0 +1,90 @@
+import fire
+import ray
+import pandas as pd
+import tensorflow as tf
+import numpy as np
+from tensorflow import keras
+from utils import *
+from model import Model
+from constants import *
+
+CHECKPOINT = 'checkpoints/temp.h5'
+
+SEED = 1
+np.random.seed(SEED)
+df = pd.read_pickle('data.p')
+
+dataset_size, x_columns, y_columns, dataset = dataframe_to_dataset_temp_precip(df)
+batch_size = 5
+epochs = 500
+
+def baseline_model():
+    model = keras.models.Sequential()
+    params = {
+            'kernel_initializer': 'lecun_uniform',
+            'bias_initializer': 'zeros',
+    }
+    model.add(keras.layers.Dense(x_columns, input_dim=x_columns, **params, activation='elu'))
+    model.add(keras.layers.Dense(6, **params, activation='relu'))
+    model.add(keras.layers.Dense(y_columns, **params))
+
+    model.compile(loss='mse', optimizer='adam', metrics=['mae'])
+    return model
+
+model = baseline_model()
+model.summary()
+
+dataset = dataset.shuffle(500)
+TRAIN_SIZE = int(dataset_size * 0.85)
+TEST_SIZE = dataset_size - TRAIN_SIZE
+(training, test) = (dataset.take(TRAIN_SIZE),
+                    dataset.skip(TRAIN_SIZE))
+training_batched = training.batch(batch_size).repeat()
+test_batched = test.batch(batch_size).repeat()
+
+logger.debug('Model dataset info: size=%s, train=%s, test=%s', dataset_size, TRAIN_SIZE, TEST_SIZE)
+
+# model.load_weights(CHECKPOINT)
+
+def predict():
+    columns = INPUTS
+
+    YEAR = 2000
+
+    print(columns)
+    print(df[0:batch_size])
+    inputs = df[columns].to_numpy()
+    inputs = normalize_ndarray(inputs, df[columns].to_numpy())
+    print(inputs[0:batch_size])
+
+    out_columns = []
+    for season in SEASONS:
+        out_columns += ['temp_{}_{}'.format(season, YEAR), 'precip_{}_{}'.format(season, YEAR)]
+
+    print(out_columns)
+
+    out = model.predict(inputs)
+    print(out)
+    print(df[out_columns][0:batch_size])
+    print(denormalize(out, df[out_columns].to_numpy()))
+
+def train():
+    tfb_callback = tf.keras.callbacks.TensorBoard(batch_size=batch_size, log_dir='temp_logs')
+    checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=CHECKPOINT, monitor='val_loss')
+
+    model.fit(training_batched,
+              batch_size=batch_size,
+              epochs=epochs,
+              steps_per_epoch=int(TRAIN_SIZE / batch_size),
+              validation_data=test_batched,
+              validation_steps=int(TEST_SIZE / batch_size),
+              callbacks=[tfb_callback, checkpoint_callback],
+              verbose=1)
+
+    model.save_weights(CHECKPOINT)
+
+# train()
+
+
+if __name__ == "__main__":
+    fire.Fire({ 'predict': predict, 'train': train })

biomes/utils.py

@@ -16,7 +16,13 @@ def normalize(v, o=None):
         o = v
     return (v - np.mean(o)) / np.std(o)
 
-def normalize_ndarray(ar, o=None):
+def denormalize(v, o=None):
+    if o is None:
+        o = v
+
+    return (v * np.std(o) + np.mean(o))
+
+def on_ndarray(ar, o=None, fn=None):
     if o is None:
         o = ar
 
@@ -24,11 +30,17 @@ def normalize_ndarray(ar, o=None):
     tr = np.transpose(ar)
     to = np.transpose(o)
     for i in range(tr.shape[0]):
-        tr[i] = normalize(tr[i], to[i])
+        tr[i] = fn(tr[i], to[i])
 
     # transpose back
     return np.transpose(tr)
 
+def normalize_ndarray(ar, o=None):
+    return on_ndarray(ar, o=o, fn=normalize)
+
+def denormalize_ndarray(ar, o=None):
+    return on_ndarray(ar, o=o, fn=denormalize)
+
 def dataframe_to_dataset_biomes(df):
     rows = df.shape[0]
 
@@ -67,26 +79,27 @@ def dataframe_to_dataset_biomes(df):
 def dataframe_to_dataset_temp_precip(df):
     rows = df.shape[0]
 
-    # elevation, distance_to_water, latitude
+    # elevation, distance_to_water, latitude, mean_temp, mean_precip
-    # season, year
     input_columns = 5
-    num_classes = 2
+    # (temp, precip) * 4 seasons
+    num_classes = 8
 
     tf_inputs = np.empty((0, input_columns))
     tf_output = np.empty((0, num_classes))
 
     for year in range(MIN_YEAR, MAX_YEAR + 1):
         local_inputs = list(INPUTS)
+        local_df = df[local_inputs]
+        all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS]
+        all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS]
+        local_df.loc[:, 'mean_temp'] = np.mean(df[all_temps].values)
+        local_df.loc[:, 'mean_precip'] = np.mean(df[all_precips].values)
+        output = []
 
-        for idx, season in enumerate(SEASONS):
+        output = all_temps + all_precips
-            season_index = idx / len(season)
-            local_df = df[local_inputs]
-            local_df.loc[:, 'season'] = pd.Series(np.repeat(season_index, rows), index=local_df.index)
-            local_df.loc[:, 'year'] = pd.Series(np.repeat(year, rows), index=local_df.index)
 
-            output = ['temp_{}_{}'.format(season, year), 'precip_{}_{}'.format(season, year)]
+        tf_inputs = np.concatenate((tf_inputs, local_df.values), axis=0)
-            tf_inputs = np.concatenate((tf_inputs, local_df.values), axis=0)
+        tf_output = np.concatenate((tf_output, df[output].values), axis=0)
-            tf_output = np.concatenate((tf_output, df[output].values), axis=0)
 
     tf_inputs = tf.cast(normalize_ndarray(tf_inputs), tf.float32)
     tf_output = tf.cast(tf_output, tf.float32)

map-generator/index.py

@@ -4,6 +4,8 @@ from matplotlib import colors, cm
 import scipy.interpolate as interpolate
 from scipy import ndimage
 import math
+from io import BytesIO
+import base64
 
 WIDTH = 900
 HEIGHT = 450
@@ -27,21 +29,8 @@ GROUND_PROPORTION = 1 - WATER_PROPORTION
 DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)]
 
 def s(x):
-    # return x
     return -2 * x**3 + 3 * x**2
 
-# class Point(object):
-    # def __init__(self, x, y, z):
-        # self.x = x
-        # self.y = y
-        # self.z = z
-
-    # def is_ground(self):
-        # return self.z > WATER_LEVEL
-
-    # def is_water(self):
-        # return not(self.is_ground())
-
 def is_ground(value):
     return value > WATER_LEVEL
 
@@ -156,7 +145,7 @@ def constant_filter(a):
         return max(1, a[0])
     return 0
 
-def generate_map(width, height, continents=4):
+def generate_map(width=WIDTH, height=HEIGHT, continents=4):
     ground = np.zeros((width, height))
     ground_size = width * height * GROUND_PROPORTION
 
@@ -181,12 +170,16 @@ def generate_map(width, height, continents=4):
     ground = ndimage.gaussian_filter(ground, sigma=4)
     ground = ndimage.generic_filter(ground, constant_filter, size=1)
     print(np.min(ground), np.max(ground), MAX_ELEVATION)
-
     print(np.unique(ground))
 
     plt.imshow(ground.T, cmap=greys, norm=norm)
-    plt.show()
 
+    figfile = BytesIO()
+    plt.savefig(figfile, format='png')
+    figfile.seek(0)
+
+    return figfile
 
 if __name__ == "__main__":
     generate_map(WIDTH, HEIGHT)
+    plt.show()

map-generator/static/script.js

@@ -0,0 +1,18 @@
+const mapSettings = document.getElementById('map-settings');
+//const board = document.getElementById('board');
+const map = document.getElementById('map');
+const spinner = document.getElementById('spinner');
+
+mapSettings.addEventListener('submit', (e) => {
+  console.log('form submit');
+  e.preventDefault();
+
+  spinner.classList.remove('d-none');
+  map.src = '/map?q=' + (new Date()).getTime();
+  map.classList.add('d-none');
+});
+
+map.addEventListener('load', () => {
+  spinner.classList.add('d-none');
+  map.classList.remove('d-none');
+});

map-generator/static/style.css

@@ -0,0 +1,14 @@
+* {
+  box-sizing: border-box;
+}
+
+html, body {
+  margin: 0;
+}
+
+aside {
+  text-align: center;
+  width: 300px;
+  height: 100vh;
+}
+

map-generator/templates/index.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>World Map Generator</title>
+  <link rel='stylesheet' href="{{ url_for('static', filename='style.css') }}">
+  <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+</head>
+<body class='container-fluid'>
+  <div class='row'>
+    <main class='col d-flex justify-content-center align-items-center'>
+    <!-- <canvas id='board'></canvas> -->
+      <img src='/map' id='map'>
+
+      <div class='spinner-border text-primary' role='status' id='spinner'>
+        <span class='sr-only'>Loading...</span>
+      </div>
+    </main>
+
+    <aside class='col-3 px-4 bg-dark text-light text-center py-3'>
+      <h3>World Map Generator</h3>
+
+      <div class='panel px-4'>
+        <form class='mt-5' id='map-settings'>
+          <div class='form-group'>
+            <label name='preset'>Preset</label>
+            <select id='preset' name='preset' class='form-control'>
+              <option value='desert'>Desert</option>
+              <option value='forest'>Forest</option>
+            </select>
+          </div>
+        
+          <div class='form-group'>
+            <label name='mean-temperature'>Mean Temperature</label>
+            <input id='mean-temperature' name='mean-temperature' type='range' min='-50' max='50' class='form-control'>
+          </div>
+
+          <div class='form-group'>
+            <label name='mean-precipitation'>Mean Precipitation</label>
+            <input id='mean-precipitation' name='mean-precipitation' type='range' min='-50' max='50' class='form-control'>
+          </div>
+
+          <div class='form-group'>
+            <label name='min-elevation'>Minimum Elevation</label>
+            <input id='min-elevation' name='min-elevation' type='range' min='-100' max='100' class='form-control'>
+          </div>
+
+          <div class='form-group'>
+            <label name='max-elevation'>Maximum Elevation</label>
+            <input id='max-elevation' name='max-elevation' type='range' min='-100' max='100' class='form-control'>
+          </div>
+
+          <button type="submit" class="btn btn-primary">Generate</button>
+        </form>
+      </div>
+    </aside>
+  </div>
+
+  <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
+  <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
+  <script src="{{ url_for('static', filename='script.js') }}"></script>
+</body>
+</html>

map-generator/web.py

@@ -0,0 +1,13 @@
+from flask import Flask, render_template, make_response, send_file
+from index import generate_map
+
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/map')
+def get_map():
+    res = send_file(generate_map(), mimetype='image/png')
+    return res