feat(web): auto-generated form

biomes/model.py

@@ -99,8 +99,8 @@ class Model():
 
     def save(self, path):
         logger.debug('Saving model weights to path: %s', path)
-        self.model.save_weights(path)
+        self.model.save_weights(path + '/checkpoint.hd5')
-        return path
+        return path + '/checkpoint'
 
     def evaluate(self):
         return self.model.evaluate(
@@ -119,15 +119,19 @@ class Model():
 
         # map_callback = MapHistory()
 
+        extra_params = {}
+        if self.class_weight:
+            extra_params['class_weight'] = self.class_weight
+
         out = self.model.fit(
             self.training_batched,
             batch_size=self.batch_size,
             epochs=self.epochs,
             steps_per_epoch=int(self.TRAIN_SIZE / self.batch_size),
-            class_weight=self.class_weight,
             validation_data=self.test_batched,
             validation_steps=int(self.TEST_SIZE / self.batch_size),
-            verbose=1
+            verbose=1,
+            **extra_params
         )
 
         return out

biomes/train.py

@@ -4,7 +4,7 @@ import pandas as pd
 import tensorflow as tf
 from ray import tune
 from tensorflow import keras
-from utils import logger
+from utils import *
 from model import Model
 
 B_params = {
@@ -45,11 +45,61 @@ class TuneB(tune.Trainable):
     def _restore(self, path):
         return self.model.restore(path)
 
-def start_tuning(cpu=1, gpu=2, checkpoint_freq=1, checkpoint_at_end=True, resume=False, restore=None, stop=500):
+A_params = {
+    'batch_size': tune.grid_search([5, 16, 32, 64]),
+    'layers': tune.grid_search([[16, 16], [32, 32], [128, 128]]),
+    'lr': tune.grid_search([1e-4, 1e-3, 1e-2]),
+    'optimizer': tune.grid_search([tf.keras.optimizers.Adam]),
+}
+
+class TuneA(tune.Trainable):
+    def _setup(self, config):
+        logger.debug('Ray Tune model configuration %s', config)
+
+        self.model = Model('a', epochs=1)
+
+        optimizer = config['optimizer']
+        optimizer = config['optimizer'](lr=config['lr'])
+
+        self.model.prepare_for_use(
+                df=df,
+                batch_size=config['batch_size'],
+                layers=config['layers'],
+                optimizer=optimizer,
+                dataset_fn=dataframe_to_dataset_temp_precip,
+                loss='mse'
+        )
+
+    def _train(self):
+        logs = self.model.train(self.config)
+
+        metrics = {
+            'mean_accuracy': logs.history['acc'][0],
+            'loss': logs.history['loss'][0],
+            'val_accuracy': logs.history['val_acc'][0],
+            'val_loss': logs.history['val_loss'][0],
+        }
+
+        return metrics
+
+    def _save(self, checkpoint_dir):
+        return self.model.save(checkpoint_dir)
+
+    def _restore(self, path):
+        return self.model.restore(path)
+
+def start_tuning(model, cpu=1, gpu=2, checkpoint_freq=1, checkpoint_at_end=True, resume=False, restore=None, stop=500):
     ray.init()
 
-    tune.run(TuneB,
+    if model == 'a':
-            config=B_params,
+        t = TuneA
+        params = A_params
+    else:
+        t = TuneB
+        params = B_params
+
+    tune.run(t,
+            config=params,
             resources_per_trial={
                 "cpu": cpu,
                 "gpu": gpu
@@ -62,6 +112,5 @@ def start_tuning(cpu=1, gpu=2, checkpoint_freq=1, checkpoint_at_end=True, resume
               'training_iteration': stop
             })
 
-
 if __name__ == "__main__":
     fire.Fire(start_tuning)

biomes/train_temp.py

@@ -1,90 +0,0 @@
-import fire
-import ray
-import pandas as pd
-import tensorflow as tf
-import numpy as np
-from tensorflow import keras
-from utils import *
-from model import Model
-from constants import *
-
-CHECKPOINT = 'checkpoints/temp.h5'
-
-SEED = 1
-np.random.seed(SEED)
-df = pd.read_pickle('data.p')
-
-dataset_size, x_columns, y_columns, dataset = dataframe_to_dataset_temp_precip(df)
-batch_size = 5
-epochs = 500
-
-def baseline_model():
-    model = keras.models.Sequential()
-    params = {
-            'kernel_initializer': 'lecun_uniform',
-            'bias_initializer': 'zeros',
-    }
-    model.add(keras.layers.Dense(x_columns, input_dim=x_columns, **params, activation='elu'))
-    model.add(keras.layers.Dense(6, **params, activation='relu'))
-    model.add(keras.layers.Dense(y_columns, **params))
-
-    model.compile(loss='mse', optimizer='adam', metrics=['mae'])
-    return model
-
-model = baseline_model()
-model.summary()
-
-dataset = dataset.shuffle(500)
-TRAIN_SIZE = int(dataset_size * 0.85)
-TEST_SIZE = dataset_size - TRAIN_SIZE
-(training, test) = (dataset.take(TRAIN_SIZE),
-                    dataset.skip(TRAIN_SIZE))
-training_batched = training.batch(batch_size).repeat()
-test_batched = test.batch(batch_size).repeat()
-
-logger.debug('Model dataset info: size=%s, train=%s, test=%s', dataset_size, TRAIN_SIZE, TEST_SIZE)
-
-# model.load_weights(CHECKPOINT)
-
-def predict():
-    columns = INPUTS
-
-    YEAR = 2000
-
-    print(columns)
-    print(df[0:batch_size])
-    inputs = df[columns].to_numpy()
-    inputs = normalize_ndarray(inputs, df[columns].to_numpy())
-    print(inputs[0:batch_size])
-
-    out_columns = []
-    for season in SEASONS:
-        out_columns += ['temp_{}_{}'.format(season, YEAR), 'precip_{}_{}'.format(season, YEAR)]
-
-    print(out_columns)
-
-    out = model.predict(inputs)
-    print(out)
-    print(df[out_columns][0:batch_size])
-    print(denormalize(out, df[out_columns].to_numpy()))
-
-def train():
-    tfb_callback = tf.keras.callbacks.TensorBoard(batch_size=batch_size, log_dir='temp_logs')
-    checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=CHECKPOINT, monitor='val_loss')
-
-    model.fit(training_batched,
-              batch_size=batch_size,
-              epochs=epochs,
-              steps_per_epoch=int(TRAIN_SIZE / batch_size),
-              validation_data=test_batched,
-              validation_steps=int(TEST_SIZE / batch_size),
-              callbacks=[tfb_callback, checkpoint_callback],
-              verbose=1)
-
-    model.save_weights(CHECKPOINT)
-
-# train()
-
-
-if __name__ == "__main__":
-    fire.Fire({ 'predict': predict, 'train': train })

biomes/utils.py

@@ -105,7 +105,7 @@ def dataframe_to_dataset_temp_precip(df):
     tf_output = tf.cast(tf_output, tf.float32)
 
     logger.debug('dataset size: rows=%d, input_columns=%d, num_classes=%d', int(tf_inputs.shape[0]), input_columns, num_classes)
-    return int(tf_inputs.shape[0]), input_columns, num_classes, tf.data.Dataset.from_tensor_slices((tf_inputs, tf_output))
+    return int(tf_inputs.shape[0]), input_columns, num_classes, None, tf.data.Dataset.from_tensor_slices((tf_inputs, tf_output))
 
 
 flatten = lambda l: [item for sublist in l for item in sublist]

map-generator/index.py

@@ -7,32 +7,112 @@ import math
 from io import BytesIO
 import base64
 
-WIDTH = 900
+parameters = {
-HEIGHT = 450
+    'width': {
-RATIO = WIDTH / HEIGHT
+        'default': 900,
-
+        'type': 'int',
-MOUNTAIN_SEA_DISTANCE = 50
+    },
-MOUNTAIN_SEA_THRESHOLD = 2
+    'height': {
-MOUNTAIN_JAGGEDNESS = 1
+        'default': 450,
+        'type': 'int',
+    },
+    'mountain_ratio': {
+        'default': 0.3,
+        'type': 'float',
+        'min': 0,
+        'max': 1,
+        'step': 0.01
+    },
+    'sharpness': {
+        'default': 0.7,
+        'type': 'float',
+        'min': 0,
+        'max': 1,
+        'step': 0.01
+    },
+    'max_elevation': {
+        'default': 30,
+        'type': 'int',
+        'min': 0,
+        'max': 50,
+    },
+    'ground_noise': {
+        'default': 15,
+        'type': 'int',
+        'min': 0,
+        'max': 50,
+    },
+    'water_proportion': {
+        'default': 0.6,
+        'type': 'float',
+        'min': 0,
+        'max': 0.99,
+        'step': 0.01
+    },
+    'mountain_jaggedness': {
+        'default': 1,
+        'type': 'int',
+        'min': 0,
+        'max': 5,
+    },
+    'mountain_sea_distance': {
+        'default': 50,
+        'type': 'int',
+        'min': 0,
+        'max': 200,
+    },
+    'mountain_sea_threshold': {
+        'default': 2,
+        'type': 'int',
+        'min': 0,
+        'max': 5,
+    },
+    'water_level': {
+        'default': 0,
+        'type': 'int',
+    },
+    'mountain_area_elevation': {
+        'default': 0.4,
+        'type': 'float',
+        'min': 0,
+        'max': 1,
+        'step': 0.01
+    },
+    'mountain_area_elevation_n': {
+        'default': 5,
+        'type': 'int',
+        'min': 0,
+        'max': 15,
+    },
+    'mountain_area_elevation_area': {
+        'default': 10,
+        'type': 'int',
+        'min': 0,
+        'max': 25,
+    },
+    'continents': {
+        'default': 5,
+        'type': 'int',
+    },
+    'seed': {
+        'default': '',
+        'type': 'int',
+        'description': 'Leave empty for a random seed generated from the current timestamp.'
+    },
+        
+}
+p = { k: parameters[k]['default'] for k in parameters }
 
 CONTINENT_MAX_TRIALS = 1e4
-MOUNTAIN_RATIO = 0.3
 SEA_COLOR = np.array((53, 179, 220, 255)) / 255
 
-SHARPNESS = 0.7
-WATER_LEVEL = 0
-MAX_ELEVATION = 30
-GROUND_NOISE = 15
-WATER_PROPORTION = 0.6
-GROUND_PROPORTION = 1 - WATER_PROPORTION
-
 DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)]
 
 def s(x):
     return -2 * x**3 + 3 * x**2
 
 def is_ground(value):
-    return value > WATER_LEVEL
+    return value > p['water_level']
 
 def in_range(p, m, size):
     x, y = p
@@ -83,7 +163,7 @@ def continent_agent(ground, position, size):
         if not is_ground(ground[x, y]) and in_range((x, y), position, size):
             trials = 0
             size -= 1
-            ground[x, y] = np.random.randint(1, GROUND_NOISE)
+            ground[x, y] = np.random.randint(1, p['ground_noise'])
         else:
             trials += 1
 
@@ -91,22 +171,19 @@ def neighbours(ground, position, radius):
     x, y = position
     return ground[x-radius:x+radius+1, y-radius:y+radius+1]
 
-def away_from_sea(ground, position, radius=MOUNTAIN_SEA_DISTANCE):
+def away_from_sea(ground, position, radius=p['mountain_sea_distance']):
     ns = neighbours(ground, position, radius).flatten()
     sea = len([1 for x in ns if not is_ground(x)])
 
-    return sea < MOUNTAIN_SEA_THRESHOLD
+    return sea < p['mountain_sea_threshold']
 
-MOUNTAIN_AREA_ELEVATION = 0.4
+def random_elevate_agent(ground, position, height, size=p['mountain_area_elevation_n']):
-MOUNTAIN_AREA_ELEVATION_N = 5
+    position = position + np.random.random_integers(-p['mountain_area_elevation_area'], p['mountain_area_elevation_area'], size=2)
-MOUNTAIN_AREA_ELEVATION_AREA = 10
-def random_elevate_agent(ground, position, height, size=MOUNTAIN_AREA_ELEVATION_N):
-    position = position + np.random.random_integers(-MOUNTAIN_AREA_ELEVATION_AREA, MOUNTAIN_AREA_ELEVATION_AREA, size=2)
 
     for i in range(size):
         d = DIRECTIONS[np.random.randint(len(DIRECTIONS))]
 
-        change = height * MOUNTAIN_AREA_ELEVATION + np.random.randint(MOUNTAIN_JAGGEDNESS + 1)
+        change = height * p['mountain_area_elevation'] + np.random.randint(p['mountain_jaggedness'] + 1)
         new_index = bound_check(ground, position + np.array(d))
 
         if is_ground(ground[new_index]):
@@ -118,21 +195,21 @@ def mountain_agent(ground, position):
         return
 
     x, y = position
-    height = np.random.randint(MAX_ELEVATION)
+    height = np.random.randint(p['max_elevation'])
 
     ground[x, y] = height
 
     last_height = height
     for i in range(1, height):
         for d in DIRECTIONS:
-            change = np.random.randint(MOUNTAIN_JAGGEDNESS + 1)
+            change = np.random.randint(p['mountain_jaggedness'] + 1)
             distance = np.array(d)*i
             new_index = bound_check(ground, position + distance)
 
             if is_ground(ground[new_index]):
                 ground[new_index] = last_height - change
 
-        last_height = last_height - MOUNTAIN_JAGGEDNESS
+        last_height = last_height - p['mountain_jaggedness']
 
     random_elevate_agent(ground, position, height)
 
@@ -141,13 +218,22 @@ def mountain_agent(ground, position):
 # def split_agent(ground, position, directions):
 
 def constant_filter(a):
-    if a[0] > (1 - SHARPNESS):
+    if a[0] > (1 - p['sharpness']):
         return max(1, a[0])
     return 0
 
-def generate_map(width=WIDTH, height=HEIGHT, continents=4):
+def generate_map(**kwargs):
+    plt.clf()
+
+    p.update(kwargs)
+
+    np.random.seed(p['seed'] or None)
+
+    width, height = p['width'], p['height']
+    continents = p['continents']
+
     ground = np.zeros((width, height))
-    ground_size = width * height * GROUND_PROPORTION
+    ground_size = width * height * (1 - p['water_proportion'])
 
     # position = (int(width / 2), int(height / 2))
     # ground_size = width * height * GROUND_PROPORTION
@@ -157,9 +243,9 @@ def generate_map(width=WIDTH, height=HEIGHT, continents=4):
         print(position)
         continent_agent(ground, position, size=ground_size)
 
-    ground = ndimage.gaussian_filter(ground, sigma=(1 - SHARPNESS) * 20)
+    ground = ndimage.gaussian_filter(ground, sigma=(1 - p['sharpness']) * 20)
 
-    for i in range(int(ground_size * MOUNTAIN_RATIO / MAX_ELEVATION**2)):
+    for i in range(int(ground_size * p['mountain_ratio'] / p['max_elevation']**2)):
         position = (np.random.randint(0, width), np.random.randint(0, height))
         mountain_agent(ground, position)
 
@@ -169,7 +255,7 @@ def generate_map(width=WIDTH, height=HEIGHT, continents=4):
 
     ground = ndimage.gaussian_filter(ground, sigma=4)
     ground = ndimage.generic_filter(ground, constant_filter, size=1)
-    print(np.min(ground), np.max(ground), MAX_ELEVATION)
+    print(np.min(ground), np.max(ground), p['max_elevation'])
     print(np.unique(ground))
 
     plt.imshow(ground.T, cmap=greys, norm=norm)
@@ -181,5 +267,5 @@ def generate_map(width=WIDTH, height=HEIGHT, continents=4):
     return figfile
 
 if __name__ == "__main__":
-    generate_map(WIDTH, HEIGHT)
+    generate_map()
     plt.show()

map-generator/static/script.js

@@ -3,15 +3,26 @@ const mapSettings = document.getElementById('map-settings');
 const map = document.getElementById('map');
 const spinner = document.getElementById('spinner');
 
+function generate() {
+  spinner.classList.remove('d-none');
+  const formData = new FormData(mapSettings)
+  if (!formData.get('seed')) {
+    formData.set('seed', (new Date()).getTime() % 1e5);
+  }
+  const queryString = new URLSearchParams(formData).toString()
+  map.src = '/map?' + queryString;
+  map.classList.add('d-none');
+
+}
+
 mapSettings.addEventListener('submit', (e) => {
-  console.log('form submit');
   e.preventDefault();
 
-  spinner.classList.remove('d-none');
+  generate()
-  map.src = '/map?q=' + (new Date()).getTime();
-  map.classList.add('d-none');
 });
 
+generate()
+
 map.addEventListener('load', () => {
   spinner.classList.add('d-none');
   map.classList.remove('d-none');

map-generator/static/style.css

@@ -10,5 +10,6 @@ aside {
   text-align: center;
   width: 300px;
   height: 100vh;
+  overflow-y: scroll;
 }
 

map-generator/templates/index.html

@@ -11,7 +11,7 @@
   <div class='row'>
     <main class='col d-flex justify-content-center align-items-center'>
     <!-- <canvas id='board'></canvas> -->
-      <img src='/map' id='map'>
+      <img src='' id='map'>
 
       <div class='spinner-border text-primary' role='status' id='spinner'>
         <span class='sr-only'>Loading...</span>
@@ -23,33 +23,18 @@
 
       <div class='panel px-4'>
         <form class='mt-5' id='map-settings'>
-          <div class='form-group'>
+          {% for k, v in parameters.items() %}
-            <label name='preset'>Preset</label>
+            <div class='form-group'>
-            <select id='preset' name='preset' class='form-control'>
+              <label name='{{ k }}'>{{ k | replace('_', ' ') | title }}</label>
-              <option value='desert'>Desert</option>
+              <input id='{{ k }}' name='{{ k }}'
-              <option value='forest'>Forest</option>
+                     type="number"
-            </select>
+                     class='form-control'
-          </div>
+                     min='{{ v["min"] }}' value='{{ v["default"] }}' max='{{ v["max"] }}' step='{{ v["step"] }}'>
-        
+              {% if v["description"] %}
-          <div class='form-group'>
+              <small class="form-text text-muted">{{ v["description"] }}</small>
-            <label name='mean-temperature'>Mean Temperature</label>
+              {% endif %}
-            <input id='mean-temperature' name='mean-temperature' type='range' min='-50' max='50' class='form-control'>
+            </div>
-          </div>
+          {% endfor %}
-
-          <div class='form-group'>
-            <label name='mean-precipitation'>Mean Precipitation</label>
-            <input id='mean-precipitation' name='mean-precipitation' type='range' min='-50' max='50' class='form-control'>
-          </div>
-
-          <div class='form-group'>
-            <label name='min-elevation'>Minimum Elevation</label>
-            <input id='min-elevation' name='min-elevation' type='range' min='-100' max='100' class='form-control'>
-          </div>
-
-          <div class='form-group'>
-            <label name='max-elevation'>Maximum Elevation</label>
-            <input id='max-elevation' name='max-elevation' type='range' min='-100' max='100' class='form-control'>
-          </div>
 
           <button type="submit" class="btn btn-primary">Generate</button>
         </form>

map-generator/web.py

@@ -1,13 +1,23 @@
-from flask import Flask, render_template, make_response, send_file
+from flask import Flask, render_template, make_response, send_file, request
-from index import generate_map
+from index import generate_map, parameters
 
 app = Flask(__name__)
 
 @app.route('/')
 def index():
-    return render_template('index.html')
+    return render_template('index.html', parameters=parameters)
+
+def parse(key, value):
+    t = parameters[key]['type']
+    if t == 'int':
+        return int(value)
+    elif t == 'float':
+        return float(value)
+    else:
+        return value
 
 @app.route('/map')
 def get_map():
-    res = send_file(generate_map(), mimetype='image/png')
+    params = { key: parse(key, request.args[key]) for key in request.args }
+    res = send_file(generate_map(**params), mimetype='image/png')
     return res