diff --git a/biomes/model.py b/biomes/model.py index 80fe648..d581ae1 100644 --- a/biomes/model.py +++ b/biomes/model.py @@ -99,8 +99,8 @@ class Model(): def save(self, path): logger.debug('Saving model weights to path: %s', path) - self.model.save_weights(path) - return path + self.model.save_weights(path + '/checkpoint.hd5') + return path + '/checkpoint' def evaluate(self): return self.model.evaluate( @@ -119,15 +119,19 @@ class Model(): # map_callback = MapHistory() + extra_params = {} + if self.class_weight: + extra_params['class_weight'] = self.class_weight + out = self.model.fit( self.training_batched, batch_size=self.batch_size, epochs=self.epochs, steps_per_epoch=int(self.TRAIN_SIZE / self.batch_size), - class_weight=self.class_weight, validation_data=self.test_batched, validation_steps=int(self.TEST_SIZE / self.batch_size), - verbose=1 + verbose=1, + **extra_params ) return out diff --git a/biomes/train.py b/biomes/train.py index dc885ad..6a38c03 100644 --- a/biomes/train.py +++ b/biomes/train.py @@ -4,7 +4,7 @@ import pandas as pd import tensorflow as tf from ray import tune from tensorflow import keras -from utils import logger +from utils import * from model import Model B_params = { @@ -45,11 +45,61 @@ class TuneB(tune.Trainable): def _restore(self, path): return self.model.restore(path) -def start_tuning(cpu=1, gpu=2, checkpoint_freq=1, checkpoint_at_end=True, resume=False, restore=None, stop=500): +A_params = { + 'batch_size': tune.grid_search([5, 16, 32, 64]), + 'layers': tune.grid_search([[16, 16], [32, 32], [128, 128]]), + 'lr': tune.grid_search([1e-4, 1e-3, 1e-2]), + 'optimizer': tune.grid_search([tf.keras.optimizers.Adam]), +} + +class TuneA(tune.Trainable): + def _setup(self, config): + logger.debug('Ray Tune model configuration %s', config) + + self.model = Model('a', epochs=1) + + optimizer = config['optimizer'] + optimizer = config['optimizer'](lr=config['lr']) + + self.model.prepare_for_use( + df=df, + batch_size=config['batch_size'], + layers=config['layers'], + optimizer=optimizer, + dataset_fn=dataframe_to_dataset_temp_precip, + loss='mse' + ) + + def _train(self): + logs = self.model.train(self.config) + + metrics = { + 'mean_accuracy': logs.history['acc'][0], + 'loss': logs.history['loss'][0], + 'val_accuracy': logs.history['val_acc'][0], + 'val_loss': logs.history['val_loss'][0], + } + + return metrics + + def _save(self, checkpoint_dir): + return self.model.save(checkpoint_dir) + + def _restore(self, path): + return self.model.restore(path) + +def start_tuning(model, cpu=1, gpu=2, checkpoint_freq=1, checkpoint_at_end=True, resume=False, restore=None, stop=500): ray.init() - tune.run(TuneB, - config=B_params, + if model == 'a': + t = TuneA + params = A_params + else: + t = TuneB + params = B_params + + tune.run(t, + config=params, resources_per_trial={ "cpu": cpu, "gpu": gpu @@ -62,6 +112,5 @@ def start_tuning(cpu=1, gpu=2, checkpoint_freq=1, checkpoint_at_end=True, resume 'training_iteration': stop }) - if __name__ == "__main__": fire.Fire(start_tuning) diff --git a/biomes/train_temp.py b/biomes/train_temp.py deleted file mode 100644 index 1efab31..0000000 --- a/biomes/train_temp.py +++ /dev/null @@ -1,90 +0,0 @@ -import fire -import ray -import pandas as pd -import tensorflow as tf -import numpy as np -from tensorflow import keras -from utils import * -from model import Model -from constants import * - -CHECKPOINT = 'checkpoints/temp.h5' - -SEED = 1 -np.random.seed(SEED) -df = pd.read_pickle('data.p') - -dataset_size, x_columns, y_columns, dataset = dataframe_to_dataset_temp_precip(df) -batch_size = 5 -epochs = 500 - -def baseline_model(): - model = keras.models.Sequential() - params = { - 'kernel_initializer': 'lecun_uniform', - 'bias_initializer': 'zeros', - } - model.add(keras.layers.Dense(x_columns, input_dim=x_columns, **params, activation='elu')) - model.add(keras.layers.Dense(6, **params, activation='relu')) - model.add(keras.layers.Dense(y_columns, **params)) - - model.compile(loss='mse', optimizer='adam', metrics=['mae']) - return model - -model = baseline_model() -model.summary() - -dataset = dataset.shuffle(500) -TRAIN_SIZE = int(dataset_size * 0.85) -TEST_SIZE = dataset_size - TRAIN_SIZE -(training, test) = (dataset.take(TRAIN_SIZE), - dataset.skip(TRAIN_SIZE)) -training_batched = training.batch(batch_size).repeat() -test_batched = test.batch(batch_size).repeat() - -logger.debug('Model dataset info: size=%s, train=%s, test=%s', dataset_size, TRAIN_SIZE, TEST_SIZE) - -# model.load_weights(CHECKPOINT) - -def predict(): - columns = INPUTS - - YEAR = 2000 - - print(columns) - print(df[0:batch_size]) - inputs = df[columns].to_numpy() - inputs = normalize_ndarray(inputs, df[columns].to_numpy()) - print(inputs[0:batch_size]) - - out_columns = [] - for season in SEASONS: - out_columns += ['temp_{}_{}'.format(season, YEAR), 'precip_{}_{}'.format(season, YEAR)] - - print(out_columns) - - out = model.predict(inputs) - print(out) - print(df[out_columns][0:batch_size]) - print(denormalize(out, df[out_columns].to_numpy())) - -def train(): - tfb_callback = tf.keras.callbacks.TensorBoard(batch_size=batch_size, log_dir='temp_logs') - checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=CHECKPOINT, monitor='val_loss') - - model.fit(training_batched, - batch_size=batch_size, - epochs=epochs, - steps_per_epoch=int(TRAIN_SIZE / batch_size), - validation_data=test_batched, - validation_steps=int(TEST_SIZE / batch_size), - callbacks=[tfb_callback, checkpoint_callback], - verbose=1) - - model.save_weights(CHECKPOINT) - -# train() - - -if __name__ == "__main__": - fire.Fire({ 'predict': predict, 'train': train }) diff --git a/biomes/utils.py b/biomes/utils.py index 0abcdd3..e1ee7f5 100644 --- a/biomes/utils.py +++ b/biomes/utils.py @@ -105,7 +105,7 @@ def dataframe_to_dataset_temp_precip(df): tf_output = tf.cast(tf_output, tf.float32) logger.debug('dataset size: rows=%d, input_columns=%d, num_classes=%d', int(tf_inputs.shape[0]), input_columns, num_classes) - return int(tf_inputs.shape[0]), input_columns, num_classes, tf.data.Dataset.from_tensor_slices((tf_inputs, tf_output)) + return int(tf_inputs.shape[0]), input_columns, num_classes, None, tf.data.Dataset.from_tensor_slices((tf_inputs, tf_output)) flatten = lambda l: [item for sublist in l for item in sublist] diff --git a/map-generator/index.py b/map-generator/index.py index 3bb7147..118fd26 100644 --- a/map-generator/index.py +++ b/map-generator/index.py @@ -7,32 +7,112 @@ import math from io import BytesIO import base64 -WIDTH = 900 -HEIGHT = 450 -RATIO = WIDTH / HEIGHT - -MOUNTAIN_SEA_DISTANCE = 50 -MOUNTAIN_SEA_THRESHOLD = 2 -MOUNTAIN_JAGGEDNESS = 1 +parameters = { + 'width': { + 'default': 900, + 'type': 'int', + }, + 'height': { + 'default': 450, + 'type': 'int', + }, + 'mountain_ratio': { + 'default': 0.3, + 'type': 'float', + 'min': 0, + 'max': 1, + 'step': 0.01 + }, + 'sharpness': { + 'default': 0.7, + 'type': 'float', + 'min': 0, + 'max': 1, + 'step': 0.01 + }, + 'max_elevation': { + 'default': 30, + 'type': 'int', + 'min': 0, + 'max': 50, + }, + 'ground_noise': { + 'default': 15, + 'type': 'int', + 'min': 0, + 'max': 50, + }, + 'water_proportion': { + 'default': 0.6, + 'type': 'float', + 'min': 0, + 'max': 0.99, + 'step': 0.01 + }, + 'mountain_jaggedness': { + 'default': 1, + 'type': 'int', + 'min': 0, + 'max': 5, + }, + 'mountain_sea_distance': { + 'default': 50, + 'type': 'int', + 'min': 0, + 'max': 200, + }, + 'mountain_sea_threshold': { + 'default': 2, + 'type': 'int', + 'min': 0, + 'max': 5, + }, + 'water_level': { + 'default': 0, + 'type': 'int', + }, + 'mountain_area_elevation': { + 'default': 0.4, + 'type': 'float', + 'min': 0, + 'max': 1, + 'step': 0.01 + }, + 'mountain_area_elevation_n': { + 'default': 5, + 'type': 'int', + 'min': 0, + 'max': 15, + }, + 'mountain_area_elevation_area': { + 'default': 10, + 'type': 'int', + 'min': 0, + 'max': 25, + }, + 'continents': { + 'default': 5, + 'type': 'int', + }, + 'seed': { + 'default': '', + 'type': 'int', + 'description': 'Leave empty for a random seed generated from the current timestamp.' + }, + +} +p = { k: parameters[k]['default'] for k in parameters } CONTINENT_MAX_TRIALS = 1e4 -MOUNTAIN_RATIO = 0.3 SEA_COLOR = np.array((53, 179, 220, 255)) / 255 -SHARPNESS = 0.7 -WATER_LEVEL = 0 -MAX_ELEVATION = 30 -GROUND_NOISE = 15 -WATER_PROPORTION = 0.6 -GROUND_PROPORTION = 1 - WATER_PROPORTION - DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)] def s(x): return -2 * x**3 + 3 * x**2 def is_ground(value): - return value > WATER_LEVEL + return value > p['water_level'] def in_range(p, m, size): x, y = p @@ -83,7 +163,7 @@ def continent_agent(ground, position, size): if not is_ground(ground[x, y]) and in_range((x, y), position, size): trials = 0 size -= 1 - ground[x, y] = np.random.randint(1, GROUND_NOISE) + ground[x, y] = np.random.randint(1, p['ground_noise']) else: trials += 1 @@ -91,22 +171,19 @@ def neighbours(ground, position, radius): x, y = position return ground[x-radius:x+radius+1, y-radius:y+radius+1] -def away_from_sea(ground, position, radius=MOUNTAIN_SEA_DISTANCE): +def away_from_sea(ground, position, radius=p['mountain_sea_distance']): ns = neighbours(ground, position, radius).flatten() sea = len([1 for x in ns if not is_ground(x)]) - return sea < MOUNTAIN_SEA_THRESHOLD + return sea < p['mountain_sea_threshold'] -MOUNTAIN_AREA_ELEVATION = 0.4 -MOUNTAIN_AREA_ELEVATION_N = 5 -MOUNTAIN_AREA_ELEVATION_AREA = 10 -def random_elevate_agent(ground, position, height, size=MOUNTAIN_AREA_ELEVATION_N): - position = position + np.random.random_integers(-MOUNTAIN_AREA_ELEVATION_AREA, MOUNTAIN_AREA_ELEVATION_AREA, size=2) +def random_elevate_agent(ground, position, height, size=p['mountain_area_elevation_n']): + position = position + np.random.random_integers(-p['mountain_area_elevation_area'], p['mountain_area_elevation_area'], size=2) for i in range(size): d = DIRECTIONS[np.random.randint(len(DIRECTIONS))] - change = height * MOUNTAIN_AREA_ELEVATION + np.random.randint(MOUNTAIN_JAGGEDNESS + 1) + change = height * p['mountain_area_elevation'] + np.random.randint(p['mountain_jaggedness'] + 1) new_index = bound_check(ground, position + np.array(d)) if is_ground(ground[new_index]): @@ -118,21 +195,21 @@ def mountain_agent(ground, position): return x, y = position - height = np.random.randint(MAX_ELEVATION) + height = np.random.randint(p['max_elevation']) ground[x, y] = height last_height = height for i in range(1, height): for d in DIRECTIONS: - change = np.random.randint(MOUNTAIN_JAGGEDNESS + 1) + change = np.random.randint(p['mountain_jaggedness'] + 1) distance = np.array(d)*i new_index = bound_check(ground, position + distance) if is_ground(ground[new_index]): ground[new_index] = last_height - change - last_height = last_height - MOUNTAIN_JAGGEDNESS + last_height = last_height - p['mountain_jaggedness'] random_elevate_agent(ground, position, height) @@ -141,13 +218,22 @@ def mountain_agent(ground, position): # def split_agent(ground, position, directions): def constant_filter(a): - if a[0] > (1 - SHARPNESS): + if a[0] > (1 - p['sharpness']): return max(1, a[0]) return 0 -def generate_map(width=WIDTH, height=HEIGHT, continents=4): +def generate_map(**kwargs): + plt.clf() + + p.update(kwargs) + + np.random.seed(p['seed'] or None) + + width, height = p['width'], p['height'] + continents = p['continents'] + ground = np.zeros((width, height)) - ground_size = width * height * GROUND_PROPORTION + ground_size = width * height * (1 - p['water_proportion']) # position = (int(width / 2), int(height / 2)) # ground_size = width * height * GROUND_PROPORTION @@ -157,9 +243,9 @@ def generate_map(width=WIDTH, height=HEIGHT, continents=4): print(position) continent_agent(ground, position, size=ground_size) - ground = ndimage.gaussian_filter(ground, sigma=(1 - SHARPNESS) * 20) + ground = ndimage.gaussian_filter(ground, sigma=(1 - p['sharpness']) * 20) - for i in range(int(ground_size * MOUNTAIN_RATIO / MAX_ELEVATION**2)): + for i in range(int(ground_size * p['mountain_ratio'] / p['max_elevation']**2)): position = (np.random.randint(0, width), np.random.randint(0, height)) mountain_agent(ground, position) @@ -169,7 +255,7 @@ def generate_map(width=WIDTH, height=HEIGHT, continents=4): ground = ndimage.gaussian_filter(ground, sigma=4) ground = ndimage.generic_filter(ground, constant_filter, size=1) - print(np.min(ground), np.max(ground), MAX_ELEVATION) + print(np.min(ground), np.max(ground), p['max_elevation']) print(np.unique(ground)) plt.imshow(ground.T, cmap=greys, norm=norm) @@ -181,5 +267,5 @@ def generate_map(width=WIDTH, height=HEIGHT, continents=4): return figfile if __name__ == "__main__": - generate_map(WIDTH, HEIGHT) + generate_map() plt.show() diff --git a/map-generator/static/script.js b/map-generator/static/script.js index fd9b225..61a7a56 100644 --- a/map-generator/static/script.js +++ b/map-generator/static/script.js @@ -3,15 +3,26 @@ const mapSettings = document.getElementById('map-settings'); const map = document.getElementById('map'); const spinner = document.getElementById('spinner'); +function generate() { + spinner.classList.remove('d-none'); + const formData = new FormData(mapSettings) + if (!formData.get('seed')) { + formData.set('seed', (new Date()).getTime() % 1e5); + } + const queryString = new URLSearchParams(formData).toString() + map.src = '/map?' + queryString; + map.classList.add('d-none'); + +} + mapSettings.addEventListener('submit', (e) => { - console.log('form submit'); e.preventDefault(); - spinner.classList.remove('d-none'); - map.src = '/map?q=' + (new Date()).getTime(); - map.classList.add('d-none'); + generate() }); +generate() + map.addEventListener('load', () => { spinner.classList.add('d-none'); map.classList.remove('d-none'); diff --git a/map-generator/static/style.css b/map-generator/static/style.css index eade526..9cd689f 100644 --- a/map-generator/static/style.css +++ b/map-generator/static/style.css @@ -10,5 +10,6 @@ aside { text-align: center; width: 300px; height: 100vh; + overflow-y: scroll; } diff --git a/map-generator/templates/index.html b/map-generator/templates/index.html index fe31615..aef86ab 100644 --- a/map-generator/templates/index.html +++ b/map-generator/templates/index.html @@ -11,7 +11,7 @@
- +