From d9654749742cb2cf07e23df4abedf479a23271d6 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Sat, 18 May 2019 16:17:28 +0430 Subject: [PATCH] feat(map-generator): use biome models for generating the biome layer --- biomes/draw.py | 24 ++- .../index.py => biomes/map_generator.py | 174 ++++++++++++++++-- biomes/predict.py | 14 +- .../static/bootstrap.min.css | 0 .../static/bootstrap.min.css.map | 0 .../static/bootstrap.min.js | 0 .../static/bootstrap.min.js.map | 0 {map-generator => biomes}/static/script.js | 0 {map-generator => biomes}/static/style.css | 0 .../templates/index.html | 0 biomes/utils.py | 24 ++- {map-generator => biomes}/web.py | 2 +- map-generator/requirements.txt | 0 13 files changed, 204 insertions(+), 34 deletions(-) rename map-generator/index.py => biomes/map_generator.py (62%) rename {map-generator => biomes}/static/bootstrap.min.css (100%) rename {map-generator => biomes}/static/bootstrap.min.css.map (100%) rename {map-generator => biomes}/static/bootstrap.min.js (100%) rename {map-generator => biomes}/static/bootstrap.min.js.map (100%) rename {map-generator => biomes}/static/script.js (100%) rename {map-generator => biomes}/static/style.css (100%) rename {map-generator => biomes}/templates/index.html (100%) rename {map-generator => biomes}/web.py (92%) delete mode 100644 map-generator/requirements.txt diff --git a/biomes/draw.py b/biomes/draw.py index 7008d8f..4c74baf 100644 --- a/biomes/draw.py +++ b/biomes/draw.py @@ -2,26 +2,33 @@ import fire import matplotlib.pyplot as plt from matplotlib.collections import PatchCollection from matplotlib.patches import Circle, Patch -from utils import logger +from utils import logger, to_range from constants import BIOMES import pandas as pd import cartopy.crs as ccrs -def draw(df, path=None): +def draw(df, earth=True, width=23.22, height=13, only_draw=False, path=None): logger.debug('draw(df, %s)', path) biomes = {} biome_numbers = df['biome_num'].unique() for i, row in df.iterrows(): - p = (row.longitude, row.latitude) + if earth: + p = (row.longitude, row.latitude) + else: + p = (to_range(-180, 180, 0, width)(row.longitude), to_range(-90, 90, 0, height)(row.latitude)) + if row.biome_num in biomes: biomes[row.biome_num].append(p) else: biomes[row.biome_num] = [p] - ax = plt.axes(projection=ccrs.PlateCarree()) - ax.stock_img() + if earth: + ax = plt.axes(projection=ccrs.PlateCarree()) + ax.stock_img() + else: + ax = plt.gca() legend_handles = [] for n in biome_numbers: @@ -34,11 +41,14 @@ def draw(df, path=None): ax.add_collection(collection) ax.legend(handles=legend_handles, loc='center left', bbox_to_anchor=(1, 0.5), markerscale=4) - + ax.autoscale_view() figure = plt.gcf() - figure.set_size_inches(23.22, 13) + figure.set_size_inches(width, height) figure.subplots_adjust(left=0.02, right=0.79) + + if only_draw: return + if path: plt.savefig(path) else: diff --git a/map-generator/index.py b/biomes/map_generator.py similarity index 62% rename from map-generator/index.py rename to biomes/map_generator.py index c0424d8..9a99973 100644 --- a/map-generator/index.py +++ b/biomes/map_generator.py @@ -9,10 +9,15 @@ from io import BytesIO import pandas as pd from shapely.geometry import Point, MultiPoint from descartes import PolygonPatch +from constants import INPUTS, SEASONS +from draw import draw +from train import A_params +from model import Model +from utils import * parameters = { 'width': { - 'default': 900, + 'default': 700, 'type': 'int', }, 'height': { @@ -34,16 +39,22 @@ parameters = { 'step': 0.01 }, 'max_elevation': { - 'default': 30, + 'default': 1e4, 'type': 'int', 'min': 0, - 'max': 50, + 'max': 1e4, + }, + 'min_elevation': { + 'default': -400, + 'type': 'int', + 'min': -1000, + 'max': 0 }, 'ground_noise': { - 'default': 15, + 'default': 1.1e4, 'type': 'int', 'min': 0, - 'max': 50, + 'max': 1e5, }, 'water_proportion': { 'default': 0.6, @@ -109,6 +120,16 @@ parameters = { 'default': False, 'type': 'bool' }, + 'mean_temperature': { + 'default': -4.2, + 'type': 'float', + 'step': 1, + }, + 'mean_precipitation': { + 'default': 45.24, + 'type': 'float', + 'step': 1, + }, 'seed': { 'default': '', 'type': 'int', @@ -156,6 +177,9 @@ def bound_check(ground, point): elif y >= h: y = y - h + if x < 0 or x >= w or y < 0 or y >= h: + return bound_check(ground, (x, y)) + return (x, y) @@ -190,7 +214,7 @@ def continent_agent(ground, position, size): if not is_ground(ground[x, y]) and in_range((x, y), position, size**2 * np.pi): trials = 0 size -= 1 - ground[x, y] = np.random.randint(1, p['ground_noise']) + ground[x, y] = np.random.randint(p['water_level'] + 1, p['ground_noise']) else: trials += 1 @@ -218,6 +242,7 @@ def random_elevate_agent(ground, position, height, size=p['mountain_area_elevati def mountain_agent(ground, position): + print('mountain_agent') if not away_from_sea(ground, position): return @@ -283,11 +308,11 @@ def generate_map(biomes=False, **kwargs): ground = ndimage.gaussian_filter(ground, sigma=(1 - p['sharpness']) * 20) - for i in range(int(ground_size * p['mountain_ratio'] / p['max_elevation']**2)): + for i in range(int(ground_size * p['mountain_ratio'] / (p['max_elevation'] / 2))): position = (np.random.randint(0, width), np.random.randint(0, height)) mountain_agent(ground, position) - norm = colors.Normalize(vmin=1) + norm = colors.Normalize(vmin=p['water_level'] + 1) greys = cm.get_cmap('Greys') greys.set_under(color=SEA_COLOR) @@ -311,12 +336,6 @@ def generate_map(biomes=False, **kwargs): return figfile -def to_range(omin, omax, nmin, nmax): - orange = omax - omin - nrange = nmax - nmin - - return lambda x: ((x - omin) * nrange / orange) + nmin - def generate_biomes(ground): width, height = p['width'], p['height'] @@ -325,7 +344,6 @@ def generate_biomes(ground): width_to_longitude = to_range(0, width, -180, 180) print('generate_biomes') - INPUTS = ['elevation', 'distance_to_water', 'latitude'] data = {} for col in ['longitude', 'latitude', 'elevation', 'distance_to_water']: @@ -341,17 +359,24 @@ def generate_biomes(ground): data['latitude'].append(height_to_latitude(y)) data['elevation'].append(v) + print(len(points)) + print('buffering points') points = MultiPoint(points) - boundary = points.buffer(1e-0).boundary + boundary = points.buffer(1).boundary for x, y in np.ndindex(ground.shape): if ground[x,y] > p['water_level']: - # print(x,y, Point(x,y).distance(boundary)) data['distance_to_water'].append(Point(x, y).distance(boundary)) df = pd.DataFrame(data) - print(df) + print(df['elevation'].min(), df['elevation'].max()) + print(df['distance_to_water'].min(), df['distance_to_water'].max()) + print(df['latitude'].min(), df['latitude'].max()) + + print('running prediction models') + print(p['mean_precipitation'], p['mean_temperature']) + result = predict_end_to_end(df, boundary) # fig = plt.figure() # ax = fig.add_subplot(111) @@ -365,7 +390,118 @@ def generate_biomes(ground): # plt.show() +df = pd.read_pickle('data.p') +print(df['elevation'].min(), df['elevation'].max()) +print(df['distance_to_water'].min(), df['distance_to_water'].max()) +print(df['latitude'].min(), df['latitude'].max()) +def predict_end_to_end(input_df, boundary, checkpoint_temp='checkpoints/temp.h5', checkpoint_precip='checkpoints/precip.h5', checkpoint_biomes='checkpoints/b.h5', year=2000): + batch_size = A_params['batch_size']['grid_search'][0] + layers = A_params['layers']['grid_search'][0] + optimizer = A_params['optimizer']['grid_search'][0](A_params['lr']['grid_search'][0]) + + Temp = Model('temp', epochs=1) + Temp.prepare_for_use( + batch_size=batch_size, + layers=layers, + dataset_fn=dataframe_to_dataset_temp, + optimizer=optimizer, + out_activation=None, + loss='mse', + metrics=['mae'] + ) + Temp.restore(checkpoint_temp) + + Precip = Model('precip', epochs=1) + Precip.prepare_for_use( + batch_size=batch_size, + layers=layers, + dataset_fn=dataframe_to_dataset_temp, + optimizer=optimizer, + out_activation=None, + loss='mse', + metrics=['mae'] + ) + Precip.restore(checkpoint_precip) + + Biomes = Model('b', epochs=1) + Biomes.prepare_for_use() + Biomes.restore(checkpoint_biomes) + + inputs = input_df[INPUTS] + + inputs.loc[:, 'mean_temp'] = p['mean_temperature'] + inputs_copy = inputs.copy() + inputs_copy.loc[:, 'mean_temp'] = mean_temperature_over_years(df, size=inputs.shape[0]) + + inputs = inputs.to_numpy() + inputs = normalize_ndarray(inputs, inputs_copy) + print(inputs) + out_columns = ['temp_{}_{}'.format(season, year) for season in SEASONS] + out = Temp.predict(inputs) + temp_output = pd.DataFrame(data=denormalize(out, df[out_columns].to_numpy()), columns=out_columns) + + inputs = input_df[INPUTS] + + inputs.loc[:, 'mean_precip'] = p['mean_precipitation'] + inputs_copy = inputs.copy() + inputs_copy.loc[:, 'mean_precip'] = mean_precipitation_over_years(df, size=inputs.shape[0]) + + inputs = inputs.to_numpy() + inputs = normalize_ndarray(inputs, inputs_copy) + print(inputs) + out_columns = ['precip_{}_{}'.format(season, year) for season in SEASONS] + out = Precip.predict(inputs) + + precip_output = pd.DataFrame(data=denormalize(out, df[out_columns].to_numpy()), columns=out_columns) + + inputs = list(INPUTS) + + frame = input_df[inputs + ['longitude']] + + for season in SEASONS: + tc = 'temp_{}_{}'.format(season, year) + pc = 'precip_{}_{}'.format(season, year) + frame.loc[:, tc] = temp_output[tc] + frame.loc[:, pc] = precip_output[pc] + + frame.loc[:, 'latitude'] = input_df['latitude'] + + frame_cp = frame.copy() + + columns = ['latitude', 'longitude', 'biome_num'] + new_data = pd.DataFrame(columns=columns) + nframe = pd.DataFrame(columns=frame.columns, data=normalize_ndarray(frame.to_numpy())) + + for season in SEASONS: + inputs += [ + 'temp_{}_{}'.format(season, year), + 'precip_{}_{}'.format(season, year) + ] + + for i, (chunk, chunk_original) in enumerate(zip(chunker(nframe, Biomes.batch_size), chunker(frame_cp, Biomes.batch_size))): + if chunk.shape[0] < Biomes.batch_size: + continue + input_data = chunk.loc[:, inputs].values + out = Biomes.predict_class(input_data) + + f = pd.DataFrame({ + 'longitude': chunk_original.loc[:, 'longitude'], + 'latitude': chunk_original.loc[:, 'latitude'], + 'biome_num': out + }, columns=columns) + new_data = new_data.append(f) + + #print(new_data) + draw(new_data, earth=False, only_draw=True, width=p['width'], height=p['height']) + +# TODO: reduce opacity of biome layer if __name__ == "__main__": - generate_map() + # p['width'] = 50 + # p['height'] = 50 + p['water_proportion'] = 0.9 + p['continents'] = 3 + p['seed'] = 1 + generate_map(True) + # print(normalize_ndarray(np.array([[ 5.59359803,0.99879546,-90., 45.24], [ 5.54976747, 0.99879546,-86.4, 45.24 ]]))) plt.show() diff --git a/biomes/predict.py b/biomes/predict.py index 5c382e5..ca3f330 100644 --- a/biomes/predict.py +++ b/biomes/predict.py @@ -147,6 +147,7 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000): all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS] inputs.loc[:, 'mean_temp'] = np.mean(df[all_temps].values) + print(inputs['mean_temp']) inputs = inputs.to_numpy() inputs = normalize_ndarray(inputs) @@ -158,6 +159,7 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000): all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS] inputs.loc[:, 'mean_precip'] = np.mean(df[all_precips].values) + print(inputs['mean_precip']) inputs = inputs.to_numpy() inputs = normalize_ndarray(inputs) @@ -168,12 +170,6 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000): inputs = list(INPUTS) - for season in SEASONS: - inputs += [ - 'temp_{}_{}'.format(season, year), - 'precip_{}_{}'.format(season, year) - ] - frame = df[inputs + ['longitude']] for season in SEASONS: @@ -190,6 +186,12 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000): new_data = pd.DataFrame(columns=columns) nframe = pd.DataFrame(columns=frame.columns, data=normalize_ndarray(frame.to_numpy())) + for season in SEASONS: + inputs += [ + 'temp_{}_{}'.format(season, year), + 'precip_{}_{}'.format(season, year) + ] + for i, (chunk, chunk_original) in enumerate(zip(chunker(nframe, Biomes.batch_size), chunker(frame_cp, Biomes.batch_size))): if chunk.shape[0] < Biomes.batch_size: continue diff --git a/map-generator/static/bootstrap.min.css b/biomes/static/bootstrap.min.css similarity index 100% rename from map-generator/static/bootstrap.min.css rename to biomes/static/bootstrap.min.css diff --git a/map-generator/static/bootstrap.min.css.map b/biomes/static/bootstrap.min.css.map similarity index 100% rename from map-generator/static/bootstrap.min.css.map rename to biomes/static/bootstrap.min.css.map diff --git a/map-generator/static/bootstrap.min.js b/biomes/static/bootstrap.min.js similarity index 100% rename from map-generator/static/bootstrap.min.js rename to biomes/static/bootstrap.min.js diff --git a/map-generator/static/bootstrap.min.js.map b/biomes/static/bootstrap.min.js.map similarity index 100% rename from map-generator/static/bootstrap.min.js.map rename to biomes/static/bootstrap.min.js.map diff --git a/map-generator/static/script.js b/biomes/static/script.js similarity index 100% rename from map-generator/static/script.js rename to biomes/static/script.js diff --git a/map-generator/static/style.css b/biomes/static/style.css similarity index 100% rename from map-generator/static/style.css rename to biomes/static/style.css diff --git a/map-generator/templates/index.html b/biomes/templates/index.html similarity index 100% rename from map-generator/templates/index.html rename to biomes/templates/index.html diff --git a/biomes/utils.py b/biomes/utils.py index 8d8059d..71e1401 100644 --- a/biomes/utils.py +++ b/biomes/utils.py @@ -6,15 +6,17 @@ from sklearn.utils import class_weight from constants import * import logging import os +from math import ceil logger = logging.getLogger('main') logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) +EPSILON = 1e-5 def normalize(v, o=None): if o is None: o = v - return (v - np.mean(o)) / np.std(o) + return (v - np.mean(o)) / max(EPSILON, np.std(o)) def denormalize(v, o=None): if o is None: @@ -132,8 +134,28 @@ def dataframe_to_dataset_precip(df): logger.debug('dataset size: rows=%d, input_columns=%d, num_classes=%d', int(tf_inputs.shape[0]), input_columns, num_classes) return int(tf_inputs.shape[0]), input_columns, num_classes, None, tf.data.Dataset.from_tensor_slices((tf_inputs, tf_output)) +def mean_temperature_over_years(df, size=MAX_YEAR - MIN_YEAR): + means = [] + for year in range(MIN_YEAR, MAX_YEAR + 1): + all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS] + means.append(np.mean(df[all_temps].values)) + return (means * ceil(size / len(means)))[0:size] + +def mean_precipitation_over_years(df, size=MAX_YEAR - MIN_YEAR): + means = [] + for year in range(MIN_YEAR, MAX_YEAR + 1): + all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS] + means.append(np.mean(df[all_precips].values)) + return (means * ceil(size / len(means)))[0:size] flatten = lambda l: [item for sublist in l for item in sublist] def chunker(seq, size): return (seq[pos:pos + size] for pos in range(0, len(seq), size)) + +def to_range(omin, omax, nmin, nmax): + orange = omax - omin + nrange = nmax - nmin + + return lambda x: ((x - omin) * nrange / orange) + nmin + diff --git a/map-generator/web.py b/biomes/web.py similarity index 92% rename from map-generator/web.py rename to biomes/web.py index 3fda4a5..df44e89 100644 --- a/map-generator/web.py +++ b/biomes/web.py @@ -1,5 +1,5 @@ from flask import Flask, render_template, make_response, send_file, request -from index import generate_map, parameters +from map_generator import generate_map, parameters app = Flask(__name__) diff --git a/map-generator/requirements.txt b/map-generator/requirements.txt deleted file mode 100644 index e69de29..0000000