thereadme/world-ecoregion
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(map-generator): use biome models for generating the biome layer

Browse Source
This commit is contained in:
Mahdi Dibaiee
2019-05-18 16:17:28 +04:30
parent f79c63abf8
commit d965474974
13 changed files with 204 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
biomes/draw.py
View File

@ -2,26 +2,33 @@ import fire
import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Circle, Patch
from utils import logger
from utils import logger, to_range
from constants import BIOMES
import pandas as pd
import cartopy.crs as ccrs
def draw(df, path=None):
def draw(df, earth=True, width=23.22, height=13, only_draw=False, path=None):
logger.debug('draw(df, %s)', path)
biomes = {}
biome_numbers = df['biome_num'].unique()
for i, row in df.iterrows():
p = (row.longitude, row.latitude)
if earth:
p = (row.longitude, row.latitude)
else:
p = (to_range(-180, 180, 0, width)(row.longitude), to_range(-90, 90, 0, height)(row.latitude))
if row.biome_num in biomes:
biomes[row.biome_num].append(p)
else:
biomes[row.biome_num] = [p]
ax = plt.axes(projection=ccrs.PlateCarree())
ax.stock_img()
if earth:
ax = plt.axes(projection=ccrs.PlateCarree())
ax.stock_img()
else:
ax = plt.gca()
legend_handles = []
for n in biome_numbers:
@ -34,11 +41,14 @@ def draw(df, path=None):
ax.add_collection(collection)
ax.legend(handles=legend_handles, loc='center left', bbox_to_anchor=(1, 0.5), markerscale=4)
ax.autoscale_view()
figure = plt.gcf()
figure.set_size_inches(23.22, 13)
figure.set_size_inches(width, height)
figure.subplots_adjust(left=0.02, right=0.79)
if only_draw: return
if path:
plt.savefig(path)
else:

507
biomes/map_generator.py Normal file
View File

@ -0,0 +1,507 @@
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import colors, cm
from matplotlib.collections import PatchCollection
import scipy.interpolate as interpolate
from scipy import ndimage
import math
from io import BytesIO
import pandas as pd
from shapely.geometry import Point, MultiPoint
from descartes import PolygonPatch
from constants import INPUTS, SEASONS
from draw import draw
from train import A_params
from model import Model
from utils import *
parameters = {
'width': {
'default': 700,
'type': 'int',
},
'height': {
'default': 450,
'type': 'int',
},
'mountain_ratio': {
'default': 0.3,
'type': 'float',
'min': 0,
'max': 1,
'step': 0.01
},
'sharpness': {
'default': 0.7,
'type': 'float',
'min': 0,
'max': 1,
'step': 0.01
},
'max_elevation': {
'default': 1e4,
'type': 'int',
'min': 0,
'max': 1e4,
},
'min_elevation': {
'default': -400,
'type': 'int',
'min': -1000,
'max': 0
},
'ground_noise': {
'default': 1.1e4,
'type': 'int',
'min': 0,
'max': 1e5,
},
'water_proportion': {
'default': 0.6,
'type': 'float',
'min': 0,
'max': 0.99,
'step': 0.01
},
'mountain_concentration': {
'default': 1,
'type': 'float',
'min': 0,
'max': 5,
'step': 0.1
},
'mountain_sea_distance': {
'default': 50,
'type': 'int',
'min': 0,
'max': 200,
},
'mountain_sea_threshold': {
'default': 2,
'type': 'int',
'min': 0,
'max': 5,
},
'water_level': {
'default': 0,
'type': 'int',
},
'mountain_area_elevation': {
'default': 0.4,
'type': 'float',
'min': 0,
'max': 1,
'step': 0.01
},
'mountain_area_elevation_points': {
'default': 5,
'type': 'int',
'min': 0,
'max': 15,
},
'mountain_area_elevation_area': {
'default': 10,
'type': 'int',
'min': 0,
'max': 25,
},
'continents': {
'default': 5,
'type': 'int',
},
'continent_spacing': {
'default': 0.3,
'type': 'float',
'min': 0,
'max': 1,
'step': 0.1
},
'biomes': {
'default': False,
'type': 'bool'
},
'mean_temperature': {
'default': -4.2,
'type': 'float',
'step': 1,
},
'mean_precipitation': {
'default': 45.24,
'type': 'float',
'step': 1,
},
'seed': {
'default': '',
'type': 'int',
'description': 'Leave empty for a random seed generated from the current timestamp.'
},
}
p = { k: parameters[k]['default'] for k in parameters }
CONTINENT_MAX_TRIALS = 1e4
SEA_COLOR = np.array((53, 179, 220, 255)) / 255
DIRECTIONS = [(-1, -1), (-1, 0), (-1, 1), (1, 1), (1, 0), (1, -1), (0, -1), (0, 1)]
def s(x):
return -2 * x**3 + 3 * x**2
def is_ground(value):
return value > p['water_level']
# TODO: should check as a sphere
def in_range(p, m, size):
x, y = p
mx, my = m
return ((x - mx)**2 + (y - my)**2) < size
def max_recursion(fn, max_recursion=0):
def f(*args, recursion=0, **kwargs):
if recursion > max_recursion:
return
return fn(*args, **kwargs)
def bound_check(ground, point):
x, y = point
w, h = ground.shape
if x < 0:
x = w + x
elif x >= w:
x = x - w
if y < 0:
y = h + y
elif y >= h:
y = y - h
if x < 0 or x >= w or y < 0 or y >= h:
return bound_check(ground, (x, y))
return (x, y)
def continent_agent(ground, position, size):
if size <= 0: return
x, y = position
w, h = ground.shape
trials = 0
while True:
# if trials > CONTINENT_MAX_TRIALS:
# print('couldnt proceed')
if size <= 0 or trials > CONTINENT_MAX_TRIALS: break
# if size <= 0: break
dx = np.random.randint(2) or -1
dy = np.random.randint(2) or -1
r = np.random.randint(3)
new_point = bound_check(ground, (x + dx, y + dy))
if r == 0:
x = new_point[0]
elif r == 1:
y = new_point[1]
else:
x, y = new_point
x, y = bound_check(ground, (x, y))
if not is_ground(ground[x, y]) and in_range((x, y), position, size**2 * np.pi):
trials = 0
size -= 1
ground[x, y] = np.random.randint(p['water_level'] + 1, p['ground_noise'])
else:
trials += 1
def neighbours(ground, position, radius):
x, y = position
return ground[x-radius:x+radius+1, y-radius:y+radius+1]
def away_from_sea(ground, position, radius=p['mountain_sea_distance']):
ns = neighbours(ground, position, radius).flatten()
sea = len([1 for x in ns if not is_ground(x)])
return sea < p['mountain_sea_threshold']
def random_elevate_agent(ground, position, height, size=p['mountain_area_elevation_points']):
position = position + np.random.random_integers(-p['mountain_area_elevation_area'], p['mountain_area_elevation_area'], size=2)
for i in range(size):
d = DIRECTIONS[np.random.randint(len(DIRECTIONS))]
change = height * p['mountain_area_elevation']
new_index = bound_check(ground, position + np.array(d))
if is_ground(ground[new_index]):
ground[new_index] += change
def mountain_agent(ground, position):
print('mountain_agent')
if not away_from_sea(ground, position):
return
x, y = position
height = np.random.randint(p['max_elevation'])
ground[x, y] = height
last_height = height
for i in range(1, height):
for d in DIRECTIONS:
change = np.random.randint(p['mountain_concentration'] + 1)
distance = np.array(d)*i
new_index = bound_check(ground, position + distance)
if is_ground(ground[new_index]):
ground[new_index] = last_height - change
last_height = last_height - change
if last_height < 0:
break
random_elevate_agent(ground, position, height)
# takes an initial position and a list of (direction, probability) tuples to walk on
# def split_agent(ground, position, directions):
def constant_filter(a):
if a[0] > (1 - p['sharpness']):
return max(1, a[0])
return 0
def generate_map(biomes=False, **kwargs):
plt.clf()
p.update(kwargs)
np.random.seed(p['seed'] or None)
width, height = p['width'], p['height']
continents = p['continents']
ground = np.zeros((width, height))
ground_size = width * height * (1 - p['water_proportion'])
print(ground_size / ground.size)
# position = (int(width / 2), int(height / 2))
# ground_size = width * height * GROUND_PROPORTION
# continent_agent(ground, position, size=ground_size)
position = (0, int(height / 2))
ym = 1
for continent in range(continents):
position = (position[0] + np.random.randint(p['continent_spacing'] * width * 0.8, p['continent_spacing'] * width * 1.2),
position[1] + ym * np.random.randint(p['continent_spacing'] * height * 0.8, p['continent_spacing'] * height * 1.2))
print(position)
ym = ym * -1
random_size = ground_size / continents
continent_agent(ground, position, size=random_size)
ground = ndimage.gaussian_filter(ground, sigma=(1 - p['sharpness']) * 20)
for i in range(int(ground_size * p['mountain_ratio'] / (p['max_elevation'] / 2))):
position = (np.random.randint(0, width), np.random.randint(0, height))
mountain_agent(ground, position)
norm = colors.Normalize(vmin=p['water_level'] + 1)
greys = cm.get_cmap('Greys')
greys.set_under(color=SEA_COLOR)
ground = ndimage.gaussian_filter(ground, sigma=4)
ground = ndimage.generic_filter(ground, constant_filter, size=1)
print(np.min(ground), np.max(ground), p['max_elevation'])
print(np.unique(ground))
print(np.count_nonzero(ground) / ground.size)
plt.gca().invert_yaxis()
plt.imshow(ground.T, cmap=greys, norm=norm)
plt.gca().invert_yaxis()
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0)
if biomes:
generate_biomes(ground)
return figfile
def generate_biomes(ground):
width, height = p['width'], p['height']
height_to_latitude = to_range(0, height, -90, 90)
width_to_longitude = to_range(0, width, -180, 180)
print('generate_biomes')
data = {}
for col in ['longitude', 'latitude', 'elevation', 'distance_to_water']:
data[col] = []
points = []
for x, y in np.ndindex(ground.shape):
v = ground[x,y]
if v > p['water_level']:
points.append((x, y))
data['longitude'].append(width_to_longitude(x))
data['latitude'].append(height_to_latitude(y))
data['elevation'].append(v)
print(len(points))
print('buffering points')
points = MultiPoint(points)
boundary = points.buffer(1).boundary
for x, y in np.ndindex(ground.shape):
if ground[x,y] > p['water_level']:
data['distance_to_water'].append(Point(x, y).distance(boundary))
df = pd.DataFrame(data)
print(df['elevation'].min(), df['elevation'].max())
print(df['distance_to_water'].min(), df['distance_to_water'].max())
print(df['latitude'].min(), df['latitude'].max())
print('running prediction models')
print(p['mean_precipitation'], p['mean_temperature'])
result = predict_end_to_end(df, boundary)
# fig = plt.figure()
# ax = fig.add_subplot(111)
# minx, miny, maxx, maxy = boundary.bounds
# w, h = maxx - minx, maxy - miny
# ax.set_xlim(minx - 0.2 * w, maxx + 0.2 * w)
# ax.set_ylim(miny - 0.2 * h, maxy + 0.2 * h)
# ax.set_aspect(1)
# ax.add_collection(PatchCollection([PolygonPatch(boundary.buffer(0.1), fc='red', ec='black', zorder=1)], match_original=True))
# plt.show()
df = pd.read_pickle('data.p')
print(df['elevation'].min(), df['elevation'].max())
print(df['distance_to_water'].min(), df['distance_to_water'].max())
print(df['latitude'].min(), df['latitude'].max())
def predict_end_to_end(input_df, boundary, checkpoint_temp='checkpoints/temp.h5', checkpoint_precip='checkpoints/precip.h5', checkpoint_biomes='checkpoints/b.h5', year=2000):
batch_size = A_params['batch_size']['grid_search'][0]
layers = A_params['layers']['grid_search'][0]
optimizer = A_params['optimizer']['grid_search'][0](A_params['lr']['grid_search'][0])
Temp = Model('temp', epochs=1)
Temp.prepare_for_use(
batch_size=batch_size,
layers=layers,
dataset_fn=dataframe_to_dataset_temp,
optimizer=optimizer,
out_activation=None,
loss='mse',
metrics=['mae']
)
Temp.restore(checkpoint_temp)
Precip = Model('precip', epochs=1)
Precip.prepare_for_use(
batch_size=batch_size,
layers=layers,
dataset_fn=dataframe_to_dataset_temp,
optimizer=optimizer,
out_activation=None,
loss='mse',
metrics=['mae']
)
Precip.restore(checkpoint_precip)
Biomes = Model('b', epochs=1)
Biomes.prepare_for_use()
Biomes.restore(checkpoint_biomes)
inputs = input_df[INPUTS]
inputs.loc[:, 'mean_temp'] = p['mean_temperature']
inputs_copy = inputs.copy()
inputs_copy.loc[:, 'mean_temp'] = mean_temperature_over_years(df, size=inputs.shape[0])
inputs = inputs.to_numpy()
inputs = normalize_ndarray(inputs, inputs_copy)
print(inputs)
out_columns = ['temp_{}_{}'.format(season, year) for season in SEASONS]
out = Temp.predict(inputs)
temp_output = pd.DataFrame(data=denormalize(out, df[out_columns].to_numpy()), columns=out_columns)
inputs = input_df[INPUTS]
inputs.loc[:, 'mean_precip'] = p['mean_precipitation']
inputs_copy = inputs.copy()
inputs_copy.loc[:, 'mean_precip'] = mean_precipitation_over_years(df, size=inputs.shape[0])
inputs = inputs.to_numpy()
inputs = normalize_ndarray(inputs, inputs_copy)
print(inputs)
out_columns = ['precip_{}_{}'.format(season, year) for season in SEASONS]
out = Precip.predict(inputs)
precip_output = pd.DataFrame(data=denormalize(out, df[out_columns].to_numpy()), columns=out_columns)
inputs = list(INPUTS)
frame = input_df[inputs + ['longitude']]
for season in SEASONS:
tc = 'temp_{}_{}'.format(season, year)
pc = 'precip_{}_{}'.format(season, year)
frame.loc[:, tc] = temp_output[tc]
frame.loc[:, pc] = precip_output[pc]
frame.loc[:, 'latitude'] = input_df['latitude']
frame_cp = frame.copy()
columns = ['latitude', 'longitude', 'biome_num']
new_data = pd.DataFrame(columns=columns)
nframe = pd.DataFrame(columns=frame.columns, data=normalize_ndarray(frame.to_numpy()))
for season in SEASONS:
inputs += [
'temp_{}_{}'.format(season, year),
'precip_{}_{}'.format(season, year)
]
for i, (chunk, chunk_original) in enumerate(zip(chunker(nframe, Biomes.batch_size), chunker(frame_cp, Biomes.batch_size))):
if chunk.shape[0] < Biomes.batch_size:
continue
input_data = chunk.loc[:, inputs].values
out = Biomes.predict_class(input_data)
f = pd.DataFrame({
'longitude': chunk_original.loc[:, 'longitude'],
'latitude': chunk_original.loc[:, 'latitude'],
'biome_num': out
}, columns=columns)
new_data = new_data.append(f)
#print(new_data)
draw(new_data, earth=False, only_draw=True, width=p['width'], height=p['height'])
# TODO: reduce opacity of biome layer
if __name__ == "__main__":
# p['width'] = 50
# p['height'] = 50
p['water_proportion'] = 0.9
p['continents'] = 3
p['seed'] = 1
generate_map(True)
# print(normalize_ndarray(np.array([[ 5.59359803,0.99879546,-90., 45.24], [ 5.54976747, 0.99879546,-86.4, 45.24 ]])))
plt.show()

14
biomes/predict.py
View File

@ -147,6 +147,7 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000):
all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS]
inputs.loc[:, 'mean_temp'] = np.mean(df[all_temps].values)
print(inputs['mean_temp'])
inputs = inputs.to_numpy()
inputs = normalize_ndarray(inputs)
@ -158,6 +159,7 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000):
all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS]
inputs.loc[:, 'mean_precip'] = np.mean(df[all_precips].values)
print(inputs['mean_precip'])
inputs = inputs.to_numpy()
inputs = normalize_ndarray(inputs)
@ -168,12 +170,6 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000):
inputs = list(INPUTS)
for season in SEASONS:
inputs += [
'temp_{}_{}'.format(season, year),
'precip_{}_{}'.format(season, year)
]
frame = df[inputs + ['longitude']]
for season in SEASONS:
@ -190,6 +186,12 @@ def predict_end_to_end(Temp, Precip, Biomes, year=2000):
new_data = pd.DataFrame(columns=columns)
nframe = pd.DataFrame(columns=frame.columns, data=normalize_ndarray(frame.to_numpy()))
for season in SEASONS:
inputs += [
'temp_{}_{}'.format(season, year),
'precip_{}_{}'.format(season, year)
]
for i, (chunk, chunk_original) in enumerate(zip(chunker(nframe, Biomes.batch_size), chunker(frame_cp, Biomes.batch_size))):
if chunk.shape[0] < Biomes.batch_size:
continue

7
biomes/static/bootstrap.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
biomes/static/bootstrap.min.css.map Normal file
View File

File diff suppressed because one or more lines are too long

7
biomes/static/bootstrap.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
biomes/static/bootstrap.min.js.map Normal file
View File

File diff suppressed because one or more lines are too long

29
biomes/static/script.js Normal file
View File

@ -0,0 +1,29 @@
const mapSettings = document.getElementById('map-settings');
//const board = document.getElementById('board');
const map = document.getElementById('map');
const spinner = document.getElementById('spinner');
function generate() {
spinner.classList.remove('d-none');
const formData = new FormData(mapSettings)
if (!formData.get('seed')) {
formData.set('seed', (new Date()).getTime() % 1e5);
}
const queryString = new URLSearchParams(formData).toString()
map.src = '/map?' + queryString;
map.classList.add('d-none');
map.width = formData.get('width');
}
mapSettings.addEventListener('submit', (e) => {
e.preventDefault();
generate()
});
generate()
map.addEventListener('load', () => {
spinner.classList.add('d-none');
map.classList.remove('d-none');
});

15
biomes/static/style.css Normal file
View File

@ -0,0 +1,15 @@
* {
box-sizing: border-box;
}
html, body {
margin: 0;
}
aside {
text-align: center;
width: 300px;
height: 100vh;
overflow-y: scroll;
}

56
biomes/templates/index.html Normal file
View File

@ -0,0 +1,56 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>World Map Generator</title>
<link rel='stylesheet' href="{{ url_for('static', filename='style.css') }}">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
</head>
<body class='container-fluid'>
<div class='row'>
<main class='col d-flex justify-content-center align-items-center'>
<!-- <canvas id='board'></canvas> -->
<img src='' id='map'>
<div class='spinner-border text-primary' role='status' id='spinner'>
<span class='sr-only'>Loading...</span>
</div>
</main>
<aside class='col-3 px-4 bg-dark text-light text-center py-3'>
<h3>World Map Generator</h3>
<div class='panel px-4 pb-3'>
<form class='mt-5' id='map-settings'>
{% for k, v in parameters.items() %}
<div class='form-group'>
<label name='{{ k }}'>{{ k | replace('_', ' ') | title }}</label>
{% if v["type"] == "bool" %}
<input id='{{ k }}' name='{{ k }}'
type='checkbox'
class='form-control'>
{% else %}
<input id='{{ k }}' name='{{ k }}'
type="number"
class='form-control'
min='{{ v["min"] }}' value='{{ v["default"] }}' max='{{ v["max"] }}' step='{{ v["step"] }}'>
{% endif %}
{% if v["description"] %}
<small class="form-text text-muted">{{ v["description"] }}</small>
{% endif %}
</div>
{% endfor %}
<button type="submit" class="btn btn-primary">Generate</button>
</form>
</div>
</aside>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
<script src="{{ url_for('static', filename='script.js') }}"></script>
</body>
</html>

24
biomes/utils.py
View File

@ -6,15 +6,17 @@ from sklearn.utils import class_weight
from constants import *
import logging
import os
from math import ceil
logger = logging.getLogger('main')
logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO'))
EPSILON = 1e-5
def normalize(v, o=None):
if o is None:
o = v
return (v - np.mean(o)) / np.std(o)
return (v - np.mean(o)) / max(EPSILON, np.std(o))
def denormalize(v, o=None):
if o is None:
@ -132,8 +134,28 @@ def dataframe_to_dataset_precip(df):
logger.debug('dataset size: rows=%d, input_columns=%d, num_classes=%d', int(tf_inputs.shape[0]), input_columns, num_classes)
return int(tf_inputs.shape[0]), input_columns, num_classes, None, tf.data.Dataset.from_tensor_slices((tf_inputs, tf_output))
def mean_temperature_over_years(df, size=MAX_YEAR - MIN_YEAR):
means = []
for year in range(MIN_YEAR, MAX_YEAR + 1):
all_temps = ['temp_{}_{}'.format(season, year) for season in SEASONS]
means.append(np.mean(df[all_temps].values))
return (means * ceil(size / len(means)))[0:size]
def mean_precipitation_over_years(df, size=MAX_YEAR - MIN_YEAR):
means = []
for year in range(MIN_YEAR, MAX_YEAR + 1):
all_precips = ['precip_{}_{}'.format(season, year) for season in SEASONS]
means.append(np.mean(df[all_precips].values))
return (means * ceil(size / len(means)))[0:size]
flatten = lambda l: [item for sublist in l for item in sublist]
def chunker(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
def to_range(omin, omax, nmin, nmax):
orange = omax - omin
nrange = nmax - nmin
return lambda x: ((x - omin) * nrange / orange) + nmin

26
biomes/web.py Normal file
View File

@ -0,0 +1,26 @@
from flask import Flask, render_template, make_response, send_file, request
from map_generator import generate_map, parameters
app = Flask(__name__)
@app.route('/')
def index():
return render_template('index.html', parameters=parameters)
def parse(key, value):
t = parameters[key]['type']
if t == 'int':
return int(value)
elif t == 'float':
return float(value)
elif t == 'bool':
return value == 'on'
else:
return value
@app.route('/map')
def get_map():
params = { key: parse(key, request.args[key]) for key in request.args }
res = send_file(generate_map(**params), mimetype='image/png')
return res