diff --git a/README.md b/README.md index 3014c33..ddbd4af 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,22 @@ After reading [Evolution Strategies as a Scalable Alternative to Reinforcement L The model learns to play very well after 3000 epochs, but not completely flawless and it rarely loses in difficult cases (high difference between two wall entrances). Training process is pretty fast as there is no backpropagation, and is not very costy in terms of memory as there is no need to record actions as in policy gradients. -Here is a demonstration of the model after 3000 epochs (~5 minutes on an Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz): +Here is a demonstration of the model after 3000 epochs (~5 minutes on an Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz): ![after training](/demo/flappy-success.gif) -also see: [Before training](/demo/flappy-lose.gif) +Before training: + +![Before training](/demo/flappy-lose.gif) For each frame the bird stays alive, +0.1 score is given to him. For each wall he passes, +10 score is given. +Demonstration of rewards for individuals and the mean reward over time (y axis is logarithmic): +![reward chart](fig-log.jpg) + Try it yourself --------------- -You need python3 and pip for installing and running the code. +You need python3.5 and pip for installing and running the code. First, install dependencies (you might want to create a [virtualenv](https://virtualenv.pypa.io)): @@ -43,3 +48,18 @@ the updates become pretty large and will pull the model too much to sides, thus To try it yourself, there is a `long.npy` file, rename it to `load.npy` (backup `load.npy` before doing so) and run `demo.py`, you will see the bird failing more often than not. `long.py` was trained for only 100 more epochs than `load.npy`. + +Web Server +---------- + +To setup the web server, after installing the dependencies by running `pip install -r requirements`, follow the steps below: + +``` +cd web +npm install +node server.js +``` + +You may want to use [pm2](https://github.com/Unitech/pm2) to run the server as a daemon in the background. + +Now you can redirect traffic to the port the server is listening on (8088 by default). diff --git a/draw_chart.py b/draw_chart.py new file mode 100644 index 0000000..9f24304 --- /dev/null +++ b/draw_chart.py @@ -0,0 +1,39 @@ +import os +import plotly.plotly as py +import plotly.graph_objs as go +import plotly.io as pio + +def draw_chart(scatter_x, scatter_y, line_x, line_y): + marker_opts = dict( + name = 'Individual Reward', + x = scatter_x, + y = scatter_y, + mode = 'markers', + marker = { + 'size': 2 + } + ) + marker_trace = go.Scatter(**marker_opts) + line_opts = dict( + name = 'Mean Reward', + x = line_x, + y = line_y, + mode = 'lines' + ) + line_trace = go.Scatter(**line_opts) + + data = [marker_trace, line_trace] + + fig = go.Figure(data=data) + + log_layout = go.Layout( + yaxis = { + 'type': 'log', + 'autorange': True + } + ) + fig_log = go.Figure(data=data, layout=log_layout) + + pio.orca.config.executable = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'node_modules', 'orca', 'bin', 'orca.js') + pio.write_image(fig, 'fig.svg', width=1600, height=800) + pio.write_image(fig_log, 'fig-log.svg', width=1600, height=800) diff --git a/es.py b/es.py index 1002b5e..b4b88df 100644 --- a/es.py +++ b/es.py @@ -69,5 +69,5 @@ class EvolutionStrategy(): # np.dot(n.T, A) scales each noise's contribution to the update by how much reward it had received update = self.alpha / (self.noisep * self.sigma) * np.dot(n.T, A).T self.layers[i] = self.layers[i] + update - + return R diff --git a/fig-log.jpg b/fig-log.jpg new file mode 100644 index 0000000..34b59e7 Binary files /dev/null and b/fig-log.jpg differ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..c9ed7c1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,713 @@ +{ + "name": "flappy-es", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "ajv": { + "version": "6.6.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.6.1.tgz", + "integrity": "sha512-ZoJjft5B+EJBjUyu9C9Hc0OZyPZSSlOF+plzouTrg6UlA8f+e/n8NIgBFG/9tppJtpPWfthHakK7juJdNDODww==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "body": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/body/-/body-5.1.0.tgz", + "integrity": "sha1-5LoM5BCkaTYyM2dgnstOZVMSUGk=", + "dev": true, + "requires": { + "continuable-cache": "^0.3.1", + "error": "^7.0.0", + "raw-body": "~1.1.0", + "safe-json-parse": "~1.0.1" + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "bytes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-1.0.0.tgz", + "integrity": "sha1-NWnt6Lo0MV+rmcPpLLBMciDeH6g=", + "dev": true + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "continuable-cache": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/continuable-cache/-/continuable-cache-0.3.1.tgz", + "integrity": "sha1-vXJ6f67XfnH/OYWskzUakSczrQ8=", + "dev": true + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "error": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/error/-/error-7.0.2.tgz", + "integrity": "sha1-pfdf/02ZJhJt2sDqXcOOaJFTywI=", + "dev": true, + "requires": { + "string-template": "~0.2.1", + "xtend": "~4.0.0" + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true + }, + "fast-isnumeric": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/fast-isnumeric/-/fast-isnumeric-1.1.2.tgz", + "integrity": "sha512-D7zJht1+NZBBv4759yXn/CJFUNJpILdgdosPFN1AjqQn9TfQJqSeCZfu0SY4bwIlXuDhzkxKoQ8BOqdiXpVzvA==", + "dev": true, + "requires": { + "is-string-blank": "^1.0.1" + } + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "file-type": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-9.0.0.tgz", + "integrity": "sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw==", + "dev": true + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "get-stdin": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", + "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=", + "dev": true + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0" + } + }, + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "dev": true, + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "dev": true + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", + "dev": true + }, + "is-string-blank": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-string-blank/-/is-string-blank-1.0.1.tgz", + "integrity": "sha512-9H+ZBCVs3L9OYqv8nuUAzpcT9OTgMD1yAWrG7ihlnibdkbtB850heAmYWxHuXc4CHy4lKeK69tN+ny1K7gBIrw==", + "dev": true + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "mime-db": { + "version": "1.37.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", + "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==", + "dev": true + }, + "mime-types": { + "version": "2.1.21", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", + "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "dev": true, + "requires": { + "mime-db": "~1.37.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "orca": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/orca/-/orca-1.1.1.tgz", + "integrity": "sha512-5rqQJpxpNOWdk/Z0FCbK9xEOfHfv120+xxW0niGFBk7FrmVlE8/D1tYYRCAZrRKIkgWxNOWzVUpySzvmM2SGqw==", + "dev": true, + "requires": { + "body": "^5.1.0", + "fast-isnumeric": "^1.1.1", + "file-type": "^9.0.0", + "get-stdin": "^5.0.1", + "glob": "^7.1.2", + "is-plain-obj": "^1.1.0", + "is-url": "^1.2.4", + "minimist": "^1.2.0", + "read-chunk": "^2.1.0", + "request": "^2.88.0", + "run-parallel": "^1.1.9", + "run-parallel-limit": "^1.0.5", + "run-series": "^1.1.8", + "semver": "^5.4.1", + "string-to-stream": "^1.1.1", + "uuid": "^3.3.2" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "dev": true + }, + "psl": { + "version": "1.1.29", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", + "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==", + "dev": true + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true + }, + "raw-body": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-1.1.7.tgz", + "integrity": "sha1-HQJ8K/oRasxmI7yo8AAWVyqH1CU=", + "dev": true, + "requires": { + "bytes": "1", + "string_decoder": "0.10" + } + }, + "read-chunk": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-2.1.0.tgz", + "integrity": "sha1-agTAkoAF7Z1C4aasVgDhnLx/9lU=", + "dev": true, + "requires": { + "pify": "^3.0.0", + "safe-buffer": "^5.1.1" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + }, + "dependencies": { + "string_decoder": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "dev": true, + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==", + "dev": true + }, + "run-parallel-limit": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.0.5.tgz", + "integrity": "sha512-NsY+oDngvrvMxKB3G8ijBzIema6aYbQMD2bHOamvN52BysbIGTnEY2xsNyfrcr9GhY995/t/0nQN3R3oZvaDlg==", + "dev": true + }, + "run-series": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/run-series/-/run-series-1.1.8.tgz", + "integrity": "sha512-+GztYEPRpIsQoCSraWHDBs9WVy4eVME16zhOtDB4H9J4xN0XRhknnmLOl+4gRgZtu8dpp9N/utSPjKH/xmDzXg==", + "dev": true + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "safe-json-parse": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/safe-json-parse/-/safe-json-parse-1.0.1.tgz", + "integrity": "sha1-PnZyPjjf3aE8mx0poeB//uSzC1c=", + "dev": true + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "dev": true + }, + "sshpk": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", + "integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==", + "dev": true, + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "string-template": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz", + "integrity": "sha1-QpMuWYo1LQH8IuwzZ9nYTuxsmt0=", + "dev": true + }, + "string-to-stream": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz", + "integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==", + "dev": true, + "requires": { + "inherits": "^2.0.1", + "readable-stream": "^2.1.0" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", + "dev": true + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "dev": true, + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + } + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "dev": true + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", + "dev": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..077f999 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "flappy-es", + "version": "1.0.0", + "description": "Playing Flappy Bird using Evolution Strategies ==============================================", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/mdibaiee/flappy-es.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/mdibaiee/flappy-es/issues" + }, + "homepage": "https://github.com/mdibaiee/flappy-es#readme", + "devDependencies": { + "orca": "^1.1.1" + } +} diff --git a/requirements b/requirements index c921e12..16c3e6b 100644 --- a/requirements +++ b/requirements @@ -1,31 +1,4 @@ -blinker==1.3 -chardet==2.3.0 -cloud-init==0.7.9 -command-not-found==0.3 -configobj==5.0.6 -cryptography==1.2.3 -idna==2.0 -Jinja2==2.8 -jsonpatch==1.10 -jsonpointer==1.9 -language-selector==0.1 -MarkupSafe==0.23 numpy==1.12.1 -oauthlib==1.0.3 -prettytable==0.7.2 -pyasn1==0.1.9 -pycurl==7.43.0 -pygobject==3.20.0 -PyJWT==1.3.0 -pyserial==3.0.1 -python-apt==1.1.0b1 -python-debian==0.1.27 -python-systemd==231 -PyYAML==3.11 -requests==2.9.1 -six==1.10.0 -ssh-import-id==5.5 -ufw==0.35 -unattended-upgrades==0.1 -urllib3==1.13.1 -virtualenv==15.0.1 +plotly==3.4.0 +psutil==5.4.6 +PyGObject==3.30.4 diff --git a/train.py b/train.py index 36379e4..3649941 100644 --- a/train.py +++ b/train.py @@ -7,6 +7,7 @@ from gi.repository import Gtk, GLib, Gdk from datetime import datetime from os import path import os +from draw_chart import draw_chart es = EvolutionStrategy(fn=play, noisep=50, sigma=0.1, alpha=0.001, layer_sizes=[[4, 500], [500, 1]], input_size=4) load = path.join(path.dirname(__file__), 'load.npy') @@ -14,13 +15,15 @@ load = path.join(path.dirname(__file__), 'load.npy') np.random.seed(0) # if load.npy exists, load the parameters from it -if path.exists(load): +if path.exists(load) and not os.environ.get('SKIP_LOAD'): es.layers = np.load(load) # show the game every n iterations -SHOW_EVERY = 100 +SHOW_EVERY = int(os.environ.get('SHOW_EVERY', 100)) # save the parameters every n iterations -SAVE_EVERY = 100 +SAVE_EVERY = int(os.environ.get('SAVE_EVERY', 100)) +# number of steps +STEPS = int(os.environ.get('STEPS', 10000)) # an id for saving the parameters in a folder run_id = str(datetime.now()) @@ -45,9 +48,22 @@ def timeout_kill(win, game): return True -for i in range(10000): +reward_scatter_x = [] +reward_scatter_y = [] + +reward_line_x = list(range(STEPS)) +reward_line_y = [] + +for i in range(STEPS): print("{}: ".format(i), end='') - es.train() + rewards = es.train() + m = np.mean(rewards) + + reward_line_y.append(m) + + for r in rewards: + reward_scatter_x.append(i) + reward_scatter_y.append(r) if SHOW_EVERY and i % SHOW_EVERY == 0: play(es.forward, step=step) @@ -60,4 +76,4 @@ for i in range(10000): if i % SAVE_EVERY == 0: p = path.join(path.dirname(__file__), 'saves', run_id, 'save-{}'.format(i)) np.save(p, es.layers) - + draw_chart(reward_scatter_x, reward_scatter_y, reward_line_x, reward_line_y) diff --git a/web/server.js b/web/server.js index fee7836..1a2e399 100644 --- a/web/server.js +++ b/web/server.js @@ -4,17 +4,18 @@ var express = require('express'), io = require('socket.io')(server) exec = require('child_process').exec compression = require('compression') - path = require('path'); + path = require('path'), + router = express.Router(); server.listen(8088); -app.use(express.static(__dirname + '/static')) -app.use(compression()) -app.use('/assets/', express.static(__dirname + '/../assets/')) +router.use(express.static(__dirname + '/static')) +router.use(compression()) +router.use('/assets/', express.static(__dirname + '/../assets/')) var record = path.resolve(__dirname, '../record.py'); -app.get('/play', function(request, response) { +router.get('/play', function(request, response) { var child = exec('python3 ' + record, { maxBuffer: 1024 * 5000 }, function(err, out, stderr) { if (err || stderr) { console.log(err || stderr) @@ -26,3 +27,5 @@ app.get('/play', function(request, response) { child.stdout.pipe(response); }); + +app.use('/flappy-bird', router); diff --git a/web/static/frame.html b/web/static/frame.html new file mode 100644 index 0000000..c464788 --- /dev/null +++ b/web/static/frame.html @@ -0,0 +1,83 @@ + + + Playing Flappy Bird using Evolution Strategies + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+ +
+ +
+ +
+ +
+
+
+
+
replay
+
+ + +
+
+
+
+
+ + +
+
+ +
+ Fetching a new game from the server... +
+ +
+ + + + +

Note: we have limited the game to run only for 10000 frames to avoid infinitely long games, if you want to watch the game indefinitely, get the python version and try it yourself.

+
+ +
+ + + + + + + + + + +
+ + + + + diff --git a/web/static/index.html b/web/static/index.html index ba7dc49..4e7f315 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -3,9 +3,6 @@ Playing Flappy Bird using Evolution Strategies - - - @@ -66,6 +63,8 @@ + +

Note: we have limited the game to run only for 10000 frames to avoid infinitely long games, if you want to watch the game indefinitely, get the python version and try it yourself.