thereadme/flappy-es
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Charts #2

Closed
mdibaiee wants to merge 6 commits from charts into master
pull from: charts
merge into: thereadme:master
Conversation 0 Commits 6 Files Changed 13 +808 -39
8 changed files with 808 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 3df6e470a6 - Show all commits

10
README.md
View File

@ -7,15 +7,21 @@ The model learns to play very well after 3000 epochs, but not completely flawles
Training process is pretty fast as there is no backpropagation, and is not very costy in terms of memory as there is no need to record actions as in policy gradients. Training process is pretty fast as there is no backpropagation, and is not very costy in terms of memory as there is no need to record actions as in policy gradients.
Here is a demonstration of the model after 3000 epochs (~5 minutes on an Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz): Here is a demonstration of the model after 3000 epochs (~5 minutes on an Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz):
![after training](/demo/flappy-success.gif) ![after training](/demo/flappy-success.gif)
also see: [Before training](/demo/flappy-lose.gif) Before training:
![Before training](/demo/flappy-lose.gif)
For each frame the bird stays alive, +0.1 score is given to him. For each wall he passes, +10 score is given. For each frame the bird stays alive, +0.1 score is given to him. For each wall he passes, +10 score is given.
Demonstration of rewards for individuals and the mean reward over time (y axis is logarithmic):
![reward chart](fig-log.jpg)
Try it yourself Try it yourself
--------------- ---------------
You need python3 and pip for installing and running the code. You need python3.5 and pip for installing and running the code.
First, install dependencies (you might want to create a [virtualenv](https://virtualenv.pypa.io)): First, install dependencies (you might want to create a [virtualenv](https://virtualenv.pypa.io)):

39
draw_chart.py Normal file
View File

@ -0,0 +1,39 @@
import os
import plotly.plotly as py
import plotly.graph_objs as go
import plotly.io as pio
def draw_chart(scatter_x, scatter_y, line_x, line_y):
marker_opts = dict(
name = 'Individual Reward',
x = scatter_x,
y = scatter_y,
mode = 'markers',
marker = {
'size': 2
}
)
marker_trace = go.Scatter(**marker_opts)
line_opts = dict(
name = 'Mean Reward',
x = line_x,
y = line_y,
mode = 'lines'
)
line_trace = go.Scatter(**line_opts)
data = [marker_trace, line_trace]
fig = go.Figure(data=data)
log_layout = go.Layout(
yaxis = {
'type': 'log',
'autorange': True
}
)
fig_log = go.Figure(data=data, layout=log_layout)
pio.orca.config.executable = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'node_modules', 'orca', 'bin', 'orca.js')
pio.write_image(fig, 'fig.svg', width=1600, height=800)
pio.write_image(fig_log, 'fig-log.svg', width=1600, height=800)

2
es.py
View File

@ -69,5 +69,5 @@ class EvolutionStrategy():
# np.dot(n.T, A) scales each noise's contribution to the update by how much reward it had received # np.dot(n.T, A) scales each noise's contribution to the update by how much reward it had received
update = self.alpha / (self.noisep * self.sigma) * np.dot(n.T, A).T update = self.alpha / (self.noisep * self.sigma) * np.dot(n.T, A).T
self.layers[i] = self.layers[i] + update self.layers[i] = self.layers[i] + update
return R

BIN
fig-log.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 350 KiB

713
package-lock.json generated Normal file
View File

@ -0,0 +1,713 @@
{
"name": "flappy-es",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"ajv": {
"version": "6.6.1",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.6.1.tgz",
"integrity": "sha512-ZoJjft5B+EJBjUyu9C9Hc0OZyPZSSlOF+plzouTrg6UlA8f+e/n8NIgBFG/9tppJtpPWfthHakK7juJdNDODww==",
"dev": true,
"requires": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
"dev": true,
"requires": {
"safer-buffer": "~2.1.0"
}
},
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"dev": true
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
"dev": true
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
"dev": true
},
"aws4": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==",
"dev": true
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
"dev": true,
"requires": {
"tweetnacl": "^0.14.3"
}
},
"body": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/body/-/body-5.1.0.tgz",
"integrity": "sha1-5LoM5BCkaTYyM2dgnstOZVMSUGk=",
"dev": true,
"requires": {
"continuable-cache": "^0.3.1",
"error": "^7.0.0",
"raw-body": "~1.1.0",
"safe-json-parse": "~1.0.1"
}
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"bytes": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-1.0.0.tgz",
"integrity": "sha1-NWnt6Lo0MV+rmcPpLLBMciDeH6g=",
"dev": true
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
"dev": true
},
"combined-stream": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
"dev": true,
"requires": {
"delayed-stream": "~1.0.0"
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"continuable-cache": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/continuable-cache/-/continuable-cache-0.3.1.tgz",
"integrity": "sha1-vXJ6f67XfnH/OYWskzUakSczrQ8=",
"dev": true
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0"
}
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true
},
"ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
"dev": true,
"requires": {
"jsbn": "~0.1.0",
"safer-buffer": "^2.1.0"
}
},
"error": {
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/error/-/error-7.0.2.tgz",
"integrity": "sha1-pfdf/02ZJhJt2sDqXcOOaJFTywI=",
"dev": true,
"requires": {
"string-template": "~0.2.1",
"xtend": "~4.0.0"
}
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
"dev": true
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
"dev": true
},
"fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=",
"dev": true
},
"fast-isnumeric": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/fast-isnumeric/-/fast-isnumeric-1.1.2.tgz",
"integrity": "sha512-D7zJht1+NZBBv4759yXn/CJFUNJpILdgdosPFN1AjqQn9TfQJqSeCZfu0SY4bwIlXuDhzkxKoQ8BOqdiXpVzvA==",
"dev": true,
"requires": {
"is-string-blank": "^1.0.1"
}
},
"fast-json-stable-stringify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=",
"dev": true
},
"file-type": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/file-type/-/file-type-9.0.0.tgz",
"integrity": "sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw==",
"dev": true
},
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
"dev": true
},
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
},
"get-stdin": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz",
"integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=",
"dev": true
},
"getpass": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0"
}
},
"glob": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
"integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
"dev": true
},
"har-validator": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
"integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
"dev": true,
"requires": {
"ajv": "^6.5.5",
"har-schema": "^2.0.0"
}
},
"http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
"sshpk": "^1.7.0"
}
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dev": true,
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
"dev": true
},
"is-plain-obj": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
"integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
"dev": true
},
"is-string-blank": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-string-blank/-/is-string-blank-1.0.1.tgz",
"integrity": "sha512-9H+ZBCVs3L9OYqv8nuUAzpcT9OTgMD1yAWrG7ihlnibdkbtB850heAmYWxHuXc4CHy4lKeK69tN+ny1K7gBIrw==",
"dev": true
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
"dev": true
},
"is-url": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==",
"dev": true
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
},
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
"dev": true
},
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
"dev": true
},
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
"dev": true
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true
},
"json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
"dev": true
},
"jsprim": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"dev": true,
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
"json-schema": "0.2.3",
"verror": "1.10.0"
}
},
"mime-db": {
"version": "1.37.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz",
"integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==",
"dev": true
},
"mime-types": {
"version": "2.1.21",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz",
"integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==",
"dev": true,
"requires": {
"mime-db": "~1.37.0"
}
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
},
"minimist": {
"version": "1.2.0",
"resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
"dev": true
},
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
"dev": true
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1"
}
},
"orca": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/orca/-/orca-1.1.1.tgz",
"integrity": "sha512-5rqQJpxpNOWdk/Z0FCbK9xEOfHfv120+xxW0niGFBk7FrmVlE8/D1tYYRCAZrRKIkgWxNOWzVUpySzvmM2SGqw==",
"dev": true,
"requires": {
"body": "^5.1.0",
"fast-isnumeric": "^1.1.1",
"file-type": "^9.0.0",
"get-stdin": "^5.0.1",
"glob": "^7.1.2",
"is-plain-obj": "^1.1.0",
"is-url": "^1.2.4",
"minimist": "^1.2.0",
"read-chunk": "^2.1.0",
"request": "^2.88.0",
"run-parallel": "^1.1.9",
"run-parallel-limit": "^1.0.5",
"run-series": "^1.1.8",
"semver": "^5.4.1",
"string-to-stream": "^1.1.1",
"uuid": "^3.3.2"
}
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
"dev": true
},
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
"dev": true
},
"process-nextick-args": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
"dev": true
},
"psl": {
"version": "1.1.29",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz",
"integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==",
"dev": true
},
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
"dev": true
},
"qs": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
"dev": true
},
"raw-body": {
"version": "1.1.7",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-1.1.7.tgz",
"integrity": "sha1-HQJ8K/oRasxmI7yo8AAWVyqH1CU=",
"dev": true,
"requires": {
"bytes": "1",
"string_decoder": "0.10"
}
},
"read-chunk": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-2.1.0.tgz",
"integrity": "sha1-agTAkoAF7Z1C4aasVgDhnLx/9lU=",
"dev": true,
"requires": {
"pify": "^3.0.0",
"safe-buffer": "^5.1.1"
}
},
"readable-stream": {
"version": "2.3.6",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"dev": true,
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
},
"dependencies": {
"string_decoder": {
"version": "1.1.1",
"resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"request": {
"version": "2.88.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
"dev": true,
"requires": {
"aws-sign2": "~0.7.0",
"aws4": "^1.8.0",
"caseless": "~0.12.0",
"combined-stream": "~1.0.6",
"extend": "~3.0.2",
"forever-agent": "~0.6.1",
"form-data": "~2.3.2",
"har-validator": "~5.1.0",
"http-signature": "~1.2.0",
"is-typedarray": "~1.0.0",
"isstream": "~0.1.2",
"json-stringify-safe": "~5.0.1",
"mime-types": "~2.1.19",
"oauth-sign": "~0.9.0",
"performance-now": "^2.1.0",
"qs": "~6.5.2",
"safe-buffer": "^5.1.2",
"tough-cookie": "~2.4.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.3.2"
}
},
"run-parallel": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
"integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==",
"dev": true
},
"run-parallel-limit": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.0.5.tgz",
"integrity": "sha512-NsY+oDngvrvMxKB3G8ijBzIema6aYbQMD2bHOamvN52BysbIGTnEY2xsNyfrcr9GhY995/t/0nQN3R3oZvaDlg==",
"dev": true
},
"run-series": {
"version": "1.1.8",
"resolved": "https://registry.npmjs.org/run-series/-/run-series-1.1.8.tgz",
"integrity": "sha512-+GztYEPRpIsQoCSraWHDBs9WVy4eVME16zhOtDB4H9J4xN0XRhknnmLOl+4gRgZtu8dpp9N/utSPjKH/xmDzXg==",
"dev": true
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
},
"safe-json-parse": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/safe-json-parse/-/safe-json-parse-1.0.1.tgz",
"integrity": "sha1-PnZyPjjf3aE8mx0poeB//uSzC1c=",
"dev": true
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"dev": true
},
"semver": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
"integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==",
"dev": true
},
"sshpk": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz",
"integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==",
"dev": true,
"requires": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
"bcrypt-pbkdf": "^1.0.0",
"dashdash": "^1.12.0",
"ecc-jsbn": "~0.1.1",
"getpass": "^0.1.1",
"jsbn": "~0.1.0",
"safer-buffer": "^2.0.2",
"tweetnacl": "~0.14.0"
}
},
"string-template": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz",
"integrity": "sha1-QpMuWYo1LQH8IuwzZ9nYTuxsmt0=",
"dev": true
},
"string-to-stream": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz",
"integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==",
"dev": true,
"requires": {
"inherits": "^2.0.1",
"readable-stream": "^2.1.0"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
"dev": true
},
"tough-cookie": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
"integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
"dev": true,
"requires": {
"psl": "^1.1.24",
"punycode": "^1.4.1"
},
"dependencies": {
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
"dev": true
}
}
},
"tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"dev": true,
"requires": {
"safe-buffer": "^5.0.1"
}
},
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
"dev": true
},
"uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
"dev": true,
"requires": {
"punycode": "^2.1.0"
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
"dev": true
},
"verror": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0",
"core-util-is": "1.0.2",
"extsprintf": "^1.2.0"
}
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
},
"xtend": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
"dev": true
}
}
}

22
package.json Normal file
View File

@ -0,0 +1,22 @@
{
"name": "flappy-es",
"version": "1.0.0",
"description": "Playing Flappy Bird using Evolution Strategies ==============================================",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git+https://github.com/mdibaiee/flappy-es.git"
},
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/mdibaiee/flappy-es/issues"
},
"homepage": "https://github.com/mdibaiee/flappy-es#readme",
"devDependencies": {
"orca": "^1.1.1"
}
}

33
requirements
View File

@ -1,31 +1,4 @@
blinker==1.3
chardet==2.3.0
cloud-init==0.7.9
command-not-found==0.3
configobj==5.0.6
cryptography==1.2.3
idna==2.0
Jinja2==2.8
jsonpatch==1.10
jsonpointer==1.9
language-selector==0.1
MarkupSafe==0.23
numpy==1.12.1 numpy==1.12.1
oauthlib==1.0.3 plotly==3.4.0
prettytable==0.7.2 psutil==5.4.6
pyasn1==0.1.9 PyGObject==3.30.4
pycurl==7.43.0
pygobject==3.20.0
PyJWT==1.3.0
pyserial==3.0.1
python-apt==1.1.0b1
python-debian==0.1.27
python-systemd==231
PyYAML==3.11
requests==2.9.1
six==1.10.0
ssh-import-id==5.5
ufw==0.35
unattended-upgrades==0.1
urllib3==1.13.1
virtualenv==15.0.1

28
train.py
View File

@ -7,6 +7,7 @@ from gi.repository import Gtk, GLib, Gdk
from datetime import datetime from datetime import datetime
from os import path from os import path
import os import os
from draw_chart import draw_chart
es = EvolutionStrategy(fn=play, noisep=50, sigma=0.1, alpha=0.001, layer_sizes=[[4, 500], [500, 1]], input_size=4) es = EvolutionStrategy(fn=play, noisep=50, sigma=0.1, alpha=0.001, layer_sizes=[[4, 500], [500, 1]], input_size=4)
load = path.join(path.dirname(__file__), 'load.npy') load = path.join(path.dirname(__file__), 'load.npy')
@ -14,13 +15,15 @@ load = path.join(path.dirname(__file__), 'load.npy')
np.random.seed(0) np.random.seed(0)
# if load.npy exists, load the parameters from it # if load.npy exists, load the parameters from it
if path.exists(load): if path.exists(load) and not os.environ.get('SKIP_LOAD'):
es.layers = np.load(load) es.layers = np.load(load)
# show the game every n iterations # show the game every n iterations
SHOW_EVERY = 100 SHOW_EVERY = int(os.environ.get('SHOW_EVERY', 100))
# save the parameters every n iterations # save the parameters every n iterations
SAVE_EVERY = 100 SAVE_EVERY = int(os.environ.get('SAVE_EVERY', 100))
# number of steps
STEPS = int(os.environ.get('STEPS', 10000))
# an id for saving the parameters in a folder # an id for saving the parameters in a folder
run_id = str(datetime.now()) run_id = str(datetime.now())
@ -45,9 +48,22 @@ def timeout_kill(win, game):
return True return True
for i in range(10000): reward_scatter_x = []
reward_scatter_y = []
reward_line_x = list(range(STEPS))
reward_line_y = []
for i in range(STEPS):
print("{}: ".format(i), end='') print("{}: ".format(i), end='')
es.train() rewards = es.train()
m = np.mean(rewards)
reward_line_y.append(m)
for r in rewards:
reward_scatter_x.append(i)
reward_scatter_y.append(r)
if SHOW_EVERY and i % SHOW_EVERY == 0: if SHOW_EVERY and i % SHOW_EVERY == 0:
play(es.forward, step=step) play(es.forward, step=step)
@ -60,4 +76,4 @@ for i in range(10000):
if i % SAVE_EVERY == 0: if i % SAVE_EVERY == 0:
p = path.join(path.dirname(__file__), 'saves', run_id, 'save-{}'.format(i)) p = path.join(path.dirname(__file__), 'saves', run_id, 'save-{}'.format(i))
np.save(p, es.layers) np.save(p, es.layers)
draw_chart(reward_scatter_x, reward_scatter_y, reward_line_x, reward_line_y)