From 6da6e4fd090057479dc665c76205e64a325b7641 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Tue, 11 Oct 2016 16:28:09 +0330 Subject: [PATCH] feat(pca): implement PCA and visualize data using it --- README.md | 20 ++++++++++++++++++-- examples/word2vec.hs | 32 ++++++++++++++++---------------- sibe.cabal | 1 - src/Sibe/Utils.hs | 13 +++++++++++++ src/Sibe/Word2Vec.hs | 20 +++++++++----------- w2v.png | Bin 16438 -> 14316 bytes 6 files changed, 56 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index c83ced5..741a071 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,27 @@ the thief robs the man the thief robs the woman ``` -The computed vectors are transformed to two dimensions using SVD: +The computed vectors are transformed to two dimensions using PCA: `king` and `queen` have a relation with `man` and `woman`, `love` and `hate` are close to each other, and `dwarf` and `thief` have a relation with `poisons` and `robs`, also, `dwarf` is close to `queen` and `king` while `thief` is closer to `man` and `woman`. `the` doesn't relate to anything. ![word2vec results](https://raw.githubusercontent.com/mdibaiee/sibe/master/w2v.png) -This is a very small dataset and I have to test it on larger datasets. +_You can reproduce this result using these parameters:_ +```haskell +let session = def { learningRate = 0.1 + , batchSize = 1 + , epochs = 10000 + , debug = True + } :: Session + w2v = def { docs = ds + , dimensions = 30 + , method = SkipGram + , window = 2 + , w2vDrawChart = True + , w2vChartName = "w2v.png" + } :: Word2Vec +``` + +This is a very small development dataset and I have to test it on larger datasets. diff --git a/examples/word2vec.hs b/examples/word2vec.hs index b09e1ca..457d7b8 100644 --- a/examples/word2vec.hs +++ b/examples/word2vec.hs @@ -32,32 +32,32 @@ module Main where sws <- lines <$> readFile "examples/stopwords" -- real data, takes a lot of time to train - ds <- do - files <- filter ((/= "xml") . take 1 . reverse) <$> listDirectory "examples/blogs-corpus/" - contents <- mapM (rf . ("examples/blogs-corpus/" ++)) files + {-ds <- do-} + {-files <- filter ((/= "xml") . take 1 . reverse) <$> listDirectory "examples/blogs-corpus/"-} + {-contents <- mapM (rf . ("examples/blogs-corpus/" ++)) files-} - let texts = map (unwords . splitOn " ") contents - let tags = ["", "", "", "", "", "", " "] - return $ map cleanText $ removeWords (sws ++ tags) texts + {-let texts = map (unwords . splitOn " ") contents-} + {-let tags = ["", "", "", "", "", "", " "]-} + {-return $ map cleanText $ removeWords (sws ++ tags) texts-} - {-let ds = ["the king loves the queen", "the queen loves the king",-} - {-"the dwarf hates the king", "the queen hates the dwarf",-} - {-"the dwarf poisons the king", "the dwarf poisons the queen",-} - {-"the man loves the woman", "the woman loves the man",-} - {-"the thief hates the man", "the woman hates the thief",-} - {-"the thief robs the man", "the thief robs the woman"]-} + let ds = ["the king loves the queen", "the queen loves the king", + "the dwarf hates the king", "the queen hates the dwarf", + "the dwarf poisons the king", "the dwarf poisons the queen", + "the man loves the woman", "the woman loves the man", + "the thief hates the man", "the woman hates the thief", + "the thief robs the man", "the thief robs the woman"] - let session = def { learningRate = 5e-1 + let session = def { learningRate = 0.1 , batchSize = 1 - , epochs = 200 + , epochs = 10000 , debug = True } :: Session w2v = def { docs = ds - , dimensions = 300 + , dimensions = 30 , method = SkipGram , window = 2 , w2vDrawChart = True - , w2vChartName = "w2v-big-data.png" + , w2vChartName = "w2v.png" } :: Word2Vec (computed, vocvec) <- word2vec w2v session diff --git a/sibe.cabal b/sibe.cabal index 0f84197..f3bbc51 100644 --- a/sibe.cabal +++ b/sibe.cabal @@ -16,7 +16,6 @@ cabal-version: >=1.10 library hs-source-dirs: src exposed-modules: Sibe, Sibe.NaiveBayes, Sibe.NLP, Sibe.Word2Vec, Sibe.Utils - ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: base >= 4.7 && < 5 , hmatrix , random diff --git a/src/Sibe/Utils.hs b/src/Sibe/Utils.hs index 72108a6..d90cc0b 100644 --- a/src/Sibe/Utils.hs +++ b/src/Sibe/Utils.hs @@ -3,6 +3,7 @@ module Sibe.Utils , ordNub , onehot , average + , pca ) where import qualified Data.Vector.Storable as V import qualified Data.Set as Set @@ -26,3 +27,15 @@ module Sibe.Utils average :: Vector Double -> Vector Double average v = cmap (/ (V.sum v)) v + + pca :: Matrix Double -> Int -> Matrix Double + pca m d = + let rs = toRows m + means = map (\v -> V.sum v / fromIntegral (V.length v)) rs + meanReduced = map (\(a, b) -> V.map (+ (negate b)) a) $ zip rs means + mat = fromRows meanReduced + + (u, s, v) = svd mat + diagS = diagRect 0 s (rows mat) (cols mat) + + in u ?? (All, Take d) <> diagS ?? (Take d, Take d) diff --git a/src/Sibe/Word2Vec.hs b/src/Sibe/Word2Vec.hs index 7c52669..5b7586a 100644 --- a/src/Sibe/Word2Vec.hs +++ b/src/Sibe/Word2Vec.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE ScopedTypeVariables #-} + module Sibe.Word2Vec ( word2vec , Word2Vec (..) @@ -9,7 +11,7 @@ module Sibe.Word2Vec import Data.Char import Data.Maybe import Data.List - import Numeric.LinearAlgebra hiding (find) + import Numeric.LinearAlgebra as H hiding (find) import qualified Data.Vector.Storable as V import Data.Default.Class import Data.Function (on) @@ -59,21 +61,17 @@ module Sibe.Word2Vec let computedVocVec = map (\(w, v) -> (w, runLayer' v hidden)) vocvec when (w2vDrawChart w2v) $ do - let mat = fromColumns . map snd $ computedVocVec - (u, s, v) = svd mat - cut = subMatrix (0, 0) (2, cols mat) - diagS = diagRect 0 (V.take 2 s) (rows mat) (cols mat) - - twoDimensions = cut $ u <> diagS <> tr v - textData = zipWith (\s l -> (V.head l, V.last l, s)) (map fst computedVocVec) (toColumns twoDimensions) + let m = fromRows . map snd $ computedVocVec + twoDimensions = pca m 2 + textData = zipWith (\s l -> (V.head l, V.last l, s)) (map fst computedVocVec) (toRows twoDimensions) chart = toRenderable layout where - textP = plot_annotation_values .~ textData + textP = plot_annotation_values .~ textData $ def layout = layout_title .~ "word vectors" - $ layout_plots .~ [toPlot textP] - $ def + $ layout_plots .~ [toPlot textP] + $ def renderableToFile def (w2vChartName w2v) chart return () diff --git a/w2v.png b/w2v.png index 0183921893d3ee2742d52c1152322fe7d88b3d08..7eecb27b7c24553c5a6e34c364e485c402c2efa7 100644 GIT binary patch literal 14316 zcmb_j2|Sc*+n?%`Ez~K5q7@UFO15mJQcdDmvNJ*mk$pE_6%|t1ccmd@mz^OYB+3%m z_mF)TLN8I6D10TqP~3TyetZ}Wj_kF zS&4izJVUpejvfBkrgi1wdDIf}UqoKCHwv{Mb@}{Rd7F@-7JId8mibF#BNE0(J#?`F zO`Ci+TkN6?`=YVq5mP&dE$0nMCRRVaZ`Zwx&c$7fW->8n5j3!!)c5O_5#s3;e$;iV zT$_)Q<9csPQUTS;jvH5^nhwy(HxR|@w-?b?^3At6q#F{$@RJQGeD?Smd!>*WWy3LL zdjARnTvmiv_#|?b?_FKC!oL`|QlU_1cqr*nC}-W>T=0!b4keV1kB^TM7rlG&cvBlG zU9gQ`OblP*!N{Zg{`Tq6%DH@$!$TsxON4SyUtd*-pd6alX<;_~RudLO<}7IX3)LC( zmk9}e8ZJ_7-vv-^uF!XMVCq+7r(r=A?$6A})O=mWYWhEtq%B zsiqq-9@jh*%y-M>&Yc|V5mCJlA7dxE!7GwCEN9Xxa>!=$Y>RjRNhYxdd4mOz(^D4E%@FZLJ;!m;H;wtPr(dv6o-pDkoZ2`U-JEh#syXxFfkM%~&L zhpnl@n`7kIy9&QGrkZF-UcdeZ5#gzY$<6*cqgBiJ6HxH=|A#uy*CP=DFjmoMLa-CH8&x%6dd8tyf6BdbFeZd~qz#>Q-=qWAIj zRnIcJ0srLokGoOf7pIt*Z!^~U8Hc|$nVM%mBNZ;%V&m@3E}gBG(An83G%L{2(Lv)@ zbie0)AWvOg9TX2~t_q(ph^Io$oTj9A#$%T`n z67fXYSo{*)lqx91PlPMj7`<@Ebj3ssc-JYnoOI(Bl_dju_ZxHLJ+vZ|zsbwj!rjNH zWk&cgi8(p6QMzcnOdV=9CBaZpDSRbpb613t0_&6%dyn3f!CX8aZ< z85vl(v@mJ5OUUZmZ&p=;iBMwn!N&=4n1$~QBA38YiTvl)xza{$uJRv z(^^nau=st3xH#QZCHno_Ia-lZmILzEtu)S<2q-f4E^M5xX5bgPax z_pmX}&ZwN01RsO8{JgQW`HFOpqAS$=Xu?3Ek}S3^e8<9Y&Ipsp?1ALuM2$QjSrg)7x@qeU6o4HG&Bn&Y#U+V;eY1v*;L~C}QWNbZTb{W)1}Z?JEU9Ve z)|WIH>@i$Eveh(h+JJ;5oG%E3mT}*%&Q3Touq~rSv~jVqYTSyEV)Hpiy7t$@(-jpJ zgG)EL3ie#Tp=Lqw$u{jOnI39#+sEUc*{(LN{E@|ClLUVayXi(f&qSWw-34vDeRNZ(T-@Tl^E-F$ z1a$d6LgXi%mqJgKb0*e`-D*s^!M`NRc-lf=T|IP=6>p`NE1ow`JZMxCbrp`~AT4>z zcp^EqURBtwkI(Oor6s86@HMAt6;pO8`4nqW(9$O2U%NA4vYloV$pL$>AMMHbt)vky%-`CadP z`+(JPQU>to(xpq9nwmyN-|!2i_*beK!@aw1Pa&HFjpgpWd+Iq>#||9m0JQ7!;${;T zb|`S(RMHh4?v&jTHP%&X(vr=4>{$HOU_J(JHT|~y#rDEG{@m(JOiZypbOZO{VcI+hxtM^=0MO^ZaKg5c1hP^X}~p-}n~h5z^khT?^QuzODF zKdv)?qdC}F>@JeQ2L0DQsJO>0Yp(X;!-oKE(<5!fl$4VvPM8kX$G2o#PJHczj~l*l zRlub#%e;@Y_??sR=FJ-|Ev*kBf?dhi;3}GnhP#kaNp&})Y5h; zgBy2oaWP-;RXe);``v`cx%=Pvi;v4TOXgEVlN%ZumU=LHKY7w2l>@hKRQW20($=k8 zq4}$%=sc)1c&!o-ePX6L1zEKE)y!>%P!1vETTrN38A&t#1J>O6ci122?taQ}epdF9 z#>3z;`_a=D+-77bRQ`iQO7Nst7-H!1nB2%)M^H z@v?ouqPAW9lY+P*E*b6(tG=YMqXT{(EonBR{>>8pxpo=%E)T=Sp8VH7K;uN&zJm>e zFV@`~#ob)SzHsLpc4U-8nN)ko-CemqlK02*|Kw8E%0AN^da7>f)qvyAR5OeL_s|?1 z0Eg-%pU&>xy?eKg@aV(@ zkJKDIS~5+~pp1CvqrBnz>fUs+u>xw@qOXH1*I=4X+|b|gonqxF7_InNg4`@%aW2l8Z@YgIh%CNO}+Fi zEKZ~CXS>E0<|f?T-F;<&mqkm3$|8)>0D6IEK7lmn&34tG#CJ0)i0-py;EpawG|%JgIdh`x{cLI=AEY!XK5H*Ju#_ zA9C>2fs*f7sM5o$>*M#1)@iR`W9EBp?aVI5N?3MU=o+? zpww(_ZSCyr3=9VRj;R729r}3Vxa0c!dt6`s%G_SDUoK^X9)R^$A&)ko zKfxyX{U2{6eddjZn&DPSyM*@j_Vx}81QHG;E7P>gM>Z=fYiw+cFZfpP`#>xjPEl$j zn`A&WpzsOp5~$dQp=n?U-90^w7~j7?i}N0>Ug4`?^;=TVIcUY#uGM*o9^>V;1VVru z5p;BoZRcBxDhzeyVL?G#V2aonxJ->Kv!3~>!F%`am6eqd2m~lhPA;xyP-3uf(zrj7 zg8;2PN;1%_``r^@RaX6<2p29~V0D^(CPVa*HLQOH{CX)&#Gd|`%47y{@haMKyt@o7 z7xfB9%m-q_%xAP?Apqsf_6!OQ)J;9{U%wZPWP&R^9EQO$ydW{uZ z9{|*G!G?fD0&XoWEsdXPv9_Q5dOcYQ_GMIzUSuT}<1d_oJIZzkOV|bImwd?z#v%ukR{nBjF?0pN$pl zS^j!z!l*DsiEBF(m=XCUsTt+hY5d5esxGg?plzPO58kyiZTJj0+`zm7chvN)Yjl)% zBpewwbn#~hIeQ1<<9CXt+7s~SIQnqMADxh&aP&=tOk1%AJ9PXtG6VVnn)mY8RlcG9 zH0RIRK{PpB#ZPqcXLhiiX=)=oD3;r52=WndDXG!uD(+W+X=#14!XN?D&fdykk>uU* z@yB|r!k5CA=EAKPhRpju4AsXsb(eXM%+f6c&r?H5lzOoeyJn#OkE(!E(PNeeSdMd= zyP>cDbsv%D=e|PnqvulE!N9<;UmXEr3Qg$!5SpX^@gPB7O6r62rY%0QAOgXb36VW| z?AWmrCqC+?#;aw?qI+%&fD~tw4imN+Z3lb`5j1Cm<_C?QpLj30jDL?P8-FgCf~=vxyn;<&&zOOLk4M4tjA01@;O(RH}pF?bqUd zZMFm405HAPmM_A^^$N7v<;#}=lT7mGdf~FHzBQNvo4!Lv>oi`*4$os_Kv|_3ejWy( zEqnLwzNcq=LV~E>D)vgCqV=BKL~2cFyXll zH&#)H+0pwu4`o}Ag!%eTl`xxdsieIG4h1r_HRXmok+`IYbDZjr4tI5P1NX$o&yNX1 z$HFpvTVPoCM3@K=u;ZT9)zzwDCpP6?1Y9U0+q7vD=sZq^S6^$Qs}6YyKm7qO&<^P7 z=z4m3;Kb|d=*TN7auQU)nlU|cHGi8Spz;cEE5uoXf5GIx88s@-E){Y~QnHzd$3qtY z<>@O+<9290e>*|)ssUO8@)jsqN^0szZ(@eCP<^`{6%CHNFf%vl%dIwof&Y-Pa9}V?;1J@cQ^SFlVbQ!s4N1MYSlZ4~lAqb~ zgURX0v13C3#U)+9jhJXvUK?0W{8r_vxVN=;GZ(+Np5zu(dp$G+!`CVxu1s3x@dWqq z@HS6o{#kFfh@D`)w$}s0+e`|Cw$zeqJAp7SjXcCS=v8KEuS7J$?AZec z4j_VBMMVW}+WgGO(m82Rh^kgrR#jmlLa-Dlvham&_R6eF1yr`T<>}0mUfk9Ky@AM; zsSaG083Ar5FE8(%ZPY5cHm2g@E|Z@mAFk1j0OtHYoE|`+tOki z!G8-m0JSXkD)ct6KUnO5bD}WajfP2}`%$_nnuRW4(<#T-kB*MMc=5u+!vne`7*?DF zpf0=i?p+wfE!n~G1bE6b?UDfRrwz^yv~LMq7I0^vQJ~$yFG*C-nT{4)L<yBD!Ct-*AJQX}@5mseDPUzQ7R2R+dm>=b_fncl>g`1fdaqLbt21a;WU z>fG=x`ZNo4&nQfLBA1n2W#kG$fI?KjdhQkcG^@2pe(~HIo6D9R7^sWevTb|j9A|7{ z$S0tjZ#HaNkHi;MWhLENrowV*qWD1i^4!GCJ=U)(gMuU8JT?9$39`D{4d>@k$<_-Qbv1+}$(6xvsAUbi@1azq2CbP5+=$ zer5>sYg_X_egbG_r1`UDL6_7oL5uIwnh#lL23)UxIkHjSEJr1bKLV3g-t)I=8u@@_ z86FX+N99-1^IVfx?B8C;dP}M#-s~qPR?NB0(=2Z9&JF%UG3RplL;HL^NpiU%?lI%L z!M6AlqKA;MWxBIL`PdXk$gpL(Yi^K(8p~Jauw!G~v`Z^Das6$ELxOVN4uQn)bZym( zsJI&-x}JCCYE^S&lcYHV3 zuH`+g>?-%=9Lb${G}h2@8(8k0RnZJ(swqWHMUS#!hXld?{eRJ=oJhX${cEHbxXHSD zdY@}+FC{WKV>)%KP{Swss=}qOUOmRmt$-EWOA}X@4Uz?L5`^o4bFR?Sz~96JZ8>}P zY)!QENcFQKDopPFJROv1X{veB{^uzvPt7L|UeY)Zoe6}QudG#09{3X3+1a_d!XhH- zXiK13o3?DbgsJYHOI}MgCzo7j%EgV_<8c$ptJ-4)0eIszf#85XIe+0o-t-qk78M}t1$hY+TY=At z%ru(=2#Wv(1S?lXUcNi)22@aUPS9^jG%UgmU~SLN+Tig^aJxZu*%~WnGH*2H=E*$6 zL>;r`qNPf@K7hI1GXLbYxi{5ufKAuW!{Eh5=_!1(hLz{oa$V( za8uHPZa=K9s>1N|TSGbo`I5BQq6N`uQ8~oo^ag zeM_RYAo6#s5*$;s65HDiXKqCR39mC!;VoLDDDFs%pv=KzLJs1yKNm#*#4i1VAi9SD z!u~tUrfJYBY)kD7`g2M553I4jmvlcm&!azI!smYgReMeiT-64D+@|=jHKBI0za~o++WIJ(#NueV>PT<=%##QHy^gq2BBmQ<{lC%Ji#ge{ zc=SPe&+Z4C2HcBgc4&^XOe@Wuajho){k5!?`&u3RSMy-~)+pCd)`O{d+TyEbYNK%4 zaxkQ?^210-+x_*;#s(R4c^HfXz!e))(=jjNDTL}dOLwkNJz=4;7Omu3u`s?{UqGh} zEP*-SI7+*(=*=4@7ISS*)y8s#^c3HCqp9BbjG04vW8f(y>6HCSkcSorCim{;ZFhR_ z6>{)>S(zYsu+RZLt?t>27iD|)o?lNN+tLh^Bp&Sz3<*(FQGr;)VwP<^<9MICwL&ZTeZYyL zNE7&G`z^pWHExEjadR6_nvKp=p<#**RBHYGUK_0W2T@VU3it+e8~>CQ{T2BA?OFTl zEDCYHHuz-^A^qr45d33W1b2}2{06l7TYkQQbso0<>5Pp%JR2!##;}p9GN|K`!nt#q|?~<(8i1d_Lz8)p%AwdUeEd#Ee^~2(c=C~^tIj! z{KI-K5AXQHYL+`i`<)+2$}h{8{Eun$AG}OtZy=dk5u9r&&Q^B>)C$Oc^gUP@M9(MJ zr#3d{#-*Zp`1lm;=qf#zfBX(aD75=d`ve4Vn%A1a!ZUkwr&EeFziD0Sga!EcibVkh#&a zIt|8W_{d(nb`AWSo0F2kmGnDzg3AGONpqdW@_>wxxwxdHqyQz9S$Yo<(bV8);`~4ycu`3H?V!8P>jxtJ zF@xOkJSd0jvOJ70s##v zsi_@%1NFccwSpP7CgBOU{NkGM3y1ihekQu+rm*=)Yo1B&n1nn1F&Lq1vmeO|2ndj$8C6_~DOP00g4ST=XTGAir@%D{v@q zL0^2kg5L%Wx+G+*NCEE~81qOIX5g~3PxoJg&%x=rx@@qYA^T}=ZU)jHT*+tNE7?Wy zIT444I}B#6D=jMt2JwVKKcuClfdm6{5kj1I+$<}kdX&S=DfaNy?nHEfcEY43s?qVP#^90>Qw-LTC zZu2Dd@)MDHFwi*Qv`y40;#xkw5q=J4Tt=fovJw-CM^u#Lj>*68?ruNV;}7C-u!BIc zQJc?GDZuk!tR?zTQc<}-cpzxr`+gsHrT3rZOV}7VR_9jAm~GGVd9_G=*E@)p~osDen{_24)ZQEN5b) zM|D?PisCa(&|{^4Rt+OBnFH)^|tv`~MiP z{cs)DW)D`Ic*wz(jtFc2F?9TM6aOdTeccIkolmqF^$>{2}B^ z2?+@bSV+Q;u(5qHN(UeJC;_rpB-O61t%VU)b@Xeip)VI%9Z#J&aY9HaCn~C>-y#Zr z9v+7egUf$*Iu%tg1f=PdH_{~M`ob}2^i7z3{P?&Y+5#LYWR688?W+m?dy|RNL@;Ey z3kwTvr0EPWK_K02U77#NvFLvq9GnACF*`d8i2;O!`p`cOR{m!@hnp&~S(&B@9> zl`S{U@b~xUB=~VDr`)KZ2P*_pGqwW10t^O{Wksxdo}DGkS;4GtsGzwn zOzY|D>cSbiEBXs6^WK^%n8C1juaPi}ToyxyfegpW%lr8IGwj(T0FfcQl-D6;dDg)O zf@)@`F+iUN&e0kxeMbJQ#kh3Sp;BcdeS>p5a{O@LsVes`m9zLYpzFioi z3^4tDWg+kYR4~wW*?C!z?E<<_x=RYHA8G>i6j;6C4bjghEiSmSqOX z3IvU@g+0oMJI15C-Cu9h=&x#bwiYyr%yoN%$7Lz77l1SN0er)myyVmhpI>MJp3+vz zigy6t9|mC{LW0J|K>(nJPnv){rN4|lTpmk^iZcF^7W3+rBpPs3Iz#}8G$Ccw3$xYz zl6DiIXa{M#s4E0P%foSku;0Kf)D}2gx<@o5CEwS^$boG+WR?YQ>|v`2c}!w#?0mm8 zejegC4rF59k6ZmTU>IBu*)P9vsb!8rs@B-pcuXaYoQB3}ByZ-0*j+v`~2f6Mtz1fzHJ|8}FEK|*;MQ+RP> zE4|)e2gA%Q#*o=tWT{DuOx42CgaOVWjiAfJ;z)rY5$EzmW@eznu1A_+aswd50K*5e z_wwaeZ)wpi_a0F70SYxzJ0W}LBn&J+EaD47&ng?!)EM;okAJbY@V4e|6~dJ-M)OVx z+4Ws{kQ-Uzf6`HH#$w!9v{DF%9T6I^6Uvj$iEfh_>HfgRs`(8PXRtuPVS%xzsGi-= zgrp}v_>bSS))~v6c*NCe-8vSg;V?lwU7KUmV~`C%P8uYV27$U6FQu{KO~5R* z_Y}HOwls}NhCubm6+aFFB_-u)%K;sSWEj2g@9&4;_eqN}{Eooj{9)L}na6A)2VcE< zm1{R;aQ^(=n*N2xDkqr6>joeEOQbu{uuO)1YgI>_g-jA4 z3YO)k@yLXc-&oC!W}gvX@T9oI6%7pl5J~vO3B$xZwmkZ_H?+CrPGwhank8r5vEP~9 z50VB(*a6sq9Fl+zidE(9WSZ{{Hi7l4aesz>2Y11u``yldy1B>D3o-e#&jE`*QH>q2 zw6ugFF+rfoVq-(FouqpWEE>;lp4G-A|pi+GzF& z=FOs3SEGw1v;>IhAy3hQES$yh(5cQJ%=R7#+{CS;`xWn0>=#g?X)DCoqK6rnL$?|N z3H`YIpk4}p2Sb8U(b4$Hk1*y=?s*EEpPg;jkaS%xN|I67CJ7S#VL!16eJO>wDZuTL zF32|OSSvS6df>)Wnz{H+Qz^hW514LVtYWR8)5N%WJV!M=BvB1QJ$)jiWef$`D+19RuGi{JQz_^UR#?E%-g{S_XYPGszlhira^LYX25RE literal 16438 zcmb_j2RPOJ+dt_+p;SslG|32+mDw))>B#0-MJOXgW+izl@{~=ebnKZu4_V19t8g5$ z_dLe2-up)b_57dr|6Z@ID;LiBo!_{>_x)MlM}Nv++(Na73WY*#k-BsajY4hQheDC5 zZX$!< zF3tBmm$aE4#>@C}tG&G{Woen*Sxw%ih8t{WLyOz&{-)HS2L+YX0ZY;1_rAe!6`bs|Nf`(0=yYH(ss^BL%V$)yQNBCq7) zjK1U2bgRAD{LI)!%B?gsH13|B+G#hwq?$BN%1slTam9L%+}-DU!#syGdsrkTanGOc zkgIi?N)Fw^B4(R^cf*}Kcbw;%30Z0tEt!@_>C5BNTkZNj9bO2?wV&>%{+wc@*h6;w z#0esKi{mt&Nyv1NM3rqb5+ZEzbts*{vTfToM#kC6S_Llb1&=-0&^)yC`Sa)1xP2cy z_@lA4>e<1I;j#fc=;`U7j8tkm6<`@ZgQ_p z%Vi9vwW-Ojw`M=K?sJm951Q3^er$H4#|JGr-%fkcW6v&n`i47M0}IoH5eI=M#p9%b zgwmp-bLY=%Wm!cfC0*tqe%AN&U_8MlB4YRP^?5w6L5Vw~bd*i&ot_UqTyQiCN5@=r z|Hq50O456U%}*^c<<~|iCVs0>5d)h_QRy6E_$N1RTWohxTB06}~pL6b5nwyWpGkhVchqdd831ExH+&QyqiKq zZHBWbwb=y)1y7z-<4AR!oBH}y4_i0cSGzDjpS9=>=Y_E$6y7JNhQgbS6A%y(${X~U zR?i&0?TxM$o2dAbZ0Ni+M}U%YsEfQT9i+QXB zA#J)5V}G+b?PgDPsO0Uz&yRO;O^yuZ99&GUmKbmlFeFq#F(NA?MkPT$kiRd$Fl4r! z);UtLdK8UXk{N$$hw5&;bMWKe`_J;rW!sLuUSz)|9c38mm}Z#{M;0mtsabmQr?#%Wy?ymp%C`N&F63=XeL2#R z+FJ$*xFx&|Ax<)ka2Fc{E0T3_)Xk!!WD5gdK7k-9C^)qnH*?;?V$5TAn~3;C#R-@$ zo40I%Rd0+&e+n|ooAr3InVOnO(kW-rw!0!gOGtd`v;GKf$w^tcA^+Y+CMKp4D7ejZ zY&oR2f&{{-?Q6@c;0KlPZbQGBP&>HLhH#dwu>%mV~yVqW*$pddscy@vd?o^#1+(Nhbt@ zD&HT5v+)l)Uz-xtqEwsG+uQs6`SXm73|QX+CJk@G!&@^Hu3qhQV|8MnqobpzH%M>x z!0^EZd^u@B5%c!#abaQMW5+%=HVVPa%*dE2afG&f=ImKmo9-I>QK(a_9p?&+uV3Fp zOWVkL@!UCne*Tt>Y8cjr75=XzP0h?aA3b{H=}E}2(68PNkf%mvrgc7Eip9X7rJR34lLji z&%L4z#DMN@gM|VFu?)9lR`tLoV-yu@Cx`*({`|8Bo&`P^x${DNWm;c`rm}~#OyFgn zV&m`ORCq8sk=~q~oYJzgvX?HY!c;={zjLJ~)mOt1h>eW}Un4JvLHCZ1i4ihub?AKm%nxHH#bMq4 z&J|X^X1d{-Gw+~385ac>-@cXdV%x$j%(rjfK3-n4k=EQ`dp36VkGrI|v%Zpp9+ zwr^-`q$DS=3>1h-OdK6;FT@YdS`!XT$HVFi5}sChko&>I#^vy#LpNG;W}_lP9=p3c zj1}*uq+v+ZDYy><&?L~;&(AL);LB|3^If}kNiOs$%n=86GcaUWbX*9kyp0bLa>^sZ zAs4$Bk2Q9GdUx#)o<}LjGa}pFFyPfRz>Y9J}SsHx)g|>H^)GzLLVTC2&gbsb8hhM<4mb>N7wdA4; z%kT%c`_K20<`3Cm>%JC0VRQxFK@#a+At5pkU}5}VBwbj4MfO#1V&JD@H+P^q__I!z z=?cf6t6H-EBP!owJ>3tH3=|uvf4(wNLb42hhzo8)02zAPO{rYukYHD0uTx=}2xVRQa zDj=G+wl*%6cz4DV-LKqO(9(NFZ^yoQQ)H0LfqnPx-R+Uqq1N<`lh#mb74Sg(e0+T5 z0{M-A#z}MJp77Ws5dHe~S7>7!Hf$&>D}%K$H#JmSZ4{@CTLi{2K_rShk|u#66ux`+ za0^p-Iv468-{pe)hgP0DCMYd5RJ)N92K?EFG+%)F85{Ql9??=)xBFW32(B9;lsP#$ zDJgXTN7&gTu-Nk))&BB;iK}8BcPKL4lNql{fd(K!>J6KkntJ;5 zDZE?T`M9{aW7h{C{nDp$3JL_kPJeAb$W1S!sHk}5iZ>3ov|HN&VHXpuL&5j5x8I|- zJZfI0v*9lLUmhNn<>lp7Re2LtqPU6RkzR%HhzMSdv>TJX9~b7PX=!Nc%<==p?u-K! zV!hM(Y{N#1L+tF?*7bKwkC(Hf#?@e*p%N*OJ3b*w8$sH@(3zxAIu;Us90(mF0$|4VS z*KJR$ zq_HgSgjY(uxyU*dYt#fxRz_O->({SnMMZY`7l0I-GE=p2PDCg$%zis^m%FQao4?7u z>0TOT;N#}z=6ZUs6B5jzz$`2*6fj$!ujv;4OjOEe&z?m{yzqeouV1}-HP8T+RF!ob2793-elEzXQNQQ&SVDEXS?;uVCw{f<-j|z6AzO zZtXP}ZUvnLsIGnkwTsow-fgFTw#Rt#*0B3s9UV~V{5P9YUY&J&$|{M$Z4x$Vhz$=9 zPd2QEiU#OFupv0k&dvq|1c=X`=3_36SXDEYM@L!kE?Dmm6%}H)2UQyl1A~Hm(7L+1 zK;6V0EZ`yK{k7ajS4{-X#cailhd&32CUpyU;dX{vxS3)PA64mqTyt2)H^Y7x{6f6G z7I%Pu8z>w9vDQufJD=t~S+ic2LB8xSU3*Q7kF8d?r6PS4ljE-jW&ScjADIVt3o_1a zT3yCJTK%`Va5~JKY|Twrmf!rzR#S|CNVp98aQDZ$+Y(pZu7L@!7P+@O;6qrBuUk^z z_{qQafi>4fw~KpuwpX_e43j&!o(_Ayc1HVmZiOqMV#`9VsEu*`K>wH7oG784=#MOE zv5l6?^V#6_avy6A`=A%!kjWadm9x{b|G_G7;c}tn zDAI%a7?slEV)4=ZjZk+`jZv_m-oLLAItuL-X03G8P;)xLek2!$3cLU&mXL&gdb7U1 zJ~b89ElW%20iQp8`dVW12g6gC+U94ht*tL#zI^_CF(`P}y*00#GZeJ6@LARaAY<9U z&rtgGyLRDc+|ki#X}4Q*?ugzVWDvZeDkb%3a1LeX_lRfvnH}G{$gmQ$4lg#jegY1M z?q8mYD9ydYyhrrlaqbWD)E|(l1}p;)km?8Jm`Z?N=kyWcx@KO+aaV0 zJxDPzXXm#f%B|E(vRW9g5*-=r?X6)2nMZ(+@6Xzl7cX9HVG^_&PBy9y9|%cm9)dna z(g#Wgje)rMPW%sCI|AMic*6b$i(Ocp#^J2`K6RD**$jSu<-JGD*4)w25qLcyYa^p5 zH0QyC`aM;_pzU3e77!8&1`UENn`5W61dA0A`ksYuP2 zvs%WoCsxBl26Di?3z${Y2MD3s!-IU`Fg9#ykP7;th5uL|IM*Gp%RWu3)32L4Z4L7K{AzhK7Qy>k#VYeoN`d5!DqV-sE{htbWet3uLn+4R#rlB z4Xe|(k+GZJYUIBOy!G(m!!XgJqN3>7~ zyWPjQTwi_fH;Jv(^F@#7U`4uZ-J50J{thssH~MzAxS(L#%a{9vglqt^9X)zfI?9Jj zrKGqR7ExDEPl8%85xz)B*9Dkm$8)b7R#OV{$&ty_&fCMG65l|ktM>)-|gZFT_q8zf}52+AMG-Ejvm zKAkL)u6Rg55aMvl;dfP~d-v`Y8m?(+nIA2r#Z5JtEXz?Yl(ZKvTmYJDU}#ucS_%Nu zdWl5;1Uq4&tJ@ALp9s!Mm|1J2op#HXA(H8r56c6cC5`xmH_soSU$jyT41Xo1dROY* zP0h_jxsM+|K700Tt6N7}v#F?g+bKD@N&v$4V_%DFYHBE19cofW{)%Hstmlw-AZu$c zbc6M%c=_@xZ}sfkLm+S{$7_DcOvPdki;B)sO3oB?cuG3qg+P}9RSXnHoAj*`Pqv|6 zdZRy>rZ3e>E-2dC+5&h5eB^_MGU4FhsIRYwX97_9?OQZQD%=~8labB?F3t={12Wh; zYTle}%*-tdOPL$43R^tl8O8X5x1fiTw_Dspmi zc9d{4M)KxN_4M@2Vp&ja&&bHgK!+6sZ~;_ec$yjP1JRKj%h~ZRAQd^p294w88}pvg zk(a}*jf}hwqUBkr9^gApWo8YrYH&<@C}yw|NW=kRCJl6DVf(2Yc6N5Qwle^0{6a%R zfp(jiZHVu+quNQf%-sgC`9RG_-9Wj<`eB+fEe(aTx3EfxfjSssKO6v0MSQx&viW8@ zu|!YSb-4gceW@)q5A0M{Qu6ux@3vtj+wIGox`ky1dZ}415uSg;+z8GYCI;w@rkd`k zbjsnI09($nPSK$5QUk)bf%W9;>&tk;z}6xAjP_}mgOfJ_xejB<%*Bz1vMfIGc8%-M zu7^J9z5|LEcJpF#u9oevNq{l<@}jI2t*=xx3&?u37T*a1fFTer*Y%;bg9Ksj_o&yP z5nyKYPh9r{`p@6UN4Xxh6KAD{+0;O~IYk@LyJJ6su~xwMcPsJVDZ(lQ-%^GY4;s`wg#_k%9T2GVZy*qpcIRJ9DzZtFnrnDfV^tVI~ zj`sv%e)%W(t^?D`*I^Sua3FG~{}(j;OWmr5hIYn#tLNArOTMrHbxOMwDIS!|FU7tx zKEpocIn0i8CFH!8OluxGc)^{o^O`sz;pt{H8C;q zU9^RbMGhh+`N<=hFF02H68gw>g6%~R->0TJv01pQm;+oUV2mnVx zMOD9YrE{1_#`mmsPc2M`Z7eMDl28Bo3kv&g9uN4GpwLQ3fjJ276x;#$e{k>ywho!D zo~$Fo6i@jS0MaE zM|1M?Pl9?4!zM1=h*olb6u@7o{ct82#ywqK99TiK))t)e5;zw&)Nqd$LW2r;W)YMxLr1tDkN?=;6Q z28KwpT!-0PXU`%Q#4yal-5A?~p^?TR4Ii|Ul9GXjkD-~hFf*H`PlV}ER$=8poYe5| ztpuBa#0<-D2TMy!7&&k+VYGq}hPW9F4F1oaje&RcP5UJz8+xKSB_tMwaI=+Ro)T~) zNZ`pI+qX{#1~I4%F#0b(Wxmsg_47vCsg%Ee-#gz!af-G6(r`94B`0He#n< zyM{#@7e3sASbG%#Cjc}nDk}?G_11uBZ`o4?=)D3jL!m>TYfY>CD^;(90xW^T3F0^U z1a8hMxiJY!Hk2V zaeeRrsa-hjMm&Q({$?V~da&XFK4Op8*Crjl*knEdwbjvYIwsHjX^_Uzh4 z?2k!+6Wz)x;Q%^#ghHsV_I^}tf1Jz%P}E^|)rQNN7#Rhev@}pxSLa>Ww}CL%It{+G zX?in}+7XR0CEa1gpcv^CIAo(3k!Sm<|-JYi;Ig;^Y1RQI;Pl9n_{o@ z+)O0(#RGwO`SN8^QBhT`*07@qDyxy$V7)NugYGY553OTG0u{6pGcTEZ$@3@e#*#Ok zSn*cu69`P7K7H~9~A;YQ{aIc5(*($Xu@87=%?JZR74jLD2 z3xE@zfVYni*xAzqjR`q9^Px+#p`zO65QJ&OQ;6!9xI^rPG&cuU$VmZ#`kQ7TIZW=M zbNrml`1wW}IW@gIi@17i3P_9<6|&eX4i57G$Gmgl`Ue+vFU zg?Dmez4!26|I^w!b#VrX5qLUHh7H`F8ChB!iP!FLB%fr`I(Xm!)Dz}Tmk$^>2iTA= z15jOq8yAlsKTZ#F836FAW*sOg@X-w_1FZq?FvVs8)Ist_8kYNM1$He$}%K}>)Kb;&P-28*ZMMZ}W zALhVbaA(-g$~w}~VF}ZzuD-rcqcK5ysnvdDmP87Xa3sO31=V+j-%3Y){`|QLUz3i= zHoaQl{*X4Zva&8Dd3shui}Xe#5fBLaf#!k3Im^pGbvj*BU7cX*>?kyOvI_5_q^=H$ zB$yP?$?Ax+z%dEVOAE>|)}Iq~0V36g(rn(G1X4&$y+JZ#8V1(*rFDMI@Nic?pZrd4 zxmttjkX!(8AOtv$7Em@Wu)s*724i4udIZTOLdGWCyB{tEnl^mN$$1pST=4w4LR#s( z*cIYqV(dGgT?G1=+1c3{G)qsm8~U`o<^eF~S7eea(l7%dLlhAn-X8>F8GL!8!7vaE z8iF}c=tgU!E3mD@TzE!wr_@Skw4DY=0ER!Y6sT|=>kNqmHZ&L36xb$`I?FQaYgvMc zm@`Jc8IcP`AqDd4)qXKCO=srq+dtOSn0H7(@(K}0fCxu;8?Q=uge8|lz4d0l5O%4z zc0Y6RVyTBHzmO2%xx$3pDfK`9{2Sd5#y_8|?eW_2Z{Ywwiq^efl9-cc@APTTWVi30%IwKSzWQ_7`Fk`a2( zmF~rHtM!^hB(8ggnQVpwF+zOAufPVa#3R?%MGlgK0`a98z8~~-B2L~^>u+F!Q zm9pZ7&r?xT$0-khR+z0pX_J4 z;sR(acCdOh)YT2DUSxusf~|Xb+6D3ul91l3sHmtCO<7zZEa453ySlqEIkq1Ogr!I9 zAv0wPPUBk!mI>$RPJ<``q|3A&lbu9t8e|+mGl5jDuU|jOFmlCz@nqbFUV#9-hMU0 z%EY7r&o?MY)Z**8g%gxo9a8tEH^-C9gFE=WAx^x>udS_}9%_#8j*g7XupQF_w+P@O z$SQ!DLDYg|kpx7mK$jXRq?NoPt)!w-9V9e@Bu5Y=s-o?f3Q)=hHy4b-d7~jkAqt2W zI0BN3gUF*6hmwYds99?cn8h?qf{0(-b>0r$=6EKQ7sQBbWzP=}W5`kY_G|oM$Pvaz zMt)8=(}wdo!hqk?dO+nGlIchfn*FC{k&L@@t9uGMq079pRI^D)oFnQ z9Ss)eiM6%gw$c@sQteY4_8zU5D~nu)W}+rm+Rg~AGVg#0E21qC>9T-Y0e z%|Nm)z_QmX`U~6vsL5gNtK*$zKzEQF5wvDMQ11t+#JSBUy!<;`?NHzOOG*U!A(@3Fudb?2x2 z4r&+89-pFranjc~Xy4QFn8$0J!EfP@{AF|91%YXZ4q{88ZNuBfrFq~FSl9(foMx@e zSi9zqn}MyoB!;1P=@`-vNHeGVUr|gL@l#*fWE! z9)?$lU;W|et#qkxqS=2Z@+{wWc>g!}hd|8Vr33y6(*K;cNV@Nz@F)aBzQuKZdWEOI zn{wY`OKV)20iy!WqeoM}+yLwE3H{M5Q^iY{{(|^`dYoWToSP(^?@QXgkFF=>YQdxX zksoM`Rr5mf0|F4YGEFB8imbRf=RK=*l$4bzC@I<5*hnN&_wu-BO68suTnjygX!hJ( z6T`H^H?8t5NWQy0FsZE(ytY1nxex=Ecvb0Y`Nq*6naoQD&*S26 z#zjXTZ;uwUj_H3V7Nf_dvE@rNk{CbU94%%aQ`pW2e{+gae33U{MC`8z9;_lg^d-5p z-5iif#Ov2*I8tv9H7Q2O$7yDcmzcr!f*%pg7MH^i!2B(vlh;1LedLJn@*FNLEG+yF zY0Rd&x;Hkb+zQ8rvqylZip;q{;gKR)v<^&;jd5^t1`6I(hYTZ}4e`m{WP{4+_;}MB zH!3PCb0G)~2^=~Fv1b7RMm6_!Y5c11yc1Z%EDbXGN>>Quic0!(_sfs+Um=}2=NnJ5 zu1!Cfe)k0bk!OuEBZ1H^AC<|{+u4pqe{I)F7Zef#ZO{on%>xm15d9UziC0E=K8hkx zca&L+y20cA{rhRAc#^e-Hqdj}1k=mw14K}YHfS?so3|w!0Pw}ZWYS+>@?-Fc@fy6O>VXN3V z&tg87|_k#QYr{Lu}%&& zA{z^Wt052R<&~^kNFfwgJ`vE%vdr3Q&VoRa0kVY9u`oHTIJ>c#)A1i57${Y-z;Qgiu{9& zV4Vz`6d;b)l1o*{w~n*d@h!&*0U_tEkGR!(>GXqrzqffAr*PBE~=D0sua(m+xATSTxe{~pjn z;DE68ObOYeg?;np2$sch@*|M7!|T$)wwCOk%d*AoKcG z&6Y)p9Ps9MCS7vTk`a@{#|jh)C9(H-~t5cmxEj zA<6{Oi_kC~>{*2HLP=54^6s~(AT3H9&s(H*{RmjD?0JCf?>d*stR(`{y0F`WjjiO1 zwQ%tlqpk8o1bfjSW%DSaiQ2UK0WOW3Q?%-*GFcmZYPK} zUwFRQ^Hxbe(0<{Q_&hK5ImXdK`JHsj;@8+WO!5^4{iOcoNQF6U?SN<|bn%7qe~43b zb?@$|tI8#PjXgJ0thSk}PlKzkez)P_*rFlc;lo@O1wuPxS8EC}_3GN%36ik;k(GtT z315YbM3B)aXjx*NVJ}0J_v6Qp1CL)r+R8Nflkzg?XZj9>;;Q825B|H($^!aj3-2oLdKsVF(8qrIgbu2N zv|8%*kFN&=A(Z7D1edC+`nl%SSz1k_%A-4Q?!U zy~sWxwh@ww}YyuB<=d}A=tm*mSva) zutd&jTTe3Mn%~VdYWUylD!NGyve~PNj@RIrfX^o)1nH7_QNnB$t<$IrXs)nhxdV1d z-m2Xreo@GX3kw?^M<+s;a-d-ks|FhOOPJ%Aw>XO15gez>cq$9=vg5s*ozh*$64LoL zmOpBy@@?okXb1N#%QOpcG+4LYRlyp7!;G3hTrF1VG0Ktxg`uvr)NNq8q@T@P%hGZJ zWR>|QlLWZH&GF?Ej{4)h`+pwtdI9X4g{OynojvPPZycA9&={HkNiA-7X%5)lOHU7e zZ+Az>1ni3gP9n|oNx$W4Q&(3P3>)36SFf6ynnKjkn?tcBrBz~tZKH~GYDx-XtWAK+ z!>4;+RJp$_OMykabR>;$kw|ZF^<KK~Fn!p25M z(7HG{`f3#<>-mQOK91oQ2XI*{bzlv;NspjZm=)a&XjrhpdfaSj%4`@DW=L`?D?IhB zz9@X@(szL%D*-ZOZ}0V(dJ>?A{%RU#VPq#VY$Yb(B##|C#>R&1G!SuK%wgKycPvXv zqjbM|FgAkfwuVS W<+HbSlTN{rQBvpS&&8k7x&J>j=HZ$E