From a46c80b1a7313988afa3693225ca5591879895bf Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Wed, 23 Mar 2022 14:13:21 +0100 Subject: [PATCH 1/8] Add decoder support for wav2vec2 acoustic models --- .github/workflows/build-and-test.yml | 22 +- data/smoke_test/LDC93S1.wav | Bin 46876 -> 93638 bytes native_client/alphabet.h | 37 ++- native_client/ctcdecode/__init__.py | 63 +++++ .../ctcdecode/ctc_beam_search_decoder.cpp | 212 ++++++++++++++++- .../ctcdecode/ctc_beam_search_decoder.h | 130 ++++++++++- native_client/ctcdecode/decoder_utils.cpp | 34 --- native_client/ctcdecode/decoder_utils.h | 7 - native_client/ctcdecode/numpy.i | 201 ++-------------- native_client/ctcdecode/path_trie.cpp | 6 +- native_client/ctcdecode/path_trie.h | 4 +- native_client/ctcdecode/scorer.cpp | 6 + native_client/ctcdecode/scorer.h | 2 + native_client/ctcdecode/swigwrapper.i | 7 +- native_client/modelstate.h | 2 +- native_client/python/numpy.i | 220 ++++-------------- setup.py | 4 +- .../coqui_stt_training/evaluate_export.py | 2 - .../coqui_stt_training/evaluate_wav2vec2am.py | 200 ++++++++++++++++ training/coqui_stt_training/util/config.py | 3 +- .../coqui_stt_training/util/evaluate_tools.py | 6 +- training/coqui_stt_training/util/io.py | 19 +- 22 files changed, 726 insertions(+), 461 deletions(-) create mode 100644 training/coqui_stt_training/evaluate_wav2vec2am.py diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 02b8a2c25..2a8437e2a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -274,17 +274,17 @@ jobs: fetch-depth: 1 - uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: "3.7" - uses: actions/download-artifact@v2 with: - name: "coqui_stt_ctcdecoder-Linux-3.6.whl" + name: "coqui_stt_ctcdecoder-Linux-3.7.whl" - run: | python --version pip --version - run: | pip install --upgrade pip setuptools wheel - run: | - pip install coqui_stt_ctcdecoder-*-cp36-cp36m-*_x86_64.whl + pip install coqui_stt_ctcdecoder-*-cp37-cp37m-*_x86_64.whl DS_NODECODER=y pip install --upgrade . - run: | # Easier to rename to that we can exercize the LDC93S1 importer code to @@ -540,7 +540,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} strategy: matrix: - python-version: ["3.6", "3.7"] + python-version: ["3.7"] samplerate: ["8000", "16000"] env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ @@ -700,7 +700,7 @@ jobs: - run: | python -m pip install --upgrade pip setuptools wheel jupyter - run: | - python -m pip install coqui_stt_ctcdecoder-*-cp37-cp37m-*_x86_64.whl + python -m pip install coqui_stt_ctcdecoder*.whl DS_NODECODER=y python -m pip install --upgrade . - name: Run python notebooks run: | @@ -713,7 +713,7 @@ jobs: strategy: matrix: samplerate: ["8000", "16000"] - pyver: [3.6, 3.7] + pyver: ["3.7"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -779,7 +779,7 @@ jobs: strategy: matrix: samplerate: ["8000", "16000"] - pyver: [3.6, 3.7] + pyver: ["3.7"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -830,7 +830,7 @@ jobs: strategy: matrix: samplerate: ["8000", "16000"] - pyver: [3.6, 3.7] + pyver: ["3.7"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -874,7 +874,7 @@ jobs: strategy: matrix: samplerate: ["8000", "16000"] - pyver: [3.6, 3.7] + pyver: ["3.7"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -914,7 +914,7 @@ jobs: strategy: matrix: samplerate: ["8000", "16000"] - pyver: [3.6, 3.7] + pyver: ["3.7"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -950,7 +950,7 @@ jobs: strategy: matrix: samplerate: ["8000", "16000"] - pyver: [3.6, 3.7] + pyver: ["3.7"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/data/smoke_test/LDC93S1.wav b/data/smoke_test/LDC93S1.wav index 86cf5b49eaa9314664e16269c5f87134e4dca3a8..62b65f8fc67dab9e3b46c5395daebf74654c9ce5 100644 GIT binary patch literal 93638 zcmZU+1-w+n`}jX|&hFkzcXx-3uphzep8iXx#R2x6d!AO@l$-3`*+ z-3|BdIWxcav-90QeqMj}bvI7TJo9wTp;PM?Ef)3Fs%zse&7bc3a`Eg+Db4GxhJ1NL zDNkiry`OmbiAh|em9-v+mGb%1Dzt{*`u~nZRq_?#{GIQb?=ntIMfes;e0uy2t>u0> zCgU0#@LTSb>tcLHxJyRy6IUwUWhR+3qKs9X_c&*)9MSwr&iLO?f!SSQ5?9@U%Z(i2 zyo~Doe|O6ffl}a*Z}Li>jU|2weC~*R%9Zk-yiy>KaYQGM3%)eJl5h$vKELE1hYxq3 zyvtD;(T$lLMdlO;p=? zfrYLf?~X47cDYm5A?Jk?Wn>xCR{<*(%n2{Znq~eZ_GJ{ATcDCTQWC%A8V8Zgk;I;Z zQ0OD~2o!RSTcuoohi-CC=8#bZg5*p1)?wFSQSK6G1d1dL9419B9Hwl7P9j$_zQdiY zU+^OL%Ss)FUCR^m9&TA5GIIN6a;fyv><@vGeB)+rQlzmi8Brer+f5LvIl>2NLl zBx5Cyx#I$X`zH_x-0r-?j$q62l^jjstm8+y%0Vc!`Y(Jko|6-~#!+7QKxTG4BxA@u zj=t`UaE8OI!;0WTFeTRt+>Srongr8wzq>-_cmG5>1W&>p4zq%Dfyb>|cr%Gd!H?YU zu5@@6OgnBCOvp>-cdHa^$czGwtWM6kRmg9dLq-!=k}@hs1V@5DflKtO-0$Qei9Na7 zjV!cut8&-5waPp)hRo%7%E_7IM;XhFDG8+s9Q-A`|cVir2?C*!p$MF981h0pULa*tXICt+Wz;5 z+$*bcy2-)oq*<^hD{@!J>fKELy=>{?>lq2^K!;O zlpo7ma)%>_4=2A4J96KhbIFw_KOKYuiOl8haMX7r z3uLl-#}7$Kl3#MYT zgsu)lj?ZN^aZP6>^Py6F8lYvsQGr0-qU>CV@~qaItP8VIns%I86Lf!I0c75Xez?jYa3)=Bf-T zC&yAadkqa7=MFcsKrC{qR6G$E%KBt3fj~x*GtSP)8fB&=4rN9;BC8U<7n;jD1b!LY z$+WE2X(9)YoEP{+3%IKVKkk>%->pJ)w9M{2C^_#|=6KcV8HaO`0C$(%>2T(FA_I!K+X$No_g)UphGqB;_p0C33YJN7g9U-?=||&aGeY;pif>I2_2k z)0+ahj4fO!a^dvJ9k`RD%3N}-i+ThK`R?c;W62yzJ({FKl3(4Yz$q`cYPYJSG$)S= zUft>)7bR!^-%rQCNnZHhb9a6@N;poF@nuB!DerQ(@Pf1a61`+nHxia0HNo2k-WN^~ z-VzU8*689i!KK5bSeYd6%X(x~M@_d1M;F1A;}lsa|N7jzWhQ}I za3}ao{&x2}dJ5FZaonfFy`z-lFbA!pr$8<9$qaJV&F$uqag*nr=5*_jYutL1pAH+z z`Q#pXB}a3=?!3#r&aS#+PM^uuZbmmt64%LJa$Ybk9OsS+FW!M$&L_FdQ7L(qyH1Wd znC@IDJTL1OXas+9hs+|7xVhxh=|?&4tf}yUXh-MAIXuc8GJ`=UI$2N3EkBt-sN#+Y z#7W3xB!^*lBsu$k@p=bucfL6|1tLdLxlg!N#+H@Hl|noBEqR}tHwm3kQO>*HNp4E6 z+Sy4N-(kqDJn0W5@$N=4kojdyS-F!p z!I-;Bu6D8zV#S1#Zq0IsjP3Zw%_a6a<^McWS#gpZ9M?p!u7Y_P!L38yMFM0EZXP)+ z_;Ba%Kp@ZvwFN>MSMHKe;R%6GpqACjcQ>BDnb}?GAeY(QpMy(QARf50#SRY+b9XpM&I-<*4=Z#?u0uFn&IQ{>AMzn~Fbk3!cde9BB#MId}zALBbcx2&^~3T}B|fhQA~u36G|8m(We_ za?*FxW@nrHmM3v_3ZubPBD3#zJWuiaxSs_JUF-C?UT!%lvcR|o7~TND^GgY#>)49CwhRyDN;oZX|Y+9Rx` zl1f#V>>2Q$9XR)cf$OX>8|%pg4ze(IX0?edLNirQea1-Lz{70juBz79ZMFuqnqv#7 zhN^=4*KW1tRRPYQU`2IRHb%1c6lafskE&`LbCk!UUB$YZ0J(U!m+VpSn@gPszKZH@ zwUsfesha9He&5Xqmw}}=5N-q~cX8b^Mk@zgGwpWU9jvTn1Y=99-=T0}b($4bRu#ej zE_k6HM~;DkCxPQnTO8_K;Htc8KRi=}&wT2X{l}J5b<}kGudSfkaNiMo*4_p6Hv(~a zwb|xU_cDehchZ-Y-hc)dp?5RpU13{u{Zjk4?V#GK@vN;o7#PXX$64bBdk=7yWv=D6 zl4_(@15pQchP77#2T}Eh-3)#O)0KF?4doj{@%grts>`{JwinQ^v}^4X;9@p>(tuUH z3J*<#BRc|BcV>CVPPQ{u503A)uYvy_&~CVzV#mW5eZbH=j4}z#9JbBD?`iN{ApucQ zxHmJ99R#x%Y(`ax)$X;ez}5(xSIy*(vEb)j^{icJ-{#tn!EXaqTMe<_*l&P+H$2pd zJBM-g7}lI-_t}S8MQdAKtydjvTRTKOrzYD!>>H}Jns4XYu5kB6yV4GTf)i~9^%Ro( zJNO+5Z+>kz*$Iqk!N@IW`KPVJ)xDrZKh;HjVSljCsh05f60nqm`>G=o|G*7{;DxVk zH&qgDUIdr51XEdo;*i~GPuORGb0zD$X&WLzpL4dinqX4mj z57;?e^%it(rbgR|!1IPG3cO#zAv2#S1+Tu(qVj8i|_ zb@mISq9>f#U6oS9z*`?xSuIBPiXyEWpkqxiG2T9Jm%<&7*iY^Is-Kz)M{2M!z&5d~ z)ll`KebJs(_p3ilU%Ohrqh2(_?5Fw#HQ8*n+4R#ude)uN09lg=%icl=PGdZAXt7GIe!WrQ3mY41-HM6c9{s|-@x0)?0xD1H3&Vs zS(Q+!$jKhnRb{u2**1Ef>Th=2wQ8Y#4K5!AW-_1|vqRfO@WmWBryd;KfR)Wgnx-&w zbvp&VeiLqNXxFLl)!m%!sfw#N;o4_aN4p+wzh!GfLq2(Cc9M)K}LqM-t*{_OXhP{JsI8|iH6!@>f4F>B6t66x~uu9{>?ez3FEYjE0q@WC#){+byN{TGAb zuIhO+7@jzU&VJnvX6>io{&D6fdqJ&&GYVh{-m#ylR(hIUXzo*A>&dng*ssBz4?>5( zOc}0giB3CXWxwg8DA~lljq{QTOXdYz6y^ z%AiY_#r8?Bv8`uT*`vCps%{3_>3V{?VYb=*aQRR$^}Ri3a;gK!Mi#r)%vFW-O1r?! zx1Du8H3n(8m(|S0+Lct@O?Nw07tvpsYW7Y2koN6JbDwUeW8BvjoUK6HwT2_lnv-_8 zzF{9S71TkUQT=Uh*fCJy2U|;xMnnE%Uj_%OO&4{Se!+Gz*X>KXzPf5UqUX-4R!Hl5 z^%4B~D7rBVIQmkTRxL~;J5cx1-E9N&gleRpM-LUVpCFsHRYdnv)y!ox6NhHBDS~!s zpr)vkb{ewV*VfT#>Z19{EK~jTPxcQ}16?}}S?^}^LWKf)l6uRWH*?_l-B|NH;J+Fi zu-#??juhPj?UBWfRVNv9IoKF#KU9CJs>n(zzRx_gnYO*`A1WOQDvK`Y3za?su58eJ zkg5X5mB(rogO4((7TD)+xci(Pj_fu<9?QbTbL}mlqH3L8ZI(g#C2A!4r#RN^Fn-HuEa(vEmRCJxU$#Gh z^X+yGy85(TW`9;K^dOrP`|~vO{|Y6Zg$sV?>fG3)7gY}Zs{PXxR@Wt-JI^9IW9L6tfgC{V@uEx$hz}mKfu|4<{#j#!|;EjA}Q1P8^0bw(Ktpd8T zSX;wd&LE=~@PouF_3#Pn;WO-JRTa=gg;~M(tgHZ7zRIPbhLBg&e}a8^9w64>?}jFS%6W@4Oce99Q9Xtfj|$^h-oK@JY#dH%~>>+t;M;2XUHzxT2aVz>L- z`)ws#-Zr;gZA&z4B|M^6;DhhbqdBm5EuijuYPs5kKCG-W>GSGt-B}mY8FVFGP_IB6 ztmN$X@ODQiA#=^OQ|&xB2XzSFb|yU55k7x_zYo+WYO7kIKEq0 z|Fkk*#WcLAt@w?77_SQu)O(z5KcFw^AN9lfu|{E?d=Z28u?ejwyhCimD&YLi379I#6#S^%Q{my3oAbuFCda0cHJFxMj-mjn0!*xe5hgTO) z-KpdH9i36n#40yaBd}5nY+0yY+vbMq#jr^YY(9I`+&1svt?aXJV&ksixjqLJm5{7A zu|oCGCwb8_eeDf%(|Goj8D|=pGhxJ}U`_kl_GX=Fi*NoI*Iri3RAv31eph$bS-qy- z@A^70-PT)ld3?p!;r0&d6{3jdV853wXt$bE<{5aUtUYLAwhh`MGam9Q__i&eZYOj= zH!NLuV6O^KY#@qCg&($>3ucVTXU>H?!}ra0F!&a~t8iUYwORGl^YKUi(YxT({N9J& z5nV-3)DdqhQk+E#e&w|v3Y4&AD9`Ms#q9;#%5H%Bvmkv3h^mGYE#0)e)nKHyrJAa~ z#)97i3>tsFlf7!zA{XzQX27sJ+#J4SCa{Jtk(5-{_cHR=QU9b$>g-Tqf-dJx_p*CK z^hPkUTtBXx=ugp*m5|uQ=)Fs(jLl|?+FNFyS!1@FE#___g;98?zr%?c&<*3k*jwr= zHCa6mHcFwB-$PGDz}O&D!`yGGo5|tyaE5u+9y1@Yf+)IV0iI2$v|g?Ac{}w9-Q61v zHGJ=YUIB;9)z$Q1G<9{X(@3;&OLW#>=6kcz957!&%}nrH5suHo`?yU!JP7`J8ej7y zI;1i8u_?^4mKst5N6#P4i2w|ZzAE9>=iTLjKEjlt7a>s9@byz>$<9Umsi7k z!)xJX^isUi-bwu$di*mavm7JNg$oy%Qdon%rnTu~ena9~q5J+ZMZrL$MC{p>sBbPh z|9gCs=IUX*lQBe06YXVF(NyJ*A!be3KO7f6WileIqs%AfJ^L$KYrM*ZuDYVNH_&^; z8|;1Vz2k*C#e3Ad4nM3%>$HN~E2_V3U=;MoZC@}g%&#V57n{G#YM}a-_~jhdrH0Cl zcKMKq=r8=ZCxLt?vGQ0Xc(EyE`kKOaiy0DL3O)^onaucGO_7o&Xy`>?!l)v8D=;+i zUiH$wFT8SIUay~bH!jCf^ymQm$?0h4&3Hd0@eO}9r_FU_@&of6-|vF!7ua!lU@>B} zx8a3p>I<;(F4p)u_UdQbn~3T!Q_DPUa^NYS2xo+OOij4?DimF6nqq^>s*S3pu8k#| zs0TBDHZSVc^)BfHx`ww7ThSJMH5nV<6$zb<9M(c2n}dtRW+t@gZAT+5PotwcqO%Kw zt=?c@xOxH~>27RF2DEZ*?09b?kZNYODMyrbG&~=cM@Fug8(5RM<|Q&ZPbQ?JfUclN z>8rZ5H%-sclk^5XLw}Ez?SWOF4d2d2I@{tM^~Y0E=$C@X=?lo?cW~tA_)+P2%=OUw z4WV0SU@3)u%8XCA*nXI>oVD@%f5+-%L(6Y55mOt>mYXqB(BVa~uO;!iYJBwg|Eg=im-Wzw^wWA9_?7w9mF_~j8WwV&Vj;CtSM zeoVn9-AzRLfqfJ|=Y3m__@E$Cbf0}3FJpvx*ThYJ>!U-fV2A$1{(giXHVs~S06q7& zI<0PEKfeV-d)2E@WGPxiEca5*y-(iY6S7qIq6b@&U&(@pdmW!O760@K-WxdzH0OLi zZgO2J~&={lDgV_6z@kU<<)(_BfUE!9YDvn;rgrzURXKnn~3&`wZeEvhs z{44aEg0<^}-g=i+e+4&pL1&L8*YF}xwg)R?z(rfGc@pgXgdd&}zTO1KeTSFVM7@J; zP6OL}(5s*DTA-F7Q=h5t@Ca|>3Ae%eJgchcu4IqigA)d*wdz%}fbZbvbOoN0>MitX z0G0-0=?~!*k3rTI_$f>D)Dt~c5GW7fFQ3! zld(x#>99J3?=}W6bP)Wy2X0w}EH8jYPa|zRfUpk|+W{*sGV~|H1b%EOWbY6eif`~$ z8e^qf!L`}+Rel%KrO~Vf!9y#wTUL07xLy@TvhG1{vI2Dne6lCtyBT&1G(HHwNdMN4 zU}6`moD5%{6swUC7J1WR7KrLmx2er#Kuj~x9|b()#K2r@2Owl z)jde2MA}Q?jgOF>HNcY!RwREC5CyNXKZ5lgXu^ZM)&tEeU?VG5ZI9W2_xCt{M_M?T z?CkT{m4D6O#0dAnv3201OVl%b{i=Qze{;LO*E_Da>E`%<+p(A_WCnT@Ej*?^Wt~~k z5UY{inds66WMC%}uf(~#2-^KGX8jUrSPAdHht_zOvBhKFNo?5DR5k@o%v3VxfFR90 z2}HY14}7wmVDWzR@s!AVeU!zdJ$_(N&fycub58wXhlsXFfh$@AYLtx@3cNTv52d%mTK5tAUIG67)k}aQv<*la6>0j}3JK%>k z)7f^8WTy_=s-DdVPV`)6@cf)^)!nT_UCM!PpE_}yRW~QB`J|o)M zt(WQdyyyH6{HtCwZw;A)6X={F`1v`Y-EU@O1QF<|Ql<}JsSd&4$nv6&5bd`7(70{?%aj(N4c4qgSXq^^i{Elrm0 z8`j$1Y!B~dWlP9Bv?2qN-L6F5zQG?phgW$yG~p-ViEwe4!*sSUdF%WG{y_gHzpDR| zUnTNjWQEsL7gBT0&M+rYK`C=6oXNEnO=;pOY99E(1N0O!7!ACh!~_++^12EZWC2uu zgqSL`86RphkZ7z1(%vv(g;I%AXW>bH7p@BDgd4-v;nU%QFt3{JkByX!O!I&Di}|t0 zr;$dHI$i_)irp7p3l@ea!@tA#!#BeLVHPs)X?Bt7PNw7_KKD^Q%=_N^&1>zo)@`uJ zkJu?h_9x5?b0GXD{4H!w6jdERdneM~3!R$R_BWfDb80v`d^{Wzyc^Ut>%FI<-J@sx z>3%hTp?@**dh{cIp1xo&h9iPY!O!7KVY{$Gn2T3wGP*C5+wJL<^={}>`W>%^-^2I( zN4(Qi;EKSBw~(izMD#7fXT$8~uz3PXR3q!RogBm0cm%!8mauNvCM+1n!p=eY;6C%A z|6(jlY@Pq7-^M@a=Z~gE$NLR+ZToe&B;F%v5Oxb!1(8sPr-KFIKr(8F^%8H9*NCWN zsJ9aCeA|zEKj`o5=VrR8iba3WJQnT@P6pS*MZ~eEsdKf)&uofy{0iObn|s5}!7st+ zV1NAG_|f2JFOpIwrCem7|FNGfQZKqKHZSspu3{I4Z^WO7&k6E|LxK;2UBQRJ`CyaT zr*eB8{h|IbuOLxy=vDTc`TM;=y1goGJDY3a8l<m1t85r3Y*$^8MTkZX1YnX?Ed()qew+n0Uo_=6L3yyBA5_o0>KHt=}+mHL^SUL&}rUo_ehL zILsN}la@EWBK}vrU~ntQ58Q8<5A`+gsmSR_k4OuDq(9ED?4R_0@O-W9f-o9>9~=oA z+vVnkFl+d2INg*}kLYP+uU;cMu7}@PljtYa4i7KH7o<;1kEiF02kFz|opqbknHieJ zHv6X{1EalT->0;QeW8CeCa4+Dkk&Q*@AO*ncj8Ba96^B~i}^~o_pe8Oigt;{{W<<7 zsBzMZcps}Z=85pfpmtC#Y+?Q|$HLFhE#;_3@5EN_(bch?|KdS5qMDJ-7BJrio#T_z z=cN~pSAz~)<44rmlzTFMoRS{-GWvIPXLNEbHFjD5Zq^2w;xFCKlXf8OM*8G<;c$NN zPEghi)g%0N(br?uV25;>TypLt%TT;WZ@dwk(q<@qiiMNf%<1eWNDYG-yN%f)+M>j_Ai(QGm63gK| zY1#*2`X{#!-kzMcBK_m|`Cv{kHK=I%>aG5t(SKqkVpAgL{KI}JzoGY-uA@FPS;L&c z$MMI4(cyRK%U{hq=$~FHtM{h&y7#w!UN0wKl!i^ZgvQ+yd>DT+JzM%W=~d!`;@`&O zra-J;#vU1VM|($)MSI46i`9>P?VUF(gJJQ4X)SK2-M%Y*Xnb9eBm5@_!$azQ|F!6x zSkG9=Xsbw$ND2QY8TIUHp&1w!3kJoD#$O4hnC3);1?*09%|54Neo4QtSB1PGb7kbJNzQkBv77ZU#BLsi`|M7D`KGjSX)q$*E`Cq27hTYh z_~|dAf+o6$cfp(H8qVeMKr9pS|E&fe8;Ap3Q*B^T`yobqT4Z>U4hR>UYY~-qNaBwU= zmYyelU;3c+g(qiLH#*jy8=>j1=|vlerD;byL8!30_3n2Zz_OheOO}(||hWS7a1Z zykg!C-CO@cX89UE>OAvSSRh!Kes_A`^xSbjeky%`e2lJ_nki%3l+%%)qZ4BFQu?H1 zi*8aMn4iPz@%^{k-!76CPp=tF2{y$`ARV94AooG^xoFmCN;F@zNF+d8zpU@0mY?6e z8&nH+2Q$J^;kK}@c^yCZepOY^By*G9o2A?6&BWwQ@vRn^SHi48hWKacSJNBBucXgO ze?QpfWlJ5Bx*+;PWJL7USlg7YDP1DRRc%|x?1+y~YoAs+{ge0$fePxxgZQJSz3$|{ z6X_m#Ix07`NzmwnJ-|tt5eCz9chiT)C;>38sM(O**<`#UKZLpJ-}uz$QY5`DjUk*<&ux&hhz31m)x z@Ur<+@v`=L&ApMj236Fd0tGPtwxkK%6x#Z6Y? zfPTaU=hOjxP^ak`UR(c7|0z7ph58vf|61d*_OorxGvQO=p0FzMNMZcpOjzv)Z4V-r zZPd!j+p^}T@bmCOxFmcmTpu*ESG`e@HvTa^&gc_kTUQWM=U(TW# z)-prM9-k!px!x3^=5vrd^{sGum|`|y<+4NWAW8L!tZ)HXcwf0$HK3u&^(Xjw{iVdXZ|UVk`7f%T z_F>Zzzx`2j7O$d?$xel$nR&pxNVGl>%W*x-YEI%m?+CYs1I!?u#ed!(;TdnQ*WK?F z`86^FzihHsPcJ3|HkzE=H1o0P748cz29M$!HZqsUN}nV<^fw(NmGRymgBp$fhrG}9 zG2PC~;w{$ys6X+UA0l3Op1kj?#3=jlgXe@ctO-@8haZREgnv-MIva*z8OD2DPxi9< zPk5zhavk6oj(idM)4$i>;Z4FndjKzV9l4ETW>VNZ>=e$z`@I?VV090Y*De6R_g71G zBkyr^;ROASuJ6tDp7K@`Z!e-cewZGE@nmVT*f-1`)_Vre{cj?N_F<;5M%X%h1)g05 zB)iOU`iV+W zAmZL^Kc_C7lWOr1^*EJYFaKOn!mkj%s?^D>o+TxK10it=GRNDDHC zwZkUF2W!KUR38^pJ$p+Zg2(rG9sKvvJEQ#@URSRwBd#V+yGM7R=FxNBGl%34!Nb4Sm49trzEqT z3r)0#y2Eqy^%NqbS%Un>7gQ3j(vf-(Iod|#Io`!SjHPbVo@~o)w8LmDXA!E7bBWwP zHvQntXUO67fSzBG=crEv`UG{E+w^O-;+0P~#KzoDyx4?qg~_JP2CIX?Y7;W^hv+T% zksReLaujoj>OUc-AII;1;NXqqZdQ}MJVOtIbP}gx9nzWMPx#_2RHF(cT zj+(ASY7QHz{8dQo3+_rTaRPan-_bwc(21}K-MSGizlsdj7i4$d;?;w!UsWRZywoRi zP>0AthIRwFq9x=RerLRY&>Zvd+hzjYE?`*!oLiY=37Mjw$%Otugf$U}hmpJc6iYsw zHOxe7Z6zzZpPXV*s?Ej7`!`{|&yt~NM}}k^+18iHr@l#!Yb1Y9krr;@H{!*XUQpbCwui6)wxy)i0e@uDgkD4 zkk6M6nX~jf?qlXHK(jlM-`vjc1I%%f-!rH+ZU-OB6R`dX_BQctA*kjC<{YU{&tMtsViCOPl3Xo2B(lLZY4GF~#yQCNSE0vYsIwW`F9Vuw zd|L$;w(_|Zm`;JGUlUb?S@Z;5BujgWPajH4PR|1yg;-qS1r&a_P<>9O{a8@c-S~xcq$;?LmW+&cE8tXd=t#%+QhnaB;p9jFwLhyeZK0V5e ztAQwm46^j-9|Mb-Sj{Q&#@YDH&UiV=GUf*#xfuB_=F9{qRN=j1qFPpjY-=tcmd^4b zNM<(pRdz(kw+Nl}8F@d-*go&);Ji!F<{UZlW6UiTrbAS$w(u(>{emaKo`oK=8?+#> zTqd)1i*?-1*cr(omSUwhfwDMrq{BP)Snow7JPR{Rhrjgv=LS02*-{$m%}DmM5MB8B zxGO^ZAE&Oh9-L=Ke_mvLhsc^=W&Sv|kV`-n@-F)=EL?CLSYqU`^THFy82c9Cu$&QY5bSsCfNcNNDX009+I|5$?Tz3+vija$I&e&_|I6cS;EuFFF zIQ|!v(Ux>(UZ*y_hxt=k#dc=B3T|pLmh_mV+h@sQ&gag1IPweq-Mz44hw$AS#Mbw8>hbz!Kyp2)r#^PBZS9q4~kjI(G@*7lWWslGfDnG)z%fMv^&SWKa*aRld zGuAQke^M7(2E8i+#jniU2HxGmN>ix|v>@7<&mH%{C!?sy*8&?0xUw`?WQU`w($O!S zeZPRCQ`o{n;IB0_J;OR>XVg{r=V2nZ%XpOq=(xH?b!{K_&xgiY&iTxq6N2vu=xdNbaTqJ-Sla{sUXw5L^8cePB}tRP;1;ZWH&vrcgu#w8`189~7=))X%z+c+Tb`!SnS1N&3s3nX+ zBAy@yeIJU=RSob>c2X(tggvRxddjKK6O>s>1~@k>xd#l)qUU2K)SCbfCL#6bsWN;` zEusVU&|&DQo5;`-Fx#7N!>GDa>Q!UJ>@R;G+eEP6$g#5k=U+)M0iJT9)r#SWOzp1B<#OHYioi+{a@e?xk z72Mke9q}bvr3XC=^Wm*xXhqSI-LMMNp_r}L*MoSBadeJh~8Kuz)l9C8A` z;9g+affk8UsToC|Zf~wyiZ*!w9hnK5?%`MznbJt2?8B(Q+$Y$d)P(x>WjvVz$iXh; z{wkUzJ2IROf2E+eE>VHcPHi+RJG!($*O5G8_%~7}%VKEvU=6y~t2; z=DES~f{e2bExm&dhs*5xXpHth#p+MtXO>1|9>=q)03?5a-!}M5myq3@=!)~qDBg(d zO=<#%uSfgT24{z{D;jH+f}P5Mbk%3~FdV6pE; zFYN^S3h3yqoV|z!^so=t@EEQDZ!ykp$C5P1&p3k3$_+KMGM`4ya$%iwfFq3z)kRP4 z1h#V6(_7HxRO0HAtZy5$RfY~5nJqVV_&8D;WgP~c7=t2*(fmcYZZGRg0nhu9vvgK{ z8d{vgn~@&tG$b?wIP<_mX6BH60r^>BI;#;6w*YXIVNHd3Zv?i=@>i8nGjmm5UQx7e zoYz(GApJK7S<`ClwAjm~P;5G!DE|K+L?+YFkKbe2rsCfZA@Z1tuIr7I{)PtF2VUPm z2fc?^znA<`393`W^nJPj(N0w==a-cwzc3!VPzG4e!8;qEXG>d{xTP9VOMW84Dts2B zD&Lff)pPXtH0SR%^kr9a5+~`fOsDQPlIqel^zbxRu95SJ^ofqsMTt}o(<6AAXf>~1 zL!ag~G)`Mqx(Q40GTk~?i1^ph-By(x!mUI$W4)L!JQS?%k@DB5> zIn~n-(TW4W>qltcU#Q=`NS(Mdk={$DtVuBeG3(8+hN)}XP#w%*r<1XW+WT}jZ;Ad* z?bVCCx5+EqVn-VFT~eSGt%258luHO@9Y=&fsJ(nF6JxRjX^4x@UIhgEkb zlFCn)#c(}Bm(@?}F5b7^RdVNZh^c4kn)Gd#z;>TQ>&NUX<^?l?TIyfnaN_R>nUO7I z16G9Dh(ZxIFRIl`$+tWTB`<_Qc+mvt(OT$|<@AcTrEhtZ zcivk@h3tl2NA0I8J>F-?zl3n?PVz_3f~%LPHs55dePKZ;_8xWJRdn|qqbfazdUhA` z4^832o93u0~QF!vmI%V9er zO_{Ox{pr1W7rS>FI9{O7H=FkfxSL3qO>13CpQS6TKR!WMGISe|$X<*zksg)?rh+MN zrWy@Y)#z<{l&X7a>aH1p?RoTzZ@)9_=<}-hkLyaRlXuqJO9x&S{|#>*Gu-FZReMbd z)le5zd%*T(Fjjy*1@?A@GpRjiu(g@1Gn&3F*?~!9mu}M={UCjp8}uA+gZCyf)I~oE z-lo&9w1%#;K4`T%aQ-gz&0BCud*poteZBR7t}_~RFdaub>415V@AdFk<{>$c>+Jpw zea=n+uDbNTPo}=UNgMr{cbp9O(Mta@b*hk=$*U63k6@ExBQ)zpjxtGq&P1uTx z=(8zmC!K3`byIpW`g_~G+@7VArZ*#94}^Crt6KI`)lJ`~%b^Xu zONHU!N6FD{3=f$Sc9w~U{pcRpjD@R0mEk9Hgekfx)V!saA``Fc4)m_pAS<_|8Q-g@83Am-~eoX-|Nym}cJ~4}RD| z_rC8xO=t3C?=`Oy`??r%o2yIF?VrUP zfZWZ~W7*G=SM(W7Sj1+_eRcRdfW+ z!FqKg=le8!pX$(Gx`>^n$H3rUWP!dVr`-t+@-N*;N9c;_hUD(1YvZU%qf)Vz_Y|x| zUb31sk&|V_0>$-ldYOx(p$1YZdriNn|DuY~Lm#BSU~~$$7|AzGN5w6)drMhsW{j4&fai;3YlQhw-RR@OK&y?J7R0#XFYgY-Gl- zER@I@6~PxOM$}f8xhpbrCH%B9yjLc-REbvw{JDzQ`hxgsCGpkrinN{lAY z{v!UtE}y}3JB#loKBer9+rVore)0r?dIqeXWi@B`llOD{lJD+M=zD_ivd8f}9B`R2#5bAY3a^X6DA1fq z@RwlhG?=yIf@EW&*E*Q+XPxK;W=;3K&lVqrfVk zg4ffW70k+~yyUw)Q$?tAgOLPI_nes+@v2}oFR&H_(!y|{Kv#-Chtbj;DbBGHoE5n# zkhnG{_XwOBSgSnYCxolyX{Yk^m+Q=R5!^To3PmL|?*4=;lP`f)IN~($$sHFHw78al z@HRY^jvTos4apN#v*Ow1f}Rea#S;)FVH6%M0h|K2WE>^uUVs(l;f`$3A~nISBHiie zm)pR211w6OUZPcz^(0LHh0j4OSQYsbs$T&@S)oum&RFhAT~Rno;FIUwNbWj6cq$Ce zO7W6ixyhGcvJ}6B@z>CX`=zrcI5AH`-xY-8N&rPtX3FxdJW?c|WfL?Pn2Q3x#Im_q zk;u9{r%teUo6&E81;Lv@bsT8qbqJj!dP!;lPFo)WLWfg{g3p6((W|!@K|1N>DN^$M ziY$yTaOGn@(J>+sB@*~@ly&c=z?+k5;ZR3afh!-@W<2AphK7sb=L}?u)}Rv)09QV) zyoB!FO0{M!^^!lSTdksIvmF?&fhDO)oC4DZC{HF}mZuoWvsXn|CRXq_`jy6gii_Y}}g(ohaB6%_90ww1jZFVC*!I9Zt}4AGDO`3W-kN zo6y$Bxk{w?Ecc0&rty;JOGdG8ITI8Viil<_#H$EU3BKfA^tbR+k^hs=+zBoeJPDSv zB=|oS8Ff#+5?w92>KgJUm^cH37ZMsmblq8?6NwP*D?7)9qvWYeQbVEjBB9@?8K^z_ zQS!9?$!TTbnJM3KeaL#Yv%WImrzkL|Co<{vv3y0qT_f^zBiVsD3{PbYBWEKA^*7Wx zKo)x`xqu_=$`?Dlo?LPo`z!n6bzQ;Zd4|019Omgo?&=Ht`@4wd+YnLqAfNFs)INz_ zJ_&4B$;34y7gCm)+A&`>Y6ngE-U=!ez#EbLRvqkc1-^FzYlY$Ydw?>E^_A*KMtqM= ztTT=EAHb4cfOpoAS=D6kc;*7W)dIX(nRykS^aIplKW4|pM`V*Hkq?=`H4A`eJKlIp zDzMa7*jGK9Xz)0DI(8GU?oj)w7k@#0`W@ERjjJSsB)fJNkpcRaHU7mnY72PN3!#rs zte8gRAy0kCgm)}EYUJryyTIDuME7qQyz~t8Fr32|K7v=g5WoH?-rlohefQEmK3SF0 zkFhg;06yJRV&!b!Zg#a@V&}sdI+6y#!T}F=Fl;AFn>c1rvV&y4VWLn->i>M`8Qq4W5^_pqdVnI_V#^7Wbp=nQ}KpR z0_8*W&W@t{eYno-wZ#9==DkRsqpJ6i*WPChPqCYQ9ed7R1ES?bU@<#^xM3~6`5$=d^Afp; zwd{8+$jDKA_yhEgzfY9+1+c9*zUsj~=$X0&v2Yu9pML7~1*$3DIPU`@>H=g8A4%lS zGO(k13mJ&>;jOR>7u}g3k9TvVm#lD3H(H+#tfaxZE zJ5$&ZFbO;@A_ge{)}EoO;X^Xm9f>qEkrVic?$Vk>+m(m{TC0J?z+dV&y+LGdnv?75 z=?(E#&|#QO%M-3%CU;hb-rJv``BX47+O`p+U_ zcwFyB0-qvMy^6dxAaAh-JpDzEW)pl9As6xp-2FWg>`?=lM=VjCoeRV1dD={O%qIFl z-Zf>^$3RulyGhUXOWqDIum7^Q(~J47h`3sj>(6dKrMGD`S*b2i??|{TtPHlQB1=+Z z{DORCKK6UR2~=_Rfb0OiwnQKgAqUb=UYpFvL8v#)`~&58kg2(!Xk$tOqP+OYm8hNm z%wG8wy}~+S=derIF)VJ4y3QI)`@{W;el~xrA419O{&Bi^ivrb2Q;$yZ+VuS| zB12WlYzzw!{XL4_7)DoK2_ofp_0L4hi}hu&HI^J}X(|)}wVii}WjB%w*};w;p2R~$ zTa!q6JNhOkHI4p2-UKZZkX`ib>*UiWg|~wB!TjJ-LQps868sp< z4ThT5I**^v|Hr=)SrTa<=^AMjDd=zZ#&~1&V>X}Z$Uf!X!47)#8?Zy6K3TNwXp}yx zlfIkSbt+u=5Ha>%y4|X}iJp1c zNxhvNlyS1F8^c23#GrI=DV`%3MDON5;TUzpyX+hPt>|R>Oh-k&iM&Kk^%RlrWP3QA z89Wkfh+m423Tn`4THj=YSIW~_wMKtH#>oe(UwXI6%9N+-l?D59zg>!~eFW?c(Msiz zu*u|@2JqB`9b~^s@WhbSMEiY#Xftx0$4(|vJ`~mrUk@Io-}F%2#7_jzhQ7V6M?^|R zTSXhh7DUq{y&`e{1#dDDJ@(9u4c>@9O`mu9;99UVe8G&Po_0z#CzCtbdzik=y8a*R zFDjr1!?WGVSImH?x1$&9vR7t_sf^XA$-b&H>_zN}zPQelMOINetW0g?S@Krn=`uV< z|JUWOk{n4@WMDX%-8fyX?ZS0IpI~s%DR?`GhDXB#W}*Hpk`h}S9U5yC zYZJZb*Y~%uqjDBD@=91fC?C%dZyYZaJVQ=xoSjRLas%>5GuZ24ydwU$?6uFQYw~QY zf7oeqJ^YIedYNz^=D)xG#*ECN{fQ zCpt6sP3&Z>R%}M3tY6Pt$g`zpV|{Xkd*UVHm(xea_Xoqx4xSBH!dn8RI`}ui(38|C z-qst41hma#rZMu@;gN8sIc{GCme0r-Asuk^d+eIepqG+0Jz~q*MO1jEn|<&Zy9Zds zx8Z}~mf#EaUhEHgv#05PdtUF0^ouoRw@kj+qDT$Dus4S~_AAIhS$01>6|WgT9nVBY z@r=D&=l0f+Z+?*e?_NN2klfYZP~j#$tSiGQbaQ6Ks%@iV;b)$6b_B2Mf@;q`{(4xy zso0=^4#PcY^Uu-hKOv88O-SEqk8nrubMO_rg1SJj0rq3>&FGrgsMwO&FVO;#KfIZ` zq&iQ|?{s)~m>Tqtw~XJ4*9fnhm(?uY&pYQ;^~?AZ>B&B&KgUNJP0f5O-25r_AUFQc z68a<_M>C6do{Vo+02!;PFQOIG(Led9q;11*`xl-5j7c>g(LFu^P4q@^H0VyR?o+m# z*8^$|1)A&8rz4)fPuEa9i^I$fD}~jAe(`RU8ykm*O%JSdJE|A;{D=H4bmtrn0*;6MJbYA%zo(wI9c)+(y@E8a~QgDn!@9A?(z8 zF?a$A+Y`JO_BNr-<8O=ZiA{`6jlC94jf}*`-mWd@o7@s8|OsZZ_yU)`H} zR!R0Wed>AMY$||9u?cJKQEDeYQpd`JUsjx_hwLIl{T0>4hSYYx1*Y9=S zjWQn<%&PV@l$spgM(-33>IEZ%3&Ee^gLbI*bo7T<-`KF&uxN`&#INqbb9M;5v^~S- z!GQQf@uTtF;aroOXG9%km+NodNi1CnYFP`gsOQMPjp0ca-LVrNV7WR`f!&CQv5Bbc zex9wc0sbk+j*ER%@zd#cXpev5p$p%o64#S%@mGS@L0@+7~KbH~v*T7L-IUr_vF;2p?*q_a!vCf)_N2y||^(I-9UXJMiWU zki)Ht?^TxyOHVpdk$x(Hx3Mm%?ADe&xQocU)+H8NjJ{bCwhVs@J`dgu#s`ao4&f6f zyI$!xiM|@uu{rErneHz|W^3~F#0>UBy5Zjmj>L1Zzhpsp&{&=jSqB@@kmu1Hrp9}a z3dHwR4L-v%{zPPY5b6B~X_?5M#@GLq*pMD7`iLhJnf2ta9Ukx|qO$|k#ZHF3!>z%$ z!94U{w(#Zf4Rcx*@Y_XJM*2h-L~=(a(;2^%9j~eMh+HzoOyO`IaEHOlaG80X*z#p6 zp--Ughw2ab`v_U;#)=maPYG`vBMvxBOfe2wsg5__179~v%=Zuw6o;R?5@~AnF4pr^ zEY*bYHEdNSy2C%AEBiaDKgYdx?ClDu)4t$e#HW5ir{gKzO<&PuH1i+@;N_h&!Nky+FS@>BCFW{RMl>JgJ1wmJ1_o^P_s+kzH*#%st9uAV%JOZLeyXF~dx5NUX1 z+8>}BXC)Z~>9WX1toj@kvSU=B@=+PSOQ#V@W~Mi45IsXvs1iI$jPx9RD1-TY0)KQB zxuGq@r_J$Tud}L6{4K}7TtQU>FCmwVre(XZ$+CXBq?({=S4|4}z?YTzk2KTUX zsearhYaz28O5~U%Q+%FSS+Y8k**eL{yAnB#?bHf4Q?J-aUgHnqJ9(DGD*8;eq3`6G z)7yde0&9?*WOg`MvV!H%JWa?H+($$q&+}^n2I`W#XatYTGykgdYRvI!WWvfqgRJnv z3D$fJ%^_V(C&7Bez+S>Yq8HJu{MB(E&wJ__)v z#rV=CT#o!!8NBx5P`)zRlOlLK1@V2Qq9b*aoaA*R2bTdFNp0v7vG66}k^bOIKp+Vw-{Ug=TXTi zUx%;G1E=KvWHfmWq(tMAua_RsOw5r9xtAFfXJw@sfhr51sf;SM7(-6Z^;yex@|1b0 zvC032BTpiieqs-}?)*v5yZm2cAsJ)I@JfErWsl|A`SQei$?(hni6fN_SM_nvxp&{) z^X=uDSOSLRZ;^yM-(_Bx;g(Ms)4}Ebzf2ifY8|AKfWrL`oAS*)^Ioo%{$LqVu;4J5 z9NX0~G_kt6-SEwS?li%)t36Luswq$-e>i%~D z!G!!1_~f{ZD*t2@xk^?ebsL$(ji3C#9V}5+8nep(Q6PVEos2F2|APFJqizL4AsOA_ zUsfmFDCguzgmZ!wIqG0{5Xw5`3i%}?%b%=Dj?3B{-Q`zuG~oleLO$hQ`Q<3>Mv(Q% z|2^lfNn%4*C}RlJ@+beVk(NxjAd*=f#pKi7Si4g%p@8BgwUP$%&xcL{9rKk zm6sJ*k*ny*YV@7fMGKurf6I=G3v@`9B|}yQeQ*igB9-Wn{za+lWnt`qT)t%Sg-)4y z$^TF2YE7;lESgEAMkG))f#6DB!cBq&_nE|H@~m7TcO$-LD`l-h#Tf7MDb$uL zoQ%tef>rrnKHS|-nq@>8!5x!(e5n0mKCQYu1rEGRxe3M zhm9oP$hGeO5Rn-L{{oZ1A?tT9`JYR&V8Nu?EFB2H(mOPX+{Oudv$NPK#L2nI!Ir@< zYemfDvw!A3GLG}G>HX-B?oDMQ3-sNFuO&SX#qq~wPeTf|!EMA%Qf<44ZB0c|#FD1q z&5Ql@@L&t!=~SY+F`E4Ow^-j2>}Kdh58mB)kBgARc6d8Y@psqaK}|sp55hgN7v&zV zOl6PGe*EKOSkEl@B~sIk<7=GZ+KX^hgps96mkICjvOR?7AvIB1WdWpE>QM5(jGj(- z2~t%T>n=6sG&}_9+_=JA8SroN@+bcji|oXbr}Je4PFaKO6g!#dTh4?3DBUDk$t9k_ zvyu67Q}=Qm+r`1yF77-F3=240LKO!)W#G-d_<bNM|Ed}gj|FH1y`5G}SMq>^2-mMu{#idJb+ z+Os63%{S4WRFq0ug(O>2_Ush0lrp$Pj;QDEq+2FBK3k&BV`7x~@O=}kUCm>y0^)J> zImUQ&16)AjQbgRCJ=;HV(#en~z3BeGa8@NzFVP2y9O{>7xe z6l~O`g{$caC@RAE0QH;L=eu0({RVvx(cN{1yy4Y6`x1$RvQMX*iab(1|3$h4^3<5! zoA?OGoqfzOnb&z$?z!h`GcAVERV6>zl>E zUx>-p@TXHcQwqp|ci|Q9mth`_PEW`l-VQs%Q0eQ$2=UG`^gY|%Vi)>Lm1T8F?vxP+g>PIZF-)J& zc=ue2PnueHPhCI7b!xq*59A~i>XD6BhO!K$L^YelTT58WVqhsimj5Kl14-B<6s!+- zUxB}b-PawkSsd=}!+TfZ_K0Y&mO<=50=KF5{mOGDlJ&hH>j}cGb|n_N&qQ#1Z1Vn} zVypA@RHW=cU7%um2QInZr&oY|Dj7RZZDO3J^AAplI=8YYS~saW(|r&ch`LK+!}9}u-f_hMT&V{ zMEXvncQ(nv6!(95pNi_A&`66w5;N>efd2?wJWDgyrE?0PL`j@j3BA^3En!<&%hgHs zynd^WV2yK%CX@4B_Kp_1GO{0sbqB^tMB7PmXu_Dn-57Reh+ePvsXxd@W2@X_#Y=Ha zCtBuW&p+n(xvpRCl!83ingYLv=boRn}m8pBYpTc8>Xz-H0KZj}$uqjKe z^a6J->WWRcWi5J~0}Ivl-&A*cOp`dR#NQ6l7{9~Pxj42p3sKE_;m)VSjmdgw?GE(Ri?)?2~!4z!E=4=kPU7>nR9h z$5H+}^f?TU^H8a>uc(r%#C9zp@1I#sKHl$fFzup$F4RvqJbSLq3z@!TtFzx!)n`t?;MoA6fXAQjg@rLDm49=jD`jeFS;R~ zo9U_zWTy)4cP$U&YWlF7$ls15&Ay9GuY(V^;O@WZg{WU`O~b8VLH@A9(`d*l)*D%b zV|eO!eOkrorZOb?Z#0P6qeoUQSityo+Tl zf?s0NbNK#qJmY&iT93ll0Qde6ZGU6){sr}YR+s{3OVqdtX8I<&Wp2d9UF5%uyY^q} ze-!1`@u9w?Cri`uMcnZ;mN{mq?o+E!((mn2^cs>9b*ejYQ`F3?XP<9zzu=#?G+6Lk z8QHTAp1BxLPlJI!tZF^Y)zo|KJo9R~&gdn*mVW&hbuMy;%{bs6nxr?cs|c;Tn-*wH zkFFxId#(5%bdA$9db1D{ynkj^=EJM+LQb7Oqj&956Xl*p!?N_(d8DWVEVd{64e@YW z6l=uFkEcZj(ce+c8s~1rNm`9%l-i@p%lQ9RcdlS{<+Bn#nm&mB;Ere!{?WBOwpRSs zS?;|BFaLla>hTzQ!t6`z&k<{l&b0XKe02Ot=igyCI}^u_qOlM9>n?Iv1|%oxb}2x* zSJL;dJ4t4y9os4RAWqhMpMS9g#;ycaK74#Kx@TB{xpc)0)D7Rew7dN#Cx5S6s@^1H z7XFPK&u*|C_tR9EFXo)A-`SA0=s(L>wXEk`-!nfY4;SlE{TFtA!;ks>eFNMy1z!pH zd_@QL>tK3-2lY8l_!D0>=Vcb>mt2n1%EDh0eV4^yuL~}0YG3(TmaHZ_F@j8*M329t zFQPUH`pIf~lf>hwx>Hy4a{4j9_bY5q8?Z6ED)C2Xwoq{ApZDzl0-x1e2e zX}#NV%cVHwNgOoC6MK-pW-vGx{M)RyIy|o+Lka#yJ`&JHXT)qeN_E`mRsp z-M-ab+9z`qA2#M=j?=~ZKYi-8tYyAgozB_N8}cw+-w>zANi*f`h^)g?UHNvep+f!SGx$NK^+VV(P9{jdrfk_q?2ScIuuZRX5Hft56d^jS}7D@v_24{*>MM7q`76mwqAn|4aY!1@iVWU;HVM{(+yWvx4FIRI7@hl#J#D8uebexj2N>EXlWID za=Urrb>QHB`tdOl#Ua$I;n^`U_hQ8kgUokahIIXgr^!$WRjynynvDoTVR-mm< z*KtiVt2h^5cDC2D308Ko2S;7`tMwj4;qc}XG)hC7q_?Z?hSiuXA6emA?h}6G*C>7p zET7{u>#SuT8*~-!JfHrU?e~&+<{3|^Nmm3fcOZ9rU8jD5#9Tyf3*qDG>`+~Yd$ee3p7o5UEhCHnH#kp2li_wvYM@C$*j?+LQ*rZlP{b_h zpV8=5HYD`sk*sBj`5F9bJK|eVu|E8sB!UaeS%kFxCR({sR1>}IUxVpZvK4buYVbTy zwjXjAinMjrF`8`#Ts0sgtyvg)5Vk*e^=gnE6;ZCB>;FXiHh8P9s~XT5N!K^QZ`a|; zYw%B;@f4nA)RNAov3HTYwz%hGzQKyDue=S;E~Mpdwg2)k7@B}0hXpBOmB-!vWSTc0 zi9ZT6tN97vk%Z_JS?<|MT$$jRHvoM|as^k^cJ;ZeXjRm|1YFhO?;P*1fcp(7@H;H+ zwC-c^PJ@;>RC{6+T*XK^chY|Cg z*ez+~nQ?|xHU3KZEJj=T{W8&8OdM`$WxXxg{pmMT_{^e`TYigXV zXm9!hJ&_QdDLo0thN*Do zn7YL>{GhgGQ-4TO%b4Tc)qLl9`nNVD&s6dHu;<*MORuY5uEX{kui&99(gWQA?6-rv zo@$x5-9M8)*&SQ6ETOl9b{QV%%l3eWsr-3SJlkAE|2Ga9r)D#enV^RBWwmTKtC=a3 zu93cy7xbM7F;_%?n@X^%DyXhW7D}~Ey`Su-rf|I;>h<>4&sKfAUe9k?68EDk-X`<4 z?OCVnh?i1dkfV~N1Ksp8uW=>|eFuJe{+G1h!K5J4koj!|Dqvs?*r2 zQ}$AKTMdI}=^4K>k%H&Z$#J@;A4%;r18rsUAP5qv6;AleF`4l#{>$&8-idb7&&qtE z2I4wZUaQrAY?EUvXqR+=eeLg9!jC<99sIt=78D2n=Q?mFkcNF^oGEFt|e>$0WqInw_d5tAHh>j0B ziK%FMZr%{Dsd|~81;g#+`X;f^ZnN=rn{s?dYO+e5*|5#V8--c=_lx%9&f9rTZWmD-%@Z(?UFRlb*kDV6Gw{0`-M zivb^(_o<~ux1UPrU8;y~Qm^!)Dh#tN(f%>sPAf1k(t)}^Tf@_wM4hE$aT!}2=lNc# ze&iihxT*9vs-m7yCEH$=>9^^g^zg&9%M+xsrb)WBQ)f8&=3m|Cr>V<+M)zYyXA<>S z<8{A`)owh{SyysY+ci*|l+rgj8Wazb%Um_Yab|9Fn*RsVUBE!g72l( zcF_sjO&#o`{^5IdRs-^`G{^Dbyt~saScGS3xX0k5cyfwP=BlvpPwE>pzb{A)P2B)L zr>5RbZiU%fS++r{2x?kWU){r}fa3zy6rbVOQQ+vP=HYYrct?Np9yO1plE?HM_t)2a zSk=sjY~Ffwt!{Gp6Rw+{H&kWTLr?O7_kZ=bY(eqRo0`Tj*Inn-02G-QR0!JPOmwL&RP6WHA@x z)lw>Y&Qh;?IgA{TN1Q;P#Ke#)X4H0}JvQe}%p0Y$_J+K+xc0krKmN@pJhT~Vv)U*3 zBpaZ~1RD`kRWr9#b$4FsFSSB_l0VVym-8swW{#+h{t}N~j5_D5a2=>(=!^6wbvpOc zW&1>1Q&o8#z;PFtr+=}Uy@IA2%*8pycrV>E`$2xO`sqpPcAm+*Kkpi~uQhcdKTa#` zkRe`#s-Go`;E}GWt*IUOw27Zurt+t@nNeRVIf~5Ganfx?nank~vm2_^K@8IJS2Yje$<&`DxLN87bv#wgnVhPxe!aLi@)q56QXHo_Zq=dQj5h0n zF0a#Ozojoi|52poLi%^3c?ZRuL2-JrtQn4Hs;B-OBr)^qezjV!yY?LCXT3viUsj*= zWVT-JR&ujS9Qh-yIv}~rq|H-Q)2>QAXud-S9Q&7wo(VYbTQO;C@k?8KCVSE1W;OFo zREyoD%J2*Q8u`ifQ>^)J5zElTN-|g*1SM4#Rn}EJR}IYVypBIu^n-dju2F5c+zgD* zRZ|U)DqtAx1t;6ZB5T?9eohkGlsrlO>^Cq{k@VJfdensENK+X&y{cWdcol271s0^Q){Duww4|YfKGCuS9`k6F+ z`n2x+o_OUo9>y|t$5qJh7@7Uu{Nbl@|39#Dvg-=s)*sz}49k8B>-!{6VI-WKZW{Dl z{rKZZa%mNlO`Xp8q8Q^j6QoYZm#@M=dsTktrw*xlu8~@l{8}G)V|1QkRVRszyQuK{ zTd%{bV7Z31nXEc%zKNG@aqDOi@ow3GVYKV6L|P?uL8sL1wTc&5%o}-xYuJIKDr@_b z;e~lWm{am4TXPHAI&GOh5>qP2lAO2oH+Vv(%gtX`#`tEd!|zSr0QI6K90Ow zXZL>A?jxG)CVKL7^^ogT+IO|ukN9O9VfY?)ca`0^qUzuZX8nNS_6U0MCU@ECsFIlm zlCw>9U2V?EPvE%|ZLeaj-shk726KMYxy?M*x~c0`VE&RkL$Sp8)P1)dNry}gm*2GU@zMI76Ek)Z? z)vOiK8Pk;o-=s2gL*AdJ|Ndpp#I;sao_)ND|I%Aca3?(|9nHBJo7|6g-%~C79-nQk zRZd~ef5dUm*&E5@buQ-X4WZ%IsH#55KO4wqOrzI!i!6Kd5x-5WwnJIm9F6bok_^w* zB32b!J%Ub0(R+H4_S19&lEZLp zDmmTm)GhqtLHHr6-uLj$=FpWN@zUmjs;X#bIL{(ZqR*4R8^H#C36_(x`gMkV)7!Fp zfv3nB#mVN!QK-6;I{q-@b(J~4r=t7gc=ims;%a<#4oeYpRqCWRn3S{J>pqo_1Nj!K zP=2b&b)iT*X3ZAoukP19^pR@om85u`y2+tDr0F7t^LVWdM5&{7z%Aez{D(dtfh*2W zj&8g~+0_)!JZ?fpb+wvP(WrO2yr}D0nxGI^n#+|Rrlrs1$yYP!jp4#?aH} zfn^UmzD5J(%e==P-f)^y5oHspb{^>RAv)hzu(Cn3qVwvhdqoZ`nO2&a*<%1Tlo~fWIemK6vs^C z+uvu+alX-d*{eHwemnToJ^AGMog-OFjp-=9=aXX5IIXjl733!8JMApy3AZ=-<~#Es zjytR7TpaTjPJEtk{=U7-!MvUds){3)%1>*VSDj9aRpJEPA4E9kiGZIsG522B`HrtJ z4@C3X!%1wy<#IB|#IH;6dguSsn77i~mBc$$XwUxe`w8q_k3$=yZ6S0ngi0mM9=VKE zzmQCVsk-Xvm(+@WMlz?t=gT_Z&S2e2kc%Utu6^{(A1YF>V;9GWM+)McJ7~v8e0>A2 z-_d^ad`-pKciUg7C?oQVnEy#0ZY})$vFStCtF4W`xsTJmv-z1H#4LC6)$XUOj`PFb z;nlrrZp%=dT$BeCUh@Co|8@D2`Sk3Qv}I*5717mjOyqMY{j1m6@(*{@04v4N=i!0d z$n`sB{2r%`XM=1N*oJxUP7zuq@!Uo_@IkR*8y0sTo%IsP`tc@O`YJ@1r&!a9qKsFw zURh~+tG$?Po5>rk@b|@{h;8ifD6|+R7rhO~+)O8*i?@%M({+HKn+s=~#i>`Z$|J?7 zN9p87;;XxH&-?tmGUmVJcaGB>_WJ|-xozRHoJ{U+{%tq@^Tnp9Y!*qs;sl*1^^aY} z=KP%=BeII~sUKHyJIP&6b-u{uEY&Z%HM^;jub#|f374{nx8js`@;0M+Y|TV|72*GH zb}t2UM@8m;r_ZsYK2q-EcX4nPv43x`hw17SVw8g-upRDnh~EC%d(Vqzqe@{lAMXWP zzn$IpYNGj=W3`v`tY&{!lI5AS%mQ3HUjFMDKJlsy#WZD*hPb z?|o>QVl>bi_&dv_lKh!n&VO2m%Z`YhN|Mf^qW%ZvdG^X(Um%_v%6hzP#?Ug7bFTH4 zbg#DVI1VR1KrW-=H#)`I$r+Ro-+jPN4>fzM6P`OF?tV;k_m`~ES5AuFgo1B5cc3w> zt!Hy5hj+@_rjpy7`8A8}8E=RAPjS^fBGVe+IZ2np?|NQy^q5ZK3xxIV!4nxJ-x!(r ziu|zFx`yZIWLb|6>-hqI;J#V12%~W1DEi=3)gs5l4l_~uQBnIKcW=j=ZHZUz67R+7 zy&uR8Z2{+DvCf5ZUr(WCfBDvTyoP3Z`+dCmCGVx9d|#ZkUK;P@NAGpA0ME$sbi&)s zO&OO`4`(!83=2Ww=G&yhuHf%|?+B$B_N(+lKo zB^-W1XFY(so|nIhS;YIr`EeTJ^XUFGJ#!1G?w+;OU&@O-#cIv6{@rwF?7%j%Bj1!% zw?pG0?B=7S=q6uxki82$^8)#~)8TXjIg2Xk&-njic|xDbM}F$(@t(AjCmyvJyXE+z z&%P*Ws|A`C?tUr0xq^1R+;7pB-ds+tvi;Ghsfwx00r6VdyO=5Cd<;+c#cxr;5IcJZ zS*}C$b4))zF^IHQ@P}ckvb8`A}~4F%%ml z{^p z6UqH}HntBdH4Pr-z)2x1t7n}>tfvatnu9fFs8<2$$*kjcnTyb8k)xXmYrlBSaqS*i z(wK@8^B_x#1TTP}dMtNKm=F6Mv}i5U8Z&=lYDwA^8*ocZ(D{L!O`)@XGOvN~z6vo_^n9+QSb;L}Om{ko9Ks`)u8>-W;jG09{V9^1;U|A3oEuvkNB z!uOJ>&U(EEaiyG356hdrzEv{hr$QJn8JNW6M>ax`zWul}Dd{yv7Sin+*T?8*(3 zyH2`dwpGL=`Ts=)G@_iJ)ZpDvp*q8uhMfL@zsX-{)MZT!eh)= zImt?6`cM;V>r7`{tHau8Gx%h=u~gT+qqEOTU-`wfe8(7nIYB!PHWnMd4sI< zSW6OR z*ILB^GBKH4jYiQ|WHScP{3F@cm%P7W`3OPQZW z-=({Azctq&!8f??eb(13ndW^hl3jXVX8nkq-FNKHS{A7>jGJ`qiD$5AG2!MLU=IjSwPvem2 zKRJ^%5BzM&QXqqe{?0zPq&(1nekzZKL9KFN2`dm2Wlv0o`5hjY;=P!p6WubuT0=}k zT5Bb1tb3jHt+D!FL{$^WXdiw-8#(gRNopN7XuGJdAYLkGWzUF+rmNW-4ze*c?h+pJ zM4UF0m)jvLGtKFXDzw0H@D)&p5z}7^SYhmYZSd@W@KH>lkJ{)$VBhQgs8WfUVn5Ka zKln`ieqt)yQtyWT{KfahIARWI4ll01Y;0>g8s}y%6`3w2nbAK})Vc@B&y0~(x=+So z45(&!eGTT>&H)at>xlN}k@%?IEk?fujD^5g5}h`~#WvcoC~fy2xHEn_1k)SAuoM>O zvC6UA7PXi$QD=qEulHIGe@kei@BDo{j189?=mxTj#6_({eG}j?4Iletd~RXC?vq#P z3cj~kv1u?kRs6cz{>PKNn>+0NHAB7V0IGwgXW^w{H0v33O4Rut1XI*oZ*#Yp0v7Xx zfAHKN@aea(GtcjFUh4|qH-SB-4lRYd)!>{@Zr>xfkJ?9T1G<>cSD(iYSDd8$6dAr9>z2dmm%(4qw>W&p?&ASZi#bPO z1%Axxi+J|C?EGR7{tS|Npjw-?upwnj{||eg@_vTeH;V~_Emd5fCHlHa);)I17s6m; zdgN&|eiRfRkmWz%Z4Iw~tDok>;EQnB7Y5Zg;Mmx2j9FPRhbkcg*hx-`qU-^b-r)b3 zOcQ%>Uy}A|zTN`kEWdxBg>aeA{D$*Z`*c)Y2jt(8GP>aS95@ zxe=>Dd(7{P;cF^eJR=(wyy^Kfd__INw}spxftRO zQnSdtV#ZblzEpQ|(o;tEZ902C{#(f_*dyv$N>)DRosWELFle@C8(j9!!ulbH$%b!G^8{u!OuN8j#RE@$A(0BK00mkSIxRBNf z$tXzk1cm?f#O>hw1AH^kZiZ)1A)T*gZR-Sb@Bxn{cEe*@({d25BQNvebRh_1hkOJW zpODS!t%ji|oqanC`H5Bi?7oq6Y2)d=Ny->IG1Ejx%gM`TuXS0iJ=q@N<7`atY=qLB z1lOixO2AzyvRsI*j|o)=toGNeR-OmWpWS^N9-QWDg3QkZufXl+=)VNVuJDTP&*;GX z2J~;zH6v-mfpSf^`|4*G{{`5c=W}~_8P#aI>&R=wg)zx0D7{+Tz7D15qx3lX`!U|a zwbpW}&o%~QoHA3CPAtxL?17Q3Al!j>PS}p{tv~aGZ$SDH2tP#g&tULNGBg(l#`!tR zvM`2~pW@x|e3(~BO!yU#vn-E*_f_xD_o?lycQta*Eo+OPgvXg|$sD*`OY3Z~JMsk` z`5x%+u#VI8ZQLJVi@}ZWK;b=QOr0FoI$rQU~wk+ zW6I|*KEIlFT7cF8=f^PphAiYO{MYAr!NcKpiqFPmvmNZ$MW`5ky#3kavGmMTp3Ypp z&oa<{!~c0h{Q4NFj{e^0q^-%mL?>rVkUU0`mxD9vVHSg6Hu|h`@9Dg*X`UYRod{2p z{Uy$CnvNfq;m4Q_y3*RV`Z+ZEk9alyK8c>6V0~lpfkJg+WAO#{ELZqCuZ<%BeToFwhAOmycT)h z5>E|HJRPxp@<(NWTZZtMt;S9-O9^Je6yK1j=v?NX#Ce2C~ui;l1iOY@IsMxXM!taJpt ztKn@m*w^_vw9Y!vFM!WAXc2P;qqcu1k2EIu9k=S}Lru`VMcgmWY&fho>kz*zPC7`5 z7Zbi$B^CL2vJsue`8P3L^(@vedcfi< zFLyow%Q3n2Pwzzs`d@yEe#Gc|`Wt*(y>rn^K)TWL?>Df$vM%{7N3h4G5p2oMoQE2F@>_A zJ+C+!D-W)sV|SOQr~Mqcq43Q!*;S-Ld6;w@W3M7^jA@Z^wo}YcjLD2A`Z(iMtK#kv z9zyi7otU>6b0DKPE2iZ}cUR22jNDIj8OCn~{2mi&|9{t8e2=_9&^!6Kyb#)|!rcI`Lo^($z=ln7^~0otc`b*R({Z3BVQKv$1!R75dQqleNW8KjhHjKNg}=p9lq0F z){?=fsEdx;m@0gdYhqS#^wdTlTS<5*gQC&Fe-v-Tl-c05m?Bpv=c8p z<}X9|7ca8_1?b}*WbG^d;s#&yz%-kLZO!gn{O9{TUhWNSP*wDd`P}>9c`>Pb)o#(X zGR`-<*C!wdJMjm-ygkdUWl^*~-#vCjI?$=bP$Y6wC$#leGBHseG<@$HKy;1!^z!O1 zQ+kD1riEP675@5-o!WcI_Pf451lKAuzQumje|EXjc36Jr`AlG|V<$X(vQpMQ&z;AD zZUjH-Zqj>??=SEI=7TPUBg>+1&8+=z1+wn!=Pmwkg$t_V^_1+!Ds=pWm-8YT4i@3Z zDG)8anyTrj zf9zN<#)nH2sL?{Q6X}_2(uduKIo^L zSo)aMavn_|ba(@&4aoYjLwMYs#8)k0vNbNeOvK;N_ZHr15Au%gy;dd3TVjLtJnxP0 zmkX{mjh<9}af;oq!pT3#OkdI*9bt#~OCO=nQ1`zRAGUGN#waxatv}+=tfViBlkQ>s z&yVnTe-bv_F6c9IMwip&aT;Wr>`WyEgJAzoywYE;f!OF4|hY@l!B&HTW*UH+AF+ zFC~dj+`2U~`OjeU?tY&gxQ+TXz=j$?BtUkKzRG;uKDL!mEQ#BP?&CN`^P=2@uIQRJb zb@(wnnf>(aTWEW=^;MM*tbi_+vKgbt$kKei;zsfI4OaD~nDS@)y6=k4Uxmw8WQy9# z8=OJ^Zh`avfw3)2wsxn+Svt2u**a)_wtc>${D#x)jn{;Q9`rzeGP4IS?xzP!sak32 z6yAI6UN;151NdA46YbcyBINQva{VNlRJKoas$I@P;H>J--T4n6vkA+1f#>qxXC{8N z;(&3Yh-eawz9~lUh_5Ts4vVelVOp@YJ2k_5XQRb=Fxb>?c12m{VyYU7c(;aX$TDC& ztXHL=xtq!44%)k<9qha8T?|v3cd@F3YyIU@8aU3qEKV->;>Hf{v)3+mF%a%gC#?5E z`sZ!&(qx$YPk-rH>n*I7VX^!{ob$V!mi6sO3vRBDk|j(n+hvz037#|UzHUt~Ry*^PnulW`zm!+=d!n&Q^^=l^S-w-$-W)a= z_pqAJ7KT%l_B9a~^%r5C!UB%P9nK&!4{f`uz=G=8qK5phbWJgP%qodhtO&+Cj)+f$P7EV2tx-L~g{q9Zd%q8a9l};`| zsV=#TZTJ7scqJ-u4ojyoEt19*(XV%O@omJ^O^u}I6UYD64 zy5Gdkf0J`lkE#3q%_O*!RdByTs@H?Bt7s|mzAf#H{EHTGs>~qJ^;567U0qX8G#kS5 z?qvstk>mp8?S3btH>16hEWx$9AlBl}=(7%apTja;KxS8iW0@+7k!lFu*2BjV<#2nWB4sA=#tU+x_~Kn zzp8AxMNL^p=Nx?nmql@UHCg_tqJ_SEjv4BcZh^nfxezQ zG;uW?{4IG~qbhSKi29lY+tNJVt5t2ko`todnx_iMKiT_V)igIumNT#QS=FZd@!_p1 zJkzRYO5wnPv`kFW8%aN%q(1Fa71}*ijohaqaJp6gL0(#bex1tKhe_HV+Tbj_(asrb z9i8>p0e^4Af4#uhRGjb;tJIKw`OUQ1xhV0J`qeJ#FRD1FtBb15K33j4-NTgK{K;!+ zg^hH=05illf$itysm`%~OPy|s)L1^wS~;EQ)7r>4zYJ{`s_XCJ>H$_b5GRlKPGJ`C zEgJO-`Kf34N@paSs%tq1T=(L`T54LB;k{>Q>*nfnpH&yo#m>}z5-&~>o~-_|xpVzifG{^T9N*L@8_mUwCG-hiNj4U% z@*j+MpH^?uO5M$i=3bXz`(70X6lQM+kgnQ2zc~vhg~v zumr90l}gTmXqg{m`O=lutDd8Vr*--ZeR;KYDz+r|#Z1K9m^#K@{E&QCZFf<1_{mf) zzdxk9suc^@Q9Wfhx^p#HqCWL?a5Yq6J%Lo51F|P%H)D#yld$s%x}K@3s6Cy3h=mWm zzl7)23LXxM&|YJ2(zM;{H$*8a^akS?iT;vci-KTjXjKRJW<8C7vx6GxJLO{TwD z*G9M0ax>JQQb*fgmGQ&Lk-Xvi!Sw^X&|1=GH}2X^zAh(MGrcyZPlL;D{QfwzIIU)5 zk(~H`x^oE*E3O7DPdxF0yPS>N&ebJ&74IoSpS`7uy@IZq@@84)q<80CobDh78^V9v ztvmDCM1OVpeQ32=bnsRa%Ui25zE{W7d*uI16_-Er(tqaP_2#2gbI*0AB;TW!c40a% zeI?%`&f|NTzf={?BQO3Yc)sVYEOG4~R^wUw(|3tY&vKW0;5@AKEL{)gfWuQ8uZz&& zDS7Nl>|8tfjjs|T@co*^&B>!GSSKWFq2*w`P`}Ym`^dy!;)fdIx5+emUpP6amT3xo zzLJG4NrTp56Wh@%%d9iz%G4EWyv;g%tFLPTZtq8SZWS9;LctF__X-yIC$&ps=&>HK zb&ZaUFVfv`K{KB8I-Q>b`45$o|M2RDf@35Nw%Q#1H;!Bu0F@+@F$m9Cg znkISuLT}w@8hQnuS41#hvN*-Xi0jz5liAR=u+q=@00Y#CzfSYl=Wmu``6uypZe~BD z`*aQur<1(Xk@Tsse2zNFvVOaV_Y{*Xm$OgNMc76Z(GUgAy_MyxEuyF*zTE6H_wX!h zxhgz^J#^O;I%0SFWD(y)5&2|(U}Jl)bMWJjY{*q4?R@a`=VA7u2hUAr*z~kLho`N& zsLr|4`gI4{e+-_g?|VBkbQ6E0i@4=OQOQ(3|8B8&VLnP@Fg2qk;tUz*LtFhQ*WSzT zJD)Gvz&aY>!Th|`M?`32Q6R2uMVg-G(Jv5TZ|AENXUqSl?bAGhJ@o4?wrMme?k%@} z70Ts#+RxVV3>wWQulImvBy5lJ+XwuN$>&1p!a@~gK32~u7=r|x^0#VCp(#>%;)VafsJLbHOb_QfB4@oi0tRl;5BqPj}t=&i$VOtN1#H+Ua@Z$}NlJ(yBQH{$-+7r`gQMB$ zdU2ZYNjT(9m@dMjKSj(R`^H1;c|OKtkH~c&7+33LXv5k>ElD$zGP+pnBXZY|+lv|C zGcU?A{bNN{>CfY`a}WFU<6istTS@#k2DS(Io}V_$z~Ehc{J?v8x~-p>ZnJFcFy3fi z+|d&~bNp05XMOMNERt#9iu09+iOd%2uFa*HX6g2fc`7S>?UeVujGegD)PzPRXk6w_ zaWZ|J27gHO_MS+8J`LJk&hS&~zRA4rHF)r#X~2inBYkNeP%-Z|GEd>w#927?Wu8fN zaQ7D}mCnM_Ht~+rsy~<2%VXhg1Bo*?MM#TOg{et_jnCPT;bus6aqm>(W*N1cVvYdV zV^ZOLwC7y4xkuTdT#-RTbYN-ByO@B>RAD5)!e{%)a23-I=9-Z4v`#lZjqa;Xu+rIT zb6CUlw4ae=Gel z-ZY3UB98-b`Ha~Cy~*Y4G)GHbQ7zP(4udb#$Q@YKpFz^gJxb~x`b(}ZCev2ItsTT6 z`&{u7`d7E^8hEZ695;}yz1Sp_m3*fIv}7N=Rn@$Nak%Vh_nCydr>LSf+ZlyvB-*~x0-7V*N1!tyU<@@w?~b@kia2vfI;_=>p8B=_Eo@;~A|y%FYb+(kx~ zCi=lxD;DQ2b3W$4Vo4P{r@B*A+J0`%%Vs`AZ(jvL))8OV150EXPw}iX$;Jfyw-d)D ztYA9)?2rwbgUeT-=jr(6J9HXL_x?iG+mWYS5?Ca0yD2CCd0JE_UE=x+(K04hKIi|< zF!~HGs_nCb{4K#seeby!!0Jc3NuNZCvY=RKuE%oHbhC)6SLVrtlP6sJb7pnowRDN( zY&<-LUi>Uk-V~o#)fvP@xz34w>3Q&f2irV5^S$XJl~|BzF+Brau0-i_u-7}$GQ*0% zYs{4>On=1mNO6^~i+%DsT-Om7?N;ZrG&94Trk>V(Hs1Umq}yQQVSJU;GxZM67;m@Y z)y(~-mVIOJXJ?;^IaBw7dmU}_SXP%U;=A9J=x+Y}J9w)HJu{fM^$rb`z@4#!H8}H} z)zzdc5A*bA(Fv!Ko*HI$d}-EG4O(|Od>(`EUMA0cXy(zIR`Hl=H8IPuBHVuAQ)PGn zg>lAD-aUrn_K?Z5Jo9{idCKZagKZ)0@f*HB7l-em^VY)l?{Ig-w3Scs>T$o1#Wfo} z=^mI_LIYgH({4>e+)h6~WSU9=lJrUD#YAg))|n(@3>_#kaSp|LYnkc}^FXtL3@pLt zm*MLJG};51nIy8D-!{+@pOBjmt@0~#2QP=`7jRANP)0}Tg)lu9kHnexTV>eO>ATSF zX54oiG`nP=hmom!-1Q=!U|&zb886!B%OSeK}wEEwrwnuQXToEDt7sBX{4!QA^&^ z-Sp`1cs(QES{L04Kprhs@vX!QZaCQDsU@0lG5t8-Jq)-Evlzk}+qtmeqDE2~pB}{f6=Cd5R4T=@JA#v9;>k|(|DY9JL~6%bpeYB5Hop?tS)JX1Q&6u8%7%Zn-j3RD^x1wDlreYxW**k5?tE(2W_3?g zB4G>MYdGwe#>sJ#VuB~K3ywR(>u@s~t`UjXBX3RFt<8L=i&)J1?Aqx#b1n_O#QndZ z&%Qv}R-SQ3Hn)12&ju9F`P{GY)r$pvA?wpuW-l(rEoWPkd6;be%QW{&966HZh??vp zJn-A$=r=y+Tr##0&i1oxJ!t3Vc(*-%$;UH|?#W#6b^}QRyt$f2h+2znzTV`gA2Z`< zi*;YjW2=PUL!%|&F={8LSWPDwUqHrNfvW(E|0~LucF!5S=KT0<59!>2=gxq)I-XI2 zuFsH_wXUCR4M$koZM5s@C~_1;n>{zqmiv`fs_Ka)`3EsazH;{dwb|YMsJz^j1=)*I z_@o@&8kOjIG{uGN$3Iy)EeQ*!(#lb%5_{9j>F8?o;b|mz7d|}=G_hCkFR3d{&+Q`X z2k6c+C|=AvapLL$?`=ZUPP9tYPQ(eBwbAZ0f614<+fMuw^K19P$pOzQ=DNdt*aP%x!rDQPuY_`$=))Rjg#luwKz|Xdx`z4VWO>S{E6TBsQ;rCF3QH>wcNS19j~hL zs@I7bAEKcv(mGYaR1H7g2lCluMoeJ$XP|6BOE^7QhGzo}bf2#-?(>ff(QJA4vYF*_ zs2`^<5iJ~oje+F8foC5Q^}Z-Rzd-)FjVxe0xvVC3r-~=1%J1xz`MnLsqqe*Xihml$DD!SkBU)-PF!bK^f+4?8;Z@a2MXskB8pFGb?2T@{36t(A2TtR!*+D zC%XQRZ8>Ho7vP=iC>66chS&=`Ti&*Ha;X?-Gl*}*l`;45b2@FjOnGab zc~^JtECP$(wBe}L56^7N#^5m-zEE~2;%u5VP66M^vcH?9bXEM4XGdVFoc0pB=?yu? z6*NS1IqhP}2gPjZ#3jkyiImK7@#OwQefx27hRHJfR6W>*`s~hhPnl_->MESyRvx(x zSQ?_#m);F}93y=t@k}FjrxU)8KC#PiQshozzi^98!aP=@1zszj*=#pvolMSXne5xi z?(4X3CVhJv%X}DJo|5ruV-L89J%A?mT07gd_}32R8eH_atVvH=XesXc*1l?2U!CpZ z_Ts%ZB$H3b47@E{@dYcG!11L~xF+2j-CWg3+qtk>kT<=6rM}CK+G)C*|CAGt^FA-3 zPbb*({hZGIgSGn#_dkMrtJ^*QL+43dJN=iT*eS_5;`n*^;9hI3Nu$R6tf$;>pp{%K z1J}dXjh^s?s6OU~ub{CqaDO&giYl3wu;0ke%Qa*lYL!G7RK-m zV=7yIobfu#Jd!-c31Fr0ZaEM&$3=x@N%QRXj#eko*ZrchW{oSOSF8hQt_Mq}Y^}?4 z;ECA%Q=U|sevh+&E(TBZJ=R63nA3eS*;-EnKSw)Ol?RMdu4ci|s&r#GABoGuo)*S; z!#wSO;EDd|+A@=6J+=Z8;cef?}yj`~HRk^jVNc~nMa z2;N?%rz_WQ$GNrWr*oMh^s94xO%pxB2a2h)LOHaiOzp?z2aZYr%_M#(u;MAy;dV~Ca ziyPu(?x+}za}mx2Re2bSezG{zJx)E^XH}EPR)07StG^TdR)Xzx+U;SM^gq#1Gguk{ zrf1on*>Dl!ribr@Ke*!wW3hyF7>J zZrXyMqUy8+9T_!oitD51sgw$gM(qAT7>haJ z=d+5B(~KYDt+jISYe2Hie#!euYCH;-9o)cgo$)}UncY^>y|E(X<1TI^eUYva)RX<68=q&>3GTfi3k77x-4y-4D{ zetL>#{gSlSDY~x71Ie^b>dX6=;V#=*Au5^ZS?dn=(*^mYyoTM!de;MuXMj> zVe~ICN(Q1fRzwjE4yGEol76`-x6MmBo0%ewZotd|?r!Tb3BvvELJ;ofXkQ{LhE{KF^ed6+e`s#d^xUjioPs0^c$?ixYXL`F$cAIg0l?gr9H++Y=tqC3HYkMuyFQfgJP` z75AZK@56oL$?RhKE6#&EQOz3|i&M$s!Uv~4dOVtC8!+dT^ykGmay96P~cV`XA=}X zk2hZ%jr-t{kLbnXDAk?L`WSDmv_JX>_%@M>nV_2ly1}seps4q1YiPv+)X7TfpZuZE z@yp{l_h#!3X?-kP>#~UMPou|~r1Ckxt)owBf)kzuIylkDlG9i^izIz zcr~6F&kMMNm(vjscV|}~CV4YR=|L6r)p5%e_@FJHqNOKP18p(b3%H^u^;dUT2+z~p zWt26>B#KMukeKgT-2Q3o<-S6;&Tx-Gtmba9Wkr5PCAOfX`lYpG;$u|3TLn`WPY6${ zdR9Xnb@$jye;RCE-KVitw*}>3nq(19F9l;K`sKUxrP{gnV)!eE(_&uuKdz3h*u^aW zJi6p{nl@?(o1$q{t)<1M#njT=#lCEi+vM)E&Q3Nx)?9413n#0V~^l>_h+M9gZIRVjoWG7 zg`$wiMtww6J|il>p3hQ)Kb=zBvx_ZFklX7>$_m+)es(^OtAFb1T+UHWBkv=^xBz`O zcz-aOM~8j|T%EK(v&D+PrqP~Z7uxc!&J*#TDmJ{Bm4C`k>n0vU?5vK4^{IFxbmV=c zsRilCCwIJ(L{I1KjAMykc9$X97@?*6oTk3-PkY*n?Xv%uKA)UTz#-k$3d9VZL1w_s zQ-Sb_xo3CUS3gACy+T@=h^V4NB4j^$3*W(OH)nUqOYmO;{hV1jl z_;;@;{2<<4NxMIkt%t~AujawkXp%FOB;AXex8l|-#S2wc932!fuHmuG5XB}@d9eGP zYo|7myju0auWB`lrA|(*HqGi@5$x~m*)4QOF}aNOH0EUX=@~Y(r@Nh~y4a@nZd>|z zx)P}vfy>Y0iNEk#f9cHoTF%3WIZL;KtBaLh zPE%GPSw(m^`EhY2{?}dPF3#Jyhm;@WBe%r8J$R#KU#1fwebJN zEPN}_UGDlu`0O;|9 zHcefucIgAPJ!|+#xw5bKs$V>n7Amjup@}9eulpe{VsYg?vrRJJ-IWTnqmoK96 zM(`eXll$Mud~x`Dot}?ckI!W>LXs=tmG+>XN7jb$4{NK}d`VT({bZ!3n#!x`%$TFz z*?kJ*m8dHZiyHC%5We(haJPVT#VKIXr508F3uvD!)GKW_arh=vODClp@f(_`j-2TP z@ZVC&oWePCQ};Nly=L-$yONh@)~Lh#Rt8`xdvgb#nJmhB-d%2khpO-q^Do!SxsPYF zJD3>xmHCJRX_?=7XgBboM&Q)GELQ`ct&B6;y6#Fat82x5zwlfNh=f}EYZk*necdClB;qrpT>DaMex3IX)y_&$s8?Z1?%~Jp5R<)HC zQ-{^)^-NA@)jlCxm*BiT__#h@`WQ(VjEf_?bGM&frymd7WqOe3JC3y<55`hPGP}(8dObBQr+>b2`L4^i2DRIynk6f#zj#?~R#J!9 z*SzL4VIgvSwNPxJTDG-zD<>u@s_^~Cgw&=gxW882_$PdfqAz!dp;ySe{+n({rawT# zuUO)*d1q1i{+AuEru>M{$o)7Ta8!o9jUPTV=XQPGAg7`?$ZMR}Jnyr-h3RUEuF0~g zi*jzrS0%S=Zcc8ueC=}jq{=2$hWc!!ylKY^=k+xcvrzhhbb%~KzDk$PM36lv2NrEYVa$sLF1=H z4~=B?#?no@$Z?tUm*xxiaANuZ)4AWyYm@#Wb4KzKbEoU%E0;St_v75sx#^sbQjaGO zTG2asi;h<~e){pr$D8Ne85h(x2O>}8TxMrHIum6&x?ol_U4X6Ym7OJ6n2 zj5f73OYK+Ew;L zk0-9CmBz7)m6FqVvgh#2x5zBarG3W1;c)uqQ2NL8o9c}FsJ5(Af!f zOr~aXdTLb8EBW5dt)2h;{Czb~6;2IGe3U*tFZcLU$I2dSa;)j`0eK_R?L`ug>s~6) zd-;o(^>?Bkd0xUZDy^H~1KQ_3`)9wJYyER}TK6)P`YrvLs-#PGmApst2g?TTLgC^n z61LEVU(u2OBU5eJ{d_dtdn%5j+I0h7o+)2G&|J@|=>vHy@`k6K{+7Hnr)IvsxmEJt zoPS>K$b3UnZzeXUOXcMqyYEhm(Smx13jd%mA%yj);eyB3N)`J~-j;C-x z9kLOR)>6m4GVeQ++)qpYFEcgyd`^peXXYO8YMNUm-<_#SiD~JF^4>pQ_xSC{#~ja} zcOY*|dK4a?$i7_2EBabJ@O_|c#Xj{X+ZQC;sgtRcd4m=?TV4H2>2vA+nPRK+NZPx6 z-70#BYP08clmCJ94A#7l7$x>XW`Xh`U3DM-?J<79FjwBdhx=Sj@cHSa%IQy6?gO+|ADnt6kFKtqdrIL_hD*UE9^c!?bj7j=%Y@ zdfz|ODSqeU{IG}N=POay3o@YPWo0|)D%d1nR#6q-gKTUk8Q2#5wzjNj_|X^eLF(}v zF7dt}u(#WQa5N|aGX;JzpN#1Lr*(iUzg`YT0KSdR16_oIQ9s6aY z6K|)!&pDnmI^U6eo$__gxjWT5xmFzdg}#C7RMrn+^Yf?w$}6Gkq#EnpFVRNL`WErV zK2cZ&9{0($%+~7=@xlO5 z&SqKmtDGJIpFi+xJMtn=$8)#f&pjfB7v!KW;XB@f+ONyBO%mxwF7`Sx=rQ@1hqBqQ zOT~y^r|+fFhw5p#n(r{y(|$Hb{wiM7*$Mk-PCWTLRV3%koX^FwU&{0kbe9DGtC(5# zSLt22hXj8w^D>yW?umxq*a6tYQnvBg*gf4UBN6j6)s&Ie*n3my#2bH|1{=p#9_8cJ z%;sVm(7fHn9cRcyE#vPGv#L7c$30G_mX{-r`>wh-;oQU-U2PlkOslD0Nwj~4ME;*_DM_`uO6ej{4mg7@A5(H!=By*&HdUQ^Ne1)j!>bX9M< zA$DQA(q6BV%%9lfnBep_`|&b+oL_&>L6PlJ9>lZRJZu*}RTEi(8&nysRtFQi0DHu3 zE#-m_WIeZ}EJjPfaxRj`B|oG4m=#<=QT!?4`hypfiw1Sd&+e7^K~2g4a3i||9C0Ry1+XF zTAksE|CR_=`FR;aigbP3a7oq9N*7C$v9c<7kR*#(Hr)oyj_ZE=z3T! zSaFq)agJ@Aj+-B^#(8h+RD#x4+xdqRCC^G6P3wZS+UEFi5E$;3$G98C9w!gF!qq(P zbk=Y~^`p-@E3=PE@TMw7$EY3Zg7Tx_{|nXr*Qsipt$K8%toE+-Ty^YSRm+Z4Ki*Gm zP*;`HZznqPp2o0pZCyE41bRQ~SdaZ(MMuw+<+?;2YY{S5T)k>7yGUEf!VFc5Q`9G} zO%F;OOI#~+d^j`NXL^fi_sNS7;9<^3$qM$_W|EG|bWWAz#KhM0bSHUS&NKQNkL0@d zKDaxkBjA8_l#>lE zL4GPPyQBM;r?=xN+=~adB$~;o-9(aa!$0%YWX3tDAIhLKQ!`ahHR#i5_N!|8iJo&c zNk5YQTF*v!8E=bjrs;D1k>53v7ceKYRNk?!XWWW!@58eN&@yb@=Om^fo~e%7zlyAn zXS#~lB1?A`O;=nd^?&5=EgpRxTE81!i4*J927tIIyiMWd?Si}Jr2aax`LOQHYZ689 z&5QW`V_J1hc270B1eVzmO^Nf~^HV!~zLo~4&l-10R8Q7P{F#1CJ#P;k4HHz`HdROb z4qkdrt?Kdg7-t3z=6%`4Np8%XhyH&hcV`BtYi2sC+4#i1K`TDcM0#uznJdWO|DE-k zs89I)#FXSu;+!LyN=avhp!)-8w9xm5NS>23Y3_gY9`;qkJr^ZTqjg8KNqfNcDhypu z7E7wn03W;94)%_r#IN#f)9AyLyvg~AuCk&R%TIhFvZ}(DsLv8y>O9|z>8P)1n|jvT zmX#_@{>qV#I=HAh?QtY?G(D8OJ+BYvGIF(B-Csp=S~1xvaVyCh15?9g^Gd2yImP*^ zyG+L~;T@-6+i9pt3+L%=n8VjDZfF1M#7S0lFP*=_&eK5|vK{bOmDYG&Uqe~hu_kiI zZ@H=*$!)6Y_Y&E+_sQ&??9xJTZc5kl?5S2hoPY2$9WdFm$H_Og^3D`GaWT&B0VAF1 z-J8kkv;2>%aCj${bq=n4U)<)rV_r;A_?b%zmf(P^RkS_JZyhKP@FZwHrju@$k8+ZQ z-Oe6PGY&}~1*T>+R(|p}T)uICx{VWMC-Z>E+wpD3`u>BHM$iOLWET6aCp}=UD0z+< z+#l$1c~k!C4$^oJ&66K237O0MDAABS&%^t-!9p)jeb^q-oNO-aCpa3%f^{ZAFY|KN z&<9)S@40M7)I8?nyZ%FC?Z}M6w~Oeo=vMreM^-@$@hM-Yxcud6+FbPjEN98f!0Qbt z{(|goRW%@k_3red`~C*k^L(B{EcDA{{YR1E0M@c^bZL110`w|?Up~!R@qBpqZxR`^ zb0hDc@yX_Vk(lSY%Bou1lYfxZccM#vqpfmrOH3E9ru%P#I@Bjf@}I2RA(T1?PnE&* z&2)#nqv~3;!P~kFca6s5Q3duc>^(%I7bhbZpm@xwjd|!XIW|Z3`V;oH5xV7g)~6zq z3t5_YF5fuJ*FFC3ZJ1YPvk6oQ5H9^ z_NkZ;R)+3gX@${!yC%Dzk+9BFz+4+0qb zZqCoG>FIg=w14c(UB!o4Dk9uKs@sv9A}~>#6m5pz?{Ppaca46!)^zkQ^m0_DRpX5{ zg2RzCdCUu4!$UcXeJRDdZN*E`8xa+UH9!?L6=}Of|F5Pyfw!^h{s4Z@xyY31n&(+% zNQMxZ$D}BvLW4@wixMFkOi5H!QZ&4kG%L-KCM3-$sZhq0d7kdM|L;2Y|2m)R4(B}2 zv-a9+{noJ8Ub~s6#u{*VYh~$E1G5unS!KYz7Ivjjq_Q73#h zyu9d+y9IXEA>Uc%E@gc;(=|0@4E`VyEm+3r)w!1zbTyVLe%j?3(Z8@1i?@R;RfS`* z^Eh_Eb#mY6liJud|F~P!-S4D{Cs145JJFSLBimG;MpYw&wLCZOy_;|H1I`!n)1T<9 zM%O+gj|F*7Q5}>+E=##@^c*bh_cdldy0IQ%=VJ|gy;m`uiEeXe;n5Z`%od}KT?d75 z?NZVaHIduh{ZF!V#Lpo$6;V(bMn)~r35a(Qx-Kx^g`xfHq##zR3i(@>tB;xKIE5kl zidQt-htW{d-%q8VbJ93Y!Gt8Oh)5^;^5wBd@fTdFZg%4&C(ciRC9II z@Wh$ki{V3Y^H$qc+ezZro)#zRp6gykajL9&K17}m;zb32iC)Ojr}KzWWTGWHX&x|! zsAGyV0P^5hR+@Ja+a&xH9q#`1bAA-Y4vK>A80%cIgJY*zJ#1E^&aMzSKI|@0;Zq`A zM^hOW4%7bV!nw_Ovd|R$<)c5~0k4B3Wxugk_4!O3$n8QHG~|UcCCp0H5XPAw$9y(U ze#q-n(fzWtcal8xs1Dof>nOcC?x!O@5wA?+j9pLB?=pUlK9q&%M4Su}Ra+-q@sC&B zF-||o^w)!4(bqDs>rUWBUZadomr+L;H8u{3MNRa~END(LT^FB?vxlQY_W{p7k)}K^ zJ{)oH=%O1>D&U#X@3OcnqEF^A&&c-wad%9m>CW9>8vRnEZ(iIddUytxb9*r+AyaNm z^Eq%ZnU<+I3+13s#%SVmh0qi|IZvmb5cO0AMQyRmBS+1 z#%H4v?SI$C-lyFAMeop{Ke|rF`5^Ix7)kuLhfMwL|LAPHH_elye%?z`;{A|@=sOy; zMBn?8WFby*j+(8r=u3TCoZCx1_6VQjZ#BJB)%$TOK^gN<63w}NOZStUZLlc5{_+)h zk`1&fPK#b93Vg%sW%wUEGoFC3(RF!>%)fw*z@N-^5jRW9W*vCeM!w7Q9w zs>NH#I_6DmQ2{oah3^d8s+f;s#r`E}ZXNqJ#g?f`!)=YLrUI(l5cj$t0maafE<>P-B zt8{}S`Jwm<#6~_ouG| z{`#Wzi!q|!Ga=laa=T}WL_V`x{hjrSJiOwl0(}SiME)WACPW|ek9Zsp@;`1uWjoJp zETjIEx$7;uJe4uRsc2iAHTYNJ9cvA7BHDEJhL_KluJIMS#G;~hf^1YhImLt4@3$I% zb@|Yzpk8zS$0t0Vigqrvuu76l?6#)%B)_OLEUrOIeis>k>9eSGmTg0p5zJD0I#Vue`ggsTUk zW3(*K1h1#b!$YFDCe}6nR%5VIcCe(#>v4A(CXOtp;$ed7q6b8X^;GRIKl1rrvXI-YZ@(#rxhZuup7+O# zD-#Jld2UILaK7g$D_nIGFY`qH<>B4Tm%rESk0rs+d5zWf|5kp#3sYxIu1TBQ3RPiZB&PRSU= z99Zs=OIqH zze?;m0SE52mT;l9r)(>X8-YOzq^<|K zDdkmG=DdN@IBs`$U7wZnn9(^SM4wO6(1#kIJ7+CJ|ERw*wu z`%_SR8>-8Z)`c>Bkq!GV-@JiLdmgn_`FyWpym9*Lo2*D7RBhu zlL;>>SADaXaTO1t0gtpViNB4^T#t(`7_4{&ol|7nA3=2qvfouSTus-X zk;&JR%acDiVY+?t2fcR|%U8Wh6Ar+*af~sR}_Xjv_P1cw~v)9w? z_h9sqR9RYf4-7ku+UrGmlSEvzeZSUy;slOmY|%yLbT=-&iq@IdF&ix+$w+lDjkU!se-3W z68{z-T*2_o2u0pZ^x1zN$Sty3Xr&1MEAB>X)Uk3e)i~Met^M71~^Z?5- zl!u=oww+?;idbKJ$rH!1KjW=ky^XggQk{8;lT;jZv8p%ES=L9B`7{4~=gVbyRr+{!P{bR+Hhk(T6T>7yU_bW(aCqo!Vh?MOr`tLxw3Y#(vT&#>z} zE3QRJ?lv~GC^Q&rR(?Ti)M`&v)f?G@@g$;`yX|7%CqbH$p8Fd8n(Xx73Fh|s)Cw!M z<)D3Al?ToBw7XMvN*%9Cncu22nZx2&m5tk?t6gcGxGFfyb%0$|#Yxq@IJ=I&ke^Ps zH;0qtNOzO}i`46OVXey~R*Tt7$tpZTqdKTJImp`ew3DHqYvZKCf7AK7x={UEtDbtm z(6g)HLJi}4h;|px`CT>FIMp9RbcCBget%VgTOQY!=wP7Mz>4XVWckb%nYA*{$efof zn7lTzKs?(t=V=vIo2_7$!rfu1^{GyLoRHX(@G)ww2gsE?Z4Gm~%C6SzZ4LW=|0Ih| zMW>INhl^#Hj>43dq#`Qnm+|5Xi%OcgwksNM!PzlnxHH79COfhKQr3Y9;fajaiR(6f z`yPV|o6Sr;Yfo$Bi_Ue{=JCYhWarE)GMi>*X3lbsciqGQS&zM>c85BQ1KAbSP|Z!9 z2@gLLjU;(lu?O)Y7&Hraf6It{R~NFmIV$j9z_-}n+Xlv#he);f0@v|ccAJyD^hu6f z#Mc!Pw1W-T@`o;iz0s3;y$sG3>i)K|>~C2!yc2JqQWx+GdsG|l-^zNNWtZ3Oi6tsK zb|){+JU_Ffli^DxuS`5ArZ13lkNSXRYBFkC(|*><=191ohd)}^)4GcMhsj@kCi9y| zU3oJrvin(`!J?`q_*B!p);7CQiTXS&sKOtJlkVcwftq++i)YXY<=5eJYhLkI+@E5U zIp}+ru06)`K1dc;z`>xm1DTr6QmZXU>_`kF50f$nX4cGnKY2#7a-v7Zq?~V@iT=A9 zvF+KNb)MWHxBU#iH~}Hg@w`^nR%Tl{-lwji5$WyBS6ps}%gJ%v45d#vS@cAzJ6ju> zgDmnnoNl&FN5~a*(7T$v+b(jDCc3ts6T_`n&7|t@|~V#cx<7hqYI1SzZJjt7~)k zuw_u&7qxXD;8QZT-@$}BJlVWZtOiT{me{tlyhnRq*FlX|yqG%Vzq6m)qPC70?f?t% zG%u*5>Wiu@a50(ezxk!VK+vzzyOB>`mo`LCkV?GrwTY(5=aOYI+meUhk^_>x6T|6h zYZYi0>uOj+#m1-V1-jCh*F+p^QZncY-=F6HZdCvLqY=S>|oOR9+INt*JL$=AgbE9H!0{jH)5 z&q!DCS!(Go@Yz_QeuSib!amhvl~%G8wbfN#56jlG66ac-dQx;Zh))|k zARe~1|Gh}8BJRfd3yo0Q0lh=f+tKR5HGJ|TP%cgbeTjZdAVEQOTV7RB7IC*#`?b(F zs)ZYRFDgk#``V!!&`pVC@&#Pp%13)C*)Ul>(O&MdXwJ*3nZFU)9b@@>lg>LuX0bzQ z1~00Y75;y%0ktI)5qz%HmL9E73VeQWo6=Iwhu_rcyzTQRaM||!JM7qm$f38 zqNJ=_I(}+{>IrP@PZ^7J-CUF0lYC!2Q^n*2zCh>b7$avs+_<|Ncj25L;Q3cj<{Wdg z$!g`})?s4D(I&R;0djd8&5QnlJz0kgNd6pOHqMM5B9|Nc@9W5yM6Ra{X=p$fdytG# zuDOmrrqbT>ed3BqWO^9Mh;_W(tn24+c^QmshJz21wCF;8o6OA0jKzuTl7A-uO3qGp zOP=6!eQ4%3lA3$T?`#rr1^xY)OwWNg)!p?sx%6SyaelM2@m>DFC|33n^$n|dnU9H7 zOIZVmy6^$G7U#b0O6RdQs6?16bMTKG?xj4niRAx2vVA^;+r{5}haA03$LG<+`XYwd z;k?4v52U&?-$WpEZ}%u~<)bMixVW4qOZRY*h!{u|9&e!1j03+iS%yqqo$ z;71)y4UrotOcKvG!yQS)F!a7>XXYPb`J=4#BD~AAo-zktVlQ3f19y_}8^tLN*y~qh zf*L{S-bUBkJ!(Pt$Sg(&pX$bYgg)Gi)0@;DE{w}UM9dev_W^p7Yn2L0P^kK7%}Ys_FH{C-MwR*eNd zEiyZo-Hh|WW|Dxr$jU;q--Ors2~TB%9bF4W%cI%t+x$P-&o@Je98||T<+-HwD%ZV9 zN|vPMVV*hoK^9R3yPQ(24%GGOAED&^toI-YcQyTMr%o*|?{_T=Q;CeVu_An(I`P@+ zE{iAD^3Q))E!vP}Ym_($5jsMok8x*U+B!Aksa{9QN0ZpTFuV_KkNq)&Rb?$h_e(td z@$hUqJ&h`!*sXpAY+h@OSNeQ(@tBF)4@g~{o;%uVN_6jx)sW6c*O08FWb(ft-P6f- zoI}@#PIY5bvvQ8Wfs*2_GHh87eP16(bK%5Sp7J@#ti~@3&taAsi@tWRq><=6_V`?g za{~@fMEMZ+98Yr+tnefB@&kU{!}c@Y;cJ?|%oGg`hbB#7+8Q*(nT)aHR!2V8W|di< zYo_L_3dy8Fmv}|J(FxcTRYZ?Mwi{?)H*(U9Ojm}!yG7xxygNj!H(!nR!gL>xHFRo| zxVVliWd6k4xRk}VJ|hp*6JD2wB6ZUhWn-XIoVh-h?K>)a@VIr-&8pMif*zCL!W5p< zbTnTB+3Jh>)|0S-{D||}hp}|x+jOSy6;wZg>xZm7^yWGDGQJ{^~XgBG}N^{A{udaJ%*HH&8u|EZh5E%OwV`G8L`4a#@>@%0Mv#kID#H(m zP`8Mr{$hcwTOFA#GZY)0e^KBx`=249iyXH66tR;6OtjUG%T`kQB$Rvv53ZDf4!oTP#_pr@@HW$&vQS^GJ>|iH6xrqc0z~#$n;icq1P6I2fPUHprY@SBB zmAJJWuh#RNj`EyzyQ8DE^*vaWxH|c|J!lstx7&kKCh?aFmD%pum{z<9;~J5y4@G4+ zsoH)aRft~xVlB2gzI5bG-z?T%2PZ2fs?pwUdK64ULl(dG$uu_J z`!}RzHtv<uF=MDqvcW|7+W>rtI)6(D+czsw65 zoz}!cqL#y?d572TREEq`bJ%?+59M&q7&+^Wi3#@ny_M{nJR+BzGbUXy5hrqmxWz-tOf$FUJnS9l9@Fz}M`VW3Q zN`I&lRE1~$BcAW)Ezag4H>L6OY4Z;L z(mMM+@AK>dW@&^Nsu#rDiEDS!!XBRAgHDfOH-55$7TK6~xP2#$c+@NQn@4Z_1}xF< zJ~Nd)j5Bznqr@N-T!GS_aJwa$ZZ2CE^}xeKpi#N=ct&Tat?NGjsh|q(rileSy02VQ z-dM{Sb0^x=fcJ7-#-Xn3-jM}wB^HT&{n@@gXGe#!gPnbKW0`C7m%fBSaSl>+UIRsJ4)t1+&P4rM(^+l@xi#aN=w@^^Qhh}Ut(Q9H9VgiK8K8imU@@KC$KhtBNt zHF&;ML~=OexkN2l{4vQ^$$i$y4wKOB=&K^eJd@{iq4kO`X005(IM;n2gw|PRdy%-H zxb?ih%=Rc$G=qBe#M>pw>aO$((gXA{o;1+>FBRoQAL=Vf#_N2S`=MSt`WdHnMfdj{ zb|<%M_dq=B3@iJP^J%>D7x^#a>Bk@=YD-_^%%)-dE}i5OUnP3jdpk;&@n3dlInVc5 z(s_z~DnQ0s;akjCGoDm0wjla)*JQa@Hzk?7T z;^510;#t?dN5b3kY3K8Wr;+kOM$ynZM4V{50_73)z5xaAA`yMeaAPtW8g@2nqnBbG zUQO)ezMJ>9)Q;0m$uHy;PRN7|O}@>C+D{^0wHx*_XTUtdD~~Mya5(ZPDV@#p&em7( zd0FV1>GM?@h|}lN$5Z@|=}=6!;k3^ZeaQdonpgv0U=ET{;}+O?I-{5qGM4caJEv=W zJ|aQW&CLRze?Cq1aJ=qAR&F6(F}ttP*Ed=1el)ECYf>fclWo$$^nN=a?@zud^1Ufp zBH2GNMzr0?iprz%?_cu?YSWY%__f;gv;Fjr5jQ8P&xvM|GLV^xpWtzvIq)TqB~D46 zljiSQ{{DIh_zUc;MgAiLeLnoSluq_w(W7#rfU9v+o5E3atA4SE%B_(US&J1TG$ua0?*&$dEJE09W32a6dd#X zUi_9f?C)w}4X|D!Bk>yVqM((LaU`=Xr0qhUUhwHS2`tW-I3>%_O78td(avi!*iRed zAgHsMJbh$*51OMpY48&`KiT|DB+38re`Fno)9XQg8w>@b_hWrp{=WGxX0J~Jy#6ft zy}j~#p;~u4iwovVLUV6k)HgDc<8UL+Nr6x2W$;G|Nmj}jlC#Jjt*WVF&Z>D(gxp`88r4+) z@JQ;j4RvvLg1kbKk6YdtH`1~;tmD(}T1yntl%7YAzHfQsZSeFCUtNuB?AMOt8TUZ%Tw4C)b}Xe-Ni1EY_Lo4?g62@#9eV@wcbn zz@lDA1IDt7>t!D%l7k7j+yR~!_K6k7v5%}Y;yqp|s}QF->^9qD;8{g@+&HaoEulfk zLQS4(5w^O7?}hm|`c!4iOcb%gz0cZ3^UNLg#Ye^RULGn2^v9ylTQ=H86kF}vDkh%$s&qo?+;Y)8ZNSurjdsfE6ja8z?l8MtS%oKRG z+MHfs7HdMJOK`3YEA$M%_D-nW3&meRuNUEW>{V$3N4i6hZmjwD>`F(g9yRdtMCxR! z4*EN?Bj>{Cu;1qxM*$;w)M)}8tju4)yKb6!XJ%zP9dD5-KOxQ>YCq)q?Dg3ltm3!u zoPF{`UC|i6a9-SwYTvtgY)eT+w!LMqs=}O`7{d+MglIZk0T-*Ql6r?&hUdFz&uaK2jF&BIC|7FAt`D$vP;k?9~A6 zoyk@7UMwn}NyxCBvhrApmQ1w2_kA{Sd$NCK|IDn+{>c~ZpsndtifMM2@3U*Wv7BIe z&;H5UNjp#JL1GeW!=k>wA3yB_e%?u6tJQ#COdj8bQe8;I<;J*)ml1vGqi&?Cm7onY z@c`Q6+@VVpbCDMBnxFo#ntZhu$~Xg`Td8Ioie-gGKMB!=Aq@V)3;lf-;#YWyS}c# zC5?Y4dFa8H=?;st_H^Gm^>{C-uIR~NeGh2TXA+|9!q&P4USbbhJ5&3`(Zu54zf%=0o=Cof1op7>NGTETi@)zoo&pa<9u zTGVIbyoRV=Dofj5OxPB&TKjK{bzC7-@J;dxHXMEF2!f7pQvJWey+JaW@fJ;y=Swk5AfBaX1HdW zuMN_ffREVKUw!=!)vjYjCa`^-jeQgD*3a2)<>^r?^RsBo%jp`JimOcl&F>hQ)Kcm+ z+Iv+Jb&Vst@6wt(c(&ocOeeW<#?_BJ%@y*_UyJL0;B7WcM^TTe-5Jf6R5u@!$jxcI zTwxvLIr{dw%+YsV5d&oy$87tpzR$Q--=%kP_EMD^zb1Pm-xMk5@b3Pw>%5;G|NoV3 zs>08k&GY+66flzq|B6^>s|x+QX<22t_QBR-FQRKp$Ydc^0T1(D<23S%cv^81*esH< z$kl7<+-cgIF=wwV&(ofKuM-ldI!$x2{pTlBmzvevc&}3v z_4(~(Gb?8P%?qxYe1tz)+zCllayF3kw{Ufjs)P2ttpVw5z~$n`$>=`rgo-!pYVN2e zX(ryxA!%#Spx`6AkyW%&}8>21#1U`zc1VZeU&JrMlb3t-FjoshKqCf%vIn@ih-+r6hZ>>A=vJ}WnY5rYJ)Y#G!)0pX zcJp~}O#b7^Ul`{`@7`_RuY@O8@|T;ke!=B8;Pz%*zK2#1WED!NF}qx);d!bDe&UL+&4hhEbY&yg1IJ@~%QuPcUBkD;i=IoPY zm`?iI@wz9FfG0(aDplBo*>Z(XtClU2e8rQO!1mbLlb>yg%;O8Rq?XFBCyj8h=wdaS zb}v7rw^jE|H1i05H&$4lkeiroRcZx$wwaGJgmrQ7~uLL!5^)weZGHCNA69R?5JH{FfW?#a&j0{x^L)P-X0&v@I9 z$;v)Y!#kq)9d`b2^KqpPVt>bW7>X}Y zt8=l{qdmA1CofzBMdM_T4eZ0qc=(dn`?Rwho9HYu9&dh8>OdH~9Ae$0Dri2gREEtJ zAjKE_=xb3pi*Aje9aqrfD(O|#uXxcz)0yije5jJL$J=4aK{lZk8$E&N@fnZ2fVib6 zgnb*v9?Xc7$nzyO_-(9l6q9j(#f(1YEAo1`rPr&jqn-WD@}pwqKlr#sA@4b`^|)B= zb(XmoWGcpf9b$3+A!lQFY7Oz>M>=tp&UI~60yVYH6zG^~#d@oCuPJlS$}RJjc!u zeSf$x6rWb{tIwi4<4Ncu(ZZYJ;Lqd^VwHCQ{ceU^ zabo;(qn+f68))cIcIRdoFg~3bdfm)^BvbW{ao_7XR~SndxueKO-eAs3@b1RZ+idlK zc~sw5v$|W%nr=m&(ry`DTWtRW;ePE{FCexJtc z=sDBRGwwuRSNI=!uzS_-$F7B5a4*i!T&0UvS^2{<@bDnK*=)6b8q4tvE|s#zza8h^ zWtp7AfcsqUewU!FF5B0ghkF)pwH5j7LKZUUaal3Pg+~4g9W9ec z=n|A;6|kh0%d%E%e;~P^kzT8LHB zhl_rF%d*@-YnzFkB9>n6-VZ~)@Qi!=x;c&ABXQhHddZ`xxPD_ zbs5hy>MtWpcv2=H!|tee&Hn;YK9GENHvg^VDUOIiY8d}yw)sw4wv5jl`rg;w`p7jD zXMsLK=V9|*3Pyda{^AoA0#5HDV{K#`!m|AWhn_S0_gf7Z0qt&v+PA}$C){;?8fAaB zU$zA*1uxJlPJ8I+x5l`6D;Aqm6rl9jW$lT3bSCJ8njQrKOJ)1n- zCMvFBe6gpaqM12rMS2>po))ofrTrO+S6G%UEaz>IW-OhJy?8~fQmnb{7lzALXx8Q zcw<)i9DJ=0ML(oPv0`_cXMPqBr8b$lRWz|j*6t3-Swi+_3tw$@I;XlChTg+^Ue8jO zAzS6_5P5|zJ>?0nkjuU}@erOo$eMhCrwhHeRR%ly(?^8mguFC*7B?du{L}$&;#7iK zxb!fgzfPC%wmaXN7qV7BDdKR!EAS_MVJ& zYBAo2Omp~^(~P8!5ycwgpE&n{u`Z&C&%o-b5IZ`&&n20^d1sC%M0H3W@_d|4jrENS zWCiMbN;f?1;4ZNzw*q>%(7JW*bsw!CMoR3hGwpF`0Vs26$mp6;-Q zG+dBgbBlfG=fbz1wEJDL>$EgSpR+QRK?0tETd!L?c?2?CO#WMXQVG>iYiPlj?!FLZ zPq9f4p*nV9euBzzbZnytBkG6p)0)Wf#o9%5@riy*=RkotF(-Cu$DZ|dIQIsbe~Fx4 z2cx3fd*rJd)6tEd(%P#iD_@#T44czKt;CviG`I&>UmyulBQg$+pF*eGy}ILRS==w9 zFU=zO_AyRAOim}UeRp}%1E^g^PNP5aVZ1yDQ;&*@n?QgIaWqbU>xZA``K^udMn8fx zBLF9Z@|-7EguD^+NROJGEI$B7Tc15v&l_;Hli23yax9+^Cahx&@GVZD?Z;Y z(lALh{D$uXAWT@lnzVHpk8+W_$Ldb_XR*UI>e-?n(r1u=0nekVjO!NMi1m&*5h=Pi zMJ;4>-fD}}fl`${`wyf3#{9>c_+2C^Dhyk*@zFWB1Wh|b*NW2Y-nC+#60VB=@1xa$ z?Kc05Gs>ZHAvtIy`!SNO{{okX;QaZhEI{`U8GoF+HU~P5F>AN8Qn#D^8RXzywri~? zuTVGD6h-+-W_14O=K9v@tl32_MwO)NE%+lN;9RVL_hS8n$1Q1pwviW;0eCAtM>Fy6 zR`%{f{A}kdOAetj#Cpp<&X>d^i=3wWfGeBRpQ$W=Nm1D{d$i*0y$vX?Y%lB>zOYkW zalR$r=jU|B_GA9&4YW0MU;z2Qi2pM*tzq-=G|q`jsF^se-spDx3<Fm+>4gG5fKvwG*7aI&G_-^zMFfPJReq z$oOWWHvFfkm3>?$`)YAZ55849C>^yFyCBjMs23;1k3ely&cwP(8GcGt8Qh#y9@h0K z@#8O0Y=-`stJwEJsC}F7cc~qLM_JYmeAI0$ZMnoBcycw}yUq+8=6}X13zN)$H`3J5 zoufm4oM6(&vmRmtVt3eKR`nA1j8m1YM!SDczeTO={qEG5&5N@*^IM@Kys#f8+$J@B>b#E>^SCME7*w46Ql4vwt+B8LCV8o-eVL zC1?$^+LM-8d76ly{oT8damNmU^Ppizzg|tiCo9?H3nE7l8s9IxVGg^2>Ss zBThdmmhP39CMyv;+uK+ziTkf7&t2$3!uTS;73-Bvq2LYfcqo-Y8(QJ<06O-#*9t4) z)qFCl)gFh?b9j#v(~rg#F^j*nOP%z&u*31=JZ-@;+{G3K+P?xSTH~rLSwt>T2ex0Yr`wJ2^PFQDPuL^;E3frr#^s48 zRa4bX?Bl`a#rNms#h$?7-`qKR!pw#+73E_Jiq@K#vyF0Gakj!J$op$LX03v^(fxF` zc;{)}P;Jr0L?ifxr};JuQHCAs2o;u*_WRxSY1h|6(Ma)SUC7mdY;;Xa#`)%Q0BLKA zzfJK)RR?QZR#&x283WK+S0<$Z`MpC``qg5BEoSmp&m4;48`$zV3-D{Ub||0yq*ad+ z?pqhlsnlgG!~|%x(cYR}9KGt9}K_yA|SLiuM&bDVtjvDtlt)$L>Cm!LH`dpW=J7WDPOna5bn-t=Q2L>gyC zo8$CZbf-S8nrTJlZ~XzE#os(~LZ#$%uT+zHj@|j|QT;eX>yFpeoT%LaLOjI>sZS%e zlj72lyc48|KFrUNiX-lLA-Nw&D!!Dj`Gs%%rEz~oKDN=hUeMuOavrDLKaY|z_;9be zkMl*YM9&b2F`cwT6#9T!ox|(B(r5tl12Da=oi#{>tQRA|HiGU7Y-lms@OTxQewejf zjq9(%$R%XuS~B*lDuOEV+D}05EAaGoQhX~O_rdKrXY6CM`MU4n`Gr4ypLg#@|CRiS zV&=7=D6%hZvmY?;13Sv5(7!r~m1edHv>Pw0xd^UyVPWU`{xh1dLSs+7Y3&X>t=`0r z`pZbiJXE|c8+0Dadk-mik}fRa@f}sc7Im7_$k;< z_Q7~OR}}KA`>f@celHgh>kMzX_jtI}9*-)>aPF4-o#k}!ciE^F_R}9_OJ3KjB5&ed z>%ybtkOw4=$?10`CDD)d7#@s7cMJBY4rI!)LVYN8f}~tWLf+@AM?FVJG=5Gd-t_u~ zL?`&tAF_Ev&Do{&u?zh83WdX2(J$fMFt|_$w;yM*mw3XX#=l)Is-C<^eGS`hngEL`7bnTs!(bekA5_=vu+U3@lMVu)^{~4|C&BM5X_4*)fSKesG`c&|#W;ELY&BMqIUSD1 z_sK@&k2;asPuwBu%03Y%6=#{EH^gb#n(tZHci7k~%};5zJa(q8_BD_$EtCy9lg`Y~ zn3%|){EPM8B{R}i-u`Vn&>!`zI32kYOE!m;mxb5+)wL(os&6&=qp5my>RAy{84*-d zG(IIBEGS#`uibXvl9qqO-gDd`2j)drp=~5#5?!o;{-{!4KtnIXl_l(RbqEmsp^DJ( zfw)%H&aLSAx)2_;L-|qM`bx)$C#{YBlhU~m^&Lc**JLzJWLs-m<7}6FY~(mKHLuu% zbqU*0oKEGqLt9?V?`reEwEJrd+K-U+O87C6k946ZUinu@(+DhO7TaY&8bM}a2|s-z{p^7UZ_>by)tmR1 zM?QKUEjygCCNVkrcJk8XQtJ~%NX;}fd@G)L+*4y;W_fFbZ`#*zfw9klj+Iaz@zsa) zbfg?tZI+;{Q+`Xs*UQ+m>WMFLy`c=(G1<_E@Vh8|+nFi|Q}2LsUA-P)33~84MnQ!u z?T(mB+J1tkH}Z0-$~YY%XHUVPdhqyFE2tB#+0C*`VT=wJ)2vwb)Mv05l({$2+|%bK zF4BSgOXn{q6R-=AFp@=eeKK!FJyiCv1zre<{eipcPdqpZGTL3mPyvnMeEHX>DaUPg-YfA zq<*;Ud|NaBvT?SeB}H;JqkojDiN1Owj5Y^xuI#?lWZh~f>O{08aW{G2nYcN*D)~Y3 z{N#|tJ3N3{=KehVFR3E(0i6{7W8Hg-YTtQQ>hr;`QS_>lb-1NoKd^D#-Q`}MTb8|T zPr%x-yzU%hyc++C)AF@DWcy&`D-?uUoy=iG^{AoNq0OR8{p$-OU!QzJuBav+Zc;7zy0y8z@;0LqpC%TopB~SLh`RevAl=;6N23Z>)ZRpPlgJN6pX#_14|)jOY+t16y;Hge$VG4?YBYzV)_+hC3(Qc(L#YsI`zXm9b zJsNwV@xwg2(XMz#)@^~=e}(U`nH-)aavGA}&pFAsZ#1GlUY(3#lg~~t_p?aBG$X$m z-LYOdo(>LTLwXv;P@j0zyV3P)4!Q3sdYbDri?^(qr`V(-Y(iJaI$gAP4{t4Yl-xqD z;|z|o$zlgweJ#Dd+ua&TPoC0caAX1h=W7~zJNj$OZk1={vmsA-@p0x#thl|#8~YT- z?lFe)Jdc57FV2@8Sv*VAG$_IUq(Zs0>la2Bz zt4GgzPG?c}-Mom|5O$xu%XToc0vQyJA$@7mSnJC(@IXe9oS1Y19dJ z<}=SQ8*k8s!R%7hMMsCl<9Pn5eAGN#o9laY-HQ{$3&>|RLi08B;Wo3-j|b3!o@aUD zR{9brHO_=6_n|fN+@sj$oAB{E{ESt|Tk$_Ks?Xrjm*V@u)(79wk?uVfHA8-TJLx_n zu^vvZ<+DWO-i8E5?9iV?^>w$Gd5IGq{*ZfCJ6RXDe<83!f-tSQ6GvWvuw_L05iPAZ8!vdudUF}OUpg55NF$#?PVXj% zFQKsjI>wQPaYk?vI~QFpMw9LSq$ER>@{JL!5JSdU(aUi5RXkfspJN|?d!LW~WU-Sy zviPx^{Y+WzA8F7VsCp6QGdw%`O2_KeL>!DtpleClbh7h0y?WWF;uOMzITy30gPm*f zoK?`hG8eN*=zCsMN$eZ0YtEWMhsCxIc$WHE{efS<{K+;6i>`r}S)GB|BGI zzVk;j94F0x%{%_dI6gIwb#y3$3`Ng|e)!W7sx|f%XX_T{QLKfhk_((toUS43~D@x~s6ZZc>s(w={ufl*1GdJUOhAr9QgMs6nCo5;^TNE}^qtMDt@ zp}n2RJ~HreW_O&{u)-SaD`|f+R-+&B?8#|qk9E#TX^ps#6g`{9gir8fCtWO$XA7X{ zc`Do*v$8A2I?(|mcH0$|7kJm~O?8*~=#C8HuX4GYX4t`@cs={yv>(bV|6?zIblAx;(2iUgbe%<5#lfM(VJ;!(mkWY@qT}cyG5_r zqkdXzr0ut^Ah4>Ns z>~|R7Dth;eQ6BKTlExTofoDUAvrrxTRNCQk8C0Bd*EmHx$(PxQiXHS}zgamzN@DFi zzq*7o(7M&@uzMBq+ez~lXA7^S8E@fkR4+y^mdSMWzv9dv`5OhTe3UjH(IGbKi(8qg zsBZ2_B8G^5Zo=PgINa8F8j!_o_x}wAaV|v>IuzZtD;RCdG%hqpd30bc%Q6*!(MM63 zds2DyqK)3VN6;Fz3nx*&5q0a`>zEpmeeN1PSy$oUcjhq8yNi=vUWXZP92I>$ z8sTv#^w-3*W9WIX9(^B;Y9Bd7B<+5VPQ*yObaF1-&a11p2Z-%$Cx zQN?LA>v1=_1!usSe?`ghTLB!bY;GIU{?O=7G$?i<#j0pc?-jw%N`7u&)vN^S!V{~? zPR7`x!*^*kl{A_b=II<1=CvlZjr2stS#%bTe$)S=F%U2`Cr(=a!w5d57t2ufGpg35 z-~Ac)qqA~!1U_I~Ax}G9yBC$QZ#2H5`)LMadWcwRNMRq}gHKSkfE(q_IIM9MP8 z=-Hd;_t=F~+5AT@lXM}LUzWVtRWX@>S9?vnC*^!$0Sig^DNPVF+5!tPlL zpHBHp%<~RZZ#9|&-j6QBvA^@jG`d7r=2*3l%uE5fmm{voCpQv0R1jwl(B^EbC2?w5 zMU{&!d`m9GM z`VHxvWnR3EXPl&!#XPSnEem-q>CRCr7rD0Yd^%rR&ZDbBbha+ym1P7mTTz=8oZ04n ze;8@8^gLC-_kZaA4&y20+Op~S+~tmeexXlA&0R^OI_w!?oA%;;@F8@hESie>IX~(u zdDcEM^FAEhfm_ANZUfp^$LGQpMJM-N_)-8B+v)Y6K2y#Z;xyE{?s60_4&%%gBZ*$} zg*+j8>qm$5Y_n3)m}{Dks7l*IvsU3>;C?Yw)bL*PV&6}8<7~B%pyK#>+8uYhXVPzF zd@_&EM33%lTnw!b%iP4PI3H}ItBXT~ayYjUecSw=i-Ga^=yQ9Hr$pE5=ujSKYi&zQ zOzg!8+jbm2M}P9@;2r3@Gp$X@G?$|PQ%M{u;d4<_9lJpP_30R6h7QP)X)R+`3*f>b z{LO=-710({?n7^m&+c-+{XSR0ee%}SZH|lj zYc`7iG{zJjJWc0B8t=L8itG~;2M2@sqN`w~)~L!tut3Zt&1 z>*ExK@Iu2o4qgP-WQf$ap)K&QI;1LNT;WBTQ=64HJO*F)=zqBrm(tWIEZoE;I^n~}Cad+ zLYp|LB6RZ~caJ*P5~QarE{BzgQx&qk8yFN8Di<^2WPt*Zt&;nM{SS0XxLcee5E+Iz z%^*&|*yovHKMvzfphQF?VKvIRGP)KPOn)jol>EjV`IV60ki&=%a?fB0&52QD8%@+N z#%UvgX>s1czmOwjHjlprqQq$*aZW?n&(O{TBrJUSz_$JF82VDi{o(|PqFz}(8p6!^w*qt9b8k(0oi!p0) z`a*`M2EN2NVy;i(U}=2Lowbmbe|;`S9#%OlYeApSjYWdD2~-qvk2ql>&XovFiIX_O zvV~3rXTysOu7+g9?%y4r8&*8x+VHZ&--_qQxg3!T3fw&8Zh;$lTo*hHJc{?iQw_Nf zuO&Vi+7);ZKjmszplJ%dp={&+8c{P!I?ld=1@??*)7=x{$@^qB~>w?Gd-- z&S=PdNMh~`hrS*3Zu}PS2ZCjLUU=uZ91gn|smh(L{G`7E?u8x( zB@sX7o(2+g8TZb;BD5}^5;9i6=tB!a`hp|Dv)p-&^EINXH`nWo83^3Sy-y%|c%;Gm zkgWe9bD(3Oa=a7YV-_M_$e)&z4BQCB{@*x*n%sGfxe5N|jy|{^(PEs&64IK>r?@I; z&7Ft1Z&q5{V{StC;wyJtK~3Bteh3HK~1h_8Auyf z#@}+~BM&;_6}la-I8!s1^JjQ^*!nonG434v4t)xa$C&>ozu_$f=L78$J{Niz6oh2N zS1#hkX*+>K@uXZC3kpKRVr01p9sCF$g|@}rf)8QaLOMbw0&7B6gPVb}afgt;;LHC3 DejC}J literal 46876 zcmY(s1)LN|^Z4C8yJv8R+u`o+1b5e<0fGerM34lx1PGoGAOr{rmf%5x2M8YA-QC>} zZg;xhZ|ywy%l~CRcegt`Q(awMUe!&ry0vPh+Ne~ksx51F`{L`Y$&^x>SCuSEeN#y( zMVDWfJ=g0ts@1sehqam~;eCTn{kxSaQ@BjA!o`XfRiAYFy3^4OT;`uVZLH>< zPlfUF^OE1r|9zJ4eSCJU`oHhOl;xk|8-d{Cld&qKTe;%r8+m6#wDjid&D47@<2WH)WIS*7^4X)cV4d*t z=#&6eu##^Cvd~|i7To0be@}QYJ*aZ8K=tM(PkT6eqkHu5W}Ps)@Qnw-o3D3|ToE{O zuUt>KM^;`a;6e4?3m3@!-YhMY@GuoDJ!*L{yyrX$dN_Jx$cTce&_S*X_5xpEdT_nj z%3b0AH+I5`di*2QkWsyvdv|&;1gbZi1ZsFZAtQOK?u{sT$qeK!?~ecX`>@cr-ktJW zo))Oyti30_-yRMg_6gW{>*?VkzrC^LTMy@iIe2tR;Klzi^wwJL^5`MY$xj0Q^4G&! z-g(%26!CENuKjmM!t>s^38Vi9&zqkI$HPJ}^S<{+_V_|(pHAiEo0s+;uV?m!jki%M zwMxlK$vhMB6K~^e0G<^+k$}m6RFbO#$CCnYv~Zr5myGCrC;ad6o`;Eti${OqIPaB! zu{VnMNhtjPu1^9jy*VaOFo6>CS!gWJ%ANnwQ%3MKRs!^dzuvtb{_?Dci*Ud2t>`oP zkr{Xz_rE*jNx@Bidsh=iPQcy!obbt`y?jc*TVQ+l$fzDSdK40P|NVHg@#y7ASppps z?v^`b?gCvPdXgZY1-FE`$~PWO{=4R3CwIu*@>j-5fa2kmfTPS-^niz{w-)l0+$C@X z=LBB%=9vIpAPD`1THZ^Z@$gC*&C@{gNk$M1JQ^fGl`&*Y`J6y=fhE|;bp<@pv+~&+ zEe_sF!qs;n4i@@|WimEgMR1qszwpps;SG7#gY3;eK}I~v$dmG|aDt2`RPa{a8(X+k zFq3-{<{?;m^i06q!#3fnP|914gm>P^9(N?rF#)~@JAuXmF@cK`C?aDg=z4ENk6!Xy zMoze2J`3$-&IvD}gNIGR6%SIva|xLH{zoy-X3DIE4k8_bpNuNw$asQH!e4K-JbHL* zE~6x1lrU588*illDB)o%&&oCVE6@G^fB*O7|NmXWecqe|CwZ4ZNg3C}*rS#QTOfNB zlhFi10)F23GB58b@3TjJ`R&Pu@Sc21n7_x39whn3v+&*s-YgO*F7n~c)1#!&TJG?E zd*gVpy?J@#dz6;ZJI|ZYJ)tA2sIC?N->;#%7U?5cV^q%;Y zo|csPiR^h&;D&rZkpu4;!N`**4_l9?1+u`B*?Ke*>% zsrRTYl=blPVYNJ6BXARD=jm^O>rqPn3N(v85cuLPdw3*_?yeKZD@o@uDk}WwLxC-HU^ECmS@v4QX=khphu*N1fq-;wC@-{ z2)uVZpA!6D@~xk7l5 z2z+G(4a98fjeQJXytP@lo}4EmfSj0bk^w2TI&5=r$1D4uAHPbk9^0JiJy^VAjxp+< z%?!TJc;>tx#yG|dqttp^9h}k=-R-p% zLipTf7SHWXTUXuYcYdf`fO~T=yXRp1j!(~_U`pt46PhGb$Dv6sDD&DDQ5S7SsI|`y zRkN5)HmDTAmy$?ww2jr@wk_9Rv5s+eE0kPjTc{&IRZu4vc66!jtbXSDD|j~#ba-Oxsrj~#8gB>j zUC=&(&+ajg#34TNLz9wfTnI*1#(59s>*0>2wjBOt1e~)ItgER%fUpaQ@9ci&TvZja zXVpl%)UHuE>>K7+)8^5K&ChBNT>F{&&~9Vg`F4!zV&|yQ_K;ny{PtT_N%gZ!RC@cf z%BV`(M`|`yY^|Ext*Q^t4N>`RGgfzw{ar1`%Uz`^@XW9FlK;4O zwuUQjLb-r#2oKc+;~{E_oo&BTRT<;D{nuUrat+m3WwIw#LtB~IJ+mkIKA+v8YS|Ic z{4wwwvEIp56T6qu=Wyi)xGGy%eF!IPRv$5wf^b+tX!wyj#N7L;Pi;SS##Um54neo% zcBkqK)jFsXP`Ic40X`aOGwP$prw`d`_LZ7yr`q-EiJ1jG2STkXww21SzO;pP7*C{E z8Tobunik{Kz8hE})jJgB}O zI&XwO?!h6oRT(>0RaBd78dV56I&06_7pkp20NrX>k=Zl$6R<9#>O=V~Q0pJYjD*4u z?NeJ7+PX-3B2^TsT!RCbpy`rvPYZRKtFdrLbLJPT3Lx`!)SpO68gQJbK>ZvNLdhm8tKIc)n(QzkuVym5EKeHd%ZnmhcYU|lNb|?~&6iN+H zk5oz>txM~yI!dor|Edwt>6u+Y`UE9@Gw>@ny+un|_WqAGro6J_kmS4UNQ zwZrzd7lCxr&CXoEGJRDO)lL=AAM1^}lAfw(>S1cO>WFq3YQMH$GWUky-G-l%a7}LF z-anbaTN7^Ixt}uAc(Y#3Q8!d2-AiB9DfCCWnXU%Rx3&d%J78bcHp4dTfdju&MN~i4 znYA8m%RIDbE%{s0H?VctTxy}HUo5RYR{Mn_JSQ{yRhPI zfRP0mOl9wz&1Q@Hfvsb&n}ukGM2x*mr*b~fyL30*iu+F5DmJ}+W%}A{rUP6&2^sFF znyW4=ru< z92%h^dcbSzt+IcZu4c8n%G3mhj_Q{3>%Xp6$?Nj*z{17{w#uT3uId(hl5gY6i& zx)rmXp$5XMLv0%Sx%tq{cjucJ@EooVs5<(Q*3J+o%sHz6(&JQd^&5KZJeck=ugwQ& zv&?Yu6m?vEgfB3|-ZG6%Co|O@Y|_~q=1X;6Rn-Aq*y-S8auPYy^=wrai+<1KwB5{V zQv>d}WizQ_>a-fKdZD={0->xKXPUZSnJ~M=j6xzN=z>ls=O-tNQ_$(F`>8(gRTOh* zZeGGWh47uu*>h^WnuP3Zv5W0&GYO813ywF3OcPTC*ma=7Z_aEW)Nyv{Rm|4djL_Sd z*=B*6X~!Ya->V&cD+K6bF`Dfd7(ei*>pXXRkg6i z;K-G%XD4h)D|E-NXuz^aZ)+s5hk0(&m?WfTCzx+-H*{=GeNUg#v-J)=1o_yaM#KLH z?RPd7oO{8O*|AD(R2j6eWo6pf_ef4rTg=Qe#q4Rb3m<2UDyoy|pWv)LY6+_`MJ+%_ z#X`|=tmJyE(i<#B8GM49aPI1kd84{ib-|C&+i{6j3gX~)5E(ucewJnPtDrei;;jH*s*1RmZ^nve2sHFOPHC<)X zf56Lss|Lv0Ah=9??EUtl9dCc)`3C6DqVW4X;Zb0fwHwi&snOsOYM&~P=Blp?s*m+{ zRYBcBa}HEP(cum8V|Jr+ccH~Pqx*hGWA$XM+So2gUma%ngms&XPM(B?Sq4QHshw(* zdZ$XNKU5LD+JyuDhhepUz759q$4%;!(Lh#4JbHu;%PU7lZIo7k4%HmP zyXrmEsfiY@3toB9+tpQPtjT>K-N%04w56a+Z#44v;8BD3m9R4VLcVVo^vWEpg4mU_ zQ24Ss#_Dv#uAPGdlhGb0!1_AezR=dTt{I2K2h3;CI5#V{7!B7-?NY;4VO>u@R_%06 zv`JSTz@wOjCc6oyC$T{rv0@{kOg{V6G`9bmS0+DF(F}dK(~eUw>>p~Vc2yIdQSU=X z)JMi{;)VT+9*IWh4a8O-!4jT@5^3yQ>`GhcaLKer*DI)8Q>{V^mPcYn>cLQPt@;A3 z9Rofu@f4CFtK02gcr_Po2e{&}ISr>jHh)tR6gAl-FH`ipvS=J3$XG7 z!Q?yE!(#W(+Lc)FbKL!*y^RL{(8Qo0W9?c~2|Ie$rpCG+(2kSDiO}cmc<`QQXQ*rF zs^zK~o|{$+p~D5oemCVz(7kHjpoiPzwSKKu>pyfFr?B&t9-yYoTW}Vr>NQwj^of4DX{sOut+-z zju+a~tl}^%{XgcM$!B`I1I#ef%)B>~?NYrPjWy9%$!UjdWxyhAG_&!ycB%DxzV5Ai zz^Mc6K9d+tn8+?P9ZVuq-jp)^(VhXDhZ$dSzW0rGqVzjsY1 zBH8O~9g`kARF;+g7kb@yZ@AmsG$zW4^_TW-@xAwr(!ZOeriPj6);728dDTzf(fyn# zr!f)8M>d(c(dK;nro&BT2 zKlDv?^8^kAdIYNm54jnf6#go{{JufHGdh`l?mDKK+t;mZ4W4f zu!zX;uD;XZu0PHc3#^XM5Nr{g7mU;s{iXa}d~JQrokn&|FcEV4E_lKe(3|yEl@C9C zojW$ZeV}gq0o^VtkAHSp!-y76_h3x?C-IqszF-COi*I~bUH^1nVqZVi*KHT93ZGnY zXDPqaMxV!Xcw^26+s6+IOp5#K{n>{l3ZjnBchJ zN9M8br|{GMxxOw=I+e{`5SZd#HY3e=^@DRm|4Za`&G_9C@i*e1#4oo`qJ9bgDlALH zEaynD&-)ec4hD_{bDB{=s1*`ypvf0wX z#(^9r&J47R^)e^cS*W_&>1KTJ?E4IXn7|Mlndnyd-mtL|Pjor=-uq*5y8;J;=iObt zYT@m|#`v%4bn1ZnYoM6h#{6M^RLPvQPBZlUAUoY%6+bclMf}IXH4*0`uKI%!pZV{Z zX@TPJUdQ(jW(oGypN8cOyW`L9YpNUCMZtZ+18z8aAiaL3*K7POwZY^Jj*R~z@JakG z=bOlZ{-fc0!&~dU?&J57@%@6;0&Pq$Utj-Pe@_2wr@roOg2BY5mb=ys*n?Q^dU`k7 zWujX!@OL0@{7BU(BGNxS>~wf~Uuj!7@F4zJ&Cl?xa9<_uD|qV0HCQuX5V?diyH* zMx*CW0OLXMocq?yBWC&wD_p_e2>RU4fmx2 z^>J(-TyQ|@5kV$7Un{1{9UUDJyb0V~CHr>xR z(pe2HOZ)QZ(O9mc#&NT`)s15d+b`8wUB=hd*`^yhzp3u_l<94%5l=*c>tuI>`_4^k zGU#yMa%ZKltp8KrSe=jD+;G#;?diU9J77&_+ZnpOGfdZXJ_C;rh*Nvn&&*->k~_xD z>eh1?pm(RL{KNyF`Lg&o`YJlv^*PnXCN`JR_1D~&?h`XnJyB29ZJo}Ur^zef)0MD& zO{5v*z6z#y%enzKjf&OV^-gEJuf1=G^Gg4sTdN{qJIX|xr|wNN)(*fAdw_==L2@{U zUQWKUf!&R5|J9x4PDFRFwoO!=I;z_`Ih+XRp`NQxs?ubPy4xq_wyBQA%#G*kCreR5 zjU=}D5Wjc=vGxk{qiJg%VGXa7HTqSR(k1XrYUqkOy*DT}?-{P%a3(e&e zzSezvgx}lY?5+6DhxvO6FZ^DpGLZrQtRPQSMTayXHm*&)Se1w-H~BIj{?aqzfjhvu z2wjeV%{CzXi?4Z%E4T5=pA&gFX!9IUUUD!s`KiUbl0?WELsfDg5&vs@3pnR^{uo#t z;_qo*=ehEL*+^9&B{|V_*tQ(3NO4|efl!3cS;#p=gNueTZyE6h{@X>KJI~(>c*Qr# zO+Exd9FagW?u_OBtmKt40VylL(}k`?lf_HKng)nqp7Xi~=J$E-KBK?j+IzmS#I*`< zT~;zNv2_~$CgF2p-o@}9+CqyIk3Z#(f~6^1r@RbH#2j6$-QC3 zO=pQTZW1X(a$O=UiF6-<)qUVdtvLq#@-y>vd>6sWNLAxC*!qE=hPzVn6U&uX+;JTm zIb6>{+>w{(o^bCCa;dL@WZ|G3War|cme9{a$9PuNVcZDTK0R@*1N_%e=Z>uggk=CH zh7nVPQ8?E`0$(9X5zIFTwcqmHV|Z3-kYUU!1sF!)TjYjY@{&C(N$xxa??pqtCZ4^H zw*LnWyOQWwat({Ak$gvnw=c2&J^Lwf!xmLRf2xb?JmhyTzzva5=V!7JIc#?9cq~!X z$7DFZB=3CIMA+_Tn;lG!=v%!+`>}Touy>hsW^it9liLlZjyZ1bnV^YPRa94U*?GwM zrN*nSiv6B$hnbpYh;iN5ZVuDbyfd{;9`cgY^&V%Rlf(D9b5XSnXK7ZR$zu5hflK0-GK$)>V}zRrjpUBsxw%J`Lg2$X7sh!cZjQZ zkQv$bWB{=f|~gZtdw?iIHxv1M3ygC6mKsFn8TK<~wHFh&=mc)k@cOhCAbROZ|dX ztz!F_FHAQRM!bE=B(f>&1=E5EJ)7O)e5TVn3!IM5Fvmq^7N|H=*<|6)^=2{f8mK|) zsw$>)I$t?I>MYt|E$Z0sfi|AITA66PkK8suR`oOct4Z%1Adk4rspa%{M(f|GD1BpV znCd3Z9b+y-kANLZrm!IEJ=xhz20SNu^snq5^NCqvN)nSlCqGe>jNm1+*iJFU)o>@N z?o6Jep;OtZh}~(YR*}z00o9T+qfF!kKT%n98lBAf+DWN%>ka6mnzpBz1NIs5`*M=O z=|PS%9a+65W~N?C-sL;zz8>rx)7$Mdw9-;D#XN8qlA~&BwxgYXCg)g84}lv4L<}v7 z0e!X=d^iwpJZ!Q;|Lk@V(bXW6S~a5dJQ&P6IU}7<@UVNR<7AGlTNr3%Oe!o>J>rG8 zDu>h7>7;Ax1*$1^%gN>pv};8Ec9ZD@?fx>!sQ~A zbK2cwhU4AM#-CV2_Pv@O;jGfr$QBR9k8#bv;8maO%5d|{)VA-)EoZdr%p7|JdgLH8 z(Z-qSye6Xmfx2I`Eyumb$jEFkWAKl*;oHq14?X~RugP&{R){BR-LpYQg<^_xsN{J^qQ3%ls=$Yrw7yC6EP5Wn+> ze6|eu8J@o*T@a7)lCFlbUI)oi@4$oH$!jkZzkr;-!nURYPCiu%PqYTFn!IX&Q(1ht z+<1U7=ubDKO|Mh?xP)DmO5SOHpU028jAwb9*BvbOE3l16FGt}E#xkG0U{sh_0kF!& z80qosQXyl~Cn9qb%`5(a_;XJ}BRl}t2l#aISAO3E!$%<>^)1%f4=$%=1pcECz7K)>5}!c&@jQ!@7AVpckP~b@sIiPA z9)fhZNXNSP)z5kUF7&(ezu)rCgZh;31j_)t;=|q~3DGrk2%^kB8<4a857Q(e3n>ua zUI;%gppHFM$<@e-p{7 z$^0@x@66Chelmq6Cd8eBUnzPdK()w-hNIsy9=3fk-W!vv{h6&BEPak zeEE>F;>26@mN3(d%r7a_dC7e4L8A*moJMT?A+hYIM9D*_oBT{&=|1_Eu~dHcsE7D8 zCyB0y5)BuC{yV81Z6a1&0c~F4Z)HVdrPFE!wUYu?Q~5Yf{(Lo6l)tEUH>5tdLHnFK z`aV^~e9W}0eQaW>OjNbsGmmNNFg51;R3;XJ>tp6o9LqS)OgAIQE04r4IctX5F4%}% zPF<&n)7cq>fAf~PCdF13Fx#+BJby;V3dXYmO>a=wJpnsw@Iq!c=6n<49yrM8FZ?Sr(KCP$FNwAjnYe_9)5%zK? zm4r{BPF+SGO>H8FDM8L6uQSg1#98g+aavLVNMnyMM!Y+odhlcVg|<<#xk}}_p`J+G z{TubzZPZ1sn^Nd`iEg`aCE9E!SC5aXr#Q#RT|Rg6QlZF5?(u7?w^v*$`cyoxtI~R} zenlPsEL=4gI{%HQa)+w%Jb17y@>9cpXhygN-Hq-B)y6479xmMX)|soj+b_)%{6*{L z#`_tra$u=eQHlPIN>~h?JTkj2CIzeUlHQ+|==$+?g6Zx)2~G+QGdFcvUo!u8UuWMQ z=W|t(yiXftHW=Susd>EI3swhBHE zCeh#ddipo|kNEt)S~`b05ZoH9uG^x7moj{D;njCDc@> zA?3Yo7gNkl8Y~`M796e~`BH^74lCr};Uw0HOiVCMFv4wOex{3KsFT(=)tREFQNz7p zhMLaQZ#EH~omLt3PAWwwvCEB3dAC5&21W;*3L1PT;=8smJHqtCRX8LCBm14rSc!q z^fI}3;>QFgx#MhcorPw}PQJ`eV+GISjh-`aO`JWWTI(MA8nQUi9>p(O;$8{X46X=H zbC2t@VKu`)4twLwS4nM$;FRQD-}zRMuc@;8E~ryIenYjym0) z>$-yKVE%P4y920FehYu~(<${FBycX8V2HWrR&<*)vkPuvXLk7J@ZYUfvHJt!viu>FYCg(VXDs(;l28$32 zU!A4GH`|?ngw%DH+O%O0!~Y2@;LEGX$hw0A?*g~YNPXR@EItn^c zGmV7H=cx`riAOg*cb^CA2Hyv(o7BEb;qAgR`^ncEvh~3pfj``3>T~#ck@J&&MF&|f zGs{hE_o&U}^14&MJ&CvQn3`h&y!_|El);(7neGT@Zg`dOk4UPFvG3iFnNuz|ow}gc zIX^r1h*E2)qeLbb@sy>PX_6vN(mk;_Y01|=bB_lbB43x>jZUZV!r`U;Cv-1c+I$}z z9Vp}~wL?!J?%SciLYrlz*Qg*7RXO?^8dDk1LDqL8R%9#L@$JEh!8O6TX1ecK*yyl> zzR$>Q_cFtSI=IKZYa8m!&P%;VuhawSFxiT>AB~LuK@RvF)&36Hn0Ca-ZQb93&w^dt zld6+HZCH8#9p{P~Yh%s)V3@n!kgFr29YoaCL9bV>$YvZj^N9euU^(j&!?(p>eo0Ky z*{$PtbJyFG&Tii!-+j7&CX$^w><*xY`T>1xOVxJr(x1}dlvxi~t?5R3O9a!482>IE zZ80jdZ9&yEFS`Dsy`UF5dz@TOTYVVo9&WcAO>KS${#R0Ryw_A_ol8$9mYhtNVtXuc z7djbk^C=~i+GBPTf6xOA&X@F9qWVNs+U}4yZel;dde6ew>8$3f9q7#z_#Wx$Kw1Qx zPUIK{BiDI}ziM#j2(lpy)MJ$%342fVc?7XmW;&+Ulb6_y?J7lnEipb#RxGWb+{tOO z7T?o%@)bV97es}FvC@}`R~k^W->810M`IMRWFLOY@!SKv_j7n$&+xXSpI!2)l4;)j zzs$!hF#Vl8^>HkPWP=-#>u9Au$1kYG`-a3UMeya~v1_;R4J4!c5ISDLpF2m^`w(#c zCR#cc>f(D&7AysL7NK{bJbC3x*qusvI>q=~63-_I)VYeMb{kLRA@I)O!)+&DemMkN zq8!OjOTTz-{K@noFR&m!p=7Ib@jDi;EDUKCuU>X`N!Cv~4=>@pN%k-Re&X*+L=a8@~MY386FK-uwe3IwAeG7zwj4Hq7 zn%v=KaOK509JiK+5wUd<)swb=gZ$o%^_j?$6@H`AX+`a4ajEpIA6wNya`7z{~lOQ|N z+blaVBnu$p$jjpg!C5|ecnT&`^^rL17kuNKDnKX4Ncz*)sF`GD9io+%jIfW{mLS(~ zkNEF8dFJ#yCG*aNrs;`QuTD<22=^yq<)YEKo1k+=qQd(`QCNCLxJ`s4x#^5}*b*^C z^Q`nSKLSb=*od}!NgSRLZT1rF;6vsmYSieZYTR*?J5%vQYR0`wbatJc5t-SoaFBb# z(00eZMS3g4+dogMcJ)lWDY_yNHnv7kTK~?hx+3gJYBG0ihPnp#> zWOaU|w%P<7y7J@#df%lpC^7TDPCxVj^(|4J*f22PGd;Cy1j?k(7$feeQXP^(x7ZjArEYeN-CWF!yKdG>D*v@d#XF!Q?V)_5fV# z(Gwbutt&^rS7xl+WjjPuwk*}rp# zoJItEQl8a)ixj`-&U;*!8bwAhdx~w@Po?D@W2V7=e1I2p9gb-TO>e;)8HnFcP*r_S zF5((bl;G(`%=9d9q8LZ|Xl&f5>FyC*vl@g^Y~UzvBXKJ+N6T>MUL)*p`UEWb@GqN_*P@b zEqBX)Bgt7vbzSa~Y{(s+l8ob7#>^Aa85zM=s;t@gyP5lM0zEaBxExoL!Wq|)Cdo0< zA&WK7#(UYpnHD*}%Iu23Gm`Op&AmzZMrtv#=jtAFPXiC{rw=#{Sjt|g@*(MsW-W!= zQ{W?=f`-w^kL)Bn!V>|gBzd;m%p?c%&ql2H9OyZaFv(@UfG74si}&FB92{gv)(vDi z9r7tNycBx&130k`TImEHavWDuBi|38dVL`7=gId_ViV6@1X>cl%K_F?eRzwmk$ zsF(s>^oF?|XKce0>G&MP7nI!3Db^zk>m;?lyJ*93^g>DaT=vLahDt}^ljl%5j829( zKu80fo`6$I?#l;nY~#vL$erZ7B6+4IoG1H9Hh`O0o~I#cNR4L6%Spo*-VudboS50WTi3+P=eGMuXfqS0ttiy9J(F9qUNi?+1POV@L zeHdA|Dm@>`nU!LF4nWVcP_P2?ThEg}QnmUKx)mS~7NlPy3H)#yx;{gCDuDeHcrpeU z@^>@ql^M$+mE5v$@GEp%O86?ksA6A|uv%r%SWVSuPT@LWkg9ftav zb+3a4Y6GOPJT($MU55HpePs9*Biv*acEBUwFtf5;zsMaE&`4vbHur(@5%B#pIA|DL zf0prk0c#W<{ZpPg&C1O~6S|>(@(J*FR_I%WRmlnWBxXfU6M<}pZfVI3l!8{Vc-J2= zav#POt(6E_zXNyYCg#eEjh1eM%V3$8S+7H%6k{FN@NGR-O){ZU+mI^oZuHN7RwyUY zM|oi8W^TK%MLR=!X&im^DWP&Uw8d>at5N8&hs^dWvUY^nw-U1W4&Lg`4w98fLUn9> zajv)E`a#w*FV*gEfOUm&77#PF!U|O2sVdMmCs?@f^+l*Ok{R3uQYrSXbbuqoA|!<$ zB&)p-e72&ip23}Uxt0WZ$%KTJBxA1rdUB$c{%fJM(27L>`^~BVJ{v+Y3OrWIaG)% zkVPyU_> z;B|DhamKL`cHrMJ)S{dbZ_%HS+UyW7Y<{`U#lThofA8&si3XW$=Ppm zm`F9fZAQiR1~|7QqAjaVsk!W2ctDNxV|<5eU&)%*pNdT{q(a^Y)~#TRTw7w~iQnMrNFrh{!L^D3Z>`d9aIN;!Av zrMsfz@J)75k3T?c{}q1Y68h73)1m(sN^5dyYnjnZI^LU_b?$fM5RS1oA>Lji+uPGw z>6CZsQ~hj25B**G*shsalZj0AFI4s?+KTKmc*1_C>V!-KsY5)YfBYZ!8@GzPz)XW5 z4y(0vNx>Td;=sUu0p{IBT7n)bWqe^;eD^mUqpkWc}Kbk2z@CA#H!s9iLxU(|PGW$j(mA zpQ*P0K_7T8SHn9E>9>1JM(Bjo6nex{kN%K-r_aqavL%(wV7RP0JTQ(OYT4o20{RTr z483F0nw6|=LwC8Ei3r^#b&}J)It&g8V;|55rWk9pmYn<&x>s(JlP^aH z!vppkY-D#tC9~X}MsBC98G;0!P>a|V^{+F>DMKFqJN*~ZT9lmr$0iRs^xup^yX7L& zlZu?4NYF%VN?zOBoO6F5oBh%)ft=4$ovBhxB{N#vDda5EPpNc%#V&@?>>~J_YQ$Z- zD9*FL)`6mVso`%V|Iw40cnY(EZpvxyB{LttE*+JFdQ<~ifyYHU8b)I260`rIp{Zwn zrea-(Uc?gWqRPeEWY9;+o!mrX8=152dbhvZ-Za1-pHQD7Z)2REPEo4jU0ItV^cOv% zpRz9NyN2F_iFlaf*msl+JnkU9rRbYjN*&>-JCy(`6;j!cdj2w92C5cdr`R0kJQSUs z-R`32;Bz|8ADU0FuEo@TxFkxyW=~>AG<}q9U>>`h+`rtz=2I-gC3MIg{Rug*7;^I! z^xs&LR^aZ zj^ur%UlJ88VyDd!>N_K;p(jC$?8fS>gHMu?f1ZdQzQ^ixB^&v#`;hM8FYrCKs0Mn! zF5$#cw^+(9&Gq=PpA*BZqc37HJ)%O_3;6p-k>Ln-K6PXDwz3bTAswi@-Iml_rlZd; zs!sa84tGxL!+M~0;DgKT@=0#jn3?3~BG9P2;HFQ=8NVhkDtjd}plKc89Q8NU_TA9+dFvW zCH=dp$;3Ue9jVTBq$aeK{-16-oxXu5F^~O5k@gB1^PMKW{S6CM37hvlyPqynOX-Le z+lhslO}EQFcdGdT8&eSudaX87T^d0qG#wl@7k_XFb+|b8|6HQR^bxU8cGjhvT1ggs zJ9)!$c<3wHg*FXIJmtQ}UfiRzZWg)iQ~Fo3sdbQ_lkD@{$ZoeRHs1K`IXarU;t`Z! z@8(ce`zRWJJu>8@?%9bwS6|V&Qk1<)TksY7kl}8@PN6|quH@`#SO;HCr#s5Wi`@+- zI~~PG@a(=N_TR&5?O|_LZTza$>|{av(dSW?O4lJ|eh`qV=zoCJ3_kxCPp1%`=7(^) zz`D#%z5&?5V%VJ(bUB|y9)6{Fw*drGpxR70L$Q{}Am;7I~hnOn!Ymw)J!KB|1Eodv-zd)~x?0 zto~Cx%AR<}vIo+oUnCM}S>f2L#2p=>)=Bv2DSWVliY(TWwQoibQxrSzuAo!?rJ}e5 zpZzbiPbS?&Z=$Z`LqF9cQaZ$5GEI%~Hr+N+NZl^H%a5?ubJ?wuQJ18W)sOh&3H({n zp2UXrA=m9=-`Z^=@*`;7UhGa9s5_EvUMrOgGURoj%0qBmO_x_^WLNq%YqPKPD&1Ev z$tf>jhn{qM?qDa?AmpL~c6J!K?A!2j2{go)j6Rk5uA)D>E}mpKUhXD%wh{eu$?RVA zt)f4!3!d6eIIXfy~mx*<$dO zEAgtv6VmfshIk+XTYa8rZwLPH3B0>2Sjaeh_%uX4(#?>C-hOqxh}2!0RFLWMRgt%N%O3Lb;fe#oxY! z#IFzauS-1Q5?AI1>zw?g!1tA|qBlJI0v*K;8fFvA%(AkgnHWRPQF#FLD^TesGkd@| z($$=b`?7FfEUy&2ufh|V@WCaDh$fyEFaAEXxCs50kSFTPTHm4?f1kJ}J^cL~TXqN^ zeKh-MtAoR4JohJPf>TISVYoOY)XK={Cy8_xu%6SYtn2|I`;Pg(4v|q4yuSLx5qTI- zIzDd@8(iY&0koG+3CU{888I)oa+7ZbcRA}MBh;ydhVH4pCU&TUSDrsK(jj2&;bzl;N znnk$3HCl8qb(_!80Uxt2Wmy+$Rp{xHtlVB^JqM|pfS+Cm`<4++-Vyy&fjQp5_t=U( zIEp4;!PAAXPLfl+2qhj7DX$}@*^I~a0=*%9^e3sANN03b=2UU6HxHhL4XDMN3Zf0l!Z}<5T?J z&YHHSgDp8HGf01N4K&|)wBTYi_gZAVCUTyNu_X^B`P_5NH-?BQ3@aktYf>A!4qsg+ z%DV`>!$d*@(B*~cJCqzt5~O+tG4gb9-AdN74{Kc-JX0|vi8!ypThc)+Ie)=uJ9Oi8a5)%UkiIKM< z?;H4i1Grb1^--YiM+!&L2T}&!Sindx$VRqeMgGJRokSDPVTC^=nye1z7epd%;Y$Tr zqae?2VvQEV-v?Q(^Q`!OUh7%&zF4KQ$&#pteZULG^(~=pv#QW7~lXlpX;;hUWMyZM&X^-{ZMUC|yf2X1UD#H0W z;EdEnt|_qr*P-4aaM?jtWNS#voq>+BlVc-TPeA*0fr1~yqr1^&9~1A-b^vh>yU)8|A8JFv!_1`)5C`Gs z9mRIuWQ}K$r)Z2ODTRc)!ul*V6INDFn z5BUWQuMo$cB{m$#3e`uGOHNWUs=_rlfwvnR=K^moRNKx=IOrgw%))~7qh{KU3hEZ9 zBVAZSSd0DiLtTe!R=}0*;Oz?NzDRiS84~dp!>1RC+G9kLNAv6WIQ zl)hZ4aGwI|g%AY~gZWXe+y~xEe%@iJrScYqrV=Y8Ik2QyG}(2L8Cl7VROiP|X6JJ{ z)=18)kp5Tc$9%%`QgN0FxR*PYj@OsyFR4*WOUf?I<6&AuT;Ve?G z;w(B?pugis&LDfw9Odja@2ceY70*lkTBY;Dp~e zCFYsk7&>!c29eWsIBPRL;h_}ZKJk=phUk5ccMr(Q3b$Nr!)d9gooZkJSqqnW{7R#)uT6V~Yp5X6GI;2}EyITT63O1$H*w_7TTxr&dI zh9?VwUj}F*l}qVZ&W;5;3KbKPDL%t$Jw_7Nz*F~tbDD3?GDapSD7A?~$YD)zDFB}~ zft#vAh1`4=-%Dx>=R+%U2430^gwtq{DR@g;kOk4^v*0T^w=)q>btl*LH~wya^wDqF z*Lv9L*I@gcXE&h7cEOWUy*SM(OaAy2RG*9%{ucdMgP$>2g@feXzQe+tB0@Tj2lfWr z@Cmpma_`6K0BwtxupEq}_c$4}Y(y+tjCFVmXWm5jc0)6^;fEc!z^ui%ed%9Wf~LJm z3~upO7NIXILZvgC}h*y2Q zh+Jd~YoW)}5pnJzmP*b}RKE_>Z=qo+IA#y@9m@ChnAKEpd`FZ&1MGX?6*grKG2AhX zJ4WI|R0qosh-#{n`}>eJpHFmtooLEXuue%=&oUq;##5byPHsn3)rwi~fX8a1LC31E zI18mKJd}wF%S+0I>v}{{@(LWwg>15WW>8|iIa{~Dc_2RwT8Jb0b*JFss)T$56>hA>r#^L zyfAi`WPmgOP+9e5&I9|B3`;$B1@A;-WTG}ykGuw{RdVHDB4huO7kb9YFssQ&9A@67 zu)5!|vQ5|*{DzL-6!syxlxWUiiQ`$j5}h9@rR*V zvS1-&$+#`$jF^jfT~hz(&U_01?}bTAG_j8^*~et9_pz(xyedde>%DHSHTj8F#A!3h zVVohuE`8L6(4uLOl?|!{S;t3smn*pE79L3zvUyQteIKw-<%;=-tnVpv(YDZO^+mN_ z*KyXtIf@>$52(bZqsQ(wQTuP?0VZHc3c%S(IbSRbBff^7spuzYjoc+BBfb&bj*qJXUp7CX`oPkx(`N! z;UoHi*OARyO*VcNkjL3ff^`V~I(9rQ~fdxLq-CyO$WU1rnhv%YFMNs6rDFm@u%qW60PCvaqEA7(Xj6!T0i z_P0HN-W~B)e`3CabxrCZJ?T|0ghc;Ev~r4W>>tR8R-{XK8ok2x$sPW|$ta7cGtA+{ zmt|xUQ_z!8mA=3}R0F0#nP}p+?k7zntkS1&UKRGPe@^XT51EcKoVVDJ zz3+dpBPlN@$#f?tGTtu4TQ5chLXib6%Z{@Knb&R}bB1$2kVxW{8|3S5aP>xsl?(2TVgHjKJ@gg!Jw|;(Msq&9`0`L$=w<#!#}=g8GnC%@Tdw1#@p+OWcNm*51ag)txyPg?GBX(vMaNSUdX9G zC+TFO1vE+)F?Dv1oHJcTyY1#wm$l<7nb8we5I9W&O3A*Z>FB+Lwipn zTd`Xu(T!N=v2@=z#R?rmns1|nn%Qvl7w6F+*?-_s)qv9NiO~wc&o{80dF@qX?H_7p z1Mu6eorv9zhM!wAuhvvv?qdg1g456NeswflYkF*^1f1re%OxuP0X!~Kp!P2$I!^?nvKF$40vB-N*(rY-pD!_;7u$gN5^v}%J0m)A zujGPff$MxAiYHnfAD{vl?7$;UMQ*4Ok++;vbdNn5x$zA;P^Ff=@)3;N6u7d_Ui|M8 zL~;_P4dp>Jvk!qx0j2?SV!sgUwEfWbP-%kSIekW#{p(dXYsQP368iR`?aU zh4py5n~C}iz1jt#e+3?4z9XuPzv+_M16 z6(XlBasCZppC$JDfGEZRbJ@MO6hHVjb=3Q;N50eH8HKJoDz^HRMH|8YL5X0$aa^gT{)?rs@&O7)%9nh+y zkKU2p_y=Au##nDy=~Q$a$vF$syCeI|1=4nUQ3|TntSFs=_~NqjUrtqchKFl`n4NEh zE8j4uWYk>uB@|CgQu$xXN;}+N4}No)>vN>$0zWyZsC@&UO1ISmZKBi!WJ2bA4c?R8+Og1HvJQK=eg}NY zK(Ui(n1*kOl{#y_B4ic>xS>kcw|9sFS z6ZDX4N#IJ!4QNj+X4Hvf~%Q`!zU8|-sW`PXey15iGHSF!=sV9nPex+<13`YGUvj^Kf+^( zA=22)oCfj4E6!pahz&2#&XM!P4TX`KDR{D3vDI>>Y)@pg2CG*CAEghrY%1I=C#Uoy zqPk7jPBgXCity7Y*7OZ|x(uB2TTC}5jy#NanFYQ|tiGWB(3dEw0`bFm)_fsaZWZx% zQKH#xW*J;Ihu!?y@IZ4=M@&h)c!3?8c;wcp5o44S1*7Os?W5>is*2^^y`PrNyV*PrmRw=y?Hu;Um1bn!x#kc<3G)w=kBr z1Mnp$!kOg6ckie*FC_+8fwy$c&Lqe2glbD(Vt`WmI1yhjtjJ+}h8C<~Q}Fo-SjUiz z3|QzYSkN4B^ep0zIC^@{o5{%D9<0$4;>FZDH83jDKk_H1a7u^ZFQM#nQ)0l8tf}Ox z3*sr&LV^p0G}*UM?lxyB+%Oa2g)7+3MO3K@P&KVc{Wp%>ID4u&tMf)iYD4UB?*!!^nZ&r&90(8HmEf;>EzokfR;M{2%jdH|F&*)RbL) zf1|O*I@jVSH#+((+?zXe>f}SypY^cVpW$R$^3n@YoUWEvRBONE-skYh7f_}q9{*%$ zeh^&}i8iPSJ|&UK%Xl``=}KCUFIk&0z7;RyNyrLR2B(VXm3w$19f&9|;3ubHtUJgrgR)ae&QU7F+{!}hM|j3vshK@6huF>9AH7~3xtmNC`4Q1+DIh(^Gwpzu zc#r?{3l-NE_@2;;Wc{9wO4K%&!le-xx@%k?}LgeGFQM z-ZXqnvGGfRG!Tg&j@P{a3SUN-i#98QtjIYxw~?H|XoD#Glz4{i*@{kSN#-dQJ@FBI6NX-xPK0tE zdAY_)9wm01i634BF8>QZzYo#b6nJ4D^pcu(JQg||*kwoC-{6iR#H{(*0bLdyaS$o! zPo3!$(9WP!cA;gZM%0QYZo-58S*hM&xCU7FkUgnHCuI$@@*`e@%(FIlWkQ2ZCSIt) zvs=h5o(8M!Xo7jv3_eFwr$fSihhm>F(@Dt1B_PQj6xnMdSzM_QK0*6`54Eb3>8i@T zf3wmZ$eZpXn{tHg$0BAu5@^kl6F;_P8a{Y8F#H|+@*3@u600QjCpiI0DyV0{e*n5O zFRR*|yTakldCc@WRh%8vaK5KUtvk>vF!}|es4wAf_MSqww@8qjb|L@2n76ma#V-CG z$~P4!&sCe%x{MVZ1-))yV@?oD%|Ks&8mfe4#B#1dOLt?o-|+4>cD?{IpC2m|#)`ke zn%qL;&Bpdr3+3wCA$u`!?j|_HK}Q@5^+$9fhw}k+eu2Iojf^+t{uyY%RLEmv>KoGG zF8|}lO}vJGnd@-YH7B{DoJjmRe6vj|G0|`=`ww?eXX#2Vu>w3Gdu=8X$5vs*zC{nF z1d^P)Tm$^11N|1c(gR>U9$w2!tnLz9|Av3Il>ALYG<+_7lQro|%vS{Zyk+F?u@(8r z=ygDXlHu8Wi>>|wD<(TU&ce}adA=VsOHHQip(%j$WrfQMkmnhJ28z(D$KX$Nu?l+t_g z|D>n+9GYh&qo&1LMM0T8_<&2XQ*Ypa!L06mY|<3Cr#B;2ASZN-{+y$x3KB7!exEZs zxpS5NsQYAwV&L6B;oA0eNWC`KjB6UBJvzZf(jE8&JD!g_er2@2ta)=X<7v^{7s#e? z3Oc@JxZcW124(1GtDt{@H~++|X$Q^9V7Vg5nN|mryhu$}PHC!2X6`F|*$H?KMM62` zI?T1G^@kuv;#q#7x|4J1!(;Ma#4_kb>C;k-QPM0_9pL|5dr$AUmHh=P!|*WXv%co=Y40 z7zYvuG^bbTwTUHrJp%o=5IZ=K(X-)uwT0Jjt2erx(~iv7DSWz?Xu-^Ak$%jn6I|CE zZpcbrI49jtpF-cE#9j5VD^E;%{EjhTbO?>Nn$gRG^9?hJigP1fo719dIbC%tP6>)4 zTT=!t_8oeu9=^^fe9UQNXWQs@oRrWJtuU>aN<}qGKgOw zPqpB{53wq*)HR*a*`aq3e^q8JOQG9GK;5qy^&P(SJnBX%`TtaEnn`Y5P8z#vSL!9c zW&VS{YED5lj{e#-oV%3@Jy3|%xy=8)GD&~HNoE~&70zwCg#TI&EwvH3+(DFB8S5|y z?K_tq;>e_& z@FH7cr3UFJvc5aW_m;O&W~)2dET&I35&6EhXv>TtUv(imb%ZgTfx3W%eU1jd#Q6#*b&Rt}FXFW3 zgYGw?-Do3xiFKNb2yx{5=p9^dT>QXU*!teOJQ5lUuQoG(b4G6`V#Sx}<+NmBf5a+( zgO|1uc-7HBKLT@vc}E9x9`mQ}=ie3fMOYBH4k|?&C`qKLdhYxG{@(uQ zdF4FgcVEBj`d;7h`Ht&*WnQMe$ME#F>tFU~($Bp2IB8uKFPcR@;Mt_S=nSx|){s)Ch6k3z?&HPWFgaHLxG6U(IxvB9<$Mo&#QOSo8 zBnqEeh8sW|!y1f5bP}(uqK85NP8kH`Y9jW*4 zz=9c>BJlEH-BlW-homn{zLkUeP4(<}GV_Wvm#0AL+wIV3daWZFiwtLT|DUffZ{^Ik zg4vla=@;_{73z@RDY;W8y$1^x6!exM7?CZdgWjO@e$nembksC9!Se&Gx>C!3bCy2|0>J>%8XC9%3qd$TVB1SLiTBId95Ht=l$1BHNT_h&}0m> z^DRB|XD$md39|hSW~&G1R8-mAoJBejH}w~zAB5*`<$o+o_12fZTK>c7n^XwglbKbp zxu9dF7oOZspPT(O zIVo>xUgf-*Nfl9`894S-=PD*=-_+G^yt`wva;4v~DfQW*qnWAnYkw81&%seW*{Ix;=-tNzFZXP z6CTNhFi}r3{1z8c+1dI}cEQI%oo_KCLrM>Wh= z@_r{I`aREfBs_IFZMg)tD@<$bq~q?Tk@rFrAMgN%!}Oo0ZX>fVB>m8PHXAhu-k-tF zea{QI0Dhn7w>QIKWq9fjvZt3DbAh0m z*eq^?30v|`t~ImzP<0JESxMw4AFlcp(zzf1&Gc`QnMHr87g>?0$c>15RBE>O`ObXM z?oeeX*sg^co`>2Omy|d~}G7ZnIIJTM}wL zDvDYb8mArY~@I(=@!xP=rr)PsNouC7QYtJ4f<|Z#oSlC^Jz1$ZS`sU z^@2Iom%RuzP!gm=amK;H*Q-Q z>-<6+QONi4$YgzQUzJxGECPNaoWI_BqiSc2*lIhR+}bU4cah;QV6;BCXERzvm+j$B zv8+V7dE&v-aKM>P1(%>1ZlEVh7&VJ$&hTU_)T@mS52MNm6gR`&84GBMicBVO$Mo}C&hi}bX7 z$9}Y~!y9?p9`*3H*W9DGGc_o;R~7kVU&>{iOoAqfa}QD>GQ;zo>{k`L8a;8FSz~yX zb41?XmBE=ui!RVX{zfSG3p*Hh4(_E_PmsI>{-;de#?W}tRx}}o{OjFU}1thGR z5$DOptT)bQBJz>Zsitp1R5zDkt$T{Bj><0AQJ@D8Y!Za4x=MU;7wvH)ik7E27m1Yb zphUb>S3|FH_d6KxOD}b zvKYTFB{yfH)IlqHz<;kGNs-x0j2eB%y4l3^vCHdIyAEl$~=sd95>i))S zH?S28XsF#J|5Q}0;OQwM+K-9f*Vnqp+!*4}36(EnP)DL(U|@lG|*O2&-d0xA3*pNzV`==XD+6+~9FI875BP~)t6yyF*d zIlCX~noZH7f4%{Zv+jX_xV%4|>XI(eXaI zqeMP4-m}k2qDM>IkQ$vmVTn;R{5H+~;ok}9BCEoiR+zB|po zdh|tfSqRQ5u(HTmZTGbiE?7o(USOLa5^_kiza%L$QjJ#eyCX>8YOE zhu2_&YvGfzEc#bw6Fn+ErkT!zi~m&7^ex2Mi@nN-4~$6n){m|!gd5!!0?Qub5#5w| z9L9Q`#`=bLTP-;%zWF<}HOhKNiZ+(<>;g=!-{gmK@H zhnR)4*0TSLpr#(`Znmnby28rtP@#C1DEuz*f-WrFCe~>bY+i*Qf2KI|yU@dDBKTcJ z%r|9jrt^Cy7pEtU4p4m)>!~KL^Orm5&T>=ZD7l1rWaLpR&&$>J>Fi04n?fMx<@DJ^ zZ(5akN+hEu-C85LD7`d2CB2ki{IclEE;*$dVi@)LAcJt)C8`#$RK?R1a_`4h_7O|p zEEdobKjyJJ!_d1Od3;#b?+evo(`7hb7jsyfyqR9EBX|3xiCD#YeMPrq>Y8mAlz5J& zcvA)RU1DS9k|*f(Yecd>NAt$w=7+>gZ^X}6(=4Cy{`Tn_Jty;|9N-*YtauECzQr%#jp4ZEX+<>RA5s4@#o_RhA z`8LVpUDP&qnfEycAH$x7;@iSj`UHHlwR!eB>VC_uWhWeyPGqQx}|7kayZ>2Z&mui zq)uvslb{vNxSZ#{CGD-P{8zmf&rM1v*TdH%`I0qo%6Qr!k0lu(3Vl6~YZI^e2Q`0J ztE~IZr&ncDD$d0)yO zzK@TXknWl8YyKlCn|G6&f*K?fv+eZrZ;kGAj9V8DfLYqq935UCDgzqiZjq zH6MaJP9?dg&|>GPW#}!&Ul*MMvmBsFJD_g~aoTU4W!>oH^8i$Q$4ydaQtqj>*}qwYoEK!k5Nvj87WDPM3K@RT44Xezq1{O}TU7Yxj;e zB0Fod-I6=f?ec2peUrF>GL!FAeu9Q;SlpAG94V}pu5dCMO6h9f#?jf0*^w$dyYof= zhZ+5UqQjSx&SvCox7j9Ql7mFPr??a0Xu53Pfb_$u=qzxH{PS3Adz9t7!`()Gk_O2n zeqWr3odDHlXsX)0-#+BQZAmo78+?Vq=G>6Q+NcX96L?!Mra52XPM%WV9Pe>>z> z@K(0I+X#M7PfK5ttRqjy$=ub=ea+JH}M zxgX+glUz=Imb}ix?D}joyA_toKD^qPbQy> zjoqs5d<0zdA^NTosroDPrJZVHy|Y=lJu>e7`Qx)7&O$tfCUpNnKK11&5>+*|AilR$ zzkj3;fUdDJHYd=Gm7$VB=`k|mpNWRlS8H7^`-D~Xf^|1Q6ZPdF3zO<+NJKHnwKl|G ziA3BF$wduQC7QSk@3jP9Ml>0`8XoD5vkPax#9?Q`33FsUJ~XpV zD!M!Jv-)LEhZAlx!ueu775UfivJ?|#C>P1&oX8J4*WX_Db-UgAnUB)mYPP^C*V&6p zN$Nha)(Wt~L!@kxlR;Gde;6swN^2BvT`ufrjH= zx!aIi)y;gF6{1#4$@@TDxXT)DMvI!XY@TZTH{CkF0~XmL$5+?f`>~AO$Z3Who@ngJ z<~(1X<1+|$8j5s=d6#=%!{ARe-YQ~rlJyLP#NVP<8nLwpAkJO5Z@3%pO3`z_s=)|d z`5lP~ytXgX4X89(``PH%?`KVAE zUtCH~JM+J08u`1-rs;p&$BP3QR-eY6sK8Eq5&#G=8>BpnEFKT;SNyDxSZB$#0NNlM_SX_0f${SpP?kFtAGjopc}H#zKyli{|us9KfYya&cwtQLL?y0(NaDw4TZ`COf?|7TKouaiiF zSoGGS7aLg7A;!LhSN4=$yb_O|CX;dvOg<^yCOs^vD}J~yb27bjj=x>U+qz9f-$9wh zt~jA69r{&n<~PxL^;Li^R+-SzdUins^E`h6n%rpIE~Kg|%DDehFaAYtS>G&^wHgN% zX5~8Bna82-pZOU3;*5`(MaAnpdvqLNyyx>~O?<7t&`v=&doNZIN|d&lhZJ zcgpJ@(~m`1!%Mvc0^A1k47RrW;G-_8QnGmPCbscow&5$gFkeN-5>cDS?d3ISwT`x0 zi&{6^$1`|K^>IU85&S!{OWn=$clxXJg5-C7mb$^D#bAZG;{GW%;1rptJ$CXw6tra1EM8chJ7_K~?>CQL3&RldhNBUYY$E3Oty( zlrCCqWd}qXH$Z!9Y5Wy-YmSv{g**0(?zU#VuB2(>r08x!e_5%%gcFv$R8}5=*%srz9uuyymp9SRG{_N(SR%P!ZM$^N)+i& zbqg2MSaGuXWLT%8*yJmC?jq7Ukew+Z;!wiub)8Ce&_};~-s;xP|XFT_)2?Nl(Y;&i_SE4(_D;hkzKgP->x&;bNIYfjo%qw$fW+2emnhr z`rahp?OQc7y=j>5A>WXbwruSWY}hC|`5JT_hZeVpdbXiSqqAjm8oMdg(4$Qw~`k7i;X@WJ-Xqvs9UnR9t6Li;Vs0UJXs%j@b z8I+D(WIDM8-tI|S?=;&A_@yGucp8qoh(G(a9V@A#^(b9d1+P^^iJ!79=>BeQ30aa9 zO5MRfZsNV$NnID(tr`lx#}n8>ewWzm(?zw)q0l9K*DW+(X`XCPsNptie}gsJmZ@sa z^=Q2MYFy^0FXhkoO&-a%&3s?bH`878`C|S=LK6(MYc-(C!7NT=HfJ2YTne>Mf;uiR z{v~AZW!f!HU>0IE%ArEO>Xa(5;}rh!C+XSAZW-ev^4b+?;Ep&mPR@5%(N`1dcmP$} z{J%GU6yAtV_1%np4q9)Z`Ra@Le3@A;(soE(aI`GJUCDpb4fFn;z9xC!xE0`y6IE=T zEUsCV2Ydq>q~+YNg3h9DbSim%2@c&ucMgLauI4d3Lu!i2=I++RZ?mf5LUhl8RIPMX z0oCH^S?-MQ=O(@1GI8(A4*EMqORQH1Ts3)`7d(_?Oy^6Sg66wX;~V<34-4CoXV_G& zz&#?EYtXlvobSlg=;U2o_D1^VMBlf2YAz4IR7SIrg+ot2A3 zO~heWsmEPPBhI3=&u7=>z{y|1zO8r{qj6aV=T*Ol+dQMllV8L#-UXq&;@so-q+Rkj>+mo>7zRhgN$Z!OfV1J2*?1+7Zm1=iHXFW- zZf`?W(tMcgl9j(ozraH{z6kzkiKq98SH=wxzoZ5jwRSQQLi~olc@~d$CZ{)|L06u| zbg1+bA#hKyrA(hrLq&SWR{CMg>W|x zc-m>n^2y9(ih7k!G|5M7)fe`7AM9KXl6wT#HWY_j2PqUuUh>8baOs0~>;`(JFC?)E z-RoK33-+Tl|7K6-4mq7~L_RJQ4{Vy`JBvL-<>okERWZ7_r%x`F*XYaU>=O5XffrL% zOkp|i<_f#j$M~zgb*Cu9V`TJfpDXE&7s6lXCC!pjNgf|>t2yQI^)L2`IP2fr69?$q z$I$Wvm}iWgs!YGW0J{xkF>i!Fp2iU`LJF6&V5h^xB_Xy0>~%$6MJ2xU)AU5#8$Ff= zYDWjVsgV`yfT~aMOCMnq+tZD^_@a}IItI5+;vYTj&9|C$Cp_~I|M^Z8)CXyelVmg2 zv83H`$Jf@@orUXV?0I&hl2K=|VmINRzIHoq{faJR*ZCR^V>~YAHV_x~WX0#mb&WKG zA9?yGTKzXxcQ=jsjol0{c%ZdMjqxt~be-LssM=x(K97#!zmn5wo*G6UTuTlvvZ_`r z*O%~_`)^^OnmQNkbG~7%)it0=-}U}JDAd>LpMf!|(97LKHlwRkFKDBlQQu=nXBx8) zJ~_>3P3RQ$GipxH;4>6~HIFmy!PHk~a<);IlEPbYVHdM~3by^6-nh`Sx1d$@?`i=x z@3K=1&9#fYY-h(C@fy10lI=3B8`QRcEM`zaE$bh!Uw=qvC+s%`JqPjQhr(}hKVDs< zUVs8udE#6WR03DMY)?kwy5?lF0?aa-F4O@Q@|~m>t}#EPEFa}wGi^=(t~P^_kXcu< zJ)h24M_%IA>%Kg!&U~J8M8yt>4$i^D9jv;7Xi^J^{1$%mEN_297VAsr7fw&!z=yN> z(OEk)$8KJM{#V1-U+|6M)~=I`)|)i-^!*8JGt6@lv*dRbO26h%eqD_PkFh% z{rK@&PYfc<)kw%J@_I6vXpGbUP4a%moonp-qh=EK0epZ{p0l!Gr?;i7fQAPZEqdv`lzQ$)7$d13qSG&(Arjx7Z zP_77@A6@3EqIlE^)wKTT0`{`^4krJBJ6|IO(MP16EYDoK6er4-K-5pzw^ro-bE7UG z%@3gKJNPy5^j?(?O?|cWbuu0;PLf}>i^K8AS>okik)<8IgOO`Hg;r^6)RW9~AFlYXw~pe!oCB|}Bdt5c_A7{zyp*fh z{K3hQsUjYI>_AOO`XxJbyERRawR|lXpV%+oeirF$39-ce&=p~l)o62@o$3m0vXS_p z63M8`=Da0Fwvi{XiLQ9X&K}Q``Y*rlcG5A_PCZPUEVKT&KeUIxck{2aH%D#L5*lSJ zPHj*A^Q|d#>P>iR0=x4G8?i|=;!(cCMEhC*f8B{whueejaW|ks^ni-JZ z{{`hAv{T(#>xQZ@mz%?4)jZ93QhymUW6V_YFuA>ob=qn72YI@m=dWdVUNh!z>T?EJ z|2_Q0b3_g*$wr1>J{E?#OuV!uBsG~f8zM`1o;^Py*_*1G6k;deCsPx64XaW0d^GBc zv!NHVA3dph=3?~sTTM4Jt|NE<9jZW397^4mc@JkV%6A5_y=)Tv7Ec`^KuD46C&=uYNyNZ_Rq9W;ef9uDFR;B$TPTEEm zpaL4EWJ2Ra*-c3mJwvvUul^`jNWD~NG&q{6iwl2iv1@D;ybIF@jDwn%STR z>$B`Ra#h>(XC5RHy41+s#oyD6_ml{PtoyH(RLf`^$b5(hyKIUKb*ZXxh45je12l` z4E~%(8^$V6S1p_J+`Xir9!)og*89V{CO~r6vP*x_qi3_p>LOX_P4?|e7H>Y?(jxR_ zX1?wpTe82VmO-#bQ~#Ey=?Ntim`z)pUnskSC;kvrpNBV=@1+z}6WRei>bRUN&xLw^6@ACtfK>&;_Q5b4|ZwdaxbA{j1rO zk7=q6{@%>!%W%facCrpTyb;%LWz%b#=_>lIm6*Yr%%)5oI3Z7;>2JwRhh)CWgjzm1 z7kBQHyZ9w@Br_PoZcL7Tf`Znv@iprli=7m*EK0}>6w&yuHyXBexx$*iq8exan_YEj=woX4E;OxONsMF1O8kxRK zv~8X@4TV>0vOk^0ZHANj9Xzsu{KRuvs9vd`lL2br+v-y@5AQlt%^SW+M7BWGR8MGh zS;aHG((c?N`d>)J5hD0fxnay!$DwJyZ(Z=j3L89cugFhJbZRMz~?BLiJgX$(DcEz7ikE+`{I zezo|@G&LC0Gh@uKNotl;wT~pLWlDD7=+{-yE^rcfKJV{8nR7)Cy1)o$>4H>CEM$Wj zF2EPH#Vw9ydWx^qg3uPSQIl}@aMfc&XpF6DCN4MnGBqGGQorkKy**i(ZkB!v&oxSw z%HC;h)5K<@nyCZX{E8e;l|6bGzaKMt84-#+G`nVJDw7k$LjdA+9G|vLN5WP$a!GQC9{smU%PgP@` zy?F>5beeasruFVZsWGhERp!~Bb(uw19mrNpeo+y!p1ofO6W7bxwgIU4A92MoXnX~I z^$;Dg-+I4;(jMjyd`r$wf#n9}@_Kb3zqO(bBLl4B8s2sj)?uZYMyK3YXsPJTe0~mzMxUYR(EBJ0@Q|-z z^wdo`?ftxrMEBHHdcW;uMcxoKSja|nL&I0C-~)M#Icid(tL{K3d>763p|?h6c^6K- z(D*HRyP=bo!g;e;nMdW-w6nUEH&RFy; zPoKt{SFs;kjo$zd_2B28Vf?xxgbPt-6$-6^MH-ny`B>x!p~`8iH@NB{F(uyphkU2Eju>(@+hFbMQ0M^a?SP#BFmlvCmq*3Z`HR(z zb|MezAJ4I{_1CceDt5M>r&4Hqz#8%(z3sf{1N7czb6k(hzsFr$XoBdr@&})}yt%}k zsA;&YntiX%?RAsP}~mpBhHa_mx_}3lTUO8Dnr)ehRYiEzOqq*N1{u9hSWqnB5L5HV!FJ~ zM(5#wA|hYYtwF7ts1O`bJV(vFeA2kNGERHNt!1avV1LlO(Z4PFRYfnMdY&xshUh#V z(iSI@D)8I48KsI{sOqo3usBDIwc9%?TF)`7s3Hy#ckM=(w5lj{7-foDL)?(MlMK`{ z?g1l4@1Ml155qpuJ!m~z=E0sN%|GSo=x@B$tfB_|c$z4>Qk5}!^ks<3?J}PIJ-0VO zqY_pS_wMZ$KP+#a$8tI$v1idKK4l(-Q8Df;jNEhyZ-{#vqk=x=nYfFtIGRSM;Fx=y zzp0I;DR26pr;1u{ z`kxNkA>VPgVNp9?+FU{tMilr^Za0E^qK{SZY4AauqBv;O_=-;Mv9c1L2rCumJ>m?< z-}bD85#sD#;)&3XpPKkYi}?mE;w(+v*_ZG8IMc&PMw8gh z*yW&hoVN)%4~{SHuklno8#p#}PUy0@*E!}BpN&0=ImQeWwwZnGb)21vo@H_7^QhlN u*Yu!uNiz#w7(d0m#Id&6$LRDIavP_2{*i*{{~k|7oGWf0i8W=-<^KRlddKMi diff --git a/native_client/alphabet.h b/native_client/alphabet.h index 166e10425..1cd5a052c 100644 --- a/native_client/alphabet.h +++ b/native_client/alphabet.h @@ -36,41 +36,40 @@ class Alphabet : public fl::lib::text::Dictionary size_t GetSize() const; - bool IsSpace(unsigned int label) const { - return label == space_index_; + bool IsSpace(unsigned int index) const { + return index == space_index_; } unsigned int GetSpaceLabel() const { return space_index_; } - // Returns true if the single character/output class has a corresponding label + // Returns true if the single character/output class has a corresponding index // in the alphabet. - virtual bool CanEncodeSingle(const std::string& string) const; + virtual bool CanEncodeSingle(const std::string& label) const; - // Returns true if the entire string can be encoded into labels in this - // alphabet. - virtual bool CanEncode(const std::string& string) const; + // Returns true if the entire string can be encoded with this alphabet. + virtual bool CanEncode(const std::string& label) const; - // Decode a single label into a string. - std::string DecodeSingle(unsigned int label) const; + // Decode a single index into its label. + std::string DecodeSingle(unsigned int index) const; - // Encode a single character/output class into a label. Character must be in + // Encode a single character/output class into its index. Character must be in // the alphabet, this method will assert that. Use `CanEncodeSingle` to test. - unsigned int EncodeSingle(const std::string& string) const; + unsigned int EncodeSingle(const std::string& label) const; - // Decode a sequence of labels into a string. - std::string Decode(const std::vector& input) const; + // Decode a sequence of indices into a string. + std::string Decode(const std::vector& indices) const; // We provide a C-style overload for accepting NumPy arrays as input, since // the NumPy library does not have built-in typemaps for std::vector. - std::string Decode(const unsigned int* input, int length) const; + std::string Decode(const unsigned int* indices, int length) const; - // Encode a sequence of character/output classes into a sequence of labels. + // Encode a sequence of character/output classes into a sequence of indices. // Characters are assumed to always take a single Unicode codepoint. // Characters must be in the alphabet, this method will assert that. Use // `CanEncode` and `CanEncodeSingle` to test. - virtual std::vector Encode(const std::string& input) const; + virtual std::vector Encode(const std::string& labels) const; protected: unsigned int space_index_; @@ -93,9 +92,9 @@ class UTF8Alphabet : public Alphabet return 0; } - bool CanEncodeSingle(const std::string& string) const override; - bool CanEncode(const std::string& string) const override; - std::vector Encode(const std::string& input) const override; + bool CanEncodeSingle(const std::string& label) const override; + bool CanEncode(const std::string& label) const override; + std::vector Encode(const std::string& label) const override; }; #endif //ALPHABET_H diff --git a/native_client/ctcdecode/__init__.py b/native_client/ctcdecode/__init__.py index cbbf06fff..e92ee71cd 100644 --- a/native_client/ctcdecode/__init__.py +++ b/native_client/ctcdecode/__init__.py @@ -178,6 +178,69 @@ def ctc_beam_search_decoder( return beam_results +def ctc_beam_search_decoder_for_wav2vec2am( + probs_seq, + alphabet, + beam_size, + cutoff_prob=1.0, + cutoff_top_n=40, + blank_id=-1, + ignored_symbols=frozenset(), + scorer=None, + hot_words=dict(), + num_results=1, +): + """Wrapper for the CTC Beam Search Decoder. + + :param probs_seq: 2-D list of probability distributions over each time + step, with each element being a list of normalized + probabilities over alphabet and blank. + :type probs_seq: 2-D list + :param alphabet: Alphabet + :param beam_size: Width for beam search. + :type beam_size: int + :param cutoff_prob: Cutoff probability in pruning, + default 1.0, no pruning. + :type cutoff_prob: float + :param cutoff_top_n: Cutoff number in pruning, only top cutoff_top_n + characters with highest probs in alphabet will be + used in beam search, default 40. + :type cutoff_top_n: int + :param scorer: External scorer for partially decoded sentence, e.g. word + count or language model. + :type scorer: Scorer + :param hot_words: Map of words (keys) to their assigned boosts (values) + :type hot_words: dict[string, float] + :param num_results: Number of beams to return. + :type num_results: int + :return: List of tuples of confidence and sentence as decoding + results, in descending order of the confidence. + :rtype: list + """ + beam_results = swigwrapper.ctc_beam_search_decoder_for_wav2vec2am( + probs_seq, + alphabet, + beam_size, + cutoff_prob, + cutoff_top_n, + blank_id, + ignored_symbols, + scorer, + hot_words, + num_results, + ) + beam_results = [ + DecodeResult( + res.confidence, + alphabet.Decode(res.tokens), + [int(t) for t in res.tokens], + [int(t) for t in res.timesteps], + ) + for res in beam_results + ] + return beam_results + + def ctc_beam_search_decoder_batch( probs_seq, seq_lengths, diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.cpp b/native_client/ctcdecode/ctc_beam_search_decoder.cpp index 179ec467f..99ef38eab 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.cpp +++ b/native_client/ctcdecode/ctc_beam_search_decoder.cpp @@ -31,6 +31,7 @@ DecoderState::init(const Alphabet& alphabet, abs_time_step_ = 0; space_id_ = alphabet.GetSpaceLabel(); blank_id_ = alphabet.GetSize(); + alphabet_ = alphabet; beam_size_ = beam_size; cutoff_prob_ = cutoff_prob; @@ -54,6 +55,57 @@ DecoderState::init(const Alphabet& alphabet, root->set_matcher(matcher); } + init_token_mapping(); + + return 0; +} + +void +DecoderState::init_token_mapping() +{ + for (size_t am_token = 0; am_token < alphabet_.GetSize(); ++am_token) { + am_token_to_scorer_[am_token] = am_token; + scorer_token_to_am_[am_token] = am_token; + } +} + +void +CTCDecoderForWav2vec2AM::init_token_mapping() +{ + if (!ext_scorer_) { + this->DecoderState::init_token_mapping(); + return; + } + for (size_t am_token = 0; am_token < alphabet_.GetSize(); ++am_token) { + if (am_token == blank_id_) { + am_token_to_scorer_[am_token] = am_token; + scorer_token_to_am_[am_token] = am_token; + } else if (!ignored_symbols_.count(am_token)) { + std::string am_decoded = alphabet_.DecodeSingle(am_token); + size_t scorer_encoded = ext_scorer_->get_alphabet().EncodeSingle(am_decoded); + am_token_to_scorer_[am_token] = scorer_encoded; + scorer_token_to_am_[scorer_encoded] = am_token; + } + } +} + +int +CTCDecoderForWav2vec2AM::init(const Alphabet& alphabet, + size_t beam_size, + double cutoff_prob, + size_t cutoff_top_n, + int blank_id, + const std::vector& ignored_symbols, + std::shared_ptr ext_scorer, + std::unordered_map hot_words) +{ + int err = this->DecoderState::init(alphabet, beam_size, cutoff_prob, cutoff_top_n, ext_scorer, hot_words); + if (err) { + return err; + } + blank_id_ = blank_id; + ignored_symbols_ = std::unordered_set(ignored_symbols.begin(), ignored_symbols.end()); + init_token_mapping(); return 0; } @@ -93,11 +145,11 @@ DecoderState::next(const double *probs, full_beam = (num_prefixes == beam_size_); } - std::vector> log_prob_idx = - get_pruned_log_probs(prob, class_dim, cutoff_prob_, cutoff_top_n_); + std::vector> log_prob_idx = get_pruned_emissions(prob, class_dim); // loop over class dim for (size_t index = 0; index < log_prob_idx.size(); index++) { auto c = log_prob_idx[index].first; + auto scorer_c = am_token_to_scorer_[c]; auto log_prob_c = log_prob_idx[index].second; for (size_t i = 0; i < prefixes_.size() && i < beam_size_; ++i) { @@ -127,7 +179,7 @@ DecoderState::next(const double *probs, } // repeated character - if (c == prefix->character) { + if (scorer_c == prefix->character) { // compute probability of current path float log_p = log_prob_c + prefix->log_prob_nb_prev; @@ -141,16 +193,16 @@ DecoderState::next(const double *probs, } // get new prefix - auto prefix_new = prefix->get_path_trie(c, log_prob_c); + auto prefix_new = prefix->get_path_trie(scorer_c, log_prob_c); if (prefix_new != nullptr) { // compute probability of current path float log_p = -NUM_FLT_INF; - if (c == prefix->character && + if (scorer_c == prefix->character && prefix->log_prob_b_prev > -NUM_FLT_INF) { log_p = log_prob_c + prefix->log_prob_b_prev; - } else if (c != prefix->character) { + } else if (scorer_c != prefix->character) { log_p = log_prob_c + prefix->score; } @@ -164,7 +216,7 @@ DecoderState::next(const double *probs, } // language model scoring - if (ext_scorer_->is_scoring_boundary(prefix_to_score, c)) { + if (ext_scorer_->is_scoring_boundary(prefix_to_score, scorer_c)) { float score = 0.0; std::vector ngram; ngram = ext_scorer_->make_ngram(prefix_to_score); @@ -176,7 +228,7 @@ DecoderState::next(const double *probs, // that matches a word in the hot-words list for (std::string word : ngram) { iter = hot_words_.find(word); - if ( iter != hot_words_.end() ) { + if (iter != hot_words_.end()) { // increase the log_cond_prob(prefix|LM) hot_boost += iter->second; } @@ -220,7 +272,7 @@ DecoderState::next(const double *probs, // Remove the elements from std::vector prefixes_.resize(beam_size_); } - } // end of loop over time + } // end of loop over time } std::vector @@ -261,7 +313,10 @@ DecoderState::decode(size_t num_results) const for (size_t i = 0; i < num_returned; ++i) { Output output; prefixes_copy[i]->get_path_vec(output.tokens); - output.timesteps = get_history(prefixes_copy[i]->timesteps, ×tep_tree_root_); + for (auto& token : output.tokens) { + token = scorer_token_to_am_.at(token); + } + output.timesteps = get_history(prefixes_copy[i]->timesteps, ×tep_tree_root_); assert(output.tokens.size() == output.timesteps.size()); output.confidence = scores[prefixes_copy[i]]; outputs.push_back(output); @@ -270,6 +325,73 @@ DecoderState::decode(size_t num_results) const return outputs; } +std::vector> +DecoderState::get_pruned_emissions(const double *prob_step, size_t class_dim) +{ + std::vector> prob_idx; + for (size_t i = 0; i < class_dim; ++i) { + prob_idx.push_back(std::make_pair(i, (float)prob_step[i])); + } + // pruning of vocabulary + size_t cutoff_len = class_dim; + if (cutoff_prob_ < 1.0 || cutoff_top_n_ < cutoff_len) { + std::sort( + prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); + if (cutoff_prob_ < 1.0) { + double cum_prob = 0.0; + cutoff_len = 0; + for (size_t i = 0; i < prob_idx.size(); ++i) { + cum_prob += prob_idx[i].second; + cutoff_len += 1; + if (cum_prob >= cutoff_prob_ || cutoff_len >= cutoff_top_n_) break; + } + } + prob_idx = std::vector>( + prob_idx.begin(), prob_idx.begin() + cutoff_len); + } + std::vector> log_prob_idx; + for (size_t i = 0; i < cutoff_len; ++i) { + log_prob_idx.push_back(std::pair( + prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); + } + return log_prob_idx; +} + +std::vector> +CTCDecoderForWav2vec2AM::get_pruned_emissions(const double *prob_step, size_t class_dim) +{ + std::vector> prob_idx; + for (size_t i = 0; i < class_dim; ++i) { + if (i == blank_id_ || ignored_symbols_.count(i)) { + continue; + } + prob_idx.push_back(std::make_pair(i, (float)prob_step[i])); + } + + // Blank must go last to satisfy assumption in decoding loop when merging timesteps. + prob_idx.push_back(std::make_pair(blank_id_, (float)prob_step[blank_id_])); + + // pruning of vocabulary + size_t cutoff_len = class_dim; + if (cutoff_prob_ < 1.0 || cutoff_top_n_ < cutoff_len) { + std::sort( + prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); + if (cutoff_prob_ < 1.0) { + double cum_prob = 0.0; + cutoff_len = 0; + for (size_t i = 0; i < prob_idx.size(); ++i) { + cum_prob += prob_idx[i].second; + cutoff_len += 1; + if (cum_prob >= cutoff_prob_ || cutoff_len >= cutoff_top_n_) break; + } + } + prob_idx = std::vector>( + prob_idx.begin(), prob_idx.begin() + cutoff_len); + } + + return prob_idx; +} + int FlashlightDecoderState::init( const Alphabet& alphabet, @@ -463,6 +585,26 @@ std::vector ctc_beam_search_decoder( return state.decode(num_results); } +std::vector ctc_beam_search_decoder_for_wav2vec2am( + const double *probs, + int time_dim, + int class_dim, + const Alphabet &alphabet, + size_t beam_size, + double cutoff_prob, + size_t cutoff_top_n, + int blank_id, + const std::vector& ignored_symbols, + std::shared_ptr ext_scorer, + std::unordered_map hot_words, + size_t num_results) +{ + CTCDecoderForWav2vec2AM state; + state.init(alphabet, beam_size, cutoff_prob, cutoff_top_n, blank_id, ignored_symbols, ext_scorer, hot_words); + state.next(probs, time_dim, class_dim); + return state.decode(num_results); +} + std::vector> ctc_beam_search_decoder_batch( const double *probs, @@ -509,6 +651,56 @@ ctc_beam_search_decoder_batch( return batch_results; } +std::vector> +ctc_beam_search_decoder_batch_for_wav2vec2am( + const double *probs, + int batch_size, + int time_dim, + int class_dim, + const int* seq_lengths, + int seq_lengths_size, + const Alphabet &alphabet, + size_t beam_size, + size_t num_processes, + double cutoff_prob, + size_t cutoff_top_n, + int blank_id, + const std::vector& ignored_symbols, + std::shared_ptr ext_scorer, + std::unordered_map hot_words, + size_t num_results) +{ + VALID_CHECK_GT(num_processes, 0, "num_processes must be nonnegative!"); + VALID_CHECK_EQ(batch_size, seq_lengths_size, "must have one sequence length per batch element"); + // thread pool + ThreadPool pool(num_processes); + + // enqueue the tasks of decoding + std::vector>> res; + for (size_t i = 0; i < batch_size; ++i) { + res.emplace_back(pool.enqueue(ctc_beam_search_decoder_for_wav2vec2am, + &probs[i*time_dim*class_dim], + seq_lengths[i], + class_dim, + alphabet, + beam_size, + cutoff_prob, + cutoff_top_n, + blank_id, + ignored_symbols, + ext_scorer, + hot_words, + num_results)); + } + + // get decoding results + std::vector> batch_results; + for (size_t i = 0; i < batch_size; ++i) { + batch_results.emplace_back(res[i].get()); + } + return batch_results; +} + std::vector flashlight_beam_search_decoder( const double* probs, diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.h b/native_client/ctcdecode/ctc_beam_search_decoder.h index 2176565ca..eea2df4f1 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.h +++ b/native_client/ctcdecode/ctc_beam_search_decoder.h @@ -11,7 +11,7 @@ #include "flashlight/lib/text/decoder/Decoder.h" -class DecoderState +struct DecoderState { int abs_time_step_; int space_id_; @@ -21,15 +21,17 @@ class DecoderState size_t cutoff_top_n_; bool start_expanding_; + Alphabet alphabet_; std::shared_ptr ext_scorer_; std::vector prefixes_; std::unique_ptr prefix_root_; TimestepTreeNode timestep_tree_root_{nullptr, 0}; std::unordered_map hot_words_; + std::unordered_map am_token_to_scorer_; + std::unordered_map scorer_token_to_am_; -public: DecoderState() = default; - ~DecoderState() = default; + virtual ~DecoderState() = default; // Disallow copying DecoderState(const DecoderState&) = delete; @@ -55,6 +57,8 @@ class DecoderState std::shared_ptr ext_scorer, std::unordered_map hot_words); + void init_token_mapping(); + /* Send data to the decoder * * Parameters: @@ -77,6 +81,47 @@ class DecoderState * in descending order. */ std::vector decode(size_t num_results=1) const; + + // Get pruned emissions for each time step's beam search + virtual std::vector> get_pruned_emissions( + const double *prob_step, + size_t class_dim); +}; + +struct CTCDecoderForWav2vec2AM : DecoderState +{ + std::unordered_set ignored_symbols_; + + /* Initialize decoder + * + * Parameters: + * alphabet: The alphabet. + * beam_size: The width of beam search. + * cutoff_prob: Cutoff probability for pruning. + * cutoff_top_n: Cutoff number for pruning. + * blank_id: Index of CTC blank symbol in AM output. + * ignored_symbols: Indices of symbols in AM output to ignore for decoding (eg. , ). + * ext_scorer: External scorer to evaluate a prefix, which consists of + * n-gram language model scoring and word insertion term. + * Default null, decoding the input sample without scorer. + * Return: + * Zero on success, non-zero on failure. + */ + int init(const Alphabet& alphabet, + size_t beam_size, + double cutoff_prob, + size_t cutoff_top_n, + int blank_id, + const std::vector& ignored_symbols, + std::shared_ptr ext_scorer, + std::unordered_map hot_words); + + void init_token_mapping(); + + // Get pruned emissions for each time step's beam search + std::vector> get_pruned_emissions( + const double *prob_step, + size_t class_dim) override; }; class FlashlightDecoderState @@ -162,7 +207,6 @@ class FlashlightDecoderState std::unique_ptr decoder_impl_; }; - /* CTC Beam Search Decoder * Parameters: * probs: 2-D vector where each element is a vector of probabilities @@ -183,7 +227,6 @@ class FlashlightDecoderState * A vector where each element is a pair of score and decoding result, * in descending order. */ - std::vector ctc_beam_search_decoder( const double* probs, int time_dim, @@ -196,6 +239,42 @@ std::vector ctc_beam_search_decoder( std::unordered_map hot_words, size_t num_results=1); +/* CTC Beam Search Decoder + * Parameters: + * probs: 2-D vector where each element is a vector of probabilities + * over alphabet of one time step. + * time_dim: Number of timesteps. + * class_dim: Alphabet length (plus 1 for space character). + * alphabet: The alphabet. + * beam_size: The width of beam search. + * cutoff_prob: Cutoff probability for pruning. + * cutoff_top_n: Cutoff number for pruning. + * blank_id: Index of CTC blank symbol in AM output. + * ignored_symbols: Indices of symbols in AM output to ignore for decoding (eg. , ). + * ext_scorer: External scorer to evaluate a prefix, which consists of + * n-gram language model scoring and word insertion term. + * Default null, decoding the input sample without scorer. + * hot_words: A map of hot-words and their corresponding boosts + * The hot-word is a string and the boost is a float. + * num_results: Number of beams to return. + * Return: + * A vector where each element is a pair of score and decoding result, + * in descending order. +*/ +std::vector ctc_beam_search_decoder_for_wav2vec2am( + const double *probs, + int time_dim, + int class_dim, + const Alphabet &alphabet, + size_t beam_size, + double cutoff_prob, + size_t cutoff_top_n, + int blank_id, + const std::vector& ignored_symbols, + std::shared_ptr ext_scorer, + std::unordered_map hot_words, + size_t num_results); + /* CTC Beam Search Decoder for batch data * Parameters: * probs: 3-D vector where each element is a 2-D vector that can be used @@ -232,6 +311,46 @@ ctc_beam_search_decoder_batch( std::unordered_map hot_words, size_t num_results=1); +/* CTC Beam Search Decoder for batch data + * Parameters: + * probs: 3-D vector where each element is a 2-D vector that can be used + * by ctc_beam_search_decoder(). + * alphabet: The alphabet. + * beam_size: The width of beam search. + * num_processes: Number of threads for beam search. + * cutoff_prob: Cutoff probability for pruning. + * cutoff_top_n: Cutoff number for pruning. + * blank_id: Index of CTC blank symbol in AM output. + * ignored_symbols: Indices of symbols in AM output to ignore for decoding (eg. , ). + * ext_scorer: External scorer to evaluate a prefix, which consists of + * n-gram language model scoring and word insertion term. + * Default null, decoding the input sample without scorer. + * hot_words: A map of hot-words and their corresponding boosts + * The hot-word is a string and the boost is a float. + * num_results: Number of beams to return. + * Return: + * A 2-D vector where each element is a vector of beam search decoding + * result for one audio sample. +*/ +std::vector> +ctc_beam_search_decoder_batch_for_wav2vec2am( + const double* probs, + int batch_size, + int time_dim, + int class_dim, + const int* seq_lengths, + int seq_lengths_size, + const Alphabet &alphabet, + size_t beam_size, + size_t num_processes, + double cutoff_prob, + size_t cutoff_top_n, + int blank_id, + const std::vector& ignored_symbols, + std::shared_ptr ext_scorer, + std::unordered_map hot_words, + size_t num_results=1); + /* Flashlight Beam Search Decoder * Parameters: * probs: 2-D vector where each element is a vector of probabilities @@ -252,7 +371,6 @@ ctc_beam_search_decoder_batch( * A vector where each element is a pair of score and decoding result, * in descending order. */ - std::vector flashlight_beam_search_decoder( const double* probs, diff --git a/native_client/ctcdecode/decoder_utils.cpp b/native_client/ctcdecode/decoder_utils.cpp index 76f8b3a20..35a1ada8e 100644 --- a/native_client/ctcdecode/decoder_utils.cpp +++ b/native_client/ctcdecode/decoder_utils.cpp @@ -4,40 +4,6 @@ #include #include -std::vector> get_pruned_log_probs( - const double *prob_step, - size_t class_dim, - double cutoff_prob, - size_t cutoff_top_n) { - std::vector> prob_idx; - for (size_t i = 0; i < class_dim; ++i) { - prob_idx.push_back(std::pair(i, prob_step[i])); - } - // pruning of vacobulary - size_t cutoff_len = class_dim; - if (cutoff_prob < 1.0 || cutoff_top_n < cutoff_len) { - std::sort( - prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); - if (cutoff_prob < 1.0) { - double cum_prob = 0.0; - cutoff_len = 0; - for (size_t i = 0; i < prob_idx.size(); ++i) { - cum_prob += prob_idx[i].second; - cutoff_len += 1; - if (cum_prob >= cutoff_prob || cutoff_len >= cutoff_top_n) break; - } - } - prob_idx = std::vector>( - prob_idx.begin(), prob_idx.begin() + cutoff_len); - } - std::vector> log_prob_idx; - for (size_t i = 0; i < cutoff_len; ++i) { - log_prob_idx.push_back(std::pair( - prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); - } - return log_prob_idx; -} - size_t get_utf8_str_len(const std::string &str) { size_t str_len = 0; for (char c : str) { diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index c51ea046a..343339fec 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -52,13 +52,6 @@ T log_sum_exp(const T &x, const T &y) { return std::log(std::exp(x - xmax) + std::exp(y - xmax)) + xmax; } -// Get pruned probability vector for each time step's beam search -std::vector> get_pruned_log_probs( - const double *prob_step, - size_t class_dim, - double cutoff_prob, - size_t cutoff_top_n); - // Functor for prefix comparsion bool prefix_compare(const PathTrie *x, const PathTrie *y); diff --git a/native_client/ctcdecode/numpy.i b/native_client/ctcdecode/numpy.i index 72d5824de..0ef92bab1 100644 --- a/native_client/ctcdecode/numpy.i +++ b/native_client/ctcdecode/numpy.i @@ -114,17 +114,12 @@ if (py_obj == NULL ) return "C NULL value"; if (py_obj == Py_None ) return "Python None" ; if (PyCallable_Check(py_obj)) return "callable" ; - if (PyString_Check( py_obj)) return "string" ; - if (PyInt_Check( py_obj)) return "int" ; + if (PyBytes_Check( py_obj)) return "string" ; + if (PyLong_Check( py_obj)) return "int" ; if (PyFloat_Check( py_obj)) return "float" ; if (PyDict_Check( py_obj)) return "dict" ; if (PyList_Check( py_obj)) return "list" ; if (PyTuple_Check( py_obj)) return "tuple" ; -%#if PY_MAJOR_VERSION < 3 - if (PyFile_Check( py_obj)) return "file" ; - if (PyModule_Check( py_obj)) return "module" ; - if (PyInstance_Check(py_obj)) return "instance" ; -%#endif return "unknown type"; } @@ -529,7 +524,7 @@ return success; } - /* Require the given PyArrayObject to to be Fortran ordered. If the + /* Require the given PyArrayObject to be Fortran ordered. If the * the PyArrayObject is already Fortran ordered, do nothing. Else, * set the Fortran ordering flag and recompute the strides. */ @@ -2007,7 +2002,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2015,7 +2010,8 @@ typestring); SWIG_fail; } - $2 = (DIM_TYPE) PyInt_AsLong($input); + $2 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($2 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $2; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2035,7 +2031,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2043,7 +2039,8 @@ typestring); SWIG_fail; } - $1 = (DIM_TYPE) PyInt_AsLong($input); + $1 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($1 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $1; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2497,9 +2494,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$2), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$2), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2567,9 +2564,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2637,9 +2634,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2711,9 +2708,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2785,161 +2782,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, - DIM_TYPE* DIM3, DIM_TYPE* DIM4) - */ -%typemap(in,numinputs=0) - (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) - (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) -{ - $1 = &data_temp; - $2 = &dim1_temp; - $3 = &dim2_temp; - $4 = &dim3_temp; - $5 = &dim4_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Utilities") - (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) -{ - npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, - DATA_TYPE** ARGOUTVIEWM_ARRAY4) - */ -%typemap(in,numinputs=0) - (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_ARRAY4) - (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) -{ - $1 = &dim1_temp; - $2 = &dim2_temp; - $3 = &dim3_temp; - $4 = &dim4_temp; - $5 = &data_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Utilities") - (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_ARRAY4) -{ - npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, - DIM_TYPE* DIM3, DIM_TYPE* DIM4) - */ -%typemap(in,numinputs=0) - (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) - (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) -{ - $1 = &data_temp; - $2 = &dim1_temp; - $3 = &dim2_temp; - $4 = &dim3_temp; - $5 = &dim4_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") - (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) -{ - npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array || !require_fortran(array)) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, - DATA_TYPE** ARGOUTVIEWM_FARRAY4) - */ -%typemap(in,numinputs=0) - (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_FARRAY4) - (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) -{ - $1 = &dim1_temp; - $2 = &dim2_temp; - $3 = &dim3_temp; - $4 = &dim4_temp; - $5 = &data_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") - (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_FARRAY4) -{ - npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array || !require_fortran(array)) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3013,9 +2858,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3089,9 +2934,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 diff --git a/native_client/ctcdecode/path_trie.cpp b/native_client/ctcdecode/path_trie.cpp index e68b1ca79..fa50b7f06 100644 --- a/native_client/ctcdecode/path_trie.cpp +++ b/native_client/ctcdecode/path_trie.cpp @@ -158,7 +158,7 @@ PathTrie* PathTrie::get_prev_word(std::vector& output, void PathTrie::iterate_to_vec(std::vector& output) { // previous_timesteps might point to ancestors' timesteps - // therefore, children must be uptaded first + // therefore, children must be updated first for (auto child : children_) { child.second->iterate_to_vec(output); } @@ -218,7 +218,7 @@ void PathTrie::set_matcher(std::shared_ptr> matcher) matcher_ = matcher; } -#ifdef DEBUG +// #ifdef DEBUG void PathTrie::vec(std::vector& out) { if (parent != nullptr) { parent->vec(out); @@ -244,4 +244,4 @@ void PathTrie::print(const Alphabet& a) { printf("\n"); printf("transcript:\t %s\n", tr.c_str()); } -#endif // DEBUG +// #endif // DEBUG diff --git a/native_client/ctcdecode/path_trie.h b/native_client/ctcdecode/path_trie.h index 255c18973..80e03e1b6 100644 --- a/native_client/ctcdecode/path_trie.h +++ b/native_client/ctcdecode/path_trie.h @@ -78,10 +78,10 @@ class PathTrie { // remove current path from root void remove(); -#ifdef DEBUG +// #ifdef DEBUG void vec(std::vector& out); void print(const Alphabet& a); -#endif // DEBUG +// #endif // DEBUG float log_prob_b_prev; float log_prob_nb_prev; diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index c8c99e288..0ee3462b0 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -83,6 +83,12 @@ Scorer::set_alphabet(const Alphabet& alphabet) setup_char_map(); } +const Alphabet& +Scorer::get_alphabet() const +{ + return alphabet_; +} + void Scorer::setup_char_map() { diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index ecf47c1ce..3d51c3864 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -79,6 +79,8 @@ class Scorer : public fl::lib::text::LM void set_alphabet(const Alphabet& alphabet); + const Alphabet& get_alphabet() const; + // save dictionary in file bool save_dictionary(const std::string& path, bool append_instead_of_overwrite = false); diff --git a/native_client/ctcdecode/swigwrapper.i b/native_client/ctcdecode/swigwrapper.i index fa1763428..441ff4a4c 100644 --- a/native_client/ctcdecode/swigwrapper.i +++ b/native_client/ctcdecode/swigwrapper.i @@ -27,15 +27,19 @@ namespace std { %template(FlashlightOutputVector) vector; %template(FlashlightOutputVectorVector) vector>; %template(Map) unordered_map; + %template(PathTriePtrVector) vector; } +%ignore DecoderState::timestep_tree_root_; +%ignore DecoderState::prefix_root_; + %shared_ptr(Scorer); // Convert NumPy arrays to pointer+lengths %apply (double* IN_ARRAY2, int DIM1, int DIM2) {(const double *probs, int time_dim, int class_dim)}; %apply (double* IN_ARRAY3, int DIM1, int DIM2, int DIM3) {(const double *probs, int batch_size, int time_dim, int class_dim)}; %apply (int* IN_ARRAY1, int DIM1) {(const int *seq_lengths, int seq_lengths_size)}; -%apply (unsigned int* IN_ARRAY1, int DIM1) {(const unsigned int *input, int length)}; +%apply (unsigned int* IN_ARRAY1, int DIM1) {(const unsigned int *indices, int length)}; %ignore Scorer::dictionary; @@ -43,6 +47,7 @@ namespace std { %include "../alphabet.h" %include "output.h" %include "scorer.h" +%include "path_trie.h" %include "ctc_beam_search_decoder.h" %constant const char* __version__ = ds_version(); diff --git a/native_client/modelstate.h b/native_client/modelstate.h index 324efd033..a0968b0ad 100644 --- a/native_client/modelstate.h +++ b/native_client/modelstate.h @@ -9,7 +9,7 @@ #include "ctcdecode/scorer.h" #include "ctcdecode/output.h" -class DecoderState; +struct DecoderState; struct ModelState { //TODO: infer batch size from model/use dynamic batch size diff --git a/native_client/python/numpy.i b/native_client/python/numpy.i index b8fdaeb1f..0ef92bab1 100644 --- a/native_client/python/numpy.i +++ b/native_client/python/numpy.i @@ -80,6 +80,7 @@ %#define array_data(a) (((PyArrayObject*)a)->data) %#define array_descr(a) (((PyArrayObject*)a)->descr) %#define array_flags(a) (((PyArrayObject*)a)->flags) +%#define array_clearflags(a,f) (((PyArrayObject*)a)->flags) &= ~f %#define array_enableflags(a,f) (((PyArrayObject*)a)->flags) = f %#define array_is_fortran(a) (PyArray_ISFORTRAN((PyArrayObject*)a)) %#else @@ -94,6 +95,7 @@ %#define array_descr(a) PyArray_DESCR((PyArrayObject*)a) %#define array_flags(a) PyArray_FLAGS((PyArrayObject*)a) %#define array_enableflags(a,f) PyArray_ENABLEFLAGS((PyArrayObject*)a,f) +%#define array_clearflags(a,f) PyArray_CLEARFLAGS((PyArrayObject*)a,f) %#define array_is_fortran(a) (PyArray_IS_F_CONTIGUOUS((PyArrayObject*)a)) %#endif %#define array_is_contiguous(a) (PyArray_ISCONTIGUOUS((PyArrayObject*)a)) @@ -112,17 +114,12 @@ if (py_obj == NULL ) return "C NULL value"; if (py_obj == Py_None ) return "Python None" ; if (PyCallable_Check(py_obj)) return "callable" ; - if (PyString_Check( py_obj)) return "string" ; - if (PyInt_Check( py_obj)) return "int" ; + if (PyBytes_Check( py_obj)) return "string" ; + if (PyLong_Check( py_obj)) return "int" ; if (PyFloat_Check( py_obj)) return "float" ; if (PyDict_Check( py_obj)) return "dict" ; if (PyList_Check( py_obj)) return "list" ; if (PyTuple_Check( py_obj)) return "tuple" ; -%#if PY_MAJOR_VERSION < 3 - if (PyFile_Check( py_obj)) return "file" ; - if (PyModule_Check( py_obj)) return "module" ; - if (PyInstance_Check(py_obj)) return "instance" ; -%#endif return "unknown type"; } @@ -485,7 +482,7 @@ { int i; int success = 1; - int len; + size_t len; char desired_dims[255] = "["; char s[255]; char actual_dims[255] = "["; @@ -527,7 +524,7 @@ return success; } - /* Require the given PyArrayObject to to be Fortran ordered. If the + /* Require the given PyArrayObject to be Fortran ordered. If the * the PyArrayObject is already Fortran ordered, do nothing. Else, * set the Fortran ordering flag and recompute the strides. */ @@ -538,7 +535,13 @@ int i; npy_intp * strides = array_strides(ary); if (array_is_fortran(ary)) return success; + int n_non_one = 0; /* Set the Fortran ordered flag */ + const npy_intp *dims = array_dimensions(ary); + for (i=0; i < nd; ++i) + n_non_one += (dims[i] != 1) ? 1 : 0; + if (n_non_one > 1) + array_clearflags(ary,NPY_ARRAY_CARRAY); array_enableflags(ary,NPY_ARRAY_FARRAY); /* Recompute the strides */ strides[0] = strides[nd-1]; @@ -1999,7 +2002,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2007,7 +2010,8 @@ typestring); SWIG_fail; } - $2 = (DIM_TYPE) PyInt_AsLong($input); + $2 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($2 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $2; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2027,7 +2031,7 @@ (PyObject* array = NULL) { npy_intp dims[1]; - if (!PyInt_Check($input)) + if (!PyLong_Check($input)) { const char* typestring = pytype_string($input); PyErr_Format(PyExc_TypeError, @@ -2035,7 +2039,8 @@ typestring); SWIG_fail; } - $1 = (DIM_TYPE) PyInt_AsLong($input); + $1 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($1 == -1 && PyErr_Occurred()) SWIG_fail; dims[0] = (npy_intp) $1; array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); if (!array) SWIG_fail; @@ -2489,9 +2494,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$2), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$2), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2559,9 +2564,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2629,9 +2634,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2703,9 +2708,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2777,9 +2782,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -2853,161 +2858,9 @@ if (!array) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, - DIM_TYPE* DIM3, DIM_TYPE* DIM4) - */ -%typemap(in,numinputs=0) - (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) - (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) -{ - $1 = &data_temp; - $2 = &dim1_temp; - $3 = &dim2_temp; - $4 = &dim3_temp; - $5 = &dim4_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") - (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) -{ - npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array || !require_fortran(array)) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, - DATA_TYPE** ARGOUTVIEWM_FARRAY4) - */ -%typemap(in,numinputs=0) - (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_FARRAY4) - (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) -{ - $1 = &dim1_temp; - $2 = &dim2_temp; - $3 = &dim3_temp; - $4 = &dim4_temp; - $5 = &data_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") - (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_FARRAY4) -{ - npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array || !require_fortran(array)) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, - DIM_TYPE* DIM3, DIM_TYPE* DIM4) - */ -%typemap(in,numinputs=0) - (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) - (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) -{ - $1 = &data_temp; - $2 = &dim1_temp; - $3 = &dim2_temp; - $4 = &dim3_temp; - $5 = &dim4_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Utilities") - (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) -{ - npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, - DATA_TYPE** ARGOUTVIEWM_ARRAY4) - */ -%typemap(in,numinputs=0) - (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_ARRAY4) - (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) -{ - $1 = &dim1_temp; - $2 = &dim2_temp; - $3 = &dim3_temp; - $4 = &dim4_temp; - $5 = &data_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Utilities") - (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_ARRAY4) -{ - npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3081,9 +2934,9 @@ if (!array || !require_fortran(array)) SWIG_fail; %#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); %#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); %#endif %#if NPY_API_VERSION < 0x00000007 @@ -3139,6 +2992,15 @@ %numpy_typemaps(unsigned long long, NPY_ULONGLONG, int) %numpy_typemaps(float , NPY_FLOAT , int) %numpy_typemaps(double , NPY_DOUBLE , int) +%numpy_typemaps(int8_t , NPY_INT8 , int) +%numpy_typemaps(int16_t , NPY_INT16 , int) +%numpy_typemaps(int32_t , NPY_INT32 , int) +%numpy_typemaps(int64_t , NPY_INT64 , int) +%numpy_typemaps(uint8_t , NPY_UINT8 , int) +%numpy_typemaps(uint16_t , NPY_UINT16 , int) +%numpy_typemaps(uint32_t , NPY_UINT32 , int) +%numpy_typemaps(uint64_t , NPY_UINT64 , int) + /* *************************************************************** * The follow macro expansion does not work, because C++ bool is 4 diff --git a/setup.py b/setup.py index 9bc041d2d..308768f67 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ def main(): "tqdm", "webdataset==0.1.103", "miniaudio", + "clearml", ] decoder_pypi_dep = ["coqui_stt_ctcdecoder == {}".format(version)] @@ -67,11 +68,12 @@ def main(): ], package_dir={"": "training"}, packages=find_packages(where="training"), - python_requires=">=3.6, <3.9", + python_requires=">=3.7, <3.9", install_requires=install_requires, include_package_data=True, extras_require={ "transcribe": ["webrtcvad == 2.0.10"], + "onnxruntime": ["onnxruntime==1.11.0"], }, ) diff --git a/training/coqui_stt_training/evaluate_export.py b/training/coqui_stt_training/evaluate_export.py index 69d1b6b16..34cefd638 100644 --- a/training/coqui_stt_training/evaluate_export.py +++ b/training/coqui_stt_training/evaluate_export.py @@ -79,8 +79,6 @@ def main(): worker_process.start() # Launch reader() as a separate python process processes.append(worker_process) - print([x.name for x in processes]) - wavlist = [] ground_truths = [] predictions = [] diff --git a/training/coqui_stt_training/evaluate_wav2vec2am.py b/training/coqui_stt_training/evaluate_wav2vec2am.py new file mode 100644 index 000000000..7e6949488 --- /dev/null +++ b/training/coqui_stt_training/evaluate_wav2vec2am.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse +import csv +import os +import sys +import io +from functools import partial +from multiprocessing import JoinableQueue, Manager, Process, cpu_count + +import numpy as np +import onnxruntime +import soundfile as sf +from clearml import Task +from coqui_stt_training.util.evaluate_tools import calculate_and_print_report +from coqui_stt_ctcdecoder import ( + Alphabet, + Scorer, + ctc_beam_search_decoder_for_wav2vec2am, +) + + +def evaluation_worker(model, scorer_path, queue_in, queue_out, beam_width): + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = ( + onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + ) + session = onnxruntime.InferenceSession(model, sess_options) + + am_alphabet = Alphabet() + am_alphabet.InitFromLabels("ABCD etaonihsrdlumwcfgypbvk'xjqz") + + scorer = None + if scorer_path: + scorer_alphabet = Alphabet() + scorer_alphabet.InitFromLabels(" abcdefghijklmnopqrstuvwxyz'") + + scorer = Scorer() + scorer.init_from_filepath(scorer_path.encode("utf-8"), scorer_alphabet) + + while True: + try: + msg = queue_in.get() + filename = msg["filename"] + + speech_array, sr = sf.read(filename) + max_length = 250000 + speech_array = speech_array.astype(np.float32) + features = speech_array[:max_length] + + def norm(wav, db_level=-27): + r = 10 ** (db_level / 20) + a = np.sqrt((len(wav) * (r**2)) / np.sum(wav**2)) + return wav * a + + features = norm(features) + + onnx_outputs = session.run( + None, {session.get_inputs()[0].name: [features]} + )[0].squeeze() + decoded = ctc_beam_search_decoder_for_wav2vec2am( + onnx_outputs, + am_alphabet, + beam_size=beam_width, + scorer=scorer, + blank_id=0, + ignored_symbols=[1, 2, 3], + )[0].transcript.strip() + + queue_out.put( + { + "wav": filename, + "prediction": decoded, + "ground_truth": msg["transcript"], + } + ) + except FileNotFoundError as ex: + print("FileNotFoundError: ", ex) + + print(queue_out.qsize(), end="\r") # Update the current progress + queue_in.task_done() + + +def main(): + args = parse_args() + manager = Manager() + work_todo = JoinableQueue() # this is where we are going to store input data + work_done = manager.Queue() # this where we are gonna push them out + + processes = [] + for i in range(args.proc): + worker_process = Process( + target=evaluation_worker, + args=(args.model, args.scorer, work_todo, work_done, args.beam_width), + daemon=True, + name="evaluate_process_{}".format(i), + ) + worker_process.start() # Launch reader() as a separate python process + processes.append(worker_process) + + wavlist = [] + ground_truths = [] + predictions = [] + losses = [] + + with open(args.csv, "r") as csvfile: + csvreader = csv.DictReader(csvfile) + count = 0 + for row in csvreader: + count += 1 + # Relative paths are relative to the folder the CSV file is in + if not os.path.isabs(row["wav_filename"]): + row["wav_filename"] = os.path.join( + os.path.dirname(args.csv), row["wav_filename"] + ) + work_todo.put( + {"filename": row["wav_filename"], "transcript": row["transcript"]} + ) + + print("%d wav entries found in csv" % count) + work_todo.join() + print("%d wav file transcribed" % work_done.qsize()) + + while not work_done.empty(): + msg = work_done.get() + losses.append(0.0) + ground_truths.append(msg["ground_truth"]) + predictions.append(msg["prediction"]) + wavlist.append(msg["wav"]) + + # Print test summary + _ = calculate_and_print_report( + wavlist, ground_truths, predictions, losses, args.csv + ) + + if args.dump: + with open(args.dump + ".txt", "w") as ftxt, open( + args.dump + ".out", "w" + ) as fout: + for wav, txt, out in zip(wavlist, ground_truths, predictions): + ftxt.write("%s %s\n" % (wav, txt)) + fout.write("%s %s\n" % (wav, out)) + print("Reference texts dumped to %s.txt" % args.dump) + print("Transcription dumped to %s.out" % args.dump) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Evaluation report using Wav2vec2 ONNX AM" + ) + parser.add_argument( + "--model", required=True, help="Path to the model (ONNX export)" + ) + parser.add_argument("--csv", required=True, help="Path to the CSV source file") + parser.add_argument( + "--scorer", + required=False, + default=None, + help="Path to the external scorer file", + ) + parser.add_argument( + "--proc", + required=False, + default=cpu_count(), + type=int, + help="Number of processes to spawn, defaulting to number of CPUs", + ) + parser.add_argument( + "--dump", + required=False, + help='Path to dump the results as text file, with one line for each wav: "wav transcription".', + ) + parser.add_argument( + "--beam_width", + required=False, + default=8, + type=int, + help="Beam width to use when decoding.", + ) + parser.add_argument( + "--clearml_project", + required=False, + default="STT/wav2vec2 decoding", + ) + parser.add_argument( + "--clearml_task", + required=False, + default="evaluation report", + ) + args = parser.parse_args() + try: + task = Task.init(project_name=args.clearml_project, task_name=args.clearml_task) + except: + pass + return args + + +if __name__ == "__main__": + main() diff --git a/training/coqui_stt_training/util/config.py b/training/coqui_stt_training/util/config.py index fe7839957..702ba11c4 100644 --- a/training/coqui_stt_training/util/config.py +++ b/training/coqui_stt_training/util/config.py @@ -8,7 +8,6 @@ from typing import List import progressbar -import tensorflow.compat.v1 as tfv1 from attrdict import AttrDict from coqpit import MISSING, Coqpit, check_argument from coqui_stt_ctcdecoder import Alphabet, UTF8Alphabet @@ -39,6 +38,8 @@ def __getattr__(self, name): @dataclass class BaseSttConfig(Coqpit): def __post_init__(self): + import tensorflow.compat.v1 as tfv1 + # Augmentations self.augmentations = parse_augmentations(self.augment) if self.augmentations: diff --git a/training/coqui_stt_training/util/evaluate_tools.py b/training/coqui_stt_training/util/evaluate_tools.py index 9ab5669da..fc63a1527 100644 --- a/training/coqui_stt_training/util/evaluate_tools.py +++ b/training/coqui_stt_training/util/evaluate_tools.py @@ -8,8 +8,6 @@ import numpy as np from attrdict import AttrDict -from .config import Config -from .io import open_remote from .text import levenshtein @@ -113,7 +111,7 @@ def print_report(samples, losses, wer, cer, dataset_name, report_count=5): def print_single_sample(sample): print("WER: %f, CER: %f, loss: %f" % (sample.wer, sample.cer, sample.loss)) - print(" - wav: file://%s" % sample.wav_filename) + print(" - wav: %s" % sample.wav_filename) print(' - src: "%s"' % sample.src) print(' - res: "%s"' % sample.res) print("-" * 80) @@ -137,5 +135,7 @@ def save_samples_json(samples, output_path): We set ensure_ascii=True to prevent json from escaping non-ASCII chars in the texts. """ + from .io import open_remote + with open_remote(output_path, "w") as fout: json.dump(samples, fout, default=float, ensure_ascii=False, indent=2) diff --git a/training/coqui_stt_training/util/io.py b/training/coqui_stt_training/util/io.py index d99f0a8c8..93889a400 100644 --- a/training/coqui_stt_training/util/io.py +++ b/training/coqui_stt_training/util/io.py @@ -5,8 +5,6 @@ """ import os -from tensorflow.io import gfile - def is_remote_path(path): """ @@ -21,6 +19,8 @@ def path_exists_remote(path): Wrapper that allows existance check of local and remote paths like `gs://...` """ + from tensorflow.io import gfile + if is_remote_path(path): return gfile.exists(path) return os.path.exists(path) @@ -30,6 +30,8 @@ def copy_remote(src, dst, overwrite=False): """ Allows us to copy a file from local to remote or vice versa """ + from tensorflow.io import gfile + return gfile.copy(src, dst, overwrite) @@ -46,6 +48,8 @@ def open_remote( with open_remote('gs://.....', mode='w+') as f: do something with the file f, whether or not we have local access to it """ + from tensorflow.io import gfile + if is_remote_path(path): return gfile.GFile(path, mode=mode) return open( @@ -63,6 +67,8 @@ def isdir_remote(path): """ Wrapper to check if remote and local paths are directories """ + from tensorflow.io import gfile + if is_remote_path(path): return gfile.isdir(path) return os.path.isdir(path) @@ -72,6 +78,8 @@ def listdir_remote(path): """ Wrapper to list paths in local dirs (alternative to using a glob, I suppose) """ + from tensorflow.io import gfile + if is_remote_path(path): return gfile.listdir(path) return os.listdir(path) @@ -81,6 +89,8 @@ def glob_remote(filename): """ Wrapper that provides globs on local and remote paths like `gs://...` """ + from tensorflow.io import gfile + return gfile.glob(filename) @@ -88,7 +98,8 @@ def remove_remote(filename): """ Wrapper that can remove local and remote files like `gs://...` """ - # Conditional import + from tensorflow.io import gfile + return gfile.remove(filename) @@ -96,4 +107,6 @@ def rmtree_remote(foldername): """ Wrapper that can remove local and remote directories like `gs://...` """ + from tensorflow.io import gfile + return gfile.rmtree(foldername) From db41947cc026659e9ac7c64854a1cb74e67fb85b Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 29 Mar 2022 23:20:24 +0200 Subject: [PATCH 2/8] LM optimization with wav2vec2 AM --- lm_optimizer.py | 50 ++----- .../coqui_stt_training/evaluate_wav2vec2am.py | 51 +++++-- .../coqui_stt_training/util/lm_optimize.py | 32 +++++ .../util/lm_optimize_wav2vec2am.py | 134 ++++++++++++++++++ 4 files changed, 212 insertions(+), 55 deletions(-) create mode 100644 training/coqui_stt_training/util/lm_optimize_wav2vec2am.py diff --git a/lm_optimizer.py b/lm_optimizer.py index e23ef56d0..e5a1c9b4d 100644 --- a/lm_optimizer.py +++ b/lm_optimizer.py @@ -1,46 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function +from __future__ import absolute_import, division, print_function -import sys - -from coqui_stt_training.train import early_training_checks -from coqui_stt_training.util.config import ( - Config, - initialize_globals_from_cli, - log_error, -) - -from coqui_stt_training.util import lm_optimize as lm_opt - - -def main(): - initialize_globals_from_cli() - early_training_checks() - - if not Config.scorer_path: - log_error( - "Missing --scorer_path: can't optimize scorer alpha and beta " - "parameters without a scorer!" - ) - sys.exit(1) - - if not Config.test_files: - log_error( - "You need to specify what files to use for evaluation via " - "the --test_files flag." - ) - sys.exit(1) - - results = lm_opt.compute_lm_optimization() +if __name__ == "__main__": print( - "Best params: lm_alpha={} and lm_beta={} with WER={}".format( - results.get("lm_alpha"), - results.get("lm_beta"), - results.get("wer"), - ) + "Using the top level lm_optimizer.py script is deprecated and will be removed " + "in a future release. Instead use: python -m coqui_stt_training.util.lm_optimize" ) + try: + from coqui_stt_training.util import lm_optimize + except ImportError: + print("Training package is not installed. See training documentation.") + raise - -if __name__ == "__main__": - main() + lm_optimize.main() diff --git a/training/coqui_stt_training/evaluate_wav2vec2am.py b/training/coqui_stt_training/evaluate_wav2vec2am.py index 7e6949488..74c00a7af 100644 --- a/training/coqui_stt_training/evaluate_wav2vec2am.py +++ b/training/coqui_stt_training/evaluate_wav2vec2am.py @@ -21,7 +21,9 @@ ) -def evaluation_worker(model, scorer_path, queue_in, queue_out, beam_width): +def evaluation_worker( + model, scorer_path, queue_in, queue_out, beam_width, lm_alpha, lm_beta +): sess_options = onnxruntime.SessionOptions() sess_options.graph_optimization_level = ( onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL @@ -38,6 +40,8 @@ def evaluation_worker(model, scorer_path, queue_in, queue_out, beam_width): scorer = Scorer() scorer.init_from_filepath(scorer_path.encode("utf-8"), scorer_alphabet) + if lm_alpha and lm_beta: + scorer.reset_params(lm_alpha, lm_beta) while True: try: @@ -82,17 +86,25 @@ def norm(wav, db_level=-27): queue_in.task_done() -def main(): - args = parse_args() +def evaluate_wav2vec2am( + model, + csv_file, + scorer, + num_processes, + dump_to_file, + beam_width, + lm_alpha=None, + lm_beta=None, +): manager = Manager() work_todo = JoinableQueue() # this is where we are going to store input data work_done = manager.Queue() # this where we are gonna push them out processes = [] - for i in range(args.proc): + for i in range(num_processes): worker_process = Process( target=evaluation_worker, - args=(args.model, args.scorer, work_todo, work_done, args.beam_width), + args=(model, scorer, work_todo, work_done, beam_width, lm_alpha, lm_beta), daemon=True, name="evaluate_process_{}".format(i), ) @@ -104,7 +116,7 @@ def main(): predictions = [] losses = [] - with open(args.csv, "r") as csvfile: + with open(csv_file, "r") as csvfile: csvreader = csv.DictReader(csvfile) count = 0 for row in csvreader: @@ -112,7 +124,7 @@ def main(): # Relative paths are relative to the folder the CSV file is in if not os.path.isabs(row["wav_filename"]): row["wav_filename"] = os.path.join( - os.path.dirname(args.csv), row["wav_filename"] + os.path.dirname(csv_file), row["wav_filename"] ) work_todo.put( {"filename": row["wav_filename"], "transcript": row["transcript"]} @@ -130,19 +142,21 @@ def main(): wavlist.append(msg["wav"]) # Print test summary - _ = calculate_and_print_report( - wavlist, ground_truths, predictions, losses, args.csv + samples = calculate_and_print_report( + wavlist, ground_truths, predictions, losses, csv_file ) - if args.dump: - with open(args.dump + ".txt", "w") as ftxt, open( - args.dump + ".out", "w" + if dump_to_file: + with open(dump_to_file + ".txt", "w") as ftxt, open( + dump_to_file + ".out", "w" ) as fout: for wav, txt, out in zip(wavlist, ground_truths, predictions): ftxt.write("%s %s\n" % (wav, txt)) fout.write("%s %s\n" % (wav, out)) - print("Reference texts dumped to %s.txt" % args.dump) - print("Transcription dumped to %s.out" % args.dump) + print("Reference texts dumped to %s.txt" % dump_to_file) + print("Transcription dumped to %s.out" % dump_to_file) + + return samples def parse_args(): @@ -189,11 +203,18 @@ def parse_args(): default="evaluation report", ) args = parser.parse_args() + return args + + +def main(): + args = parse_args() try: task = Task.init(project_name=args.clearml_project, task_name=args.clearml_task) except: pass - return args + evaluate_wav2vec2am( + args.model, args.csv, args.scorer, args.proc, args.dump, args.beam_width + ) if __name__ == "__main__": diff --git a/training/coqui_stt_training/util/lm_optimize.py b/training/coqui_stt_training/util/lm_optimize.py index 70a7f3d66..8c484f202 100644 --- a/training/coqui_stt_training/util/lm_optimize.py +++ b/training/coqui_stt_training/util/lm_optimize.py @@ -63,3 +63,35 @@ def compute_lm_optimization() -> dict: "lm_beta": study.best_params.get("lm_beta"), "wer": study.best_value, } + + +def main(): + initialize_globals_from_cli() + early_training_checks() + + if not Config.scorer_path: + log_error( + "Missing --scorer_path: can't optimize scorer alpha and beta " + "parameters without a scorer!" + ) + sys.exit(1) + + if not Config.test_files: + log_error( + "You need to specify what files to use for evaluation via " + "the --test_files flag." + ) + sys.exit(1) + + results = lm_opt.compute_lm_optimization() + print( + "Best params: lm_alpha={} and lm_beta={} with WER={}".format( + results.get("lm_alpha"), + results.get("lm_beta"), + results.get("wer"), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py b/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py new file mode 100644 index 000000000..734078022 --- /dev/null +++ b/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function + +import os +import sys +from dataclasses import dataclass, field + +import optuna +from clearml import Task +from coqui_stt_ctcdecoder import Scorer +from coqui_stt_training.util.config import ( + BaseSttConfig, + Config, + initialize_globals_from_instance, + log_error, +) +from coqui_stt_training.util.evaluate_tools import wer_cer_batch +from coqui_stt_training.evaluate_wav2vec2am import evaluate_wav2vec2am + + +def character_based(): + is_character_based = False + scorer = Scorer( + Config.lm_alpha, Config.lm_beta, Config.scorer_path, Config.alphabet + ) + is_character_based = scorer.is_utf8_mode() + return is_character_based + + +def objective(trial): + Config.lm_alpha = trial.suggest_uniform("lm_alpha", 0, Config.lm_alpha_max) + Config.lm_beta = trial.suggest_uniform("lm_beta", 0, Config.lm_beta_max) + + is_character_based = trial.study.user_attrs["is_character_based"] + + samples = [] + for step, test_file in enumerate(Config.test_files): + current_samples = evaluate_wav2vec2am( + Config.wav2vec2_model, + test_file, + Config.scorer_path, + Config.num_processes, + dump_to_file=None, + beam_width=Config.export_beam_width, + lm_alpha=Config.lm_alpha, + lm_beta=Config.lm_beta, + ) + samples += current_samples + + # Report intermediate objective value. + wer, cer = wer_cer_batch(current_samples) + trial.report(cer if is_character_based else wer, step) + + # Handle pruning based on the intermediate value. + if trial.should_prune(): + raise optuna.exceptions.TrialPruned() + + wer, cer = wer_cer_batch(samples) + return cer if is_character_based else wer + + +def compute_lm_optimization() -> dict: + is_character_based = character_based() + + study = optuna.create_study() + study.set_user_attr("is_character_based", is_character_based) + study.optimize(objective, n_jobs=1, n_trials=Config.n_trials) + + return { + "lm_alpha": study.best_params.get("lm_alpha"), + "lm_beta": study.best_params.get("lm_beta"), + "wer": study.best_value, + } + + +@dataclass +class LmOptimizeWav2vec2amConfig(BaseSttConfig): + wav2vec2_model: str = field( + default="", + metadata=dict(help="Path to exported ONNX model for wav2vec2 AM."), + ) + num_processes: int = field( + default=os.cpu_count(), + metadata=dict(help="Number of worker processes for evaluation."), + ) + clearml_project: str = field( + default="STT/wav2vec2 decoding", + ) + clearml_task: str = field( + default="LM tuning", + ) + + +def initialize_config(): + config = LmOptimizeWav2vec2amConfig.init_from_argparse(arg_prefix="") + try: + task = Task.init( + project_name=config.clearml_project, task_name=config.clearml_task + ) + except: + pass + initialize_globals_from_instance(config) + + +def main(): + initialize_config() + + if not Config.scorer_path: + log_error( + "Missing --scorer_path: can't optimize scorer alpha and beta " + "parameters without a scorer!" + ) + sys.exit(1) + + if not Config.test_files: + log_error( + "You need to specify what files to use for evaluation via " + "the --test_files flag." + ) + sys.exit(1) + + results = compute_lm_optimization() + print( + "Best params: lm_alpha={} and lm_beta={} with WER={}".format( + results.get("lm_alpha"), + results.get("lm_beta"), + results.get("wer"), + ) + ) + + +if __name__ == "__main__": + main() From 0c4317023248a136e5dc43efa29cda65e4c3f6cc Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Wed, 30 Mar 2022 11:23:35 +0200 Subject: [PATCH 3/8] evaluate keep pool --- .../coqui_stt_training/evaluate_wav2vec2am.py | 174 +++++++++--------- .../util/lm_optimize_wav2vec2am.py | 7 +- .../util/multiprocessing.py | 33 +++- 3 files changed, 113 insertions(+), 101 deletions(-) diff --git a/training/coqui_stt_training/evaluate_wav2vec2am.py b/training/coqui_stt_training/evaluate_wav2vec2am.py index 74c00a7af..c8546a7bf 100644 --- a/training/coqui_stt_training/evaluate_wav2vec2am.py +++ b/training/coqui_stt_training/evaluate_wav2vec2am.py @@ -14,80 +14,75 @@ import soundfile as sf from clearml import Task from coqui_stt_training.util.evaluate_tools import calculate_and_print_report +from coqui_stt_training.util.multiprocessing import PoolBase from coqui_stt_ctcdecoder import ( Alphabet, Scorer, ctc_beam_search_decoder_for_wav2vec2am, ) +from tqdm import tqdm -def evaluation_worker( - model, scorer_path, queue_in, queue_out, beam_width, lm_alpha, lm_beta -): - sess_options = onnxruntime.SessionOptions() - sess_options.graph_optimization_level = ( - onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - ) - session = onnxruntime.InferenceSession(model, sess_options) +class EvaluationPool(PoolBase): + def init(self, model_file, scorer_path): + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = ( + onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + ) + self.session = onnxruntime.InferenceSession(model_file, sess_options) - am_alphabet = Alphabet() - am_alphabet.InitFromLabels("ABCD etaonihsrdlumwcfgypbvk'xjqz") + self.am_alphabet = Alphabet() + self.am_alphabet.InitFromLabels("ABCD etaonihsrdlumwcfgypbvk'xjqz") - scorer = None - if scorer_path: - scorer_alphabet = Alphabet() - scorer_alphabet.InitFromLabels(" abcdefghijklmnopqrstuvwxyz'") + self.scorer = None + if scorer_path: + self.scorer_alphabet = Alphabet() + self.scorer_alphabet.InitFromLabels(" abcdefghijklmnopqrstuvwxyz'") - scorer = Scorer() - scorer.init_from_filepath(scorer_path.encode("utf-8"), scorer_alphabet) - if lm_alpha and lm_beta: - scorer.reset_params(lm_alpha, lm_beta) - - while True: - try: - msg = queue_in.get() - filename = msg["filename"] - - speech_array, sr = sf.read(filename) - max_length = 250000 - speech_array = speech_array.astype(np.float32) - features = speech_array[:max_length] - - def norm(wav, db_level=-27): - r = 10 ** (db_level / 20) - a = np.sqrt((len(wav) * (r**2)) / np.sum(wav**2)) - return wav * a - - features = norm(features) - - onnx_outputs = session.run( - None, {session.get_inputs()[0].name: [features]} - )[0].squeeze() - decoded = ctc_beam_search_decoder_for_wav2vec2am( - onnx_outputs, - am_alphabet, - beam_size=beam_width, - scorer=scorer, - blank_id=0, - ignored_symbols=[1, 2, 3], - )[0].transcript.strip() - - queue_out.put( - { - "wav": filename, - "prediction": decoded, - "ground_truth": msg["transcript"], - } + self.scorer = Scorer() + self.scorer.init_from_filepath( + scorer_path.encode("utf-8"), self.scorer_alphabet ) - except FileNotFoundError as ex: - print("FileNotFoundError: ", ex) - print(queue_out.qsize(), end="\r") # Update the current progress - queue_in.task_done() + def run(self, job): + wav_filename, ground_truth, beam_width, lm_alpha, lm_beta = job + prediction = self.transcribe_file(wav_filename, beam_width, lm_alpha, lm_beta) + return wav_filename, ground_truth, prediction + + def transcribe_file(self, wav_filename, beam_width, lm_alpha, lm_beta): + if lm_alpha and lm_beta: + self.scorer.reset_params(lm_alpha, lm_beta) + + speech_array, sr = sf.read(wav_filename) + max_length = 250000 + speech_array = speech_array.astype(np.float32) + features = speech_array[:max_length] + + def norm(wav, db_level=-27): + r = 10 ** (db_level / 20) + a = np.sqrt((len(wav) * (r**2)) / np.sum(wav**2)) + return wav * a + + features = norm(features) + + onnx_outputs = self.session.run( + None, {self.session.get_inputs()[0].name: [features]} + )[0].squeeze() + + decoded = ctc_beam_search_decoder_for_wav2vec2am( + onnx_outputs, + self.am_alphabet, + beam_size=beam_width, + scorer=self.scorer, + blank_id=0, + ignored_symbols=[1, 2, 3], + )[0].transcript.strip() + + return decoded def evaluate_wav2vec2am( - model, + model_file, csv_file, scorer, num_processes, @@ -95,27 +90,9 @@ def evaluate_wav2vec2am( beam_width, lm_alpha=None, lm_beta=None, + existing_pool=None, ): - manager = Manager() - work_todo = JoinableQueue() # this is where we are going to store input data - work_done = manager.Queue() # this where we are gonna push them out - - processes = [] - for i in range(num_processes): - worker_process = Process( - target=evaluation_worker, - args=(model, scorer, work_todo, work_done, beam_width, lm_alpha, lm_beta), - daemon=True, - name="evaluate_process_{}".format(i), - ) - worker_process.start() # Launch reader() as a separate python process - processes.append(worker_process) - - wavlist = [] - ground_truths = [] - predictions = [] - losses = [] - + jobs = [] with open(csv_file, "r") as csvfile: csvreader = csv.DictReader(csvfile) count = 0 @@ -126,36 +103,51 @@ def evaluate_wav2vec2am( row["wav_filename"] = os.path.join( os.path.dirname(csv_file), row["wav_filename"] ) - work_todo.put( - {"filename": row["wav_filename"], "transcript": row["transcript"]} + jobs.append( + (row["wav_filename"], row["transcript"], beam_width, lm_alpha, lm_beta) ) - print("%d wav entries found in csv" % count) - work_todo.join() - print("%d wav file transcribed" % work_done.qsize()) + pool = existing_pool + if not pool: + pool = EvaluationPool.create_impl( + processes=num_processes, initargs=(model_file, scorer) + ) - while not work_done.empty(): - msg = work_done.get() + process_iterable = tqdm( + pool.imap_unordered(jobs), + desc="Transcribing files", + total=len(jobs), + ) + + wav_filenames = [] + ground_truths = [] + predictions = [] + losses = [] + + for wav_filename, ground_truth, prediction in process_iterable: + wav_filenames.append(wav_filename) + ground_truths.append(ground_truth) + predictions.append(prediction) losses.append(0.0) - ground_truths.append(msg["ground_truth"]) - predictions.append(msg["prediction"]) - wavlist.append(msg["wav"]) # Print test summary samples = calculate_and_print_report( - wavlist, ground_truths, predictions, losses, csv_file + wav_filenames, ground_truths, predictions, losses, csv_file ) if dump_to_file: with open(dump_to_file + ".txt", "w") as ftxt, open( dump_to_file + ".out", "w" ) as fout: - for wav, txt, out in zip(wavlist, ground_truths, predictions): + for wav, txt, out in zip(wav_filenames, ground_truths, predictions): ftxt.write("%s %s\n" % (wav, txt)) fout.write("%s %s\n" % (wav, out)) print("Reference texts dumped to %s.txt" % dump_to_file) print("Transcription dumped to %s.out" % dump_to_file) + if not existing_pool: + pool.close() + return samples diff --git a/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py b/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py index 734078022..1972f845d 100644 --- a/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py +++ b/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py @@ -16,7 +16,7 @@ log_error, ) from coqui_stt_training.util.evaluate_tools import wer_cer_batch -from coqui_stt_training.evaluate_wav2vec2am import evaluate_wav2vec2am +from coqui_stt_training.evaluate_wav2vec2am import evaluate_wav2vec2am, EvaluationPool def character_based(): @@ -45,6 +45,7 @@ def objective(trial): beam_width=Config.export_beam_width, lm_alpha=Config.lm_alpha, lm_beta=Config.lm_beta, + existing_pool=Config.pool, ) samples += current_samples @@ -101,6 +102,10 @@ def initialize_config(): except: pass initialize_globals_from_instance(config) + Config.pool = EvaluationPool.create_impl( + processes=config.num_processes, + initargs=(config.wav2vec2_model, Config.scorer_path), + ) def main(): diff --git a/training/coqui_stt_training/util/multiprocessing.py b/training/coqui_stt_training/util/multiprocessing.py index f507fd039..72a885100 100644 --- a/training/coqui_stt_training/util/multiprocessing.py +++ b/training/coqui_stt_training/util/multiprocessing.py @@ -14,7 +14,7 @@ def target_fn_single(arg): return target_impl.run(arg) -def init_fn(target_impl_cls, global_lock, parent_env, id_queue): +def init_fn(target_impl_cls, global_lock, parent_env, id_queue, initargs): process_id = id_queue.get() global target_impl @@ -25,7 +25,7 @@ def init_fn(target_impl_cls, global_lock, parent_env, id_queue): if child_env is not None: os.environ = child_env - target_impl.init() + target_impl.init(*initargs) class PoolBase: @@ -51,9 +51,9 @@ class PoolBase: Example usage: class MultiplyByTwoPool(PoolBase): - def init(self): + def init(self, pool_name): with self.lock: - print(f"synchronized step in proc {self.process_id}") + print(f"[{pool_name}] synchronized step in proc {self.process_id}") def get_child_env(self, parent_env): parent_env["TEST_VAR"] = str(self.process_id) @@ -63,13 +63,12 @@ def run(self, x): assert os.environ["TEST_VAR"] == str(self.process_id) return x*2 - pool = MultiplyByTwoPool.create(processes=4) + pool = MultiplyByTwoPool.create(processes=4, initargs=("my pool",)) print(pool.map(range(10))) """ @classmethod - @contextmanager - def create(cls, processes=None, context=None, *args, **kwargs): + def create_impl(cls, processes=None, context=None, initargs=(), *args, **kwargs): if processes is None: processes = os.cpu_count() @@ -85,11 +84,18 @@ def create(cls, processes=None, context=None, *args, **kwargs): pool = cls() pool._inner_pool = multiprocessing.pool.Pool( initializer=init_fn, - initargs=(cls, lock, parent_env, queue), + initargs=(cls, lock, parent_env, queue, initargs), context=context, *args, **kwargs, ) + + return pool + + @classmethod + @contextmanager + def create(cls, processes=None, context=None, initargs=(), *args, **kwargs): + pool = cls.create_impl(processes, context, initargs, *args, **kwargs) try: yield pool finally: @@ -99,7 +105,7 @@ def _child_init(self, lock, process_id): self.lock = lock self.process_id = process_id - def init(self): + def init(self, *args): pass def run(self, *args, **kwargs): @@ -132,6 +138,15 @@ def starmap(self, *args, **kwargs): def starmap_async(self, *args, **kwargs): return self._inner_pool.starmap_async(target_fn, *args, **kwargs) + def close(self): + return self._inner_pool.close() + + def terminate(self): + return self._inner_pool.terminate() + + def join(self): + return self._inner_pool.join() + if __name__ == "__main__": From 85467525447493b2ff59d430ddce42dcc915ebae Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 31 Mar 2022 11:47:54 +0200 Subject: [PATCH 4/8] evaluate set lm alpha beta --- .../coqui_stt_training/evaluate_wav2vec2am.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/training/coqui_stt_training/evaluate_wav2vec2am.py b/training/coqui_stt_training/evaluate_wav2vec2am.py index c8546a7bf..f0bacc889 100644 --- a/training/coqui_stt_training/evaluate_wav2vec2am.py +++ b/training/coqui_stt_training/evaluate_wav2vec2am.py @@ -50,7 +50,7 @@ def run(self, job): return wav_filename, ground_truth, prediction def transcribe_file(self, wav_filename, beam_width, lm_alpha, lm_beta): - if lm_alpha and lm_beta: + if lm_alpha is not None and lm_beta is not None: self.scorer.reset_params(lm_alpha, lm_beta) speech_array, sr = sf.read(wav_filename) @@ -165,6 +165,18 @@ def parse_args(): default=None, help="Path to the external scorer file", ) + parser.add_argument( + "--lm_alpha", + required=False, + default=None, + help="LM weight", + ) + parser.add_argument( + "--lm_beta", + required=False, + default=None, + help="Word insertion bonus", + ) parser.add_argument( "--proc", required=False, @@ -204,8 +216,19 @@ def main(): task = Task.init(project_name=args.clearml_project, task_name=args.clearml_task) except: pass + if args.lm_alpha is not None: + args.lm_alpha = float(args.lm_alpha) + if args.lm_beta is not None: + args.lm_beta = float(args.lm_beta) evaluate_wav2vec2am( - args.model, args.csv, args.scorer, args.proc, args.dump, args.beam_width + args.model, + args.csv, + args.scorer, + args.proc, + args.dump, + args.beam_width, + args.lm_alpha, + args.lm_beta, ) From 1b9803898354720b029c80e6087cb9ccb076dcb7 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 31 Mar 2022 12:30:12 +0200 Subject: [PATCH 5/8] [docker/build] Move libkenlm.so to RPATH-visible location --- Dockerfile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.build b/Dockerfile.build index 7e8305b28..9a5a072cf 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -146,7 +146,7 @@ RUN bazel build \ //native_client:libstt.so # Copy built libs to /STT/native_client -RUN cp bazel-bin/native_client/libstt.so /STT/native_client/ +RUN cp bazel-bin/native_client/libstt.so bazel-bin/native_client/libkenlm.so /STT/native_client/ # Build client.cc and install Python client and decoder bindings ENV TFDIR /STT/tensorflow From 6e3e3cd893a1375df346be3a9119effbe9ae5110 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Fri, 1 Apr 2022 12:24:31 +0200 Subject: [PATCH 6/8] Read ONNX model config for special symbols, separate scorer Alphabet --- native_client/alphabet.cc | 14 +++++- native_client/alphabet.h | 2 + .../coqui_stt_training/evaluate_wav2vec2am.py | 46 +++++++++++++++---- .../util/lm_optimize_wav2vec2am.py | 9 +++- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/native_client/alphabet.cc b/native_client/alphabet.cc index 4193eef20..30aab09a1 100644 --- a/native_client/alphabet.cc +++ b/native_client/alphabet.cc @@ -90,8 +90,8 @@ Alphabet::SerializeText() << "# A line that starts with # is a comment. You can escape it with \\# if you wish\n" << "# to use '#' in the Alphabet.\n"; - for (int idx = 0; idx < entrySize(); ++idx) { - out << getEntry(idx) << "\n"; + for (const std::string& label : GetLabels()) { + out << label << "\n"; } out << "# The last (non-comment) line needs to end with a newline.\n"; @@ -174,6 +174,16 @@ Alphabet::GetSize() const return entrySize(); } +std::vector +Alphabet::GetLabels() const +{ + std::vector labels; + for (int idx = 0; idx < GetSize(); ++idx) { + labels.push_back(DecodeSingle(idx)); + } + return labels; +} + bool Alphabet::CanEncodeSingle(const std::string& input) const { diff --git a/native_client/alphabet.h b/native_client/alphabet.h index 1cd5a052c..4d053b12c 100644 --- a/native_client/alphabet.h +++ b/native_client/alphabet.h @@ -44,6 +44,8 @@ class Alphabet : public fl::lib::text::Dictionary return space_index_; } + virtual std::vector GetLabels() const; + // Returns true if the single character/output class has a corresponding index // in the alphabet. virtual bool CanEncodeSingle(const std::string& label) const; diff --git a/training/coqui_stt_training/evaluate_wav2vec2am.py b/training/coqui_stt_training/evaluate_wav2vec2am.py index f0bacc889..8a4f78840 100644 --- a/training/coqui_stt_training/evaluate_wav2vec2am.py +++ b/training/coqui_stt_training/evaluate_wav2vec2am.py @@ -3,11 +3,13 @@ import argparse import csv +import io +import json import os import sys -import io from functools import partial from multiprocessing import JoinableQueue, Manager, Process, cpu_count +from pathlib import Path import numpy as np import onnxruntime @@ -24,26 +26,43 @@ class EvaluationPool(PoolBase): - def init(self, model_file, scorer_path): + def init(self, model_file, scorer_path, scorer_alphabet_path): sess_options = onnxruntime.SessionOptions() sess_options.graph_optimization_level = ( onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL ) self.session = onnxruntime.InferenceSession(model_file, sess_options) + parent_dir = Path(model_file).parent + with open(parent_dir / "config.json") as fin: + config = json.load(fin) + self.am_alphabet = Alphabet() - self.am_alphabet.InitFromLabels("ABCD etaonihsrdlumwcfgypbvk'xjqz") + self.am_alphabet.InitFromLabels(config["alphabet_labels"]) self.scorer = None if scorer_path: - self.scorer_alphabet = Alphabet() - self.scorer_alphabet.InitFromLabels(" abcdefghijklmnopqrstuvwxyz'") + self.scorer_alphabet = Alphabet(scorer_alphabet_path) self.scorer = Scorer() self.scorer.init_from_filepath( - scorer_path.encode("utf-8"), self.scorer_alphabet + scorer_path.encode("utf-8"), + self.scorer_alphabet, ) + self.blank_id = config.get("blank_id", 0) + self.raw_vocab = config["raw_vocab"] + self.ignored_symbols = set(config["ignored_symbols"]) + + if scorer_path: + scorer_alphabet_labels = [ + s.decode("utf-8") for s in self.scorer_alphabet.GetLabels() + ] + + for idx, label in enumerate(config["alphabet_labels"]): + if label not in scorer_alphabet_labels: + self.ignored_symbols |= frozenset([idx]) + def run(self, job): wav_filename, ground_truth, beam_width, lm_alpha, lm_beta = job prediction = self.transcribe_file(wav_filename, beam_width, lm_alpha, lm_beta) @@ -74,8 +93,8 @@ def norm(wav, db_level=-27): self.am_alphabet, beam_size=beam_width, scorer=self.scorer, - blank_id=0, - ignored_symbols=[1, 2, 3], + blank_id=self.blank_id, + ignored_symbols=list(self.ignored_symbols), )[0].transcript.strip() return decoded @@ -85,6 +104,7 @@ def evaluate_wav2vec2am( model_file, csv_file, scorer, + scorer_alphabet_path, num_processes, dump_to_file, beam_width, @@ -110,7 +130,7 @@ def evaluate_wav2vec2am( pool = existing_pool if not pool: pool = EvaluationPool.create_impl( - processes=num_processes, initargs=(model_file, scorer) + processes=num_processes, initargs=(model_file, scorer, scorer_alphabet_path) ) process_iterable = tqdm( @@ -165,6 +185,13 @@ def parse_args(): default=None, help="Path to the external scorer file", ) + parser.add_argument( + "--scorer_alphabet", + type=str, + required=False, + default="", + help="Path of alphabet file used for Scorer construction. Required if --scorer is specified", + ) parser.add_argument( "--lm_alpha", required=False, @@ -224,6 +251,7 @@ def main(): args.model, args.csv, args.scorer, + args.scorer_alphabet, args.proc, args.dump, args.beam_width, diff --git a/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py b/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py index 1972f845d..01d6bdc0c 100644 --- a/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py +++ b/training/coqui_stt_training/util/lm_optimize_wav2vec2am.py @@ -40,6 +40,7 @@ def objective(trial): Config.wav2vec2_model, test_file, Config.scorer_path, + Config.scorer_alphabet, Config.num_processes, dump_to_file=None, beam_width=Config.export_beam_width, @@ -81,6 +82,12 @@ class LmOptimizeWav2vec2amConfig(BaseSttConfig): default="", metadata=dict(help="Path to exported ONNX model for wav2vec2 AM."), ) + scorer_alphabet: str = field( + default="", + metadata=dict( + help="Path of alphabet file used for Scorer construction. Required if --scorer_path is specified" + ), + ) num_processes: int = field( default=os.cpu_count(), metadata=dict(help="Number of worker processes for evaluation."), @@ -104,7 +111,7 @@ def initialize_config(): initialize_globals_from_instance(config) Config.pool = EvaluationPool.create_impl( processes=config.num_processes, - initargs=(config.wav2vec2_model, Config.scorer_path), + initargs=(config.wav2vec2_model, Config.scorer_path, Config.scorer_alphabet), ) From ceb873160161d446e25483c86e3ede245343e343 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Fri, 1 Apr 2022 20:06:37 +0200 Subject: [PATCH 7/8] Wait for NC assets before building training image --- .github/workflows/build-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2a8437e2a..8656c47e0 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1193,6 +1193,8 @@ jobs: docker-publish: name: "Build and publish Docker training image to GHCR" runs-on: ubuntu-20.04 + needs: [upload-nc-release-assets] + if: always() steps: - uses: actions/checkout@v2 with: From e36e7312f6b96b49459e1f9f1d2148112bab24f4 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 31 Mar 2022 12:49:20 +0200 Subject: [PATCH 8/8] Bump pre-commit black to 22.3.0 to fix bug with click version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 533ff4284..dc3c4bc8d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: 'https://github.com/psf/black' - rev: "22.1.0" + rev: "22.3.0" hooks: - id: black language_version: python3