From dbe62c67b8ee5a29039972900c289550b76f8e8e Mon Sep 17 00:00:00 2001
From: ver217 <lhx0217@gmail.com>
Date: Thu, 18 Nov 2021 23:45:09 +0800
Subject: [PATCH] add an example of ViT-B/16 and remove w_norm clipping in LAMB
 (#29)

---
 colossalai/nn/optimizer/lamb.py               |   2 +-
 examples/vit-b16/README.md                    |  14 +++
 examples/vit-b16/acc.jpeg                     | Bin 0 -> 19776 bytes
 examples/vit-b16/dataloader/__init__.py       |   0
 .../dataloader/imagenet_dali_dataloader.py    | 112 ++++++++++++++++++
 examples/vit-b16/hooks.py                     |  15 +++
 examples/vit-b16/loss.jpeg                    | Bin 0 -> 22964 bytes
 examples/vit-b16/mixup.py                     |  12 ++
 examples/vit-b16/train_dali.py                |  70 +++++++++++
 examples/vit-b16/vit-b16.py                   |  78 ++++++++++++
 10 files changed, 302 insertions(+), 1 deletion(-)
 create mode 100644 examples/vit-b16/README.md
 create mode 100755 examples/vit-b16/acc.jpeg
 create mode 100755 examples/vit-b16/dataloader/__init__.py
 create mode 100755 examples/vit-b16/dataloader/imagenet_dali_dataloader.py
 create mode 100644 examples/vit-b16/hooks.py
 create mode 100755 examples/vit-b16/loss.jpeg
 create mode 100644 examples/vit-b16/mixup.py
 create mode 100644 examples/vit-b16/train_dali.py
 create mode 100755 examples/vit-b16/vit-b16.py

diff --git a/colossalai/nn/optimizer/lamb.py b/colossalai/nn/optimizer/lamb.py
index 68531e92a..f7248bd68 100644
--- a/colossalai/nn/optimizer/lamb.py
+++ b/colossalai/nn/optimizer/lamb.py
@@ -94,7 +94,7 @@ class Lamb(Optimizer):
                 # * math.sqrt(bias_correction2) / bias_correction1
                 step_size = group['lr']
 
-                weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10)
+                weight_norm = p.data.pow(2).sum().sqrt()
 
                 adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps'])
                 if group['weight_decay'] != 0:
diff --git a/examples/vit-b16/README.md b/examples/vit-b16/README.md
new file mode 100644
index 000000000..83b924c2e
--- /dev/null
+++ b/examples/vit-b16/README.md
@@ -0,0 +1,14 @@
+# Overview
+
+Here is an example of training ViT-B/16 on Imagenet-1K. We use 8x A100 in this example. For simplicity and speed, we didn't apply `RandAug` and we just used `Mixup`. With `LAMB` optimizer, we can scale the batch size to 32K with a little accuracy loss.
+
+# How to run
+Using slurm:
+```shell
+srun python train_dali.py --local_rank=$SLURM_PROCID --world_size=$SLURM_NPROCS --host=$HOST --port=29500 --config=vit-b16.py
+```
+
+# Results
+
+![Loss Curve](./loss.jpeg)
+![Accuracy](./acc.jpeg)
diff --git a/examples/vit-b16/acc.jpeg b/examples/vit-b16/acc.jpeg
new file mode 100755
index 0000000000000000000000000000000000000000..43f67fd39167963235a372abb26f11c41b8dade6
GIT binary patch
literal 19776
zcmeHu1zcQ7w)Z7CL4vyk3GVKa;K5y!;1V1f0zm=+0zm=<NO0Ex!QCA~kOrFI?(W)r
zJ(;)r&1T=cot>H8_de<V)%|tfTU}k})Ttx?bLw{Db{@E=AR{jWz`?-*X0SJKI|V!h
zP>_(2kq}Xkk&#hRQP9xwFwpPbMJL9=#lj;ep`;)uAtR%rWuvE}W}zV?W8i0G;o#)q
z;i04#6cykSVdLiE`e_h2R8&;-yXZt17(`t6$?kLg)34hW02>8v{*E6!95rwU8x9^D
z?zRJ<1OPZh*ld5A@PEAE?!Y4;A|a!oqTPjcsJsW<frE#?g8+|+h=2g=?G1YlAYdcn
z+~<@;!c}{QOznin^(rbIg+{8h6<>XLpO)Li*&h{+fRKoogzf=710xd;FCV{vpwPod
z(lWAg@(LQ7TG~3gdithj<`$M#);2D#ZXkD$7oGvH1A~I!goH-NypN5G|L`#(BQq;I
zCpRy@psc*2vZ}hKwyv$cqqD2Kr?+neJUTWGnfNxju(-6mvbwguv3YQKbbNApc7AdB
z6J2ls{Lf_lm9ih`!iLdx2LS;d0r@Ao;O@A?8ay@v;(bme97#3gXHK}(T(3~@q@vPG
zTTyAa)%WpDoQKf}Xn7Xs4t^r-J7vF(F#q30*<T6!-*im^=<sl`#e>HN#DTL*h73Qn
z|KI-YJRoHWL04f3!{Jf?(lHMRC9J;0U%J0|Wl&40;)VLu%sg;;)|mL7i25<nH0n|*
zazfz&*Y2{1&K0E!TqTB$Ie6aa%H!onQ~?1v_@f9eW`YrK2k8T7<cos!Q+@M@bcdTL
z=vxm9^-c75YYa8(n?g5xHo)EkNQIZTwChHtEzw?*U)7+qzxLo`M!_3~W{9gdlhwkg
zn*phy`7NLwPl!q?KEz2}jJwc%O*6OuR`#awCi$lRv{$@B3{w8}N+-eRbk`-h-WOe?
zLGUKv7C^-c^If36LO#tsjoH2O8JIJhXwv5QMdAiGS-}rQAC4>LRz|E6)I8FopmQ|R
zDzALA3D?@X7oeJK`WPEkIn#=U`{sjqN)5NALwwTBb(q~&!g5;E=MgBAjm75X>?CP8
z!PvYt88uf{bWp2^Q`9-Op`L+~LrWZ5i=w&LlCdOIV0NSV0}tF5>!KapbEPd?Ppb+{
zQ|`nH>^*WFkW;EnqumQzFROU%YL3C@OU?thK{6C+5g1M>EVaQ(V--rcD&S-07i@7a
zOlwEekcdDC2-IQbJv227UJzumk!rtdcLYKLuwHkels)O=kj%HeVmc|NhNWnle2#MB
zoa9#U6VDF#UOC6JBh}{g#T6JYMDOxpGi(|luTZQO4|-eM-cfOKL*VG?3!#lh)V2^%
zMctIA!qV?5tJsgdDJ=(U$hoErWK3DF?An4_p7I@4$DVE&vIni!jGq)Cx48!<Uo+E|
z&6Q))T1Cd&_~JWoL*XSTs--nXSDJ&`<azpa?Tfqk9`bb2*~>V$8jzVCXDTMDrnJ(I
zu~Jpk)9!|%e(6K<Xva4^*pVQPGiIz#RX@OJob0r8$W$bAHtgp>Y0cE*C-cL@nAr5x
zIzO4Au->J5t5q6pV9qXL>lm1!Hu`z6Pjui#<Qv~*>;s1_LTHSvKFM5Ah1E;xedxHT
z*vx%vx(I1a_yH}y8myazTD+qa*vfC87zwME*-kNE86--7-B|c&ge0``svDmowb+`Z
zz9+#x-`w-S_mhF`>vy>jYk?Bcl+&qpq3VwOOt4AH*Sm^}L8WL2&fFcF7erksYV{F*
zV|V3q-Y!N4w9lS9%iH6u=!+6e=TILK`+;WN*;Wvpk)>okf#a*ldsbEM&iaO%r}qrI
ziDMVyR`ADwUX!e40Lf)Teb(mEBH1E7zAikwvT*77(bv526d9r<7h`_@pTs?1MCYdT
zRD9<a5WCtDv!4^+5i`Beg|e7haCUJ|@uue$EuDkg`yigu74+=~HzAc?raUr6a}B6R
zc{p~5H_GB9z@CWC;>5N1{KZSjXZ8tYNFmQ6h+A+S+$4kHf)6}3xjkm5pxDW{hUL}#
z_iW;|$NPema%h-e3(ha+_t1>G;dj^w>744c4;T?@fgLgvjVj|Uk`nJ%x$p?TV_Tj2
zR+_qamiJP-Xdx9>Zyiw%TxkUAR|kI;kQX_@?YxL}6rY6c1fihnY-=RWd>fFqjK-VJ
zfwu1Ejhe7=?dByTn)Za07|T4PsSaL3N4mHLY!fZ1sVVy1Xug0=X~%NO7cK{coods@
z=OEOO*N+kgK7LG@>L)rJ{E%s}a{+j7uO`)*=R^{=Cweu?^s80+U#?PC<x$9Mkk#MA
zQGpL6ORblnC~<H?oT<>^1IzN@<aw-w2QcR^-;+h4)L%K?BiPgt;}xT9-aoQlKP{VX
zbv(1aAEh=Xwip>NT;6XvS@+>(37trCiuQ211>aH?=VXn+jEag%n7$Y9Nm+pvgtL|_
z4{g@<3FBHo>!Ke&+qIr;?s>|QLZj}^PLXTps98g_A%h0ZgOQq@YQe#mCb|3L)ozP-
z2}fzcw58=yx4_Vs!K-hXJ2VfFTD;I>B0Nfj54wZ*+~z`k%>5#~!o_8)>J3N(t`Y~T
zaiB0c_(1O`G2qD`bS-PX=*?g}i5xy0rb93iChx#-HC(eTdkYx*@iAnC0)M<ET%dH2
zk*UJx6;rm8*jIl#*N_`y6EXjk1&LFgmMd+ytl&;FL3&D0ELK@~3!sd{E)R=qZvmg@
zx4?NP;OeR)`1iLd_rq<PkA=2U&sGO-fiP8MS!tKlzrCmS_rLmoWiWe1b6e4Y#xLVT
zy`r_I2qi?U*v|qgQe<S2$@6N<D;sskHz$e583}YdN`odgoj-981q_|w2md|8eq=LJ
zQXOWbQ$AZJU(7SQ!+Y8*JMa29!RRu+`s~N7lv)XbmE=rahq}#Ujh%IJy~iw=%-m4y
zA{}V3kqh+t+bwVqD!xE6=DY3&5#Jj~y#<KUFl={76~{zUcQE+A?)3=1-MPn-qIY5#
z5ZM-gMe`V&<)L!L;ka)l(~G;W_}pLkneMG6&$tYj`L^*Ez=VFQ*mo{b9LgzsO`~6O
zgITwBb5VW^EaAbG=2?6C6afM1pDP<*^DU5nVWtk)3PMhBt4cOCU%iR@I*Y|mVHt~4
ze9fSigz1=um3F7p_iv|``NPbCp{k+d8CpZ7%ncH^?9n$qR%yD`DQVmcU%4aozwQ^m
z^2(X9xvVN4FDkSt{}kgL|3F~)9<9|~KpCK?#nc_S1^&UDhldT<exTzAG9^e3t%aWV
zf}%H0i>qs{)GL@CofDCsU(^6DDTs=6v03E?19YF*SxX-vTeWiH@I~jIJEM|;K8c^p
zR6e;RcN@Gqvb+Vl7;b^rwpzD9I>hi6Xy)_1Hu2-@3``hw5~%P~axN(Ep&`viLxGDt
z08qQUm9)zDcSCX;9LzH}P6u&6(pcF=p>ZNAqGO^W_<Lhm{xHSATZ!OB2b3)zs)nXO
zI}v^d#KjM!n)gbZbcvq5e_1oFLnB-|sIYC%W5;)<647?pHn}^(ai`r0rzaC0J_tcu
z=O6JSKf{`HW3iQ|pRNp@%%G_5P5T`bVW%*ExWPPpy4A#Ic{~_i=625Ghp5lE9~GN8
z4ad{?i^}qb{{~Ud$;_}8>W;t3&HiS5@iKSPZbYeBsbw+QDRaX;!E@tG$S&zrF}!M|
zrHH8eF(c8Z)PKli{+*rqimK4bNumXRI#e>XJS}N50GNl{&^_{CZ>uK6F;rSeCXruu
z-lLg#r1v2yTHY{PMKJG&E%SGBIYR(yJDz>(;WKWEO?-;60!PcXThmKV&Sm{><5c@B
zp75RCRHvI$q?lQV)=D?L3tU|H^Ktfnz*%x~{ICCLX@7&j{=YJe+$`yse+(>aD6Va}
z1ya86DIo=B!q&5Dp?Lpe;j7-q)jE9llvrNF)qaDchkw<(W1tDBN#p%$(|B@Y;Ef@a
za0_fsty{sT?qglYG-LQquua_p(yXppU(b=D@^eAW+k|$vKt^00)1P3ff1|n2*^|(<
zH_Yo@OZ!9`NqTPo&A+J-7iY>BM0{`o->LbG@g>u8pHpPVwkw{_1E_%F+xSS?5eGN9
z_`7Q@Sr3k;XdO9cgGU?8y+}!w2u6D4+7k26_#yB)Li6J(ZMNQ`da2a`=k_<aLVe89
zM$DaOpFYpof=-8I1mgO{XBE8c2jTaN0~9Oj%F6D|K50Bo4>?NV2qq6ge4LiiMBDdz
zI<;*W^fTYAM<KB2Z|<R90@*F#=ewMIZVPP%&0R9uW)!=(h#x9n^WFka>m6?JDu-3!
zr1AdKhD==O5o2@(s=o<G4A5V<%5kg3G+y~)qW(tH?}JpsyXlkw{NbX}%bY;`)mR>m
zUKcbagm`S?dq}`_AF=%WE#RReSm@#K^9Z8;lUr{x+-un=zIB*TuNR=aOfyC`meojD
z=P49!6yhtTp1Km7WJoWaH2A`l2Y1B9IqDQK3KzLBqhnluAXb{Q$-Pm{m*Dv*$V*Z7
z^~Y&@#v_IKK&7nXmF{fY%~|SU9`#b2y3W9`Xht)G#Mfb;PvG7R=<Mgbb;_{n65gPW
zSF24?EbOJGEmWkzae9Xdv_&AJF<VF@l2TD%sYwt~z}*7T?h7SXh*Q4n*~Ou?bHFXo
zE4&Z6@!x4aR)<JSAmXY03tF#v1WJNlmfVuDqF@VU$S3cIos$%aXo_r^2(PB8?g_jL
zJENiq7%$*>--PdMF3_zfOzNt`{9<W_?s43usaU#iy_u?1c(B7+X&9FNqj8aEI3HH`
zJ8*k%0nz)7nh(#(QpZaslqWZ~a#t;P7Q9%!d9Ndi3#XsxO>z|tjNU81hbK>^?gAz*
zR@L<YYu3NWZ4Op=RZ1MhA_qEQ$%Tl3a&^xM1e<2q-&l~ujxN)>kOj%Lw9pJU4~mR=
zF~}~)&y-x>@&2^3-@ONhq7Kd3CKxB;_PTo`Ji{DEY!y82HasxAsgBg%q&o2M)r$#B
zWt#H<4s!)}1&i*ABb<BoE=2LHFGtf+iDWfxpRiydK=GRju%+_gBw7GRMCOr*-sbj0
z@<RE|k!xS>D8&+zP09^|(@F2xhEH1e>(I@^eH$iry&1O_l(cukCgVf)I?5<#Wi4ja
z;U?H~A!HCeX^L+pfpmwrfN<ml{Q)1;cf~Y1a$)z~ml5LuFE8%uL0Sk#JQ;w=nPgH&
zYQ(Hk_FCG`96^rIf`GH<n7(;Qhao>!uYb{8=KcNs^n8*>uV|#D$?;`?Ts3J6D?Bx6
z0Pp9mrkvA!S#?PQtX;$30<?SWPfqVv-2%gkM))pGKgia94N}?O0)-Cgxj*Yczy<ol
zkN=f|sjXeRNQ#9eNme~Tzo+|J7wjZJOux?9#ircvq@cib=t)u$wplaXVm5DfQX)L^
z(y7?SU59U6-(;rekY{{lFs={~SQ}k770U_dmwiGeUuaVo<}#_Y%0dwEVxXLg^bMJ1
zl20RZ`a+V!4tf0Cjc<!!9HVkyLcK40_vQ4uTO^U5UMDSyYt$1PADdS4GCt=u$b?I3
ze`uebO9zyyCKMuH8l!k9M;J3rog6;c2k%C)lp?Eqm7wz4HJSF?0DfKLOocgO>wKxG
zZltlw5SjK4o2TPB{`K4i*0-_LZsr&4M<Rz<NLz=--tlTpL2F7iH3s!%L@P;_49MwA
z;I!l7LMXN<yzQ$jeYO(@g)X>OA_1E0=ffI8NAU)pR)!qMOC!*_*zIjQ*DKwkA}YD`
z5Ku<mI#VZ}wo-K=uRQBLm}v|8E)_pX#xL9d6;kq0bfz-z7Lcn6Wq;rNb-_n%^k_nd
zz3BWBEZ3aU$ISe;_9@L5t8>XNgH%;m^orIK(2nD=h_y3yRr~bC>P?4@fTE+^)QPAI
zly{hfz;KL5R-}~hWyOibTBw_1;_J#`SNX`P6M#&K^r<wK`?RPhy4RJKD7O-%Qq)1E
z?f~(<+C$Ul2a&VP7sV}%v{-5hsclUJ4a2;)wR>y`2i<5>dQCph2o#@Qf{N>s3S<(N
z15;D;IdYjIQ?eoPO5*eHrt}TIwY)Qsih2}aQ-BUI*)SAa?7Gp}`G+#3-T6oDZ*8l@
zA2F>doBmfi%?I9Xn`fs=83o-Yao`Wbh7#q3Ze;S&=j~?CY!y4i=o?DUb)nSNO4ZSN
z52*u<rYMX8s3>aYk|-5w3(wiQbsFnxY-gm#987gf(kkCG8BvD;7wWRe_L})LVPoH}
z1<l=U^Yvy#jwa?hN_h*LcNesbYQg}t%ag4^Jh*@bdHT<gtdU9vULuE_iYrZvH7&1>
zD186YH8?T?!$6`ne<vfPl9F%hE2g!<s-P%TB#27NJXU@<5ms0FhzK`ke4bc3Te@%F
zERl5Gpci$yEP+cUs31ZPAsSVHSVX)3yCO@&^!>r%&=H=eyrB0XxdlEORYODd*Ykjk
zP=-I!;EK1%>}Wn~sw#2{0GKE3y$1w$r@Z>T`9|DFiN9!D71s1db$r-HHuM%pqu@Sx
z9gj2X;W;u>)hAs&$vv*rUn^UMi8Wrg_2jPgxaYKf7L&GaQD7Za*AOykQ_W&srZUgc
zfR@cEwBf6~X0o31h$rhUP;!3~DUcv0<E^}3f_1A`wQu48h?37`6jWrD5rGoT5M>$;
zXh{>HAxj{Ng|59%N_@T%rl~Fm!<C|cld%0HjDMcMefRN{d{69-c{YDGFn{j4q+b~|
zPx*AO(NDf?=I@(1dJY2z9uGUXJAljyj=I@n?nfwvCw~$O*i-(Z^Jr8}`hMUqy(#C(
zvrxK-kwGZ-5#q|&z_)5Ghx)^S8aMKT!A9Hj099z;{>{z0fK|Fl@J(hLG)pJJ17~~<
zic@GOw&J@?zGfJ;{`F1sf{`X8J)ydhc*&t4C%%4kE=4osA~`5|zNWeS_$aSW|MG6R
zH}eCI5gb&Foi?b}_^|Xdq3Fbhh&LKx_!+Q8$z%CvP(Q{JXi-y8^3P0#8~iaWS$p}T
zAMm&K;Fq@HU*JVBBY1SnK*2Siz5Jskxg=}C3hOOjMM|LvcA9x?X^h%A9D8}QT12}=
z6&o|S#A|jVV5Y6q6q(AXN536e<t{<#?6oy57_CI%=||mNdrH9Y?KH-J0%6|kz;FPD
zJK9sWC4h2s4494Pls{+Jlh5%fFHBw43^r=0V8}0NrfQBSY)%O8lvQ8l5VdYf?oQkS
zA|}wpd0}s1t)OiiLM};!Jr=&)8+}Uwz%p;p_*#27oQVC!OiBGvmwFNZF0=FvG=uU@
z-D9UNHmdrdwJ(ROua+*-mhTMg3_>Q$&Bhy~AVS5Z=Fars=OqmR7S!lQgm9rLjYme|
zUqf<cZaxiIV%v6Lf1Q7pgqZ|=@-AmxpROeMc&xO-nuKHfDvPR=_N;}XeLr7|QQDXj
zGN_&`Gy-~GaR?qdnrhUBLONL&d!HBYRIDO>LpZ>HD$@KuYTY1VZ8vw6N9lpLm?_tw
z&(lW3{q{Wq`UCjG66a`v)v&C*;^2bMJ-p!s9`<vZ5S|%j9pKw82Jzz%Niz72fWajL
zjZSG{1tH(KE?$lp>ze*2N7I>3OEU!E$CY(n?-S%-em;<DO%QFTjY^pwcYh<Ildo7f
zwb4X+&!}jiYEOtgwV$qUw`U$M0$Mo)1J&r$Zi<CtM~Y&H^&~Xm<emk@l2M>;8Fu?u
zr01SPI^^=J1W@E!$e8L003Sf};l#II*7{q(=x+YbZrW`4@$rp3Y>MK@HkT2JpEhF@
z?WsdO=#t--<F@+=ihF(DW)gk4_~bogXksIhXS_UiuAcM5t5#=OWshrF(ZcKlnQ-fZ
z_3CieEG{Fu`U@_uJAICFty|-zBBYtwbL}xU=h{v3lFnQ*@&MEite(wqC0|&dbqfqi
zD)45qn<JZBKrU?fr4NMXJg&5>`B|P5ptogR7iicoiu3ViE<C|!E6I2}Bc=P)y}{P1
z3lv0;sIwJtsdIl(3GbX=W@NR}fy=0x;HU?Qzarny7oE%-(IlB$-2l9hh;QoRz*)Z)
z`QcHPPxBY;*|u1@S+@S`>Kk;?Mt$ds*xBZYd>_jiE%GNi6~inTD@z<<91?l87N$dR
z)8knFd%Q$lnOU~DAMLk^iL)%HK|S@K<+mbU6#L4JIW#<3BGs%@Igi)xW!;9i;PTs1
zDJF8egs?b5kDB@~hqmKrX2Rpjz~E7cBDq5sr_5!(A_jTl7lTI{t9|h14`aYN9@mAS
zZi$Hg4$rvHbkmT~P@1XmL>Vm6BR^Y{(63rq$NFqtQJMIKV`s2zei`pSoznl)q6Z4=
z!i_5Zv0EU*{zO8lE0!cXEW5{+m!O{^n){lzp0j7Gn!GTx5xjeCus@{R&(!XoZ#^7B
z4!Lji4o8mSVzGEL1!w4_aISK0;-Va1^QFRiS)Sls>ijzrs3IIR38mt}75$ObAE{o>
zUHDota_8ImA+#dcbs^7m0xnIs)sd11bWp!Z<#21^$!-reo<xWH=1N}@jZkUtxkSiO
zr6fTl3kQqaG_8d|scys%S9!;_S~RZ^Lyj&&jp*AaE#4gUn%ejCoF7@lStjUoOBnT<
zASoj`a_g)7q_zEE&gl>Fz2V;9;v~P~4w^-G6U=c*D}BLtCo@a&^MWLvgFE8gFF!RA
zS4gm~dhAR-C%edg{|+mMgY%@wTHVdu%v`_1T4sB%(oTI!2K=D8?z!x}ugKb>erLQL
z{dqRVA`JwSS{;{e@3ohW_B8rSTUDI#$b1zb;A>QCI~LqI8^Y?NM^J0MteYu;^{E%j
z-nIpDBcfB-I!-L0Fte0fK)+JR6-xfZ_Z-O{pXu6@ZN}%Qx4~fk8!K<UZUw2}BdO8e
z!#m$xX=73~VvE5rhdEL)Xil}>E2L630EcIU;PA{@6LT7(B8}Ur?5OIyzktpvpfupy
z4g)+4*W~_!)u|(@$^&;iliadlg(lLf1^5zxE8q(I!nmogl~Q8=&bV_`CYq&yOZJP9
zX6ZNmPIuO6&_Pj03o0ENKL<INjVI%gu>RtTPGbae2Y-O{0beOIx&N8$11{fCt-e<$
z-MpjcHed8SY?XqoFspj&BbLpeS>Vb@tc|*jD(Fzaz?JZxWRkb!sQ_iHHGl`JPjUo`
za<vrDn0=aAvBQOwdC5{{#5_x_GD2IX{~0^}m%jf4?RidpmW|#7Q^fM3pUBywSfOJz
zC{BAn$bdB=tPH+y;go%IuCG#rGP~6`i>=W6Ifd8TZ|s@uW2geihs1fVr0nMk*<-xP
zQZw59sv&msVS+O&`Bc$f&Tz$jvnWM&8P$Pzq;U|W$^D4#)NPKc=H#=E-vW^QZiU2T
zS*`8U8}T~|3>meUEq~BE|5eZaXT-z}21Sll4-FxU{}2`wCjWr_{ylK4siQ$^E1#=t
zp43Sy5K48o2?>-(i=Gk?r9Q@LvbRk_ce9ofcMDinML!Xe>;L4XncOLf=-1_lG61NK
zTpML<Lc~o&_j;NsJ^OahaU;*+{ZvNHyvA1?Ax#7$^!|OfKxcJ)0=S>YNs3LT$0S-E
zAJp;TZZAB&6Y-Xjc6d%QrDACN4!7Z_$5#{<EUkB=3*>2&3#670rG^kMnQ=7_K*1Ta
zH__0mz7I_Gi}Y?G@-5cHlGy%|D{siOC#b?A{t6%IUD`KiZ=S`U#u3_zjN_mbOBDiP
zE?si1>#_**{>+lUpGP{W=ZUHn2Z1N?!LC~&DEurCTv|(;ZH=K6Eb)=LNPHV)#r>vq
zP*Iiin*xqa;wDvJZuN67WtbK{<x_a2FA(orpP~yj8$X`RY0NcF1+C(1<lUqyH~Vyk
zv$csbD^`Af6DLQ5LKJ4-t#N)*=&Gh$HC4|iPiF>$KmOLu|E0_SM?e28G~oR2PWbOq
z=LMTi`YPGR5A&X1;k^@@b`W^Nl8+HHVajV$)}Tbjb*`Zsu|gk>T>8;W(G5vPJ@`Hx
z6+yR(Cveortus=>8_um#dw})EO;L_*(LjK1M4!i60t-7@jfgdn>3S)Qv!_I_n8QQe
z=%H$WTGcq@ifh%@U}N6Eae(J@!E^I>{C!!}W>1gjkBpLm7da*QdREm?!HO|VHeTJa
zZ4(AXW5z1o*G`d(_s4>emA>s6_*mq!2N*~V?^26Vl~z>wuYHNiJbX*Nd;j$FOtllN
znw0psxGfYo8oVQ&r+41p!Kc1K0OA{#v`VCEux@COk$Uw30+NxZN5GyFEUwoHF#3=`
z;zlku&Jp~@ppZVP^pu&&^pLmkcSkc?yyOaC`7*Emjn^Cx;7m}LlQ5AVBq-J2=Zmfv
zER0pbJMXk3ew<(&efKfeboBhO##Z&Utmig!+&&7Iho`~P_~ou`z<HJUss5&|{-k0|
z2Kf3Gz-2-^So3Q1?cay_L8W~Ev@`V2)I<Jp$4P5+fnP{z`Oc(pCM{;FWz;E4tvX*^
z>m=6W`6XVP@|Th>`0+uN2Kg|;nC=Q@`Igtp^A+HLIF|X45nOrT)*RQ>tb}r6MQ#Bs
zp0^D8mqeYTB=zF&w&$7c4uh1p0pV!wh|0~ec9paZeIaZ8?CHs-H2uSG)S-J?ajp^a
zWK1>McAqpqi;8wD>{@+Sx_?Z;>fFzhXrejz3}$n@1SO&k($YH77M(rxvJ<?%u`Nt7
zulAQuTA)$i_Vyca@j><1+qQR`$`At=^sbh@YNKAJhCcaPmAl(HxZL>U8lI%-I1?-J
z_{sirS3VHv(mM+j7sa9~!||pOQAHa6t@~Iao!w91>96Jm|BPpUJh#X!Zqyj5w11lI
z&D%U_QdJpe836S}lu|g`3iIJj*KJ}rKx_`?iE?+bX8q#abAPg4icL~D%i_M8z)ED^
zRsyi59GLC_BB#`hm|R^JooV2!8rf9u$0&{UM+zZ|y>K^k6hI~_K5U3@HzrdoA(c7Z
z%C$Zn*WQ29N|7o&dtZ-2dZOh1L;&`^$5y%M61-b5lV~_wvZP2(eY!dGsq$`iS-5#q
z!Z_82O&HhRl~*Dd6RI7UP6qCTZwXQ$?>bp0w=qNP;d%Uyr=pR!BnYg(tIyKoRP4?I
z8{CbN3FD2kp^kD=uTF^pQXF_j8FLz{h+6#5zb_fva*(|@(Ln!F=Yc%9CtOy2@bKUq
z_d+`-*77Db>)W@3t2gT0k*Waipbg_q`BFctkoE(xil1>AW;ok6n@_T@&=#}x)#orD
z-(^!Q&#dezm7{;X8u&?N{BczPvu;Ef^8?ttW@9Wxi7Jga%)NABL2v@d!e&|*h7?z?
zwnnvY@KljgHiNF@U@&mq2H{yz5mT?p=@2n?Qp@M}ATnA1kNr=d6{dq7FyE$!i`lYD
zX^+?yExe$vw!lsH!zB=W>E2CUN5TLfxF4P<y*MDX+cl+4VptVbI9T|p$>lN8g6hE4
z%q#gGaW)llCD*#9SC*gfqgU-DNqDI48cBIM`eg8{`jU+OuC!GidvRP}+u{y%8=EJ1
z9mzKv^@$|Exl7Xi;H5)|h;06ehGSJ!sucv8@<VY!?Hs&*W%KJ#v&!bRGmdMcwX~G|
z_`}+jsZAprxijg>h*uSJMm{%Ep$wnGVV7SCy#A#7|6?H4VuNljV%IBjHhu?g)%u7B
z;bKR!N!1Hr-cbmny?CV<^~~cJ!SYqQ8qhlVUhz0-TUmJ*5ahZUm*lcLrYUvpocxiR
zECM6C0*=5aQ{Fl-kLHc1Ez|ldx%^+t@&B&-e<F@tGhon>us2mXw{68j0(mISUDV@z
z#%7B1UMdTr4SzXk`IxBq?M}Q<L;(J7<njv$N7iT3cOYOw@M*)bD6-EnT;})5@P1wQ
z?8>Kq<3^69#iub~jlh3rwU-zdhme$5??WNR$4zGx5uJxLKzKAVa()^*)^CmdQ5@JZ
zO3PAqUEZXJXSr-6*}}JKgE1c1DX)MA^S?sOed~(OWLE~4ln$mNF7bq3K7Y7LsHPw0
z)+#98e^MOHFhUf~Fv>(7s45IMQh9Fe+1%>e%6VRJ$=-Pjv`|X_AGcueVPg&a#W(dv
z(5{qLJGTxs;pup+=Zl=!u389Us_Y0QH2DM8^5!KIR_+NtVXk8w2AxRnA?Vl7q)Ov~
z<1QIw(XI3hKfzQKYyF%^JJzOjb5mlM)qXdOqiIDK#;d^x>oLdc8~}heGS(oRVJmGu
z=IV8KgGN0M!nbi>ZT&8M=;-gzSAt59s!Em`g*uUUT|4+)N_Pj^u(phhoSX2?8zBv{
z1&o-{hDCJ(%h)7Kk@nJVT9tMrk+9Pfh)@e2H)Zd+Ys3o}6r!8>uPFfep{V4K$2Gsy
zpubbw8J2{zE4EsvPZ{DhjIz;$$C6@}TCO_e5?yZ<6l=)c^<LTm?fbQj2o#w*NXZyb
zhTF>Y);7WZEGpc#BCT!%YAKz&aav~-rVVmJNeg?X=_~G{ZIw}OR}Hd>R_K8tE)V~u
zzWiVO{y)W#e_wX|w#|gNW^eCBJ=aFH=jlVS+6*GE-D=}|arr;!nz0e<tWcc@@{Sr9
zDHeG+&+d+xTC=VcD3T?@e|VcmHNWqa0F3-BxN9P~UU|mW?pyiBmD{Ivu>*O@nU7ea
zdbFJH!3}w82mMOE)sNBRhqAX{$uC*_lrHOsrOS35how5Pu4Dsm<O-T_MlwfU3R)9|
zu^e{MCD)gK?(IF@IM>1=^6h9lk{!~Cdf~GARea;UFDLTW4Rc{pYP~$$CG_nHr}2sQ
zmG#-?Eg+E*%8dK;UxftyIPM<{Tz(NM!1dWcvFaO}#%RGOna>;wyGrLePn~G^2rbs9
z949(CYVX&<hmALo*zF|oJL*5_diO#e|Fw!;tsN+5*0H4rwY5t&$5uTQSZ!cl*fEgs
z8fzS9bx5mAG02G0{XlFUb;j8nRo3Xpc@B~J?j!%D@b0%22jFAOd;w`|%qTdKaaM#M
z!#{XklE!bR?4WWBghY0Ts&9Sr^sWHqdD%OBY{u?^c~Z^rQ}UF-+NVy?m^_6u=TxKa
zNHArxYiyC_-Zxk9F7#t8;;xHW-OVkq&jN#A%xqwn|2pmXXFdOKFK=<75R8)@o(JEz
zE8EoJob(rF6g0gwJt$78HCVdIKEP70GByR<&915_4IbuQ9`hDaMkZD@L%w$LLNSdm
zy<r?<bKP%?+m}U9tP&Wdj+8)x<?Wei{%s=v(-!#C#2!#H{w?Vnoet0Nt!svegB1_9
z;-Vdb#Ye(=2qh$alH?qprccu_Cx5mdwqYZ#Q{q@JrR~_|UfDgj=ZSIRtuUmDGDj<k
z_|p0Tcg{hSIc@AjaX_TNz)C+Tqxf}vr3W5;=@G5Q1-t}a!;gvZzb}X**)5eB5k{0o
z{1_>L?74DHROIO<>M2dN$xqu^<FcIFmkK#7Dza4|8UJv~?!toIBXdN+)t{oX?mL)m
zP+c85GRf?5PZnJP8xNL~z|V+QwYI+nX!r>kb55UpWW`g{OZc}K@4tYWf0_|TSJG(H
zr>OKgt-qq=GbxA+F(S}HzHg>QB?m_zgy5o`Dg~ysd(>+cjUkQCHr8t%s|9*hkpveZ
zaHqm54ZVo6Jwe+aES75DOWXP=T9Nz9HleVNOR2ET_e_|>_xc67cIS7PT$~bQXMdaj
zDi)U;5;4fMsuD74e#2wo|8<6SiOU*?fGHHu?Cq<s$Z3VmCI_iE$Q9z(xc#tFSspl8
zS-f$hc-XNx=B9r2X;(1+M3Gs|VRM<e(G^}5_h~@bQZcjS^J@!D%AFl7*wHPC7rqw{
z{9x7MNmqnnx4^n+`YkY@@&qcMxoymS3#6UG4(9m(+(+o&+9xz<smsM9;!F-YG1UF7
z=xObdZ<+Z;Lsu+lX@|_TZ!5*!%O(4iofviy$LrxO&??e=9V78m#lQF4F5hDcSaN<}
zg&kG|I{a_!GY<L(tY?6)$f&I>KlrHXTU{Y0djq#}3oIu*DfJkH224^yQNMo$*4K+&
ze&BxOzpG2RMa@By7){xTcoh%#X`<fl%en23K2KOPjB2iA1&=(}!kexEP=$`*4{Vj4
zXn>vWk=(($1rT6$zK2TRkGuVQJMoSCaHFZ5o(8wT1otO-(7Qy3azjLfb2jIuNXW$P
zDkGNqpuMqG4yqG_lSob}HEEb5p=)BDOX8PUR=(%-@v!o9s~`438fL`5ughX3b0qNw
z<Wjl>wsq-G!@&mj(BAhiJc8uiO?P0Wj)h$UTPb7z)5QKi*<wE{UkGuAp~s-!=eVsa
z(}%(4W1UqnVKNZY93<GjNpt&_1o|pigloFQg#w)&y0``MzVBho9}<pQi3>L@>po;)
zo*0e~u#~pLYT#6}N#Iv5rzFqu*x%$>S=j1##b@c~o@RZBSZK(4BL(Kk@>h7Rs#k+7
zfE)@t)}Za#l!|}ydP}sTc48}PXx<--${&U55T)3*lTg)~f@^=bG_U9LM$|)h`!gKJ
zcYRPjsqTzrny^$u_xzVg=yg7YFW>ardKXL^PVTDLXV3!k6ki^4oM5CJ!;KC90#DZ#
zD_jg`mT3;{+SyIm)D7#$uY3=|_)JD1rq72YM>gK+<iiF#AE(c1#K!9lT2dSx8OmaH
z@Yt^nD-~yWauV!(Gffc?3P46*YNc&pphIFR)TdkM0R5hkH^IWmT8h-+-EqJjl}5XV
zJUSe*A~-DXdeVo7mbs$dN$#SQa18OT4ZCp{oDV?z#WFMWQI=Cg>u34mzeJKJ?a53C
zUeu83Jey}0P8M(Rq$`?M!aJBOUQgYV!Md-K4!nxA0oL<=aVWPRBJwc%+<tJ!v#7lt
zS)8lw@F`rhqhAu-SL?c}`Yh{lj@~?_4-R5-8M?+giOkUmHh1~SQiR+-m4uKaM<nmV
z%AgFUdh8~*RBHU;m;;Y!Ir;EiMpG3!grM*0Ac5BslYtEhJlgzxDjvlo4D=3keimem
z_laDN_aw+&Qy7L#h8D&#E_q*n!C}PK`x2(i#{}qBNs>woB1bUI#uiR&fqZSjYq(Nm
zUEkEh!UUN5TJULx{A>qU$@G$$JKPlxtb#lEtBp<5?~<u!g^+#0no;i-6cuT)E!;N?
z@y<U_)UH@lYHF6Le230bPZdrfeS-Kt>})Cl*s}Ts7ka@WN?BVZ;p3^M&WA?&dDd!a
zQ5;#Z2n4D`=DKFMBZV72OvAkW6XPvL1NSlWg{{F^9>}YAv?Xf#E)S($s!|w+%+Bk*
zacl{cXf{Bnc^T$(IeKd|M3l3{#N;mu^p(1zEKAlwDd^+HB^hxq5KlhTS}%m&iRhdl
zd-Fl;3tYAEo7ObU9*=q{p}|v~sqV?^J(?Sh3m<##m5UGGns1WUCabo35x<D<hvvR6
z^If1NS@$j95qo+IC>=H20%s!4iN4BXTdsdi`Doi0l~`Zi4|Fj5Aad1E5i}|Pv1*C_
zxlK1bO{(Wev1OMrXZe0|QJFk%+1wmw#ISufwschZ*IiHQW!&jyxZbq8haQ>Ikq_VM
z@gG#a&Z2duE+;f`#MivjSsqn^zDyyIW8e`l1#OAZ3_4_}m{K==S7ApPErZnd8cBN@
zqh^pMLfR?Yb%tGWKPMkEA2T<5(x!achH(Gg@I_fS2Ul_}T817cw1)=Ft9;OTFfJl%
zRl@g%_v&sYt(8JQ9y3WV)2p-h9LO{!aVj>zu{VtoxN>6ETMXoKcRNmy+9NA0>K!o|
z5-~CjO~P=oaC>~*<zR$cqp4!71(>97d};^#hz!iEFi>%9SFnjnQL|$d{#O<A(;;Ql
zpden(+{&e_Ro`+XuNT@hY)+r$be=sAGZh@L*Gaa4G7nwLF1qbsCkzB;M<ja@*BupO
z)7935x3h5O3r><@3`~b-)&k_BbD<S-yun~|LKox7GJ<;>;IGt{Pu|BrV#>-tSyxj0
z<WVR+WKV!0v){=qW6t!hA&>@{bj~FCZR+B4Of256^R6Z7&-LT%Jju?xSOgpo@nb97
zV|e=fWO6Vga{c6yffQ+j+Oh2h!Nnb)r4IDZZ0XOQ+g0CrbcOJ0QDu+A;;K>kK!W_(
znRb{qs;<Evo|&=dspmcGv4HtR<|5&iN&Mu)vKdKJG`a9GImyG+T$Q%!zgCFfNy*Ni
z+~7;KuIGAlhIog8LYwv!%`fIc+v6a}-!-3?y?Sw$F|fW+YEkR`6843(`>r67QLSL1
z&~C1I_C_S#X<nU)!pp3Blgxf+RAd&kwScoA=Kk$9MLP>td#~J4S1%(+MX|Dj`NtH)
znL5vbdQ+49S_xzCkTfj$>VpS?H-TbB_{(b6`@*QnnO0vMQncRrwf2gl^wK&F=#B8s
z2pdIMg4ZlLntDy6=|kmuiQ%GJp1pJC5-bsDaC+$$i9N|Ykb&XTw%$vEQ`U*rRi=>U
zy@C$cv$Ya4S?7ysY_KIJIDQh&rj_{Emf66eLiUTx<0580@NiXO9<Aq6B>>|2vm*BY
zY#beorP^HR?EcIr>%u{VJ(txfYxkg8c~BS1#p{q3k;5^ZP5rLay!C_!;pV*Qqw2eQ
z2FA&|*};8>pCxhmH1}xi7bXtAN(gT<2fpnh=^O<m81kK}ZAEW#Pbks^+qxcV*304D
z>%gB1>{+Q5QLhej^H}s|-Fcp9Oj5A%tZTtu@!?vCQ*_IV7Z)x`VU~@6JC6?hVrhv?
z6M;bp&&CPm&I-1OnuyxdDV2Uk3lV2W(j-h7Ti<ri#+#F`CAl_lpF6nr#|CCsYU{3O
zhtq1Kb;Jt1#Et<M0S>>kg}(rZ{;GWb8}7lqE%tEZEv&y=Ia>0G?fK&%VmpzB>#fRH
zI5hCxl}~NNutx106w@EOg{xa_FEp^`#lLzG2&*k(KzpToFLI1@uF{=H6&dF5pvWZE
z8C7QK@DCdrOgBz@`YVVHuZg+Wnq-l;zM%>;4Rd7lT|fmZN1rM=d4d}!n)p}MP9rNY
z5fc4csDPR-&Mr<PIy|`7BHUylxi{q+qwMCTjaSc|zYgzO9!`qp+Q3y~Dg$I^g;n~}
ujX}Bja~+2t-H6ro0<bF;WJ<#u962(JUlgJJp)(xc3l{zZ2ZX=<_P+pRWdKP4

literal 0
HcmV?d00001

diff --git a/examples/vit-b16/dataloader/__init__.py b/examples/vit-b16/dataloader/__init__.py
new file mode 100755
index 000000000..e69de29bb
diff --git a/examples/vit-b16/dataloader/imagenet_dali_dataloader.py b/examples/vit-b16/dataloader/imagenet_dali_dataloader.py
new file mode 100755
index 000000000..a39d73e26
--- /dev/null
+++ b/examples/vit-b16/dataloader/imagenet_dali_dataloader.py
@@ -0,0 +1,112 @@
+from nvidia.dali.pipeline import Pipeline
+from nvidia.dali.plugin.pytorch import DALIClassificationIterator, LastBatchPolicy
+import nvidia.dali.fn as fn
+import nvidia.dali.types as types
+import nvidia.dali.tfrecord as tfrec
+import torch
+import numpy as np
+
+
+class DaliDataloader(DALIClassificationIterator):
+    def __init__(self,
+                 tfrec_filenames,
+                 tfrec_idx_filenames,
+                 shard_id=0,
+                 num_shards=1,
+                 batch_size=128,
+                 num_threads=4,
+                 resize=256,
+                 crop=224,
+                 prefetch=2,
+                 training=True,
+                 gpu_aug=False,
+                 cuda=True,
+                 mixup_alpha=0.0):
+        self.mixup_alpha = mixup_alpha
+        self.training = training
+        pipe = Pipeline(batch_size=batch_size,
+                        num_threads=num_threads,
+                        device_id=torch.cuda.current_device() if cuda else None,
+                        seed=1024)
+        with pipe:
+            inputs = fn.readers.tfrecord(
+                path=tfrec_filenames,
+                index_path=tfrec_idx_filenames,
+                random_shuffle=training,
+                shard_id=shard_id,
+                num_shards=num_shards,
+                initial_fill=10000,
+                read_ahead=True,
+                prefetch_queue_depth=prefetch,
+                name='Reader',
+                features={
+                    'image/encoded': tfrec.FixedLenFeature((), tfrec.string, ""),
+                    'image/class/label': tfrec.FixedLenFeature([1], tfrec.int64, -1),
+                })
+            images = inputs["image/encoded"]
+
+            if training:
+                images = fn.decoders.image(images,
+                                           device='mixed' if gpu_aug else 'cpu',
+                                           output_type=types.RGB)
+                images = fn.random_resized_crop(images,
+                                                size=crop,
+                                                device='gpu' if gpu_aug else 'cpu')
+                flip_lr = fn.random.coin_flip(probability=0.5)
+            else:
+                # decode jpeg and resize
+                images = fn.decoders.image(images,
+                                           device='mixed' if gpu_aug else 'cpu',
+                                           output_type=types.RGB)
+                images = fn.resize(images,
+                                   device='gpu' if gpu_aug else 'cpu',
+                                   resize_x=resize,
+                                   resize_y=resize,
+                                   dtype=types.FLOAT,
+                                   interp_type=types.INTERP_TRIANGULAR)
+                flip_lr = False
+
+            # center crop and normalise
+            images = fn.crop_mirror_normalize(images,
+                                              dtype=types.FLOAT,
+                                              crop=(crop, crop),
+                                              mean=[127.5],
+                                              std=[127.5],
+                                              mirror=flip_lr)
+            label = inputs["image/class/label"] - 1  # 0-999
+            # LSG: element_extract will raise exception, let's flatten outside
+            # label = fn.element_extract(label, element_map=0)  # Flatten
+            if cuda:  # transfer data to gpu
+                pipe.set_outputs(images.gpu(), label.gpu())
+            else:
+                pipe.set_outputs(images, label)
+
+        pipe.build()
+        last_batch_policy = 'DROP' if training else 'PARTIAL'
+        super().__init__(pipe, reader_name="Reader",
+                         auto_reset=True,
+                         last_batch_policy=last_batch_policy)
+
+    def __iter__(self):
+        # if not reset (after an epoch), reset; if just initialize, ignore
+        if self._counter >= self._size or self._size < 0:
+            self.reset()
+        return self
+
+    def __next__(self):
+        data = super().__next__()
+        img, label = data[0]['data'], data[0]['label']
+        label = label.squeeze()
+        if self.mixup_alpha > 0.0:
+            if self.training:
+                lam = np.random.beta(self.mixup_alpha, self.mixup_alpha)
+                idx = torch.randperm(img.size(0)).to(img.device)
+                img = lam * img + (1 - lam) * img[idx, :]
+                label_a, label_b = label, label[idx]
+                lam = torch.tensor([lam], device=img.device, dtype=img.dtype)
+                label = (label_a, label_b, lam)
+            else:
+                label = (label, label, torch.ones(
+                    1, device=img.device, dtype=img.dtype))
+            return (img,), label
+        return (img,), (label,)
diff --git a/examples/vit-b16/hooks.py b/examples/vit-b16/hooks.py
new file mode 100644
index 000000000..b6c306ed7
--- /dev/null
+++ b/examples/vit-b16/hooks.py
@@ -0,0 +1,15 @@
+from colossalai.registry import HOOKS
+from colossalai.trainer import BaseHook
+from colossalai.core import global_context as gpc
+from colossalai.context import ParallelMode
+
+
+@HOOKS.register_module
+class TotalBatchsizeHook(BaseHook):
+    def __init__(self, trainer, priority: int = 2) -> None:
+        super().__init__(trainer, priority)
+
+    def before_train(self):
+        total_batch_size = gpc.config.BATCH_SIZE * \
+            gpc.config.engine.gradient_accumulation * gpc.get_world_size(ParallelMode.DATA)
+        self.logger.info(f'Total batch size = {total_batch_size}', ranks=[0])
diff --git a/examples/vit-b16/loss.jpeg b/examples/vit-b16/loss.jpeg
new file mode 100755
index 0000000000000000000000000000000000000000..a16c333cc8e945ec9e3713017160d57d273ae3d8
GIT binary patch
literal 22964
zcmeIa1z225(my;n1PdA*5-hj}hd_b{cTa*taMvLu!Ce#FU4jk{3GTs$Fu3dBF2jHB
z-o5+o=I-vhckkW({@?R`@HEYw>2pq3byxMTs;m0$``seozPz-YGynks0bmOM1>DU5
zo&nHNQPEIQ(9zJ)Ffh>XVG&|uVPayD;uGK!Qj$?qQIb(m(9p3n(mZ0NrJ#7s&&0~f
z#mmb}%_t};z%9bg!^{2CLl7`9Ft9MONU*U<xF1qH<o?Y+ckKW?bc97je<XxQ07N_l
zBs_$>P5?CkfPez;?N1&4^A7?d5;6)Z8al>3On8IZ`v61)BqT&+Boq{6WO!>|`1b&0
zJQVzgToR}Rs;|%<ITCWejmbi%m8|?iq&9j$$Mf1L0OKC<0}@g)dIrYFOw7D|Pxu7{
zg`Pc^l9rK`lULW!)Y8_`)iW_QGq<p`vUYZHb#wRd^m_OHLts#FNN8+ad_rQ<$4|-G
zIk|cH1%*Y$Rn;}Mb)fo&#;+ZnUEMvsef?wO6O-Vn@6(W_<(1X7^^MJ~?Zczvlhd>F
zi_5E@=t2M>{Y=(hDEomfJUCs5$jC^@Xg|?~fane{NO;I754lkBB~;N~ITAeLev3{h
z8Ix7{1%sAH?SSaD)95{7I^HGv!=Fg|6J`GzVFCXZWq%>;f9Qe$u#gbogNK9%5C@!J
zJ<j&O_y3pwE)Tj7*<0=a=+F<fQ`4=Z8%7c~puuU-phHUayK%uDp!IY)$FfbSALaPm
zcV_m=*Sa4U5jH9q{DrwCi_B>N+;UKjnp+$B$z;uOS78=#@Un!hP_Oz<O{zBIA){Jd
z^M!YbjEOhlh0f5|8F^CJ$a+DtLxI3_lh~y6$~7j8nEGAY66rN=%G#`N&MvvzZCw9~
z-@?*DwVn6L{!_}vi+57KFSbfbzVlrOD;739dn}g1lQ9<3ERj94%~GIT@?FZO6Q#~O
zURO?(Gk3jtMmBcgtF{+kDz^vFaBSZdkEm>r+;4_lw(DITKgN@dL~<p;>>~vr`n<9A
z{l2AJcg3d3=IK|qZFZ5m^nhaOkAl)7hzI}^5f!u&Wf3j#k+Mh_1s9XMo4~zH)3^(5
zN~;w%Z_OG*W!Z!GAR&!+F5Ck?(L+LRF2RvOg|qE3p)-ep8IHvl9Qxx>Zacd-=AjQW
zn!C`E4!17A#vhvXGsOc=u0!}(6g2B<Wy>EWYetmK5u!?~Iib^p0=8@`pyj4VTAKP<
z`0{6i9YYcN0<#Uhn=*^mJf&6v51A%xk<EeCXvb5iZ=grV8Er8UEzSlJbw#hN4i(hu
z8=C{s?RoWeGh;2#%ZrcpXi{Bfc}~kY%kn{s3Men>t6B~#Xzl<M_AU#9WPsBhN_Uxw
z^`kd@`s76pLsgjX-M&U+w2LCpP`?PR0)e9(9TDfAj_adXt>7b$q9O-$6bB%5`?5qF
zMzmA}?z!ON&pgAftxI!XHN3RMld8Fh2<b7iXKTB+TFcwie4$%>5&lXX<SM(*uJ_y=
z<2Aj;)AsxF=#z4QNYX|Xv{I`pa6-u^)RdxWqmPkv*StFn9s4z@n%l>=xUz1CiBUyq
z-W|wY$dzTSF}ABMgD+Ze_sN}#?sY_R?HhJiI+o}|axBk?r>c`WWE`Dds?>`F;8i(C
za?Shm+GLE1XxbE8oogq(CPxlZO;J{7UBT&lDD{QLsVDn-C3(8q`x9%Hcdtp{S~ThF
zNX6}lS=9F4^Y#jgCQ#`9(&<{wKqFJL>1h3kCjUoXgzdwNsAq|g^>tATC2w!_un?`S
zd!4={8<-c9A0~Ov#-(#XXZfAwOHK^NER;M=0zE4Rd?P_8Q5ObYE;a@Fth7>z#!Rmf
zFmnqFyJV34cM+h;y>%0>zWH!12jhS3wgFzq>y4q?qq5na*F5Km>@Dr8-;az(*t&8n
z%ilhzftY7mdE=**o{h8mQ%h<jzipPuqC0NC0;g8%M>B-a(OCAq%;X%F)_FVPgyNYW
z#kOlqz^7niGeqV+b+B759cA}AYDkx005|DP{j4tziMN*M`OTw+gZvxeRo|#<?UCx?
zxmF-VWa1cf2Y6t3IRdH$cuVJ$Zs|P@-}XvKi;iBx9^9c?f!2qEaU*@~TCv97@F9=S
zsD^H@K7XzTWlkX-h{V}tCWz$OF^D+PuD(${q;NLcB;I=kA}%R~kcQ1%0{P>8y2#!4
zXHh+OAn{gK`J3s;<9C1>TRvgesW~<MSJ!197%v|rE_oW5o^%1+-CG$07c>r_Wz7oC
zICXop^}3vRTC_ATxsFY-Tq&Kk_iBe8b)T`Yv}kqD?-_VHC^&@BZsZfDCLg7U^E|(Z
zS&BkauFp^Nbrws94qpqhtugb==AbHP5@`)6HcY-g)XdK+x_Axobj6HsHdGp{?Pp>%
zvFENBsJ4<sMHR@zm7BP6&Dxx23X`0$hq4($_15y!*Jw350QdK3V=-vIlWnd#eXa@4
zmn2Nk6!-qFo$}1y!eV6~Grz#q#sfEqE({v`Va93u+f93VZg_gZJENp_%F3q{ZGk@8
zg*JNYMS0d<7-M#<uLBLs+^Qew@~;R>Ban+K>oFV-eTdiyUYUs2*WavqMxD~;Y``1C
z;$IZLH((1O$gm#xb}Ec^QMRm>1RkWXu9WRfP&&KaM@m<I)L{5bzKxo&Sz+0m+#X7s
zGY4r-HgR5O(aSdtcRAK@;+cx(v_zV$5#G5~kPqEg9mh%GZ7{2|^6AllQQiQ^DYa9*
zz!9pWwPBP%Incz(G}v2r%1HpajlL_f7};H+f*?isd&^X>G|l3&Kc$odjAY!k#o}wS
zbdlo<n;AoMRp7EzB)Mo-8q$3p1cAQOGUE&BTH`B9*OBXEi#tH~W2BtyhLInx-2vX)
zXx;&`rVQ=?ZBKwVul<FDSW<s=HR^}k{@?wmGGPAq?Yc6X!fE|^eJ<n<NBrLRkwIAK
zr9X1)SA6j2u!4~sz~Y3J^EbWBQU?Q{#6rddz87uh7E(9}oE4qi63CC~X^p!KurS20
zKeM%~OuB+1@g=~q0l4H2@T|Py4&e9Z4sg*$eGp;Yb|SQsalSrs2MAX|ji?B!QNRSZ
zRF9rqd)EzH1Ww}(d)>5~e>gY`PEI-twY>6Lu$w!Jc@to8wb$)@0d~Kk)vLJ0Y23KI
ztiA)R5CXsQ72g5A?UVx-Qp90gPbn$Qm6wKawr`AQEgM~?{8tU+tRonJ^R#>!|3Y_a
zVNv<k`Lu;wVD~_RMp{S2zRy?Xiw=JAZPR2gLtc})79>{R7OD3dU{p5^B2fcH_IA>3
zXkmC|+OQx)XV}g39pErje2Hulxam42zCQ$iCrKvuOp}7mwJw`I+j2c9-ZqvgT1eo8
z=D=!n<_mU}z7&?@Og1{0UWG^l3k4~YWQFMMztGWypr8{&cJ~aWF;>=fU<-XqmGsal
z4Hh`UIpc3xtorMR0bE=au4!K!VNp+m)8#Ut4B`*nKDpZ+b49f@rOi1Hnce~ZmZ>s7
z@EoD>4K!kVKE*?EtoyZYKlTy^a-m&49a5Ba@Rpuqz&6q=>sL+A^qqe4&`*&F79<@6
zZ~IP&qJG_zUrmR+#E{&r@~BsN(|T18BgimaBnxHfrO^P<7*v_s58M~aOmrr-p<qAq
zs%o2>`liOuA6h5fD(hXP`@<3Y8-$~jSTcV#dw(Ux+Nz^9Ew5BvMV|~`ng~ibBlZ(s
z+o2I2=4~yrSbq3W7&&=9QxRML;j^@g$(jIT%(e~n|2D#(YGyFQuDvfy%W|7R$Px~R
zOtCc5IAB^@+Hz*heca8)w0?taGN})<o1C$;`gGxKk`j-qfqy$sa(;LTU~+<2kx?r^
zfRBy4|BR$<vd7sjmf05xDYU894ca~vZbI?8Vc`cA^(*TAS@kb?zJimu3}!rRWCA@t
zia7O{;8$_3-j`g(+tI7kosn=9?R;|3Bl<<1H`t#y7Ynh|$pL^QT7vmQe||NsOBBWW
zJBF_*(^^0o4-<_U=#{xfwBF#zor+8*Z~V~uSHx{@mOKA1FTjZm{r|*ca_zCVzsSKy
zZpvU0I70p>Ek;iiws<|0rtrm!@^&>}za#~nAyE=AyXznKwte-itQhb(JM`}sX2H%O
z`^O*`xqduy*}MdO3Va0Q@Uz~l(AZGGs>_^uC-5NU4)EG&&Pv<u+q7@-x`GYUL(73D
z->2AE!czseMtyf|v<`_wvJ=8I!K|#NF$#}nXX(s>3bCZE+k$FrB7`=y+uA&GTPN(v
zw7BpkpJ&p^#iBh2qzhGR4Tj!(@>GN>+n-Ye=?*}*-(h@)S$78*RWKxSZbi5Q#JVq4
zT%*hYH}lFv{qM149F`5uMFp~Ru&#Toy<KrqSxlHkpngF%9|4vHO9^TL-tAha+7dm|
zs9?mKP?eIC`29HI@8S?-Z7em2mU`elS>CQ1#T+&!ft_{g!ve!QKu_wz9$Eh+aEbm}
z9%yn0F!N0rAa9*ox1WnkwOt|U+0i697B=eaHz+=9C_0n91GI45-q;`J?8e*yyo|3e
z7FG<-Z?u7@9Kcm;@#UsFK#}(yfL8OGW1pi9hLBMQ+qrIAOv9a~g`KpWAGUQzRmt3L
z0ryvc5zY&T@b^!^&Fh$woZ4%d>4(?GNoQ&AhK{PM-2`XS9g{yjODgvJ@YQFI?3BjQ
zf~ZsL$Y3}9W%px<6Y1G0geNXMd?ioefBIC;-vu$O%UW5u7nTy%tGUwjVrG5-p;w9)
zh2-Vw)&!21mQj+!8xg$Xi#YX(+OPusk^vW~R#)^4Ys*P(MoXHv$EgXF=tx8;=^sQ?
z^k)V{0-a*A+|UhY0H#aYbQv!(csHTTv3#ypb)S*#%+U_fVvG!l%Zj(4%eQRhroj@K
zqPy~ZRvbq21_Hea3A%FRO0%gi@xsB)Pe6L-VTt`KI+HZD#<+UQYmm8S-Gtnao}biH
z7yU#m1PBr4ePhM<7485)@*67}a~rMMu3**%Q4fYH5!}MrZ5ZmP{l4ijdqItK;tLaR
z6J(gEjot!GH1`hRc?bBEiDeJC1M~?WOx*@R+fLM`qzv}`fvag3Hn1;l3s+1w*_nu;
zkN<H=z5{H}Y+6lB-2t=~o;{_w1N`mX<?ij`M@lz*cYv2o_P2z!qbjJ>VM{z7=1>WN
zOLv4>QzUhd*Bem32jgapY-g_0;?Tzspo;Z;2`+>;rQR|2+h=M6M}zZ@ohTlO^x1UV
z@>~^Bq*pz#_co6~P&2A6@rIf^0IwBYOT2N4v6rIXiygS=PYnJe`TxDw1NUJf>vsT$
ziIQ`fv26I1jS_5|<ztSiafbL4hrWnXm6j>YM9Ge$olyIRA}S+4UhC~>U$k`w`W!e*
z+wj3u#n+)de0*G~GO}+sc6iQ~+U<gv9(^W+EukAnj_zs|%-*tsX^guAF;8){aev$*
zXE1<LQ%!P)5YB2jQ=L91lwK?3(I)Bp!fMCM=a@g5q=P1S;0_>iGt|Vt5z#R0(QS!1
z30Fi4vPR^#)XITOk2o7;-h`HsZofz9PfZtc)>^5m4Frm@SgP4Gy*ei5Bp*<d&-a2N
zr{cV0s9HWWj}XhE59b+S9c}6!qRbtlIzg@qLEL5K^gt6N*-`O$)&-tt8w2l=>!(-=
zE>L`yFBUUbt(_(^-dkM}*tA4)gMjDzA;LB~j{?xAtflZ#G^qS>0EJLE7KJFEXIv&%
zYnJ`xe?a#E?oC`9Ht;+9%pE|A%|&zg0u3g&5ZJc+!1fN1o!H3yca``rgUD0?D{Bo*
zQv{ycp70`!{CMw%&yQ*rfNc753Ul$@nezP|Jx|ftr=m|1pWU)Fmg7tjixaL!6+p6M
z{F+%azJZnHzxhriEoN+5wUBxWv~^WX?EhEh)uiwouF^!FTsicn4-o~;8%O9&Y`uK8
z$VOLFT2B3~qJEQ8*!3!fJ0C&8y|6^<oxysI1pXm3i&b;rSNd$=$S1hPRsrMv>koWI
zXHCG%ed^m&83=qFHkyIwlP0Gk;y2gEPJCDMw3iF>gKZbjYm_gK#m`USHeu~jps)vR
zApH!5HIcNlzX0J#Eqr)JJnjJd#bQ(TcL33qJ3woRybF`M`+K#M=Ne^d*)^*RC4&3-
zde$t<*qAz%<7*T6Jx~nZ`+a0`A@>0tgJ$Xh%|}90Y`GUg+r2Ye$$?QN^F7#`pX|o<
z%xYQIFGJS}7_pl1v(Q<Ej*Sw2-C=7}M=c?vYOCds`U>L}Cj+QWiTabtXH2&A(z9}r
zqFtlYy>^Ii<5Ix|Z=tS~jM8LcSp^CRHJ(olF5$8n81s<`(iMbk;S|}jlXY#_c5@yD
zyUKyO-2sm0#jm&h8H&Mh(*(+RJ#sT7{+G+2>~N<^qNSGyc{b&Km!4z}tSy<b)Ds;p
zaBuEFTrI|`j(w_0wHl4qkcoNnv*y$>B0EIsZf5kP8CMhN1x#BjxIJ^%hGa5Pb>7-I
z!MzxCIJoLhNvM9n2@MNBXz!;e4t@;cuMg)V+)$AQ$P!Pm8*q<_=C#odnT&+Jg+B%j
zf3nX9Yuy3J`txJoLFq5Hu9BbW3;|tXWUwGlDQ+fW%h>0F+A1BJR;NWDRiy|4|B0ez
zOBIk93Q#QB6lZ=?k$F>OnTZT(eRq^&eTCcqPru0drY3$>cSYt3Ur{DO-@c)VFH&DJ
z`jOoMJ{y8yp?aGoX+J0xjekgiw)jc}qB6mSu<F>uqnqgjGOmYTt7%I<Fn*?gI@01o
zsDkz|%L+7hQKx8aOK1t+DF_p<{zPm33ABghQ>AU~|Fmyp(1Qq6;^F=r96C_EdG=sT
zI#0FSc1$wo8!;MD(Qi;pMExFDvm@Evgl6G^Tz3FOHI|Z6Wul53S<`r|k7;z&=ijKo
z4Tnk-X7uC64Fy-^O9|WE9OW5aHm7t4Wh0Oqmb{_VIJ;2g#)yJbq6&iwe`<@M^~zzz
zh}pV3K%CtjV7jEZyuWyoGE$!XWO?Cbb|?$a)9ld4fd59xZ~LQ0?5DY?ZhWini!Jms
z@UTS6_+hyJ&KklT2*S<Cn(vwTs1sF43QO=y-lU6&Y2>7t(1;SD@Xj|BbW&8P7HC6P
zJOkd}CVqbdO8hq8bJ`7;%$~Lcx5<rY!OzVHlb0)IosDlcrW}Q37Hz(57Y#sE3)Gvn
z=WBp51X0yxY8h7I4cl#;*`ZAS{|TA-d(Zssy!d79{Co9pUrz~Dr2r0){4~Dxgk?;Y
zSFku+H&ApGu%%}ay}Vj>A{)#bW-E#JJ`Ks_x*&`CEwUy#aQ346P}o2(x6QD?*5X3o
z4q!2B4MDmovv3-dZ2jEE5_LRZ{beU8UOZNLukD7=h~vd`i<?7Z-?l}b68Nl@{HJL2
zpCQ%X`rU6ww*SzRt1>s6mll7nhRTFxON+AolvLoR*t){cgv_-s8G|wS`SntSJMTL*
zW;7?S*Zw@bxOj3EU4%PX+iqIbPpi8`N8h~({Z0dj<w2Afq0xfFy%PJZa<q;!;>wF|
zFc5na)Tou17@KzoNV7hx9PF%!N47&SCvmRF2_h9Nuh9O+dKyq=FjPn@{T{)lP}vT5
zLV6o_<tbtFkMQX4SfTJeW6o3K6Z`vC{JuofWwUpHW-()Mb+aznmR@4gvOr)5)(*-z
z4!`7$^Uk}Q9qrCTo=vCzj2*UV^Jkk9cL1!(lpR^uAsl@x@kzsH;a8@n)a{EfqDV5M
z5dwjhw2b*uxW#AiLNuQW+pfITYR=CLR--MtICE*wO9X0!!|Q<uHgaEJM-I4aQQ$_h
z%~S)^<-v#a0!8duQiK}uKh3}Yc=>sOR{B|s0XA;_HMaf3?RJ|#jbFjfq6=__{qHL}
zq3YB7bM5S<>mX73h`3{t(gmDZd9bpX(1+x{=0KWNPNaAGL~pTjln>2j)k>I+nRyjK
zYpj{8<_$L939r;wVcve?pC)A%@`QkVM+*lxp{xrBQ5T!lH+4c|Z=0$NH~_%*pmpV1
z*V>T_aJT+GmI1jdo3owPGGz{<Q|jS<I5<C{NTOsb+E`z}a?RlI`#H6WM}?%{2_a1R
z<VKu!q+oBTW4+C|=a^kpiYVms-_ulo^tJyRG#^pSGPzY=Rp|O0|Bw~;aIsLdP*i{I
zX6g30Ce7kivRdKRkfIAsRdXP0Sh$d!tCw3;rn21T*&QH+F^rEAuT|f>?PNw-BArrZ
zW#*e!P*8ioie@zuB1`tRC{duyQL)$)<I1l5okDb9p)K9-z!3&v9X}DRkIa#!_=)?<
z$!*(h1R}P4@;;-}Hb-u^)Qhefnug*kwUciiwN%Fqg5vQb0(joN`*Mz2rfyYWPB*R%
zcM++P{Lv%v<MaNQkzv9_>6RrGAk8cH?n(sl^yDd;vBl2oEtGzH=FJo-I5QCQ#14?U
zG=PQF{|$B3ol}vuQi|7}?l}gL4*vTy+FAu8-^Do|<(k9C`l8R28d|F>6Df6O#P=B*
zV>O)yxnc=R{qddn@r>R*Mfn*?f_+=t^5$$ndaMs@7_w|gVm%~5y#@LDBqOxp8_D1P
zbsO^+ee}uzslX$D-bK7|g0SbY_=4(D#+E+M7wkbctV6m=fezY2Gum0Q{zG2d+eDCK
zc%K!sQrp{&OISh~VHmF*Geu6JnONo&AGK5I(&p3%u*;88(cUkfaXVwc{tFC^ZPCgE
zR6F7FdYitoC4pGvcr7ma`$-=kSetgazF@Rs@lZ3>627>GcX;9naCgggi++KnCxB^7
zV@iXaN)gZ|$o9+j{SIAkuHa6fFEVp2{NqPkFyn_4s#34t{6fNitp0;k>bj5JqagX3
z32KcXrR1+r3s9mZnXA>k8gG%>hp92NFF6FiKn?j!JfdV8Cd+nLD;q*FcgXV^K~^~&
z<eXZP<p^jfhJg~j0-}XTsRyQdFOt)uKTpub+VUuRduBD%#Ljz?=@LO+mm_>J6-6L5
z8FC}5fLp><a5!DnoB>55zyyj!2fXlUs+{2)*eWTm=wvaUG8!shkgvZM(kVe7<6;#)
zCl)lPPoe-IrZ<$2`HKvEJ!@?JbV$X*lEywpL?*t!W+;*GJ~-U1_Cldy8a~x51fhoA
z6{t;I>bcuOzNN2lKZ3NB1x<FfG<sHTw0;BV{RQBeR?;0bt6}97X;Ec@Wdd^@3;N8M
z1{0lm8AaPgsE@bwHQzN`r8CD3_J1jS1bySUUO{prb6l<mTa(qe6i;6WHTE2rSt$5<
z=>WdXE76HimTj)2!xbYAC-@%o<p$gALR`hWg;00)VdSxreU(z~Mgj@irp#%1Km)cJ
zb%zm6m=P$Zwa1%T0{DQ0;*}}Y$Nf9NRE1<l%LC>07T{t|KtTh&`s%ItB2NVz;Ss=%
zlmDz;^2gXCTI>}|qa|~BJ!jn&7PW|;ZrVm_RC0YJJ0r%g^fgq_#B(p?XXL@Cg|2LB
zXKS|ebBfTBWIWP@6eJ4MY?$V^pP<X{kb}GutDU0nqOH)V=+U-w9Tz^g&`)1^!ous!
zB<=9kA~%OeITlpMveRE8x5r20?jwa-1fS%TUHBa<t7Xfnud;pPqzQ-7X!=s$66j67
z!M$htfa{jLVkdOA^dNzLcX{SKF@ty4Dy)FYldi_=Cb=PK$xv@aW86%C;(2p*T3Nw8
z57zpP487TMF}QE&pdxs<h(wpSMagodLDJz>^Z|vp*B#)h<a2prrS^K{$dv^>ld#i(
z>n#IZr<hv*#}v!&Wa{4wk^Hyt8VVe<{ZvT!kLZ&^OW94$WcMC=kbYcra1vM7pOn$A
zRFqLjpNl7cERQe5Z*3RiVPB)KA1LVb(PU3RyuZO}KFvx3%HipU0gGC9nW)+9w(+>G
zIJ7x6e%J_$ZCnp>EvSCOUD107n0x7jyN;rH&|1k}RM5;HvbO!ie%0_H>;U#~*9+#_
zU?3B+GI=9fG<y*w9mw=L9RLbplwuk^j7b-$kdECj8rg&U`~o~rjjR1eec&=k*+N8A
zv*l`o-3fHk&^FE@C1QJNU;pDG|Etd$uD*Ix&&SQ$n$7R`_)CVz%C!Tu1+zW6PAf<Z
zbR|n}Q&uZMaaQn%JZ|T$zMBtLWa{w-v7@B@rL^?Krxqqsgj(ir|D7GIpPek*KaWx-
z4)BNV25sRV{6mHR(B+ZeRBAcAOGJo2*aN^-SIU1e%8JjshWt>@=>qbAVnej@Dd7p!
zS#}a4HO{BN0*=}bvw0s~F=ZuC@+UuZy+ERl*yFAIJzKGrp{fQ2u7aL5Uamb!pG93z
zI10jB6&yb!k8M}a)9>;Tj7sBsIgfq18(Tc?WxT8u19JV#=Gb5CiuQ4SE4@L8`sXLd
zNRD>0-Q1pVbTntLK$aGR_d&l9W=SVP^wN21foLT7eX{1mLzzQE?UCck(V$ejt&ydq
z#VhfoHD~$EmfFax?&UFqOXu#<I#Kj4vkPnQd{AR;OMIWRt)SLx=V(uAaNUkPk#8=M
zRKeD*&TuV<1;64B95-kSEZ`U@E<yE_v90+D<Rfafj(ZDmvOJ%9n^1~hO-^g`t)y50
zac^4@U<n2CNvhQ6%Hn}Mms&FKZk$nuit45~v{*Jih5NmRqM)OAcB83>Uu(M*HGV#;
zG-_LS=+3oLo|d};#>~TwihkE$B<gQ!qEE||+SddUO%swVHOx;+C=lfUe)%Qv2n|Kk
zbmB0h6o<DfTHjOaR=P*GoxAi37|N6M?{7;zN!bF%`0Z#^SGU$wryIu2CEhQ`9jY-s
zh7|TKLs5m5Yo8RL2Mt=j-Jnk@L?RWH5fvob&^w(wkKkxs$eEdaYjRK81oOD0>Ash@
z`&oHX$2^iDOXmX~gC4>&7Q|dk1`?iV{#ab4pyTo+$<&9QE#5$5+Jx84!*2OC`?o%Q
znYAs4&EToV_m5^IBbvXn$qmd>4Z_~RvBi|?PBQj-s7p!(*0g8fka=)V6XW^2K*60r
zx;wyW2}n~zr8r9i3xEnBtW5%0ZPq})O>J+L8MRz}G{Of*R%d>p?<|}e4M;-gB;lkp
zeLF5NWH*Jd?2Dv2#*sTEBA_*mEUnNxTG?D;H!Qr-Ha_3vjoDjEDK>vZ@Npwtwk>!|
zzLfT`!{WjdCw%QjaIcM;yNgg&C*{MR_1WKW|DRp^8B~uK{Yf<Z1@GYb4gcmYV+yP4
z5J49+YfNK;%d6=2Q)G9-?Z=3LX<1uBg3kF?Zd%gHZD%$vQSMWgq(4jIZ`l$0aV2`W
zeRgC8GSRDkkwDTIOyL=_b4molDsn`2mUfDLY$n!e9`af@%Oll{QvUN9-$3nf$x@TI
z#~v}0%ckjfGb4knVF75=K{-?3tNiuxRj)S2g+9KvWZ{PoLIOhlu4abZPL=H$TwfG#
zYx*-wqtgbkJUZ_nQ(SeQDb2>8m&TWSvKo%XMD=l{AkR|AsH;#{${ok20rja}P91ZX
z+jvoSJXQEgk+L>vStN_}q2y*6Zcy#b7heonlX$aO4cfIgXS;A-Q@GV;lv;w5!5NM=
zj%4OK{TOZm1GD#iBe9+IfMN6pbCckxWx@4^L{w#VxDQ993XN^t-*JSIp^<{fYzU=7
z^D5_);;2uxmkTiG4#3lr?eE0>xZFASXH1zB>0xd~%%lo=0$I|*FZgyv28a#KBQCLc
zQs$U1!)^y1HI3o=MwJSqJtm4;5yzXoa94i2n;B{IDfNRc_ft)^K?n3<<`pHi*wDEc
z=9qAUlu@$A^o9b<BYpbY{;zfhxn@D6R0W?`o%aoJs#n;gdpf;(rXkw3k8Rl;xJv{u
zd!ea`SBG1ALQeV@`+DhT+ca0ME!i{Kz;SJa)F??IKTWU|8D(TheqKbbXNbP16{l{>
zb?4Q|O-YuwkdU0=sSM9q!f9BImw5XU8^a-g1aE!WdH~#Pu8$NB@+mD>M;XcInW41F
zG)9u3Dj@IiW!|z9Xe9IF4p5z?8h{$=6{>y!U~RlrP-T7I`n8HTl1zALE`FD{gpI3Q
zZwvviA%2J*L-^wzfFsW!JM>H4+;pQH3k}>aj3uq|^SFz4Ip{?+!w?ha&$_*5(&#Qj
z*m7D=gJa@nHu>G*do%lmWlR&RUfYd1Lh+a2WWH*_1#}w_890sE8vW6PZHDO~bmq&N
zlCUX|d9x(?&?xe3D%vAXF>q-=Gg(~hv6Ihc53((#5{_7cyTqYwU8N&iUMIuXOEc!g
zYW{svw>>lx5&<YwTgYbsB4Kq2Z09LZPq-F}GKy-uA;^YfaHPy%%!hxa{`dUGKT6(T
zKK~9-{0n}I^mx<|+$CZ|XHopQmyez>Sy4CQN!sWjWKcw~gfC*KKMW%w&%bJsmiW4%
zM2vDOMO&_Ov$BtLmaF}-7xh@hcB83KS$wo?QF9SquQoky;mwDa)?(x>I&3TBb2zqO
zh)-GrV8eaG$Y3R;W8~cWz))yse68`wPo@Cl)gj*PO3i{%vg_sU%_w-1Qe7T2>2h5n
zr>PCa^R%T=t@rYiN$wuEzmZJjvLDmZkh;gin7E1~h}P!i;>UV>MY?MWs>NMXU;)kg
z5QLkgCyY~kjg?nxwx8p>q8M-2u<g>*+8yUS)!!$&aDH)(pwWt--C^Zv_g?79(~b&}
zm8{b}>-`|HhgQk&IrWYE!BgI<{PLVt4J34U?W<JvWunHN2YR~tvpRi5J)a(9A*`|s
zu2gP;E1~SXw#F}_<R@KLL)N01gI|5mah%P?8yCC*l5N2zzV!Qbd+)o}wFULA!v?#h
z#USZ3x|e?9+?K8*gf<GI>mfD;?CF%T#0?y+Pun`Vx^d-u!<Nm~7Srdk1=uEnLo721
z8cqD$ud!Br1$T_XY-~}|YTUr1r4F50st+=Wi0V?z%;oB8LmwID(rZ{a`>tFpYQHW~
z(C-+BVRUa!j*Y{C>f2j4fi&?hJ=n)1ql+ZvG*(7r#-s<FjfVk3{_}T$AR^*VOKBck
zJ-VNppXilc%bwpKO3O1yD=5I?i8-y1%8n}4UkcT}Ze1$ngE8^jx9yAU$RD8Q(a(Y!
z%PU-o`eJh21C-M}{8b9vJhm1@QN)$Lxq4^d-A1V$=a1Kn3Vm@Fr0cILCp5iXMq3+e
zxfe>sY<GSK2t;|GUuJ&+RLA@(gn8jCsla4NWkM@QE0dO+{&D0pQIizW`_&Cb#wPyv
zgpac8;C{G2`>r*dXQgbA6ydorHs_zB5K|+o9yM~B!I!yJJo)XjZDC>K%<ae79|P(~
zNGwNn<<ps}-$tC~jt)E_xjE#(;yURO3uqFOR_XuKY9fklC|EDQIh@C*_QZm1F$!<a
zrVgaKeGmWW0S>^hA*6d%e4~z~xLmnYPpQbl&-E4|4$1@w)K}(sWQv_G+72_&Qi>>?
zy--$b3hLC36>O^2@AAX}dLYdEpNzcm#<iZ;NKhEgn7jNaU0MVo5`(47WU1;JQBOgA
zr1Eder*-eu^*sVMM{McU4n3}K5*;q=lXGSZ2z{Ij-xT}bP5M8^EdRwa`D5IWKdC=v
z%qAA^O`nvHiL&RY5ML=}DJCg7-IA<sCHAdtQ@N!U(No^$``P~CW%pGpkGNvR3Dj(a
zOsvUN>p(1o!TqCDZ)L^i1ct0eb(;Cub)}^H^}o;T@OkDivK;L??U)pMcn{i&ad~59
zxziG~n090OV(h>axp9eSTPo78tT;P0*%|O(vJkxcuw!VbnWS8sIA)`MuT<T%ULrhh
z;?^czM?QM&Jw$J-u}wMQ{08SkXo2|3*L_7dSk27gW4Yxt2H^^tSyTi}(gz91ZX;#N
zxjS6eU!9`+FoMHN(`0FBV0^rKN*UOVPwO1wU)(1}SalaGn6IvA7CBFjpTh8EYgX^Z
z*9Z%^Sjl*!4%81*b4!XHKlIHwtz_AbRboCo_JUzHRI&};Z$Iv14vul>fybnEx7I39
z*VXj1)de04zntQ6U&>(7stf4nR}^y)8{?L}RPtn;3axO@p95NQ1dXQbvUe}j-cEkG
z^h+sWT4DKmUa;EMd3NcS|M{y*LYex|mFp!eNZqky>#Uw&Uh;k6OH3z;O-B_Y8w{J$
zX{+mJzM1$#8BJT4F}h?jLC9Bh3y}L!buaFf$TwKFlWT8{izi6K2<z*r1D5BN`8DJ;
zr>9{?7+dLc;>>Cl;qiny*pwM*6~!R8ATko!p{@~4*EMg;{b`<x(pzz@D3zAhy5|sE
zQ*<TgrV8<od|^*Y3XN*WglQqxI2Xy!Dc&H_$6rslHL_P`FOagn(3X@v=8^wCKNE$N
zx<RtBqseNzCeWyA$8-(xw6#6ie(vBHPFF0)#J9Jz(Q-DGHD+YoVP)bBx745FtZBg>
z-iY<&X;)9NvA?e3po!G4VU0=mq`=E4b5h#^olUuc`LSdtW<|el&~W;s`jO;EkZmhm
zUt7NAc-d6uf9~6HGUPl!9lS9{rFDGKc9PM#6^Vq~S9-I*27)DAP%<=UAKd{I#`Qn0
z$M-$DD#lv8*6^cj4g}BDG^}f;lLrOznhXG(aCg);uUNrjQr^C<yyr$n-^!h6N9epK
zy!OK38+H(|V`b}ySjEiSgBFj|o};o<8^+T-N^jS-D(IBE2@$iRukL`-tzh}}$+W@d
z<I)skye$x#Nx$vd%DFqztBoCPG8poN$!AudOIq&7JtWktFwF(BliUhY+CU9vDTu0B
z@#E}T?t6K~R>VR>1=FsWuF0%{fLL_edfFIm%ls6jO+|*%VIEbB17I-SWDY1+k>OI4
zU5%(OREk6k>D$R2;8AV<p<5hrcw-U-3C({vdKH5NDcnS?*$)+DGw5TysIQ>;*_RpP
zZThYh4VJ!j+Y6j_=gTQY4hzjZb(w{R0d0t?lE$fF6IGO%JJD_U1BbXZWwPFeod6#7
zDWoN+zQ#8`a&esPx&UE%4rKmt^kaSDB(!|ar&ln(1uZ_SBZYyiTITdI{++1_xC$Bk
z>CYb=%Jvty^urb?5z!#SGT0KiO43wXnVUA~MYx&rYhjIA?6robs`T&5d=1G@!WA(P
zR(iYF=<cI$3y6F}qs^8+gYhSi$WOSE4{<{#<5c+Fl^w#Ibmf_OVxN7O*Fo~XcYtlb
zeU-PKuE|!z_Doe<QUTI@kKp@dEYr=~Vk3%!j2l|6E_?g3ZFgcvCe6Bfecw-v^dYhd
zVHS+^4RjlP-~JS}^ZR#Ce?fRZ)m+1tKc_VlSk96oIV@5*E5PGJ#)1b+HHeKHWmkQf
zzLO3M{+aV7-UWKwc-#gO<gt0rh)|Nmf@&SWPIEIC0T6|_7?Vf3P2QC4)B}8*uFurQ
zGACHeX%Jp}hQ={h2vTC4mSdUYO<L!P($${Ul^hBmc+|=XC~KUYv_>6pj2fg^Ya5aB
z=)Q<EV#Iv|odX6FK!;pD8_mpiA;<K;S8B>34^*z8nyjVJVCJXH1sS%Dz(~H<k-3Z|
z9(u#Qr&E`4qf$g;&Oh1MgtdkYBf6E|!kf=y_-QNEw{cAjM3)X_0#`pbw6@@HJ>f^>
zcX2Y(`MAO^y!g`B*J~<!jJu>rOQj!fOwxdBw(Bh?r%o#)<e9w+EN<-_AiyYko07g$
zVf*Uu<m#^GfXg`~=$=>HOemCfFQ>ICV2|Qby}OFRYcV`(Lx+m_-~%Wa`V#S(JzuLN
zHwOx^A*$VEtU>(VMeC)pY@KEBm<oM(xdyQO4V2~<K%G%*w<TFBSX9E-2y6PtT%9K-
zU%s__2Pp3;9MwtI4t2RT(P|mDz4;<q!YLetg8<Q0&bP5^FnjR8ing35JYjhcK8ycx
z;QAlu+WQH9{IT%j7nt$S>i4L~_t8ZooX)9;>cdp|lWhkbb{!5n+o5WH=Szl)L?3+k
zgtnZ^>NbZqmzh=VWA>b5E{e%zJIS&|+(Q_Xn6p=bZ6{@h99p%L(80PvCmd38grsfe
zuCBb6F>d91vZYN*OEWvVNig!7<`}w=^wF=o=I_>KTi#y1)Do#LwwKOrdIP&{8=OhO
zRJBeu_MMt*yCjR5(;wjJpg3%<ygkuEu9$EuvPl-BY8Zd6t!J=3<(PxvGYV^C6gmEK
z2iQ@oD=FRD$L=n&N7Utc{p>#L=tjO@JgY(0jR)U)t=2Ix;}T=ZM0~MN+>r8Coa0zX
zo`rj%lT35CJt?NhK^Dg?9Y=WLEIBV>dl-CN!ipV2o=J5FQ1;r4@KOV@#<eALCfuuR
z<JG%-z=ElFF#7;Touv3#r*vVE;CX*NCe53Ts7^mV5Jd>94V_(iICp6QO{l9qMt#ll
zAR`Lh3Z3WVURa@rvC6kQKqc)1el;n!n%MpVD|6eK6iPI>zpUMRLd~iXo;kAD+`<oi
zaJ;RxW}2W;mDhcfB<c_Gp2}0zFV$!)IX{L2c)@vMakJW)0T{)qkKl?d$6C#p{j>6t
zwfYPi6rA?~xD9}0c!=oxK->#?c&MRI^}27y@G-w*>g}wr;E<V3D`1>2C9xr59=xpk
zK2G+Xa5p*))}lqZ4Ym(;`4)a(d072EAIlSU{Nw9Q%_j+B5m6QLt$GBL!u;bSH^J+j
z&!o$fj5&@)8A4Z27O1xxBeN5~k}gGzvxtPfpMT)|@EqypWVj+oL;@{@cd7*xC>vvD
zS=T_if+pgPXK<7wz|u!~zf8$))VzgIRX6!p-nu`|8~0NbRq}6+`A7J7Jo|s|9%*h)
zxX$M`Iap5x@vF-r4%$&W>Jr}1P*zfua9{dp9-pEM8D_dK!>jtZGgQbIx5aMMU<5;=
z+(`4*bAPJgTczuG8xO3xPO$5s=Cf>-o^^HSE5_Z$!dE18t0p2Fe2>TY;@KYA44c~N
z8UZviUZtarfeMRsm|2z-y`HqGr9Y{@D7{sK$1Ej#0|{mgJe6IMnL`cD-usl!#cyPc
zl~T$Wf)vNMFqjK3!+8*u>#4^;mRYY1>{isYPutepda0Nc<RqV*fK~?q^RzAD%A|Ew
z#?8@HiEd|aPHvb`t4q2brKI5-vqlv33C{p4KHmWZ%L78i%J@pH;q#91g0G-gtnxu_
zQHjsg3UF}INA~q0WikY4ZajmWGKk-Us#C|g7%5EE05#|w5KHkE+M0Sm>@Z`18DAY8
zk0tj46`r2qJnZE?RW)IctY$G7LQV6oL1OWgsMLs^iC)_R#0(l~OZpVP@*%}`Z1Qy7
z{nD>hhCbh8oui6*`|#4;yWu`o5n8mT;8xUzUQSoC@^q~Er+Cm!^>>AeiF*^xRiL)&
zMDQ}bhpA+Et{W|dmo_DTN4)OC&=Jih1%7h&G40v0$29)+030bk`I?MF{L`pQ&ICPG
zJ#~8++<fPCIk<QymoB;m1dyO@*ajU56wPRiPZABCe%9Lbnf!8VGmto<dvH7V0bEu9
zPbw57`(9k!J08gX%%qi@E7gT)%rbfE17FEbJv@JX20#Mw;YcnhQt`U)%e)cPEoCH9
zWjMu^CZdF+G`1eLyX)Gc{#hH{o@Q33km}{Oux7k#3{noN8aKKiY6Jhfuf#_>-Q5qa
zfqiygrOkY(8qgvgcrG;84~DXkzZ)6oMRan8s%cYp3u}uWI%=mr=xdwQ*;2sZw^Y0t
zyCrRtdYN%gx5N(A&<wK6&27=IrZ_L4MBGFyDmVVnExq11(r|K?1DVnvzs_ybcsH%$
zt#Q2!N9HN<TKC6KFH*d$59M`Uz5fhA%jeRolXFXKHyWUbGhNFuz|xA}?zbDp_L?~{
z)yN|$DJs|kP7?pM500-g<X=gK_(uqmP*pLSN4h_%viG4r>6g<kz@KAnZ9X2PfeRs^
z;1j5_y6>~y2aYm?75AhiYw&ud?;}i(7$i?jo?eSml)7_n<BUC|fovVmBAjyNAY6Py
z;rO@R|GzbC^Pjb^a;3($gp@uKZV@BC$5~oZ8*d?k7a-VoW1|=97#G%P+RrRMrY%s$
zzcd%mGj|8D{M=M4oKxc;i_DccH&woSP6ewazu?gxhpe-q>0A?qIErvODSrfzu-%SF
z0rT8>wH9=Xyg|;?#bdRxppEl%FPPAz%N)dYE!<tGhM+Aq#!NE{v&{)comn<y(w(Sh
z9%5A}L2k$g*$X5Rpnp0*Fhwq7>;>^<?p9QEeNgwzt@7jhg7WbhN*1AkFINm1g?-+(
z=6I{p;oWkNkzBu~f1>bZZrE(%r@jzhRi<}!$pem$46@(6%F(IyPv5}w3Xq+}DlXd8
z|K=Pu43a=Lr3l?9YbMBq^tp}d-Xy7dDwA_DKM)jl?%hB;ASq)be*x<qSL-!Ybcl(o
z%%@2(IB0t+K4Lv=oz^mNGi22WIv=}yu=Q72H+(TgziXz2lN8;1`o=+a7@Lg{VZf4#
zVv4uSTV=xDbwIfO6oXhl`1N=)L4%3Tlntpq5;`}h^N9S`!k@ywV*=M`C`@1b3-13o
zv;1N*JwVc<Z(N#y_brD|(i-)<O~>e1H3=X0JwQsumC*L<S`oj(wFF);Cc%>~E#b!&
z)VFv6NjLo{Cu;4X5jgghdVP*-K-2!FGp08dP&CpW1FeARi@`$i83;8k%pYz*A1MM{
z;<W|OI7s2i_1^G^zAqweH*qrGo8f0FNI-G#0LXuC<nrHcM3g`4F{0Ww2?f^i!h@@;
z%M)kZ&t4P^4ss1NQCqIunsbf3I4|R<^caD?gHXdT*6skS$$+2IjdzWB?f{u*K|e(U
z052J${##9|U0I??x2CS7D$crt7|$Zc_3dDN@tz)ma_%Njcs^lCx4=&N1OjB#VB<&r
zQolaTxJ9cGzabca?=JEpjQ&5|#(Xo0iU<6Lz7h8Q+tReFqb=x~L2D#|r>jZalG4qB
zvO3ZdsTO|T#^175>{s*gztf_=s9eMXH+Z)u3L1fRn-BKR;J;=tP8zUGvJ)E}_L*;3
zv2P<Y{=baHw7M6AOg{-C(q9SUgUpc?QRj|EyRzHLtd6r8ojs9?-B0@Fr`X}QnHHow
zp`+Ii!tVf^qFHx<#dKqsT;@;7^FQ$YztzO%33(Lh(6aCx?X}4>YV+EFjUiZH()ux(
z=1f$LV-!od=Nz;b&E`+WFW-o7SU2#53U1*XD09G*#>dF1Nuy=X&Voj6C4LrN_rJ}z
zEnUFfvUql|{wep<`|f@)3y_z6?wXL@F)n-sT=S1(U;fHJkHn(~=52X2DPBw0>5jcd
zCrSJ%j}BksQNU2UV7rKB1ixDK<jF-vDWfhlZ@9-GG6?y;C-Qv>v}h@3qsNW!WDk>q
zTqC5Vf?kA@H@Vlxy${qr>L(U`@Q}jvYNbVCk<)DaRDV@kUmvQqygucWFo#Ej1~_|7
z8dUh^JQJt<ub1!7`#y%bI@YmURkgcGj?BCvwgt0F_n)|}<wzq5h+b>0dG>^_e|uxk
z7uF=|wm05oB#B;TYiH~;#=K5{gocl(gCnu+Nr*6Md{NSI^(6Pq-8}@mgU~bd>uyog
znmS8y!iz<KL1X~Prk|tsaSfEC`^u>1t!tVADVuFT?39sNsbjIsEwsF}*=5HlCBK8M
zHOUGY+Gx4-JbFX$WhOP)pY5-#e}p*KWm|>;{kqXTc0-PG9agzjS<R0D?e>^86f6`3
zkjzOR&81rw6$M3UDE=~A6~Ss#DS1^^+Si0Ak?=~&os7ft;2K+D{e#+;y&@}nUb5m@
z`PhEKT%+eZ&F@B?o{kiiwkT+gpPU{J<?RI3gH;p?Xw<r|_-kcHU&HPIgQc887NXby
zccGq87h6=*Ak`+W{=-jhGMN2ZoDPNoOprDx1+Ld*sLsX*h7VxbT$SE!ClV?NwE7C+
znzmZFBb9@^vl{)waFiJZw=6$K@<W<fdeIqHoYB5HL;kmK%4_x!HnrRfM`ydt$eL{@
zaYm0a9LEnbaRPrCv7&9Sy!<FryM0ra_xBrQA(C>Nh=p`8#BxnV8THE1-Sl&{=D@K@
zbZp*XJPAf5y6bPKKCNt{ufDxG&2gpGF^bo>2!-g9N!|fmkTu!jqU2-Y2PQ>5&(LI}
zPkYbWEz?jN`~6*NT;-(BYKD^Bo{j&(aLy8&@{)4U%SqS5SCM@$aSjbXCS!c9R2j}C
z6hN!rQeYdJoC@4$R0tbVkQ}d_k%R7;W7EY#a-JVWJANV}cf=l+OjmjgcNj-_D^97I
zk8fC5lCW0x5p>a+hDY=+a`vv8I_qh6U-+HP=pXKtm0GElURjPq%M=HFdRY69D(`tc
znRG=yn+x6jAIT~e74IgaD)1#rD_i-rj`a$EW&nTaZQNRQp?3Sm8e`~eh0u7jcGksI
z7g(*Q^gDfens5rHaSdaJr&#whhQT!z7EaahI`|nQy=W@)%m)`!+DbN^*G_$BFXt6@
z@hPZThGsr<wO_hhyzLUY?t&1f6v3ms4Hts<5Tt-}!gNxwO?N;sK{r2MYUuGLdluaF
zbP&S`!<U&N1us|~lk>MaWZaH$U^f+z?5NrBCxZ3FBb&wDFu;5YpOU+0tT<fgLx>ra
z_fcrksNG);KSr9|EL@a{oTrNc31wo<7f>X}{^b-!u2XwLD<S&!fGf{LWsadoEV^dj
zKhZi9su4PP3T!D(h-l_)Reo=t!tV!d8%^ydj?I430LUTIn}l-;o|gj2JjTYQ8s14p
zsZ~`@?3-|$h4pFD)`iiL;w<w&k+_#rM6rmZv2W``V3y<ICY<I0nbC#Rc-c<yLvcT*
zXv*O~BoAp{5!uKCxE$?VNvCA%FAVP6T0GRrj3_ge!w_L+)py@`cLW_qldsL&1`AC_
zH;-=+Na|z>%U69CVD5j)EU}G_+I+O2(iA%IRcNn)wfpPX6_o5~U9|L(Sq$5H{7c(@
zC`vFjd=q|3X0eF)=(sbvtPWURQOm(SS@|KJm5WxkHM5al^e`E)WBF${GgEATwB%8*
zLvY^56#=BUu?$rM11T#Tw!!@>5|v(P_|nrp@dSYB_Y08J1Up>-OF^j4GEbGEexKb7
zlaeSdZDww6>8C?v{2?`LZhKPWcF&W<)Oy|94|!uumy*09wqiX=DXyndF5*GK`+S7o
zYJ(2RhjNXjv-u=h-BBJA<siuUyy-Bjr#Ky6>&~)T-oL-q)8P~%Hjq0{xA)G{zrV87
zk!yD*KCML>0@&eQcwngqKSbkuseNa4^bpl+S6#_eu88j_(Ky#~WsOI-)GF1JLH;!@
z{Nnaw!QPva_raElBLuncF67)a0jJvn6)OMuWQbq%!tbh=n%#Nt05kLCwGn)ti)nj4
zGq&spGkxsezfg0=2%x3wWg9B!jGs~xJRdF1uMrA09AL2-Z<+XfX{3ez$h<#o)pk~x
zO0XBrSIVK%qeG`La$F&BzF*YIP}&>A)B$RjE}vwD7(f!x<j_C?Vhq(fjI5wg&TA=)
zv+2+@WEGfY>$CJZWarkw<q&6gjtPgaQ0t+KK3aU6Q(p%h8PddZZkw|iLm5L^$*nHU
zSbHLskeuK44%uJWYqzyFOrAhhZ6o$R!H=}7D;}UI9?cE?q7z$y8^G=5nu+~?<C&~n
zt|K`4weGLn8WT=G4248@uoWa_q`7^5^!%mASt+*6m#c`W5ErJ%%3?*zRQ?aO_y=^~
zn$u;a5L8$Y@vd^b&&t<B0qbo`=;KKWbv2>uS1e)PNlj5IaxxH+jp1`vjd%52$avw7
zktiHD4~4A8jQYws^xT^sbl>!gjh?-2k8>bF20S3?7U*Wlg=ZcH1)9XbhriC+>%(px
zf17QjE<|(?rY}<iefngnuN^7<)NMc(^m*b`_#$1MJ%?F=^YxP=SuhtU5|z+(gfHul
KOAZ9yP5&Qd8v;fE

literal 0
HcmV?d00001

diff --git a/examples/vit-b16/mixup.py b/examples/vit-b16/mixup.py
new file mode 100644
index 000000000..822bc8659
--- /dev/null
+++ b/examples/vit-b16/mixup.py
@@ -0,0 +1,12 @@
+import torch.nn as nn
+from colossalai.registry import LOSSES
+
+@LOSSES.register_module
+class MixupLoss(nn.Module):
+    def __init__(self, loss_fn_cls):
+        super().__init__()
+        self.loss_fn = loss_fn_cls()
+
+    def forward(self, inputs, *args):
+        targets_a, targets_b, lam = args
+        return lam * self.loss_fn(inputs, targets_a) + (1 - lam) * self.loss_fn(inputs, targets_b)
diff --git a/examples/vit-b16/train_dali.py b/examples/vit-b16/train_dali.py
new file mode 100644
index 000000000..fed39c3cc
--- /dev/null
+++ b/examples/vit-b16/train_dali.py
@@ -0,0 +1,70 @@
+import glob
+import os
+import colossalai
+from colossalai.context import ParallelMode
+from colossalai.core import global_context as gpc
+from colossalai.logging import get_global_dist_logger
+from colossalai.trainer import Trainer
+from colossalai.utils import set_global_multitimer_status
+from dataloader.imagenet_dali_dataloader import DaliDataloader
+
+
+def build_dali_train():
+    root = gpc.config.dali.root
+    train_pat = os.path.join(root, 'train/*')
+    train_idx_pat = os.path.join(root, 'idx_files/train/*')
+    return DaliDataloader(
+        sorted(glob.glob(train_pat)),
+        sorted(glob.glob(train_idx_pat)),
+        batch_size=gpc.config.BATCH_SIZE,
+        shard_id=gpc.get_local_rank(ParallelMode.DATA),
+        num_shards=gpc.get_world_size(ParallelMode.DATA),
+        training=True,
+        gpu_aug=gpc.config.dali.gpu_aug,
+        cuda=True,
+        mixup_alpha=gpc.config.dali.mixup_alpha
+    )
+
+
+def build_dali_test():
+    root = gpc.config.dali.root
+    val_pat = os.path.join(root, 'validation/*')
+    val_idx_pat = os.path.join(root, 'idx_files/validation/*')
+    return DaliDataloader(
+        sorted(glob.glob(val_pat)),
+        sorted(glob.glob(val_idx_pat)),
+        batch_size=gpc.config.BATCH_SIZE,
+        shard_id=gpc.get_local_rank(ParallelMode.DATA),
+        num_shards=gpc.get_world_size(ParallelMode.DATA),
+        training=False,
+        # gpu_aug=gpc.config.dali.gpu_aug,
+        gpu_aug=False,
+        cuda=True,
+        mixup_alpha=gpc.config.dali.mixup_alpha
+    )
+
+
+def main():
+    engine, train_dataloader, test_dataloader = colossalai.initialize(
+        train_dataloader=build_dali_train,
+        test_dataloader=build_dali_test
+    )
+    logger = get_global_dist_logger()
+    set_global_multitimer_status(True)
+    timer = colossalai.utils.get_global_multitimer()
+    trainer = Trainer(engine=engine,
+                      verbose=True,
+                      timer=timer)
+
+    trainer.fit(
+        train_dataloader=train_dataloader,
+        test_dataloader=test_dataloader,
+        epochs=gpc.config.NUM_EPOCHS,
+        hooks_cfg=gpc.config.hooks,
+        display_progress=True,
+        test_interval=1
+    )
+
+
+if __name__ == '__main__':
+    main()
diff --git a/examples/vit-b16/vit-b16.py b/examples/vit-b16/vit-b16.py
new file mode 100755
index 000000000..ac51e226e
--- /dev/null
+++ b/examples/vit-b16/vit-b16.py
@@ -0,0 +1,78 @@
+from colossalai.engine import AMP_TYPE
+from torch.nn import CrossEntropyLoss
+from mixup import MixupLoss
+from hooks import TotalBatchsizeHook
+from colossalai.registry import MODELS
+from timm.models import vit_base_patch16_224
+
+MODELS.register_module(vit_base_patch16_224)
+
+LOG_NAME = 'vit-b16-1k-32k-mixup-light2'
+# ViT Base
+BATCH_SIZE = 256
+DROP_RATE = 0.1
+NUM_EPOCHS = 300
+
+parallel = dict(
+    pipeline=dict(size=1),
+    tensor=dict(size=1, mode=None),
+)
+
+optimizer = dict(
+    type='Lamb',
+    lr=1.8e-2,
+    weight_decay=0.1,
+)
+
+
+loss = dict(
+    type='MixupLoss',
+    loss_fn_cls=CrossEntropyLoss
+)
+
+model = dict(
+    type='vit_base_patch16_224',
+    drop_rate=DROP_RATE,
+)
+
+hooks = [
+    dict(type='LogMetricByEpochHook'),
+    dict(type='AccuracyHook'),
+    dict(type='LossHook'),
+    dict(type='TotalBatchsizeHook'),
+    dict(type='TensorboardHook', log_dir=f'./tb_logs/{LOG_NAME}'),
+    dict(type='SaveCheckpointHook', interval=1,
+         checkpoint_dir=f'./ckpt/{LOG_NAME}'),
+    # dict(type='LoadCheckpointHook', epoch=10,
+    #      checkpoint_dir=f'./ckpt/{LOG_NAME}'),
+    dict(
+        type='LRSchedulerHook',
+        by_epoch=True,
+        lr_scheduler_cfg=dict(
+            type='LinearWarmupLR',
+            warmup_steps=150
+        )
+    ),
+]
+
+fp16 = dict(
+    mode=AMP_TYPE.TORCH,
+)
+
+
+logging = dict(
+    root_path=f"./logs/{LOG_NAME}"
+)
+
+dali = dict(
+    root='./dataset/ILSVRC2012_1k',
+    gpu_aug=True,
+    mixup_alpha=0.2
+)
+
+engine = dict(
+    schedule=None,
+    gradient_handlers=None,
+    gradient_accumulation=16,
+    gradient_clipping=1.0,
+)