From 50aaa711ea8fc129af30ff618152a87fc0bfb4be Mon Sep 17 00:00:00 2001 From: Hubert <42952108+yingfhu@users.noreply.github.com> Date: Tue, 1 Nov 2022 18:54:06 +0800 Subject: [PATCH] [Docs] Add custom evaluation docs (#1130) * [Docs] Add evaluation docs * minor fix * Fix docs. Co-authored-by: mzr1996 --- docs/en/_static/css/readthedocs.css | 6 + docs/en/_static/image/confusion-matrix.png | Bin 0 -> 51804 bytes docs/en/advanced_guides/evaluation.md | 104 +++++++++++++- docs/zh_CN/_static/css/readthedocs.css | 6 + docs/zh_CN/_static/image/confusion-matrix.png | 1 + docs/zh_CN/advanced_guides/evaluation.md | 3 + mmcls/evaluation/metrics/multi_label.py | 127 +++++++++++------- mmcls/evaluation/metrics/single_label.py | 118 ++++++++++------ 8 files changed, 269 insertions(+), 96 deletions(-) create mode 100755 docs/en/_static/image/confusion-matrix.png create mode 120000 docs/zh_CN/_static/image/confusion-matrix.png diff --git a/docs/en/_static/css/readthedocs.css b/docs/en/_static/css/readthedocs.css index 577a67a8..fbf96d37 100644 --- a/docs/en/_static/css/readthedocs.css +++ b/docs/en/_static/css/readthedocs.css @@ -25,3 +25,9 @@ article.pytorch-article section table code { table.autosummary td { width: 50% } + +img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} diff --git a/docs/en/_static/image/confusion-matrix.png b/docs/en/_static/image/confusion-matrix.png new file mode 100755 index 0000000000000000000000000000000000000000..a1dc7ba6a73700ff55f81e40d00bc16f4da26b31 GIT binary patch literal 51804 zcmd3NbyOQ)*Dn20N{f^N1&X&p3j~T6FOcHyPH~4K!KH#EP~5$^6WpaO?gfHNXmAZ~ zL2mlK@3-#y{=Ij7>)u(DHJLLrXZG3W%s$WF`#DfG6`&yJKNJ?f(0FkjVa&8dFH@@m|kE)5Y4u%goIROViozqnW#h zl`+NW5*F4=EP0TmmUqV9l9v(bCOz&UOp3(v35go$p_ZDuIx&b9;s+~Dv>$J&$QzwE zxTu_-9%q|YKP#GU?}6fDt|#Q|5q{ZYig|c?(Z}@aGd?x(ZJX(hE2Ci6;`)KO&q4Dh zmH*}8XN(kBSRIapyDk6vhXp38e(|q=|LySUS;T|CrH}tz`v1!*B%w?wK*3rvG65T_ z^qsC^2qc53qjL8PMpAh7>f}h75H6e?`>)_H|J&i2IrrZ}tXKao{U10bemw*Jt1)pQ zW-$XGudcd-97v0mwaJ~~*xTMX`3aUi z0Dy*uj=l3@zJ!%&*}By~{&s)6`n!OwmwF5oDwmoEPsxMJOBcjLRm_bBJsVHwVNMl7 z!h@x5_kxa$y2(Z&d{q$gM$ze#a0-N#Kvn_1AV zHAIghjZO0xRM$GpfJ*l_Co&g*H<$nz`~-}cGfx2Zan0N;jN_s4z+w!;M!0uEXcnuf&t^~4CNJp zVaVy4K?DC>R_mrFiiljPpVBjsAkD;<*@g`B!rU%9H7&hUf#Z=eBKI}xKF$5fOo`(6 zdCZXsTHTQz5jD%}xR)&Sgg~{$Sbb~@-hj=@9U%4V5;no_7xc%+IDrpbo<1Z`(Dxjj z{ZkRT#RH@le{N~%ES&v9+qXRK{k7UlR{!(_@VVKs;savh62S%GN~q+(&TFfoyN$k{ zE_xD^u=`^WW}`akrPQ+%&|=QLpm!rdjCRD1YVcV_fx`*|l&I@9v~~Unr~v@iFQ0de z4Ga+x4!yl%V`;BC<)b>uFwk@0QvJ}<(x8(HQ)$^Pzk7MYgX@@P0VqeSnGR@ zm(;;y8Xh@+|E1{+7&#%i-T)?_sm;W)YhaVn_4PyUC%q!Evw9r^fQ+)Wszw z&$z3b)OndnN!7a^QL%9pxdga(_2}$eFFssR8DTHx6TZb}RCj&DtsQIlY)`sB9;yu!m}qpa|5*uQP8yGy z98l==O58M=mf}~iG8a-Xs&pEgj!+(aAv@<>K49Cd#3AC7_e$Epqs&_JfwnRFm%-(G)#y~>+N2qaw9c(L4ceYrL3yH%T-}pe-16n zmj&77dZC8sP}0`cJ=bQu_SJ2RAv{!UhVzYk6>m=1cZ-uL%cuB4yIm-^8eZ0RBkz_H;tK-jd_F@nx0T=DkglCjq9+bUtUC z?P?|Z4oh=lb%ug*-GhtyrcarO)f?}uBfjP1bDV2aXUvc9z zo%KeB&2B$^6LWR+Lz5Ps>2A7cnQAQQIcjwwsEg=uF)zQ5$V-xRZ4dmK6so*sWl%7P zt7iJwTK?#Rf0rj6$vIa>?+i>mpvYfuO5~pkD_{G@;Dr$PlU4o(zGk(OB_=k7)#|v8 zSRJJ27tHW>d(DuAf4M;hlD)XKf-whDWz;B8xVIq32prPw2eo`UD8$=oUEe@4Mru97 zb>ysp+hjQ>)?V8_9(>K0c%_F5iFOG=S#2o2mnW8Y08F5X1g=#KpPPjEumd{jimO)6 z*Zc1ofuG!KYepCQqqbdy`{Y`FQNQ5Z+*Ql{;)s5Qb{Y&VD;b{Dp2$O#Yy5c*lL1tg zhRc^Uz_GG%yUnq%iUUfiA_~*l{dKig&fh4z1`(t3uybeppdtx!(JErHAkgBpXgkss zuj?z1!m#9vjBJC9O{%S(64G@N;(EJ$7Q@E0*jtylJrDZa_T0NE9&|O{NQ5#xZ9s`p zX(S<=m-xw&b8zAn_VE)2sZ_L~{f*Pl=te)edy zV>`#1Ia8;^Y}=6FfjH5>MsLwF#kDv`ZP3>?FN$_OVg23w*M4}K2NSyUFiMen8Bte` zX|=e3{N;tUIG;2K-jLl{8J4NO=0+#Hj-yAvv$r1>mYc%bpsk-m7GE|aqj(Ifi8~dP zD&fXg6bms-VF#If;`*m!17nPA@J>#a4&L7n_1N~N3be77gD%WX9 zoG5X18fhjt-u?6SWj2^YJS*ZT2cUBdsuhj;r!LvyZx1D~r^ZK;~?5IOZ3r-C-F@TeinM|mgVma0KU zoU+IL-*!sdg_0!$#rRhE&1oa@*cwamNZDrQaDGrc`1`V*0<9d!eHW%{enoa~E^5Xr z;mGC$HdDK$@A8Rkf$VQK(}PGpYT z{cgmcs8ew@iB5OSp1o^|{|xnj!}q-BUe3kLI6%e^3l%{5%S}h!#Dxy*m)%Vh8oU5Y zl!Zh8E`QAka@1Zq0a`3c8gPen8J^ZJwHMHuSZ7ZQiL;aa$|(xa;BhKifZ5@YQ)N)T zin&V8dCfkh;`%|fB)*IHyth(K3^^t)+b-KeUG~HV*#UWO*X1sJ7wD9 z9&zVbN0(NindQeK6smFx_Vy}W3&K%f~R7rcz+Ecl4ycTEa!+{LDwv0iFekMnK>UQv<<>F*9^ej%vHf|T&sTo!W zUojAab*|agB=^qhN>AU=9J9`$jP+?XpSQC^tWewKGLGZ2 z6_D;eS281__b9igO@-epUwk6_iv!`Nzb19_xERTj(%B+{b8YAD@oO7RRi6MqDEHDI zPw+NmDa%)w2RZ+mq+l3JP|v>z`PuPga}6)P1*IKGXIM@a&X$rKQp6>5$Ca~JEn&%2 z>w1I95aCJYh6p%2%hVkI6&6AAg`o+>CAK1ogPUuwAO36wBAI8=V3QN4+{F)PL89<&e z+91_PZFXQ!mA7{(cG{FC%0yO(+`UD6efgEYX5@9B;O7#dpsXD$L}C48nW9vn{PzY{ z@fekVvB|~8>r@(WHB;-0QWb=4(ENsAPB7d5w8sM}{uxF@ zU&{N}V^Jo|BoD<|2Y@x|EA{uxL>6z&rXJU0TEhhUUdSS?mI1f zSZWVuo&Lm!=n-I@>8{P`i&6Ly$L@O~{6?9<#xz*0^qz*79y!D~SXu_5efGFrGQRT$ z`!+@j<-5NyJpq4=(ek^3_LgcOV>0BS-6owZ&5Ht4cKp|nGoRw@ zsW1Jt&}Cvmprt@-Z86qwK8JM*6;5ll-lZN*$6(^^?3mb6y7+^tIZ2SC<7 zyDwur!H1BCsV){>*d#Sjty6>aRK8UQ){d_wFL|$=q`dVMFm8pMH7E+9*yi6ySBVxp}~4Bb*PU&P+86ov0cn_RHsM5`HA`KX*; zJ|RBoh?)5fXF5dgeWS7nLqOoRDg)C&Ps1fMP~Ev(F*}80f^QFAnHVK3dmX^ODZ_cR zZ+>={RH(roESC^ZPD}GF`8|C0M{cI8K7vU^yN$#t9r`J7K9^2zS$r&;7~}hYz8m_TY0WDJr<^m293WYr>+; zQO)Xn{SpuItTX1|VsAOqc3@%NiB_Z*C5Mwbz#TgrsCCB;GGI*w|H>WC16g?*st-D3 ze-ivBb+iaa{;mTSk23+(lki%2L@(}NJbLmJ^~ zbV78!*5B4Ud1Zf>B{qak2!gwt7K|*`3%K1-eJi*VpG0uKBeFTwO@oXXZgONZu#>&b%k0nv<|vJ+5GF*D|%}h4A%g#vOWM!3_GQpk0H;NkvW;hD;@85Im21A;dr30_s zXBL<70za^(ls-f+{iNvNjq@u}XmvUr7hj_ACK)ehzoPc-MwkeE+{uv}D6L&O@Eg-F zdFr_^VbeP(g_)R(4n9OdHZsgwM*I; z2(uqw;);TevYim8FWfo9!Biw6+399ecMG%}S6nlFR#6rgloY=PxoR>9vMZhnd<|lY zg+f^cF9$cOjYDrpUuK$(W?mUzE6uGN2$RNJMn($Lan|}PFY(L}s+uYqpWm|uwh&x^ zJ&pB>;%w%PWCMlo#Z(ISA{Zoh;%_w{=|u^xWUEhexG(AoowAN%r1hqwRfX3ef&+)| zKljl;e4X-k>(BT*0061HZKLTqz8gscbGlVh3@QszFQOo-LYy+D2b2(^D52AE%~3f=lRd zZ33r)jz*AA=cUgHws_pu<&%_@K+Qk<$SS({}Ie42h&}up4+l}Rb=}nG$3x{UN zt4JZl;CLo?IyUTAh@O)^Q1{nQm4)qN36a*iH69nba*cx*>3NN+}{8gB!n0c ze#sS=iF1PS5xOcHpNV~^?mQ?PgJ_#s6yAZ1BHkCZk*@L_$H&K;dS_%gyb^Du9`=gp z?^hj-DZ3*Mq0bUnDMAmw)^A6j8tWdQ-}7b%ph{m&5LTQ2;OHUx^aSnxt3TigD1Y3k zaOi&2bP1jA&NcbjNf}byyn6X(SWfbO*>^-3+~2-BOnno=;O9>lAcU@@6aJXq`AIHs zA@{HR*wQA;qE&U&q-r>^t=4z@hS#mH@kH+0^VW(&WExXBkz8i|L+QBFgdaL|6WoJR zSAXHVgcp^vggppSdR}sysdJX#1h?6#_wC zC@BvghHBXiCUC)oPY2sX^!*k zqfcC6RqW`X23SR)k|rt1L!TUGip8ef1kaU?CoPb z`rOOF`^JvV{@IK~FWq)!(d+M7+RbbEpQ*kd`fWeB z>~Mc-C37r%E+?6(pIUmooy&O(Hh2f>&OSpFj07mvdxN^i-)y}#A1OG|yaa)s1l@uK z^#korH|FQxM?V}!2Da{^I(#(lp90f)vIB;ZC(-n@)B|uCyvkAZzRp0nM>Rb7;e#V# zu69~Ci`|5u0oRdNe|}J+Ov|AVxyJiCR7$+Z>>pg19`jS5kHIC;6IX#BH&v0R1_56d zjUJR-AlB&IEb8M6>@>yN8`6Xv0>VWq!9>l~;P6Yu z;+bi;y|KRbtbK}2#-EWw)CX*{B=kc?)eGVNPe-i-hvR?lrH?I5M_1dI70qbD3PSd0 z&yr2TjP<8e6{$)i8CShr=g$__i~htWH>hkcjms_Sr03;W2!H4~D~q@sW)bB~B2X4??x)dp z_`vs6gojJ-Jh@6|^rV{Wfb<~DoU5QTn{&N~u#ng?{c4g%v2F~0u*;Vzv}qbZ2e_X; z>FP8gl_PJg(kuzh%n+~$x_AF483GCJy8|Yc7I-6WP`w%s4WT(y4PCw5EVI8mjRO?g zf$h(SvV~_iCf~MCKg?P^M{nBnHzustxu&7I3GlK)WQSB&+;5W0hQ((uP}1%Sg4OFO zQJ6_wa5(%qjnGcvM@4_e>>tkTzP`Q`q~TX5cX#z`jahG7!~%~VN6pU}N*{-ZPF>PJ zsxxn=jI`iYYm0tt0_xGWJ#zbEbZ;p@A2j}F675vg^^`f*>JcasM{zs#9ycBW{w?t~hbtwmplk*~{59yD- zv)6kKYwnv^C(=?jE%EJ-c3Ek_6?mSv>TS^=#=>MV<_O)!r@W>XlS~X1`V_@ZVhICH zUs}1oMwpHcS$h5upe0ZY`p4M#sAG?oqmjqKzL6=t^Xctx`JdM%LH{nR9j>(WoU1flge=|mthSl&ir#$WbE1yNk!G}3d;J|& zr!RkObL;qrs@%>f-{IAtYB;7}G0Zl4-NNg*(bsPr=b&e2O&^F{@9pz^**h3Vwr{3h z9DR4OB`f7~7%i$=kdg6*Mr$y%%&aklPt+`Hd&VWiu>2WT>#W?s>y5Qu{Max%a&4{< z?(J2}mv~#CWzp(%@gzV|dtqq!d&hpmR^tmS`=@>O*MAZP!(0fS#k|=$Mcx%9AJH;b z7zhQ_0nz+@!VH7*#MvY!&M%xB({JVUQeKG@wte%K){;SKfk4lx8SF%C3K4Z~1;mV3 zXmMZf%|z25{4RD8P%knkDt&_~iA>k94&k*_nYDl^5mycNW;VvynToH&j!q(!L(U^j zYOW|ve+zNHAZ!rEp7ZlC;4$H_ppIP~AE|7jX5$nRd{2{5s?!r?l@p59v_%9T#v)81 zm%odKKSe>aa^Z3y8nIOY&>ztAB((ydx~{!hV}T$sSLy>^hW0_)JW%tW2M2Tq;(>v5 z7X@UK69u}Go=KM35&}~PQC})0+?ZA$_I49PgR&axDV17Xo{i6;#dm2c*2Md$FXDAE z@pzl1UcC+QUc=sBFPPcR5Y}{Hb=$XSr*x71ODp;-{v*1ZR2RWbL}wt~@0tS<2b=C5 zoba&VlGs1%+HCb4<6lY2asl%9AZ4u~2;(v6iC0iIzmH3ab53ayRd(gVC7A#jIkej_7wlu0J4`gw$n{@bfJO=Z`lF#Tl;cAlxfgUM#ChhJ#Xa$P=1-@pYc@Nl)7$`vACi9)HllP(%4?$nxEFv=V z#s(fcoDQ15qE+Rg>~s^ZDFQj^I{K@nUNq}Q?1_}Qhg~Y??sQO96;axF6@dU6Lqz}> zzAs@shvs9`1&e3qoY}G;ADdl&u9?lvaVdq#G|QX0bU*pD*<@9b8w#G+UUY zlFJKFex-}`l!G_R08VcC=Do;{dTcK>N5WfCzyspgt73ZGI!^aRYq6q>5EM?|F|F% z9z*W$6iZ^TTAk1PBUpE$*|8H1d% z-K}N5j@w=09smGB3;_p;T)glRrJr0Gs+z#Ph8~2;Sd4e>vDd<0tAk89Z}rbLh8a(S z*dRBX({UEUa{&+iObLA z8_%TJ+vg~~+`hcF76s&1X)}az2nFtse)aRBTWnpWd=Z~rkWFc0Fjh za+-q#R$B}n%Q_k&$El7g;w`H@axDfp9}nK!C+0yrnv@;iw-Q!nq#>LJn(md3NSRDw zmuI%yKH(asd#*e*UeIl^41`wAmn_+2Un2?_TCwu^eFiHeu+0M0_<$OFaO-5QI9`nRS%c^LpC3lV zwT}g1AGiO;WGDB8?q8lX%^1-q_l$P+MN*184N6`rMrTpq4}#A=;12*59-Z7}=`Z!JQwijCpB>M5qVO32 zNEI3z!wUCvo4l*v^$U#Aq06^Uhyh%>ko6~g?hML=XkCIYF&+bMySLc!n+cPd?N2kY< z?YZ%3vRE{P)eQArT=POSDn_*~H=|DP9xa>vZMSj3bYs0?)sw4!g*f)j&h-W*a>>)^ zo)ft=tu^*N)eO<>hoP^)*!LP7|8nwjgeQwwi$$?M859`qnAy>TZBF${18_{d2Cn@*I~~6 z^(|#fOP(Xq(A;MYrNI@sTZI}=lpkJPllTA?r}yp+@MA@QiEOy8(iKNHcwvbIqsJCH z`;%4=Z^Qx*?JwMfziFa6ckix_YKT6NVrHR?9{iKJS^NI%3F>rjL}ujDIP6=8t^;kY#57!7?+(ul@qjWn*zhDQz%T!s zw+!5Tt#f8Bd)hDurlSuU69l6gtKk!ym?R1F)5Yu+XLdXh@m}BRFr;bjqXe5SStBX^1r;4Rn^2TP zUFM40sxuQ0H}Q7#vX^}zd*w~U-#{+EmJvW@7d7gKaKF+y{=s2pZ|SXPi`kC`w3luc zQqLceWfwH8?x3R^qk;O40Dwu^gEFJ`(FoqZHHrIrb;egW{qvfIn}aW7pnGQnad2@u zyOH1YQ~pFf$TZ7ABX|5ICX{?@kynX$UYKh$V{7A^;dd+|-V-&lHOmK2-|~t*hSZ5#jKYACSy=!@`&ccShrR+OeZx0b$-&kbgLNU^xe?iu$bJ_FskGvHMxi! zY=e%WTC%3=*sa$0-9vhy=k~9K2QG|a$jLC;N@aeDJlfY;IudWrFGKd|advc+xeV-^ zzeIG_G;~=L!G z8h4vMl3dH0b-mIiKnC+mrCur2Xgx5sfe0of8k zJEXCSAZ3xW+WS(?1@gV!?=O!MVPp8Juwo;?*|biSX9MkmeaW-+heq>3!Y>>3$F^%{ zuY+Eatb?w_ypacFi9;N@MNH^@ezH3cXi;yvYO^9}qf^Yg>UTzc-Gr;{ucO%@+ys-& z+>GF7omDHzbg(!GB+b*U%GSkxeEHoG#Y;?TC^q7AGras_*yo(A{lVslPtkO(3n}q7 zhtEpqM4`;+g0@a=ea|{9hR5q2lw_ejwOKmZ(@7+qQfr0XcpR1_6^ARxOl+5;cUJA5 zAoh=c3noA>DEb54d^yY1%3Jd-jdXkS?APYc-NtU3G+#*PnhV|Ss@r{q3>sz=Wwm|( z8G;x0PXL`n%hffo_AzeQsFR-8Q%UQ7cEs-)uD0Bi@T0;^nqdRVeYQDh=s0OkH87BC(v*GQ$dAIo|-;yW`ZvQCc9W~r3 zON3b8`Jz*(N9(8FL0KK{J_CTNLt<^XxTVFc@RD!L)R)ECu2GmR6g1Ln{z<{Rrmg2U ztrQ?_BkfSu6h~5X{$qj4Me;ntY5XrWz&`Z!XlhR1C)~M;M65L^ijt?Ex2!e2pOovC z`9S5T{~V-cv>OXtBFDJw!n^$1{{=Yrf}hLU_MHvydxohN-|D|{HVoN2RuY}1G}L(r zw%C+w9gks?^IvcA|NXJBi+Op)YL#uVL@pBFZOJEsa;NohS9NEg8jIdA-5L3GgHwsB zRc1aiG?&dYj3W;IuvAcTKV z@B(XRF+RlsZ2a&OS6b${8i3@|s2D?+_aFDx$@|}Ad9Bul*{hXZs@J%zZvHJo#08e* zaIoZxW^m;POqKR{|DBHVA6Cl$XG;VAA4s*+>4@kIT>zsTXHc8C17KrvtWRV5%e5A} zz}`Ds?yLUH(-uzrcR;nu-GAP-GYvvdPr^rZ#`zm}ek&Cm48~qxUuPb^0Zc%HbiE3& zie2y!?N81E(uXGPr{;pk$_8nIUGhxAtnv8MJojde$XMdJq&&a3n%WW&+x4y!^Ox)* z-_7BjbbXc1a~JZhPzHJkfsNCyAIS0)1MWM4XeS5@010cOZ1`1EIfR|v&}~>dL;Iv; zk#C1#W8Y|2y}}buYd7xZUVjIfrM3emlrFo|nLf`f-+8Mbld;wK!4yCoJN;{ahCEH% z_fbi-;M*HdRXGsc{o)7_YB%-qL5>D8dsOc#Th}>twfF*ku_CuSc3x-te51i@rz3^~ zpdpyiY&qJdvJ);l>gTx^ZI`Sm*NNyflrfl@$x{+Cp)s`Gjl=K;_t#wgGQh^kMJAVy z7evKQb0tSejH}&dU?3AUmNhVW7r~&s3%=i@v@%ju{2cv?X-&r8(Tc@4Ax7WD!u8kJ z)jm<=+%h!!?4<1wH`bsPde4!5@B@~Yci1`%I#RRtlq~TgrN60j%66fnJTq%mN5KEF#B!td{9fkrBi5t8umjSnqvK=Q=mx!Y4U4wNp3aj?lhtpnU=nKp zXI;%DOgrp;8L$ra6&6m`XyXrLcaZBIU$?GU zklJtvC#vZQ_DncA4H;>z*Ewhlj~8BM<3~n*i#|IHm(zTMP5U(p=`>$lFS%;K#sp`q zlyu*3;p~!SAh|=`W@O$@waDA4Ui^# z47GJVnuxyd5{wfeG7WyULmt*g_9U3GDe#At(M%$3!cL1QxNz!l>gZg^2Sr^fFbyx> z{ROAM%oS{_#V6@I+ecI5QS{$Qnr_b916$fghfFAIP=H|<%((o3(pkAtMM|~dqDTF^y|V3Zlqb?_B^L2VHGBuA&9j|iSwROEAVZy zM;Pi7EMy@Z4O|l~Bk(1x~!OkpVbg^oK9RzNjH;9RrAXdLw&;oRP z`Q_aa=)=y4$)irKu~RSHEjc+29;jBlVSwL{J6-ruNX0?=zwdv~xxFTEaVge0w&gN@ zg}AFV%Wq-`Z%Y=@b#9N7P2hU6V|K4}{U`X8&BL%r;{06T*?s(lFT#adr6?wJDmSRe zi$Zsddm#c2>@;le`Kk7kvhd9;c$;R1G1q!B8I|(8rn~i5qdM1tpF|64ois@nG}3Zj zwQslkrnn+7(Kq%h59Cl_!H?I*~&LA|Y3Q0;9*4j@r*T zI3ofG>IZfOfJt^s>QF-B`;RxjYGDw(S|*BAPE?IIKuonH(y-dsN_XH$c|t;d2o>mg zJ0Ih-f7Ww`xJ#DGM=z!Ibh7T1Gu&>_ci!Ke(w2BFbOg9{l87B?JJrKUAw6o^N71Tc zL`=Ddzcq9}y+3M9Cd}|<)zG`qu)REC4-1;wYS`I<_6V#moe^((G4UB4BxZD1WgzeA zf(u?ldcMSH;`iNu6jx9)Z*KQ$rq6_%{M6S@_!!i6D!-65H2pd_7o#(@lMCxAwEDWYU(Ix6`}8YYuZHwyG_CrYo8gCjZ4JCB zv?>wk2r2QpxV_qsRZIOU64Gnbw(5ISgz;RR^)HIBJ4?e%m>h~INuzE&rQ}Y}Psb|7 zP|@b7vK)f#S6<{?BHnB!j3@KyzjaN}=Lb_&nL7=xwiL=qC3v}XcU80R1KG%RyEMqX zY-O~r2xFhSa?5lK#kQUPPC?!u^jcE%A@g4r2F3j@<^Z^IIKS*Ci1d*@wH~P~LeT8Z zgxCAO{_Lk3h6xj=;;pM@Xd430`jg#*)iLyYkv3(_DyYTY^?c*2CZvcR`JrpN*codNi&6 zH|oaXiOzIk$o0@kJw7@%IVzWH)`%TF7Fz=k2*jD zvg@{#{8L(hK}F26D;59=^Rh|PiuUi8ubl?{*`G*YBX{y`;c$-@q8+C~S)W+0gQe*H z=a`%J??$Sga3q7*wmeh$$LEBzS;yT`tB(P>y-B3k>5Ax7+qpdDIbJms&+pV#UpWwC z%gVWLR0Ylu9y>vh+m{j$FWu6D-o{yQ8R@1Iv5R?>4sMya7@?LsX&dY+xG+L4#oKlM zTMDq%%irN}J)_ZH^&2`C!`@0UBV5`L5D+l-fJ^|mL<4D4UjYU$JoS0AI!~q#@rnj6 z{Z@V)_b}9-7oukY6H5{(-G5dO3@os zsO81xT{Mq0=vV%2#AZnO?16Bp4;fK=nhtw%gGlx36o_bJX7jSmUCdy2y+?M}l1*#4 z-#ZPb30SlR1+4;}Yv&E0SQu;HiCXlNfkqW2*M#gM*7H%h&Guo16}xnTorNAP;OaUU zCh{6-S{e!T9os5>5AR=AXftPTe>J>k)UgR7v1)5U!&C)`)zj%D0+`xOn`0xbgp8%@ z8|lr_n;TSp?FG{8>?`p37}>U{`oa!6QU~-LJgZ0( z;iqg|JKm8|A}Yvv$O!lfGk)pW-8Zc_l+64fs=Ap%ZL~~6~j+auM zamvEbF=t1`+$^is%0zvBwhD^-HNb?mW2IhGpL?Y(WkYV>BlR5Le3)MQBSUttH%Q(k z<3KB%DbFZ;u->`&s|GG#c$nJvq!3Z=iCrEMukmMhiM=~jFX+bxuf$sgs>B~^%l^hDYzE5fI@wWlG4|rMd?$4f z@q3B4zg@PMyh)l}7Q7}(zjcHSjoZ(zCE_MK`71JdWC!_vyr(QAQ=WYt9i;aoSR!im zi*jG5Qw6)plTp7(22YsYQCT!(xkW|9d{qQt#QdPp^cwyB(t{_ke)DD&S+X%AeAb4r^z{QQI&kSkEj zEHo$(t}%`h*oMenc1k-`qd{4>*F_gkmA3)pCqZK+hm-429 zItiENJ?@6KQe;|=qnTJ-Gm^dSZcBgrOnVFEnfbb?$I`)Kdq_U|ZYfm()LpOVX&#kl zWZ_(S_d{qQiJFx7$*u9|qZxJ8u``g-WoA)ec~FN-M83uLSyuLz=WnM<<$=Po!N}GC z5i??UW97@=dAX{E{?u$8)8>sI)bLZOc@p>tS}PGjFSq=zN|6Z|8VGNC>#9kzKEt|w zuuyNiwXx9$5Qve$D3CvYC(OiZQmFGSPpj0;9YY`cJUSqDrunOG_qOM{yp7!@ChNp! z*KWJ3O4Q7B+ApFm(n|?l)rL{1+yMy2bBhOhV^?eG6 zY(&A4^9xjuYPs&w7MpSVjHl?ggjS&%*cNf?%XrO~7qplT<1!M@Dgb<*H2VdxHb{~q zq)%Ti&n^_Qow^~btm^Bd$1wQg$7_3QjK1qS&upR}fIta>lW1*a4)w$7fSzzC@nMNx z-Xf1!9lG|D7WcFg`Kfzr0Yfu6{UO)$gz4drhi5E5dE664``CC{I!`T2?df;>I^3vR zpD2~#kK;7XFflm(*_G`}#r`@q(;i;7oad6^rvL&O@UjpJn9V(Qk`Xy$1mq5rcI4En z?Tq#1S#$*&ja}PZn5rGw4o22#lX+~FE%r(pCC6lHUzc^OB(Qe~GO(1Ll1Z-1P#q&W zzPBc)IGp-#J{c`opo!NgdSxMlY0}R7&SWx?H%iVug2RqTRyhE8)6cPMGP zgdh1cDrX3AR3zd`vUK=UNn(^0?6 zOv<+Q#c<7>ufRqk+Zy<$63=vBHD;LGRBlmxDmfEGb}S*qu)~M)T&zd^BeI>vyhW7y z1=jm=7OU^oHky5MSBd=B{QB7ZH^84?8^BDABe<430>C0^5C~QDHX`cQ{lswY%@woQ z8$6=+a*_@i9Q3i8MiLEC05YCU09xroVNHe5UZ8)Jc(Sh;xG_qj!XLCXmvsAkp*uea z7XzBT$rAd7WpMc2P1d^WemygiSUD~&&KQTX@*U={vCr#D@-A_bBft5vL)JepSo?Aa z2D>M8BX(0xy$TT8-|?t(%{_n_e%#VO{pb>BGvgMGQ%x7rCifSfyJXtKlCF{NE{2Zz zxxo0F2)tvIXt)F|u(XS}cN4ZBOs2w~p^7iJ(&z-)p8GsBQ<*W_XkmO_kw32&c6OGc zdocVdHci8g$8T{Uk>=-d@Z~hTM8|i&-n~hIfW-8%ZRPYEV1Z@wvu~pEIC`I}jwudN zv?d{)popK)S|h}GYDQ( zJau|_R}Fo$+i|{-8VdEK3z^yS=NeaMe!A+t)Qn>Mh#p*HNXEvmK%-!$F3FA?Z>Ej^ zVWf60kB}no*&t436h%Q$6IP6K3f7qvsr3Zs#wzq~$~l*wsumILRJrxBxLxxob({}- z7~M@foI5`YN3f7<3D(KYTFz11gCkJ^@P2RuiQ&x?1`}!ai<;PLhVzb%3d?dZZGITI zxamlYZvlTHRS|#-w7+7cAf3ErD_?I%6ZkhOt4? zL%4^-v*L9{vbdLOvVxK~^z$sV`1GwvB8peRyx#2!Y0xUZDTiWKYrwhMGB*aa37PC zN=ik~IOr~c9Ew_91M{GqMSG{^vFSQFn?d4cugja-auy@}K%-{@-l35>GZC?bWed;` zVdbd&Jg`yuGdhU^^cocf;bkc&DsyC{8VU9}=5004x*3j4cd`&T{dxyZxr;Z+58LwQ z=N=4m`$t6U^%#0mAM@QlryZKIQAG<=N2-VQ&RY=5cBdL0}uk#`qHD2R02 z9UtFT({*){zPiuUYAjivK26h4z{#PvWo6-t8d(!qYQYug8Vk4$YmLH z*)V(K9IPlFbyi*7>=4@<$!%Yc5h=fgWlXuod(?!qzDTY__tw>vc(sBipYM~U3Ca$7 z=8ow&e1`^x2N{k}KeD_QtJogKEstq0rHhGRzHC3}d9~h50hz9?G^-(g*WDDQ>xoc` z89uYV$Dp1ATRREOx3blFzvLW|5q(B;9weypay0Y?9Ul~L|E|Pmb-n(Jwp`rw$(DU8 z?DF~hje(D33g+g(d&cqPR5D)1)AhzfZ$k4o+1lr$!Y3lmx{RlH)Y>|m$G8}-=unu^ z#m+(dy1LJ*uy|DML7ndbj#YWT@2eAU@&aj} zRiBa(RE$C3@Yz>-{_0aI=*;AVMNA!L!TwuUzPiMo=+w1!J6>zMJNLGFla=W+mos<(_wZHj%x1~`($0uK zn(At=oDg18aJl^>kx^ZpbkOVgnrAmk*f5KXr(;CXeo zp4Wyc-Vd_Km+D>d!MgaR#x+e7ExSrhnKtx~@{T{86t5cl(8qG9hO*3d59FCWD!PUh z#|PZi!&(h(?0B42R1mTXd9yP2MwgT0NHs&%kLaJJpH35wud<*1F!4CKO558PWo@`v zP0V~%e8UOI%C|N%`*M62RXkzAAl?ukQ=9^Wya5CC1K(U)+%kWyF&^{9iA_QcYQf4N zgrs=y4_C)G%6-)8NMDT;cG`w>O#FlLa=rewpJ*7uU-B)yMXp-(#}8XVQ3V8e_e!OT znCOV+;_?l(@Q~&RU4lmLfu^*59_I^TCU*rT6Eg|OaRpp`xym}znP(s}%?H12`E)2T*Z1oJ~VYO1(l8P_!g`;)GDw+wI|w@Qafuy^hyQL1Au50lg~@>UsSBNX;N% z*y}RJ_I$jeqc>9q$tOlOFj=ag{7f&q5Lf=%Mx;rdz@2Wgsh+QlGcudXuWPK%zGyio z-b>?%z*R0A(8!q0Ncr{98O1?5jKe`GKc%NG()c@z_jq~vh<7=niYa^_jhXRi+^Uec zJm)Bgim?kALik=$o{GY=pvl@V6HyGNxNbR-9#)?~Jj#biJaC`iKuB>QnnT*_}Wr>_4;1;94xuFXuVIbk9cg%)zNZNu`Ggv zp74Dz+#(cf!)l135_D$h1^8?a*{ypZi}c3g5qimvlwZ@V*W&+P%zhkJSo>o|no z_p17RV0xo+BC+bF{cKYnSspfOEkRf znjXtqb9c&KF)36U@<(|9ku*&bAZu*-RgL!uPB{d&x2S;oZe4uD7Uks^!3R#5-B z>1`{-$$DNk~b+@jx0J*joHbaxcdDF8_1B`QDBd?k2EF|nSHbUeUD+i1#c0ekjG zlhQn3dt7n$Lujmc8pW1F?u!-*i?~<>JT6v_OjKEZYOAk0M{d zWe~B>oqMU160S2D`HzA~k*aomzl_fpIt*wvsfV27bSr$QDZ2-#8hq zCwYD2SqGEL?I)C#)r?YYWQ=XSPY9ke%9?mV&;30;VF;Q_yQM`_Qv!Kq!3lZq$Md{~ zZx+9vB=Vpy8hg3quJYNZXy8qih5zC)G(Wud&eXlRz|r9iRI=5DOsFI2$cHYU!1Y z3g6b7Fud_j#bY)PRN$0nn_*XQKXo30@Q>{_R65P!UyXG8SYYYU%qJJ)wX+gArD4`8 z;;lPJ6WCK-laDx)^3xm6!u>0)L@JBuxJ#=#1W9?RiILi-%v276H#9s%tluZRYiVO0 z4r@%WH}AK;-DB<;8QIb-;whE}2lLfj9JP=4$K-i51rur|HmxPt1x7dY+gy1mI|(=n z-qayWOP3y+>Ar+cH{J*NOSi{BejZv!Z+&aKvSXPpzGYG4Z9(MVR@NqW>t$EkVA8Mz&jcC2WE@3Mj zn|Yh^(sUJLNa%XLXyGc~aUXY!taH-LKKq85|4C=H)iIO<2XM1LMnGk&CU21{D9PCi z!?|C8stEQxjmi&l-Ebn+f9ZG@Qx)@$F{!|^wy)qM3W1-usmqxy`Xi+}MzGM9_nR3Y zYw+;gF#dAprqg&tu)f)ulR7O^vKf?{e{e@|R*Zj;f%HSq(m7?HW7xPboy0&-o9AJ@ z+l;WYx{3wu-G@cfVo=Qi=He%^Pi&P-Nm5-Gw!>Fcl%;#5^x{RFLTo63iP|v(=_pmh=F&#qucWy-4E#v0W9qF zPMGaKV+b9)m%#z|dS#~+jGrcR>@f()f5ejct-h|gaw7pIUY`|WD6uUXYFT>Ar~u~; z;8tG_tJZi`x5n}K&`}&+BC^AAO@T74%-<(1^NpQ#EgUd_(T;NxH&l>giTShG3jTTY zTit>BUPA`VO!!Wj(6HwZPcM^g?(JUGW%Wc{oDpXy$}-y#HBZ$Zo-Uos{;~6(ntX-^ zIIvbb9v&w(d*dZu=2K#9)J%yLIxQZjM$hdzsLBGD^31wvhpkPvSPjM#_Ql1Jqc9h1Zj(&bwMnc z%q1`HkhW*^7D~bv#`%1s$<3G)_=_h;40y%|+7jA#TE#1Y5m22PERN_)vo0Ud@~>!vb}B_5%EkWZ#Hi^x>lo_k-QR-DX$RE(N#Mu&n{~R zQ-1#NIMib5Xc3V6Ja5y7rs>=xIECct6I>HIc$7XVu!1W`QPtz6i8h(4c4l1UyhBb$ z;V4>j2TmcS3P?S1cBopPoEoROUGlIpw;4RYGrQBMEtzk6MB@bPKdy)NzmD>*jB+6s zW-OO--{H41*agUN;tmSlS&Y+9^EL!&hL;gHIEU>7G|$$84GE>&J`KY+UZSC$c)CFM zf}Vk^sq^@*w-^N2uuH1fj8+>F&$o80Xk0{|&M%uS!ZV=Kp?I0ti0s>69gO;yjV$`w zKB<>CTSWypQf1jw`n}^Fqo?v$elxk<0@&7QJ>=x4}G4ai45xIXv*n->6>71HwKqu_Po5gb zgK|Xk^ol}sn(LK3UUyKmcW-}&mZAyb&m&r4))_iEcC4173P*-_Y&z7hysC(9A9jFK zTCBm!RP+Ix&3N{VgNvis7?PQy)wW^G;l3+1STpS6DJWFni93N zhIZlAq@D}8a0>soNt>mA@1mX%7hpeqz7=c4VX5+%dbMrTY4tPx4_@U9y*%_bx8RZ+ z#iqo(_UOc8DTlHBqvN`LlijJEaV(|p!ADA_i!V3K4@W;WQ2WS{Yt3qLn-7fy8lQzM ziUOW$G-ZHJ#{vpqLxB90bH<8PVZW^4@9cK))%@4C(8)uo8Lu+StnX`QIS|u!&0%{&GstxX98hw(uijj8S zLgVoj59JD~Ox1aoO0$e*8O5Aa4o9iu^1RxW?ofatYb&cv>N~Lv!nLNZ@;d9m`pP;@ z9+|~M=cfx_RO7vuX?tGcVz6bI>@@b-HBal3LdOg?R_nvRAw>-~7(1VnG($w8B_z!S z@tC+=W<}P+w#2E~sZUPWz9##?E)QjJc1~HXteFpN2_v#cW@j^zieC%D095S*r4qe? zn@hiYROD2kr>Emd66fj6dOLO!f+g{3f;^^C+%npmmp#22BDl#jRHqKVdY|q!B<`WH z)8JE6v!3$pHx-rvs18|=rsf|b30hVW;AB|h z8xAN}+Wfn+Qwjw72=kar)&Xr8|T)xhy=iGFOl8NLS=t9uMi2}1 z>n)9@*53Qtc$fzdZ9l>{RxE14gW2l=mKP@+n$LO_~s@ku^8A1PmDVzbVEnjztFFB!YNitxV$d@o7VGf&DZHdhXQ z$4RqP_9d6AR+%yJK`lr8#@*|qFb>8gM)pWvAHN@bSpNXI)IE z_IP()^Ix@YLO8Kst!3DAeEFHtW!)oD!)^&T!E5@?uplBC zSZPH&7)bI_BdbzfK-5g=gK<5%0Nc_(%~03tXGUW@hoz}!OTrFSKrf8JZh9Yze{e)& z!P3#siieFxV>vO=DUj!1h!Pfdi=b7kYH&gaEmBBa;z}cLQ;RrKzsfB~CC5=@A_@U-?4fs}zy&OE;J$3+JgZG0}>AGJiwglHB_72K1`*3}3sa(H?*M z8}chklRQ!alS_|AMF&Mn%W4@rdY^Uv2R(I+xCM zoSxx9bAa)4P#2?K5oOZ9@`Ju4w)F}b7-B=ZLFGAx>gpea-+la-LF8{Y!eCzTQ(PLY zzWuL41%vJ%;s0yP$EfEugVjt5yxJ({2S#yO%^zfSCQHKq+J8g}%?iDTvxurtq9r~; z(==Cc6zN}UZ4k*7p!y-l^x%dZy9A#1%eCWDALol?XaLtXpD$x8L@@`NsejFouJyNU z>tJg(S@Pcr&dH$1d~&I77#1oq@3*EHY^D}*!^94iAJ`QgY)1JN#ZAw*v$GUP1nlW4 zDJ&Xprg!_seMw}%|3tv!#3DLpMUp#bw9fDJ0QGyRx5nfLlrEEsh{;hhaxl?_d}_RH?iG8uj)tP01ndpDV0S*L7qjJ<={&+1WbPkQuZv&4@2s~IKV>&|UwLJb|4>+>0$J!R1~ zXD}Uf+Jw!l%c`-u$ST;cnQdpJpum{1PGv6e8uH~~scFo_bFJH;42fI^Bg!!zXpiOj zN4XVloR+mI045B={K7YAQa<9p@b2Zrxj#KHzw@npSZ!L(7#A(OA?X`EkHGI_K5RMZ zT+t^Ox(TDMCup3wIhRpzIrgG8y7-LW)KQ6{_{rf~hzjtD@pwHz!$a&A`awL4p5%rr1FKFoXQ0!;}Z0&kH0s3&_Oi&DO)8u zz$neuJAurR_HB`kT}I}QVK^Vvv9Z1Ql|cV0C$4cjUo@NB;qsLXwz8H;Q&^w*br>6( z24qq)Ou!oe(iCdPJh-MU<_#c0GPAt&_%#k)G(|LPRC+hT(AZvAK6tSO;;niDktzg3fdYKSG94zso=TQZ{jSaqmq8&x&>zT#Q>sN&WY=SKSD zsC~xj0z>@UxS7)aNd0u)Vc{PC(29W4V`4C2P?)?8lb$4<+63%$mIT@4JC9!?8ic&s&|34aOi@R%?!^$1AiOROyO%6LeDA!}bP3 z7(f`MQv&;5eCNd^Z5Z_BfHT?(&+-`tOm4?R7{I_wrnd|Hv4NUQ$p3QzVw!(-8SuAJ zoNRgU8Q15%PZfGw)NPjmY)1o35L!-3Ji=6XwSUR@W_ z8Ew{Dz2=X4MCKBmP87>K3fAIu{%plO5xu!=nveQ^+1)VzRI{1vxF=bb!fZ{z!Sbcb_H_?5`Pv z>JK^G`mPSaDf#OE zJDdI6ZLb`)Pkc)$fFGVXO=x&ums?wH$f6xa5ga<>)dXrAoFCo}FE!PO6ic6V8-x?} z;c11ub9(rNRC~E=zx&)kz&LkpnrJSzhJ>>64sF?e-s&7gu9TQ#iq=?bgjMFce{a5h z50^-|g@N8ay?-+1C`sS<`Qf}`vUff-E~tfMlf1dAagc~K1aZ}$FH&%@lJj&IC}mRa z5V`f151SC%?sTFelbv zrrlk%gdcD|#JBgfwr+ano<-ePHU_KW%a~Jozv4Y}#cQ#Q79qQ!Q);`9dK15;n#FIh z&W-S`Jt#Cn?F?@I>S{(i%>i6w9!DS|tf5TOP=PL6)v11-QYc1Z4{JB?BE}Ioi5SK{ zuM|$khVwln=2P$`l)hGpv43#2*sq%hWX-$p{TdTZ^uzX}`?VZ%_g!Xw644Nj{t(zR zyY%55<0_xWR*g@odr32o$a-tbXuFZoh&yClyuY_yJ6}arwwqvN79&Fs>dhLDRHr#ItU`71?@h(y?-Mg}?p-?`}`{2?- z?J)s-JyDriF96voH}bLH$v%J51&eBzxc;1%{fcMl zA-$%cFeYLJ8T1MP&<0%a5 OE%afY>byOZy~7q&KTP%JYm-7RV`(n`+SHOcl~64 z$k-Ppwam(rHSmcsedbd+-)B`edO`KAUiGuK&CZ`|F_Cc41{X>9;w<_lDaopCC)1mq zGwFgyjBD~9vdC>S?TbSDf{68@4@m0&`Bva_Vh0yikqD!{(=lz!==w zv~%9>Ihc5=mgq~dqrzVJbfbl8HPh(;MICngAEfO2nV70#z~daUqi0_Rk{4Wv9}i!- z{8WJ(#CIE|9Rb*ZtAG#ZRj%Y1N$lzFgWkfxSJmVDFOSwIm#cq15pOpMoqKa|$J3qR z-LDG?N|ze7ZPYGT^OwO<#zD9qPM*xK`3zvj0>Lcox1P1jx(?qx%z6y~X3R&OlUCb2 z9Et-Gp4ZIZWNUBBcRU|Ej4JP_Z8V{nWK0TmMx{W=Qh#&j0w5@EfO{2rl!T6u@R=mlz;N5aBX!}(Cz~d8$od>X6#xKkDu67j zVq(C`Gas@%55)2WQr;6x4k}+{jcz6CZ_2%^n6Bv5%Oxd>8v zHxETxk7uT@OK?Ue<*gs~8+yxqhZPuk)ycRNS|*22Yxx5MXfw7JWnuzUzCYn;qMt+r zoE6#UNw%0go@#og2M3=*JT@98tKui#QFBzMdu?PqJ4;u~T>VDbAEISlG+L`$4$OD9 z&V64wld8VxXpsj4pj%cLPJ&Tiv4`-{XsIXcpF5@<(mJKE`!Gb_s(93XXPHYNOv|i-fPFn*f_dbi9aLbGGBcqCDU-2IHP>?|8mc7P4&RmRs@1ln!`I%!X3i8mhQ8MBi>e(=5LpPk?|Mr9%ftYiP z3;{baWaK?(TV(P?%V11dd6~$IVCDM&#w8xv!04f{>LiAF^WCIJolhqy3Ni z?iGbm(Taksfj2GyMf)pIh2yW8>=QacvEXcjUZSo>VSEmvkKnj3Lu3rJ2 z*Ew3?W}nF8!ts#K%S;g zZ|kz4Xr-+sQAo&f#mM-}>(X90=!fL>jhYSNfN*F*yCDYtU~Z2A0ECYOgkVd zD(aiQEtvRTp9wwY!*fQ*%wnxwj=zfrwRw~O3&hAsm4A7s`?Nf^%VQ-7rmQr?c8jE5g3tqMD zBxf){R{I(19KTBz^h|zv2-nVj31S&-fYg9I4t22r0>b1vwtgj2@&{Yo?p=d=<3ExY z`JPtrHEM82Me3>OUK4r({l6lwE=ehY0usUe8y(#|ugyC+@!Eo-tigGG$fP$)vC^Ul zfG%CnC%B7iTY3VMBP$3!JZHzSXHoY_f{;FFsm;(m`S8~gS`9iXC{(u9{@BhPlv;O) z00*;yiJ8!lp&tGmb{NUC1}JZS?hAj3AcDa$m2Z@_$8`Nkk;_YGg%SFC-4}sxOkG~j zP9lN{Pmf|n)>eEfJngg^Qg7J3C?*%=c=o89ALH zoy0O@sx*mY?AIOKj_H)=tpUYEGV*!VYVF;+S&5OhD$~tNpP~gatXq9r6>g8Y116{J zspRR;ozEBP1GcdvKBX-<$?uW!E`?hZDE-o0&=cThloJ}W15z@6irhO(Ny$B`uZEN4 zMz}FJ;8*3xC#`O;_(xmxOI53YzB$Nngeh~lGEU^rRf;f5@iPi}g;uDPnm?D12t5oZ zRb%xGxjCEAdzXjj+gG%<*lq+eq3VpFguj1;)Uwh-N+gxUqSEsUi!Ib!)2J^mNXvMCI1k$Umc|J&s$4t@As#d|*_<6x&EaH92ZYUir4v>F|!Q zB^79IG!UZn`y-0^=%~~dV;x9gSgAL5DCWMBlbb_<6IInjOI%4?bdA+&GB`8I<#=#8 zq<~JsZFjh0{kYjQZy|V{raxCk&eLR&@urLwA3Cb11OkPU`71Er$&hk+utv8m6&ook)|oae`tpMElkWKsAv?H1G?(Rzq5`V zCj^oYdM4*`zE&m_V~-NoiLG@TLx^@W1;Z+b^l zUnO_}@^kF0tjw%4YK)9KwMV@`+p$C7%6cg8XU3N)BFbdUBPW*h;wkchBTh3UjOz}5 zSKyb8B`#FJ!T9Hi=wIhhW0;ED%&qVj8Tbrkk+hiS>Gc4~+q`eSbXtna$iHGiuU z@Rv}HyGX_2akAp%Q}E#B%P$~{XS2JK-hiii80DHm#-P7wbd?j!Hp*&T$LxT5&A5@5 z7{$D2*9eA<sfl&uXF6_xIu;y+T4Hv3K=s--F*heu1gGFhaxQ!}k=5qUaBP3)jx{<5 zLV-RlC_}IHxq;3mwJUCCF%hGrs5`r{VG zXTF3oV!K@Hdrsoh-Ihc{eFq%FMuBPFk5%Ul#R@168pOwzzZ|}%G{@*rb-bjE3}EJO z4|4(c?BDIuHe?ozG$uEW>Q)-c^smE4{OB9w$AK~G4ZK#$CF^p2dP|UAw(kB2EWla7 zqExQrrdy~3bzI+l0#qReI`OW%&)jb#I)n0kTays210%H2(`2Wc_936tcYj6S!S2^C zSH7Tpxa7tJOt@{zFBhN1&A=axaxZO}eD&ct2V!NJr#JuVYrWOSUUIf19~6QAH6vDR z#Z%|MuLbzI8O-}>4iqSSc|wRK>4DL;b5fPIG&MfsmLsb(ZQ;{?S$d{9^(v_(uTTv~ zqhav7eEP6%&|Xe`yw|xJV1(FGhU`Rp*&1^TeG^6|**M{})K6;6yMZp%O`4ye+EbG-dF#Tx#MYG zI4>;ZtI{AZ7P>GX%)ErUdh)PgDZ@cIFvQPo>ur_NfW^%2?P?h7>h=0Rd@gGhY7Pvj zVOCylJHdiv4Z!_3D@MTkGOx3|h}<>D(I#Z%lVY;a=FzOh)dWVAk1q}2&;54!=upSj zc0yb2hcuXrH@bav)hzLj?!QF5-1=Tp=1&)6R1M7 z)*3sGD+qt0d(k#y_WN_UH9W$*mf|kyy5y2-t-Gx_F?vJzU{k*e77W;#aJR9q@^s`j zWK5jE9$BezYQ$*|f_}O`qz=39-CJqrfAed$r45uQxp#T2I8%9nUQlL5w!b!v5O?C; ze%pB(S$51}tgAN4d|=)WypkPWF#-6C#|U@i(=y2tOz#qn;vmGxC{M!}a{ zNt}&R$+NDQQIpe!1=Lb5)RqrLye-=1c6iEoObPMAq~lI3A}RT}4n{)yIGT`SQsuTB zgzXFFsAHbPXoytA`Mlipyk|(F=^7#@v@9&nsUYz(d=0=N z5;8L}ug{OJ4^r?70s^PpB5-`1mW4g`Lk0m6oLei#3U2{< z-sDJ|Mg>M!Sy`odC5nGeo1j#v+LtcLvGwTZABPL^i2yU1wwwFU_^uro>isugao_l0 zcu35lJbuGOAV@Rz=Edl|u<9Q}`JmwBpO~7Qat}|(MkX`pZH4_0m^3de&MQ*bL<~wz zk5U0pfewp8U*5PL)>jT^Z~~yOI=FBuX=_WxS&FIIqFs3%hK;^`9f`TS6SdaXBnS?+ zNUN^b_ea-vEs?Bl>*@b)et@egKYtHaVl??J=YE;9#mV&|je=ZbcV!w^8=gB*9=+wi z725pL?`l#6Y0O|_JB5vg?^WfaFax(S!oG$2VVyr~Dv-01JMzK4yD!bQ%2FIuH4a=e z7Gl=fCse)i(x+VKhELUny@}M5Mkrr#i|5QTT@2-tQkfP%f^&yQA+VAN9j=)jPFLby zdIsj_$Bs6RVPecv+IioOJUyp;!BC5O68xD5_XD8ave{DG&=&LzX#F(a#zO`G$RIeI zYqX4FfWA#E(nH_#$_utfR8pqX!=(S#Sij=@!1n{vRg?SMll;A#g>vzDSt`IM0r`}E z=e1e3-IZ>bzeGz%8yn`~VMXGItDV8)t$!QR5-tirk_-W*$ij+jH|BoDfK<;i@CtdP zm(byi_xs%@@*2tJJLr}L`pFa`T6r*C9Js@tUWtXt8bot1aE?>qsqv6IGyO($4B?#2 zpzuiG+2fL~d#!o2}lQlb>UD6{^WA(9c3se=Lh`t0?f(Pu?n_VwU3 z+z2qC&D@pJ4miR=3YGBef-YDqcQ7Ll9Vg!#9BTr(oZ0Hk3_+89wf5&QN;pIpa(jU9 zsiU^nOLHY|lzXvVh>_XJf(b2(O$M&H`;`MK_yFC^`;ZuL*wrT5ss}*eInWk0oJKwI z!{hgrb>nzc09+$tTO2r0UVQ8t!#Wu-Vfge}s#s9rFAKPI-Q{`4$+~8#WfbIWW<v}zMJWk$Ce>?mhPj!mJF$uCin`q+~QfxHP z6CgV-`E&xN}>ZT7eWPR^t#>+;}`uFB*sSxiOM z$N}i4q`(XcdxJmpKIdfTR>07W2t+TgB)#JSpID zRexM?IzA4tzoskYS6!|Nt-_46P%o`y!ax+g*Ck3YNLv|2B|C~d|HrQ{1d?Gknd4Pg7H+vC>&+PIN!SrR8Uk{JXm_Ej@aIX zs15kuIMRxHD#v%B&q^xDNXf~}g%gV)a$pl}BSLiz`Z77^_*(UqsX#Kj4Z z7b~ax{liyhc&?H}(1#@;*}v%>J0WbqNSxcv>ocE|lN2+o|FU&MvoVNkJM%_&(AlmU z{8v}{oG#1QAYWM5ccHf^Pbr0;ROi=}>*&BWy3jBG z;>6P_86_}5D2ua=4N$Gw+x)Mlv5A$9ZdhIt@s1e(rL&4j{1S4o((?x8v^ZmQ{?|fn zfw_2uY*1-}D*8Y8*iA^c%OKa1KbiEbsZU4KvPxAI0kBh$YuT3aK}G(BZn<=@!m~C^ z9f0=xH3|BIQQ{i*5+M&@$9eW&3!RM7&v=I1k|_~up8g{%G)#J@8AKQii9UC zxiV!9(f@jFAYmo2x3oxy)#1T((B0u&N&aV0d}DN`tk)Lsch+yNc0)f+3!?wJ4;i|L zR-LiRp%1dX=KPb{TmJBc6d2pF&^e;qk-z*u?O;Ly@zhGT&;%?zan991-(tIcJP_J{ z0%D;8u=`YPPe{uNZT-AXm05dv#r`Q|C!;m6ck=P2$^kGjF*9W;WkX<}_pzjMmX!jH zBNJ3A|MH=MxHw_yztl%?P1F+z?HUShw!-i=?BN{s=5gbXiUZ!czwt_L04+CR zmQ}rLMm+*r8gnwlj%sMJB-Pe$NZ=&vbBtHvgv8^a%n&9QQ(^lWDd2*DhJ;}FqZ$Q8 zAI?~;m`2?+=f6RuC^q0}!_;U4%rKj=bE2&`S}oJd*7f36xX-^9mmE)Y#1Pl%S^4&t z*W)Ps_A#Fc+yh;yjaNSiwVSJSIMdv>^#a-v9$VM(lCZ|Xny+sD+SWmkK*z|Yx@o%q zN^lf9&a9bNMH8J)W-@q^P_*1 zZ)++h#(&dW0h%^qOzxRfeGl6!!C2|PS+z=juFn)hMCrZDSWM$=^JTjJ-x!Z-mOSg; zs!+=?Ex1n9(0OJ%rZ``m`EP1|gc==$Y(rU59LzTW%o$dbNb4mfi9UMLf6I+@82fSZ{pQ&O}6H&k1T)nHNYVX z^(GjBJ?W+9tp4=`iDP;zC6rL|x1ppk|8KB<|IfHZ*$_7*t>)BNBPTd4?@;FHy2#bI z{!8p3{8y_SKMud#IuH&hx?J%mg#0JQgJ*{?INNU8m5Ozy>@MLyO>s-Fb#DzlKE*3i z{ZnZ>HFQ4_tr&%DCe`X@t?K3RIv;k>lgPF}@!#;>hc4Y|VAtn7htn(5rfCj!gM#wu z4*=!(%O{gu_itsj;r}%r7uxu33lK;b^jPx6Q#)bZ(iDi29srP99GpCO&HMGRWKQtk zn8K%mKAX@O&6LQG!#r%Mb4tBeZH4+@locj6OZ%F8Y~rAvs@b%vTz>e!aFV+4E(4f6 z1krcomX26b8tnf6mC242ihB}#s-hD5I-ezITukt9=tQ|modvG2{q55n-7^{*OIkxV z!+3<=NKCJy?{S3xBt|nrt6Rd6GN;q8J``TfQe(bn>e~DFpSqBsike(H`Eh+zG3|)t z@DTV-eLy|x>J~JcA{$gyzY7YX9Zc@I$?q-cAG$kJi_3QN|6G86@4nPz%~~d_{VnXj z54%0tbWYSF8*KlZ#^P#max`(12>Y%?yhW9~Zu8f#1w=ydKwwxT^br3hL%T7%} zesV$$>fcKb!Nj&JuJs(<+Wkurq`t1qPRMAkexIU*cfOsI>6CUAp=PnqeLdj3m!f!{ z^>yWbC#G$jznCMnV=M5SM|>4tuvkY>_S-k>MPQR%)$C2}(PbI7^*2py9Kb>fH=g8f zg;rE*O1#>Fyr`IzinU>8>BP48sqJ&zYVLG)nf(Q2BTxHW|JAvfvek(Pr4lCj>e zlsN>Up=9$Ju3HkE;wczw{5TZMy>45=(r zb+sZip?Jl(J($_3!f^#yoH=FA-d>HzLuv}r^q41KpE(d)5iUsfGJQ6kD8eo3JnaOb z6XIl?61GG2>rOmw78gyq~`=gc|A8?C+$uQC(1(`)=Xbvsq64O}oy#sk?v_GvqC z8p+q4UE?PWA89NMF)#p_yDs{y1oTqd8bE)z+E)0om%m+_n>jpzn4q#3YbLsei@%s& zwsQRes7s+iTFPQeQ-!}=?S{e;wM;d_D>^aGlN=JQeZd9?J{vr@(w>%wTaz=CQ`3Fd zB_yoK$gn;5En71714g~fwbDvlNrOX7_Af1}qCyzhk>E%c1c!wD0F-L`xsJrL!;i_D8(%Wt(XE@Q5kADYdqsJvYWAKWbWmWg- zeL(T?X0^jrsJCDBY_p6b6~?_6^$wys?`39zlYRXyGVcw8aNFpW{(29O#y+k?8jq`D zVyj}L317=8dK3)6%3bzIE&sXF^Iw~JHk$7`$2oR%lneAX*rpE`s+6W4-a zqJCIC=*?x$EzWp-9|5gh4-96gEd2C+^dWjdP}Xu6kEdqG-{7to;raM{ZsDcAVmQ#6 zSqkdVsLOKInto_d;JrArX~djzc7ksR+D#7V^`$q(8ED43O>R}oCWf-^#D1HfHznd+ z2X_D9wRTs?a}X*PaKDPoaf`wUBs zL1Lm)2rfwJiT%!6VSTIHEoI0y^=(G{rqBWm`#NEEQT?_fYlTh9H-3(U{p!i~s0u+F z0!n_%CoW;&U}0hsyeR!bcHy!(Sk$DK%V0uh=6aYIs*J_5fLxfLjqX&ksWiSGhC_d*Rbuup%mBnFkP%vSmh6I$g4UFA083Uxzfbt)lyw?n9E|9>~^9#p_@Z#Ez>=eR}n2jH*;tR1uxMcRo;Qc`D#lJ#hf|?X7j`U+???0tshKZ>KV^ zd+XYd0FyKChUXdx+g>{ZF4>=U;c*^`&R^>EuXWFH28GBX;%-XQ_R1GH2a7xht#TNp z%~}@Z9-=ogpOas;TUS3ujxV1Kaiu^^LP=UA-&;Mom-eprLXPXsPxFEeif7`u1aJGm zrf1N{nkH*o$K9cX8(1LNZsOS3?vhyMZ`AAS^2(H{*q8oLB$QUC%e9|@6~Nqgk@$*C zXKUw)j|>c_wYi?=u0@%=;~DC9XRc`*lsL+%;YsYb4Z-Qsy>N+-wsQ9;+->tf#RD6) z-p1!F9jO3RYZ6CTn(tZ5co%N~twC~hEDx1m4@bN8(9*cEl2XWT3Il%L?r<4mqnr_+ zVdiok7Zjw9U2XtAbx?a#dw+lRgNz<|7xy%lZPS_%GQMW^Zqlaazg5I0`kXXY+Iq3k zOYowcJaHR=fB(kkwK*1X&hZ1rSbOt=b&5MP;Mlby@{Tb&NjvJY=eet|| zFeyBp$@A2#%2cNK%hMS1a~mTZIbP!?KM(fF{d+@Q@A#i$5tg4jXY3_AYJN-wXaUqQ z#&mQiRC+SD#vv!wTku0Ack)qpTYm-#D0#t(A+JJz=0JQaj*|xavZFK*1dFA}jOn|x z*9HK`#dtSkvrVO9Nslh)yPsl_gxuJl6Et6_<*jDSNAefTBSsy2M5W;mZ7Q11mp!h{ zL0pD}EjFmS`-S6l_S>0{&*{zfo4RI}!7|G4-R?6xx@P}KY8}WMBZvJu#bf1X*B3ztDbN#N0#h?U zOFNYdHz6!o$}Fq6oZC$P7=_?48yT>HZD+`*=EOq8x$n~f7@k26T?^9UaMbE$&Y*XLF#eQ?|3s$)Mv5goJJ5TAtJ!Y5Z*=(lq9vb_ zR5qo1;>>2lVf}y2ePvf$(H3ne1=?c8i@O!~;-#dxyHmWlyS1gb6?ZA_65NYB!9756 z2oO9tFTMA^5APqmjQ!z^eX`Hk=j@T4v(}tz&QXHDsi|Fmw43-Y6J?j$9hV~-c4NO*h33gn)JAw zsEea9!akU29ns7|p}h}o*cbnCd;1Gf2Z*z|1{!=4h(`k4K@@(PnHqgM&4cY;8^D;! z5aDOR=j`Q{?&Yf*s~3>@atV>Ol(BDk2_X;E2t0Nh^5#aTK6CULPbyUrJe;x znTSr@K)`YKRGj_QU>~5F*n|k{d82tFR(LoG9H_!eUhDi)JU~2PajSCHcf`-#qI7)B zLik0GoK#awPEK~ZaNtoCi^)MAzkHytU*Bb#xDlHG17Y&!MK3TVF7T-3hPUyO!CN*! zTL3esb9;Spq!TNzEH1U)^$WK`^Y`rU%of>}1>zmJL#$V|%&Kq7;t@r!^u`jMUr8d- zplP+F8ttZ%!Snn@@;p7yQ4a(6QQzUAyB|aYiP1miU)*5U87vLQE%X752V zX2Za4B__B__#!sImggEDu=wN$`yoH|6Jl5S=8f0Y;@dk1E}}5;J?=05oB#z;qSm4k zN=XS#Y2!Yv(i4`uTd^>dL_LhmE<0S# zBAZJ@DQaOqZ8_jM`uLp9&w02nGH{`_BW&jdq7JMV)lDVZ-%V7f*20~@W9ZFNqhFyu zH=u?TZXn{EUe^%E05Z$DKN0nRP!fqH4fNi1F^tXUL0A)`mM=vc2~E28vQfNszX5+k z3iTzGX}6asSXfpKm23v&E?Y0=(IeZhSl0<2;9xtDuee-kV!6O5v+LCM{0{t!R&`tM z7Fj3&$+GJx`U96X#p4qSel9bQ``Oa91_%qS>l1@TL(DfLdJmD(N0GMPfZX}jfWudN zot5L`^-CK?pF7W<+@DK}y#XBNbJcr7C)tz9-sDl4G1({C_b6VT^`rAFfXTbiawr<1 z`yFWRy-#Smr!!xF5fkA1bB6-Sl6K$d`EA-Bo8_>{*Y(X&RSV-}ABCrCb{|m;`&qiM zaAHh?$~Kaw@F;)n1jRD6_?)hnljdXJBc%lU zTx7WDj*)#xZSD&hYjhP@TlwJiP{oAxw$1kG`&4WI$l@*$QNI*&2;fC=HLvAuh{7mj zvMjH#FTtQ{Vv1AMoSB>~`4QdYPYJLqcPC=6ICA)7w-`8iGwfj2%6hd0;JU|NO<(UV z+}?W#y2rjqOKl!*&+u$oLhnJQ?PV6e$0Y6SM96IFI8}r1Aif(_Pavjd%~)diJDW;V zaetHH1GcD3jLH~HWLp%m ze7NuRm1L8HNJX^>>=7|A%o}?-?P2M6fAHvtnMv^^n}P!uycbOWw4Gmc8Hw%+Ywpfh zyh;O88g6r3@6WZ~oMA6qUznLTF17kY0}nMa+o!r5pZVg}))j@lgo!pn*jF!Vo@25< zi&Gbh*~Tp%O8EfJ8(up8OmA(B<8_X+To7|z^t^WYUcVx+*6dNo`Y0a)-yK%nEs|rg z+S`QlEWr0CIusP_ggLw%16>1<>!ULV4GtYWP5(}NcZ<+9gWD!)y~vX6Dv7%}J^_-4 zQ^`{pJg3xOayrU491AmUtX~Zxo?%}*sSm1uF&T6YwvL&g-LXM^yQrLvB@mjSbx!4T zEzj80?*1V9r|qh%4k?RhN%q67=|ePuZ`uafX`r!7K<9PUhLF$JE+WXdx1A(`d!0Hw zde0VL9Cm)X_dB`TLlhnGQk)eJ+L1D3^G*^sf06Xepj99ArK8!qe)?tI`x9Ieo+QtB z{cZXH%?DMglHIe>K*(Z;9`U=n+YNnff4hM4nU77B4^a00!JS}^&;7C>TvCxdMAv@# z{>8Ug@C?`=Ex84s4&{#1zn^tMj64$RrEzUh3f>kA|G~GaBYRVpnUes$*w{+1b7Ua* z7d_a|_?%)2S>LmG1vyroRH0;E5~$9!J@2I#7x+C%_)V!gSf4Dde;c8~D_`||GP@i! z=C2x=15r*~eG8E9T5ojix=8qG=1JaWe7NRRX$BYDfOcrT#X|<@>{dYox6uF#L2{40 zo1#y9DO3oQ#aP)9!~wA=GV40u!1e+B5%fRT!hN+qjEP=p0DM|;<}don22mvyusdOK z84dj2mfNkgQW5<*8aeoYU)XF?l3RhgZ2XN@yNYR6;vMssctc2A|min!Y z300NQ+Ou@PpAOftuJjWk%MgbpkI-Qg6_TlPZPk)2yz~2*CIxv$yJv61;_27pBbJZK zFtU+Vq`x-??W~tviRTrzQKoIb%~G|jDEw6rWv!gs-S$sOra#@=@<%c@No35|1*~-=nnpiUip6^1LK{Uwo zcnt5ad0ANQ{TRXTd*yt4Ii-?vYXLXZ=cpIP{jbFH7vQmM>%xd~yt(~r$d}#TgeNL) zt-MD0fj%2f@q6(_c?6+zWac^b!b%X+2BBh9hR__T9`@I%fPC7|8dH`v3gkcBUpa~u zR7mezn4OI(htm)lC=jM6$afc@xL++D-#F@5gFNL)Jv~mwSW((p&IAXAa}pd=3#Xui zo^M6@huvnC@zMaHuJe0o3OrF)d)ZIEKBL^@=I%$LnO!$@Wx!Q1dTV<|OY(=O4~-oq z7c=`}v%!LGP;8@x@XG@sY8Ga8R=w@6Kfl1Z$KU5lh@U%wK1wm>XneDO*BxFp6A^0v zT}E+aZ{~tu+9wO9h50yY`d%LNwl`^i$}4|ZJKr?+QO;;hIv_?_F@@D{=Jlt6k-^+) z;|x`iM?fOuxq9-Nba5of%iz6!jyyLj!ytSgjCT_+Pli)+ekMl9zoBt3urxYuME4=^ zybJK!?~f9j&DrMtxQ)_W+K+Dw_^3Bj7n`R=HTQAd{kedBnV^~xA;CW?1^_^&@{jm{ zl~aoYg`xl|TB`c@NWaU*Ghz3fEq9K}UQx9d+HG|wHc0r!A>yu;uZ07U@NJDn(B;m%{c_WZR-gDJ2 z;9v(C@CDoBP;>S`|2C!2C;>r?&#Bb%i^vRZ!rS^>l^o}FXm*y8Rg^v=AbwuuI;uq@ zseh52Cq;=I?)dO2tx!U;HBwP*wuY29UUyR@Cag>ky}YQfA$&*+G{)K%zPfRY?pS0C zo^MI09pd|AGan-U?xw%^-Bh_>5HCiQen!~CLUm~B3u@EP#d6EkQxiTU7sS`=Ggb}o zgjttC>(8BVQ<^EP#zxZ-LeTU+n+0#ei(9!im%qBqN4ClDLx zI4o0)YiLiY#$C8xLiSHZ$ST=S1UX+`L^Dz1w`g&b=2@Hi6=Dtk0pSfH{YVb%@Zg)+ z!8Rr(OL@X)W$iTk*CM3{jl)T?e-(ZpCF=3MXXMaPUspgdc;Dhupv~snCC%PASyGRf zJ-5c@`KnLnbXpPiY^fvF>DlO#3)o+*kx!4gq~xT%yA~6BFSMx|m0L~6BVw?eU$#_X zuyD2+Vx{{r3f`T)#<&6}Z*W!;XLL2h`5eZh)|(pq;Tdut4}30RvFRP0#sAov7SobO znfKYR+gc|^ti%22quF07PSN*KU6!FzdOJtcLnX{_(N25a0(=nz4%1>5RH zdoO(5uCC#2rnxg}@C+6D@AZ_l*FRvhp(ztpcnNf8MyFnGdO|9AhUx1fTD>HP@8y+o zSLeO_E`}UUf->8xyF34iD07h~O0t7OS7F3R$EA}--9 zPuzQ{`WD*9P>B2^=TfS21-Tl@XtF#Oy_KJ%q5si#OwAULYxs0E>Sru$b5rbRM+Bk4 z33rdpp^?o01c|;aWFwvRDnv+y5Lu{vjdpr_f?xIUZ<|AuD7SDJ_s?!1G?0hQJE31@ zK&$UDTBFC)YMQgdAE=ldk{&j;C(yXFw8cNtSPI(AV3(iTe1I? zam{4#54|$!>IC=S8$Wv%K@VS(zw|#w8OClpY8yb2!#3R#&-$0OFplwhX)$ZsljY&5 zjkel~zR@2{RJ%nYU_!x;5XT~)9+hHZs6c3({d;JcYQOYn%X3F;%EK5cK2We@E#vKNAqjytUJCz?tm@v9pD+hP2`nNbW-StUdvfJu< zEPyKdTfs@c9tG#gT@vG&1Y7gtZCVpk(@yWV?z7TuiqZ?r+OuPyZul=AC*T)Z>9F>$ z9N7KEQ>KjnXW;7C{*JK}?A<#r4~zA~pw<$fhxc;AgSwALHu@6yqq4>2<2tI??E;Qr zB%BQy+tQEUQ?D~EK+jNfnzh>*co9N5?Uxg3aCK-?>cmHaarWhg*slv7_sckj{gqm6 zF+N%NwuX|flY1U8V>Hznt8;}&ecXu?>f)kx*@pSpEYKqz*x^%`rw19d+Rt8_`D~LP zJNsN&AQb7*JyEXCfg)DMOzP9n3Tx(;iu=M2v2U*pv}@*3MS*ubQ2xC95+mB@?PnaEB@WCM5#ral(Gky7 z=5I;yMwe~_n;gp;T29-$(^6KY?V{f{IUk#^=(j|_@gOydKl6INW|N-~-NUso(i5MV zx*2h5ydNqle?4WEhdNLlT~-OA3Z-Qij+L=P)3(5Fib)S^UA_8nh+Pc@i2LioflRS^ zG(n#n%9=Fi%;hmz^z(FRhU*-1UpK&VrST!Z*ieR1fm@wedh>J;R1j*w=;KqGQ(jof zS9tem&J(}JYWm%BGSC?aEXsiqjbUhu@6-+kENjXEUCv$nD7y9~`p(djz#|83Y)iL= zDFF?NF@D1XeJ+-DozKonp8kJJGjcL>#>Zsf)J*y8GX&M0DP~`!)eyr|K9AN%r_zV&sr!y&m6vZYVmyj(<$ry zEjRsKGj&t%XfQX^_I+$klJe}hwDPX! zyxtY?&wwopJ%2#pLt6gWFIB#03sNF@1ukBolN?s>(hDc&!A+11qDZEvT>Gk93^PDq zPoNY#a{V#m{Lp=;I>&8#LGQ$|WoF%v^x~p)XI--4Xl3*jgmR-;#9_9Ti4p~&P|RN+ zw&X2wWX|-O$fNG@R8rJeE3pyY$=qg|x_Y&o=4RFwR=tLxb@Kwv4)#g*9TkAUhpl_%bt0k>&ju&6nDQH& zC?N70h{BkjCeJ7Y^^Hx)V~4n56Zii4xl zAKxtXn*4wr4T;pG1%<}t&ECRDIAi$!{w5>2L33g_)6QeDuLdReY=7$bQk3kFos4Nx zRrZ%3%c--MsCLF^f`d3m3Y4({>x5c5gUeLmBV$OaqQQmmDx(y)`avjBVfAM*Rjq+* zwDP9%qpHLPGSReKA0N=JjNQc=hDrl$kWhCkcmhJmQN~qhveI~Lq1LS+6^lEvmvB@9 z&825b{yde2DMsz#MBX>su!UwA;1xZdt75oet;@5`e+WHR1Oiv_sJoR5{l0yI7A|D5 z89ZOYpl5R|YOW6KkCww{rU;Qj;V36sgcn1wKO-o;qOTI6M0v0h*hS^QmO)LM7I2!V z_zX$TwW_b=#xtc@Y&HcyGI|H!{-@D5ME_LLz%{tD+FMvvm)*SOJbPp(zgo17EU34_ z^hdW)%wz;P4waHR|G;0JvW4~;C6WHg0}2BOSa8Q zB3qt0jJZF5Z5G(QpID;2YCmU5H%@o?Yqj3_eW3*id{bphaf_@nd_om`)* zB|Ilr=yCX<*>)*=!_m(nZ!}kSQbb?h{S{t_KEsV^8WlZ<6t~WJpg=fEa2b+aC+zM+ zvv^E@R!$BH5)V%0U?w1(*vRx}U<*UrpmY=BF3!j9Y)FdZg6pR^R9ma(V;cTFYM!r2 zupG}|(hDIDXE63UCc3Fj%h#6yP$NP4%sy$uQ1&0qA58a_IIqR;6lIkj?=!-~S}#31 zBZmY3FBiaUckpjEge!75R|QUH&TMlUW4R8q;lhb?Hc8Yh^>P48xtQ<_paXF>cdkxg%U;clJRerEA0= z-+8-wUyALUzF-?#4&(JGaDFXCv5D!%@%MUiO}#c>zLJsiE~K~nxj*@}sO9csPzS=2 zeD6G>AJ6v|8@eN2kPXG@{R<yRr;m;#8M`HmSw}a0Z<6i`JoI#Ov;c*s~qG)y9C)Zc{)wnou8T#0&L0BYr zjr9)GV%ssQ1~t1M!0T<1p~)3D$N}0~1|XQZWkLPe?DV!qR;N(&X&HR^1H0;O_-z&A zZrx|n5%in_?aUVhQM(kF=~Sje>6sm1jax%sZ(uvrH8V;p?JG;yn9>;gY<@G@ ze$}_4%3&DFL-K6fcsPoa#SVW;oPeX)zYYuCSw82hya)4Vwz z`3u1{)z74+P?6#N$z6>+JFJAA)xi3!)bJ>ho5p$@N@*Ph;OaCoBi{3If!LZ6q%5dC zy@&DKPWrZOVfFmQL^(9pWuo|EO8LQPmYgU%mh-{qyeX6-85hU-Vg@&Okds-Mb+yHH z7~ns(qppcsh0s@+9;!?O0t>;5alUU{uGrNq1l`w@>mGMkH?)=-+7*z?48nh(w4UFw zT{tqb3nh=xz8{Yft1BNUu|uI{#KUPo%H=P^>Vjn_3%J7jxoOrSeYL@maxb}I2go(b z{ImZdxyEzK3<22Z6vk|V3((m%oiRg=g*&E#bjkj;zRRo--CVJES z^0dJeKnGtHfAZiets>K6Cp!_4;XuVhW#m$;voNKT7Z6DA4gM*O#Ny_@R3r2SWN}}{ z_1#Q?nzU& z(`^!18Dfiib==9xJt_+Ed1SZ``tv_l|1r!nQ$*s+-~c=D8a$1*;w|~*f%d{p&~VGE zX{+sR=|n0&dk_3gstZu2r4cN)&RA&m)S{v>GFh&B!Hz>9F6OoU=eF1a*isy52mzZz zz3gdd8rCkJvvfar_-@P77&&j=KEL;znRqIvMUR$b7+Ya$jXqv|F|Qn%)|4R$yySjn z=1B@^ulNan#LRre!SnrtK4FoIlRPHM%vC%$*YYq5&b3{+Fd%iq$vHyo#9fZ$HqmJyYi18G24VRth4=PJ6&E2GUQeM#l< z{2Pk0)hbZb8ax{KSVm|qAQV@mk_+8_r=^if&VO;FdrmUjbEJVj5Qvob6~(l?wt6y! zSUuki*mfPF*0gMy#rD|N4@{2FHc3iw2|dZL3ch0yj}h~{T5BmEt58kvfcRitd6Zd< zG-2%`!=Iq73nikJAa~%%rCCFqvxQZ^p;!`Xe@8V}=*3xrg8J4%|KK zc@{kD_(42}6~AY_h*~Mp_~ZxIq&Tt@;(s@!?ic6(j+9@>V^wRME9)rW#EGkrTG!Nn z&T_fm{60$LXqkxXe)3oBOH^d2BoV!Cp4x6aAqs6G3p|FeClizh&4`UHw=Mliz!tb$}MSA))n^KgHCm{g8S$r}nT-BR|=kzw}NjD0IR9-Vl zE}YvVq4S{7RQw+iLVf4?{EVnFBE)ZHhI9i@eLH{D!wE-ilP)p2|<0@E18@G80w?v}0c}6+bTfMqZ4LhP|IDHD8t)R*^-F`YMJR zG;n2n0p+7my$+^lCrT9ZY&cmNREP{KVdY~#_t4Wq(zjEzU5>b=;46#&6iIh%;>^4! z5vP|D5uUY^WN^G}&;mC)m_=Ocv=({<;#Tn9%bY_y>7+sIY-y_0gL!3@GfH51X!TyS zfYkWJ{HNt!(@j`gs-uuW?>9D!*@3=!Cj&j3n$&PF#5Dkil3Hzpd(Wtf!xPPvgaj8< zg1eTe!XC1N?OpkrO#u+{iqA?&m~0Eur+e=w9uM3yC{aB*6CEXibPMwf;MvCJ_T-kC z8}Cfbzx103g(bvWEsZX{z2x9oNX$3{>nr{=1>vZN0e@+N?pzt1``pDRy+P@^V7AlO zYxog+ne*QizRDJK;s(cSJz5D@P8LNk+OYiVrHV#mV^IGcm7rN;N7glecBEem(^m7? zL#{{ooC5%Qtkwu|Kk#FKr-Q2i!T=u8BV-L@``?hC|5g+Lcoq3?tpIQTYcBpv1FL1G zkw?cozl333YWH|$^GDwLGh8IvU&>v493z0Jyv{cXqTY^60UJvZy$&`B2=Lmk#~ zu2H2@T34MMKbl_uB(1Z=&h+SXyEdW5lmoNk78bS?I8w%|e_v;dsi#+UJA$m$(98yo zMmHO=9Ig!~Auo4EOYk6-zeNPCvI;VVCnmgKc zkk>pD_WAdg595K_7`Y0d(9v5Qys)3cXG!*Ej_r%hWMR48QVE&F0NaOP*1aq551B0+ zi2!2mTcLi7mF|ANrU28?crV;m>CNS^2KF=Wc9y#P5`QMZq)&T~eJ1A$bh{16n>Erb zOjd93$obfHY{Gbx2^ow*5E>-v0>;t4?wn7MFXbo#UBq$(ZR%pr*mt$6hNyB zY1cbV)3meMCCu+n46r;awDfn0h;^3`@7~d?&9N9=bX_$^n9TJHYU*E48UehW^+267rF~TeGWG!|48c(fpo9#@h z5)i22;W(bzvhrbU%SGt|i|llsfRY}fFYJ86FBZ4jNBcZ+Y`DW5B$0n#aq~z>K>4Oq zV0i6!RqfN{4;gLTV7rWwL)#rkV;K-LOt7nDlgL{)NXm9b`aL2eQP}dmq2>B#U^cI( zK7a(0G2}Y986Kt~4s&GJeq15~Pa{$S1~+9RNxmj){!?6BIA?-swfX@@mVaAZTz`a( z`nR0;cg-`>M6CO>f%*$|U+{LGr`B}DhKJ1-ebCbeRymH4{*u;s+O7Oj`(X>Kc+)wc zuc)wX#umZ8-r@1x>MHd%D{+Oz(T0cJbQe2{rL-$f`W2Knz0->l@bjoZQto?=oSWoJ z+_txYPDAtfHvnRzKjri16{X>nGBdn}ANNrKoFjnQNbsO`EpofPG$U?Qd*-1_BKkZ&NkJ><-Q$b}bc$ zELPHX9ciC{vugo)#hUjv;P-mM5sD&mw~t>IYha>60=dC*9=+|`F~%F}m)&*B9WO3P zPH`t$3s_j$1GUo>Svk}n#dU9%JYi+Ulnp7}0xvr2jL%V^n(&b1p~{$qIGy6@9p#|f;t z86iM{to1|x7crc;IBiGt--C0kA^ujXX2&1INC}}{gU1GxfHg<6d=c&r=`p~Aq zD_zT(Z>(V+SpFjWJ%V|O$Wy~eJ`C|)FL==plLV@-*AWIa4_?c=D=6LC z)5)1*t;Scfy;z_mLEmccxoi}`Tc$Q==BThRjlt`Z1<#{<=$8|s%SZ$XG{=v1TOyiI zp>m)M4`(Xvl$0Z*NEI-?Ww9OY3IrDLkKOhD%qXKJx11N{mHM?}rt8zCqfkzkW%+=YQ6HS&T5dRHC;l>;^)oOd*9Yvp7O3WtNA`)31^PI0Gi29EaZ{FD z9z$tKG^`ZIO#wWwKkc@J6-O`d-qaMZva(0BRE?Qy&d+BWb|vPOB)XV}F1#V??pIN?W~_)|ABicbZOxrWg+&0Q?qWEYYpejVUKqg91rC9Q@`XD$>oLl5FYO>zZQGmvjpj=5faWM(f#g* zQFgjn_@{5%w$rfur(eg&VcJOzmCviP1sphcmUeJz2?x*x(AXy=njb z7_=ll5sfMzjR@YO}kG1C*_{A zDK^KsheviGwft;3kyI7=*F!#6`47^0M%rSGn@ti$$y?gl?Z)Z!z&NTulGX*qjJlTKKeRG!V}25zkAe7hf>Q zHTK0F(-*=Z_q#;mbFI`BIt?2#y5%X300V7lI4OeNHmydJTHb>-A3_Nt`CSnV)}BcP zb{~u#)9>h0B7irH3hg3>@L$faJ@Q=&V7)=Kk*ku&aY0G`Zdmd(%-Y! z2L$4gnZe6y@2(TzB^}lVH1PtLhM=wG@t>DBKYjN+Do(jI5OMSJ_3&C(zecQj&ZA>C zB4cM+?kr9j*`Dx=g3{d{irtB7cW7}R*JiqfH@%hMfk~Kd`TO(^S9yKMbCI^`+_1*N zc&Lqvd?vUNlnDj6Oovldv95HX~gyW@Mon-(pYg_-WPi{TKrA*R(dk;e{i+po#L zr=X*w?s!+A;ca;*X*n#>L`_damuoJ5aq*;Zd3y7w$~9iuGPlBF%+kA$fnLCJbfY(} zN)8_gv}iLOwYzU99}52!m(m0?tM5RA4PSr?erUsUI@zgj9R?zN4v$%dhRa7AOF>uq zYsV#rkUTZ|of;{_{h+i0*=17bX$ZC;--(RZ$UtK6{>N}R_gE3vGkTJ#KrDwLe` z-tpR>t@!{ z7Ul_DaX+WBWOf&3HPnBu(o=9sg>rJ<8)@9$+x z_}kBlhQD^tSg~`4=Oy0sn&x@N5zR=t&}B(&LLxKU=ZK7DT(!AAhn|=tsP5l%1D5mV zf6df`1k{PH6vj2($Cq9%JrKpmO38FJXALv$-}EbZ#tHMfk%Rk6GhN}q4&nsnJbOS) z-+XqyMlS9zzLk#jTH%5ehAUtx{9d9!8+s6JYScq#+Q=&V7I_Ok3q!At)phshybSq* z9X@NaI{d)X!+aZMElU*z(oLpTq)8IXf^gS?BlNoUer^G=%G%&+>VP zT{&|FWsGrAf;`VPOhNhGs}2#zP`ukN|Aee-zy2gsYo+%LdBxbYi{Wh2EsK1nUz1Y} zU3L7(jp#z8IpurrmLshr;=YTj(&2n&w8GP@R?<~tFl~lJayQc7rT4U?v|u~b41bmM z_l?B7(TN1_BT=EU_6BCQC45|<5 z2VS0-m@lvTjyz>&W$|rmllgj9VbI!X|5-n=|`CF?Nk2F zgr@Jf9L>a2tgZ_uPPQ!%?RQynYLo<}JQr@=_j;%#Kr&psFub_TD1~vfV52|oe&2f5 z><#h?M71gXEHt^;oIaxl54eE`+Nr)LI~*!!%*%<4Ldva_jpqC`5DDy%Iiv)(l`YCw zxw69~H*D~GtDbNgOOM!EQU*O4=cCKoglloOrprGE+sJe?y4npnO2S)8E^lQ}Mbu?o8IN9tJXh(9lxH8~V7pWC7uQ7$SQZ zYGEHn3#CVi?3~}Eo6+uf>{uD;-1UnaJk^K2-+C zeb*{a7NgrdYV~p~mE;hV6YfeG&aFv5n5vAa{9UN7hJT5>vyC5sP(<#|E(Qr+7A#?j zj#4GzC%t6cLsWY!z9ZzR^ZhxtVyK`&)_!tVVzzd*^qw*9QK77KlXgi+#)AIMTS@g4 z#r~avf(5IP>y<$P?xX-d%g2kZIpngwKA5cJ=r=>3bjGP)w^ob+x=EjbTDI&3jE3Tk z7H2=dGZa}e6k&DF1R4Y$K5Y$kSHrIspDpH)mM-&ybqtvS;C06cwb-As7lzY!b zYg5`q6(41AOkH1=R)<%I?{nqgKf(S&wpIaLuI@S0n7&Jagx;9>CN=^ywzU%uhcUo; zR0-2s$}LWYD7-FDRBF@4UjJTMdNCY?cI6ezliIRC#_$IG4Ar3%zZ#%MNG;A_aBx{P zUNHvwvRm|4$T(2t>!~2^wbqwYGGR zi!)=7ih=Sp^z_-Zju7s`ZRd^xX!osZ& z(hNa8W_E#>1TrYZcr4erP!AWQ$LhUwfH+Gu8|ujyyB{XBRE6f4ic3l(kKJ*fpn1bOy2Q{ztAgCCJdod`nYhT~cyYq3uSk z21T47k-G<3+-+asRw|W{>xzsz>4Nimo);_n3gQX!~`!^usTtmah3VXQuev;l|Co zav`TIq*0H1>oEJip``jW`s!r#wWL(!oIR^ardnYR&<*@9rgFt zAgt~UcTq;Zp~Sjhn>S87nNhD0!cH@nnG5@sc`TWEB_1B* zEl)p$ll$4sFX$X;Py;Frwec8fEe{4B(Z*{66?48-A8M=ZTO87Q{DJ&($lJ3GO-3_0 zla_{9Tpeto`@XTsOR{8SV5DcT;_=3Nzl#U|=|IdbL#+bW!@I6SORyXrj}>mQLZ~XY zDXr}}g=?$^wzzvkood@u&x{zty29l^ONLBCZ*>H2i>!Ay>n>X@+1qBnK2YwxQ|#F96xKv-9!$Swmy)2xsvf{`i; zfw}x-YhbQBSN*%;hXnkA0jv312fX(_WTP1GeqA({3_kb*DmAr9jV*-+H&1i}w;c)>8 zt7skqn&o-}5)hAp+G5Eqj!XES-pp}r2p z;<(aRGL}NviTlI*gr)vw@Zbk+TY2pT3zOI1hvDF?d)WL{#sWnb6-{0hmSo$Co2PRP z)WKv{)$5&M&$abMuNQFp#!6=h!>pS}PtOjPy7J9ACp%}&|ACr9b@pj8ky)x|XAysF z@R^v)Pxj$Tey^9NR~a(=1->&oI5#?4sBh(r;>KIyhn9{nLa!NGbtT$VdYyBXzw^Bx zf&1YzM4yC83{6YU_d%pjQmDkHw(4RdUbN0v+~(RG+t_L)N{_UP9vUAg8wMiCbhk^h zYm+6^rNU(9TJn|F zA2Zhj*`k+k&JPXL=RJ-ZkwK&ZU71>+My?a2E^%A5?m80sfk5*e=ufg%t$YO12je?9 z+s?v<8ogfY?FLtbF;!P_Sx|+AEmR(QghYJ|;F5XN(BLC(QKfTrOo^ zVPrL_v(0g&z=`W28swS3zUNF>u`X+%r*XJE!#+QxLKXLk9XOgXpgA|pZ0q?esy;>9 zA9>2wdLh{Ye1oUu^rDh0#S~B7>76VPA*pYB65>ZTvh`H2hdV}_ht$=Nc3tWJsbn|a z%_U;*8lT1`)pI+wNhu2|Y34Lf`H)@xjqc4?#gT}m(zK)j}j=nfb ziwK>!U{iK8^Hw=a-rt6G8?nfON^}l~(ql6QwvT_>w;4^2MN{XcUUd9~)Tzr#B)UFPBn-+xwx5$Va zeZQRVxh5#O{+;}fR(h6>YP%`;;A^8-Flg>3x{19y+0C?bK;7jD7Lg`ld23n{>^6N= z=TwG@qk1FWM#&3x^W0JX0q=i{<4)2t+3_U#0MKiQsd63sii7jnQ8aV$@+#6+6W9CH z3qebQ-PFsE}@w1h0ifmNFSh{D)2<+}zr{7?Jj(b=cg( z5<$m*>+WuR6>Ka2E(V42heEaUcb3hkJH`x+*0Bsl!QZh4oUkbHmLLrgA4 zf?$qmv-@4T%24bOwo}>2YX59WW6rvaN~}OeHsES8v<;HOXCr^F`jJf~{6#MPNBTG4 zk^xCf|DZzv2~v;ge-8d%@bZ-j*MEAc{|kpw{qO8ggzy*Q^X-4w=G*^`{iK8OulwWw zkk9{b2Zt@U`@*YQR{l*EFgZ4<|DS1p`|nKw{(rsYuqSNJh)mL5_gwdX^OpUr1g!e> H<;VX4@!Fbi literal 0 HcmV?d00001 diff --git a/docs/en/advanced_guides/evaluation.md b/docs/en/advanced_guides/evaluation.md index 422d9f23..565a36d7 100644 --- a/docs/en/advanced_guides/evaluation.md +++ b/docs/en/advanced_guides/evaluation.md @@ -1 +1,103 @@ -# Custom evaluation metrics (TODO) +# Customize Evaluation Metrics + +## Use metrics in MMClassification + +In MMClassification, we have provided multiple metrics for both single-label classification and multi-label +classification: + +**Single-label Classification**: + +- [`Accuracy`](mmcls.evaluation.Accuracy) +- [`SingleLabelMetric`](mmcls.evaluation.SingleLabelMetric), including precision, recall, f1-score and + support. + +**Multi-label Classification**: + +- [`AveragePrecision`](mmcls.evaluation.AveragePrecision), or AP (mAP). +- [`MultiLabelMetric`](mmcls.evaluation.MultiLabelMetric), including precision, recall, f1-score and + support. + +To use these metrics during validation and testing, we need to modify the `val_evaluator` and `test_evaluator` +fields in the config file. + +Here is several examples: + +1. Calculate top-1 and top-5 accuracy during both validation and test. + + ```python + val_evaluator = dict(type='Accuracy', topk=(1, 5)) + test_evaluator = val_evaluator + ``` + +2. Calculate top-1 accuracy, top-5 accuracy, precision and recall during both validation and test. + + ```python + val_evaluator = [ + dict(type='Accuracy', topk=(1, 5)), + dict(type='SingleLabelMetric', items=['precision', 'recall']), + ] + test_evaluator = val_evaluator + ``` + +3. Calculate mAP (mean AveragePrecision), CP (Class-wise mean Precision), CR (Class-wise mean Recall), CF + (Class-wise mean F1-score), OP (Overall mean Precision), OR (Overall mean Recall) and OF1 (Overall mean + F1-score). + + ```python + val_evaluator = [ + dict(type='AveragePrecision'), + dict(type='MultiLabelMetric', average='macro'), # class-wise mean + dict(type='MultiLabelMetric', average='micro'), # overall mean + ] + test_evaluator = val_evaluator + ``` + +## Add new metrics + +MMClassification supports the implementation of customized evaluation metrics for users who pursue higher customization. + +You need to create a new file under `mmcls/evaluation/metrics`, and implement the new metric in the file, for example, in `mmcls/evaluation/metrics/my_metric.py`. And create a customized evaluation metric class `MyMetric` which inherits [`BaseMetric in MMEngine`](mmengine.evaluator.metrics.BaseMetric). + +The data format processing method `process` and the metric calculation method `compute_metrics` need to be overwritten respectively. Add it to the `METRICS` registry to implement any customized evaluation metric. + +```python +from mmengine.evaluator import BaseMetric +from mmcls.registry import METRICS + +@METRICS.register_module() +class MyMetric(BaseMetric): + + def process(self, data_batch: Sequence[Dict], data_samples: Sequence[Dict]): + """ The processed results should be stored in ``self.results``, which will + be used to computed the metrics when all batches have been processed. + `data_batch` stores the batch data from dataloader, + and `data_samples` stores the batch outputs from model. + """ + ... + + def compute_metrics(self, results: List): + """ Compute the metrics from processed results and returns the evaluation results. + """ + ... +``` + +Then, import it in the `mmcls/evaluation/metrics/__init__.py` to add it into the `mmcls.evaluation` package. + +```python +# In mmcls/evaluation/metrics/__init__.py +... +from .my_metric import MyMetric + +__all__ = [..., 'MyMetric'] +``` + +Finally, use `MyMetric` in the `val_evaluator` and `test_evaluator` field of config files. + +```python +val_evaluator = dict(type='MyMetric', ...) +test_evaluator = val_evaluator +``` + +```{note} +More details can be found in {external+mmengine:doc}`MMEngine Documentation: Evaluation `. +``` diff --git a/docs/zh_CN/_static/css/readthedocs.css b/docs/zh_CN/_static/css/readthedocs.css index 577a67a8..fbf96d37 100644 --- a/docs/zh_CN/_static/css/readthedocs.css +++ b/docs/zh_CN/_static/css/readthedocs.css @@ -25,3 +25,9 @@ article.pytorch-article section table code { table.autosummary td { width: 50% } + +img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} diff --git a/docs/zh_CN/_static/image/confusion-matrix.png b/docs/zh_CN/_static/image/confusion-matrix.png new file mode 120000 index 00000000..7b0b3772 --- /dev/null +++ b/docs/zh_CN/_static/image/confusion-matrix.png @@ -0,0 +1 @@ +../../../en/_static/image/confusion-matrix.png \ No newline at end of file diff --git a/docs/zh_CN/advanced_guides/evaluation.md b/docs/zh_CN/advanced_guides/evaluation.md index af3a57d2..82529786 100644 --- a/docs/zh_CN/advanced_guides/evaluation.md +++ b/docs/zh_CN/advanced_guides/evaluation.md @@ -1 +1,4 @@ # 自定义评估指标（待更新） + +请参见[英文文档](https://mmclassification.readthedocs.io/en/dev-1.x/advanced_guides/evaluation.html)，如果你有兴 +趣参与中文文档的翻译，欢迎在 [讨论区](https://github.com/open-mmlab/mmclassification/discussions/1027)进行报名。 diff --git a/mmcls/evaluation/metrics/multi_label.py b/mmcls/evaluation/metrics/multi_label.py index 62498038..e0a4b936 100644 --- a/mmcls/evaluation/metrics/multi_label.py +++ b/mmcls/evaluation/metrics/multi_label.py @@ -13,38 +13,58 @@ from .single_label import _precision_recall_f1_support, to_tensor @METRICS.register_module() class MultiLabelMetric(BaseMetric): - """A collection of metrics for multi-label multi-class classification task - based on confusion matrix. + r"""A collection of precision, recall, f1-score and support for + multi-label tasks. - It includes precision, recall, f1-score and support. + The collection of metrics is for single-label multi-class classification. + And all these metrics are based on the confusion matrix of every category: + + .. image:: ../../_static/image/confusion-matrix.png + :width: 60% + :align: center + + All metrics can be formulated use variables above: + + **Precision** is the fraction of correct predictions in all predictions: + + .. math:: + \text{Precision} = \frac{TP}{TP+FP} + + **Recall** is the fraction of correct predictions in all targets: + + .. math:: + \text{Recall} = \frac{TP}{TP+FN} + + **F1-score** is the harmonic mean of the precision and recall: + + .. math:: + \text{F1-score} = \frac{2\times\text{Recall}\times\text{Precision}}{\text{Recall}+\text{Precision}} + + **Support** is the number of samples: + + .. math:: + \text{Support} = TP + TN + FN + FP Args: - thr (float, optional): Predictions with scores under the thresholds - are considered as negative. Defaults to None. + thr (float, optional): Predictions with scores under the threshold + are considered as negative. If None, the ``topk`` predictions will + be considered as positive. If the ``topk`` is also None, use + ``thr=0.5`` as default. Defaults to None. topk (int, optional): Predictions with the k-th highest scores are - considered as positive. Defaults to None. - items (Sequence[str]): The detailed metric items to evaluate. Here is - the available options: + considered as positive. If None, use ``thr`` to determine positive + predictions. If both ``thr`` and ``topk`` are not None, use + ``thr``. Defaults to None. + items (Sequence[str]): The detailed metric items to evaluate, select + from "precision", "recall", "f1-score" and "support". + Defaults to ``('precision', 'recall', 'f1-score')``. + average (str | None): How to calculate the final metrics from the + confusion matrix of every category. It supports three modes: - - `"precision"`: The ratio tp / (tp + fp) where tp is the - number of true positives and fp the number of false - positives. - - `"recall"`: The ratio tp / (tp + fn) where tp is the number - of true positives and fn the number of false negatives. - - `"f1-score"`: The f1-score is the harmonic mean of the - precision and recall. - - `"support"`: The total number of positive of each category - in the target. - - Defaults to ('precision', 'recall', 'f1-score'). - average (str | None): The average method. It supports three average - modes: - - - `"macro"`: Calculate metrics for each category, and calculate - the mean value over all categories. - - `"micro"`: Calculate metrics globally by counting the total - true positives, false negatives and false positives. - - `None`: Return scores of all categories. + - `"macro"`: Calculate metrics for each category, and calculate + the mean value over all categories. + - `"micro"`: Average the confusion matrix over all categories and + calculate metrics on the mean confusion matrix. + - `None`: Calculate metrics of every category and output directly. Defaults to "macro". collect_device (str): Device name used for collecting results from @@ -261,15 +281,16 @@ class MultiLabelMetric(BaseMetric): target_indices (bool): Whether the ``target`` is a sequence of category index labels. If True, ``num_classes`` must be set. Defaults to False. - average (str | None): The average method. It supports three average + average (str | None): How to calculate the final metrics from + the confusion matrix of every category. It supports three modes: - - `"macro"`: Calculate metrics for each category, and - calculate the mean value over all categories. - - `"micro"`: Calculate metrics globally by counting the - total true positives, false negatives and false - positives. - - `None`: Return scores of all categories. + - `"macro"`: Calculate metrics for each category, and calculate + the mean value over all categories. + - `"micro"`: Average the confusion matrix over all categories + and calculate metrics on the mean confusion matrix. + - `None`: Calculate metrics of every category and output + directly. Defaults to "macro". thr (float, optional): Predictions with scores under the thresholds @@ -402,14 +423,25 @@ def _average_precision(pred: torch.Tensor, @METRICS.register_module() class AveragePrecision(BaseMetric): - """Calculate the average precision with respect of classes. + r"""Calculate the average precision with respect of classes. + + AveragePrecision (AP) summarizes a precision-recall curve as the weighted + mean of maximum precisions obtained for any r'>r, where r is the recall: + + .. math:: + \text{AP} = \sum_n (R_n - R_{n-1}) P_n + + Note that no approximation is involved since the curve is piecewise + constant. Args: - average (str | None): The average method. It supports two modes: + average (str | None): How to calculate the final metrics from + every category. It supports two modes: - - `"macro"`: Calculate metrics for each category, and calculate - the mean value over all categories. - - `None`: Return scores of all categories. + - `"macro"`: Calculate metrics for each category, and calculate + the mean value over all categories. The result of this mode + is also called **mAP**. + - `None`: Calculate metrics of every category and output directly. Defaults to "macro". collect_device (str): Device name used for collecting results from @@ -529,15 +561,6 @@ class AveragePrecision(BaseMetric): average: Optional[str] = 'macro') -> torch.Tensor: r"""Calculate the average precision for a single class. - AP summarizes a precision-recall curve as the weighted mean of maximum - precisions obtained for any r'>r, where r is the recall: - - .. math:: - \text{AP} = \sum_n (R_n - R_{n-1}) P_n - - Note that no approximation is involved since the curve is piecewise - constant. - Args: pred (torch.Tensor | np.ndarray): The model predictions with shape ``(N, num_classes)``. @@ -545,9 +568,11 @@ class AveragePrecision(BaseMetric): with shape ``(N, num_classes)``. average (str | None): The average method. It supports two modes: - - `"macro"`: Calculate metrics for each category, and - calculate the mean value over all categories. - - `None`: Return scores of all categories. + - `"macro"`: Calculate metrics for each category, and calculate + the mean value over all categories. The result of this mode + is also called mAP. + - `None`: Calculate metrics of every category and output + directly. Defaults to "macro". diff --git a/mmcls/evaluation/metrics/single_label.py b/mmcls/evaluation/metrics/single_label.py index b18b649f..4f17cac8 100644 --- a/mmcls/evaluation/metrics/single_label.py +++ b/mmcls/evaluation/metrics/single_label.py @@ -54,15 +54,25 @@ def _precision_recall_f1_support(pred_positive, gt_positive, average): @METRICS.register_module() class Accuracy(BaseMetric): - """Top-k accuracy evaluation metric. + r"""Accuracy evaluation metric. + + For either binary classification or multi-class classification, the + accuracy is the fraction of correct predictions in all predictions: + + .. math:: + + \text{Accuracy} = \frac{N_{\text{correct}}}{N_{\text{all}}} Args: - topk (int | Sequence[int]): If the predictions in ``topk`` - matches the target, the predictions will be regarded as - correct ones. Defaults to 1. - thrs (Sequence[float | None] | float | None): Predictions with scores - under the thresholds are considered negative. None means no - thresholds. Defaults to 0. + topk (int | Sequence[int]): If the ground truth label matches one of + the best **k** predictions, the sample will be regard as a positive + prediction. If the parameter is a tuple, all of top-k accuracy will + be calculated and outputted together. Defaults to 1. + thrs (Sequence[float | None] | float | None): If a float, predictions + with score lower than the threshold will be regard as the negative + prediction. If None, not apply threshold. If the parameter is a + tuple, accuracy based on all thresholds will be calculated and + outputted together. Defaults to 0. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. @@ -262,41 +272,59 @@ class Accuracy(BaseMetric): @METRICS.register_module() class SingleLabelMetric(BaseMetric): - """A collection of metrics for single-label multi-class classification task - based on confusion matrix. + r"""A collection of precision, recall, f1-score and support for + single-label tasks. - It includes precision, recall, f1-score and support. Comparing with - :class:`Accuracy`, these metrics doesn't support topk, but supports - various average mode. + The collection of metrics is for single-label multi-class classification. + And all these metrics are based on the confusion matrix of every category: + + .. image:: ../../_static/image/confusion-matrix.png + :width: 60% + :align: center + + All metrics can be formulated use variables above: + + **Precision** is the fraction of correct predictions in all predictions: + + .. math:: + \text{Precision} = \frac{TP}{TP+FP} + + **Recall** is the fraction of correct predictions in all targets: + + .. math:: + \text{Recall} = \frac{TP}{TP+FN} + + **F1-score** is the harmonic mean of the precision and recall: + + .. math:: + \text{F1-score} = \frac{2\times\text{Recall}\times\text{Precision}}{\text{Recall}+\text{Precision}} + + **Support** is the number of samples: + + .. math:: + \text{Support} = TP + TN + FN + FP Args: - thrs (Sequence[float | None] | float | None): Predictions with scores - under the thresholds are considered negative. None means no - thresholds. Defaults to 0. - items (Sequence[str]): The detailed metric items to evaluate. Here is - the available options: + thrs (Sequence[float | None] | float | None): If a float, predictions + with score lower than the threshold will be regard as the negative + prediction. If None, only the top-1 prediction will be regard as + the positive prediction. If the parameter is a tuple, accuracy + based on all thresholds will be calculated and outputted together. + Defaults to 0. + items (Sequence[str]): The detailed metric items to evaluate, select + from "precision", "recall", "f1-score" and "support". + Defaults to ``('precision', 'recall', 'f1-score')``. + average (str | None): How to calculate the final metrics from the + confusion matrix of every category. It supports three modes: - - `"precision"`: The ratio tp / (tp + fp) where tp is the - number of true positives and fp the number of false - positives. - - `"recall"`: The ratio tp / (tp + fn) where tp is the number - of true positives and fn the number of false negatives. - - `"f1-score"`: The f1-score is the harmonic mean of the - precision and recall. - - `"support"`: The total number of occurrences of each category - in the target. - - Defaults to ('precision', 'recall', 'f1-score'). - average (str, optional): The average method. If None, the scores - for each class are returned. And it supports two average modes: - - - `"macro"`: Calculate metrics for each category, and calculate - the mean value over all categories. - - `"micro"`: Calculate metrics globally by counting the total - true positives, false negatives and false positives. + - `"macro"`: Calculate metrics for each category, and calculate + the mean value over all categories. + - `"micro"`: Average the confusion matrix over all categories and + calculate metrics on the mean confusion matrix. + - `None`: Calculate metrics of every category and output directly. Defaults to "macro". - num_classes (Optional, int): The number of classes. Defaults to None. + num_classes (int, optional): The number of classes. Defaults to None. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. @@ -343,7 +371,7 @@ class SingleLabelMetric(BaseMetric): 'single-label/recall_classwise': [18.5, 18.5, 17.0, 20.0, 18.0], 'single-label/f1-score_classwise': [19.7, 18.6, 17.1, 19.7, 17.0] } - """ + """ # noqa: E501 default_prefix: Optional[str] = 'single-label' def __init__(self, @@ -483,14 +511,16 @@ class SingleLabelMetric(BaseMetric): the thresholds are considered negative. It's only used when ``pred`` is scores. None means no thresholds. Defaults to (0., ). - average (str, optional): The average method. If None, the scores - for each class are returned. And it supports two average modes: + average (str | None): How to calculate the final metrics from + the confusion matrix of every category. It supports three + modes: - - `"macro"`: Calculate metrics for each category, and - calculate the mean value over all categories. - - `"micro"`: Calculate metrics globally by counting the - total true positives, false negatives and false - positives. + - `"macro"`: Calculate metrics for each category, and calculate + the mean value over all categories. + - `"micro"`: Average the confusion matrix over all categories + and calculate metrics on the mean confusion matrix. + - `None`: Calculate metrics of every category and output + directly. Defaults to "macro". num_classes (Optional, int): The number of classes. If the ``pred``