From 9a628e78caa9c4430923821e0bdd9afabad794e5 Mon Sep 17 00:00:00 2001 From: Kevin Sala Date: Thu, 31 Aug 2023 10:38:24 +0200 Subject: [PATCH] Add MPI model with interface view --- CHANGELOG.md | 2 + cfg/cpu/mpi/function.cfg | 42 ++++ cfg/thread/mpi/function.cfg | 42 ++++ doc/user/emulation/events.md | 107 ++++++++++ doc/user/emulation/fig/mpi-function.png | Bin 0 -> 33321 bytes doc/user/emulation/mpi.md | 242 +++++++++++++++++++++++ mkdocs.yml | 1 + src/emu/CMakeLists.txt | 2 + src/emu/emu_prv.h | 1 + src/emu/models.c | 2 + src/emu/mpi/event.c | 201 +++++++++++++++++++ src/emu/mpi/mpi_priv.h | 84 ++++++++ src/emu/mpi/setup.c | 252 ++++++++++++++++++++++++ 13 files changed, 978 insertions(+) create mode 100644 cfg/cpu/mpi/function.cfg create mode 100644 cfg/thread/mpi/function.cfg create mode 100644 doc/user/emulation/fig/mpi-function.png create mode 100644 doc/user/emulation/mpi.md create mode 100644 src/emu/mpi/event.c create mode 100644 src/emu/mpi/mpi_priv.h create mode 100644 src/emu/mpi/setup.c diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b6c63..b17a773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add nOS-V API subsystem events for `nosv_create()` and `nosv_destroy()`. - Add TAMPI model with `T` code. - Add subsytem events and cfgs for TAMPI model. +- Add MPI model with `M` code. +- Add interface events and cfgs for MPI model. ## [1.2.2] - 2022-07-26 diff --git a/cfg/cpu/mpi/function.cfg b/cfg/cpu/mpi/function.cfg new file mode 100644 index 0000000..a25b0ee --- /dev/null +++ b/cfg/cpu/mpi/function.cfg @@ -0,0 +1,42 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW CPU: MPI function of the RUNNING thread > +################################################################################ +window_name CPU: MPI function of the RUNNING thread +window_type single +window_id 1 +window_position_x 0 +window_position_y 0 +window_width 600 +window_height 150 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1000.0 +window_minimum_y 1.0 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_randnotzero +window_drawmode_rows draw_randnotzero +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 25 +window_filter_module evt_type_label 1 "CPU: MPI function of the RUNNING thread" + diff --git a/cfg/thread/mpi/function.cfg b/cfg/thread/mpi/function.cfg new file mode 100644 index 0000000..3f87ff6 --- /dev/null +++ b/cfg/thread/mpi/function.cfg @@ -0,0 +1,42 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW Thread: MPI function of the RUNNING thread > +################################################################################ +window_name Thread: MPI function of the RUNNING thread +window_type single +window_id 1 +window_position_x 0 +window_position_y 0 +window_width 600 +window_height 150 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1000.0 +window_minimum_y 1.0 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_randnotzero +window_drawmode_rows draw_randnotzero +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 25 +window_filter_module evt_type_label 1 "Thread: MPI function of the RUNNING thread" + diff --git a/doc/user/emulation/events.md b/doc/user/emulation/events.md index 1f7e955..27c8cc8 100644 --- a/doc/user/emulation/events.md +++ b/doc/user/emulation/events.md @@ -226,3 +226,110 @@ TTC Ends creating a ticket linked to a set of requests and a task TTw Begins to wait a ticket completion TTW Ends waiting a ticket completion ``` + +-------------------- MPI (model=M) ------------------------- + +MUi Enters MPI_Init +MUI Exits MPI_Init +MUt Enters MPI_Init_thread +MUT Exits MPI_Init_thread +MUf Enters MPI_Finalize +MUF Exits MPI_Finalize + +MW[ Enters MPI_Wait +MW] Exits MPI_Wait +MWa Enters MPI_Waitall +MWA Exits MPI_Waitall +MWy Enters MPI_Waitany +MWY Exits MPI_Waitany +MWs Enters MPI_Waitsome +MWS Exits MPI_Waitsome + +MT[ Enters MPI_Test +MT] Exits MPI_Test +MTa Enters MPI_Testall +MTA Exits MPI_Testall +MTy Enters MPI_Testany +MTY Exits MPI_Testany +MTs Enters MPI_Testsome +MTS Exits MPI_Testsome + +MS[ Enters MPI_Send +MS] Exits MPI_Send +MSb Enters MPI_Bsend +MSB Exits MPI_Bsend +MSr Enters MPI_Rsend +MSR Exits MPI_Rsend +MSs Enters MPI_Ssend +MSS Exits MPI_Ssend +MR[ Enters MPI_Recv +MR] Exits MPI_Recv +MRs Enters MPI_Sendrecv +MRS Exits MPI_Sendrecv +MRo Enters MPI_Sendrecv_replace +MRO Exits MPI_Sendrecv_replace + +MAg Enters MPI_Allgather +MAG Exits MPI_Allgather +MAr Enters MPI_Allreduce +MAR Exits MPI_Allreduce +MAa Enters MPI_Alltoall +MAA Exits MPI_Alltoall +MCb Enters MPI_Barrier +MCB Exits MPI_Barrier +MCe Enters MPI_Exscan +MCE Exits MPI_Exscan +MCs Enters MPI_Scan +MCS Exits MPI_Scan +MDb Enters MPI_Bcast +MDB Exits MPI_Bcast +MDg Enters MPI_Gather +MDG Exits MPI_Gather +MDs Enters MPI_Scatter +MDS Exits MPI_Scatter +ME[ Enters MPI_Reduce +ME] Exits MPI_Reduce +MEs Enters MPI_Reduce_scatter +MES Exits MPI_Reduce_scatter +MEb Enters MPI_Reduce_scatter_block +MEB Exits MPI_Reduce_scatter_block + +Ms[ Enters MPI_Isend +Ms] Exits MPI_Isend +Msb Enters MPI_Ibsend +MsB Exits MPI_Ibsend +Msr Enters MPI_Irsend +MsR Exits MPI_Irsend +Mss Enters MPI_Issend +MsS Exits MPI_Issend +Mr[ Enters MPI_Irecv +Mr] Exits MPI_Irecv +Mrs Enters MPI_Isendrecv +MrS Exits MPI_Isendrecv +Mro Enters MPI_Isendrecv_replace +MrO Exits MPI_Isendrecv_replace + +Mag Enters MPI_Iallgather +MaG Exits MPI_Iallgather +Mar Enters MPI_Iallreduce +MaR Exits MPI_Iallreduce +Maa Enters MPI_Ialltoall +MaA Exits MPI_Ialltoall +Mcb Enters MPI_Ibarrier +McB Exits MPI_Ibarrier +Mce Enters MPI_Iexscan +McE Exits MPI_Iexscan +Mcs Enters MPI_Iscan +McS Exits MPI_Iscan +Mdb Enters MPI_Ibcast +MdB Exits MPI_Ibcast +Mdg Enters MPI_Igather +MdG Exits MPI_Igather +Mds Enters MPI_Iscatter +MdS Exits MPI_Iscatter +Me[ Enters MPI_Ireduce +Me] Exits MPI_Ireduce +Mes Enters MPI_Ireduce_scatter +MeS Exits MPI_Ireduce_scatter +Meb Enters MPI_Ireduce_scatter_block +MeB Exits MPI_Ireduce_scatter_block diff --git a/doc/user/emulation/fig/mpi-function.png b/doc/user/emulation/fig/mpi-function.png new file mode 100644 index 0000000000000000000000000000000000000000..00da081e3c284cb9ab7b42ab850453629ec17f6c GIT binary patch literal 33321 zcmb5W2Ut^E+bxO}6$Kk0A|N6~5KvG$L`A>`NS79n-b3#sA|e6;0s_*d_ugwjdJ9ql zQbP|N2{jOsJEMF5|9`%7&wb9#v!AW5wK9{e`Ofzp?-*k)|JMp~7tUQjM?pbx;rX+t z$`lkQ_9!Thjh;CUj&wff(*%EKeB?f%qEf6i?dh z<>G_PbiveLJryes^M9mDeHkKq=IZ56Yj45)UFpEIVq<{`-DLeU=PJf0mLJ)jV;5iN zFiAds?%|17-RHQ9Em{H+N-n3JM1At^q%mCIU>P1C-XYdv5?5kk@lG`Ro^ipZ3-tvn z4qkBG;m-qWD{D8vcltlP-feX96pWWy4*x1L44k+Gu4-@y_R7+(uz?)(g)FbEU_7^a z&^a37*wZx3ab5HFRpWgraxUxRn(FEcaXFcp?eq2uFKYfS7a8Og$uA^zG5X&IoS^qwqS3xIrS~qL30>-GaV*oZ2!nEf<8xze891 zQrak`etSdf+{=d4`majeLid_2*YB1UJlv=rhI&{(z{&;R7%8_N`#s&H+gXiqoeiqe zSx1R)IHD%pSMu6ISX@x2^v>s-{CDE6d+gf0wu zv^&=k?suNSe%hD5a~SO#%q;0XATKPrKjnj}LJ-;zwJZ5G#j){{9=qQk`?Ll#9r`PJ zCeO!zK8ffm$j;`zbH^Nq5!1^5ed*NU$P@Axs>ekXc>eZbA%u|jLsr=!J6VcO%Y%0g zTRx*pGe}qj5nDD4t?|U;XnE8>-w+izT(-Wy8+lFKc{P|x9Q?P7Bpo8Ja4Ot&b^-Gy zuAQL}AIt(FcIgoX4$4zgo99W9h{LrCacm@?ZcNuaU-e@0XXHF-1xZ{Ilyu`FOr=b3TpM>KdtAIrzx-{qmf=Q#p^^slkr>Spz>oX~U*3Du3Qy^v33<9^K zy3|xvMXg}jg}Q4i3j9y(FJg{sIber z&WMS%wYB4%V6ViLI{OKy;TER-G33!$oyQ)tsQp|~8*`w0f^*VM4|m(Aj~n5#8l;h4 z_--47s02OQAg*sDAc%NjjHd$@6`z)tW^OHu=n6SqX+2gB2B_{v&Lg+`Uk+p_YJl(0 zFFR;)I^>3s)f)mO9IMkxQgPHW3^^c8Y{2&-G@zpH+l$6gx{*Sxdl&@CvY^g+^;eN; zqL&a+XUGM4H~}Hyd(migjq_UPo8XsxRdc-v;+84oa0JZxXAr{kQF`nBX)i|`oriLjfh=Kr zx(^Y?B4Vo#eodk$u48dbhmRh)qk-7b0N=3&LYSrjAvfkn&D3dDYTBECO+*mgu<=n| zYac&;j2CyAv}0w}-N6=-9PL(?~kv<~n%tE0a5ro_ot^%Y*Hu4fo-i zg`QaYI$U@n!Dy!0zXAh4j1zU3^xTrij>kN-%&(jY(15zl4P1b^yE2Dwf z@W33FBp$6M`7y^xxZ#qd{f6>&J=UwRCf&+We7eL17|~?$XOiVmZih&>XodUEvbIfC z+825%QLpwdtf(s88s|UDph@31L~?g>uij;$J6d?$wkecNuAgJBgEKKs$a*wWDFyt3 zo<)*F<-u|ewkLM4oh?Ly@Vkj+zkyPqx>UdMJHM0-&Gpwr;RM%hs)1U!twD?)&QM*-vJX&LN8NYN?CW1xq=|Q=a z+c7w{9QC{~gb-?9o0u#U80SF#Mv=VQaSmV-rN9rx&39SHHbUazb8IZ3h%OSi>GX>2 zx-JvoTox)`oRa_QRS8LvWqYo+FC)P2tU`uAL1G$q{_A0!{FQ z=pKf2gaLE=3y)32{QUVFEOVhed9Wr-`oIlnVjJ{U~Xqj&M zjTo`{zP>;QPnuftrP|#Bzs4E$@WF$26Vfh>qDmpi!MbD?rEeTozmKaVY8aNtMnq_7;BfpzYJ z@!ErcNtr;9jyAOqch@|2M@&O*h+g)pu^i6V;oAdsc7sIdC3(&U3Cm(+VIbw|IypIw zBzA?wiP+6dHTo>eI3}s4=gsgrQf|~`f!iDP(z~5c!R&=$?t=vEj z0_*+&l*1YFH?p*_C`MdT)pB(&One+G4Es~C0Hh#S4KN!oBkzj3;f!Qs!D50zj+PY3 zGeJeqCjs)0;LwDq%ie^G+`z34`VtKlX8ah2_@@l=aC6u1xA3PQAunWz?|z^9@HA0) ztjYlkz7IJi%gL>emUWSf(0GXF9NP%#;cs%*?~|clo2aQMEiFZmhzv))znl`m+;??# zkw13>gCwCf^m=}EG>R+4Ck%P}F}r61WTbbrtYK+sDM)>Sg2J?x>-iR_(kjnm`Vd!n zdSZn)$oC@_R$m3CmdoURvHg4}xS5SK@O!h|AfA&8Ap4Cp3*87WM)8T;@`w1y4Nt5% zSTwp7HWNGrwYMNeS6c2$0cZq1DJZxW1P2Rze;Q0pU41?DCsi+j`Cb z1PEaR;o9+{4h!Vlx4qaG#3&5)+-zZ>KUbWQ5yTa=Y&wc2t%LF@jKud88MaX~i8+F} zNh{+Zt&A<6kV!lk%+N8LNC_EtT4s~6dX_a;3yh)0Ke08)6_O!_+8Kb^2M z&!B8uk0ZzQZ13@70Ggg*lyLnG07W(Urh$CbnI3!V43A|W_Z2LW;Y_T{4qXB@1qJGO z-bD|ZM;i~8ijmX z8lBZ!PM>wQ%{&%)jYdCb+_Je>sirFaJQ0ohTGx~KMqQn51KmIxT*A9;?V^(z?8Koe z#EYtw*Q3}->Jm8??URHY)(^X~0*plUs4?{VRs+gk*hfpQ217t)(i@9gJoT>cxNyfK zbKMMKSLZx;5Wx44FKG0My$JDh*3!q;s03u*BywwsKd54Q&2V6w^LMvWM*HD2$sq!Zsf7(% zCfqB><^e)5laVulZb!O#GIjGJO4PLLC4am%(n9E{f^hZ^bG5xmIb{0{CY(OGP_kir!O4*YLgOj!_9$+Q&&}u8>%5k zTuh!n21mwrkexvlkcv@NO@1_O1sh`(1uh$tBUsQ>RF{_vMRi$&VY|QG&)DGu?|Nrq zIa5K{>V-fKX!WW-*jPemmK75~t32*3+8SReq}rUHmlI;r-o|0MXJbHmLTT%zlYKd4)a zu(dup$K>95#aJP0u*4~;Z&Xgbr;2kpar!(+1ng!%Sib}j5+Uy5HVy)E8 z8-*1;gZ@O#&CueU>FL-mcE9NPXT+tDmm7#a9Wa2z-b~gNHXoWVQ#>Xo7E$dbMu%{!13s#};J9OMvnMldIr$w{r0RsG zYLf0crRvy~)zsokuUxLqlpH<uDu!KW@UQRl0g+KO^%osJ4P|>1 z$Fg>iasyd-f4|``t4Q&`Wkto@Wv+y;Ii>lai5Fg4TwE+rF~$pjaIe=~kOj%?#Y$cB zwqcTI`Dx!1TVr9(vD1XKc!$Ol$Hlox?v5{JUwyPsL4TTog)^C{H`|!k4HcIW z8Z9OV1$;U08GJOv$a8R325kw3zlsrLl(tDFG?}zZ8aX9{A)F73mMOo~nYJjNY!H2A^^=t{C9l^@F;X$7a zt#P)Nk@0ok102n;jziCfvsVDvxCeU4$j{^)@=mq1mLLt9U{U*K(czyzZ9)IKu+2mc zo@b$3IC5t0;;qQlD$8zI|C6AzgnGI*#;dl{A97+kcVA(UJ!-20CcG2)`b? zd1OR{42k+4AOQ|Hkb3bRn=SMkzlxef_o90}O*+EZ`vUEt@aNAvy!Ix99TvKuR5!x# zCa87!?GUPTw2T-dKu={fU6d`{AOOZM#N#1V^f^oRY~TKX>;os zs!eXR!Gz&5zd**UVQri_Bwg$q@7nxI7hQ9ec_=y{&UwVoY%=V(M|e-%=a!`p_OMj$ zJLBmSf$MH75tl^^79MkA0#zJed2r=HmdbxM33=)C zr`27Qu$!Jf~I|Gh@z{2Yc6*p_h^z-I$_mEG_qCxWm%p5E=VhOYt*N zr_QuadQe$7n;4cfUZym_EbE4{)C-Of4(?dIc$As2GX3EeHXgZ-qdQ|oQ#QomqF3Pi zta(^)i8QCSbK$reefRf4sZgrhn24jZq>G!J+|h}zEju(BAg0#7$FLjK`ehf(l;zd> z3t9I%7D5WlG;Y7C{jtgd6R=kLRHD#_!;#4*CUIx0iRv;?y#S}<@=wI^7(j{jUWcup z8C2BOcLVhZGXS<)cU%LLNAXMszq zaR4;u=Qi$x~{n28LIZ*nDbkZXOr4flKU41boE$ zL=DXi5!=R7^di{P=damLH5_wV2d!0GFq5{P9w?CeYhx91{H<B(XXwK0+?5egBY@RZqHdBnPrqt}CShx0TybbXQu6i~M`6oD&frjIDox=y2B&{aV&G%}srbbpH1a3mf17}wEm^xLPVH(6+R zo_)@j9r@gIx(9hJMcB#0y`^lsGrMQWr74!n%}E<|$EGI{QBzi%6fJtTJC*Q+nNCLp zanqdf?%K+4Au6T~YZ|mR$;m5x^yp6gF7+8cxy7WUy^`uk47-z!t^3YrJXO+=YJyrc zC*vSvD~2$6CrQ(OS^Y)T2s&V8^R&x*MDnVv{^yzTN3-czbK$J(xfZwc#&A1>JuJ)F zM#JnoyWM632s*c%Tt05$1osx3X9)VNg0+QB6KG!jsHnx&B+Q#ydCRth@|r<4l!Z)A z04;;4JqUs(KoDz-oCmj-q9|FdCBPH3_;5h1#0)!;k39xg&c`tVt~Ei@gWUQLYzTqF z?^IiI1DwXJD<0wzXUNx?8Cz@6S^}nK3)}00LJtiI;j26aoXMB|pv~hR`gDnnLouP& zZnhP$qdLSuGRk@h0L;}M-%7HFSOoHwwm18mXuU$&e58)Q%Gi0{ zCh2|aie9O|%6q#(_>JNq+=Wtx&R+^am63Wms;&cxQ#aDt*)!OmuN)g4_ky^JTuWVx zXFrjkhN^IIUR_S+<`E;R%e)x#PBb1o?;+mk7H)2s7YgP9%^!!LdNrVq+%befE=E)*%Z@vsd;nfsG9WM8Gb)tihb~O zNg&tZYT5S$M!f9yCjx!^_!o;*;+Gv1B=9OPKceB?F@U50X5*%h7c3(h$bl%5|^{vQ6Kl%Ds-a;yR=;%u`yew<@h#MZyG0R2-{arH*bQw#o(5#$G` zC8H$si`NQT)Cue~bX&87=f@T;#hZxeI(nbyV-~F45+68K=G9_#Z;KbC%LKKHE9?BA zillB;$e?%lttBXZX1)41&VS_D!gy|_7mug(^Iaht-x`nbsR9$*Qz{kS(P&#g!Huj2 zoeu90Hjzf<^T(3n>>u8kaC;a;Oz%&jUZ0aFL&qch(ydQ;Wqj&T4ZOA(yPRP*AY>+7 z1r02vG}H>B9*h1NIL-T(w3N7R%}Gl3mX8%xD)z8XJ|2{DRQP(u0Cuayuuk5fCWE@X zsmLcR{y9%&C_)B9{oZRnSE;nt6FZ)(Sz=*n31$Y6&~uR?YMh&lmWsT2H%R! zK%0|K_6$3_;emic1QgFy+{!>kuZp5%#KZ;SiZ(d=-4?tT0T_{w)J)59i~>gOfaTEk z&>_FGzf1{Q?_e|i!x!MEU)4@4VEmH92Fzd5;ljV5qT&}5*E3_|`I(l$z=^Zp?J&$Lg56tc~U zN{GPk&r&(L69ZFyi<9zAU{lF`q2|2&F-mSuZi7!|>6dehlO{2y8M_&90kU?Aik zD#XM)yA@iALIPTOA0)~Bo^lQs>sS z_jAl-`J1O{vf&KdtXEn^+Kpff35>9)pSK-eyl0ZSxWWRGYRw3{n>=E46<^gK6&`YG z;Jzdy-eFdEE-yW9lE5|iW`^O+hQlTV#u=EGDf?9-XP$j^JH`XEt0_eL&AJSaK2Noo z+qo1}A6fB6t0KI|YUr7@p%cC9OMaL5>XliU&5%uwHVo@B1v(vmhSIb>VmVrVRy zhLl-`Bp8wE?LP8ospSH-LH$pfk_6-)2(U;0g;~^c0Y^;OYyLLzM0sojl;(gUo{k$3}8G&YbVxYK$IyZ?EujQASIpvzkH+4OI%kO)$B!6c7S6MB4_kQm`mw}k&aj0Q5l@gnTza8zq zE>o|G)iLba?Z=JD2!dwHO|F$GuzA9PZ|TilOyZpVO#8m$n8GZN9h+S&z5-kzGhw-E zJ^p;;iRZi3Z{H-nCOpM)iBJ`LR^LFAK^rwvAb$;%#sI73l<0B@;(lb(lP1p5G zOQMiw_OR@TFihtN9iuX7;6~i5C#4R!*i^eSCNR#nWR zF;lhdv6>GIgw+Jzl=Tqk>y|DbMKOH|OnBuRow={=_DUexHRT}3Tu(_bay4^bU-SbZ zu;cv#_HiQDUe%(5M0okB2?rkYw9$XP07ab#A3J-KDwHi`2De^MsL${PkA0nR*yO!~ zeJrVVb@R~~h%aGem+rxO#*Gx6OPlRsAxuXSjtWR-g62)vu_#2&G+_?jyHnw2zu$3fRxW>FOzQU=6ido(`OJDWZ$7CRgn=IcORl;~sx08F>3Pz!L#H|G-f^?VCJH5R+ zvIW$Jo@EJe{M|ba3rzV7C=qZ%KqQX`p!eY5fHTpv7(jB+DT?JOkiUgleL09Ypi^zs zY)E0pzuh+kGGgq+g@3@+!%hEq&IH%w($WX0E*Jd&FPdt$mH?2s9Lq6K*~P))_#X)w z4oEE)&)FIlPM{)l?MEUnlh-I^f>zZgc}?uYJD<3JPCF(s7BOM$L)rHuFGA~cdoghpPTl;P^uhI`7VZ|qy{m*cKr*}sa@qgwW!qz`bjrux2#WzEQs zN17UyY*tg^N{o}Mvb4T*Ck6E?dZ_P5v6cVDO}&r| z8Cu0J-iw=#SxU`UGL^^XFU}1v*#y?R?R}bHHFrt=W>I!SKmt1H?tT|Fb|O1;YNgR@ z#u#miIn&6RlgMVFAr~N zh`FFKHuJgq0PQ!)vS6VAsN@Ih{Mv6@gL&UB3$f&uPR&JL8II66zuH0IIhf`MiITqP zBpOzt^L_jA?De#-6Tg|Snzu;5zv3Fn4NzCz-eippuobKf0d}+CpRlC8&rgc6%m?bw z-1gD{P~;QG{t%51v_VfudExptkUGV%Ks?9j0)Dh-Nt_&PUjSnZi>M_4!Jr*(0EBc& z;2l6!0YJV3g3|GT+yQ()1xPB4XoXo{vL6fF6;x#Kp}p&;8OS3%%2@)G7$9F+u=6>$ zdjQw8h&d)3!t6yl+RJg^2WSqoq9Rz3L3{6N7TJiFI&m;KHUd>oEi&3M^VLzVsTs4|g z)bI{_j;iq&efgY*4>`(An}Nxv)S$OtA9eb_U{M%sE^4tG@P08HV9eqT zFbXXEz%Axx>^P%FCzHGVUG)x&c6u9xW4TttrJH{&mFiiRyISAM!V~)zV=THS>{aoE zAF9Cft5P=)2aha%HVDjXp-)6x8xPOQ>dEZoWE8{a)sM68^G3^jffhxoSoyBY`!^0NPP;2Py7!vX7-IR+qlqOsy!+VPz;UCnu){ zm@cS9$&A4x?*MgLpcw+96F`4DSKu0=cy}B$b&(G1037r1HD-F z7JC#h$2?8!>XrKTnaoptx0NUGlKeRnxOjPkatx{nSX_pDY&l4ECn%|P0AGNr0@8JR z2e$#}^TWNVPj9OrQie+RnfK_(kY!W9h=`Oa7Q?x}HzcfldmA_~gxJ(JK$uVNi}oW9-R+ zV{T`?QV~nJQn9`RW8D}C3hDKp=T?Qf{uWRH-VCzw{6YwNiaW!yTZ%h(|!VnzcHSf#j^rCEx9)1S~YH`vbl+)g6TL5X9s5Ma*Aik zrZSr)Ih>G;!W_)$*NG=L=y3<+u_KSUe6RJ~;U&-Z!qV3xXHW?NdjqNz@U{VgQusiJ z2x^(I^=PS4N7!eFlrw~!{OaX}SXh2$<{RL96Vpevn#RR0p za>pfQtPV(I+J<+*DT;GH^Rz2~6=Qp4xL0%-1jzmsabU)ntZ{+u!tg+e8Z9!E18gR5 z#W*eq<|$n_2EJT@*P10jqd-^x3~37gM`!UJ0QQSjQ-}v3I7G4TyQ^do8&qKuu$0*Y z=5-4tUZ9ep09<(HzDeYdwndF#aO83sf*{PIs#bvG3%GtPYHA7KvI&P?Nnj1puC$#7 zD!leRasx(@ps;-SSImY3eL)bcEzpVq^97xk4yTak{%@ej2|ai)4xBxJ^$pU4I4ys* zUIv8N(O`xYz_taz2Pj|X0qh~OZ6SyiZHzstj)3dR6S@h;o?_zvt%sQeZV0md0;t3| z;OI!o+rp7;B0|^63+dfc0B>o4aRC?s7RSI(q(uNI>4tnpn%SR7c1XasxXtC)kcY8fglB5^TW6ZXLU_Y&FC@Snqh?* zhAu@7pgkB}Uj01ptP%M+pgN`-YU9%XJ&w>4QVq~)NC;yuji6XvQRre}XXk#P>yxnP zP5MFcyY;0KvfHV}Mye7Yj9%jvIuqJYr<5eKikH1@K7V^cA=b}zYKCgZ{o8nHpvmo$ zw+oG9c#E_r;cRkW62DTT;0R_0WDEJqvGFu^3sWaxXo^knB`4$ZBlh>xrG&d2k>gdgVc)w3V*U$3Yh zJ-tc%Y^pY1xO=r~F&hHyMN5TtFSrmnzKU~enr&^rKKi=nI{K!T_G(3H{F*)unjVFD zW})z2TyssiPO5Q4vQr2_r(hTpIHf4f*VG zw-G;^dE0l7BNNJ^%JTxs0B!)UF&J{9{vmLo0NQnUaL{VJ@;;HqlWZXU`STuNXCp6= zT>!3fWHJ{3jl*dggvkH8%nM+YG|aW;1L^E9BaITJ{?)s#R(1kQTYl9-44YiA=h92P z7(g)q@0=5MJPJve2TE}M$e(gB6+_U5zsxoe#ajJVSOo%y(*0cE-2;YA5iGEW)&ML5 zH0P9td@|ZSV*wQ2NYxhZr-@QtHdXVHKqD2!+SII-1M_ETEigw>P-v#ViniaV+iS(N zCUzlKi`ry^uB$7*P!%X+xqE zTY*7>;&mklF%lD&{pPWqS}u@dYc}dV(AjEWtvaBBuH82P=pG0;wP0P%BQ?k$Z05zA z6;K9h4oHCYz>xw9(PyZCjse-E=10SlpG|&g0TI{p++WsOD^mm?pTYBQZzura7?({N zyJ3fyN$ay=(g?E}b>?`IWqiFG`EbEV+RE1&&n zQp;mo(T=BP`i~-K=}(NzC67P$Os1Zr=6!rmMZ)c3dpFFKTj^x@D%#%ymU=bjO9&I% z)i&V$TYSfg`PudIXU{ST@5k>=8l6g{O^W^s>*)Eu?uf5di(a??;iFid%jKd%b<0*g zyx>s$u|jcF9$$i=#r62!g-vm3Ojwe)2xI|Ldy%0c)$R6j4SLA-o(v?}WxkimEy*>1 zu<11_sbqzgTli9(G~Q5V;#5c6{+Qyz%8FmPXCF~3%>E4_pCH-jpU3-2rLB}?+Y@Io@b|^%qIaR8(e?PD!BMx+<;Kc&eM^oTV$b!|Pi&r; z6}RP=e15=LVVJ9QDS7llPpr>zfCPJfGhz;fm~y&A{d7h_MSp(T^4 zY~!kn%PX!T9sMTV??yC&D1S|vLcWh?#lb}dIXuLncMIyh9lV>Kcs%lAa(y}fJIynpv!f$8oB9t*~k~j%s!JcigAhx@h|Mb$RC_N^H%i4;c76e^A3i{idqCm9N3; zU2j@Ue3sGJx_4)FlOwd^v)Cgc=3o=Wvw3?qwrW`ocl{*pLj}+|`Ziuk;gL#tPoxrO z>sVnH1Nye4b|FUOGpauNUS+3mLFkJ3vuzzYv9~0bAUYJR^BHdnZ~NYer!Ev0mxn5? zRWw{_kO-=?IK8QOZp(FhedXtgr$=WTaM%>?rJG3JFfjicaj_3A)vc}9fEAciF=2eh zjy%;HI$*9zHJGFI@hS&Sg*aER^cO+X$snI*Xfw$P=ebzvSW{~fh*VS8QjYn zUXYD-ok~qy?nh-MWJ-pIUR+hRsDB|)ey64Ji}`&I?(&g^=cXs;a#Hbwl`d5e@Ko&a zb_y4CPP67baB=JUU^;gD6Q|zcrKR%o@1vgDaJ2Q)UL%-c^(Cq#m4bN0OSOd;P3yX} zIo6S#;p+F%_EXM+ZIsPhU$k;>)|pAW@)oO1qlfM5ZW&B*H|xsmctqVRd!efMe&zD3 zD>QqeX%>%{>*d|dGJl-0kO|w7 lmT=!;*xLlp6uF$CP-sN4LHQJC>l0y$I_H1n0 z8-M@#mieIybP#_$oGmbXAHufEYpWF^S4T&bWIx;>$OXJK-TUU*c`7P*Gmwz?mZrUu zMeS)_0#8OFdh&C$yMP6}f4ygp3UXYp4PWz`um0DZMSIUZJA-_8hb_d3bud2wx&;yQ zN!`Gv|GVKt4->g$yrT|}BJC6)j{XNr*&X0Wo^S!DueF5MNx&t6rW}a>de4T*p>^^h zFbMBbolxCNx%wDzxu!FqW+`DQo~UW|*f%ZS z>=^&-b4Ez&;WdHzy4q|>oIT!nr#fD_wl&s)KKs*GT^gPT{kHOSeOeNJqth2u!(_UU%cyt;vsYNM5O|8wrUkF)tNjXrCX;;K%M;4+(+EXFBb zqlb+yI*v$c()aTt7!o7K-v#7FqwB?9sO35U&4kSvjN>3}z)f!BBO3UHA*fsGRI z|4J#UEz7_R;zJ`<2~c57Obl?3YW-JIiMU#G;*>wJY209M=KRHHRte%vdZ>LwnY(i38)aJ^gx-1PNW{2aUR~gIKe2-B|f0|b#D2&dW6;Kry zIVO-l7JCx%0cvv%8&~~Wf)SVZNUH8RYAAZez{V%Ew;E=0wJ|=Cjwz3=PX2+bh=HE5 zP;3mvTaItYqB~XAxTcU;@^CgpS z9o(^~_BmXs_z$V{!ODJ{a}x{^2>xMVG1sue^4{lij{#ZZA(8*zLI&~KLFdZf-DMxEf~zePwh)ZpRb36GCltX5dW!= z+O@8mV7JjGU|`4|Z(fGWVBmKgrHW*kZXjJA^vj?F1si_?{}aP}J@?fKnw{gxb9PQ> zX#=Y48mhd-sn8w7f~-8FQL$Nv>d2hEEvdDzBPQ6((I@fkk=2PG)@)+7!zO}bkB;8+@akm#*aco+gJIClj4Y()*_Qgw0*W!i7i90SMbL z>5Q@8<~57Zgi@sS;5FmVb@JbKi+iQtVmE6x({_G7nclNs*zpI7Sz}Y zyaRAvfcVW=icYQ(} zn8!v(M|JorjJqPiC|Cc6!f&76OnAwq-c zZun3}o66}7u+aeM)a%tN`M|dXweMgDKWq8gzXlRsek9OY`gnGBc2HG7v<3yV93XQ* ze}H{@uzz885P;bV@mkv2+Q7f5qT&GVkrb zl3n_u^4+H_-Wj8ZrC)I@9rxk!R|`f%t(sYU*VQuTzVNn(4I#>;!(c~gp}35G74+rx)nYrVViRte+e~)O}CJKJO|gDrG)UyW;EG zeY?F_syeap2`o7-NJ{`6^yvN_WRtnf``R%G1mAIcU)Lg8HlngxSQhE7pykEl#r6$n zYAz=PPeRe2=-^7b>Zz;nY47&=h`n>$62>dpZVKW`#H(8gPUSbgVREkra(_wXc6Li~ zaPM+U*`g9!dSOcBjZ>HIWUS0SVK)3Da$1lobi28JCNVR?+&kH%5r-)6{1(TDJo?;t z+S#e_(>IJ2u51ncPEwy;@mbb_YT)@L(?@>zTm524GkO`xC;v4#3ttliq}hER`v4B{ec(K;C;; zG3madfjmHe{=+PQj!U(rQ|b>hJRP&&K z8BUp&%ribZG_S~hvpaR|Z=?PnB5tz(6$eyRPptHnw9B?fwl)zvjED)T32K~x-7-qS?KL9}ws|5^-b7&rKeYwqXx!r!6?baYxydS?kY?mPD68z&Urd7!if4*R@XT}exYs0Zd{)BZ zc>GzjicLbX2I>@+S7bU{UE1|9db+vf`Zw+SC&hCVZ0<1}=7kZDPP)+YBv|eItmBB1 zR!l+Kig81QR>By$%^UMwC3qRCmWXUmoTVlG)*8Z1zl*R(uf0}4t}ABVcv^Tg1x4#V zbX!xH)Jr;G^@be{%`gP}Mc$Z2E=roHoHNTbkD4vpD~@we^M1;oznXfXDk}c|URh^( z2xC$2gEEFNx^6*Ds3gPtMK%4a>4Z?+8rgq zo~(z%I)ZP1e7L|U)Ez5az`#aEXBU@1eFaWhG5fhs4*5w*)4%|8YL^+Dd512-{cELt z6)TkI`=?+&LnpZ5LC%>hH6N(7n_a#CKUz3;c6RdSvuw2-up4{m{WKVag6k0|zk2_{ z2zmc!rF?z4fA#6>oEHu*aEe+7nO=u7z^(fFkj7#@wL9Hf8VO{HJci%AV^4iF=;`rU z-aS}vxUnCPTD-CM`jElA2Q~L*#npu2$&GU*G|sK?m8U<7Y6GIu;x|5`zEG#>tXy@2 zP-vH@W1J?FZwGZv88JAm{@C-4Dq7==8e`2Ic6wT&BO<6PfYRi6L>YBwU0R^~b#=+P zYE*l9MaN#pjRjL_S-q9ZH;AP%{z}!ElTe zrB&U{9iO}V4j8(IB?=H=^#|qiC&(yhvM8qgmp={%_R8yNXpqGMP#t6wyIB5D83+mo zR!Q>CE^Sy{S#hy4{1EKsF2fs=&DN76e}`_EBoAFE_8M9r)Jq0qKMzI)`62t(AO2Cm z0Lh^DN9Fi_p$vej1NDC)c*(ymC(9XReM_{=kpGz>L)svtF zj$GjPIgpEL{rXyJQQP$+quO@j#q+6lNP$4^9}%y#;AdwpEU0M}3C_|>7O__k#)+ud z++BKysJ>p-m9B6$FJ6aQd*T(U7T8U7zS8o=(prtE(kmbB?IQ2^n#8L4I01Jv`b(Ih zl2hPuV>{JjB7IgCm(r%#yY=ToolnPQo}m3DxI3gW5T;zgXMNl!Ji2TwS-OR4)`g2uj+cBotis=4A|Jk5tKAQr!{1m9rJiM7tX!dAN~2e{A2(j=TKU?JT(_-ra&6S@wO${0pb_6FuDF!{LvSi~X*sT{; zznumiTf(nX`5Oo|WrUj-$!8;f{$Hz|CVS)xdhMWsKnc{&Sr$^I!}`HW0b()94>(6i zK3(a!M7$_h}DB!Ha?kQ7ko5o7QulyJ)PEK;7c ztwG>v1K^n!TYwOq9R|AqoV4hr$oPJ?4Y`vC6U9Q|`1yrh@{Xu?bo=dOyjb+DrKV(t2=){pfX+_0=Zj&l)3mScDuh7&KNgzF;$#V(Gjcq> zm063|L}~j1m3-_)rd+qO5iv?R9fEb8<`sCkF5t?{y~n6R-KuTts^}Fxz&Nj55BDdxg*4rBUAt zBwS+iyIC`xXJvareDm`TOxA1XIxf!3FjOM4Kv@1mQxH7w^KJ+*!x z376>RNI%$WvRf?^A&sWy%VoGDe;4!q;A@|1EeEAkL+;3+EON}6vflu$XRO}KLtefI z?0G^XelUhUEQyi$(;j%o?DImh10Mdjmk&GuW(j$O=G4utRlZzX2o+`DroUCn)(Ur;bM{)8?e6&aKyMkhhkD)i?9c22_Mc(o`ck z+()05{J*G_+sU~B?3N~{Xo5HYF<1Oo&3PVL6LT2K6Rk4NXLf~iK?t^+!G2XFzBVEB zJ4RNSFICV-M%l_fSun-Slv_yrTi<#H#|oEA1UD}=_31Hj$lR~iCXb@V^740arKM7z zEZwSp;M(;sx;p0RbJw2D2sz=eR_$=J=~sxSx+rGzn3xGv=vrq<)Fwgfre=DGIDOoM zi1b`_Jt21K(7L961ETH=_0l-XXk8{wuCYfhkawCkjQHdj2HM@+CS)Grqej{!CYdERK>6tvTutm;Tp8*=4>iT|oU z*P>|B5&62%m`jJNbZqX(L$ZSQtg5U>oR7_|u@hF^weGT24NMssU7`=0M(EF3o&A3b zd+V^Owyu4cqXH@_sHCVU3P_iTlOv8bz1fHKP~!2 zs>bG~7sD3rHT?3BU27dE?v);F-wqZciVfI~>~`S z5mpZIusiaXs<=RXLb3_)o(EftoCb|yf~v^tw_jh5j-)7?cth?6c>x*U|Ck5Q_knx| zj=yWJvRYbW|JCjMZ`YQhgKEcLY?&2pxIE)AQH@B~64Wq*M+XV_qMUrt-W%lf~KoL-jv~Cs&K|&(V>m5M4I=r5xy7U-QG5# zK_Bd%v-qGsTrd7^&}oO!9rs<+O@C&G($S!@YtmV`sAn1Mo$gHVw6U-DjNG*shDj-W z_m{4}vHU_^Z;JOQ>aMNx+(K$v-1%^3Ci^&A(JKzWad1Ca^0t$4WFBQ;7xc@Ell9k2 zeYAOLJ=)*mDQnUCR@#PHSM9ueI zRA324jmu>y|E%5oAZlfCn3_D&FNT))73!b3%QL{cQi*#s5En`DU+%64h{tC9Mz)WC z9B7u|Pz81X*1!s2XPadh&?F%$39x&TA!bxqFMFMl)%U@A3N+W^!b|9`n?LYX$fth7 z*MhjRcu3tb#77p%%f*L|pt1nv{mMY$9OMSv%Ugth#`1uzwx)P}9i6S(GySe)#7-@A z%Lb_%iWDkN9DcTtlG9o?IH-E?ABz~WoFRwn=?7j%s0AXm+{!RJC+vK7E|BL_h5r=A zM#5+L9QUy0WQSX$e+_v1XA>Uy*et?1(e(;T zPYvhmoAwx%-A@L`ret@r3t!8#y)-ANXw+)sM+;GBx<9BkU_bB#_;rUJ*_J@Nm+^gK zCV@y=s>Jou4C@fVzIKmC5yPCodn<3LPYx^ zJB!dB;RL7XnQ*SGit#1WVbo@GhYEkNi)-=OFsIqcaRzBT;Jyjr zcHEveZ)RFLC(M=Hl8#SK?aGtNu=}=^v<7lnBv&Zhh;WG5d>uoEry#PvsEIvxzAbM| zO+~Cx+$mg1u6{Ji%&{|Ds%w6m~RzjUVOwGYtZtfO0atnB%;ojIA%NA5UP93zWrNP(o zG>}Q%AcGBCyq0T~*pJ!hfNVIzxHx{vlRMhwU|3=R^_9+;Y%F6&27STInf-d%y>jmG zn)_?NHWwZq_S>7_6ACK4Qb6^7m*%oAcS*s%S!0+J5+l*mW0D^zhe(4#{Qx8Yr2c@D ztgLqGq@^xHT53R5K!s%wnn@5_m`w!3m3nDk3QsbHZRlWURjbgj z^Y;|<_jguNpn+5=G|bkl+K(B-7~VdC;+_TCW%_bBZ!b|`#&KKA_+9sTAwFb*Md%lSymr%F5$`u6P1%JTy2NQ`^;7uX23ys{41ldhzE@0e!_ zcFukiZSzzvzc=OABh_d2l}YcL6MGrGn;Z{V!mcsxWWP5mTWP4omamg+ExF{}im#IW zaS%(Ttb>ewiDIdH^!ex=ro8sG*0vy7FY}>I*HuDCTSefU6TeX>aqSv~)c)I#a!Nez6m+1vr*@&lWJFQ5OvJ@*(_9~j7xBpZl2WF$j2 zt(Pf?E(4gdo$8$`ZHhmqk4!21o!1{KF{f(3erb?+z&u>6!XC4K59_q$9D;~#H{ z_?XA+($@mQ2K@G8@6HOFMV0)t+~(HpKQqYqp@mU%^%->&qEZPPca`h80j%OL_Pz?kriQN%>Yf7Hg8#{v|he zn4}-ozJm93wMLyrbMR_0i@O-Yi7yTu$F1oa*UTO?#6M6O zT*oOQ+g_VzN>&o+>NyuHAERW?L3-Pgp6+#GX%*ddH=|Q)dv42@_H3VvIF{HAx`-cH zH}6$Q7{v1wqYocpGrd=uzHL)^k-yJ?$wMu%;ks*n)mHnoIaQ``nqzRxnENG~+%t16 z%!`*r?a=fV;x0d1r&u3#cSk-O5B5}}D^`8~Dg zIlr|&u8%Y6{G-`!S6!Y~kYA{0ID_q!2~H(5hN97qr;MQM*!{BV6PijNY6O1bGm-KUy!`@y=0H>I z(s5oQy7oMpd7b{H!wk!kPiw=(kSb$&X*jqh?r$_b;8HAaQnI}48c{47d(VCnL-{D< zP2_^6>dXxl)wDGt*+9DN=VCTxpTxb}A}Y>DXr4zuVcXK0Zl9j+w_^W{365pu+DI>c zE4Sc{P0yX49z2vT7o40vt)pq0p&D?#nocTG&RYaU`Ly_&Mwq0+jgZs$B(_(iX?TXp zaf17$K0lOVd##<%sQ8$XNkd;;%Q5NdTWsM43>|MmbcwBtN9OgY4VgHGV${rFa`Ehw zziy3GqqtokH^@nDzo4;N#*YC*0GI|MU|WBHdD-!Pk9b==jSFh)Bpz?MR;ydo65hBR~lV!n9Cb$_!4wMO{habzY5~?g zgEiYWA2XEy6>O0F4J#mr93Tg?kckVCs}Ps!t|=?~?@Cgry>|auyB~A)uh;`5pJOH* z{ZH!!lvat7VixF|4JbYXR@d1nHwRMXDX<%;{HHuZRMW(as++&R;{$5<9QL~ZaoV+( z)%uZu<$xkLH}@C_iJp+#NTK~q@BlgP05eR_$S?tQnMS#d?0ahn%6V4%YBY-+t;N=LMSeEzBqb@fRo!z!z3RtV!wIQ* ztg~G}J5<2QNOwa82M1T)!SySFE_vil&YM4*TSdD+>nV7hO61>t)$$;RhqU3k{U~`L z#X@{@)HMY|x#CN*#55^FG0qaya?ze*Jx*+om(MJo`=ui8*M^puRe$(x3@5aS&_sSv zuub@0xh4lW!`bnXQz^7V8>KCu$f+&V&8LczFOYmBI91O{Hp~=RH|*mSHBX|Ldsi!l z?k_v`CPPn8#Tbz`YZkweZn>Y;^`5n4n|kb2{KMqAj#f@v1Wy<*(d>N6n9`Y3$`V}1 z!smbRm&RUfL-QW*x|KDsTEtDr13jlA^b<%d{uCKh=KWdX=;X3#4Iq^pUYBSa%P5Rt$Z?jI%;xu=HOfxw+A;B zdqt;=&u;5$0iLAX$`Q#QiH>nka$>mQetM!-Hjj?e%>50zhC3tQypk=aEKbCv^GGdV% zY*k~a8lv_~72-Eq;&x5Mdpm}^p6dD6oSngAzzB*CSm#>3tbQ1TxUw+wGM4c03GrlB zzY1u^AHAHgmmhxS8}@aKt-sVe9`$Xy*uos@uIPa5Ku*Pn-PgRx^VO)vQZ_=TP9L!B zmg3z=8M=^d{HX2BiwKwV4RenjDBiaa_uC$YN{%u#=W`)2TK<0PeXJlhg!Sq@ z&QV(EVuszM{c_v2{Ohg!`I4r_p~mzN9_Gg@Hj#Bc@Xsg--g+YLel2z6%`5(0ThYpj zPUZs{V+#BA-e-I_psk1V1AZXW*qQ#~MY#G7Z;{5Uwd@Z$UWBcOYz*U&wp%c!;Gr& zk2cCu9+nVflp;zruvD!1u`!QTBhrhHe8^$>KjO~hIBNfNZ~v1BC4-pe;n|kx;_qfE ziI=|6H+1M^=u+9T3fjgg)COx8GweBarldLO(kmN8^2rfdPs;0~Y&nv(|GE{a8YoV* zE+dpw*(GFDY2QL?4(lPM()O6EzOQVuJ0SuGihICM)t zb>5s$OaC4c!ud5rzo0QB{nM1bENRVIS&VNeRq&HIYBj2sit?Hnv*??A`JWGEUfj!- z3B1RdL@YX@wC>hTWZp?XnP zb7z)u#c^0+p?UW}246(u-l%NC+)Fxi*nOg`N5sS0^Wi>~a|vYL_SrJwU$jw*j3~=Y zu;eo=8g74DQk*JYqQW@Im7k5fJGw4RLia71{_8-+7v6%&@M4Lj<>!jdtfl!qZRz8l zoC?MgH)zrIJe}SC5<@)&>G#VCqEsKrKWALd`VwSS7OF_q)!QqJ=wpz=di=P={y1Mz zY3FpkbiTlRRQTtoLiaUI{Xr#gYa_meG0(=s;^6VE@LCG}6_LqV zbhHjZfRA@gg5uc{fyX855*~(kSVFzViBpVwZUZOic3iq$Hs``0KUWsHy}jL8t)=pQ zXzs%tZ~O+=oMWloa<|zcD3Rb$Va+mUt2)0WTgkcVwKV7Xyo_bMb^+S zd#_<4`UF;v6CGJ48Q=xL=_YFAKdHiiY;n{5^wVfI1Avkyhr!qbDqRb}oQ@v3qZy7h z3SYdIMj!(Lej-CO!+-v3tu$Oi$S!eHxRt#f5d9xg(!CuUP90T_+i}K8EP)#Mqpf^E@>#i*lSs zt#8*vciDQ`N@4sK*84SQ3{`j^F0`j}x4T`bJZs=(C2PZ*!k^EykcEdbqGa`(%=8^R zdQ{<7rpPlk9QZ}LK=#Zao*1F;XFtV*0j1DPyy+pLnQVjER8EM;}!RwzA^~_jnf((VPFA^kfOXU(DKe&L7 zbl0VpVy9v1?38NeMEAHeKGpKV#jJd9t+* zU<2mIRcQq@x-FKAJ@NzcVw=LZ7iUr#H7j=qjN*wH(@)5iY00k(GeOrl=ni#zvs7Ff zk$QU{t+Ox#m48Kcb~bP>#l^M&UnJ7=pV(DBFUFtb(hOfzuEGF;2x$;VR(fk0kT$#* zk*5nh0AUhXLZsYTEFJ9W*@Dpw;|rO&fr0QMbLBg5r}0Bt z1aQ#@5c148y1!J2gt-@rP`Gw;?Eq>_k4i$macxzwyy^w<^>%hn?iXqvIZo`He0Th0 z946iH$B1uuv*b{L`MB~oEkf;*n(c-m$IvvBbCrv_=*`%ey^Nq3OQJ8H*6gpS&Wu+f znL}W+*>Tn`b9ioH#d4%J-`DZ3S*^j?{rL(!qBrzpb>Y~nW$98&`l{m%e|n2|ybt?+ zwVQ;_%RIH;R2pV$XSAj1D0QO5U(eE&pzxQ>auj-ZrXw|eo29k+jN^+&74{9Xs_lbw zJU+s7Z?DG(Yu}>Vh~C`aNKEaa@ahY%)QyU`6-uYKo=7=y`*P}4>xCd8E!n4;eP5)) z&fbYFZnh2GD+@TFBaSNdwdX=5*Cki6;*e|!PoM%Z{D;#tsDjN?!K|CH;VNk+=}j`< zM?J8b8N5i=c}my~IYgx@%0h(RKKebne)TkGMC`7TbnI9SKdVLD!Rei!#ToLcBnfU< z88$vLN%jG>-VJ$O5*^N08Vv~eacXUo&)iNW7LW-2Mk^SgY?p1F@3l=!Zj{dKyc$TQ_~@UWuc~Azt4?awtFEQcCVpeUjxKO#k+0~Oxu$%sqccXf*bKG@2rk756YalCw^)~#Lpbh zFuRwn)sZJ@Kn}wESX0gN=b=kCc^zN_fY`cWF8cw4$he0u1Y>zjy3;oOT3cIxX&2E) zIM{;=Y(CD0=4lj?*za-yDZm8$HCIo-O8B@D5fQz;y%2n#f-hX5J8w@;P975zW4}D8 zsHR3=K52ACwi`t|aOm z2PVU-IxP8zBB^ibTQuwVZI2Dv$S#f5@GQ-DNr;M!e%H#c6YOada3_m0rM)iId51FR zV@34!8nvy#AeB+Aqu+(w16?ttO(E!%G38;`M=YahwTc+SuRm018f~JJ_qUt^Yq|n% z9X9Xk`47gFwQL93b}rLip}*;q_@bIfq6z~iLaJQ40-Y^T`oO5qm?GQU7Xk%_t#vev za)5XO?m5`%twEhqePXyEEkF(ul8y`HcJ(*x?{jd(%^`dlmcIU;ird)Y3H1)J4QNlB zXGWm5fVhfSepl|dKTi%7aX3FxwJagZ^s;c|^bjYiT zomXOZ??@Cv&`h$^sFTJHhn#hbh>7e?CV~7*7C?_J~uj1|V z$esLPaCl6hCz$y31>uP<(>l}&tA)8n^BQ7ouiHe321m4Q}@9l^l z>I38uhDdblR!JCE0V5QM%y&=H@Sv!bJ4srdA3MA7>sKkv>)AL#y|Gx&%bz@eB)dWo zB6t~K3eT!<&Y!}7rJoS?h?P`aZjbgNWpWE9B~z-bf)F% z`kgPUsdHuJm=hLB<)F+pRQp~Lzr*zkZE5`4;g=U?H5>N9&F1nm*o2PB5R<%dL-!>{yb?-{QjFJ#lIH898y_z&YyQ@OO_M=2;eJt^XE86^ z{HlLO8>K|6M1eOi|A$hV$KBHH56!g0vI3ighnJ_=$~10VVfD-LfVKN=ZU0Ai6m$%cDp#pr*vA8r)2)4a?!0$?7{M3IJo{$2*1%eyyIhd(XGxGfV#W@rr zro+{@qg$P`s>3lbM2qz!{BQ^;rBBm3sFu(+{qyiN2SY)dHq_*{tundETDdj3Uu3 zxqR%sv9sP@k>)+xEgl?W603A1<3A1!}YQ6bdm-UGD+iS&^ zfu!2);iBGWzjzgrm59drk*6=w`J;65qj$VqZrD-n{J6}M<>T@uGSoav(t5h{;;It6 zteetu@}#s%znpsyXT=NS5qHAMkK4C9%_lq4R(pS0*-2sK6AhJkw<{OjFJ&cDQcDHN zwptQK#osS};u#}kPdZCyLcB#C5s*LreVgqHv*im@xm?OSd#uO74KyzuUcTlE`>I^e zj`km*k_lwWsj7b0(<2n&FY>xZrp5WIOWN%Bm&?B4$mjwnflILRmP0sJg2v`P}&hBm-0FBqlPa?Qb3pE<~ z;EFi}ANs>Npj_*MC~9EB(->x!6ACn~gTX$z^sr}70}wv!mOZ?2FO0h3)T{FcBRMkY zp$&%3L41#8_1-KLFHm|xGmg24tlXEK(iOTwohVpy+3IBl(D;;teJJqx8T`%MLH^I7w! zl={<*svM8ALWi?a5`LcVm01$43iB^HcT##>F1*wGQY!blUk0Q3)5I0t>*><7PY5?2 zFzdIb1S)-!G7s18Yu?5)^Xtj6kk$}Vvp3z0mU%YZ_UZk{(utUQJ+#8NQ%&1*Q)ZXir>V)iF}f*vh73b}f_fu);vecPXhJ`y+Aw!&a=x9Pi=kD|5Fn8m2g z;@N-$QJR)mHljRttuyalb0*z%So+?`&6L{T;CT5eo$$0fed?%_M}u>Ho3%6++p?RU zd(`1a?}Zr%eMp8#Ja7*<#^WM_*bBzV4VI~kGkrZhwl+2hmd!0kyG)O zIee};>gj5eE?$dejg)b)-?fhz@Kokyvz#ow8M)_u)}AGJZ!LG_+d1(x`5g4Soo_*0 zQFo37U+8h}=zCYReCH8*O=oeD^~=ZnfrRsFqtA5W4V7wDLrAvl*n=0<`OvkMe)DHf zb&(O{J?5oUpmIpIB@@N&0dlGV>XwyEe!a$hNA^6(GlcB;knQ079 zq<1@sUc2z%4|HM`S*6L(bgPaDe|DAwJ7qhD#jclS=Q-`A=zs@545>`&3CbwMpL`k_ zn}2M{bA{w_e^9=JcR$Y0ilR}v+A+1ecFu%1te@gXU!D4Vtz&55hneWz5)z*wZ<=98 z8VgTSy;~FB9<=TB?1as^_IE|KOR=M(m$jLRHi%_nT6aBr8d$eI&UtR;lEx^%8+vY? zrX56za@LKSjmzOoT<;Yy#Hq?`dgmm|<=$!gW#Suw(R_gMG*%ChXL2&hn_qPMN~>og z^YVHkxF^M~G+Vi_)Dd7ReOw#6ZH3&LWjwJdG!;V0eqX||-0Q~L;;p;H+w3yC(#v&O zBi)g+?`8aGzjJ+)T*MSBzRA0*FE*}S_APN5EuQ-+Z9FNk`J(Ks-Y+6u-mAKrMk!BM zl~N)~O6PyfOt&{{KlMps|8{BTepr)Y5E|t&xy`(B;f+SVmGKGuNrJ$iOaaJ+oK6H( zv~dtQDFgz2pP{p+>)=9o(>;Mhs9#l`F!fFFNc|DLM>L3$CI0WW>W~4whEoz>-VX>1T5Ye9@tWA!J&{MV`lU%Si1&!B-=AHK7#)EzUj$7be#g8@J zK#2}=?3+%%5?)((QT?tcFNYDNgYE5ky0zy=s52rh-A)J*61eHAkC0+PV3km|%4Lla zetqdC3-#fPnT@XhY8td$Vru?E;1L-p0Z}VM_y4{B+7y*{dmS>}&lh~))ZkJh(s)2W zb?JhJ-ov|r`XSqBRV-Vo|P&i($}{pEPruDjEZkIg-4MoIt-;GrHU zW{s5gP1} zGwyKA?{X_!pYWxf6iq+OnXxyV6_>;=iQ5yEl)TyFcF7>$Tg8QA^SGdiY4O)D(UYwW z0ykVKkQ>$A$k%v&5}Ac+WUmgctrW+-h(34+_j~#qoShjJ!&eopCr^r z`e}t3@T=T?i--IIMYfI2lkbwh9hxDPxX+K3e>I>+M|{&f!V!O`_ncr)VQ#ACYR#Po zi}PHRnz;F1@^^4%=Dw18sM6^6MfGbKDzvg!&0jRwEUsi}ezhw96f5*T4WdgG_1&gm<2fyS;NWR;_)s=7?~lJ6!?ebrYSyw0)c zdlO$tFY$Igq|Vf>bef$0{&nF|io*2RQxCFAJnDPi!;L1vOyRG`RdohENpzzU<9GZ0 zZJ%k}x7KxtX_`)}rC^YCUoyN@tEg~#L@}?tvTM2MhZee??1Iu3@lNx+AF^X|l|U5@ zKpjNb0@j(-b|PVXlLHTd(8>g~(crk?ao#r5syk5}!A6WkCWAo<2_ZG^qOQrKhRgAC zLy@6EH{kz=P`_&#$iggp0RA!xNVwJ{sBvyH?34 zmI<&vIg%r_99A|<1*kx-{ojxw?YN2393z(O)`PbtO9l0(MS8m~;$Opl(WRhhoOHhU zUX|I0l^rw1Q>}xKpX(~)awj~s6vQ^*ip}G^5p&NsD^tWs)9*?XrEwYK_{R@L4ViT2 z`W)4FtqI(fYq(0wt=;Nu67?MniOgAfWSgdQ)PoiERn>J<4ww4Cn4EW}**%t&lbK1fQR*-%VmBbiyB*Y2B<_cLl@HHfK$n?^{Jb2$Dh zZ%1j>-1}BG&Tu(3{P7+C zQfx9!CU2!6%In^aBlpm<{Z8h%rpm)jOtEpiv;g5i`i>NG+b@9m3^3@0gD>!{zr39Z zMO)1tv~AUUDb^M+4FSw+OW(bb#}abjG55I)85AB#})?iZIiOHnz=Z^@e1Q)Yfj$0iiLkh3z$tfr;t$$ zfN;>!(S?x1jQbq$Cc$9!O1PYOR}q>9Cm#9|3^HsQ2B6XeLR%7ejbmYZ$TAleZ+g0z z1;ce7yERG;T9`JHFZSe-0+K-k-#?Gl^iQBE8Oj@{-j3o;$sa*D5;M$khneEeNDm7y z7&^QQ%f{5watuBL5K-s)mpw*BP4ciAAGxuTF2an zG?#!h8-ro1a6`e(20yrzn(+SWUx=ACd5t?ifm@jSgyd`G;)G%JU_MSBgEe~$cp}_I zv(j)1!L;=9HDvrRcym*0e68bw-i3$EM=Q(9%aK$TAiXz{V$+Kq=w={>36Ip_zy6P( aTNqT2*rL*)@bwdGnV68&lN>>9kN*#@A9d#d literal 0 HcmV?d00001 diff --git a/doc/user/emulation/mpi.md b/doc/user/emulation/mpi.md new file mode 100644 index 0000000..86d3723 --- /dev/null +++ b/doc/user/emulation/mpi.md @@ -0,0 +1,242 @@ +# MPI model + +The [Message Passing Interface (MPI)][mpi] is a standard library interface +specification for message-passing communication libraries targeting parallel +computing architectures. The interface defines functions for point-to-point +communication primitives, collectives, remote memory access (RMA), I/O and +process management. + +The [Sonar][sonar] library instruments the most essential MPI functions that any +user application or any external library may execute. Sonar tracks the calls to +these MPI functions made at each point. Both users and developers can use this +information to analyze the time spent inside MPI functions. The next section +explains a view that is provided to achieve this goal. + +The Sonar library is compatible with the MPI standards 3.0, 3.1 and 4.0. See the +[MPI documentation][mpi docs] for more information about the MPI standards and +their functions. + +[mpi]: https://www.mpi-forum.org +[mpi docs]: https://www.mpi-forum.org/docs +[sonar]: https://pm.bsc.es/gitlab/ovni/sonar +[sonar docs]: https://pm.bsc.es/gitlab/ovni/sonar/-/blob/main/README.md + +Sonar requires an installation of the ovni library and an MPI library. Use the +option `--with-ovni=prefix` when building Sonar to specify the ovni prefix. The +building procedure will compile and install the `libsonar-mpi.so`. See the +[Sonar documentation][sonar docs] for more details about the building steps. + +An application can instrument the MPI function calls by linking with the Sonar +library `libsonar-mpi.so`. At run-time, the Sonar library does not enable the +instrumentation by default. Sonar instruments the MPI functions when the +environment variable `SONAR_MPI_INSTRUMENT` is defined to `ovni`. Its default +value is `none`. + +As an example, a user can generate a trace with MPI function events of an MPI +program `app.c` in this way: + +``` +$ mpicc -c app.c -o app.o +$ mpicc app.o -o app -L${SONAR_PREFIX}/lib -lsonar-mpi +$ export SONAR_MPI_INSTRUMENT=ovni +$ mpirun -n 2 ./app +``` + +This will generate an ovni trace in the `ovni` directory, which can be emulated +using the `ovniemu` tool. + +!!! Note + + Notice that the order of libraries at the linking stage is important. The + Sonar library should always have precedence on the MPI library. That's the + usual behavior when using `mpicc` tools. The `mpicc` tool should link the + application with the MPI libraries as the last libraries in the list of + application's dependencies. If this order is not respected, the Sonar + library would not be able to intercept the MPI function calls and instrument + them. + +!!! Note + + Notice the Task-Aware MPI (TAMPI), as well as other external libraries, + intercepts the MPI functions and may call MPI functions instead. Thus, the + order in which such libraries and Sonar are linked to the application will + also alter the resulting ovni trace. Give precedence to the Sonar library to + instrument the MPI function calls made by the application. You can achieve + by linking your application with the linking options `-lsonar-mpi -ltampi`. + Otherwise, give precendence to the TAMPI library to track the real MPI + functions that are being executed (i.e., the ones that the MPI library + actually runs). In this case, use the linking options `-ltampi -lsonar-mpi`. + +## Function view + +The function view attempts to provide a general overview of which are the MPI +functions being executed at any point in time. The function view shows the MPI +functions called by each thread (and for each CPU, the MPI functions executed +by the running thread in that CPU). + +The function states shown in this view are listed below. Each function state +(in bold) includes a list of all the MPI functions that are instrumented as +that particular state. Notice that only the most important functions are +instrumented. Also, notice that not all functions have their own state. For +instance, the large count MPI (with `_c` suffix) introduced in MPI 4.0, the +extended functions (with `v` or `w` suffix), and Fortran functions (with lower +case name and `_` suffix) are instrumented as their simple C function without +suffix. + +- *Setup functions*: The running thread is executing MPI setup functions to + initialize and finalize the MPI environment. The following function states + are shown: + + - **MPI_Init**: `MPI_Init`, `mpi_init_` + + - **MPI_Init_thread**: `MPI_Init_thread`, `mpi_init_thread_` + + - **MPI_Finalize**: `MPI_Finalize`, `mpi_finalize_` + +- *Request functions*: The running thread is executing MPI functions that wait + or test MPI requests after being generated by non-blocking MPI operations. The + following functions are instrumented: + + - **MPI_Wait**: `MPI_Wait`, `mpi_wait_` + + - **MPI_Waitall**: `MPI_Waitall`, `mpi_waitall_` + + - **MPI_Waitany**: `MPI_Waitany`, `mpi_waitany_` + + - **MPI_Waitsome**: `MPI_Waitsome`, `mpi_waitsome_` + + - **MPI_Test**: `MPI_Test`, `mpi_test_` + + - **MPI_Testall**: `MPI_Testall`, `mpi_testall_` + + - **MPI_Testany**: `MPI_Testany`, `mpi_testany_` + + - **MPI_Testsome**: `MPI_Testsome`, `mpi_testsome_` + +- *Point-to-point functions*: The running thread is communicating through MPI + by executing point-to-point primitives. The instrumented functions are: + + - **MPI_Recv**: `MPI_Recv`, `MPI_Recv_c`, `mpi_recv_` + + - **MPI_Send**: `MPI_Send`, `MPI_Send_c`, `mpi_send_` + + - **MPI_Bsend**: `MPI_Bsend`, `MPI_Bsend_c`, `mpi_bsend_` + + - **MPI_Rsend**: `MPI_Rsend`, `MPI_Rsend_c`, `mpi_rsend_` + + - **MPI_Ssend**: `MPI_Ssend`, `MPI_Ssend_c`, `mpi_ssend_` + + - **MPI_Sendrecv**: `MPI_Sendrecv`, `MPI_Sendrecv_c`, `mpi_sendrecv_` + + - **MPI_Sendrecv_replace**: `MPI_Sendrecv_replace`, `MPI_Sendrecv_replace_c`, + `mpi_sendrecv_replace_` + + - **MPI_Irecv**: `MPI_Irecv`, `MPI_Irecv_c`, `mpi_irecv_` + + - **MPI_Isend**: `MPI_Isend`, `MPI_Isend_c`, `mpi_isend_` + + - **MPI_Ibsend**: `MPI_Ibsend`, `MPI_Ibsend_c`, `mpi_ibsend_` + + - **MPI_Irsend**: `MPI_Irsend`, `MPI_Irsend_c`, `mpi_irsend_` + + - **MPI_Issend**: `MPI_Issend`, `MPI_Issend_c`, `mpi_issend_` + + - **MPI_Isendrecv**: `MPI_Isendrecv`, `MPI_Isendrecv_c`, `mpi_isendrecv_` + + - **MPI_Isendrecv_replace**: `MPI_Isendrecv_replace`, + `MPI_Isendrecv_replace_c`, `mpi_isendrecv_replace_` + +- *Collective functions*: The running thread is communicating through MPI by + executing collective functions. The instrumented functions are: + + - **MPI_Gather**: `MPI_Gather`, `MPI_Gatherv`, `MPI_Gather_c`, + `MPI_Gatherv_c`, `mpi_gather_`, `mpi_gatherv_` + + - **MPI_Allgather**: `MPI_Allgather`, `MPI_Allgatherv`, `MPI_Allgather_c`, + `MPI_Allgatherv_c`, `mpi_allgather_`, `mpi_allgatherv_` + + - **MPI_Scatter**: `MPI_Scatter`, `MPI_Scatterv`, `MPI_Scatter_c`, + `MPI_Scatterv_c`, `mpi_scatter_`, `mpi_scatterv_` + + - **MPI_Reduce**: `MPI_Reduce`, `MPI_Reduce_c`, `mpi_reduce_` + + - **MPI_Reduce_scatter**: `MPI_Reduce_scatter`, `MPI_Reduce_scatter_c`, + `mpi_reduce_scatter_` + + - **MPI_Reduce_scatter_block**: `MPI_Reduce_scatter_block`, + `MPI_Reduce_scatter_block_c`, `mpi_reduce_scatter_block_` + + - **MPI_Allreduce**: `MPI_Allreduce`, `MPI_Allreduce_c`, `mpi_allreduce_` + + - **MPI_Barrier**: `MPI_Barrier`, `MPI_Barrier_c`, `mpi_barrier_` + + - **MPI_Bcast**: `MPI_Bcast`, `MPI_Bcast_c`, `mpi_bcast` + + - **MPI_Alltoall**: `MPI_Alltoall`, `MPI_Alltoallv`, `MPI_Alltoallw`, + `MPI_Alltoall_c`, `MPI_Alltoallv_c`, `MPI_Alltoallw_c`, `mpi_alltoall_`, + `mpi_alltoallv_`, `mpi_alltoallw_` + + - **MPI_Scan**: `MPI_Scan`, `MPI_Scan_c`, `mpi_scan_` + + - **MPI_Exscan**: `MPI_Exscan`, `MPI_Exscan_c`, `mpi_exscan_` + + - **MPI_Igather**: `MPI_Igather`, `MPI_Igatherv`, `MPI_Igather_c`, + `MPI_Igatherv_c`, `mpi_igather_`, `mpi_igatherv_` + + - **MPI_Iallgather**: `MPI_Iallgather`, `MPI_Iallgatherv`, + `MPI_Iallgather_c`, `MPI_Iallgatherv_c`, `mpi_iallgather_`, + `mpi_iallgatherv_` + + - **MPI_Iscatter**: `MPI_Iscatter`, `MPI_Iscatterv`, `MPI_Iscatter_c`, + `MPI_Iscatterv_c`, `mpi_iscatter_`, `mpi_iscatterv_` + + - **MPI_Ireduce**: `MPI_Ireduce`, `MPI_Ireduce_c`, `mpi_ireduce_` + + - **MPI_Iallreduce**: `MPI_Iallreduce`, `MPI_Iallreduce_c`, `mpi_iallreduce_` + + - **MPI_Ireduce_scatter**: `MPI_Ireduce_scatter`, `MPI_Ireduce_scatter_c`, + `mpi_ireduce_scatter_` + + - **MPI_Ireduce_scatter_block**: `MPI_Ireduce_scatter_block`, + `MPI_Ireduce_scatter_block_c`, `mpi_ireduce_scatter_block_` + + - **MPI_Ibarrier**: `MPI_Ibarrier`, `MPI_Ibarrier_c`, `mpi_ibarrier_` + + - **MPI_Ibcast**: `MPI_Ibcast`, `MPI_Ibcast_c`, `mpi_ibcast_` + + - **MPI_Ialltoall**: `MPI_Ialltoall`, `MPI_Ialltoallv`, `MPI_Ialltoallw`, + `MPI_Ialltoall_c`, `MPI_Ialltoallv_c`, `MPI_Ialltoallw_c`, + `mpi_ialltoall_`, `mpi_ialltoallv_`, `mpi_ialltoallw_` + + - **MPI_Iscan**: `MPI_Iscan`, `MPI_Iscan_c`, `mpi_iscan_` + + - **MPI_Iexscan**: `MPI_Iexscan`, `MPI_Iexscan_c`, `mpi_iexscan_` + +!!! Note + + The Sonar library does not support large count MPI functions for the Fortran + language yet, and thus, these functions are not instrumented. + +The figure below shows an example of the MPI function view. The program executes +a distributed stencil algorithm with MPI and OmpSs-2. There are several MPI +processes, each running OmpSs-2 tasks on an exclusive set of CPUs. Most of these +are computation tasks, while the others are concurrent tasks performing +communications using the blocking mode of the TAMPI library. These use `MPI_Send` +and `MPI_Recv` functions to send and receive blocks of data. The program was +linked with Sonar and preceding the TAMPI library. Thus, the trace shows the +blocking MPI function calls made by the application. + +![MPI function view example](fig/mpi-function.png) + +The light green areas correspond to the `MPI_Init_thread` calls, the grey ones +are `MPI_Send` calls and the dark green areas are `MPI_Recv` calls. There are +other secondary calls like `MPI_Bcast` (orange), `MPI_Barrier` (blue) and +`MPI_Finalize` (red) calls. + +As mentioned above, the trace shows the blocking MPI functions called by the +application because Sonar was placed before TAMPI in the linking order. However, +these blocking calls may not be actually executed by the MPI library; TAMPI will +transparently replace them with non-blocking calls (e.g., `MPI_Isend` and +`MPI_Irecv`) and a polling mechanism for the generated MPI requests. If you want +to explore the actual MPI functions being executed, you should link the Sonar +library after TAMPI. diff --git a/mkdocs.yml b/mkdocs.yml index f86796d..e09a3e6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -33,6 +33,7 @@ nav: - user/emulation/nosv.md - user/emulation/nanos6.md - user/emulation/tampi.md + - user/emulation/mpi.md - user/emulation/events.md - 'Developer guide': - dev/index.md diff --git a/src/emu/CMakeLists.txt b/src/emu/CMakeLists.txt index 1a6f72a..c252c6c 100644 --- a/src/emu/CMakeLists.txt +++ b/src/emu/CMakeLists.txt @@ -53,6 +53,8 @@ add_library(emu STATIC nosv/event.c nodes/setup.c nodes/event.c + mpi/setup.c + mpi/event.c tampi/setup.c tampi/event.c kernel/setup.c diff --git a/src/emu/emu_prv.h b/src/emu/emu_prv.h index 6e2ac01..4ae4c47 100644 --- a/src/emu/emu_prv.h +++ b/src/emu/emu_prv.h @@ -19,6 +19,7 @@ enum emu_prv_types { PRV_NOSV_SUBSYSTEM = 13, PRV_NOSV_RANK = 14, PRV_TAMPI_SUBSYSTEM = 20, + PRV_MPI_FUNCTION = 25, PRV_NODES_SUBSYSTEM = 30, PRV_NANOS6_TASKID = 35, PRV_NANOS6_TYPE = 36, diff --git a/src/emu/models.c b/src/emu/models.c index 40a35b7..c7788dc 100644 --- a/src/emu/models.c +++ b/src/emu/models.c @@ -12,6 +12,7 @@ extern struct model_spec model_nanos6; extern struct model_spec model_nosv; extern struct model_spec model_nodes; extern struct model_spec model_tampi; +extern struct model_spec model_mpi; extern struct model_spec model_kernel; static struct model_spec *models[] = { @@ -20,6 +21,7 @@ static struct model_spec *models[] = { &model_nosv, &model_nodes, &model_tampi, + &model_mpi, &model_kernel, NULL }; diff --git a/src/emu/mpi/event.c b/src/emu/mpi/event.c new file mode 100644 index 0000000..ba8a4e1 --- /dev/null +++ b/src/emu/mpi/event.c @@ -0,0 +1,201 @@ +/* Copyright (c) 2023 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: GPL-3.0-or-later */ + +#include "mpi_priv.h" +#include "chan.h" +#include "common.h" +#include "emu.h" +#include "emu_ev.h" +#include "extend.h" +#include "model_thread.h" +#include "thread.h" +#include "value.h" + +enum { PUSH = 1, POP = 2, IGN = 3 }; + +static const int fn_table[256][256][3] = { + ['U'] = { + ['i'] = { CH_FUNCTION, PUSH, ST_MPI_INIT }, + ['I'] = { CH_FUNCTION, POP, ST_MPI_INIT }, + ['t'] = { CH_FUNCTION, PUSH, ST_MPI_INIT_THREAD }, + ['T'] = { CH_FUNCTION, POP, ST_MPI_INIT_THREAD }, + ['f'] = { CH_FUNCTION, PUSH, ST_MPI_FINALIZE }, + ['F'] = { CH_FUNCTION, POP, ST_MPI_FINALIZE }, + }, + ['W'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_WAIT }, + [']'] = { CH_FUNCTION, POP, ST_MPI_WAIT }, + ['a'] = { CH_FUNCTION, PUSH, ST_MPI_WAITALL }, + ['A'] = { CH_FUNCTION, POP, ST_MPI_WAITALL }, + ['y'] = { CH_FUNCTION, PUSH, ST_MPI_WAITANY }, + ['Y'] = { CH_FUNCTION, POP, ST_MPI_WAITANY }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_WAITSOME }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_WAITSOME }, + }, + ['T'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_TEST }, + [']'] = { CH_FUNCTION, POP, ST_MPI_TEST }, + ['a'] = { CH_FUNCTION, PUSH, ST_MPI_TESTALL }, + ['A'] = { CH_FUNCTION, POP, ST_MPI_TESTALL }, + ['y'] = { CH_FUNCTION, PUSH, ST_MPI_TESTANY }, + ['Y'] = { CH_FUNCTION, POP, ST_MPI_TESTANY }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_TESTSOME }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_TESTSOME }, + }, + ['R'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_RECV }, + [']'] = { CH_FUNCTION, POP, ST_MPI_RECV }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_SENDRECV }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_SENDRECV }, + ['o'] = { CH_FUNCTION, PUSH, ST_MPI_SENDRECV_REPLACE }, + ['O'] = { CH_FUNCTION, POP, ST_MPI_SENDRECV_REPLACE }, + }, + ['r'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_IRECV }, + [']'] = { CH_FUNCTION, POP, ST_MPI_IRECV }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_ISENDRECV }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_ISENDRECV }, + ['o'] = { CH_FUNCTION, PUSH, ST_MPI_ISENDRECV_REPLACE }, + ['O'] = { CH_FUNCTION, POP, ST_MPI_ISENDRECV_REPLACE }, + }, + ['S'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_SEND }, + [']'] = { CH_FUNCTION, POP, ST_MPI_SEND }, + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_BSEND }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_BSEND }, + ['r'] = { CH_FUNCTION, PUSH, ST_MPI_RSEND }, + ['R'] = { CH_FUNCTION, POP, ST_MPI_RSEND }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_SSEND }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_SSEND }, + }, + ['s'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_ISEND }, + [']'] = { CH_FUNCTION, POP, ST_MPI_ISEND }, + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_IBSEND }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_IBSEND }, + ['r'] = { CH_FUNCTION, PUSH, ST_MPI_IRSEND }, + ['R'] = { CH_FUNCTION, POP, ST_MPI_IRSEND }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_ISSEND }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_ISSEND }, + }, + ['A'] = { + ['g'] = { CH_FUNCTION, PUSH, ST_MPI_ALLGATHER }, + ['G'] = { CH_FUNCTION, POP, ST_MPI_ALLGATHER }, + ['r'] = { CH_FUNCTION, PUSH, ST_MPI_ALLREDUCE }, + ['R'] = { CH_FUNCTION, POP, ST_MPI_ALLREDUCE }, + ['a'] = { CH_FUNCTION, PUSH, ST_MPI_ALLTOALL }, + ['A'] = { CH_FUNCTION, POP, ST_MPI_ALLTOALL }, + }, + ['a'] = { + ['g'] = { CH_FUNCTION, PUSH, ST_MPI_IALLGATHER }, + ['G'] = { CH_FUNCTION, POP, ST_MPI_IALLGATHER }, + ['r'] = { CH_FUNCTION, PUSH, ST_MPI_IALLREDUCE }, + ['R'] = { CH_FUNCTION, POP, ST_MPI_IALLREDUCE }, + ['a'] = { CH_FUNCTION, PUSH, ST_MPI_IALLTOALL }, + ['A'] = { CH_FUNCTION, POP, ST_MPI_IALLTOALL }, + }, + ['C'] = { + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_BARRIER }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_BARRIER }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_SCAN }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_SCAN }, + ['e'] = { CH_FUNCTION, PUSH, ST_MPI_EXSCAN }, + ['E'] = { CH_FUNCTION, POP, ST_MPI_EXSCAN }, + }, + ['c'] = { + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_IBARRIER }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_IBARRIER }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_ISCAN }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_ISCAN }, + ['e'] = { CH_FUNCTION, PUSH, ST_MPI_IEXSCAN }, + ['E'] = { CH_FUNCTION, POP, ST_MPI_IEXSCAN }, + }, + ['D'] = { + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_BCAST }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_BCAST }, + ['g'] = { CH_FUNCTION, PUSH, ST_MPI_GATHER }, + ['G'] = { CH_FUNCTION, POP, ST_MPI_GATHER }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_SCATTER }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_SCATTER }, + }, + ['d'] = { + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_IBCAST }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_IBCAST }, + ['g'] = { CH_FUNCTION, PUSH, ST_MPI_IGATHER }, + ['G'] = { CH_FUNCTION, POP, ST_MPI_IGATHER }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_ISCATTER }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_ISCATTER }, + }, + ['E'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_REDUCE }, + [']'] = { CH_FUNCTION, POP, ST_MPI_REDUCE }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_REDUCE_SCATTER }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_REDUCE_SCATTER }, + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_REDUCE_SCATTER_BLOCK }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_REDUCE_SCATTER_BLOCK }, + }, + ['e'] = { + ['['] = { CH_FUNCTION, PUSH, ST_MPI_IREDUCE }, + [']'] = { CH_FUNCTION, POP, ST_MPI_IREDUCE }, + ['s'] = { CH_FUNCTION, PUSH, ST_MPI_IREDUCE_SCATTER }, + ['S'] = { CH_FUNCTION, POP, ST_MPI_IREDUCE_SCATTER }, + ['b'] = { CH_FUNCTION, PUSH, ST_MPI_IREDUCE_SCATTER_BLOCK }, + ['B'] = { CH_FUNCTION, POP, ST_MPI_IREDUCE_SCATTER_BLOCK }, + }, +}; + +static int +process_ev(struct emu *emu) +{ + if (!emu->thread->is_running) { + err("current thread %d not running", emu->thread->tid); + return -1; + } + + const int *entry = fn_table[emu->ev->c][emu->ev->v]; + int chind = entry[0]; + int action = entry[1]; + int st = entry[2]; + + struct mpi_thread *th = EXT(emu->thread, 'M'); + struct chan *ch = &th->m.ch[chind]; + + if (action == PUSH) { + return chan_push(ch, value_int64(st)); + } else if (action == POP) { + return chan_pop(ch, value_int64(st)); + } else if (action == IGN) { + return 0; /* do nothing */ + } + + err("unknown mpi function event"); + return -1; +} + +int +model_mpi_event(struct emu *emu) +{ + static int enabled = 0; + + if (!enabled) { + if (model_mpi_connect(emu) != 0) { + err("mpi_connect failed"); + return -1; + } + enabled = 1; + } + + dbg("in mpi_event"); + if (emu->ev->m != 'M') { + err("unexpected event model %c", emu->ev->m); + return -1; + } + + dbg("got mpi event %s", emu->ev->mcv); + if (process_ev(emu) != 0) { + err("error processing mpi event"); + return -1; + } + + return 0; +} diff --git a/src/emu/mpi/mpi_priv.h b/src/emu/mpi/mpi_priv.h new file mode 100644 index 0000000..980b7f6 --- /dev/null +++ b/src/emu/mpi/mpi_priv.h @@ -0,0 +1,84 @@ +/* Copyright (c) 2023 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: GPL-3.0-or-later */ + +#ifndef MPI_PRIV_H +#define MPI_PRIV_H + +#include "emu.h" +#include "model_cpu.h" +#include "model_thread.h" + +/* Private enums */ + +enum mpi_chan { + CH_FUNCTION = 0, + CH_MAX, +}; + +enum mpi_function_values { + ST_MPI_INIT = 1, + ST_MPI_INIT_THREAD, + ST_MPI_FINALIZE, + ST_MPI_WAIT, + ST_MPI_WAITALL, + ST_MPI_WAITANY, + ST_MPI_WAITSOME, + ST_MPI_TEST, + ST_MPI_TESTALL, + ST_MPI_TESTANY, + ST_MPI_TESTSOME, + ST_MPI_RECV, + ST_MPI_SEND, + ST_MPI_BSEND, + ST_MPI_RSEND, + ST_MPI_SSEND, + ST_MPI_SENDRECV, + ST_MPI_SENDRECV_REPLACE, + ST_MPI_IRECV, + ST_MPI_ISEND, + ST_MPI_IBSEND, + ST_MPI_IRSEND, + ST_MPI_ISSEND, + ST_MPI_ISENDRECV, + ST_MPI_ISENDRECV_REPLACE, + ST_MPI_ALLGATHER, + ST_MPI_ALLREDUCE, + ST_MPI_ALLTOALL, + ST_MPI_BARRIER, + ST_MPI_BCAST, + ST_MPI_GATHER, + ST_MPI_REDUCE, + ST_MPI_REDUCE_SCATTER, + ST_MPI_REDUCE_SCATTER_BLOCK, + ST_MPI_SCATTER, + ST_MPI_SCAN, + ST_MPI_EXSCAN, + ST_MPI_IALLGATHER, + ST_MPI_IALLREDUCE, + ST_MPI_IALLTOALL, + ST_MPI_IBARRIER, + ST_MPI_IBCAST, + ST_MPI_IGATHER, + ST_MPI_IREDUCE, + ST_MPI_IREDUCE_SCATTER, + ST_MPI_IREDUCE_SCATTER_BLOCK, + ST_MPI_ISCATTER, + ST_MPI_ISCAN, + ST_MPI_IEXSCAN, +}; + +struct mpi_thread { + struct model_thread m; +}; + +struct mpi_cpu { + struct model_cpu m; +}; + +int model_mpi_probe(struct emu *emu); +int model_mpi_create(struct emu *emu); +int model_mpi_connect(struct emu *emu); +int model_mpi_event(struct emu *emu); +int model_mpi_finish(struct emu *emu); + +#endif /* MPI_PRIV_H */ diff --git a/src/emu/mpi/setup.c b/src/emu/mpi/setup.c new file mode 100644 index 0000000..5aa948e --- /dev/null +++ b/src/emu/mpi/setup.c @@ -0,0 +1,252 @@ +/* Copyright (c) 2023 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: GPL-3.0-or-later */ + +#include "mpi_priv.h" +#include +#include "chan.h" +#include "common.h" +#include "emu.h" +#include "emu_args.h" +#include "emu_prv.h" +#include "extend.h" +#include "model.h" +#include "model_chan.h" +#include "model_cpu.h" +#include "model_pvt.h" +#include "model_thread.h" +#include "pv/pcf.h" +#include "pv/prv.h" +#include "system.h" +#include "thread.h" +#include "track.h" +#include "value.h" + +static const char model_name[] = "mpi"; +enum { model_id = 'M' }; + +struct model_spec model_mpi = { + .name = model_name, + .model = model_id, + .create = model_mpi_create, +// .connect = model_mpi_connect, + .event = model_mpi_event, + .probe = model_mpi_probe, + .finish = model_mpi_finish, +}; + +/* ----------------- channels ------------------ */ + +static const char *chan_name[CH_MAX] = { + [CH_FUNCTION] = "function", +}; + +static const int chan_stack[CH_MAX] = { + [CH_FUNCTION] = 1, +}; + +/* ----------------- pvt ------------------ */ + +static const int pvt_type[CH_MAX] = { + [CH_FUNCTION] = PRV_MPI_FUNCTION, +}; + +static const char *pcf_prefix[CH_MAX] = { + [CH_FUNCTION] = "MPI function", +}; + +static const struct pcf_value_label mpi_function_values[] = { + { ST_MPI_INIT, "MPI_Init" }, + { ST_MPI_INIT_THREAD, "MPI_Init_thread" }, + { ST_MPI_FINALIZE, "MPI_Finalize" }, + { ST_MPI_WAIT, "MPI_Wait" }, + { ST_MPI_WAITALL, "MPI_Waitall" }, + { ST_MPI_WAITANY, "MPI_Waitany" }, + { ST_MPI_WAITSOME, "MPI_Waitsome" }, + { ST_MPI_TEST, "MPI_Test" }, + { ST_MPI_TESTALL, "MPI_Testall" }, + { ST_MPI_TESTANY, "MPI_Testany" }, + { ST_MPI_TESTSOME, "MPI_Testsome" }, + { ST_MPI_RECV, "MPI_Recv" }, + { ST_MPI_SEND, "MPI_Send" }, + { ST_MPI_BSEND, "MPI_Bsend" }, + { ST_MPI_RSEND, "MPI_Rsend" }, + { ST_MPI_SSEND, "MPI_Ssend" }, + { ST_MPI_SENDRECV, "MPI_Sendrecv" }, + { ST_MPI_SENDRECV_REPLACE, "MPI_Sendrecv_replace" }, + { ST_MPI_IRECV, "MPI_Irecv" }, + { ST_MPI_ISEND, "MPI_Isend" }, + { ST_MPI_IBSEND, "MPI_Ibsend" }, + { ST_MPI_IRSEND, "MPI_Irsend" }, + { ST_MPI_ISSEND, "MPI_Issend" }, + { ST_MPI_ISENDRECV, "MPI_Isendrecv" }, + { ST_MPI_ISENDRECV_REPLACE, "MPI_Isendrecv_replace" }, + { ST_MPI_ALLGATHER, "MPI_Allgather" }, + { ST_MPI_ALLREDUCE, "MPI_Allreduce" }, + { ST_MPI_ALLTOALL, "MPI_Alltoall" }, + { ST_MPI_BARRIER, "MPI_Barrier" }, + { ST_MPI_BCAST, "MPI_Bcast" }, + { ST_MPI_GATHER, "MPI_Gather" }, + { ST_MPI_REDUCE, "MPI_Reduce" }, + { ST_MPI_REDUCE_SCATTER, "MPI_Reduce_scatter" }, + { ST_MPI_REDUCE_SCATTER_BLOCK, "MPI_Reduce_scatter_block" }, + { ST_MPI_SCATTER, "MPI_Scatter" }, + { ST_MPI_SCAN, "MPI_Scan" }, + { ST_MPI_EXSCAN, "MPI_Exscan" }, + { ST_MPI_IALLGATHER, "MPI_Iallgather" }, + { ST_MPI_IALLREDUCE, "MPI_Iallreduce" }, + { ST_MPI_IALLTOALL, "MPI_Ialltoall" }, + { ST_MPI_IBARRIER, "MPI_Ibarrier" }, + { ST_MPI_IBCAST, "MPI_Ibcast" }, + { ST_MPI_IGATHER, "MPI_Igather" }, + { ST_MPI_IREDUCE, "MPI_Ireduce" }, + { ST_MPI_IREDUCE_SCATTER, "MPI_Ireduce_scatter" }, + { ST_MPI_IREDUCE_SCATTER_BLOCK, "MPI_Ireduce_scatter_block" }, + { ST_MPI_ISCATTER, "MPI_Iscatter" }, + { ST_MPI_ISCAN, "MPI_Iscan" }, + { ST_MPI_IEXSCAN, "MPI_Iexscan" }, + { -1, NULL }, +}; + +static const struct pcf_value_label *pcf_labels[CH_MAX] = { + [CH_FUNCTION] = mpi_function_values, +}; + +static const long prv_flags[CH_MAX] = { + [CH_FUNCTION] = PRV_SKIPDUP, +}; + +static const struct model_pvt_spec pvt_spec = { + .type = pvt_type, + .prefix = pcf_prefix, + .label = pcf_labels, + .flags = prv_flags, +}; + +/* ----------------- tracking ------------------ */ + +static const int th_track[CH_MAX] = { + [CH_FUNCTION] = TRACK_TH_RUN, +}; + +static const int cpu_track[CH_MAX] = { + [CH_FUNCTION] = TRACK_TH_RUN, +}; + +/* ----------------- chan_spec ------------------ */ + +static const struct model_chan_spec th_chan = { + .nch = CH_MAX, + .prefix = model_name, + .ch_names = chan_name, + .ch_stack = chan_stack, + .pvt = &pvt_spec, + .track = th_track, +}; + +static const struct model_chan_spec cpu_chan = { + .nch = CH_MAX, + .prefix = model_name, + .ch_names = chan_name, + .ch_stack = chan_stack, + .pvt = &pvt_spec, + .track = cpu_track, +}; + +/* ----------------- models ------------------ */ + +static const struct model_cpu_spec cpu_spec = { + .size = sizeof(struct mpi_cpu), + .chan = &cpu_chan, + .model = &model_mpi, +}; + +static const struct model_thread_spec th_spec = { + .size = sizeof(struct mpi_thread), + .chan = &th_chan, + .model = &model_mpi, +}; + +/* ----------------------------------------------------- */ + +int +model_mpi_probe(struct emu *emu) +{ + if (emu->system.nthreads == 0) + return 1; + + return 0; +} + +int +model_mpi_create(struct emu *emu) +{ + if (model_thread_create(emu, &th_spec) != 0) { + err("model_thread_init failed"); + return -1; + } + + if (model_cpu_create(emu, &cpu_spec) != 0) { + err("model_cpu_init failed"); + return -1; + } + + return 0; +} + +int +model_mpi_connect(struct emu *emu) +{ + if (model_thread_connect(emu, &th_spec) != 0) { + err("model_thread_connect failed"); + return -1; + } + + if (model_cpu_connect(emu, &cpu_spec) != 0) { + err("model_cpu_connect failed"); + return -1; + } + + return 0; +} + +static int +end_lint(struct emu *emu) +{ + /* Only run the check if we finished the complete trace */ + if (!emu->finished) + return 0; + + struct system *sys = &emu->system; + + /* Ensure we run out of function states */ + for (struct thread *t = sys->threads; t; t = t->gnext) { + struct mpi_thread *th = EXT(t, model_id); + struct chan *ch = &th->m.ch[CH_FUNCTION]; + int stacked = ch->data.stack.n; + if (stacked > 0) { + struct value top; + if (chan_read(ch, &top) != 0) { + err("chan_read failed for function"); + return -1; + } + + err("thread %d ended with %d stacked mpi functions", + t->tid, stacked); + return -1; + } + } + + return 0; +} + +int +model_mpi_finish(struct emu *emu) +{ + /* When running in linter mode perform additional checks */ + if (emu->args.linter_mode && end_lint(emu) != 0) { + err("end_lint failed"); + return -1; + } + + return 0; +}