From 3d954a4606e9fdbd6f58d3de1867ab7a961ae6e2 Mon Sep 17 00:00:00 2001 From: rpotter6298 Date: Fri, 12 Jun 2026 10:36:33 +0200 Subject: [PATCH] Implement GPU precheck and client fail streak management --- v4/distributed/client.py | 40 ++++++++++++++++++++++++++++++++++++ v4/distributed/jobs.db | Bin 110592 -> 110592 bytes v4/distributed/protocol.py | 1 + v4/distributed/server.py | 41 ++++++++++++++++++++++++++++++++++++- 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/v4/distributed/client.py b/v4/distributed/client.py index bf0adc9..ff63e4a 100644 --- a/v4/distributed/client.py +++ b/v4/distributed/client.py @@ -101,6 +101,32 @@ class _Server: # GPU info # ────────────────────────────────────────────────────────────── +def _gpu_precheck() -> tuple[bool, str]: + """Quick allocation + op test on every visible CUDA/HIP device. + + Catches the case where torch.cuda.is_available() returns True but the + device is busy (e.g. another process holding it — Ollama, a forgotten + notebook, an OS-level driver issue). Returns (ok, message); when ok is + False, the caller should defer polling rather than accept a job that will + crash on the first .to(device) call. + """ + try: + import torch + except Exception as e: + return False, f"torch import failed: {e}" + if not torch.cuda.is_available(): + return True, "no-cuda (cpu-only client)" + try: + for i in range(torch.cuda.device_count()): + x = torch.zeros(1024, device=f"cuda:{i}") + _ = (x + 1).sum().item() # forces actual kernel launch + del x + torch.cuda.empty_cache() + return True, "ok" + except Exception as e: + return False, f"{type(e).__name__}: {e}" + + def _gpu_info() -> str: # NVIDIA try: @@ -405,8 +431,22 @@ def main(): ) else 1) print(f"[client] polling every {args.poll_interval}s...", flush=True) + last_precheck_msg = "" while True: try: + ok, msg = _gpu_precheck() + if not ok: + if msg != last_precheck_msg: + print(f"[client] gpu precheck failed ({msg}) — deferring polls", + flush=True) + last_precheck_msg = msg + server.push_status(StatusPush(state="gpu_busy")) + time.sleep(args.poll_interval) + continue + if last_precheck_msg: + print(f"[client] gpu precheck recovered — resuming polls", flush=True) + last_precheck_msg = "" + job = server.poll() if job is None: server.push_status(StatusPush(state="idle")) diff --git a/v4/distributed/jobs.db b/v4/distributed/jobs.db index 6b3caa4adc12d2f8c32071e9797ae1d795a3dd1e..6eed7aa968a314e5d75be0b022934da40b586286 100644 GIT binary patch literal 110592 zcmeI53y>T~d4PBC_U`s}Uh?TJS@KEt>0xQ^dZwr6an{L_rIRIDGT4@}%_GM=dZe?Z zJLw+6Cb;ZX@dywg5J-g-HUx@<5aR@@AdnC+q` zhj;gG)?oj;+F!kW)BVr?O?U4+Q|u)OY4` zxsB|ziha7-r-OY8>?2-zky!YDR!9H&S1Q^sa-Gs+Ic2x@Ir$O!HS!+z1ztb^2mk>f z00e*l5C8%|00?AE;84gUJzp;^ZVx6c5}uy#K^-@6FdA zx#8yF-Xqr^9KN-8)>ggGxW1Qfxq}zkc|%9%+JV*iGqY=(m>wG$^UqMeU1zn%R8?Y= zT6H30=M9DVsysU}IzBNvbr-L)WBw||CbcR>#?Hg-oojcl&ik|Xa&pR_s_bH6{`x9^ zuT@t0w(}*u`L4CASLa`G`IH}=u9W6`)K;zG#QtlJ4Oj5=?kM%9UTnjW(WshvV))h* zz0bVy$npI*-qw5F@NK*2Y%YLQ3RE_~{uw8FufO@&vANrx8IPt<(`3~PZ_AHcZO>-c z%d&O7_vlUR5T72MoSL&s91u(FwNYtfIXKhY#()`Phlx?dPq52|77_dTMg} z-2I$EI^TX?tl%sii$=#z&Qn?(&r10t*Pj@^cKF6#VQ8%aG*_)ZIXQZAj7B3<<1_m` zcav~}vepoduruJ4X>+$_R!q#=cJ3zO_?fe(X>}t`YVIaFow@v+Zo4De}RFpVW6~ z|IL#twg1z8rai9x@61=81uq}~1b_e#00KY&2mk>f00e*l5C8%|;P;EbhGT4hHx4mT z>Stp0W+u9{lJ<(+q0+Z=>Nn+Q%dY&0^!KH2%Rg2eWt)75bXr~`ACdn~JERfi^U7*z ztvV_lRsOK_Q}tHqInoaGP1-5>Mau1}tz98?YPTqd)oZ0UOP=~l^@Y-_l|Pp5mOdiC zN^MteQ|?n9RQr^I^k3>K>EE=w<}AKpyLR+l`VW1dp=>*QQ+_l6WO-yGh2w`+dRXL9EyCC)j<&9 z8J=SygE7<%YFa#lngKNewjY5RVxD0+1o@tC>ml{c$`-`MZsdBp;t2a54)w_N26%=MU=LI5>aI=td4?Z{wu_Lj`!>>f2C=AV8`RN*(A`}rLM`7l zVq5oVxT}&8S;&b@tVh_sl4m$ErPTGQ?%8xF&+sVqT@)JZK#?6h!={Kj1hY0~7~6RU z30TAMBS*)fy^UwMQD|E@nHx2tzo?%lT#niz%HlvL^!zBSWESu`Fhc{HVKz{6owx`=TTF*0x zYlneFsg6v%u2Lkxkr`3MdIMcs$?#m?A*_x7qcxQ}teDx)dZk0{)jT7nwrQH!(WysP zRWi)bi9BrSG;&u~GTg|r+$hpbhxYIcge?c-fOTXGUBWY>fSD4+jJiiw@C+JQKJnd1 zH(Xn*)Zx*P*q*0{W}xy6$8*d$LJ{lgvBEPfwjTy1p6+?RT*+W6ZR|z5V_8zANJPR2 zvA%B*QsNmlF2zgc=f`hfIt_WpP2o6--YUrE1`yQF}<$EEesHVI4n*!!4ts}xCR zrAwt9lC|J>zQwK8E&oUQne;QcO;+U%@;2F&_s+r&Uw{A*00KY&2mk>f00e-*Qm)hhZE^*6X`>V^l&15Fp)lxNMDmk?@y%nCDK zp+tI5B7IdNJ(x&)iL{$YJBhTNNRvd`N~CcjZ6?x2B8?L1fke7Lk=7IG-HG(BMEc4^ zdS@cNBaz;oNN-D|`x5CZ66wnm>8*+MmPEQYk-jXE-keBpN~AAMq&Ftg8xraDiS)We zdTk=TCXrs9NUutyS0>UuiS#9j^om4UOQh9AT1lkkL|RIuOWH?stLKacd;|7a$QXM( zqC1i1*A1?kvvJ)SCG9_Rhtzv>?ELy+?Q`0vwU22JY5&B|z~8Msp#8P>R_#9R&$U0* zUd^r#yjZ(Od!aV1oz?EpV$Ii{tKFjA$gUO~)(&WUHBYm&0qshyPwUk-YHQfF16k|R z+SFgGzfgayKBhjZep~&9`c-x%;q&UhsvlQBqJB{Q2lYMdUjIAPx2gBD>k6+|U!%T4 zeTjPS>^HlTkilCePxO|H%loeTkidpO00;m9AOHk_01yBIKmZ5;fu|&aLVGcPoQq>z zT*t*xE{?2U!^I_Btl&c9Lghl?Lgqr^qQpfv7yR15 z*2SAm8~bx!`yk&)nfY(=?cuLL00;m9AOHk_01yBIKmZ5;0U!VbfI!O$$X6H2`8>Vz zES(rVL&v5j2ZD*w=p+mJ$D#+0O^uF^4UA7rof;pZaXcE1vaB&WHFDk@Il}@qo<)|W zbZ^WR+wRLr_m&P6J3Alm_*&r;?eA&3Z}z+8c7Z}5kjq`@(N^@8wV^WIvay&u&X5$7 zPwFh%>`zuAdQQ-In)QbHTfcqY>JLw|_@H#M|GYJF>Mqucrp8}DC!Vua*TeC#c=Y7f z-Mw2`P}7+&lLOJ{+>##Os&q4Gm&ZQ$kq+>1R{eN4HC28 zwfM6`yAT>g=t7U0h7u>9R$A(QktY)hX$Xp8vquyK6eSHM`NY;zp;%8Q6sr*^j_Y98 z5m4kbl;l%uOND};OenY^D8#iKPe768KuJ8gw^S(RlL^Ia2#V>M1PLgmG!!TC^x{&X z7*8e?qY)_Vkx$R44z%)H@;l&H_=yQz80@~-cs3npKhwqJldel`Jm|@6JZMhisZPPg z2DNGI1@rw{+m2yb)t;67yE|y&L4))gw&`J`+VeIQwZ6%r@)73Y%&<#76kIpE)1!CL zibwCYgw{oM3-NKC$Tx$CB<&a8jBObN;Tijre|HCwH)vV?#PZm4*fW36<=UVB#7z*k z00Kau83O$Nf19?G7ge>_&Lo8LaNx3*}puOV9kbKE<5^v!v|Jt!e*8PCmur)xB8iDZQoC z-~GYvqs1>5WBIFHzwUZ{*RIZwcHY+U!;Y7AY%aXFaJ2m!srOWScia7KIRDA~^Kw7R zy)5@a)pefJ*C6lCyLMS`h(8r zc09XqYx}d>PULT78`SwK-_x)4-91#ksWxLfxzxs@^O@#(P@(FNyO~XDT102{H?NNQ z*|^_Du0yIby2|(HDFVs0MVchwJP8DZpHUY9hfO>-YXo+u2qYI%X_7#8^U@b?1Hucb zkAUS^Vvc_Ko?R&dMq>4tCJAIW34LJ#!YitefZ-y`otd{+zK4YZojndDmdk09Kz1|E z7bYOQw7Lk`uIm}qnT_RpcBTj<*9U5nKz7s17bYOQ#`*}@$TVu>z>XAwsdC7D>`)@ zXUj!wF+kCux29}PEL^YeYq-;*Eu-B-~S-WwwY;MSZvkbQ4 zW%ap>^10rW&B;Xs8?`xW$8DC)4S8(XG1#JoqHSN6vN^d^r6yC-QVlJyL+wn7n-GhRz0P*DQ{A2`SbGag+1+`Zojqd zyKPf#ZTUaRZ_YhXe0$+RxhVaWbg=YD>E6<2HgDkB#a|RNyZ^sD*AT^MxX6Ry;#crk z%QTALpzB(uQTzfNZJ9{SVG$;n=oaFgupRSExyWUop%MppK!ghzxk?NXwhUz({F(Zn}qsztQ!4Vh{Y z4SRj2T11;(m#G%foY!WmMYQBKnQ9UK{OU}#2p@h`scU6!UHfA>%{hL4ro9j<2LeFgX-nWxp{yO<;F)njUp@@3_Ic0J%q`G@lRZsJoUS0Gwcx%00KauwFI8{cejq2N#5~gsm49FEY&#IElV}_(PgQI99foXti#Jv z4Ze0+sxc2OOEt#uvQ&c(KB3!}Y{VDcz5}Fvd~tVCluOR`-_}eix38H}&K1p+vM+C@ z6xrHLDQinJrEqUErOeBkDP?SKrWD%LT$^&QX|9~JzqxYuzUIo2tD7rl?QO0c9%`~u##POgLxW!X3+fpY=FFe*E^W4?yRq4l&W2`7+UuJwN!B%6(puYWNxY`nlIH4W zOB$=1Es0hx?T(P=Ev>xkF0H)dEUmn4FReTw{QQ4q@f|tsO`5Adte#b6k*{8>*W40VV;z9&N+qL*Lw^!6&idLRGf00e*l z5C8%|00;m9AOHk_01yBIKmZ5;0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f00e*l z5C8%|00;m9AOHk_01yBIzZ(P;wi|i%`#JUxUO)f{00AHX1b_e#00KY&2mk>f00f>2 z1TOR(U(r|AhRSry#$xU`GpT4`V)~|K>ck7^ooDI9=ovaTH8~JWj7BHvh(9&OzK)KM z4J3n1F0>-&d8?N3h9$^P@!$f>(nbyMRnpcBv8s_WtSSUh@i>+asI=dFR6FOvh& z=;T>{Dm+D_fh8-i#=B&-v*@7>*(y?fC?4*fITSvBTm?sb!#AuLM{SGGBw09vi zh*0&wvSk@_>)B@w`>bG}b?md2eKhvDgnfG0hyR^_=l}EXE7@lQ`$*OQH?sfvasvDV zFCYK}fB+Bx0zd!=00AHX1b_e#00KbZcZdML|F7lq?Eb&|{ zO2^j=-)jF(+oSoPvOhm{U*&U~wZ8G8@^I~WsOk&AmBlC+uNoDn4 z^uhfYd5(!@F8Y+uZA#f3kdP9`4s_(ZjoOS_X|q^7rj83;xZiYa7aLWZ@48g$d%;lo zcO1ZRG2Pm6-;lE1B>^@po9eMwPye2C7@StyE*1f*%XY&xh--@DzCLBUjr`aRZBMsx zQ^(zIwe4bYqWWwnhUL_b`?{3v#I?h~qEts_Q^%dO+IF#MQhm0Y$TnPY+}EaTcLN-; zo{n_Krp@$stJSuP#hmK09l4%Ih_HQ4+IG+N9m4v1fKfBH<5t@)7MZHgcH$X^C2U`v zvfZ{~?6`iQJG80aIJ4EZi^Z?%vz?9dq&C*AO4%M$+cZt==+q<49QUk!xj8>9Vo|O- zY&ZKo&t*Si;<&F&+ir$Vzq0fG z>QUYh6hyG?%=D5AJuA{sqJW(N1hJ#zCm15?*stu|!g`c90>yQRN2*Ysx+nHv84v&h zKmZ5;0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f00e*l5C8%|00;m9AOHk_01yBI zKmZ5;0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f00e*l5C8%|00;nq-yZ^%_8G~YB>tev0BWWnMWrd-IV%@ct zyo^cKQO*{H(m=#B2E{NfT#a~kq33WKitCy##WdDQU7@cQ-LI^}pDnJu{@5;7KXH+3 zV-fJ?+B6grICf;Pn3n-ra`(wPF503{8irg zRZp0`ncX@)dIx1IRh>m%#sLP?>Kf@GmT4OzcvyM*nzY@7x(FMlZqoWYNX_onc&QiF zHDWglhUQ`sQgV8K%5KBNC=4jpJtMB$SFi0zYaG#w>Kd}!CWa}(Nlx!e+3m)TO(;e> za;?VgZjG;5v%4Yh5@TX}A~g8))hWA!I6z@+JG#wc$JV<>UTsHO<4#^w*U<4sHZnzs z?CHH}yAh3I&o^}=YNCI)#$&A6-9Wfpi-$#bQ1y7vod170r~XFyx%@-v(b6N`|51Fn z>oc7n@Az=x{p}C6eKP;~+?SX0^uINAm8XYN$1@@Z@okSCPtR$*9k<5WtsT#X+*{MK zT_QrkPVY(E?ZwEYj;qtqYuxVE__H;;8wmf)+Ogr-wg`DWeO1bC)AdZxvn(Auiw8VS z`f9ZuX^qQzQC%bBmf^Yvu8muRDZ4}44h)~VdPwRC{hM--TH}@0>~6>pWr6M4Ie+!s zw%{@6JU>*vrRIpD*svnU4RwU;Y3MYLN5fIJzUvsB8aZ!{oN0+ax`5V(4!`L+j);C- zaMKvDWrx^^0v$*7wEOvYC~Nn00j&*TK!!=g)zgBL#=sUb!Jbdp(!QvkYhv^7P}bh+ z0$LlvfE*Kxxc?Ulb{Yd)6~MNAc9udMzmCy={vFENd0jwjLl`VB4P6sO?1zDfYoY}!jlpv)WU%Xhxu??MWl O4K)PJT%+oh|NjTMI_i%A literal 110592 zcmeI53y>Vgd6;+a_U?9f9t%Q(0FqL2B8eA)xts2p?&$#mf*^1pj(8C72LbRnPvC%i zKkz7y;h>tV?$ zh3VegUd+wifit@|OA_qe+8;mqru*;zo1UJYo_-uTv~PGO=ncoGMxB{nBex=#&*vWK z?ak%n(OfQfC;h#h{)+V1MSlhQrl+BgrjzcO@T+?^hBrp`?>rMUC( zz>cE>y~hp=9y&JAJ9uE%z{%cuNA*5(pqCxFfkoMPsH=N*|Em1x{M4pq$A-q7(SRM- zT~9HV6hEYq91qxdsIXX)6H~+EQ^PY?S(06gC&>?KB*_Cd?&<7ay>V6EnV-w)8E2-J z#lqt0)&AZ{toFZ+Pxj_}Rf6v`BjyqC|&2rRHPvPi}2loxs@bqpd_r^Z-z@cHk z9(Z)%`Ek@!6S)*_k1Kc#0l5FKX|uf!#Zf?K|4L{*u{06->{L z%uKIen9n(+i{2odidN};1A7= z&*gjJA>LF#OM^c|pMYboEj*TbF*Wblg@<_KqZ1=R{UAna;UU4))cDlU==3?}&fH#LV33TH4`x=V{~mj_%d>t;*+Wg?)N*gg)C3IkPk4wg1mu?SPfpPFm5hXZl=M z<&|7_KdAhl%HLGJS$T0Thy{y}ARq_`0)l`bAP5Kof`A|(2nYg#fFK|U zywM5VagpZm_A^x6xt|Jk8x`zDey)tD=-F6SKASsUEal33rQa-%mhP3FkbkN8EApe# z|0@4f>1^q~(!SD|e6jrf@-G)liY5Pk>9?i-t`tf?DE_nj<>FtLu9dgQgVKiLxctr1 z%f;U*|AkCSe^C5@a-q0MexJNvSygQARq_`0)oIBjKGS{wRyubkqttu>ARFK~&5 zk#m^^STF<_AV)*kyu<=L-wYiq^fio~*;+v0g|_QJ&9~7E3m_0;gAiNuf?%2jAY$vb zYeCKN@DvLO5y8-gp5}#Qk_BKda6+KFnor;a3($Q)^dQu=z%<5dX#@m#t_?L4k+E7D zG)?SUH0#(LWdS}4A#nnvK?p}!fa#bfJ;u?n>0PV^U~K5nb~Q6_FR%a)L2MeHueoM0 z%mRWiplNzE>%cqD0(4?v=palp=AL5#APjZgq0YIX7qS4TyPg+%NOJ=e)Q$)on_x`y zVH3Z01PpwalGaFwJQjc;a0B4dBP`^y0D@c-=sM9HKXh1tZDS)e0*B_*c$@_oHU==z zq2{?@hy`E=hk@nknrT~)u>c3yzN=fh1|91x3qZba1V$KW*!9k^022@kJHXRyV4h|H zA#xn(QRe{gA7uf7WBHge(SRA8svY6Gp{whjMiV~C0+5Lf=(~>QLhA$z(2?h16ap>4 z=r{{-aNs(I9ng$<$7)Adj)6miX5Ao1SpaZ66k^ZRT-QBPixOG@xb$S}mUWl~7-r}q z1b}Ag{vj6N;=qIiV9oHzK^71|KrnDZ%{0j)EC7WzK=h;0vU-38*e>mZAken88* zv8NV*Ab~+>P=wAywKO2~faO{mhH!W72&|i~Nu9$w8DIglNCmnL0lOM@v4DV<{E#|G z5js0-QRulPgfc~*^I$Cs@-Npjk&;*D8dMaJGl?8cDZPbVJHfD5;< z08A`Ei0^AAbZi!YOxyH{VbJ102n*1CWcfC5HOuiV764sZqXZ<-JVG!F2n<_?7Qq@p z0I>ke45(2^G>ag!mIf`Arek{=P1s-oj!m8OO_!Er7qS3@xF*3aP1LXf3otDbV%J8R z9wMCuAlLA9DQ!Rl-?&*rCDiMIx8KL253)!CcRr)C6&rQF27QK zx%@);&&!`Hf2#cP@<+=bC_h=gS{^Msl~>BnQs2tUOa3JmOS&RGDSeQBpO$`I`aS87 z==U$AmuUX}Mfw-HM{?8enApJn@ zkX88(d7TX9``=8ZLgZBt5CjAPK|l}?1O$N=2uPiG=If(+uSE3A5&cp`pN;4<5q&zM zPet^}h&~a~$0Pb!L?4alBN6>#M86Qxha>v=h<+}jhY>x9=zc`^BDx#VorwN;L?4Ri zk45ye5&cX=KONB@jp(N$`pJlXBBCFU=*J@Z(TIK|q92avha&pHi2g`KKM>LPNA!IW zeQ!j6IHC_m^gR*%p@_aaq7Ov$T@ig}M1L@%?}+FRMD*?B5gkW#6w%FyZbWn#(LqGlBYJ;C?~CYKMBg0IH%0V&Bl^aOz9FKokLc?n`r3$o zPcE1LxfZGp5j6t>?EKK#C1Z4!1?@e`l`AK5$CZ!gDu*lkD|;$CEB9CIidpHe+*?^& z>8;#Zc}L~8id^Zbbf`a8f2e*}{f_##>bKOdtN&B|s`@4Mzo~zueop-_>ZjG;R)0f% zM*UUwm(*WWe@=Z$eUJL2`qS!9sF&19bwoXpbo0L)a~jP6{)(qNnJ;K z9Nwv}R#&QuT2wog|EK&&`JVC*%C~9%!%NCv(l_#7R{l)+W99S8v&v_b-%&n6dn1}O&h6MLC>n2A9q_Av1f6T6uh zU}6^&JDGTpi5*Nlz{GYY?q^~f6I+?MkBKcz*h~;6EG94$hzXMkg9*q4U_xi2pNT#u zG$uAPv5AR$nb^q01}4@sv5twgOx(i+d$Ruco1z?c&|Ky#UtoEy%>DDG$hOF&ARq_` z0)l`bAP5Kof`A|(2nYg#fFSTzB%oNG)A``aL@+fx8jQ_M_q$WW{yDn#A6-OnY-V_T ztp8kXg~O>JoIMwe3||cBLX8u2-jSmGQ?94y`JDP?`R}A}m0v7XJb-_SVVKAX(G)lv| zI0i2=;ks!I*6V`7N`*lu5MW*!)-J#hnP%QJ2K>5Uz-cfT2-AuF9K$_v4AIH;O=Cc> z3kH-5127?|W1u&KufNIeW@)l}ee>Nn&hoqV(|cdhi>#Q?^jS7vm$PgxILr01`!;l4 z%Rug8H$+Hun^%F2zk{OyZbAAM1pDX$JSM2$DLN|e%5|upVuWw2{H*$^IW-M%ajYO9 z2nYg#fFK|U2m*qDARq_`0)l`bkR1WZD^v&bE%!c;PtBYkABw#d*Xs}60nPrU>|D4> z%BAy#>XGI0ZeHKAtJnIoc_-S5a_6(T-u!!WrRPe1@khmy~u2Bn??Q#?Y_2I_~LapFQgl?H^ocm`s_{dV!1ARQ@lK`&E6C*i1%b~ zikCokTMI|BHpH6@hqE@s+YN^@Y@6^(7-ndL z*S;V_8@$@pGql0$+x`q~@QSuCLmT|s*D|!hFZ<039s4hLyqfF&de`3+{-N`qJ6^36 z)fLJbxmS|tE`Q&+rE78F+*_SMb+#`yPBywTOoG8ev980c$AZPp)gL4ijgzI@T>-fpicIFW0reYxZ;tsL-RCF0-$urx!eV~&YCDA?e&{C9uxVEFx@V#QO@Z)w)4ipM|W*X?>A_#-@GlG%x}5}m}ShJ zZ@Q>Y`K?~J&-s6SC8Fa&E8D6;V@5!9&#}ZsV2K{?_FM={8BbIF+ZgFKsSkx-X~v;M z_g!lhN_(!7C7`6bX6QIsfE!SZI21jy_gt$`+B1b_2`KzoHBm)k>!xAVr%+U9VXOj; zZoik9N#mt~-QEh6KR+k(n^v^x z&i_@eu%_~%N}u`%>PhAAmFvoC`Pbzg($}PmQm*{rvRV3K>3pf9_>m&+`BKk?oX_*0%6~ZDpZh}oJqep0%hOvup3vG`@UZ21FBKgm zv&8*{*RxmJW|7x$m)mBsQMa_sBEPC<+h&pXZ_Z@hdCp;;q*vwOvRs*MW5hcjrrTzb zKlV?x%_4u`o@|>%-fW*}n?>Fx9uErD{jI-CpQ|ND$KBzPU~caC+|2R%GurMzN5+jFgFd-to|pDpYu5}6YOZcPHa3)M>UokNxw z{sGKoNk%t{+jBjXb91|y=o^dY~KfI_GE7juVmJ%85y z>8@WXysz_0hnGJ<1DgI+uWqZXy}YfuzcJs=CLVU7XKR7y-gG9#5<@7nn>n?V!Q}Jm ztUd@4((BVMudm!1pF-w_CO`x@n(M-wIv-tQZHdRM_Dr5y%6J+mz=oz-pM!gS<$ZA| zq34?zn!e_MuvIASnOU_26h4k7!Mq!*4`84pz8THREpaFoBA!k>qJh>+N_(bVEdhm( z4NL|F62olFz_jB~FtGq3zOR|kX|<%ZXHM1`!7g4yV3 z=C$8z%q6qMELS%Hjg~;Zy79tx*TZjygv=A8_ZXvRM2z8DpDPdE= zG@VS2=5}qd4>l8vJkm@o>p(NH@cw3Ep?%H7GWRwU%Xqk%Sa7hJSYS^xvGj-R&dW*4 z#=^A*b~hW73^W_l+SP1KytCPu=)q=Vnmd|}X*|$uOt`(-nBe|qW9r-J3XtU1Y_ewS z)v`1|?JLWY47t24$yk?`B^f-sEXkPD%aV*SwJgcN$z@3fOe{+>`uJ;(D2P7(+*nm{ zthsn(w7GcJNOSS<#pdFn3(dtdhntIMoNq23Jl9-25H=T24_bTBvHff-Q<1tLAP5Ko zf`A|(2nYg#fFK|U2m*qDARq|55eTqdg5FkqCRh1Lg{WVl@9Jgc8D*FJlKhkM9nxo{ z{`PZ^epY!G5irwdR;*Lav3_Rj@F&ddy5 z@?m&W&xDI3vJhdns1|4OGy97>1?rZ*^0^TL}i2ALpV@RHf*jX2ZO!F1mxbLjk4x{A}x_~l^g z^j(_fjgN)H=kD6vdl$|2+<&I~{o&~eXU01p`2DS(5&niO85Bfx)L0>NA`S(6ffEAV z)qHaE^_deu0k0<%{z@+y6o_rRu@v!m9E$D(q6eX_1*XyJ$;x{5?x`T0r47o(SMuUx z4H7^hY}ru4Z^_l!vC7)%ZPmfXD&{C4z;kV=nJD?yb7O{nTZ+aFbIIFh)NgF)R(%EY z>g;ID?~vA{*tKXwK1K7ak&*VaQg4_`<+q{hpuWm-b#^4?w~sL z^oF@KeiNHC7E-Q1*s)EwR?*)(=x-(ct)aiw^e5Bb3i^8+{oPU5Svb>gr@uStPpbc` z(tp|U>?b~gfFK|U2m*qDARq_`0)l`bAP5Kof`A|(2nYg#fFK|U2m*qDARq_`0)l`b zAP5Kof`A|(2nYg#fFK|U2m*qDARq_`0)l`bAP5Kof`A|(2nYg#fFK|U2m*qDARq_` z0)l`bAP5Kof`A|(2nYg#fFK|U2m*qDARq_`0)l`bAP5Kof`A|pA;8xEujD&&67B(pde(HmqwDR3m7Oa(O7zdS@aKBr3|;;-zpZ-W#!{cYZv;jdXxQ}< zEPuNA(xX`~2D^mSGz&o*2qA+njaxV!$6x|tVF!4c4U#O(zIcYRUjB6ntEn(R-2#05 z(`%1b);_VVdZbYpLgYBmLs$dAPg@vVy?-j0(6ipZU>*Z*GYtVqFyX6dSFfFl5eOX1 z#|TmaW|9@g7t2l7%fK#bGZg^@pi$qWpnC0Oj6mqSp{whjrdxWGWgzPXVHYEiVnYJb z2LU#5eMR-^wG%M{$ixQpT}N|alI=Ve%T3nX5G-mlO&I{hz$RZ0_}cLp0UdcBMj_Aw zoMiKX#d4GNo&$^8Ohv%70pEY2dhJ+@fP(|qG3-Dy?bNr$daac81_O%`NU^T~=>x#T zR(-RD>b0XW0=nfGI5eQ<8A;XvUo1CS?9+?gY%6gzw)WiCed+yOQP!JBQc)?2nYsFsF`MJpR;cM&GVV0@|kWU(`fZf z9*FsjLL1O$B&5;psS?z7^Rkh3gKxgEG(HoIEm+_DWA6Dsms5YF{FD55(znVlmcCYe zuIEp>ztr_V3NLm3?~Z@W|1f_GKmAu{FUIzLFdfsRJ6<>%Hj}h+=C&VcY{ZatBR6g? z`FSUO^dZk^T;H=7;(lYg4G6Sd%?y&<*<%^Wx@j9Xm&$K|Z`9?p!!f@-`ebVwp0Bxy zo^E3q$+|%sH3j)Wcw|csPk)5Q$ zHt(_N5*!+Bf7j&=tEn&;boWrcms!D!W1zb?`7Yg!RwH4O^S?NTtQ)XPSWSh2UIm12 z6IpQM7!U+*09;xctmZLf-KJd}L#l2}gv~}}aJ}HfF%aaMfP&H-KTOb~UpzxuH}aOS znx-@$L#GdO^-V(ykH<0CbkDcY2prnjOR~Y=;u*@i&9{WrG#Kdp5L$ej(85q0gJEL; z1D*EgxuEINkaZJqaSW+?HR+axH#P<>JQl}*9UKOhr_+wz)LmMQIy>tY;o=xlH4II< z=Q84ZE*H+mF*v~XUHXPcgHH1pvTh(Qjv-Y$k?s}@qtE{q#O^nhjlR44bx`dVz4zIAtjrNKmf>+`xlvaXQl8%tgRS$)KS zgx{5`llx*m8_>1gfIf4(PLd|tyliA$`_4C(#%Ig6kVWS1J=Mv*F`ok~v<-*onro-- zQ_H%lo#%6k8rJHg9Sjigw851L*bP zBh|^ln9qoIGWsSa8uaW0_ttsY$hs1pZ!AsOrq5h}?#Qay#KKw^nm4B8cwt~@aVWZJQC1XKtq>cTb`;H8cb8h5q&>^`Ma;H12NxStedV$ z-+y8~$vta9R3S;_kOd7;5nzUwA1 z_Eo r1~4#c)OGjAde`)`%cTMcbNZwhK9pQ diff --git a/v4/distributed/protocol.py b/v4/distributed/protocol.py index c69e775..ea7ca5c 100644 --- a/v4/distributed/protocol.py +++ b/v4/distributed/protocol.py @@ -32,6 +32,7 @@ class ClientInfo(BaseModel): gpu_info: str status: StatusPush last_seen: str + fail_streak: int = 0 # consecutive job failures; resets on success or register class JobSpec(BaseModel): diff --git a/v4/distributed/server.py b/v4/distributed/server.py index 7859f8b..955e316 100644 --- a/v4/distributed/server.py +++ b/v4/distributed/server.py @@ -47,7 +47,11 @@ _TOKEN: str = "" _DB_PATH: Path = Path("v4/distributed/jobs.db") _REPO_ROOT: Path = Path.cwd() _CLIENT_TTL: int = 120 # seconds before a client is considered gone -_MAX_ATTEMPTS: int = 3 # max times a job is retried before being left as failed +_MAX_ATTEMPTS: int = 4 # max times a job is retried (cumulatively, across clients) + # before being marked permanently failed +_CLIENT_FAIL_LIMIT: int = 3 # consecutive job failures from a single client before + # that client is quarantined (no more poll dispatches + # until it re-registers). A successful job resets it. _SERVER_START_TS: float = 0.0 # set in main(); used as a reaper grace window _clients: dict[str, ClientInfo] = {} @@ -222,6 +226,19 @@ def poll(client_id: str): with _clients_lock: needs_reregister = _ensure_client(client_id) _clients[client_id].last_seen = _now() + # Quarantine: a client that has failed _CLIENT_FAIL_LIMIT jobs in a row + # is cut off from new dispatches until it re-registers. The client's + # poll() helper auto-calls _reregister() when please_reregister=True + # arrives, which clears the streak. Successful completions also reset + # the streak, so a healthy client never trips this check. + if _clients[client_id].fail_streak >= _CLIENT_FAIL_LIMIT: + print( + f"[server] {client_id} quarantined " + f"(fail_streak={_clients[client_id].fail_streak}); " + f"requesting re-register before next dispatch", + flush=True, + ) + return PollResponse(job=None, please_reregister=True) with _db() as conn: row = conn.execute( @@ -296,6 +313,28 @@ def get_client(client_id: str): @app.post("/complete", dependencies=[Depends(_check_token)]) def complete(result: JobResult): + # Pull the worker that ran this job before touching the row, so we can + # update its streak regardless of which branch we take below. + with _db() as conn: + assigned_row = conn.execute( + "SELECT assigned_to FROM jobs WHERE job_id=?", (result.job_id,) + ).fetchone() + assigned_to = assigned_row["assigned_to"] if assigned_row else None + + with _clients_lock: + if assigned_to and assigned_to in _clients: + if result.success: + _clients[assigned_to].fail_streak = 0 + else: + _clients[assigned_to].fail_streak += 1 + if _clients[assigned_to].fail_streak >= _CLIENT_FAIL_LIMIT: + print( + f"[server] {assigned_to} hit fail_streak=" + f"{_clients[assigned_to].fail_streak}; will quarantine " + f"on next poll", + flush=True, + ) + with _db() as conn: if result.success: conn.execute(