From 3fc4c096a592fbc0af3a8d60d2026ed673126a2f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 17 Sep 2018 18:26:22 +0900 Subject: [PATCH 001/118] Squashed commit of the following: commit 9d9594ba20097dc4598f7eb42a9f9d78d73eae54 Author: Shinya Maeda Date: Thu Sep 13 20:18:31 2018 +0900 Cancel scheduled jobs commit f31c7172e07a9eb03b58c1e62eaa18cda4064aa6 Author: Shinya Maeda Date: Thu Sep 13 11:18:42 2018 +0900 Add Ci::BuildSchedule commit fb6b3ca638f40f9e1ee38b1fdd892bda4f6fede7 Author: Shinya Maeda Date: Wed Sep 12 20:02:50 2018 +0900 Scheduled jobs --- .../favicon_status_manual_with_auto_play.ico | Bin 0 -> 179677 bytes .../favicon_status_manual_with_auto_play.png | Bin 0 -> 1338 bytes app/models/ci/build.rb | 21 +++++++ app/models/ci/build_schedule.rb | 25 +++++++++ app/models/concerns/has_status.rb | 3 +- .../_icon_status_manual_with_auto_play.svg | 1 + ...tatus_manual_with_auto_play_borderless.svg | 1 + app/workers/all_queues.yml | 1 + app/workers/build_finished_worker.rb | 1 + app/workers/ci/build_schedule_worker.rb | 16 ++++++ .../20180913102839_create_build_schedules.rb | 19 +++++++ db/schema.rb | 8 +++ lib/gitlab/ci/config/entry/job.rb | 12 +++- lib/gitlab/ci/status/build/factory.rb | 1 + .../ci/status/build/manual_with_auto_play.rb | 52 ++++++++++++++++++ lib/gitlab/ci/yaml_processor.rb | 3 +- 16 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 app/assets/images/ci_favicons/canary/favicon_status_manual_with_auto_play.ico create mode 100644 app/assets/images/ci_favicons/favicon_status_manual_with_auto_play.png create mode 100644 app/models/ci/build_schedule.rb create mode 100644 app/views/shared/icons/_icon_status_manual_with_auto_play.svg create mode 100644 app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg create mode 100644 app/workers/ci/build_schedule_worker.rb create mode 100644 db/migrate/20180913102839_create_build_schedules.rb create mode 100644 lib/gitlab/ci/status/build/manual_with_auto_play.rb diff --git a/app/assets/images/ci_favicons/canary/favicon_status_manual_with_auto_play.ico b/app/assets/images/ci_favicons/canary/favicon_status_manual_with_auto_play.ico new file mode 100644 index 0000000000000000000000000000000000000000..d8528e5d0e47052a62069192005ab46e5976956d GIT binary patch literal 179677 zcmXtg1y~f{_x|iI%_2)9BAp^7NQlzi-Hmih_ky$_h%`!fqaqDUiGp-VBZz=>H~Swx zzwe*t*=HGM?wNbeea|`XIdca9fB?9_-46ty1EORAAQk)_iTuA~HXI1>P!<5p&HwMX zG#LP7R3U)8{Qn(CYXiW}3Iq@r|KIhJ0N^DE1$cP=cRe!zeDs3?baekaMu6jUpa3Q1 z|Hgy2-l>5C2=MQB_qW3U$2UQN7HG>%RLy`dmw20-yv1X$fuL4?7nb@c+oF;w(S<)M!k~*oXsR zMnx(q(+Sh#K0f#N3o_$gW~hCX&+r)UO@Ds%o85}<5A!9M!sq32zT;DRHL<;6<4$8j zBmoLT*2OIq|85FeY%dpIc>Mj4En3o%qc-fRH?1tTU^1+w_CjY;MLp4x#B)k70Y}Jw_#I6?lcbH5ch{6GVCSI3$?8kC^k$e9iN&7;~q0g0z(L?smzQK z4yEg9&I{MaAIHIoY4084VlPPGEW(&(F8p$+AktCWUWD+Kg4W0{Gbk{Rm-1c!7vUw%!NdrXOi z2jl5_{wjNF5c917{AUpcD^DyVr9_MdvMe=8D+2i_bp>AvDL+8wle`GSHBpgwLMBS} zhu=uoWzr-8+V#WTq=;f`>PGrbOcZmu@v=SrL2a=mS&{GIxzTYuy`u355CNft2yl(t zyn0I#fntz#e_W8>!h4Q_xTisg*-ra&o}E?31jnWg#J&qx4P`lC}E`WQ;xp$ zjyqmzSs%|`;?P|0_Wx}3kQ1*Hou4Tzm0ywWq(d!q)w>T6$|gt;6oT>(R82Fsg|fGt z#C6=vVa=XYpA;)M0o?AO6gEnk5HZR*euFk*S8%qf>h75pcD}hyQFxfdK#`;X+`5)P3mHG zaZW9sh%mOE^35ib@fJtFoRz*!w_I7Z(GoGuo&d-b7rsTMMAJPa+gDz7wnFw!6-WVt zIUWk(n~J!O)0lT(*^zOT&KkJor@!ch%cM4#$|H*DVa6gJi$lZp3}x$7ZYyMwft+4S+^7~*llI!@^9pj zRh$+jlGN#$o>n7Dp*Ti~EOv(Kb$)7Di3vB{f*^rWqNs-5#h74kd{^B%q0y);0LxHD zzent42P^wB=J)g<`pq$Gy3)zEjkgq(WRQ>D{Fnk|c-7YMtpv~_;~$3fi+KQEP&~&2 zmcO{6O8k)qk1!|h;ptjVAt|IJGoj+oid^M?-+W_Wc#-_Oxx(H!f^BtKN!}T;5(feD z)zv4d=z1%tzvUw7fsa;vMUARj?n?&^8!4no2?}^HL|3s#{k`rU1IcC0_}@4xjOqia zS+4b4GF8rd8rO(3QN_(wQc;bWf;cJ3_Tm`zYbNHil0HnFHar?jcvmJaR}GR)_XCor z!p~eA!_!1^-8WOyxPtwB?V#b`Ya-&e26(V#%kid6hxBxAjP`t9YJ;)LU6FExJQtm( zqpY%~bB^NXdj>SUy?qOekG`ChjvrDMfMFRgk*+6>Qz|P?S^tepTc|vh-Zxg$xQe~; z?A+*0HZ01ULpGz%rOrDkxlz&TJXh}E<7r!5>5deE#Cye@@)quJ^0-SSCB6a}Nyv0A zZ*&|%V}rsA)nz`61d_0U_4%S2wZvd$p1%aB$fDWb#W5Ayv3!b;N@ssKRJV`#<*!6d zp1xBQBW+j;`dRugyO4Zoc7G)JTsJmUax)7?ze!{&QAj_c-)7H)(s)V-54|G8Oba>p zzR94pmwRC~1%RuSrCGM3gxAfg1YgK}IR9Q!as!HZVmu0)qpuIbD^i;qVzr3L3;DoH zWSC_~+9iv{a_wBHw>-Q`2~`w-iYHlRo+rwtzP7<99Bj<>J9DPtpGdx{Gyk|vk=^-> zXLa#EmQ;zAe&X(*nymG*+X_kt%&y7@-40i^?ICKK(g9-HGY@o*iJW1DKla9z&mI0H zqVGM$dh&c>kBsR)wCk}HAZe~ETn>Zkyw#I%lTqk&w&E1!DTd#f!=gB5&mPX2qKM&; z5z3<<>nc!oKKSTHowVI#&#>I3D;X~Da=J8FPCOC#SmE`Lg{v6FY>m14g$q-e9uYZNMWKGmLMHa*t8391;qQC( z|NaUX9h6+UfBr_-#bNgyaUKWNVJ}teRKaaqlJ4y>#WgN@)EInPbt&GJ@h3^c9uRmu zdXHDgTgd@@nqEgS=6BwU&gOF|L;hJJtNJuv{$&+uyv zjg3($4^s;QVRV79>Csop2W*i$(lVkN2JniLzp7pA$s*>IX1r&|8dEM+ zY zN`v`Aqwy8C8!*3=mn(}j&AU6ZB+`2jyRHG5x=WL;snjHaf)iO2nloxhe92n zV(JaIbLI?Ff43$I^vw`x((EPd$MYXx=IGy3UUX0HAgd34>E_B-AH-F2j+0cJk@$@x z8jt*5?}OTSBwy3_4MB}EMQ!DI6RzKUet{YeyRoa?17BcX4+jSDY*Iw-ClkkeUOXs2 zF{(U|7;8W>S&c|aCH6nSu_sPHX+-@^ij4RSW_DFC7iY=p*uTxDJCwj#GTF~<%94q1 z2i<;$F5v??%~Lf^SM0_&`Cn;;GY{|>GO^t2LrOKG?TJxObIeC#@mulCM?bWjuu=)N zDF09)GMTOzWJHB{m!=#4(-minzt>dM5(1vm}dLmhHr zBw!-U`v<~4TW3$*Y%<=x$#4FucSXtAm{%bE&HkZot(vy4b4dCjVbhe_{do>jF<7QO zRX9I><^h&uJPdHa#y*jF+gUz8z|8ghV)vAmf8tced%<#t0n2z?gJn%K4a#0g-{!t^ zvrpe1v6Api7KOlwg!lDe>Su(ihdFe_bjDV?Eh~BKj&)3*$HY@Vc7&|xH|Sq|l5|tC z&2T8or^bA9c>4G7yo z51H|*-WzE7E^z;@b)V;j>ZeZ6>{ZS!C7uo@0nsOq*jd$_AJWfN7uE)0?hv}qu3Lym zwLz4wV!mV(uT32H=yZ$7l(nmhe-MHiUwJ{en7*iD$(z?QaM4yOLY5t?gj=RYbZJnt zraJ?VIt8EghufLJTOR?YH5XN`D^)v0pIkyyW-UqgghV5rN(S=Nj*KlKV9ut-Bttts`$zO&SgZT*8)~ z;(IK0nM5r&iVpfgqipMfu~^Ib&d1_hQ_hu{-mq(jDcu7wcuog&MW=zcen(!pxpor~UPwbdxse&^Guf zo@_Xq)CiPu`QTn)8^#bHc0~~N4Ox>Ou%4ZC5Z$T#<~kE}Q3m&rlE@!x?k5e0;g33$ zMpsjowhpgywn-H7uC3lAFCsgPM#gn0tua3A@i3Zqd@TKxW+mRIs^IpG{r=FyT!V+c ztC~CF^7B^lI}MiaA`U6yDA@<3YT+_)K0@{a3v2xiT|%j{tURgSm-6$U-OIeqIaJH; zKj8%)q&d8=c$Vr@nr^UbL)R0Wh>h4i71_){)y21>*+1mj1`e{V=dQ-(DICvScGXAco(`Rm(i&y{oxSd9*s)}q zy$ZKLGo$ZKJuQm}dG`GHMF&lbzzvTh?`(qXD%3s;#>)#?-;}%w5XK^yH<@y~jcG{O zejgzlNSA1_!eNcsW*v`Y189FX9m0@*HqcuIzd|-czMLl*shurlhqtfON_t&|*m5cc zNFZD=0yo2RPj3NvsN_rMEP3%~)}yAZwTC1t5Ba5;P!-gbuk$L2LAEQqMyN}j6!yhppKfZREvi!ozKA_w^ z2vN)Meu`aq^2@*eO`b7BE=F4Q=49%Z-SDLQb?=Z{l!f2r9NZBshg)6KJkh?Z{|{6OkVmOkB!E>%;HYimdFyn zFGw~%)cOQm!z^+th#q-B85$SjFxQ63L$7JUFXxm=|s7#rEeTs2``GrJm z%utPZoXd+jWnt}L0O2oYdezI!7_!Xr>r9GPt}pcSC-QGRJ|L?D`yIK6Sb zOW5)OeUG1%kjVl+WC7x39Z1!iyh0=*(Y3~`l4fl7{lOC_Ec0&Ss&RgIErYtbo8nrV zvkNt9iM6vv6=AL5DCdrIR-865M_%I1Uub#d%$v>1Ub~P;JB;URg65>gvPdPI@weD) z>6Yn@_N&BIpCfEnEPla}F8yx?@G?yP!>AQw^0gUq>C!6AD(@-P7O(xMdsf!DhC-)My)wNgdO##>=T^LDspJ~c zl{BY(0Rg&trbNUX2MJs~SX%d-JkdKpz?lpSn;}9 zoxM@PzM0j(cxZYxnPzdi*KZgnt*Esa!|u_Z4$7miBB~wmv{N6s-S2~a@X~P4gaVKz z-y6%5THP4JFK%r6kskayf^Lo_iuX9Zp2%wf)bFZ8JX=CM*~I|)ARus=GfwRKxKc); z@aSQm`Pp9T>$8rGdbSJt$_NVHm%nBxD7m&T2-Et^)i_>0g24@{DTV%!?CI>D&wMui zB6*zZ2Q;&%*~fh-vs?NTJ={|=m(trd;3#XV%?}W3AUWF4BG5@wg&{DNx7-^ zF*0MY<+~m5?5IM&eOG9vPpelvPNhT=te0v<_oOgms*6+^1iJ)gt9!PH>6D7q$UC)d z9VO0xH%yfN_VAEM`9r;sBd;tgll61d8J`B3y2yP+JRpWcFuOq2K0=6-7f5;3uC4i9 zx$3o~8wAI7ht}>$h~w56a3I}BKb394%76Kf=5t-E(ph4l%Pdx z+f&eQG`^{I2ipcSg0f_!BqV&v#8kAzDRQYoi;39aImSyBG@{z?Kw)qay%f^1s?@Tg z>)qwZE0KsTvDp+_K)RlGE_I!8tbNZtN4gIpXU!?wF7J$yX@@62fTM2(lW%ZYDbI@) zVS?3OI-M}ZzvEL>T^^>++@c*|QxA@Mp|&Q zk981T@%^|-{K@x`3=ZR#ym~&5XkkxH`-ulo#fc$wygapV^mo z^Libkezpr3Z1RF?iVx9hj_GcM%V#~+NpwWKDW zOyG{!t#|AW8YW*8oobGuvZA6LUoeGm#!s9oaOEXahSO_2J=j&F?sRFtNa%9XZY_Qe zET9ai808$8Emb^m@4x>e7CL*%rn@2guK2C9k4gHQACe>>OwZ_PhYaIJ)v<6 zIPK%dw}J?n0mHb!--O$k0%0*Ou1#NArTfCUa+BHkvE*Zv1RuKuvgmbVi3{?oX?-Uu zm!*q{<56=7ZNuA%f%UMTQ-TXRt#{3LjxxGi4*P#CJZpafJuSYH@mdRert@4Y(jHjh z%G2jLU}Q?ZEzL}yF0~`LkBlO(h)ryiwnF9?Ro*z-u2z;fT_w=}Y!lhP4AOhaDf_7S z!zyC&*M1f^!IONfLbvE)! zlfq*Ep@2s)I*e31Qj!VcP=09>0clG9G~;?X+8?tQ6Gz{pcX->kZ*REk&JrBEs-27^ z*t+99TrkA>I&7MZ5bZltL}ymr%cefn9eYMpP=^@tf^?LyW}D6PQQ-MRMY-e(Gyv+{Qk1JUsaH_U5k%uJ!yxW$$-pi)V=} zuo~KG$k~)z16&|e>DTO{YejqYNt!${)th@ZCn^Y806c>9cgMo)1wyhuVqI?bw zCcc>3p_pHLm9C(d-O5+s_pD=vW6Xb~?n+cwBz*+)gykp_(@)(gq-sOzLFPc)X&)s$ zc^ljrI}2jK;nN8Aki=Pw+8uV2{$Tn(xojDV8K(Qi=@|iyv)qmBb=cM(*N3S;b|S4v`G&SC%CG5dQXp!vW2*5fBXXc5UYX12uhkE|`gWU*^dwPIV^u^?Czri=-u19fPS1pbSUlZYE&;Y>=ky1UgCGhjz-czHy@CHXwL2- zl(Uc2Ko?b_8v7Uy@k~-um(ArtDt(~!nU{yv+T{5sO1f1_hWdA4gmKA`& z=72wD#{0vJ=6N&SommOj7%*;EFi=UCPK9Rof~r__K7_}WMmu(jYe}*bmBOvLcm)4( zGZG*X`jwgkH&Kqkip}gDG=8sV@E(i`C0Wfq%o*r=nb&cJ9T&0}2CgiHg31Qs=FsWV z%6)#!Bn4_rclcB-56gUT=AUxJvYjljE@90z%n-=*{>v+wc!!^XeY)FSW6q;X z+e6YOWcFRpiBewp?obe@Tnxo2uwJ_+pNTPm(WDy@7MkY{(c=@xd5kCW5(PZ}BEMsU z(f5S)l6H2OcJa)N=gIbBpx~)!1DNxP>6JUEelDNB+UN{k+aLcxFZpk9YP7rs=8#DJR3nJt3^Sx(1xC*ca^*9472^C>#8} z^4o$NVJKFvgE?0Lpn-#tcJd1fDGcIKgNHS)Aa2j49biyrP@HB`HrwY0QOH5;#lqgdJ7(1X7$%OZx{vO$0;WC`o!&j`gU~TaVxs{9 zIW(+`>W{%3MVGLi&?$ZDU5cS)K~Qjcnfpp@VNv91qYfsp0k%pxnM9~>F} zJXRyQ5<8^l``W4SvLWF)BhgiCk+srN&eZD8;92eheYp9zqz4kEcRKWjMSGnIgk5ww zMe$cfUGT&b(>)-Y0k^te~t>&*BvCMBIy`D4vaWiyOJHMpT!ZGqZv+1jj zLc?CyT+@Rf$TP&Y`Jkmt?vOgt1i5`q&mImW196Gnw1$OKrW@*fKyhLeUo6)*^AmJS z0h+g%S^I4lBJO#Tm69q8e!SQVj+C#c1C-JS-dvA^Bto0ojQ0=a0koF~J+4YnKk)%V<=pYU% zgw-c~Q+I$#CPI#sb|@fnYw?^zDd%514;=9t*d0O`N>~5?UI01KH?M{*K-`FAjz5K3 z4&s8tCKQT*+Tgrd8C>CTL^#%cXrD&|`bGp_DpA-KeX4qVa~63=0eHyxQg4kk0t%U> zr}I|;rkCBLu8QC32|2Y}fH@%Dm_`}*8JjvSKsrg7VUrsczMf*<`{Zz7N`&ht7_6#& zXx;>&Z?AvW=*@A7A4+*&Z*k49x)s{9KWEUDFI#@mN&W7uXLhZ;6et%l z1rvsJ@~%R9@YG!;%CYhn+PleJ&w?51L5(#*fXa*0*TQ~48%FxsB$0ir)O_wE1-KHA zPK8gOd$w^L=y+pFjR`x(iZP}R!M;N1FtCe)Qq2Cr$iL-s{<`XW`s=2}>@FIFt8>uF zr&P+=i)BtUmFCQJqFy7LAwo!EBzvJiBHej@=<|R|7mi0HwDea&Q3+In~(x0p! z8N3-y^m@p!i-)wAtHc=q0WC~ZI9d0%VFv^7&5qJ)c(HP>c3-O$tw zE7KolBvhH_eoL5vu@%B z$3UDRM&Hz&7(TnqR1(vZ-)%=cx6$LD$qRPu%UMy?tIi*DROxx+iIL^Ja*9-eiC=2; zXD#ph9w@Y4$b$~TdCOHgn{n~fF$n`a+j4r-zn0`bLY;{)jA2 zsHW!JL6ikwn1+jzT(O&eD2U30f+0OTu?|ZfnwM}vjx!s(;=a7o zk`8mVnB#5_e-85FH@PZe{6~&c-3B=Xbp+rnf616jTY0p#GUM}#S_#y2}cav- zIPmboR~6mfXM!B+xG)F#K`N5bmDy86-jnzju6v$Tc;!{c%~tQ&>A+%%p2ZpMu1t#* zPVcRf>lYUM`cAKbmlEHb(Q_R%pAaR?+@lJ(>Z|e90 zJx`T+WxlE?n7+EYF#2PsN?`+GqntYk)SWJ>vVzhu-8bp@t;qC7&&yH)DztR1bq zC%biqWMIB5fMcrHjC4Qp4l&fYSjYET@!UQet+@*)rs$K&kvFWEu;{59wspE!p0(0q zDX47zZ%ke9qZhdBT>brPU|Z>628#iE6HyO^Gn${z7mhXwy1JkA3D5(XtmWxW*Vnb)_n}cRixcQ}K3=cx zOfH0NTPAq4xE)_ht%d1M*lcV~*ZkaV0*@;;)uaMDB*KzIhwZEI9)aUb2gMgozOb&0;j@P2U^0@|sgXQQ%U53g51M$w64i120G!0?eAlV<^)t@86`iq%CAP4M;% zO`)6(!BdA?`2Uy)pirkBSvz!&XVF}mw+QBbrd#l8%VO^u?TeAO2kE0(?|+DHg+zDi z<$zwKA(4AEJmob9w8}!9$TSr<8<==QlDy3%u0Y=s%vJpWrpA~JSLfcXj!Rk4jlv>N z##EDu5t@qfPDLf?*?D%PV4;}R_kHmU&xvF|JlWK@u%)OUthdAeK}aO_hiKk+QU|hB zR_U}I7nGptRejTYC^FHvXfzvXNolT12j@F_GP9n6U3YcH0XB1Q?&t3s&;X{8?Y>Zj zn6&U3*4&Zx(at|4VlhKLKksZ4E*j0*5P$GPv^$*Do6XS*Foin7P&nDE_WYuQ;mEMB z68FW(#wm<5?owU*q0xxXZO0uwSNX@71{C<-14sdL(&1MI2?!`X-&HIPYM$5Xz{~uj zaQgPq?9s3GOA#59B=LU=0b>mW!*KBd0P!6E7nZnUE2=469VZ%j^kF%AH^i}^7JFx~ zQA#W<=&dj-NP_BEt^#Ss*l3i=lzvho{RvIaDFNH%1k%*2VNM@3CQ8>UVg-|nm+jhy z(YI21Ch3qIE+#!%i*6Fm*giUcxN)@?4<@ox{(P&vXO6Bo129VM^a0?yJIF>X&DZqu zvD{~q&=_KJILG+kV4}5~#gE{^?1wghBy`};Yzk$rF($}QQy9Lt6J{AV;)65T;a{Mk z1e?wpCk+pa%JzD`jB|l*ukCat!#h)A5t-{jtb1i9&jH{x!jY2uY!sdJJuy&zHtEj& zPhdO`b$AGd4teyOj^3fDkh`uca}mwuLw~NvCO)UraL|zbac)-y_tZCn0(jl0W++2` zv4aDmNvUfV4`-JDybp`S^|~r}qKNv^4CK5@TEJSq&-Jo{kA@E6dCX53CWm{X9D5~V zIdUlUQ)s>U4lP4bw+}%^3A)pt*NHZMjfOuiOW>ns`nzHjpUjF%+c3Y{0Dl*4cYi~N zCuA`$DbTdC^(1&j!{M7pHOQ}OR~-UIA&wlTEe+}0 z^hyxw8}?=y&ChJy89mB-grh9H$p_9kw#vr@PqdL$*h+$*-D@x>9$%?%!9F&iK)P;- zv~lFVP@+w40u$gai%BMH{)Zle4}B1I0fJ@Qu=VE8;T^Or+FRR9k9xYBcZ2FXl9B94 z;_hc>O99UT-0Q{VPmeI5E)nbIy8_~I4>WrqkvTtD1ED4%N1=C7h?2~ysUDm#Lg5*6 z(0%KDLDgV-;Pt>Grr^C>u>^>aM^RHe{_~xm_|Z=2D@NSEBzWXnF@ zBcw!*2Rol;Koc@l>D2u@7>geBhxDZT|C3=mqXy$Ki2j9NuWUvCGJko>dxN&{yLsk_ zha&ZL+N~KB4LL=En|fv4648+T91uCM5@!_9+Tc5sGom2h%uVlWJAEEEK z$~o2pHGbw4LWhF`#F-Zit(hAn-oq0g^LH91*<(Z|6ZcSD9Q8pzwnh+9KI8LvGGn`p zzPTqt(_GfLuCkKB@|(Xe=xN8%V>B{tw-VoRKVfJsSF(38z_9ETwkNy8#=K9(Lxoq$zNp0nrMS1-?cJCkNGBuA-_H&P-3x{8M zk^}(26V?LP*u#})bTZ}>mo20ye4W}{iyqs%E8o0b&Uvv+FHJx@z7JmxFg#n@gz#XX z*mh(4DVzZ|-Azed-=C!@|7odLuHu~|>7r;nyQ{D0m$X)A-cDD&dY{Ryw23Hge9kTc z4wdF^C_d#UnxExJ6++U_kQYgvRg1_G8}a01g*HfpwO#t#j3}#rKR^%oyf$R{No~Av z@?-c$hZRB@LP?$Q@dA0FIYG||&R9Jxi@;W0U9poS3xXLhfn#&K8c(M@XqY5@dc8^C zdUE~|)ZWo>?H0f9hP6?6*Eg>XxW^oc?a8&Yi%9>DQ<)J`{Ti_;eU8iY(wx+-N)WZg zsh*gr&=&$<*t@*?&wF%QFOP=vd5$<4lsDU2Dov3%XJ$C!>%J|8%b|_kJ^nyR4ZjZ+ zv;uBQOGUj1-&c6BT29)DI%ZpV&to-uM;>E$2(LNH=idAfRe6uN#{ZW*z2m)AoQUj% zNSROCpJj=x-h1E(0OD4SKQbO59GnsSI5K-?j_IRUoS+aeLBIQV+Od3+vUHbCKe>SGvEP(!e2Gm1#ERW<4MhttUueJrG zu&|2kPUX(vq_T9$WgZ>KxA*t%;=%bcjZ@R=8_+JU44w=E=M;xpxOzLobw+uIZ~3Ft ztO3#HSXVw~-3eK&=7{8KX2r`=b8`v6c`S4zBQ5QR=o0c^>g%64%(@;dG<0sPz++eF z1K;6rvOez~B>UgD!Y2Tbt8wwV9QQLh+_UA%<$jQ4#E-qaS_~SS?5l?(ns;34$^N01 zuhjw}XqlzL(wmzUXMI~_!5c}a4v9_7586U9`g+Lu#-sKb#IHYRP=_&6Q&eKcemfuU zL6GfTL8pn=8hC_$#B%ionKyA;udTP^isO`d1}uFzJud@IP^xWe^t1{aK4qLp?+=j? z9|=os=J7BvWx2X*3Knx&VOmII-p(uF)^`MKa5p%Flq#FfMN<;o2W0Eo``2u2gT=*N zv-Y8QkBdW&IoMQ0!EFh|yr-?9RRi3^;G++v%-N*|qb}UrAA@+Em$LV!6CpxY%!0U` zUMb=4LL(?fDA%&j%xB9ez{x)PI`4P?95r`m@JRKm>EOMw7xbZc*f^bDB{hF+81wnT z7Y}lK+{`AE)gDm)HVKnDfPO0_5?T)1Teti)ytpo~GT7&G_oQT8{(`XV29_siPjFS0 zdARwcJ19POy!P(aHHhWeSbG(rL@07=j^qJ?OKEM1Rq8OI(|XK(-DMLZ^4Zb9y2=A!OJS@wkpxe4C zJkl^K3-RNVK{U7~cQ&8)-oEJoAEANF>t9Q$)72iB9tGo*12zrcozxhXSNVAFptTJ; zxwDAM*kNwpV1bDUV-wu_G~478b&z`Jj=cTK@RE3&Q?o#VWOtB}eO_g5^*ovYE5XPpw^o>_K zmZBAZ8(z|Uv(rLlZhJ7|*k2wdP`ciDV|t5`UC1|u@$>5sNjZj0oPhg={6fe`!Vt#6 zmx|d!o2;?)DX55wi%Yz;Gd&Yi@j>$@cv!e+)zx8M=NFAyW#JQ#jSqh3>RdO`D2^cC zldmED*Zyn%?+A(&FuY{3j&y|8e;?j{);o&4yt#ol;f=9z(ejMi_V_`@WWX?)xG?;cK85O49BAaXmb9&RZi22$s7*Jm>rJ2=n6!58B7}2=JV#61{#2K7Qega|)%z4y95~6RA@5(6?V0oOh}q=I#1B?c z(1W8a^or{=vv4Q+doNm8$SXnh|);?RdlBpX#Lkx5K z3k$@4*O+gq--)|P)4EIp|1+ohIW8349N;>3ibJBNH=v>!qp(G=#HL%zY=c{A-sF> z=bfA7a4u73L@P@1iL5ag#$Smyt*a|%T}6QQ>P;2tz*s!9x#M5yuO68FEn?-;owMa; z!NyW)GO}vi_=%mlcX~8F_I3l4@X)rY=j3%YGBf{5=b7 z;MCJgxCy)I36%AS>x=Fsr`F&&@g08oyb`I*q@sRQawQT=JrU+52ktvstI zI!jX4a~_~OIgTaI2u&}UbWR$FK;V?GYWnv#`ScFjbqAh5w$h-e{F~L(`{sD|ca`j=vqFEa8C^*77WbAvpB-Sl(u_%195Q6a z2%j@QChk6g70+w)n)DM+$4t*Zyx)qr$16+sq)KJOe|;l%igLABj3m`ROZd zgiT)iL5u~=#j#c(p@w2CG>LqSJUf4ebB|DrpcfLEZgz7tnB3J&OcEDT1FqUFOY65r zJHhg}#$T5uZm(;aDn-8N%!ZPKX?G~%Aq;_a8rK=F2^%d zf}r&Q(BfiA$x9!%^@2;{M6RgiP?M*$U$CyPUygzjEZ!~2tjvWPNtDEffD1T^M=zI3 zhSWsjh!amgh%9R3+}dgE zEd_2A^}=a3CnP-WK<-0q&|A>}A#Q7>L`@KOLF8ds8739RTyOIui{z?W(rQ0(?nHOg zX77)e3_Jvdfik1vebjYqlLi$RIT&shIgkY$J}6F;`l)O86tlXT=V{)p7$bqkfYAQ+ z1KC@YO0CEEaNXHOk6s@|<9@Qp@LTgHd?UgjwzoS3o_ykp*iEy0)1vN_)l;zP%^O#kYgsX};k8Y&ZG!YCR2fK|Sfz zqVOm8(*bPPX06*M3@l|NzMu1+VfMfJ#VSi_OTa}Du1A=DxDc1Ps7eOZS0d_OL&i+g zi0Qn=#hcIG`zr%Ix1XkO^wGRfR+;-?v!bfw`Y#T(lihLKJ|%b`a5Wzo=3PHY{9AQ{ zO<;~n_VHn-JQ&r_=otz=A#^&-jnP?s`lO;87f(~kAcixgpfNo6tA>`14^LuaJHwET zbewo+sg-=cYt4B|$KTsI?h{J>&=<|~^LggGa1VWVEY&4fXx(pkAG64MFW?;o4@E+7^Z zLrtH`ETQy*!!M|ta_M5fzgppXGbAvQc-S&`^9=hE-b4cS(UNJu|DU06!xPWXO-*(u z{T80cK7efb#K@vPuN3h>F*SN7FRN=X))Kf{hK_wd!UDXS^b`YIeKzIWk}!ttxLf-_35gCZ+&V(^eY?tb*MD_1+Q zG$8}}a7|(&gy&Zi2(l*~Y1^wrM8O6`DqNUceDLmg(HWHmO*^Sx6LZ)x;FmK3UkYtp zm$^8f*~?|2_``^YnfFz)<@iBjwX=Om|D~u;JIb4K>?#M-3g4%}*sWwsDbfix>Kr^B zBC-&Kg?;HMFST6Jwsr2sDVkDj{{RL6XRU0s!dDN&Un!`nM=$UJX=B*WL;WpFS(lgI zjvjYCpfu_cfJr4UbS%^!K0wm{5e%3<_)7IS$q$ZFON*^}U~D>Rt53nTf9ZZcr$=1P zt=2v(5SBWxMnpj=($jo>d$@8tsGgiNP|^G}u&uY`$;;V(7=WBZGq`usds_8k{Dyue zQw$q~m_1lp#clNVbvQ|O4@K?a$r6+~1`R7}qhYy zlDXLx-X6f~O}RSyIws!_etDct+e{TAXjEENCqF&|PG3#H+2R<)O*pnru* zj6+_b@n_%kWPh+p5o7tqT68mITJbl843+OOWi8U$K=ev4dGAHaLm@j$zcg7JuScmw zxnTCGR#2x58@k zADcSQo}{0+)aYy%mHO9=diwCut^MumKs68B_nb}KpL~Wal-S&)(qxO1$&gKbxY7c+ z|8TQ^0g}m$oi-91A*5fzwHI}j<7v+mqQ!GW%yQeftFZl!w_Zg7T+eM6zKeF(Ovc6L zH($RzrFp%2%9TPrlLw9E5Tcw_GIuRG#hULpbMDii44WDu?ma7ds=YP$#$K$WW+_pv z46Z~`{jjOz>>fqa8%TJa8z)-JpjYmc>aZ`MT=SpJ%|l1c30EeB_bQ_$jQ0StY07Si z(JlK=p#69OFQHbvz6X$$8`d6e(!0q!VWVt z#+gHwMyxCC<4;wDH3P106>nToiCO%SYMjO*t=#{9hG`gKfWYz6RW|;K!?mgTqfs+m z5RmS{8ljRxWFm(2`LG7r+H60|#O}FlwK(3C>ul|<5l^R(8H8MD9HxZ83jMd zD)P84)neY;)2;;Qi@aU)9zqnxAMdYMtm76(R0?<&$yM;$vC$-T4c0ib$X*sK1m)7O z5_bhZZP3|5t6H5Ochv`az%8S*U5*g2zCE+qJ@vO+ZqEzuIp?Zt{ z*$6{(xP#b_nU4;S&NDPO9Tn=Hw(A|dcoHFf-UO6WQ}X1$)w^6z={7 zpUV>hkSr3Dk!(GUd_!KnHY^bE%&$NtcI12A3Td&fa4MwnlUCiLVj<7;5w^)URGx*D zxhw1a*R?tIhi||1i+}mT!KeCLill4dI+B#uv?#51_JXB>;To(%$WUa~m<@wwgYmMe ztb1|7uw}PBb|=M-P;Ps8&bGFKkN#&Mng>JNu9oNJc$Q$C&>r0rUO8!Y&!4x3uYS40 zBE_)|BflPXFWiWJT;l2uc5H`W;(HH{iS!~kh@OI?>N;6^+CZ8LiP0W zt7I6TF&Enr5&(cnfA@a@PK)Fff4w7ekDZratPKL+#H?y}l|CqXnrn+n&hK{*iTQmP z;dsIKdn}G%wu;jF+eg;A_u)M{)|j;0>NxzmIW`}r5c*~k1g`y*(l;#6nnK|!Nh74u z^AC?&CKSz-j+!sTKdp$o_rX$$iH+%F?_682T;H-%wf?D22~cpTT08Xi1OjI4e6Loc1ofD@0mE7UFITU*)et*^?LRcS3kr8_e2;zcZJ!2xx_E z_2cbkbwuh>38nDLw4@Y#{j;c*Zi(}u(8Ji!GTlTsI)C)?i`Ch_C7tm*t9S|dU^UIm zmgI>xA?Y#CX_*@~QAE1x=8a1`pVM?kaRq*^rQ+xQy0RX(D#^OwTPm00#8=ATTh>?x zqEUn8uL5QB^wn)%O8%fRHFl%{;_t(zZ<$^rW%}ORGjj*u{rzfsnd+)@zH{o-sp{%eH$OA?g6n#HWB2{vzT)vN zTYoxz-uMG0|GcN--LLiTKIFjs1$Vr7*Or_AdE+zx`P5IIUNQK}mn%;x?Dza7U++02 z&~o2@|KkT2@0s*v_X|e8H|B>wJv8R(bKdN>W75F(XC4}R+`D^+ckc14U;g%2?@jpP zpZ)NKGd?TtbndL7AO5)fyJJTL2O#P|tL;}l@alt=zq|d>t`~jnwvLZ@@}8(zNu&s^!A$% zeDuRT>%VsL(HFX%KIY%2eDt;dH+#XYEl>N=z?Xj1bLc;|{V>#hX!8X#KL1I}Gf!$Y z?)yKu|G3f?@Bc@qM>m{y>Y69d?J)7S(n|to4gJ~=e{t~@7vFpQ2k+Fh|5_^)H29MT5Tg3|7c}ef%Y>UYPSpvwdgWw`RH3HMFAFYh|yVG4gNaKN)!cUv~7o`@vo^^kH24>_l?T;Z>#Q9^iivGCq0Bf0^K|HIAho8 zzwPs0UcnDeJMx1cym7&z=U;f?gzc{m>-^fT{(tzd{0{&1LW^x5_3V7_W7kf)73GXP z^x|nhxNdM>n?L>A|4zEOsAfy^CH*>#zVF)6_b+SL{-rV3-hEB0`%e$9|JCVLSGNAq z)^nd*HS>G-bXXA_wC<1PBRgF2-Sf`+-lebOjYGdl!}qS3f6E2Ut;a9f|F6MUp7W#E zh73C4r5`r%)A1)t*kKqTue^2D?U+$c^FaQ4fkpDRE*A=fkwDE=xgYz~#0+M#Uyzs+sE_veRg||Lc)2+?uf)}@R zD`>XovNJEdZ}R;se|hbo^Y43Q{!jip_`9Rut+`|K_?83iYjNDqe!uyFg|{`I|6=p1 zqUQ52ZLx0Cp5uS~;vW~}Z5o3Dzjs~Dp!~0oYX0@%9lqXS%x^DT)a;Q#-#Y)C;}(9u z;LANHy!z(pvj#r9z5K($m*lm50W!=Z-<2lA{j-}_DT^FrsXJaN!tPgQ^SH#5(l)bXY2*Sd}R z?ZThuO>9;@XGH(E&wu8I$W>?GuyyGxr@itAL{`ab-?~ey3D*v~??Rl9;SIc~VR#n# zAtcPP{jP*a zZ9*eyZ8JHTi&!=#nyl6nzG1a`{Xwhs#7%(?u;*gSi9ck*)E)BFsmDUd8L7xji=x3I zq~Z({o()Jc%GCAfdU@3JbOCLo5wwzKoAwFcXiAtjHmzvu&CsV`1MH?+?WP?LbfR5n z|Ll^vmV=Uayqp(DgC#ztVw7X*kf{@z4y_N=1=32IK|6U6URqDM2L9u*?|^5hK=8-& zVdFbP_Die|v(S!c+f`d9wqlEocBti1g0?LQGV1A0OK?x6UO1+c^mB4_l`DFs>IEGE zCT({_ACz4ZU7$^|DajXeI}I3h+%#Gh+Dercd1|7B+0Nba_A2K-Z$yKslGkz^op9@ep$l;~!L0}7^bht6 z7HucD_$ufDY~OVlCvA{<754Ai&O7ZrfsHrYn#f(*@no_}CEslRm#jWh7ff1p|6uBY z_z8v{5ChCH5ffr_tm**Td?ECFoYj8D;aXd-a8UhwoSoNL)+O_7WSY&hiS!|nF2w19 z;UCEO(e@394f=#}@DB@*B|Sht|81mI@ohdy|L#TZ*~qOvo^|U&+IqloqqYSTR>2;8 z`&iHc=yPAplkSmzo^xlMlhb`X?P5>m*Xh1p_v>yv`@rT)s-yp|580+wrlQkLzvfGH zTl)wYHv$9kcf>~oHpB=%5izscyxzB|>VVa5GUjamIo9ep^Jt(O=0alRzEH^>vZvg8 zGr^B4Wl}!7BX5OgU&y^FB{%(=FY+k50G?$GX}5tuiERs|Z4fJ9M(hN`c2kgGrEK>^YW~eqPx-k-g$u+JZ~=$CAtw z@nY&hj1B-3!#8kjfu1*_PXKJ8183!;27oTEkNH5xb{x~IFHzyjr$3+$ zAhA8g>OfA|0oeT0t#+{aUGZ*1e0KK`qmOe{nnw0#6_ zq6hE?v~MUr0qg*vhE({{P87b3M*T2j~xoPuM8q0@(Z@>Yp0u zg7q2F-^0)67&ckEH+lRjxA#=Mn;6knnqx)SQrivyW6l*xUx0A|WMH+Q+Tim87?X|- zbjI4)29W*p^FNDLuG{5nDt>H3=Iu3=472@Tx;g+EP!d2mByqI>c zbeH{^$xnvZbbaKVsNcU&cN~nKI&66p&;gDQ6rQF(AZtvN9e^w-lZMa%^!eYmI!xbV z%U=6>dQQiXdmm%&rr(>uIlDCR&Fy}xK=hAuwq{LNwtS_0UXLM&rHm7SFUQXq8w6yo zsCHaHnON;7*XuhY^fQI5&eORrI)C+Pspzw0Fq8fA;CzTUJK*>_6hG z4jefeiT?1OwZ72j2aj=ONSiU*k)Z>`JVpojz5ueZI!vp#cgL_X<794+b5b0WI@_YwfB7R{<Ku3>|$(=+V8%u(rfukF63m3r&5{8`?A-?u(`G!pSu z2X^i|9R2#JmGwfeFBoVWKy0Z4#9G?{Z3EaBQ#M#jNm-#E@NGRI?EXVC$0O@D47)F~ z*KJ?heE-|M8^h-+KJ!)nEbM<5Z_8cz>vp8umZJl}9x{mO2Ph-V$5Cc}*Z}yH7g?QV z?3ejHHO>ogy$0uWIY+_qT|XwuUCEyPX+5UXZhR~H`R+r$>OcfGrR_u2^-6cvn1~+e zb_UFez04EA7l2G;euy%{Tr6c~b(oU%+H&+I(*oVG4#DX21008H*^B%ePWBqZ>;JOR zSNXHBcRt-8Z879uO-Bcaz0v{7g#MWG-WW2Y?6R=|c)k_ZpG2JX8~7F~;2!72^z+27 zVNwY7R8{ZxT?fXD|D?fq$OxwGp{D`Ht zzwhqzsdw5AX+N)FBM1gML+!ffbTlc z`>FREjVDi7N_*4oPub!`z5sne$PzZd^S9^7e@&nZ*6wP5pZcW7dXB%xHm_16$zJ2s z<8f@K;`8s5?FXZ$-0)V8>wv z^_1Sx1~qcq*Y$tnXRD%n5BU3j=}|Urkhxv>7t-(Np6oR)cfPvGcN_4=dw4%I80!R@m<~{8?6YNVkh0`= zWsq$u8xZI;BN*s7eUtc&v>lvNV80*J=NoN)5OB$%R5M^&yIB@@Ae>j=ursc%MjXr2xW8LC4~Ay z1D8j!`+Cl$YaV>Z_xD5F_u~D~&2Q%%uMYl{1!Y3nP)3xMo)h4F9c7960LnJdacY}n zHUReF`5?v%5;qGzzka7*D|?hJvPV5RD0@xQ^9#25ZUg?YXj@M6?8pWFpwD-eYqov7)3s#} z`HSpPM-Izf(-t1GJi7SZJ-+L}J+FU~yZmeUV4al6jPK*se1L5OAX~_ovUc|Y@D10B zk6`rs9PimaU(;&)S_dw@XI=CV#_)drzQp^XZ{D#cH+3M+2B>*KW33QnOj(=qN1t#; zpzGYDF@B%C--Fy8pI`a&Tg-mJ@venG;Zb0A;Fe0AvmMKUORM=~Y2|yD2_`mcQfop&Z@zY4@9;{7Lil zl{@`bf#_qiw&orl(l?tn0P@#MVFL*H*xt>My>wvZj^Jme*V6|Tyx7?R^^TkX!%oiYAjeQ zf5@1!rVj|&2RcuG-0}Ui?Wg=@u2=Q@V)vVN+c(?iKNf%Cr}(9;7Zz^IA^AgAq64*l zpvYhM0pbI|2ZZdQ1Fy=QulD%=HqJq~0{Dabz1Wk!{}AJ+2cSb_4V??C%OJ)(a@+kgGBlScmC|8L3HZu&CX;s(qK zG&WAsPpfuI8A6ug1DfMNt__3ip#$r`it;ZQwj#O|-~Rjk`xW00H5vKOTp#E<`_owY zbA2D@e2(q*PZ?eIo9ZaX@P7XG9yl63?-y$uR|gWdf3-GD+kduy$R2Y5yMn!paX)-N zu64BM{Afdz-*-Lj+Oe0$&Gr9V@~N0>Y;>%s1C*T_2g>?Y%2xY;kU3=^=sx$L#O_gZ z{?boS{!ZVo`+UfL*s+&Ry=hf+({_KqC8s?pdTL|i;U(=;j~$54RnRp7c}92{cBV@h<;}$lo>ZEB5~@(*7x*c8|X2yZk@f zeK>mB=+(I?fAubyvZoEu>wLvF&_;ZP?EmpMz8&4L#sBX|hyD$7!i~}PS!Z4CzgGUV z|GPx~>=QUX#M-eSeSfT?|z%EAkII)>7Pcdd)=KjiN_;e&`raq+#?jf<1CnY!eU z?}8xvK)2cJ<$Zu#{_Ov$J8b{QsQl-w@jrgd!!xlaq_NwGTmFz`T>Hm60LnhlZB~^N z%Sr4%?1WzLa}3MB&$Ir2A5cS|e-wQFoRq)%hE>~tyz7Vjt6&3Owflec{c7$f7~}h^ zeoT16H*mw7emeD*)zMFO_&?_F@y8Ex1Ro_c`hhYJgg6$o0r2ac_rMtQQTAy6kJtA9 zw(lP+f1~fe0kS^^(v$N(_kFxKg>!znf(_fM_5l;hAHF_fIt04Uemu})&ZuDNVtg-( z?|fk!ocI2iH{!acF#7*6%A!9T27gUG{qX(`QO5oD^CxrucdpAFY$z+rjItB|AAX!( z6HXb^*MrOhJ?4&r{5uAFW9={HkGWx4^C|s*0{O%CH)Z)3j(9t|Y@`42qwgI5Y3|@c z8M)=J*MjlAzFq?WnN#+GqPgb=dt&`xjQrL7zq|c6aruvW>0>{ey_B^E--NapnM3Wu zC4U*i(Vyd-jwyf0JWw?6+(2=8K8`y>1JL%x{;RbgYW^pTv7hLZ*nij*w4tV${^O3d z(e3;EjSs{%Ki53`Zf@a$^(T(JIBpmFujJ3~LMda&nlcZTR7hM%*nz50f7*W}>WDG_ zXZQa~2UvC!mH$gi{og<4`9)vk9v)i$Y`2`dG377qpMD-<13~7__dh-6JQnP?RLLLu zLHW~Wi2a8jpltxlrVrS}bf691@A$pum+>W9-o7Tcun_-TwSSR6{65OxvHy^*#A?Ub z4>C98UlQztZ^NB20Bpcu`yD_S@eZ1@{Ex>w*y_#xulc{>?;q!u?Aaz0$)Ei=WvOhx ztp9|p14VOczyB;QJ0sY$@~HFy+6Uk>L-A~!{F_!E@auYbzq1H2;R^8HFz4)`*yq~z zPuYnK4f#{Xly#tZp7Z@T?Le@{oVB3=cHbbnqS}A0{L!CC|F7gv{&PHCaQFHs<2CsC zV?2pVfBR1E@h*KbWF!4QWJTYP@{ei%^zR^l$XeTeeGC@OeKbhjaOMDL|Ly*x*8Uqh zf8#1(klShIYX9HwJUZvI+~iyPWb6ZE{I7jK$WV>_WPJx^3|VWr>m%;93igE$V9H{$^P|BZGaSTyJP&_KQi#5jTeqTvIEP^R7gQ}!y2gx=*~dc4g4HQm!!?aE1B z1%E9M(E+^s5&6gZezr^@TgX`3eD^U}JhzSX0h|k@{PjE#{RndoK-#~u0XZiBA3U(Z z_is8FH{zT3tj|sPi`|1Plku#ExSLkzru^$_`)b^8%Mvmzo|`iMyU4$!>>I(}3l2-$ zhcBS#0Ui0@02?p@K8Dsi9&?}%I1}I8ec|&r-JCOS_1OD4!Cyj*O&L%okWGy37x~+E zoH7meoLA@m526FbbEk#}<2ls_gs~@L7;}MP?6pMuQh+&z0@TMjumZgMEP(uJdmA(T zeTo0?cV5IhqsHVZW!=P_*i#0Sg){HNbsLZsWTyLZ$`Wng)BZ0a|DI*ng!?Xp-vAvz z3PX>iO!NViKlLnD{*9{xY@>d^-;tRAuqi2J&{#UJWpBuzG7;HOMwGv9`;;GKiMH=~ z|EFND@@CMNC8=xx$AWP>&V`Y&n*oP!T3SV1xeYggJVJVH!(gy!}kFf{Ca)8#B}_BzuSoZ$7a1FWBO51fD0I-qsdZ2LaQzYt|Cs)v|;W2SD-ec3Dd zi|mOtF(>wr0c9cnIqW`V6zpAoUApr1l0Vq9Y+S(*^aC6Z(7#Nm0~`k;)q@V)P!Hel zRBzcIJpplgd~35l=y_bbcVy4^ys@$ue_zRivZ0K;$TE{W=m!deJmg?M%ggY7$B*{vR~cr1sqH6hsl02|?LLw0DGSIXSX%LIuX4;LAAEtn;Q@<} zhOb9I;PwSb-v}i7D0-fnyqLVDiPtrcz32O$&O7ie;90*~m)r8E>_z^-l9&=(cb^X# zP!^O)HZo23*-)>tv0?N#GC$1zhrS2>LG^ARMhAY1{y@v$l>N=n3lC``Vk>OedcfB- z;^mJ$)$N78tn-+dXq+z15 z?+Co;fYt+(U!Hr=fnzd{8wbjs7d1mr_s88kbpIFbG-QBbA|X!OF_FlNB*mZRSMg6QWX?>j=Mb#4uTKn#B~l^Vy)FB5{g$5}Tsnk{%?JJ9(t-qukjaa-7V$bJ$2*_Vn|SD0^Z^*#lEz zn``aftv8?#{y@dDLX1lbI3_f8fVRM~1GVpq935bL(0bt50=AhZW)C#ICJ&k~cbj+1 zUdH#evL}Ybl9&=(H{Q8;4S(Qx&^xto=rZ`JT8D8|?~7vlg9Ns~)PtsE58QIsGUw5h zd%`wfJGZ0P@DV#=NGu`yM)mX4)w58)1(+WwA6qbJ>CwXBS_hO*0NYmJwgt*Jq&-Y% z4`TEnhirmd=4|tI%H3t}%{I@zo_-!MBX*(QOhQdSlcIf*MmmTg+wy=I$x)b3%90Pl<{R~2HO zi0`SiPgC0;XukmC1o}bFIs$txjAO)@ae{7NX8&rQ*VnkgtiwFl^ynk#wbzf^^gI5Y z_;_k8#CdXQ_msVj39%tY#EO_TH94oE7yUtDsI+`sc+ir=g(DQtLPMLZjIsgnfPIShKrab_T#FS|ilZ|(bE>IS3`Iy&r z>V}fN`HcJAkT>ggWKNnieOl&jxl5Z@`YvtW#(-E5lThEvdRezq-!e_NeCT%z*n)?` z0~YP4E#O)Ldz@(Kf%pcdJy5zJHo?#Zr4N(?Q=C3%dAa3S*R_7mb=38U4r+PxsOh52 zZv?G)U+awJY3Df(qK>OEF?kj{2n^8X9}4xa@^!slx-w~u{1_)d2;`2r z9P&)wi9usvpG$Ry`c$-n&$F~(=o^BK-A2^8yuTp=mOiBv^lgpEe|=mS+=f^b?bIc{nS34+18bvCr=`G@JL?K4`%gUPA|`=!IKPQe5lj^-;?^rOcI$S8^9Sk9m9Y1pbx?kKol0%#Ot} zpa;!h6R&}7oEjdq_%Ppj6mran^$w1&NSol&1;=j`-%;5InQK;U)Rdj*2m<|Cg~g4juPsE>LZ+f z!nP062U8!+Bz^?-BS|vy>gQ#gPMfany7uF!huXe#J&uw&X(i13_^wgppvIDHlV;L# zbB>TH`;WL}$e4M}GH`D9<*0+Y$91=;n=}Y5q-nsygmE=<4Z~xJ9<;eLpbsUG!((_q zUKJj&aA(1grFi$ucaM-0+HwKkr7*F*O5!=>ujI>fJE{Nc{Ct*Wuq>3xvRQ}J#X3>< zv4rKZz)NW0qI}5ZT+BCjL?1i~vU)t!XThr?x4sM3hx;%56mr~!xu=6j5tVR`d-xxo zS#Qf2&+u86!Ls@8;u&(=jB}$e{GN>1R3NAK{aOiAjo}P zY5pGsjxn!6p!?eVHwkuK$B+*OV#o&r?&}!x!9ZQtG3j~pD`WFp>0XbC zeZE<`*RhZP-&I2+kgmMk`72`yPFsI!VxJG!b?w%7_jO(E$NhhIdvssB+q1j9$6b#x z8A@P3D$W1H-qe`a3G7o0gXm;p7&trL9>y>Ldl|z3>}d?axHJu%*kgl0vDXHHV$ThN zq|Y-5I-BJg1YE^yqe8i?tx&2nItZyQ8g(jAU8+;T+4l1eNB7#-xJ6u~wUj6RozBmRfBmZ?xJ?`NC>H?Eq2)>8PByo3dU0$2`p2X5v_RR?3)wvL^iB z99;^@=eHhzn$`OCp1|o@;JC(WKmCB!X>P>oTpkH@t%zD(D}#X3Io8umpRW^Smd{d*U3& zl;=9Xex7Bia+N&n_Jg`f1KS&E652>(n~BHFIxX-LL^~g5wVS#z(0QJ1yJ=H;0QVwb zUJM;C0p>_C*!BX>y_Q6ng1weR1HEyM6zuKsTs><)?~<9SPwEB@7Ht`9ob)T5=S4^> zbY)n66K)T{ojy!k_It#gHrU1-xD#{QlM*E8vn1e6oFxHc;vDR~)J~;1rknJ$a#ZE% z@>!Rz7c`I-TNct5ZCa@-?6ccUXbZd4q-)*hjc040ch#ZWm9}NJ97vurF#P(p?CHiFK zsm+95xheyUjgAL>&(YtmwKXm~qiu|~EpD3voPrIJ-1TZ3CH6_#CT*AKgR!sTxP#*n z@(5mYNe))qiRW4ECa<)*mZL92-&!O#9zF_mUiwk!JoT*L!l$ zb?&)bu~~{X_VH+=l6p$tK%QWi&_|O;^4fONdAT41tL=oVtq#+-2D(*3PvBSE{tWsz z^sUr)-IoG;V(ujDi7tC)=fY9ioT5#)4NbT7)sR66+cMh%cm%KH8G3hhW6A*a_p&<9 z+9!TI+o|-e%C<|NYV@6b-cY*!26P?geV0M+YmfDHo>94Mle&+GZPI=)#vsC@MZXzi zIN==_G=>b&{)WIeI3#|8`0=`L)jqoNUBte`Vs2uhCo?P3zPx+XSM?+ zgToCa1F<=fxnt{1J5SpwHr}x9+#9(mC*42kK7u^5Z=hdc_s!BLh%IP0sa|anWPG*N z3HF|IiyWu0e)!5 zZU?}o-K2WylTq$@R)-l|9qegyr~|;>9^)wdiMa>-PapAC^syP6BTvlS97W1IMbJmVh8VS*?5l5TwV!gl)qd(q8CTKI;n+m`mSS@< zvo|j-+-APKOY5Bb;&mIlq2X3yV9#0EevxB~FlR*UV23iAHwE_1 zkY2I6<6SJr-E3Rz^Of#9{g($jD3d8kk>Go!_FW{t5@$1Z|5rXN3783mA$&)PBk^FKqz2Uubojxl!hqY@2WQ?`&Ia zXA18CY=-cHV>ajRGUl${l`;O#v&flup7j2H1U-d;_ z(sE%7#Aaz*#IZ8%5HTc{z!bVYJXISIyWeT%I=jzRa|?V&C3=sxMIS@yyXXdFpsz_+ zCg+KsyKmwrS@p%Yz4Vboi(hnlE{7P=51=jLxBz|&+kngw5!;T_llBA9{SH<)yla-Z z9p!5}cBkR+KVjJN$UB?&XFM+3_2rT1IiueeyXYCW4D~0jd=r1W4b;vN5!((^J0xoZ zfsWHBi?7dlHPhb_Kc{}|O|tgf{^1S8KP}s1W_}%eAO*2!Y_js}^}e=6^og_`;=3D; zqkwTD{Lwd@9_U(j5VlI@lhJ?Br=njk{Tb+u1P6*;#gXSogN*A;&r>}2>e4SV?u(Be zjYPUU`mTR;dgWJRME}I@3!J`KuqDPFr^l`N4RoA3Du{Q>vM!A4F(9`Pd? z_89_J4Vlipbyal7-ovI^r*-}=_5eHyu{ax&W~qx<5i??^+JLO(!axJOP>q<=eQPfB=%^_5thxPPUdx{=XE_d`X3*$4P@LGJc#+Zy6SNA=1YFG zmk{kXF~rzd<_=_j4)3jiwTXYA)3nn9-OBcJ&4{ecL7yAy58K5yG!XdHjuQXDF8qya z^MC1_*Oj;V^~)n~ZrqzO{s#^njeM8$b4qXOs?W`5R{5#7Vuv6D=?6GR4f_tPA%oMj z3}BCX28&>Oxc*D}-2`?ASd*5I(o#Oo=eOA=*(qncOfQ01Jh1hZP+1>iv?R^ z46Fm4ruTI4?=<6C_6@XAGS7g1SK5x<=Roh@iUtQScaA)FU+X;S-uL?4GxI*ns4td> zV7~_a`2*i*_ROR9E!YM)hLSNc##0=Rb)NAo@kcvo7U(j24d+bN9Gmz&+U84LLlpk% z$b07fY|cYQnW@fCyXo!7Cp!+MJuZ{Z0}bI*JbOCINL7cQdcB)LBHYnJG42#C>PSeKYx(bm;i04Vfi3-jvZ2Sq{NtG z0bm~JGV2uJe`TNu?^w_`%X=L9{Ep8PtLLWfWTW%uvu2rforj0M6`?IxHpNq0(E86K z`jE&^ep@ZJ)TE)_&PD%$q4tG%n?{yYPSpY(K6m*e%dbDnIUi4 zZPqyQo_Rl;bHO}~^1gG|x+vZ`iU`=h$_f4?JLmt+|NZNOebE-fSNKMMoxBsnLFhZ_ z3vrH=^EsTCBj&`u%dD|7=Az%jv4XZej@`ldWoT}t2d8c(q#6AvjnUZey+L>y0^Df3 ziKXp}+87gS_5odIj|Kh{WepMEuh{t8?S*Ye>m~3PIjf`lzWaaV&1;@?^6Gj{x^ZP> z!NKTX)*| zhX8+>3*@^(H5Mc0#J<~{jk2CpY(MeGyoI(sL(z7wM;pq~^!l4S2z5q3h#=bG(f8_~ zH)5!45V6I4g#A8??`42}pnKUD0luqbzd#?AzAD>`=s)_Yp(`T9-#qf%eXaAPdq30X zSkF1PuZd!=pX!{{7Nz%iNsyeK@Bt?S(!M?ar{3Ih|UJcsu&mhx;P zo{a1d@jrldcR#qVI+86NDfy-El1TrFeUV{*2EzW+{+Rd=gZ&|%#*ydlYn>$%|tc|VXb{*TSs3jS(&)aCN%%Qe9F)!MS@_VpPb7(m1PmZQD zezrRLIo7Xd`uh;;?Y@Tna_W^oVkkC9<}m5s+dh!kpLyti%Qgak+y3x>gW>(H9x}|LnATKJ$TRI?cM%KEGbsA7C5c_{z5b z_@29LBlLf%%!~HpyDs>rdd`PF=dhKLAovuFj3e)v_p>?IWv5%lbq~K6;gV@_e(;BF zAkjJRCHmitRl6L%(ycqK{KOD8h&F}T2I$}Gx#*&KOCf`ad^aZJ0vrEY`y+Pe?P%zR z+)Bq`Py1CLf0?P-iSEVvx(n&^)Hh$lfg3T5u|KtAFk+81HrQ9zxY=_Q9DAX^2=f`p zj_o{5I*lXG-Pbx#y7x1A?pNbK6;)59*KU1}yT(KJOvF2R7eATMQ6~<>o){YX&oP+2 zCIU7e*av#d9YOmKn*{u6|7Gl}@JG8b=rn0G?`Lyvmg(jE{5#i0_Z&z)<~8Nw#((W6 z81s%=6~k9H^rw7Q@CSy(pZ+IpJJ&Nf_J8h(K+)VQgMIMs(ZrwnFTM|+gMTf!7)RbS z?`Lza%l1-6c*NV0$t!jywDTtB=D8dHHk|YEk^ar^oe%uaF#J!Eo3=mHf5<@Vj>z4(HvgB-x!1g> z-m*Hf@pEr;{uB=NUjKN#ZO`Gz**{+!!-LnlHP311ckB=SPqlVx0PH;R2j;+Ez6*m4 zo(=V{#b5jX(spiG0sDh?G(zEzBhTH}LR$v+y*~HwtgQm4iLIA&TL#hSsFy$Xjb=~0 zi@jF%NBn;S|NfZ&EGm=rCtQOPESlRh*l!8eD`M@#Agq1hS|#XznERNs4UJ$zd%Cf4 z(k1u36Fqo1bGx(T{k_rV*c-CJ=oecJ%*6h1{L6O$z?9f>o&i_~i_2=)Ulo^~7A&pW zFXJ*j|IaxH_8Z#%pzjOch_O13I+@p*p4atU;mB2y`S0v;?1>khH}f2ZtMtPMH#l_H zRUditCC?2gN82CH|ET$Y`Z>fpP%_V1e<*FBqpa^a?Jp5n|_v8T(E8@x&6o z53v=DOUmNcpO?%#J=kaAL0K<72y(Lf0N5X+|Aqb6x-YVZEc7+&&g4ARbI&cz{QC;zjpX*E^l@ zI4B>B{vt3Vb|LJ=C|J_21KVKFc}c&&!~cSz-j(Y@gD^%AKNNC9`w>4BZK{Cyvme5m z0j7eVvMsaqVn{G!uSIlx9R0#?Ax9>Wd+hP3B{ zfIqMe_Nqww{i$jL!@~nH78@x2Fy<{V7Pi|!q=0h8+OupiH+ga6fBxO;BV1FG4BNWy zS8v`QIeGNzhQ(iO0sSNBsIB|3+t6EE_r>P{rWo^;6XQex30L zR6jf{JP7k~?1%MS5ZjaJzsvso6xT=vqhO0NI5(f`ZqKh}+~>aX<_^*kLrXTad45*g zVcKS3B=a_E&Vd*LOWJ);@izp2*eLAPxd!{f!*2b{Zb!-wLiuQCwhY8}WPrKJm&X5; zpRJC(k9cmG+WejP=Kbtnt&7tIv#xqOr~R?9w|zyKZ^nBRtoalSfn~6FWlFnG+|3_+ z)VAS)OOJ@(1YZ>2K4718H6P^IAjAGZ|A}R$>G!j@W@>h#>@041byLQ5XGF8My}+LK zSjJqO?;%ENyhZGYVX#+uTJe`ChhXoDW}#jcuVDXk$Wi9x?Ec)gL(&G|U&q>5PKS#eF;+!=5ddl6jKbrrW3@qw6FTnpRv3`xe zOz>ySpDP}IH^Oop{Ec(L)3{ESFYWs=_`~J{lNg@|wwu@i`;6l6X4+rALnqP9KYmArUOLtp%euxI@HW98?b>+(ud%6ytn@+@`-HplUKFxFB& z4>1e&F3&3dq?SRbSNU_{q1q1FV=|c+5*gUz0jDo^+n*G=*YcNZBbyPoEfbr=XFlC| zD0A_RH2(P3HjG4F;xW*d6HENbpOZsei*U#{(=v zz=YTkqfnnpU*m5(cBr)C>hQqD`%o9!8f49SYx`r;RqKDVFLvucg`v}Bxo+rAOEN>?FGkM0gOQ2KR9O{zoYs&nW_H~9knW56k zAq9iqI4tuUcuym99nRPcb40L3h42pw(PkwtAp*4Me`az!sd8`^1e!5=h3@4*B0$@Vd9dr!NIu@!I!@5CU~r(v=MIMzbv>ZI+q~zu`zt?<1kJb~>d-!h9AMx?&V=6y~ zJW}7lbCkRj1C2*x98v$(;Q@=b7F0G(iR{K?ITgg#3squ*=`fIENs6+>=JC1<2T{_X6-|HJXd>zbd2o`y7!&5H@L8c;Lc~;E6u7vP(FUXZjVkENt6E8^G~{j3IEO zZD682(n;e4`+6H2FBio4T!#U4E+XY-@E+*!jc~qNXyFwX` zZK3UB9rWRKdjt*2&R`Fph3jDBhm+Q1d^Gj@;4O$Y-2unR1%sCyWIs**`JOsP7Sc9g zr#QF7_96WS3_MByzn4>PioOOc+(jqh+L;CJTPr&!-;CmG8?EHcu zOZFFB5Bo%4gY&pDpUZP3wiQXVS9K&CYdb%m6?Q667>AIQr)TfYFW%*(UXd4%QPYhUc-`HA*-$rFlRjACV3YD4XiW5F{wt4Q503yuun%eTi$<8Zl^H|Bw$JFL& zmh8MX%UAm>89wIy%36q$VqR05C!Flul(l(o>V|o4>XLbG>Yk(iYye|45iY&0)c2v= zHToPn?+}63=~R4nY};`vLz{9c!>O)J+MudH8n!A_8oVm>Z01)ST*Y%$keu2rPuT={ zSlJ4x`CdB@GpQ`KHs?aH(hliyq-J^N;BQ{upkg^R+8?XcKf_k**Dtf$Oc;SZ&!4o~ zO_^b}pSH?sKYa`K;QrF;F!QLLaL#-DAM>=G{G>d?XIsA>ZZrn!p<=7m>t|c7C){kc zoiYo&@3A^#f7`CuXSmyZJ7G`wKyAW)W3|_^FSpLeXZS43Xos@epv=~ci*>T@HWNq@duhwENU9zKgQq%7hmc%cqpWeHYjv!sEvw3$%gJEIvC z2s&E9mR3{W9bKV}NjrH^m@rnc%`4&uF>YmoB=OY~+;{Ra_ABs{vJ_TQhoe_Yr)aB4 zQ=5scGGNJZ} zHVU+%?|T|JweW>$kA+xmCtirYcQyBS6`hfNxO9%eOP;v0 z1WP3gN2fpwX_7W4y0n@!dt@)GY^bNb)oJz)(V5~Z*#lJeI6?afma2W0xEBW#MH1dq{9oe0Q!nbs<29eqBr;PNk!I2k+bwbiPf6HK>$cTyQhTcl^pbso#)h_+ z*eG5Y-$tJ&JNlJJ@JY8m(n^{YM&w0!YBxD6djxDQv^ve%5sMA#GO!8sMcfCW7kl*J zTqnk>d9mvA$mhEc=@d22^}U$mb@#6a&b4JEJ5N-fJ8x|srPqQHXg4t;Z|x>$)UV;W z7FNfZtKpl%=U45C?OAjNJb-_lg8i_!vT?1?FSu=0WM6`qjQTqT_i^o}FQd)zU8#OP z*)dstX8yQ5f;+G>Z4!7Rk5>CBY5PZ3`>9V`JurS^AIW|o2{xpOab-=~%=2IWGk%`{ zH!k{`(=~U!`f=U(q^M8VoscFIBk}~^$RqYQjSjK;%#BtCngZ|C2NJ+2efqwl4vug*N=fxM6>@J1em*Y;C9 zjYWV>fz@edb;A9u8H0>5h|^4Mepk(JwaC7w-_*Buq+N%1nixk7iG42j)1n^sqZO->>ZoSdh1W%tAKk;r>_0>re zcoN=}f27*k9jV$G@H^S+GIyKWhhFqh{1(Ir7e28+ANTQary%w`cHeVfpE+t}bo<`J zn*D@F6Q3HyygV82{@vy9+Dm;Y^H9!=d#?R(*dp=_-am(aB%MFPUgh^lU(P;|{UP@k zMw?1Rqj`_<)y?Zf=gc$eg%1M?@F={3XYq@GK_VTp zI!tK}{5Fg3Q0Lf>QwP~sB+wc2o>zPd(09M_-d+ty+|iBCK?I8b{@rWiF!EZTNn@gO z;f=O~yn<(?Lz}G*)8gi8p*sU&b;q_jqNm)SC`A%_Q~Y0I%Twma^TT`BMfh!aqMsWd z=BavTx2zpE>rbaX^Evmq@Fs1IeH!~SM~4Qubtuqz_I#NWM!$!4Nxfk^M@%$(|0dAH zeF_pMz29!K|MR}igMC(>oMXpybz_tWALbeUTup4fx-2jCrOHFz$Rl}Wzt29ObKStg z#3#^U>e+$r6-S7lj+xHAQi+G`*O{`8^z-~1*0pWge$Wk1QXlCvj8R%*&8?P??=+ED z>CbE(5-d7ScitUgPwSiQy-YEegSIGm5);OBrmS#Gl`_fG_vwwE@nOclE%=rTCq*i2H6(bo)iww{sIYb z@gvV~CfR#l5RSU=VV+;Uwh3$G)Sd@^(nOw>9TGcDOghiF40|b$2=v4_l>HWMl;BgV zC-p*$!Pf~`XN>(6$+{Si3Lnc1Nq|2-kfv$6&l=%y` zEfX8P$1Y{1&XXO5mMrpg7>hkyJ>o+gCu97|yU2g`43{#t!#$9 z4=8nq{SpoPFEs$c}wSQrc%58oM2S!0{`-L-vg( z@=V@=fy^P%S0pw)=4}ytxWBK)$H0SoASI(g-}fue=$AkCj1RH;!;CG-@yUd4jSsL8 zOo&ZU<(D#_M*oQVCAwqpQ!4v&u^#lPsgmI1L!Ls!MaMq$T1}E3bw9^FDlh)^`h>Kl zO%wNHCGWsM=5jd4NNkGdALU#cY%}+u#JY0Ebzux&#t?7#^y3HKVf<-A;ORHNdJn!4 z@qteAOq@Wv+aGNY@j)y~*{_hitMg<>p~Z(h)PqIuCZ8YDm}q=BPx8b2 z-*IR)=}Vg?@=V@=fy@;VlVY~7iY;;Y7`$SikGk>kDbMg1K63OzW0B~n@p*FIX9=+p zxn@FB96rFNq;i$?_tJL(KiVm^|D=1rMP4ULf?<;X>HKx&Y5B7?5$>s)2oLu?;@1lM?$5*k#$* ziOzk<^Ed3tp!LFyM}q5U-#^zRrzaD-)qaB72NFzxO`vGrlh{{E#uk{vpg#zFkQ5H) z-hTR?`Yq2LO_e}7L&zv>8@%=OQlSw42Oz9@im^bvt=VnmYGR<>7;%e{4e1-nih0lg92O=jRzD%a| zxp{EPp)P2AI1j|J7BC4Em&vyt!CsZK_^ym`w`I>A9g~fF@i0blJ*CMjy~pgQdUbf> zegA18kLG?uz<^i~6FvVNES`5W_CK@7r|=;i`AL0H# zvS$!=jXEdzpncUv3*v#NljoGtYa$_Y+zlD3gFrEr@2<=kv(fhU}KOz^b<^Ztv@(5Cc7OI%A+G zDA!AUDf3X*$&=l$qrCth?upZH(dWP?>G~+qpsErPZ7qvWnx@JkLl{M??w6k-0N@N`aPb@_DX|8H}@v7?XvJD`$>{l z@Ql4p>Rg{G81=4b6(0P?5sB-LxH7Wm0Anu^cghX%#mpqKa^2Ja$NnUdxAETG)D6er zxK5adQbMk3vXN(2IcE*@h zNidSQ)=BXB`2_p@x>tN2n7Jh-Hsn85eGWc|8>i3ZXka6K7JWR%apb-S;8pYO^$0v$ zAco6o;A_XqQT=@)jFQoN9^Sp|QsYlA_`HXGq)x(qQZalbz&%-g=5sN5+}MB@@nOgt zc*I^SX@6fJ?X0xoLd37xK^>Gnkg>MtQ>%FTLJ!t;2`xD4c=;0B9^>p6@y>V;*2Ak5 zHO}?Dm}BRsH##x*&9dC*iRw$5hxiIUpquj$;6Zp{TruzlY%=;jMe7;j-nU1L_k)5F zV~GhyXln*WiReXq`^fRPu8y2go1(_KdEYrV_at!hYhHWF=je}v4g4HpLmr3?dFoS{ z)%Q&fMtv#=hleaX%Ds|UzVaKz)**JG^tYggdXzjNW@^g+^I0!>yyj!ys@oNHUdC9& z27Nhs0Z*DgKaQb33q}^8T#3~sei3vC^)kk~9kZXYHSIX;jNR)_jFauY=f3umhtDL< zr|UwAfg3G)dyiU#U5_W^RS0j)MR2NB!OSH02L=p!&{kGO6-P#@!_N*qzf znY81E3Ql&sb@Z=svDW>Hu`Y4cU9pYPo?_ysGp??Voeo+_Gild+)aMcPT!?sVtJVGh zhy%$ul6HHO_?^<`V)aSlW1HfZgtZlaC7=5w*Ge>7Y6Lp61 zKtU_?CaZobT|7}=i*UaMPb0SX5j)l++neb}+VRSbHV69yel(v6aA2(L7);DO`Z< z^wMrJiH%Z6!3^iF7}hqHuDp2GiJQ!E0?SshWF1_nGmM)Gnt)4MV}?Zh`Ed^zVNAQ= z4#ZUbJO(HF+{B8pbQv30#wWI3iC-YTf#TW5ji{MLkP^;`)+e8SlpX3f|7_5HGx(SVpMH;y9Si#@Z9?`8#JTw8 zoNu82&%}Ew5AzBhh#AW=SQg7fig^d)2fdA12KWghUB=k1=yRWh4V{6uxGFqw$(Hb- zrC%Z;uB)Uk?R)qi^9)!t&W;CL#b+Bdz8~ftYVr{7F$&;f|1v(~+3J1`&XGy-jKN<7 zc9e7c;bZ(`hj6?~&iN=G=6yTl8fgrYjzgMvEiM}PhuEWMB2Hj2VpKhV zn2+NTe{MNqj(&=GefzAAvyMvQIq&g*%+q%A1M&=>ZD4YOAU5fhZjvY{KSOWG6S zk~S6&(8qklM;wXvyUpR7u>m#Dh+iVSGA5oR#>n%S5F^fhM&o7UXJZkA17m)UgbeZ< z5(mVvyZ~_;7fZXQOk()f{Nm{57qJ={yQI$KeokXZ-Enk6;Xpe9T7bg^4S@mTFb+WM zuRUU?sFRMo72le6iH+!iS1)+5EM2a|STt}T4H9Q@&K|Uhxb-o9VPdrzZy~nV6Np)a z^`nq^$wGZDk}`Ewi7IoJAD%B7;QRroqEV`>zEj`sxgzMLu>@}K(`0M zU>fxxP!3uq{k^B9le{J2D`^ zIK=d0*w%GBrw*o|`84o^c#ar^)cr#o$dqNJswd(9NE_P)XqEVRq#Zl}gS7eu@O}Z} zWMUn>!MmgDb;p?-`sa;xGmjitrW0-TK(&7{e>{zvP9>KDipo-nxkwU7PdU*x0OC}<}Sh~pt+OvHdo`F6pptIQ;?9d)m7g-l3m} z1(%Sl$(dAr1qVw>*=*=xBY1{-29 zWR$u;{;ntfnW<#ldl}V{ls4+CV4z}Z!WNK6w1c?$1H`erUHW-b??vA;j&j-ABWV*H3K)19ww z^29&B-H%>+_qwd>bLb@xyIku4Vm*xwqK}Zi9ym!HHT20&8Cmkq za#{?1E4l`8s=fRcZ$Rt|#%=SVUb_tNqI5ubWPjLs)>w!Cu4UCSp4GC4-pg}7$Btth zIQ2PooiN<+w;KOq#t}#Iq;-IO0eB|=(1SArMGIt~6}A!473N1Qw)#lVS8c14cV^gz zm<26w#y4<&(LtVQ4{6)z8-r)WfIAbh&Wc6%+26C>6DP`@@nY&BodKT6o0lK4@{I=a zM0-vhAg|H}W)>rc;sdge3T*@0huAv#yfV;ls)2aF}_xPq%g`|{O(>)7|>?6814fd#8B&iZUf!R#v|6&a^auf zd29abiEk6j@87#V%D&Fa@6LaI;*T6O{~XKF26Qi99_Ug2smy0--6u}upK+7wIXyai zs~7ggcRKIqYqFzvmtyd}~}!{^{G%kKprq`Axz2NI#eH zQMCcF0UQ^C_dp5bgh3xf_qFVOh`Z%6##qVIA9!J3^7MFWW9G5Tt_M%l0mfKmKLDP| zd!S@Nqw?Pwu^YVp`aTPv$RnUY^V`MfMRy~1PB!$!@lXA) z{1j=q@XzrM#whsqS=9sIb9nuw@BF$tig*K__`h$;W*=yw?tnM&h&adgc$mDGRE`(F zU*wOqy8LbzI5Q4WeWnV0tLx?W?v?}56Zp-0w)BuE^3Ohi_K!Xgcn=hpKM>-$rvdpN z_?&nDx5MLzZIdnUeBQ-Bc_#0HqPfMifA|hw>|c!h83V`X^zXN>j&9lI?fuZJZ|?A? z{uur_4&rOxBX21^Z%zxZ{y#cUml5`{fo6e(Ewh+ z6TYXm`#6q)z%yho>+2BnsTwh8Z2wQbDN?-idv_nxPab;La|~3(p7F14%2D$VAD6t6 zXFdND>{&53ME={Lr?suQ_3I@Xv>JTVE6Cr`@kz=o;dTL!Q#2i zf_v$?b@v0~?^)0Kao&oE?4-nkE0Q{ma|?kIm>8{{8kfS=U!fFXaawz>C;E@&+D* zy~<OqmT|=zcW+{LAG91^CC-4T{Px}6wm_P?ch8foeZAHE>c61;;48HT5Y8khA(fbGS z&QD{O^-=LIx=-2D=OHiTsdr_{^$+A9_~heTsKpXrh2NpG?ZAdOHh?;S?|kb@W$V4I zf8qH5B>2R2mFMaI68nL&S9VYOJoIOFJe9?W9aHD}7r~%+#Rd3IY!9DfEFaoW!2t6W zj(k#J&_9j&ea24l#9NB`Wo(ea7e^A>Jq-$FSBnV?~u9B1{xu+taM)Uoal(M2jKhqJ%R!IM%aV`*hRqs zx_~$t3G_k5l+m)R>xj0*SSwCK`=XAqY_YxMo3@zth%}IQ(8SmRiD>rw9&9T7gGGz+ zZ9Q}jX&7h_T>uVF{{SpVH*sGBs9 zmQ=LX=l@VQ#!OWYfRE z$eeYdPS#zYyrok%XuRNc!`6y@NsAbJE zQ8w#fUFp!)kak9ILJlCbh8swGP?BrN}Du?pBno*tdAN8ebhK< z+%%5*C}N>t#;_pZUdbZ_CIvt)QUK(n<)-E6$Q2ny&bHhcC*oqHK}b4I#F+DOQ4hcT zYyQDl5PcEjVI1N;l;PX)_pMH|cO%ZhQ9I$B_xL~NX+5ED{t%;KUKlAb#K8rF~mXz7*pDq2Q)hsLjs^>!ZZuKsq z+y1tmG~4SnL931>zRVxA%{Wp$ai2j3hPWy6EjZpuwox5^{V_nMS_ zDfY`Cw%89;f0LfgxvZCCS(I5R%H=-3Tt|;@L|Hf5#Z8&;fbXtuvbxW=+a|fA zMDjDw-^5;|VXmn#o^zkOuXXuauY?BBLYjbU$~K*U&GR9v^PCmrna^=OyIme=;QD?f zxi*fE&DbpaOeCV8yxjDBP0Vv<8SXM;{)calr7qGy94QCTmfz}?q_)-SFzqh&tsQh6 z?TNT?A1y_@l&`M!{gW{s!uN3r@hAN|_8yLYebkDWa`dyhT=P2a95e`y)Ft~Hxw{gz zXT(HkX?3gEE4G+>4MR_4-(l>-T-UMZxc5kE(%;LAx7Arj$~s63X+l3i8u2}ETt97f zn)#%B56t!7)N$%9zKfB4fcbrlO8QzK)&0l}wKp4?aO)Cpwf#NzopqIw=-Mo|_T1Jb z`#U|f1<-2Bz-sf_saAJjPZ`n15OeO?n@0NP?W-dD4kq5`i9A(*xuRS$_OE z?FRcG`hlHhj@B~3H)G4l7u&0RONBiH^*4H0Q|g@ap1#&c^WJfIC%gWW{c-w3eWDD$ zJa{yEA-;DopEaMa^PKuen?gP5R z->g+T?L9d&DxZ5Xf=2djpgGWSD!)_fE8i%h&Fb%TxM!^XKF&OsHU8><=6(11KNf!> zO_)r?x8-QG^y!UG`DU4QpX2?Ly~Z{oacMfw>>KDhcbxb(uTs78lZW5I$T~6DNBT(J=8nU+GG*dF4uEX9 z*DCi6_aTjXG4m--NYZDq&GI`Ed`}_!rK@jV%yaX9=KWOXV`tgl!jStbE!=5?|>=KV{?R7do;jEU%9Ub8P+ zh~BS|r!D+6(2n)e<76*n$VBMJ+A{8C%d(_A%T4hBB|c+9|QmA z!_4X+ZKRR3g62T+yuN{M)6NX`TY6OXG6!$6rdjPH%yo`>PhtJNg1**Acb;n=c`u@4 zF(ksj9N#D58SF^_`KpwxthoEs4fYqHpEP5yahbyomR2p3{)^ujFpreab(;1*$L4x} z{Xcy)^Bjk@0IN17_n!~od%5ra2H(va<(tpfc~06uqx4^(S^NKJ1EaBq7(N61Bic>D z-(DXtbs6i#^|d~7O}0M&<%Ca@>O5uf;Eb)Z23+UY>$%PAgy+N&Gzo2_k^4^fs;C{m zbf11|sQ;3^qE~v~9NCYAdy;rd=ij|v_I@LsiGEnm898Y*_R8~EUR*xxqa*twi|#Mk z8!VpVj2|h7Q0e?9xyPowKgGTndVRL+|7om|*4O%|)>cRK-s+^CHikca&RbpoxKZ|@ zWq#MQG1un*yAF+TZ@*V0ZaGF9-)*0o^k4eUhDOxQYNnT zk@tK2=>vyqO|wb=c&?|!Ud8S*5?!0+$~pNZ{oF5&-++KdtOt*qKT$Hc3+qbZN1!j` zzDJZb_Hr@zIdq?&g8dUe*cQJxIr-p^`!W9K-Rn}8;V#p;X5FNLy2Je?Nn7dsg!5OV zKiGXrDAa$^3huQiYZ9>D!oh>DdHlnit&X0UnCtU9Fy~82x_>rl^`A*@Zi3wv$u zb@3rGCmos>y%x~fLyfut9+C7~PuY)=>){mLvVWYE-GAX-q26;-+Q_W<=u_De>#m** z4|#)p#P1JBENMFuCH~L7znkpIgt7)MdJ=XoVZWLUy~&=%8vauukI|6zGVXsG=DIQ_ z?wiFmcJ_KZyC3AazHjHlT4y}NXZtT49qLmNv60QsJm^)i^8MxlZdGq^psfby_FkBEw&^@7!{Nbv_A>o|SDyX!{;bK=HlIgjW{b5`#(YPV0H znY%SEpZhtvW|=5kWP!R&IpkmQYNixknid|q_cjuI?9+g=8u@PANy|I&+}a}brE&rJ;^rW zq2F=p_nTghqhJ2mtX|F;f9iO=yGi*x-}RwR)*a}z=m_Rb?fiVtltwDRoGRZHxqtUf zM*Oc-SW!4-mD*b$NcKuoYy1kU$!y6e#U7_SqAE2ova&a8{WIw`8h|NMjG`} zJpTMXp7Fm@mcw_Es8j0hz3^Yuy)f?S_a$ zYyMxP?eQH2zsvL(L+Yia@$GNI@|_-@N8Nna!#0Sz1Kp;UN}La=pL0`nr#6qhA7_7d z?>`d#PsFXM^Lg+m^)sec(d<&|NB5i%?7ws$-ye91pL6_8tE1JMx%meLj#wr5k27de4O(FraJZhYmLdaoPW0aaP$nsJkxoS9#Ix!Jg{um5xC~DIRA^Z z1iQR)erWI;pM^NzBlD7q2Fwk8=eO0-D;{|_CS5oNF<9(*%EU*~&$>|-%VgQX?h|5g zH!FgE}Hm0fFW6jd1Qc4oJ!KhRW-6d&Dr&w#^4GaE^1KaI84Ttdo0ET$;h7oyC~k_%yX&cq-J9apwGA*Zg$OQ+6Qbd8@Y7 zS})q1iSJEs&3O6Lf8<=Cf7q9k6fQA4ou1lPs6&fp)dK!@qVkeQmx%9Tjys~ z_v`@Y`mb=W2}VTTSnFPvtScUc7C_{&y^lv0z|eNFRW6;ds02hrmpPK`$^hgFawN zo8_YFg-ts{uu0o&!=ftCHhGGL3b-E!UYicc%v6TsN84)77fddabINyu}X zw&crzYr%RSnwZs+p8L8N-+^ku_njt0JJ0bO)Uy&hQ!C2EF*Y0%fHEwbp4Z@=Nh4a( z&MLh$JW78|jhhiam3mY9NYrwgNw#ww4E43BqZNNqm8uQx@wP^A2N;iI@&z;X+u1Jl zi>Qn5S2jAep%s5#g>U;A$3{Ik{K|Rj`l5SIj70ZFMq)_OZ9n`;AJ)*in`K1zuJSxD z*wB{lwVg+LUcDFJSr`=`kva|}!Rc7AF0vjOoJ~+t~(e!bkP#%i03)nM8+d=$WC_fmS#^&u^ggTo|>k6SY`{@13`m zi7zi0klOR74hEo^><~W)Hd6U;i{2iv&BNbEts3VGbMza5ap~v4wq70?=J6VuQ20O4 zk|xW4h)WW$Ib1;vflg~t(w5?b4 zj>|DnXGH%2YtiI6rF3DT_$Y1{gDpLPNMQo4wFlj5=ZLou;;4J(IP&e@~99@%##whAsWk%~t=$cHCtLtDq}@ zP6pSB@mJus+nf5Mt*8Hvw&A;CNYRg>ab#>=#s68x${Sc2nl1EQtG~DP&+0r7M{ESn zVXTp{Vu@2TR&CBaU*u3mg7SoC}87n8zS^ z6BrD2JZst=k6YPRYkl!1FYr_g&IQjEV&KO7E6@W0yQ@d=}nTELyxGtbS@@`6e5>0ytF|nMqs|XSeA#fB%FsT`ciD{;b zD$lyKL0hy*+r(iE4|$Ok4lB~};~I-={tNkdj|cUV9|!fa?lDQoYfHZhy6&nbtaDzn zax=9jT0_82ljLKKaje8ENW>t-SlFjMxi6G<<3c+k%(^dATXCdY@C|sxYU`j}$#^_42blkoId5rr;2(nrPQVMS<65vbUtiu9l)B)*$bqS};)4t9yX5cx zQs`G}yV@`B(Wegt-Q|(NWL)WfbKcy)I4>@qVH=N1z6o{Rl_$r9n{?m@eVB`o?-jZoJe2A29}QtHph?k( zSR8@!{n3YEpQoZTl%7zR_0^&++K}&g{%|m(_S7J>3}cdzhwY&Dz-!a~zgH2X4kM1; PjBbbNT8k>jI;{IY2~2bs literal 0 HcmV?d00001 diff --git a/app/assets/images/ci_favicons/favicon_status_manual_with_auto_play.png b/app/assets/images/ci_favicons/favicon_status_manual_with_auto_play.png new file mode 100644 index 0000000000000000000000000000000000000000..3ca612a542dc4e5585ed2d829bbc651409bde3b7 GIT binary patch literal 1338 zcmV-A1;zS_P)+}A6Ki|(o@Q{r>44E_Jm1~j?ftjA z=7AMpjFI*0*Y9GC6*0y(0xRznQ5661`Ft%povz~AwQHlRfjkg^$z*z(5Yj>^-53=W z<;%#(5EBv-U^E&L92|_v$w|nvj5BA>puN4FQc699kWyKetL}!lZveB|{4ycro1mZ| zeMw1)n4O&sK@je#^2o>tDk>^iPfrgagfv+!mR!5t?pt!WF94poAP5%?27|u7zFv%u zj#fQNs;jF}TU*Ovt7Tb!=e_~{srdz+P8ZkO+PZ4+si`T<&(9+)EDTz$b_Lvl0|(I3 z(!wFqWm!JWH>g&?Y&KgdrC%I6bO<>)IRP)9ot=eVuSZ5k2AobOs;a86b?a70k_3yz z;@|7>c(7&57N6Vgc3CWzSY8U%049^^LS$s*i|y_0;_{l8m6f5Zs|#^)amt~ilq%VW zhlgWqYz#FuHAqfQ_CuUHbqdAB#Sld?O_HQGRRFxt8Dpb1n~f9{6!>2@Gc$vfloW93 z?`21h98r{6GG3zi_;|XwxLD_Mx%Q|6Fq_R;l+xDb=4K_+<;H-P zQj{uy+SgxsJ2fn{MneeLy2nB1Zb9@c2Go^9ynk~J6Z%l}_4O%b;bynn16q-<@m3rj z9`1HJoyn>I+}5=32@w*xbMWRgXkXimndU~|=gT0B`9Utu%wga48v$U;&jx=mI5?Y+$&^r8uYvOfKiIsOFv=m%O(i!QRPOv?eih!azAV~8xj)} z;cz&xW53(oeqZj zmKA~dg4^xJ_U+q!)6>&q0A7$~S-HK`0a&+nfsiEb5cFyiSXemb+rEZy=XOA_H5+0P zy1#PO%=hly%Y)C0qL|=tIIb>jQUx$>O-W)z^DTfij0gbrfsjB~qS0X8yJcAO-1B|_ ze&Fir>iDI@2q8B_QQWpXxM~IX#&K&}AtSG9eSLktp`jr@ zxV)6oV}c-*FUh{#UzGw&4&&DJBnGx01n~e_3$QQ w6O_{HjIlPYR{Iq{?W-OPwP)a7&i}>wUoM&|Kb2`CjQ{`u07*qoM6N<$f>P0mY5)KL literal 0 HcmV?d00001 diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3dadb95443a..c2459b3f5f2 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -22,6 +22,7 @@ class Build < CommitStatus }.freeze has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' + has_one :build_schedule, class_name: 'Ci::BuildSchedule', foreign_key: :build_id has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id @@ -184,6 +185,12 @@ def retry(build, current_user) end end + after_transition any => [:manual] do |build| + build.run_after_commit do + build.schedule_delayed_execution + end + end + before_transition any => [:failed] do |build| next unless build.project next if build.retries_max.zero? @@ -229,6 +236,20 @@ def playable? action? && (manual? || retryable?) end + def autoplay? + manual? && options[:autoplay_in].present? + end + + def autoplay_at + ChronicDuration.parse(options[:autoplay_in])&.seconds&.from_now + end + + def schedule_delayed_execution + return unless autoplay? + + create_build_schedule!(execute_at: autoplay_at) + end + def action? self.when == 'manual' end diff --git a/app/models/ci/build_schedule.rb b/app/models/ci/build_schedule.rb new file mode 100644 index 00000000000..7f0a34b246d --- /dev/null +++ b/app/models/ci/build_schedule.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + class BuildSchedule < ActiveRecord::Base + extend Gitlab::Ci::Model + include Importable + include AfterCommitQueue + + belongs_to :build + + after_create :schedule, unless: :importing? + + def execute_in + self.execute_at - Time.now + end + + private + + def schedule + run_after_commit do + Ci::BuildScheduleWorker.perform_at(self.execute_at, self.build_id) + end + end + end +end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index b3960cbad1a..e2700db1438 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -92,7 +92,8 @@ def all_state_names scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } scope :cancelable, -> do - where(status: [:running, :pending, :created]) + where("status IN ('running', 'pending', 'created') OR " \ + "(status = 'manual' AND EXISTS (select 1 from ci_build_schedules where ci_builds.id = ci_build_schedules.build_id))") end end diff --git a/app/views/shared/icons/_icon_status_manual_with_auto_play.svg b/app/views/shared/icons/_icon_status_manual_with_auto_play.svg new file mode 100644 index 00000000000..a08c43b156f --- /dev/null +++ b/app/views/shared/icons/_icon_status_manual_with_auto_play.svg @@ -0,0 +1 @@ +Anchor-with-border \ No newline at end of file diff --git a/app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg b/app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg new file mode 100644 index 00000000000..a08c43b156f --- /dev/null +++ b/app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg @@ -0,0 +1 @@ +Anchor-with-border \ No newline at end of file diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 1eeb972cee9..b5a492122a3 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -60,6 +60,7 @@ - pipeline_default:build_trace_sections - pipeline_default:pipeline_metrics - pipeline_default:pipeline_notification +- pipeline_default:ci_build_schedule - pipeline_hooks:build_hooks - pipeline_hooks:pipeline_hooks - pipeline_processing:build_finished diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index 51cbbe8882e..889384d6be8 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -9,6 +9,7 @@ class BuildFinishedWorker # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) Ci::Build.find_by(id: build_id).try do |build| + build&.build_schedule&.delete # We execute that in sync as this access the files in order to access local file, and reduce IO BuildTraceSectionsWorker.new.perform(build.id) BuildCoverageWorker.new.perform(build.id) diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb new file mode 100644 index 00000000000..448fb5bf41e --- /dev/null +++ b/app/workers/ci/build_schedule_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + class BuildScheduleWorker + include ApplicationWorker + include PipelineQueue + + def perform(build_id) + ::Ci::Build.preload(:build_schedule).find_by(id: build_id).try do |build| + break unless build.build_schedule.present? + + Ci::PlayBuildService.new(build.project, build.user).execute(build) + end + end + end +end diff --git a/db/migrate/20180913102839_create_build_schedules.rb b/db/migrate/20180913102839_create_build_schedules.rb new file mode 100644 index 00000000000..1e9d9a70b0f --- /dev/null +++ b/db/migrate/20180913102839_create_build_schedules.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class CreateBuildSchedules < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def change + create_table :ci_build_schedules, id: :bigserial do |t| + t.integer :build_id, null: false + t.datetime :execute_at, null: false + + t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade + t.index :build_id, unique: true + end + end +end diff --git a/db/schema.rb b/db/schema.rb index b3d4badaf82..581496d78ce 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -260,6 +260,13 @@ add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree + create_table "ci_build_schedules", id: :bigserial, force: :cascade do |t| + t.integer "build_id", null: false + t.datetime "execute_at", null: false + end + + add_index "ci_build_schedules", ["build_id"], name: "index_ci_build_schedules_on_build_id", unique: true, using: :btree + create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t| t.integer "build_id", null: false t.integer "chunk_index", null: false @@ -2288,6 +2295,7 @@ add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade add_foreign_key "chat_teams", "namespaces", on_delete: :cascade + add_foreign_key "ci_build_schedules", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 016a896bde5..4376eb91a73 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -10,7 +10,7 @@ class Job < Node include Attributable ALLOWED_KEYS = %i[tags script only except type image services - allow_failure type stage when artifacts cache + allow_failure type stage when autoplay_in artifacts cache dependencies before_script after_script variables environment coverage retry extends].freeze @@ -34,6 +34,14 @@ class Job < Node validates :dependencies, array_of_strings: true validates :extends, type: String + + with_options if: :manual_action? do + validates :autoplay_in, duration: true, allow_nil: true + end + + with_options unless: :manual_action? do + validates :autoplay_in, presence: false + end end end @@ -84,7 +92,7 @@ class Job < Node :artifacts, :commands, :environment, :coverage, :retry attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :extends + :retry, :extends, :autoplay_in def compose!(deps = nil) super do diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 2b26ebb45a1..e1b40472fc5 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -5,6 +5,7 @@ module Build class Factory < Status::Factory def self.extended_statuses [[Status::Build::Erased, + Status::Build::ManualWithAutoPlay, Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, diff --git a/lib/gitlab/ci/status/build/manual_with_auto_play.rb b/lib/gitlab/ci/status/build/manual_with_auto_play.rb new file mode 100644 index 00000000000..f34f0be5d45 --- /dev/null +++ b/lib/gitlab/ci/status/build/manual_with_auto_play.rb @@ -0,0 +1,52 @@ +module Gitlab + module Ci + module Status + module Build + class ManualWithAutoPlay < Status::Extended + ### + # TODO: Those are random values. We have to fix accoding to the UX review + ### + + ### + # Core override + ### + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|scheduled') + end + + def icon + 'timer' + end + + def favicon + 'favicon_status_manual_with_auto_play' + end + + ### + # Extension override + ### + def illustration + { + image: 'illustrations/canceled-job_empty.svg', + size: 'svg-394', + title: _('This job is a scheduled job with manual actions!'), + content: _('auto playyyyyyyyyyyyyy! This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + } + end + + def status_tooltip + @status.status_tooltip + " (scheulded) : Execute in #{subject.build_schedule.execute_in.round} sec" + end + + def self.matches?(build, user) + build.autoplay? && !build.canceled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 5d1864ae9e2..5277b69a628 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -49,7 +49,8 @@ def build_attributes(name) script: job[:script], after_script: job[:after_script], environment: job[:environment], - retry: job[:retry] + retry: job[:retry], + autoplay_in: job[:autoplay_in], }.compact } end -- GitLab From c03631a99618d43c66a1b9d0b4303d7253e45866 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 17 Sep 2018 20:00:29 +0900 Subject: [PATCH 002/118] Support new syntax --- app/models/ci/build.rb | 15 ++++++++------- app/services/ci/process_pipeline_service.rb | 2 +- lib/gitlab/ci/config/entry/job.rb | 12 ++++++------ .../{manual_with_auto_play.rb => delayed.rb} | 4 ++-- lib/gitlab/ci/status/build/factory.rb | 2 +- lib/gitlab/ci/yaml_processor.rb | 2 +- 6 files changed, 19 insertions(+), 18 deletions(-) rename lib/gitlab/ci/status/build/{manual_with_auto_play.rb => delayed.rb} (92%) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index c2459b3f5f2..be4a6c553e1 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -186,6 +186,7 @@ def retry(build, current_user) end after_transition any => [:manual] do |build| + puts "#{self.class.name} - #{__callee__}: 1" build.run_after_commit do build.schedule_delayed_execution end @@ -236,22 +237,22 @@ def playable? action? && (manual? || retryable?) end - def autoplay? - manual? && options[:autoplay_in].present? + def delayed? + manual? && options[:start_in].present? end - def autoplay_at - ChronicDuration.parse(options[:autoplay_in])&.seconds&.from_now + def execute_at + ChronicDuration.parse(options[:start_in])&.seconds&.from_now end def schedule_delayed_execution - return unless autoplay? + return unless delayed? - create_build_schedule!(execute_at: autoplay_at) + create_build_schedule!(execute_at: execute_at) end def action? - self.when == 'manual' + self.when == 'manual' || self.when == 'delayed' end # rubocop: disable CodeReuse/ServiceClass diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 69341a6c263..323075d404b 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -53,7 +53,7 @@ def valid_statuses_for_when(value) %w[failed] when 'always' %w[success failed skipped] - when 'manual' + when 'manual', 'delayed' %w[success skipped] else [] diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 4376eb91a73..fa64041f7db 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -10,7 +10,7 @@ class Job < Node include Attributable ALLOWED_KEYS = %i[tags script only except type image services - allow_failure type stage when autoplay_in artifacts cache + allow_failure type stage when start_in artifacts cache dependencies before_script after_script variables environment coverage retry extends].freeze @@ -28,7 +28,7 @@ class Job < Node greater_than_or_equal_to: 0, less_than_or_equal_to: 2 } validates :when, - inclusion: { in: %w[on_success on_failure always manual], + inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ 'always or manual' } @@ -36,11 +36,11 @@ class Job < Node validates :extends, type: String with_options if: :manual_action? do - validates :autoplay_in, duration: true, allow_nil: true + validates :start_in, duration: true, allow_nil: true end with_options unless: :manual_action? do - validates :autoplay_in, presence: false + validates :start_in, presence: false end end end @@ -92,7 +92,7 @@ class Job < Node :artifacts, :commands, :environment, :coverage, :retry attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :extends, :autoplay_in + :retry, :extends, :start_in def compose!(deps = nil) super do @@ -119,7 +119,7 @@ def commands end def manual_action? - self.when == 'manual' + self.when == 'manual' || self.when == 'delayed' end def ignored? diff --git a/lib/gitlab/ci/status/build/manual_with_auto_play.rb b/lib/gitlab/ci/status/build/delayed.rb similarity index 92% rename from lib/gitlab/ci/status/build/manual_with_auto_play.rb rename to lib/gitlab/ci/status/build/delayed.rb index f34f0be5d45..553d4cf8a71 100644 --- a/lib/gitlab/ci/status/build/manual_with_auto_play.rb +++ b/lib/gitlab/ci/status/build/delayed.rb @@ -2,7 +2,7 @@ module Gitlab module Ci module Status module Build - class ManualWithAutoPlay < Status::Extended + class Delayed < Status::Extended ### # TODO: Those are random values. We have to fix accoding to the UX review ### @@ -43,7 +43,7 @@ def status_tooltip end def self.matches?(build, user) - build.autoplay? && !build.canceled? + build.delayed? && !build.canceled? end end end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index e1b40472fc5..0fbab6e7673 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -5,7 +5,7 @@ module Build class Factory < Status::Factory def self.extended_statuses [[Status::Build::Erased, - Status::Build::ManualWithAutoPlay, + Status::Build::Delayed, Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 5277b69a628..1dc6c28d24a 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -50,7 +50,7 @@ def build_attributes(name) after_script: job[:after_script], environment: job[:environment], retry: job[:retry], - autoplay_in: job[:autoplay_in], + start_in: job[:start_in], }.compact } end -- GitLab From 8720edd45acc50473cb080c18c4b259079674e0a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 18 Sep 2018 14:20:59 +0900 Subject: [PATCH 003/118] Add debug script --- scheduled_job_fixture.rb | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 scheduled_job_fixture.rb diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb new file mode 100644 index 00000000000..55f50e941a4 --- /dev/null +++ b/scheduled_job_fixture.rb @@ -0,0 +1,98 @@ +## +# This is a debug script to reproduce specific scenarios for scheduled jobs (https://gitlab.com/gitlab-org/gitlab-ce/issues/51352) +# By using this script, you don't need to setup GitLab runner. +# This script is specifically made for FE/UX engineers. They can quickly check how scheduled jobs behave. +# +# *** THIS IS NOT TO BE MERGED *** +# +# ### How to use ### +# +# ### Prerequisite +# 1. Create a project +# 1. Create a .gitlab-ci.yml with the following content +# +# ``` +# stages: +# - build +# - test +# - production +# - rollout 10% +# - rollout 50% +# - rollout 100% +# - cleanup +# +# build: +# stage: build +# script: sleep 1s +# +# test: +# stage: test +# script: sleep 3s +# +# rollout 10%: +# stage: rollout 10% +# script: date +# when: delayed +# start_in: 10 seconds +# allow_failure: false +# +# rollout 50%: +# stage: rollout 50% +# script: date +# when: delayed +# start_in: 10 seconds +# allow_failure: false +# +# rollout 100%: +# stage: rollout 100% +# script: date +# when: delayed +# start_in: 10 seconds +# allow_failure: false +# +# cleanup: +# stage: cleanup +# script: date +# ``` +# +# ### How to load this script +# +# ``` +# bundle exec rails console # Login to rails console +# require '/path/to/scheduled_job_fixture.rb' # Load this script +# ``` +# +# ### Reproduce the scenario A) ~ Succeccfull timed incremantal rollout ~ +# +# ```` +# ScheduledJobFixture.new(29, 1).create_pipeline('master') +# ScheduledJobFixture.new(29, 1).finish_stage_until('test') # Succeed 'build' and 'test' jobs. 'rollout 10%' job will be scheduled. See the pipeline page +# ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 10%') # Succeed `rollout 10%` job. 'rollout 50%' job will be scheduled. +# ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 50%') # Succeed `rollout 50%` job. 'rollout 100%' job will be scheduled. +# ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 100%') # Succeed `rollout 100%` job. 'cleanup' job will be scheduled. +# ScheduledJobFixture.new(29, 1).finish_stage_until('cleanup') # Succeed `cleanup` job. The pipeline becomes green. +# ``` +class ScheduledJobFixture + attr_reader :project + attr_reader :user + + def initialize(project_id, user_id) + @project = Project.find_by_id(project_id) + @user = User.find_by_id(user_id) + end + + def create_pipeline(ref) + Ci::CreatePipelineService.new(project, user, ref: ref).execute(:web) + end + + def finish_stage_until(stage_name) + pipeline = Ci::Pipeline.last + pipeline.stages.order(:id).each do |stage| + stage.builds.map(&:success) + stage.update_status + pipeline.update_status + + return if stage.name == stage_name + end + end +end -- GitLab From a5d296e9be7cc48ddc75d04b117ae62bae7c9f5b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 18 Sep 2018 14:45:17 +0900 Subject: [PATCH 004/118] replace images --- .../favicon_status_manual_with_auto_play.ico | Bin 179677 -> 0 bytes .../canary/favicon_status_scheduled.ico | Bin 0 -> 5430 bytes .../favicon_status_manual_with_auto_play.png | Bin 1338 -> 0 bytes .../ci_favicons/favicon_status_scheduled.png | Bin 0 -> 1072 bytes .../_icon_status_manual_with_auto_play.svg | 1 - ...status_manual_with_auto_play_borderless.svg | 1 - .../shared/icons/_icon_status_scheduled.svg | 1 + .../_icon_status_scheduled_borderless.svg | 1 + 8 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 app/assets/images/ci_favicons/canary/favicon_status_manual_with_auto_play.ico create mode 100644 app/assets/images/ci_favicons/canary/favicon_status_scheduled.ico delete mode 100644 app/assets/images/ci_favicons/favicon_status_manual_with_auto_play.png create mode 100644 app/assets/images/ci_favicons/favicon_status_scheduled.png delete mode 100644 app/views/shared/icons/_icon_status_manual_with_auto_play.svg delete mode 100644 app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg create mode 100644 app/views/shared/icons/_icon_status_scheduled.svg create mode 100644 app/views/shared/icons/_icon_status_scheduled_borderless.svg diff --git a/app/assets/images/ci_favicons/canary/favicon_status_manual_with_auto_play.ico b/app/assets/images/ci_favicons/canary/favicon_status_manual_with_auto_play.ico deleted file mode 100644 index d8528e5d0e47052a62069192005ab46e5976956d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 179677 zcmXtg1y~f{_x|iI%_2)9BAp^7NQlzi-Hmih_ky$_h%`!fqaqDUiGp-VBZz=>H~Swx zzwe*t*=HGM?wNbeea|`XIdca9fB?9_-46ty1EORAAQk)_iTuA~HXI1>P!<5p&HwMX zG#LP7R3U)8{Qn(CYXiW}3Iq@r|KIhJ0N^DE1$cP=cRe!zeDs3?baekaMu6jUpa3Q1 z|Hgy2-l>5C2=MQB_qW3U$2UQN7HG>%RLy`dmw20-yv1X$fuL4?7nb@c+oF;w(S<)M!k~*oXsR zMnx(q(+Sh#K0f#N3o_$gW~hCX&+r)UO@Ds%o85}<5A!9M!sq32zT;DRHL<;6<4$8j zBmoLT*2OIq|85FeY%dpIc>Mj4En3o%qc-fRH?1tTU^1+w_CjY;MLp4x#B)k70Y}Jw_#I6?lcbH5ch{6GVCSI3$?8kC^k$e9iN&7;~q0g0z(L?smzQK z4yEg9&I{MaAIHIoY4084VlPPGEW(&(F8p$+AktCWUWD+Kg4W0{Gbk{Rm-1c!7vUw%!NdrXOi z2jl5_{wjNF5c917{AUpcD^DyVr9_MdvMe=8D+2i_bp>AvDL+8wle`GSHBpgwLMBS} zhu=uoWzr-8+V#WTq=;f`>PGrbOcZmu@v=SrL2a=mS&{GIxzTYuy`u355CNft2yl(t zyn0I#fntz#e_W8>!h4Q_xTisg*-ra&o}E?31jnWg#J&qx4P`lC}E`WQ;xp$ zjyqmzSs%|`;?P|0_Wx}3kQ1*Hou4Tzm0ywWq(d!q)w>T6$|gt;6oT>(R82Fsg|fGt z#C6=vVa=XYpA;)M0o?AO6gEnk5HZR*euFk*S8%qf>h75pcD}hyQFxfdK#`;X+`5)P3mHG zaZW9sh%mOE^35ib@fJtFoRz*!w_I7Z(GoGuo&d-b7rsTMMAJPa+gDz7wnFw!6-WVt zIUWk(n~J!O)0lT(*^zOT&KkJor@!ch%cM4#$|H*DVa6gJi$lZp3}x$7ZYyMwft+4S+^7~*llI!@^9pj zRh$+jlGN#$o>n7Dp*Ti~EOv(Kb$)7Di3vB{f*^rWqNs-5#h74kd{^B%q0y);0LxHD zzent42P^wB=J)g<`pq$Gy3)zEjkgq(WRQ>D{Fnk|c-7YMtpv~_;~$3fi+KQEP&~&2 zmcO{6O8k)qk1!|h;ptjVAt|IJGoj+oid^M?-+W_Wc#-_Oxx(H!f^BtKN!}T;5(feD z)zv4d=z1%tzvUw7fsa;vMUARj?n?&^8!4no2?}^HL|3s#{k`rU1IcC0_}@4xjOqia zS+4b4GF8rd8rO(3QN_(wQc;bWf;cJ3_Tm`zYbNHil0HnFHar?jcvmJaR}GR)_XCor z!p~eA!_!1^-8WOyxPtwB?V#b`Ya-&e26(V#%kid6hxBxAjP`t9YJ;)LU6FExJQtm( zqpY%~bB^NXdj>SUy?qOekG`ChjvrDMfMFRgk*+6>Qz|P?S^tepTc|vh-Zxg$xQe~; z?A+*0HZ01ULpGz%rOrDkxlz&TJXh}E<7r!5>5deE#Cye@@)quJ^0-SSCB6a}Nyv0A zZ*&|%V}rsA)nz`61d_0U_4%S2wZvd$p1%aB$fDWb#W5Ayv3!b;N@ssKRJV`#<*!6d zp1xBQBW+j;`dRugyO4Zoc7G)JTsJmUax)7?ze!{&QAj_c-)7H)(s)V-54|G8Oba>p zzR94pmwRC~1%RuSrCGM3gxAfg1YgK}IR9Q!as!HZVmu0)qpuIbD^i;qVzr3L3;DoH zWSC_~+9iv{a_wBHw>-Q`2~`w-iYHlRo+rwtzP7<99Bj<>J9DPtpGdx{Gyk|vk=^-> zXLa#EmQ;zAe&X(*nymG*+X_kt%&y7@-40i^?ICKK(g9-HGY@o*iJW1DKla9z&mI0H zqVGM$dh&c>kBsR)wCk}HAZe~ETn>Zkyw#I%lTqk&w&E1!DTd#f!=gB5&mPX2qKM&; z5z3<<>nc!oKKSTHowVI#&#>I3D;X~Da=J8FPCOC#SmE`Lg{v6FY>m14g$q-e9uYZNMWKGmLMHa*t8391;qQC( z|NaUX9h6+UfBr_-#bNgyaUKWNVJ}teRKaaqlJ4y>#WgN@)EInPbt&GJ@h3^c9uRmu zdXHDgTgd@@nqEgS=6BwU&gOF|L;hJJtNJuv{$&+uyv zjg3($4^s;QVRV79>Csop2W*i$(lVkN2JniLzp7pA$s*>IX1r&|8dEM+ zY zN`v`Aqwy8C8!*3=mn(}j&AU6ZB+`2jyRHG5x=WL;snjHaf)iO2nloxhe92n zV(JaIbLI?Ff43$I^vw`x((EPd$MYXx=IGy3UUX0HAgd34>E_B-AH-F2j+0cJk@$@x z8jt*5?}OTSBwy3_4MB}EMQ!DI6RzKUet{YeyRoa?17BcX4+jSDY*Iw-ClkkeUOXs2 zF{(U|7;8W>S&c|aCH6nSu_sPHX+-@^ij4RSW_DFC7iY=p*uTxDJCwj#GTF~<%94q1 z2i<;$F5v??%~Lf^SM0_&`Cn;;GY{|>GO^t2LrOKG?TJxObIeC#@mulCM?bWjuu=)N zDF09)GMTOzWJHB{m!=#4(-minzt>dM5(1vm}dLmhHr zBw!-U`v<~4TW3$*Y%<=x$#4FucSXtAm{%bE&HkZot(vy4b4dCjVbhe_{do>jF<7QO zRX9I><^h&uJPdHa#y*jF+gUz8z|8ghV)vAmf8tced%<#t0n2z?gJn%K4a#0g-{!t^ zvrpe1v6Api7KOlwg!lDe>Su(ihdFe_bjDV?Eh~BKj&)3*$HY@Vc7&|xH|Sq|l5|tC z&2T8or^bA9c>4G7yo z51H|*-WzE7E^z;@b)V;j>ZeZ6>{ZS!C7uo@0nsOq*jd$_AJWfN7uE)0?hv}qu3Lym zwLz4wV!mV(uT32H=yZ$7l(nmhe-MHiUwJ{en7*iD$(z?QaM4yOLY5t?gj=RYbZJnt zraJ?VIt8EghufLJTOR?YH5XN`D^)v0pIkyyW-UqgghV5rN(S=Nj*KlKV9ut-Bttts`$zO&SgZT*8)~ z;(IK0nM5r&iVpfgqipMfu~^Ib&d1_hQ_hu{-mq(jDcu7wcuog&MW=zcen(!pxpor~UPwbdxse&^Guf zo@_Xq)CiPu`QTn)8^#bHc0~~N4Ox>Ou%4ZC5Z$T#<~kE}Q3m&rlE@!x?k5e0;g33$ zMpsjowhpgywn-H7uC3lAFCsgPM#gn0tua3A@i3Zqd@TKxW+mRIs^IpG{r=FyT!V+c ztC~CF^7B^lI}MiaA`U6yDA@<3YT+_)K0@{a3v2xiT|%j{tURgSm-6$U-OIeqIaJH; zKj8%)q&d8=c$Vr@nr^UbL)R0Wh>h4i71_){)y21>*+1mj1`e{V=dQ-(DICvScGXAco(`Rm(i&y{oxSd9*s)}q zy$ZKLGo$ZKJuQm}dG`GHMF&lbzzvTh?`(qXD%3s;#>)#?-;}%w5XK^yH<@y~jcG{O zejgzlNSA1_!eNcsW*v`Y189FX9m0@*HqcuIzd|-czMLl*shurlhqtfON_t&|*m5cc zNFZD=0yo2RPj3NvsN_rMEP3%~)}yAZwTC1t5Ba5;P!-gbuk$L2LAEQqMyN}j6!yhppKfZREvi!ozKA_w^ z2vN)Meu`aq^2@*eO`b7BE=F4Q=49%Z-SDLQb?=Z{l!f2r9NZBshg)6KJkh?Z{|{6OkVmOkB!E>%;HYimdFyn zFGw~%)cOQm!z^+th#q-B85$SjFxQ63L$7JUFXxm=|s7#rEeTs2``GrJm z%utPZoXd+jWnt}L0O2oYdezI!7_!Xr>r9GPt}pcSC-QGRJ|L?D`yIK6Sb zOW5)OeUG1%kjVl+WC7x39Z1!iyh0=*(Y3~`l4fl7{lOC_Ec0&Ss&RgIErYtbo8nrV zvkNt9iM6vv6=AL5DCdrIR-865M_%I1Uub#d%$v>1Ub~P;JB;URg65>gvPdPI@weD) z>6Yn@_N&BIpCfEnEPla}F8yx?@G?yP!>AQw^0gUq>C!6AD(@-P7O(xMdsf!DhC-)My)wNgdO##>=T^LDspJ~c zl{BY(0Rg&trbNUX2MJs~SX%d-JkdKpz?lpSn;}9 zoxM@PzM0j(cxZYxnPzdi*KZgnt*Esa!|u_Z4$7miBB~wmv{N6s-S2~a@X~P4gaVKz z-y6%5THP4JFK%r6kskayf^Lo_iuX9Zp2%wf)bFZ8JX=CM*~I|)ARus=GfwRKxKc); z@aSQm`Pp9T>$8rGdbSJt$_NVHm%nBxD7m&T2-Et^)i_>0g24@{DTV%!?CI>D&wMui zB6*zZ2Q;&%*~fh-vs?NTJ={|=m(trd;3#XV%?}W3AUWF4BG5@wg&{DNx7-^ zF*0MY<+~m5?5IM&eOG9vPpelvPNhT=te0v<_oOgms*6+^1iJ)gt9!PH>6D7q$UC)d z9VO0xH%yfN_VAEM`9r;sBd;tgll61d8J`B3y2yP+JRpWcFuOq2K0=6-7f5;3uC4i9 zx$3o~8wAI7ht}>$h~w56a3I}BKb394%76Kf=5t-E(ph4l%Pdx z+f&eQG`^{I2ipcSg0f_!BqV&v#8kAzDRQYoi;39aImSyBG@{z?Kw)qay%f^1s?@Tg z>)qwZE0KsTvDp+_K)RlGE_I!8tbNZtN4gIpXU!?wF7J$yX@@62fTM2(lW%ZYDbI@) zVS?3OI-M}ZzvEL>T^^>++@c*|QxA@Mp|&Q zk981T@%^|-{K@x`3=ZR#ym~&5XkkxH`-ulo#fc$wygapV^mo z^Libkezpr3Z1RF?iVx9hj_GcM%V#~+NpwWKDW zOyG{!t#|AW8YW*8oobGuvZA6LUoeGm#!s9oaOEXahSO_2J=j&F?sRFtNa%9XZY_Qe zET9ai808$8Emb^m@4x>e7CL*%rn@2guK2C9k4gHQACe>>OwZ_PhYaIJ)v<6 zIPK%dw}J?n0mHb!--O$k0%0*Ou1#NArTfCUa+BHkvE*Zv1RuKuvgmbVi3{?oX?-Uu zm!*q{<56=7ZNuA%f%UMTQ-TXRt#{3LjxxGi4*P#CJZpafJuSYH@mdRert@4Y(jHjh z%G2jLU}Q?ZEzL}yF0~`LkBlO(h)ryiwnF9?Ro*z-u2z;fT_w=}Y!lhP4AOhaDf_7S z!zyC&*M1f^!IONfLbvE)! zlfq*Ep@2s)I*e31Qj!VcP=09>0clG9G~;?X+8?tQ6Gz{pcX->kZ*REk&JrBEs-27^ z*t+99TrkA>I&7MZ5bZltL}ymr%cefn9eYMpP=^@tf^?LyW}D6PQQ-MRMY-e(Gyv+{Qk1JUsaH_U5k%uJ!yxW$$-pi)V=} zuo~KG$k~)z16&|e>DTO{YejqYNt!${)th@ZCn^Y806c>9cgMo)1wyhuVqI?bw zCcc>3p_pHLm9C(d-O5+s_pD=vW6Xb~?n+cwBz*+)gykp_(@)(gq-sOzLFPc)X&)s$ zc^ljrI}2jK;nN8Aki=Pw+8uV2{$Tn(xojDV8K(Qi=@|iyv)qmBb=cM(*N3S;b|S4v`G&SC%CG5dQXp!vW2*5fBXXc5UYX12uhkE|`gWU*^dwPIV^u^?Czri=-u19fPS1pbSUlZYE&;Y>=ky1UgCGhjz-czHy@CHXwL2- zl(Uc2Ko?b_8v7Uy@k~-um(ArtDt(~!nU{yv+T{5sO1f1_hWdA4gmKA`& z=72wD#{0vJ=6N&SommOj7%*;EFi=UCPK9Rof~r__K7_}WMmu(jYe}*bmBOvLcm)4( zGZG*X`jwgkH&Kqkip}gDG=8sV@E(i`C0Wfq%o*r=nb&cJ9T&0}2CgiHg31Qs=FsWV z%6)#!Bn4_rclcB-56gUT=AUxJvYjljE@90z%n-=*{>v+wc!!^XeY)FSW6q;X z+e6YOWcFRpiBewp?obe@Tnxo2uwJ_+pNTPm(WDy@7MkY{(c=@xd5kCW5(PZ}BEMsU z(f5S)l6H2OcJa)N=gIbBpx~)!1DNxP>6JUEelDNB+UN{k+aLcxFZpk9YP7rs=8#DJR3nJt3^Sx(1xC*ca^*9472^C>#8} z^4o$NVJKFvgE?0Lpn-#tcJd1fDGcIKgNHS)Aa2j49biyrP@HB`HrwY0QOH5;#lqgdJ7(1X7$%OZx{vO$0;WC`o!&j`gU~TaVxs{9 zIW(+`>W{%3MVGLi&?$ZDU5cS)K~Qjcnfpp@VNv91qYfsp0k%pxnM9~>F} zJXRyQ5<8^l``W4SvLWF)BhgiCk+srN&eZD8;92eheYp9zqz4kEcRKWjMSGnIgk5ww zMe$cfUGT&b(>)-Y0k^te~t>&*BvCMBIy`D4vaWiyOJHMpT!ZGqZv+1jj zLc?CyT+@Rf$TP&Y`Jkmt?vOgt1i5`q&mImW196Gnw1$OKrW@*fKyhLeUo6)*^AmJS z0h+g%S^I4lBJO#Tm69q8e!SQVj+C#c1C-JS-dvA^Bto0ojQ0=a0koF~J+4YnKk)%V<=pYU% zgw-c~Q+I$#CPI#sb|@fnYw?^zDd%514;=9t*d0O`N>~5?UI01KH?M{*K-`FAjz5K3 z4&s8tCKQT*+Tgrd8C>CTL^#%cXrD&|`bGp_DpA-KeX4qVa~63=0eHyxQg4kk0t%U> zr}I|;rkCBLu8QC32|2Y}fH@%Dm_`}*8JjvSKsrg7VUrsczMf*<`{Zz7N`&ht7_6#& zXx;>&Z?AvW=*@A7A4+*&Z*k49x)s{9KWEUDFI#@mN&W7uXLhZ;6et%l z1rvsJ@~%R9@YG!;%CYhn+PleJ&w?51L5(#*fXa*0*TQ~48%FxsB$0ir)O_wE1-KHA zPK8gOd$w^L=y+pFjR`x(iZP}R!M;N1FtCe)Qq2Cr$iL-s{<`XW`s=2}>@FIFt8>uF zr&P+=i)BtUmFCQJqFy7LAwo!EBzvJiBHej@=<|R|7mi0HwDea&Q3+In~(x0p! z8N3-y^m@p!i-)wAtHc=q0WC~ZI9d0%VFv^7&5qJ)c(HP>c3-O$tw zE7KolBvhH_eoL5vu@%B z$3UDRM&Hz&7(TnqR1(vZ-)%=cx6$LD$qRPu%UMy?tIi*DROxx+iIL^Ja*9-eiC=2; zXD#ph9w@Y4$b$~TdCOHgn{n~fF$n`a+j4r-zn0`bLY;{)jA2 zsHW!JL6ikwn1+jzT(O&eD2U30f+0OTu?|ZfnwM}vjx!s(;=a7o zk`8mVnB#5_e-85FH@PZe{6~&c-3B=Xbp+rnf616jTY0p#GUM}#S_#y2}cav- zIPmboR~6mfXM!B+xG)F#K`N5bmDy86-jnzju6v$Tc;!{c%~tQ&>A+%%p2ZpMu1t#* zPVcRf>lYUM`cAKbmlEHb(Q_R%pAaR?+@lJ(>Z|e90 zJx`T+WxlE?n7+EYF#2PsN?`+GqntYk)SWJ>vVzhu-8bp@t;qC7&&yH)DztR1bq zC%biqWMIB5fMcrHjC4Qp4l&fYSjYET@!UQet+@*)rs$K&kvFWEu;{59wspE!p0(0q zDX47zZ%ke9qZhdBT>brPU|Z>628#iE6HyO^Gn${z7mhXwy1JkA3D5(XtmWxW*Vnb)_n}cRixcQ}K3=cx zOfH0NTPAq4xE)_ht%d1M*lcV~*ZkaV0*@;;)uaMDB*KzIhwZEI9)aUb2gMgozOb&0;j@P2U^0@|sgXQQ%U53g51M$w64i120G!0?eAlV<^)t@86`iq%CAP4M;% zO`)6(!BdA?`2Uy)pirkBSvz!&XVF}mw+QBbrd#l8%VO^u?TeAO2kE0(?|+DHg+zDi z<$zwKA(4AEJmob9w8}!9$TSr<8<==QlDy3%u0Y=s%vJpWrpA~JSLfcXj!Rk4jlv>N z##EDu5t@qfPDLf?*?D%PV4;}R_kHmU&xvF|JlWK@u%)OUthdAeK}aO_hiKk+QU|hB zR_U}I7nGptRejTYC^FHvXfzvXNolT12j@F_GP9n6U3YcH0XB1Q?&t3s&;X{8?Y>Zj zn6&U3*4&Zx(at|4VlhKLKksZ4E*j0*5P$GPv^$*Do6XS*Foin7P&nDE_WYuQ;mEMB z68FW(#wm<5?owU*q0xxXZO0uwSNX@71{C<-14sdL(&1MI2?!`X-&HIPYM$5Xz{~uj zaQgPq?9s3GOA#59B=LU=0b>mW!*KBd0P!6E7nZnUE2=469VZ%j^kF%AH^i}^7JFx~ zQA#W<=&dj-NP_BEt^#Ss*l3i=lzvho{RvIaDFNH%1k%*2VNM@3CQ8>UVg-|nm+jhy z(YI21Ch3qIE+#!%i*6Fm*giUcxN)@?4<@ox{(P&vXO6Bo129VM^a0?yJIF>X&DZqu zvD{~q&=_KJILG+kV4}5~#gE{^?1wghBy`};Yzk$rF($}QQy9Lt6J{AV;)65T;a{Mk z1e?wpCk+pa%JzD`jB|l*ukCat!#h)A5t-{jtb1i9&jH{x!jY2uY!sdJJuy&zHtEj& zPhdO`b$AGd4teyOj^3fDkh`uca}mwuLw~NvCO)UraL|zbac)-y_tZCn0(jl0W++2` zv4aDmNvUfV4`-JDybp`S^|~r}qKNv^4CK5@TEJSq&-Jo{kA@E6dCX53CWm{X9D5~V zIdUlUQ)s>U4lP4bw+}%^3A)pt*NHZMjfOuiOW>ns`nzHjpUjF%+c3Y{0Dl*4cYi~N zCuA`$DbTdC^(1&j!{M7pHOQ}OR~-UIA&wlTEe+}0 z^hyxw8}?=y&ChJy89mB-grh9H$p_9kw#vr@PqdL$*h+$*-D@x>9$%?%!9F&iK)P;- zv~lFVP@+w40u$gai%BMH{)Zle4}B1I0fJ@Qu=VE8;T^Or+FRR9k9xYBcZ2FXl9B94 z;_hc>O99UT-0Q{VPmeI5E)nbIy8_~I4>WrqkvTtD1ED4%N1=C7h?2~ysUDm#Lg5*6 z(0%KDLDgV-;Pt>Grr^C>u>^>aM^RHe{_~xm_|Z=2D@NSEBzWXnF@ zBcw!*2Rol;Koc@l>D2u@7>geBhxDZT|C3=mqXy$Ki2j9NuWUvCGJko>dxN&{yLsk_ zha&ZL+N~KB4LL=En|fv4648+T91uCM5@!_9+Tc5sGom2h%uVlWJAEEEK z$~o2pHGbw4LWhF`#F-Zit(hAn-oq0g^LH91*<(Z|6ZcSD9Q8pzwnh+9KI8LvGGn`p zzPTqt(_GfLuCkKB@|(Xe=xN8%V>B{tw-VoRKVfJsSF(38z_9ETwkNy8#=K9(Lxoq$zNp0nrMS1-?cJCkNGBuA-_H&P-3x{8M zk^}(26V?LP*u#})bTZ}>mo20ye4W}{iyqs%E8o0b&Uvv+FHJx@z7JmxFg#n@gz#XX z*mh(4DVzZ|-Azed-=C!@|7odLuHu~|>7r;nyQ{D0m$X)A-cDD&dY{Ryw23Hge9kTc z4wdF^C_d#UnxExJ6++U_kQYgvRg1_G8}a01g*HfpwO#t#j3}#rKR^%oyf$R{No~Av z@?-c$hZRB@LP?$Q@dA0FIYG||&R9Jxi@;W0U9poS3xXLhfn#&K8c(M@XqY5@dc8^C zdUE~|)ZWo>?H0f9hP6?6*Eg>XxW^oc?a8&Yi%9>DQ<)J`{Ti_;eU8iY(wx+-N)WZg zsh*gr&=&$<*t@*?&wF%QFOP=vd5$<4lsDU2Dov3%XJ$C!>%J|8%b|_kJ^nyR4ZjZ+ zv;uBQOGUj1-&c6BT29)DI%ZpV&to-uM;>E$2(LNH=idAfRe6uN#{ZW*z2m)AoQUj% zNSROCpJj=x-h1E(0OD4SKQbO59GnsSI5K-?j_IRUoS+aeLBIQV+Od3+vUHbCKe>SGvEP(!e2Gm1#ERW<4MhttUueJrG zu&|2kPUX(vq_T9$WgZ>KxA*t%;=%bcjZ@R=8_+JU44w=E=M;xpxOzLobw+uIZ~3Ft ztO3#HSXVw~-3eK&=7{8KX2r`=b8`v6c`S4zBQ5QR=o0c^>g%64%(@;dG<0sPz++eF z1K;6rvOez~B>UgD!Y2Tbt8wwV9QQLh+_UA%<$jQ4#E-qaS_~SS?5l?(ns;34$^N01 zuhjw}XqlzL(wmzUXMI~_!5c}a4v9_7586U9`g+Lu#-sKb#IHYRP=_&6Q&eKcemfuU zL6GfTL8pn=8hC_$#B%ionKyA;udTP^isO`d1}uFzJud@IP^xWe^t1{aK4qLp?+=j? z9|=os=J7BvWx2X*3Knx&VOmII-p(uF)^`MKa5p%Flq#FfMN<;o2W0Eo``2u2gT=*N zv-Y8QkBdW&IoMQ0!EFh|yr-?9RRi3^;G++v%-N*|qb}UrAA@+Em$LV!6CpxY%!0U` zUMb=4LL(?fDA%&j%xB9ez{x)PI`4P?95r`m@JRKm>EOMw7xbZc*f^bDB{hF+81wnT z7Y}lK+{`AE)gDm)HVKnDfPO0_5?T)1Teti)ytpo~GT7&G_oQT8{(`XV29_siPjFS0 zdARwcJ19POy!P(aHHhWeSbG(rL@07=j^qJ?OKEM1Rq8OI(|XK(-DMLZ^4Zb9y2=A!OJS@wkpxe4C zJkl^K3-RNVK{U7~cQ&8)-oEJoAEANF>t9Q$)72iB9tGo*12zrcozxhXSNVAFptTJ; zxwDAM*kNwpV1bDUV-wu_G~478b&z`Jj=cTK@RE3&Q?o#VWOtB}eO_g5^*ovYE5XPpw^o>_K zmZBAZ8(z|Uv(rLlZhJ7|*k2wdP`ciDV|t5`UC1|u@$>5sNjZj0oPhg={6fe`!Vt#6 zmx|d!o2;?)DX55wi%Yz;Gd&Yi@j>$@cv!e+)zx8M=NFAyW#JQ#jSqh3>RdO`D2^cC zldmED*Zyn%?+A(&FuY{3j&y|8e;?j{);o&4yt#ol;f=9z(ejMi_V_`@WWX?)xG?;cK85O49BAaXmb9&RZi22$s7*Jm>rJ2=n6!58B7}2=JV#61{#2K7Qega|)%z4y95~6RA@5(6?V0oOh}q=I#1B?c z(1W8a^or{=vv4Q+doNm8$SXnh|);?RdlBpX#Lkx5K z3k$@4*O+gq--)|P)4EIp|1+ohIW8349N;>3ibJBNH=v>!qp(G=#HL%zY=c{A-sF> z=bfA7a4u73L@P@1iL5ag#$Smyt*a|%T}6QQ>P;2tz*s!9x#M5yuO68FEn?-;owMa; z!NyW)GO}vi_=%mlcX~8F_I3l4@X)rY=j3%YGBf{5=b7 z;MCJgxCy)I36%AS>x=Fsr`F&&@g08oyb`I*q@sRQawQT=JrU+52ktvstI zI!jX4a~_~OIgTaI2u&}UbWR$FK;V?GYWnv#`ScFjbqAh5w$h-e{F~L(`{sD|ca`j=vqFEa8C^*77WbAvpB-Sl(u_%195Q6a z2%j@QChk6g70+w)n)DM+$4t*Zyx)qr$16+sq)KJOe|;l%igLABj3m`ROZd zgiT)iL5u~=#j#c(p@w2CG>LqSJUf4ebB|DrpcfLEZgz7tnB3J&OcEDT1FqUFOY65r zJHhg}#$T5uZm(;aDn-8N%!ZPKX?G~%Aq;_a8rK=F2^%d zf}r&Q(BfiA$x9!%^@2;{M6RgiP?M*$U$CyPUygzjEZ!~2tjvWPNtDEffD1T^M=zI3 zhSWsjh!amgh%9R3+}dgE zEd_2A^}=a3CnP-WK<-0q&|A>}A#Q7>L`@KOLF8ds8739RTyOIui{z?W(rQ0(?nHOg zX77)e3_Jvdfik1vebjYqlLi$RIT&shIgkY$J}6F;`l)O86tlXT=V{)p7$bqkfYAQ+ z1KC@YO0CEEaNXHOk6s@|<9@Qp@LTgHd?UgjwzoS3o_ykp*iEy0)1vN_)l;zP%^O#kYgsX};k8Y&ZG!YCR2fK|Sfz zqVOm8(*bPPX06*M3@l|NzMu1+VfMfJ#VSi_OTa}Du1A=DxDc1Ps7eOZS0d_OL&i+g zi0Qn=#hcIG`zr%Ix1XkO^wGRfR+;-?v!bfw`Y#T(lihLKJ|%b`a5Wzo=3PHY{9AQ{ zO<;~n_VHn-JQ&r_=otz=A#^&-jnP?s`lO;87f(~kAcixgpfNo6tA>`14^LuaJHwET zbewo+sg-=cYt4B|$KTsI?h{J>&=<|~^LggGa1VWVEY&4fXx(pkAG64MFW?;o4@E+7^Z zLrtH`ETQy*!!M|ta_M5fzgppXGbAvQc-S&`^9=hE-b4cS(UNJu|DU06!xPWXO-*(u z{T80cK7efb#K@vPuN3h>F*SN7FRN=X))Kf{hK_wd!UDXS^b`YIeKzIWk}!ttxLf-_35gCZ+&V(^eY?tb*MD_1+Q zG$8}}a7|(&gy&Zi2(l*~Y1^wrM8O6`DqNUceDLmg(HWHmO*^Sx6LZ)x;FmK3UkYtp zm$^8f*~?|2_``^YnfFz)<@iBjwX=Om|D~u;JIb4K>?#M-3g4%}*sWwsDbfix>Kr^B zBC-&Kg?;HMFST6Jwsr2sDVkDj{{RL6XRU0s!dDN&Un!`nM=$UJX=B*WL;WpFS(lgI zjvjYCpfu_cfJr4UbS%^!K0wm{5e%3<_)7IS$q$ZFON*^}U~D>Rt53nTf9ZZcr$=1P zt=2v(5SBWxMnpj=($jo>d$@8tsGgiNP|^G}u&uY`$;;V(7=WBZGq`usds_8k{Dyue zQw$q~m_1lp#clNVbvQ|O4@K?a$r6+~1`R7}qhYy zlDXLx-X6f~O}RSyIws!_etDct+e{TAXjEENCqF&|PG3#H+2R<)O*pnru* zj6+_b@n_%kWPh+p5o7tqT68mITJbl843+OOWi8U$K=ev4dGAHaLm@j$zcg7JuScmw zxnTCGR#2x58@k zADcSQo}{0+)aYy%mHO9=diwCut^MumKs68B_nb}KpL~Wal-S&)(qxO1$&gKbxY7c+ z|8TQ^0g}m$oi-91A*5fzwHI}j<7v+mqQ!GW%yQeftFZl!w_Zg7T+eM6zKeF(Ovc6L zH($RzrFp%2%9TPrlLw9E5Tcw_GIuRG#hULpbMDii44WDu?ma7ds=YP$#$K$WW+_pv z46Z~`{jjOz>>fqa8%TJa8z)-JpjYmc>aZ`MT=SpJ%|l1c30EeB_bQ_$jQ0StY07Si z(JlK=p#69OFQHbvz6X$$8`d6e(!0q!VWVt z#+gHwMyxCC<4;wDH3P106>nToiCO%SYMjO*t=#{9hG`gKfWYz6RW|;K!?mgTqfs+m z5RmS{8ljRxWFm(2`LG7r+H60|#O}FlwK(3C>ul|<5l^R(8H8MD9HxZ83jMd zD)P84)neY;)2;;Qi@aU)9zqnxAMdYMtm76(R0?<&$yM;$vC$-T4c0ib$X*sK1m)7O z5_bhZZP3|5t6H5Ochv`az%8S*U5*g2zCE+qJ@vO+ZqEzuIp?Zt{ z*$6{(xP#b_nU4;S&NDPO9Tn=Hw(A|dcoHFf-UO6WQ}X1$)w^6z={7 zpUV>hkSr3Dk!(GUd_!KnHY^bE%&$NtcI12A3Td&fa4MwnlUCiLVj<7;5w^)URGx*D zxhw1a*R?tIhi||1i+}mT!KeCLill4dI+B#uv?#51_JXB>;To(%$WUa~m<@wwgYmMe ztb1|7uw}PBb|=M-P;Ps8&bGFKkN#&Mng>JNu9oNJc$Q$C&>r0rUO8!Y&!4x3uYS40 zBE_)|BflPXFWiWJT;l2uc5H`W;(HH{iS!~kh@OI?>N;6^+CZ8LiP0W zt7I6TF&Enr5&(cnfA@a@PK)Fff4w7ekDZratPKL+#H?y}l|CqXnrn+n&hK{*iTQmP z;dsIKdn}G%wu;jF+eg;A_u)M{)|j;0>NxzmIW`}r5c*~k1g`y*(l;#6nnK|!Nh74u z^AC?&CKSz-j+!sTKdp$o_rX$$iH+%F?_682T;H-%wf?D22~cpTT08Xi1OjI4e6Loc1ofD@0mE7UFITU*)et*^?LRcS3kr8_e2;zcZJ!2xx_E z_2cbkbwuh>38nDLw4@Y#{j;c*Zi(}u(8Ji!GTlTsI)C)?i`Ch_C7tm*t9S|dU^UIm zmgI>xA?Y#CX_*@~QAE1x=8a1`pVM?kaRq*^rQ+xQy0RX(D#^OwTPm00#8=ATTh>?x zqEUn8uL5QB^wn)%O8%fRHFl%{;_t(zZ<$^rW%}ORGjj*u{rzfsnd+)@zH{o-sp{%eH$OA?g6n#HWB2{vzT)vN zTYoxz-uMG0|GcN--LLiTKIFjs1$Vr7*Or_AdE+zx`P5IIUNQK}mn%;x?Dza7U++02 z&~o2@|KkT2@0s*v_X|e8H|B>wJv8R(bKdN>W75F(XC4}R+`D^+ckc14U;g%2?@jpP zpZ)NKGd?TtbndL7AO5)fyJJTL2O#P|tL;}l@alt=zq|d>t`~jnwvLZ@@}8(zNu&s^!A$% zeDuRT>%VsL(HFX%KIY%2eDt;dH+#XYEl>N=z?Xj1bLc;|{V>#hX!8X#KL1I}Gf!$Y z?)yKu|G3f?@Bc@qM>m{y>Y69d?J)7S(n|to4gJ~=e{t~@7vFpQ2k+Fh|5_^)H29MT5Tg3|7c}ef%Y>UYPSpvwdgWw`RH3HMFAFYh|yVG4gNaKN)!cUv~7o`@vo^^kH24>_l?T;Z>#Q9^iivGCq0Bf0^K|HIAho8 zzwPs0UcnDeJMx1cym7&z=U;f?gzc{m>-^fT{(tzd{0{&1LW^x5_3V7_W7kf)73GXP z^x|nhxNdM>n?L>A|4zEOsAfy^CH*>#zVF)6_b+SL{-rV3-hEB0`%e$9|JCVLSGNAq z)^nd*HS>G-bXXA_wC<1PBRgF2-Sf`+-lebOjYGdl!}qS3f6E2Ut;a9f|F6MUp7W#E zh73C4r5`r%)A1)t*kKqTue^2D?U+$c^FaQ4fkpDRE*A=fkwDE=xgYz~#0+M#Uyzs+sE_veRg||Lc)2+?uf)}@R zD`>XovNJEdZ}R;se|hbo^Y43Q{!jip_`9Rut+`|K_?83iYjNDqe!uyFg|{`I|6=p1 zqUQ52ZLx0Cp5uS~;vW~}Z5o3Dzjs~Dp!~0oYX0@%9lqXS%x^DT)a;Q#-#Y)C;}(9u z;LANHy!z(pvj#r9z5K($m*lm50W!=Z-<2lA{j-}_DT^FrsXJaN!tPgQ^SH#5(l)bXY2*Sd}R z?ZThuO>9;@XGH(E&wu8I$W>?GuyyGxr@itAL{`ab-?~ey3D*v~??Rl9;SIc~VR#n# zAtcPP{jP*a zZ9*eyZ8JHTi&!=#nyl6nzG1a`{Xwhs#7%(?u;*gSi9ck*)E)BFsmDUd8L7xji=x3I zq~Z({o()Jc%GCAfdU@3JbOCLo5wwzKoAwFcXiAtjHmzvu&CsV`1MH?+?WP?LbfR5n z|Ll^vmV=Uayqp(DgC#ztVw7X*kf{@z4y_N=1=32IK|6U6URqDM2L9u*?|^5hK=8-& zVdFbP_Die|v(S!c+f`d9wqlEocBti1g0?LQGV1A0OK?x6UO1+c^mB4_l`DFs>IEGE zCT({_ACz4ZU7$^|DajXeI}I3h+%#Gh+Dercd1|7B+0Nba_A2K-Z$yKslGkz^op9@ep$l;~!L0}7^bht6 z7HucD_$ufDY~OVlCvA{<754Ai&O7ZrfsHrYn#f(*@no_}CEslRm#jWh7ff1p|6uBY z_z8v{5ChCH5ffr_tm**Td?ECFoYj8D;aXd-a8UhwoSoNL)+O_7WSY&hiS!|nF2w19 z;UCEO(e@394f=#}@DB@*B|Sht|81mI@ohdy|L#TZ*~qOvo^|U&+IqloqqYSTR>2;8 z`&iHc=yPAplkSmzo^xlMlhb`X?P5>m*Xh1p_v>yv`@rT)s-yp|580+wrlQkLzvfGH zTl)wYHv$9kcf>~oHpB=%5izscyxzB|>VVa5GUjamIo9ep^Jt(O=0alRzEH^>vZvg8 zGr^B4Wl}!7BX5OgU&y^FB{%(=FY+k50G?$GX}5tuiERs|Z4fJ9M(hN`c2kgGrEK>^YW~eqPx-k-g$u+JZ~=$CAtw z@nY&hj1B-3!#8kjfu1*_PXKJ8183!;27oTEkNH5xb{x~IFHzyjr$3+$ zAhA8g>OfA|0oeT0t#+{aUGZ*1e0KK`qmOe{nnw0#6_ zq6hE?v~MUr0qg*vhE({{P87b3M*T2j~xoPuM8q0@(Z@>Yp0u zg7q2F-^0)67&ckEH+lRjxA#=Mn;6knnqx)SQrivyW6l*xUx0A|WMH+Q+Tim87?X|- zbjI4)29W*p^FNDLuG{5nDt>H3=Iu3=472@Tx;g+EP!d2mByqI>c zbeH{^$xnvZbbaKVsNcU&cN~nKI&66p&;gDQ6rQF(AZtvN9e^w-lZMa%^!eYmI!xbV z%U=6>dQQiXdmm%&rr(>uIlDCR&Fy}xK=hAuwq{LNwtS_0UXLM&rHm7SFUQXq8w6yo zsCHaHnON;7*XuhY^fQI5&eORrI)C+Pspzw0Fq8fA;CzTUJK*>_6hG z4jefeiT?1OwZ72j2aj=ONSiU*k)Z>`JVpojz5ueZI!vp#cgL_X<794+b5b0WI@_YwfB7R{<Ku3>|$(=+V8%u(rfukF63m3r&5{8`?A-?u(`G!pSu z2X^i|9R2#JmGwfeFBoVWKy0Z4#9G?{Z3EaBQ#M#jNm-#E@NGRI?EXVC$0O@D47)F~ z*KJ?heE-|M8^h-+KJ!)nEbM<5Z_8cz>vp8umZJl}9x{mO2Ph-V$5Cc}*Z}yH7g?QV z?3ejHHO>ogy$0uWIY+_qT|XwuUCEyPX+5UXZhR~H`R+r$>OcfGrR_u2^-6cvn1~+e zb_UFez04EA7l2G;euy%{Tr6c~b(oU%+H&+I(*oVG4#DX21008H*^B%ePWBqZ>;JOR zSNXHBcRt-8Z879uO-Bcaz0v{7g#MWG-WW2Y?6R=|c)k_ZpG2JX8~7F~;2!72^z+27 zVNwY7R8{ZxT?fXD|D?fq$OxwGp{D`Ht zzwhqzsdw5AX+N)FBM1gML+!ffbTlc z`>FREjVDi7N_*4oPub!`z5sne$PzZd^S9^7e@&nZ*6wP5pZcW7dXB%xHm_16$zJ2s z<8f@K;`8s5?FXZ$-0)V8>wv z^_1Sx1~qcq*Y$tnXRD%n5BU3j=}|Urkhxv>7t-(Np6oR)cfPvGcN_4=dw4%I80!R@m<~{8?6YNVkh0`= zWsq$u8xZI;BN*s7eUtc&v>lvNV80*J=NoN)5OB$%R5M^&yIB@@Ae>j=ursc%MjXr2xW8LC4~Ay z1D8j!`+Cl$YaV>Z_xD5F_u~D~&2Q%%uMYl{1!Y3nP)3xMo)h4F9c7960LnJdacY}n zHUReF`5?v%5;qGzzka7*D|?hJvPV5RD0@xQ^9#25ZUg?YXj@M6?8pWFpwD-eYqov7)3s#} z`HSpPM-Izf(-t1GJi7SZJ-+L}J+FU~yZmeUV4al6jPK*se1L5OAX~_ovUc|Y@D10B zk6`rs9PimaU(;&)S_dw@XI=CV#_)drzQp^XZ{D#cH+3M+2B>*KW33QnOj(=qN1t#; zpzGYDF@B%C--Fy8pI`a&Tg-mJ@venG;Zb0A;Fe0AvmMKUORM=~Y2|yD2_`mcQfop&Z@zY4@9;{7Lil zl{@`bf#_qiw&orl(l?tn0P@#MVFL*H*xt>My>wvZj^Jme*V6|Tyx7?R^^TkX!%oiYAjeQ zf5@1!rVj|&2RcuG-0}Ui?Wg=@u2=Q@V)vVN+c(?iKNf%Cr}(9;7Zz^IA^AgAq64*l zpvYhM0pbI|2ZZdQ1Fy=QulD%=HqJq~0{Dabz1Wk!{}AJ+2cSb_4V??C%OJ)(a@+kgGBlScmC|8L3HZu&CX;s(qK zG&WAsPpfuI8A6ug1DfMNt__3ip#$r`it;ZQwj#O|-~Rjk`xW00H5vKOTp#E<`_owY zbA2D@e2(q*PZ?eIo9ZaX@P7XG9yl63?-y$uR|gWdf3-GD+kduy$R2Y5yMn!paX)-N zu64BM{Afdz-*-Lj+Oe0$&Gr9V@~N0>Y;>%s1C*T_2g>?Y%2xY;kU3=^=sx$L#O_gZ z{?boS{!ZVo`+UfL*s+&Ry=hf+({_KqC8s?pdTL|i;U(=;j~$54RnRp7c}92{cBV@h<;}$lo>ZEB5~@(*7x*c8|X2yZk@f zeK>mB=+(I?fAubyvZoEu>wLvF&_;ZP?EmpMz8&4L#sBX|hyD$7!i~}PS!Z4CzgGUV z|GPx~>=QUX#M-eSeSfT?|z%EAkII)>7Pcdd)=KjiN_;e&`raq+#?jf<1CnY!eU z?}8xvK)2cJ<$Zu#{_Ov$J8b{QsQl-w@jrgd!!xlaq_NwGTmFz`T>Hm60LnhlZB~^N z%Sr4%?1WzLa}3MB&$Ir2A5cS|e-wQFoRq)%hE>~tyz7Vjt6&3Owflec{c7$f7~}h^ zeoT16H*mw7emeD*)zMFO_&?_F@y8Ex1Ro_c`hhYJgg6$o0r2ac_rMtQQTAy6kJtA9 zw(lP+f1~fe0kS^^(v$N(_kFxKg>!znf(_fM_5l;hAHF_fIt04Uemu})&ZuDNVtg-( z?|fk!ocI2iH{!acF#7*6%A!9T27gUG{qX(`QO5oD^CxrucdpAFY$z+rjItB|AAX!( z6HXb^*MrOhJ?4&r{5uAFW9={HkGWx4^C|s*0{O%CH)Z)3j(9t|Y@`42qwgI5Y3|@c z8M)=J*MjlAzFq?WnN#+GqPgb=dt&`xjQrL7zq|c6aruvW>0>{ey_B^E--NapnM3Wu zC4U*i(Vyd-jwyf0JWw?6+(2=8K8`y>1JL%x{;RbgYW^pTv7hLZ*nij*w4tV${^O3d z(e3;EjSs{%Ki53`Zf@a$^(T(JIBpmFujJ3~LMda&nlcZTR7hM%*nz50f7*W}>WDG_ zXZQa~2UvC!mH$gi{og<4`9)vk9v)i$Y`2`dG377qpMD-<13~7__dh-6JQnP?RLLLu zLHW~Wi2a8jpltxlrVrS}bf691@A$pum+>W9-o7Tcun_-TwSSR6{65OxvHy^*#A?Ub z4>C98UlQztZ^NB20Bpcu`yD_S@eZ1@{Ex>w*y_#xulc{>?;q!u?Aaz0$)Ei=WvOhx ztp9|p14VOczyB;QJ0sY$@~HFy+6Uk>L-A~!{F_!E@auYbzq1H2;R^8HFz4)`*yq~z zPuYnK4f#{Xly#tZp7Z@T?Le@{oVB3=cHbbnqS}A0{L!CC|F7gv{&PHCaQFHs<2CsC zV?2pVfBR1E@h*KbWF!4QWJTYP@{ei%^zR^l$XeTeeGC@OeKbhjaOMDL|Ly*x*8Uqh zf8#1(klShIYX9HwJUZvI+~iyPWb6ZE{I7jK$WV>_WPJx^3|VWr>m%;93igE$V9H{$^P|BZGaSTyJP&_KQi#5jTeqTvIEP^R7gQ}!y2gx=*~dc4g4HQm!!?aE1B z1%E9M(E+^s5&6gZezr^@TgX`3eD^U}JhzSX0h|k@{PjE#{RndoK-#~u0XZiBA3U(Z z_is8FH{zT3tj|sPi`|1Plku#ExSLkzru^$_`)b^8%Mvmzo|`iMyU4$!>>I(}3l2-$ zhcBS#0Ui0@02?p@K8Dsi9&?}%I1}I8ec|&r-JCOS_1OD4!Cyj*O&L%okWGy37x~+E zoH7meoLA@m526FbbEk#}<2ls_gs~@L7;}MP?6pMuQh+&z0@TMjumZgMEP(uJdmA(T zeTo0?cV5IhqsHVZW!=P_*i#0Sg){HNbsLZsWTyLZ$`Wng)BZ0a|DI*ng!?Xp-vAvz z3PX>iO!NViKlLnD{*9{xY@>d^-;tRAuqi2J&{#UJWpBuzG7;HOMwGv9`;;GKiMH=~ z|EFND@@CMNC8=xx$AWP>&V`Y&n*oP!T3SV1xeYggJVJVH!(gy!}kFf{Ca)8#B}_BzuSoZ$7a1FWBO51fD0I-qsdZ2LaQzYt|Cs)v|;W2SD-ec3Dd zi|mOtF(>wr0c9cnIqW`V6zpAoUApr1l0Vq9Y+S(*^aC6Z(7#Nm0~`k;)q@V)P!Hel zRBzcIJpplgd~35l=y_bbcVy4^ys@$ue_zRivZ0K;$TE{W=m!deJmg?M%ggY7$B*{vR~cr1sqH6hsl02|?LLw0DGSIXSX%LIuX4;LAAEtn;Q@<} zhOb9I;PwSb-v}i7D0-fnyqLVDiPtrcz32O$&O7ie;90*~m)r8E>_z^-l9&=(cb^X# zP!^O)HZo23*-)>tv0?N#GC$1zhrS2>LG^ARMhAY1{y@v$l>N=n3lC``Vk>OedcfB- z;^mJ$)$N78tn-+dXq+z15 z?+Co;fYt+(U!Hr=fnzd{8wbjs7d1mr_s88kbpIFbG-QBbA|X!OF_FlNB*mZRSMg6QWX?>j=Mb#4uTKn#B~l^Vy)FB5{g$5}Tsnk{%?JJ9(t-qukjaa-7V$bJ$2*_Vn|SD0^Z^*#lEz zn``aftv8?#{y@dDLX1lbI3_f8fVRM~1GVpq935bL(0bt50=AhZW)C#ICJ&k~cbj+1 zUdH#evL}Ybl9&=(H{Q8;4S(Qx&^xto=rZ`JT8D8|?~7vlg9Ns~)PtsE58QIsGUw5h zd%`wfJGZ0P@DV#=NGu`yM)mX4)w58)1(+WwA6qbJ>CwXBS_hO*0NYmJwgt*Jq&-Y% z4`TEnhirmd=4|tI%H3t}%{I@zo_-!MBX*(QOhQdSlcIf*MmmTg+wy=I$x)b3%90Pl<{R~2HO zi0`SiPgC0;XukmC1o}bFIs$txjAO)@ae{7NX8&rQ*VnkgtiwFl^ynk#wbzf^^gI5Y z_;_k8#CdXQ_msVj39%tY#EO_TH94oE7yUtDsI+`sc+ir=g(DQtLPMLZjIsgnfPIShKrab_T#FS|ilZ|(bE>IS3`Iy&r z>V}fN`HcJAkT>ggWKNnieOl&jxl5Z@`YvtW#(-E5lThEvdRezq-!e_NeCT%z*n)?` z0~YP4E#O)Ldz@(Kf%pcdJy5zJHo?#Zr4N(?Q=C3%dAa3S*R_7mb=38U4r+PxsOh52 zZv?G)U+awJY3Df(qK>OEF?kj{2n^8X9}4xa@^!slx-w~u{1_)d2;`2r z9P&)wi9usvpG$Ry`c$-n&$F~(=o^BK-A2^8yuTp=mOiBv^lgpEe|=mS+=f^b?bIc{nS34+18bvCr=`G@JL?K4`%gUPA|`=!IKPQe5lj^-;?^rOcI$S8^9Sk9m9Y1pbx?kKol0%#Ot} zpa;!h6R&}7oEjdq_%Ppj6mran^$w1&NSol&1;=j`-%;5InQK;U)Rdj*2m<|Cg~g4juPsE>LZ+f z!nP062U8!+Bz^?-BS|vy>gQ#gPMfany7uF!huXe#J&uw&X(i13_^wgppvIDHlV;L# zbB>TH`;WL}$e4M}GH`D9<*0+Y$91=;n=}Y5q-nsygmE=<4Z~xJ9<;eLpbsUG!((_q zUKJj&aA(1grFi$ucaM-0+HwKkr7*F*O5!=>ujI>fJE{Nc{Ct*Wuq>3xvRQ}J#X3>< zv4rKZz)NW0qI}5ZT+BCjL?1i~vU)t!XThr?x4sM3hx;%56mr~!xu=6j5tVR`d-xxo zS#Qf2&+u86!Ls@8;u&(=jB}$e{GN>1R3NAK{aOiAjo}P zY5pGsjxn!6p!?eVHwkuK$B+*OV#o&r?&}!x!9ZQtG3j~pD`WFp>0XbC zeZE<`*RhZP-&I2+kgmMk`72`yPFsI!VxJG!b?w%7_jO(E$NhhIdvssB+q1j9$6b#x z8A@P3D$W1H-qe`a3G7o0gXm;p7&trL9>y>Ldl|z3>}d?axHJu%*kgl0vDXHHV$ThN zq|Y-5I-BJg1YE^yqe8i?tx&2nItZyQ8g(jAU8+;T+4l1eNB7#-xJ6u~wUj6RozBmRfBmZ?xJ?`NC>H?Eq2)>8PByo3dU0$2`p2X5v_RR?3)wvL^iB z99;^@=eHhzn$`OCp1|o@;JC(WKmCB!X>P>oTpkH@t%zD(D}#X3Io8umpRW^Smd{d*U3& zl;=9Xex7Bia+N&n_Jg`f1KS&E652>(n~BHFIxX-LL^~g5wVS#z(0QJ1yJ=H;0QVwb zUJM;C0p>_C*!BX>y_Q6ng1weR1HEyM6zuKsTs><)?~<9SPwEB@7Ht`9ob)T5=S4^> zbY)n66K)T{ojy!k_It#gHrU1-xD#{QlM*E8vn1e6oFxHc;vDR~)J~;1rknJ$a#ZE% z@>!Rz7c`I-TNct5ZCa@-?6ccUXbZd4q-)*hjc040ch#ZWm9}NJ97vurF#P(p?CHiFK zsm+95xheyUjgAL>&(YtmwKXm~qiu|~EpD3voPrIJ-1TZ3CH6_#CT*AKgR!sTxP#*n z@(5mYNe))qiRW4ECa<)*mZL92-&!O#9zF_mUiwk!JoT*L!l$ zb?&)bu~~{X_VH+=l6p$tK%QWi&_|O;^4fONdAT41tL=oVtq#+-2D(*3PvBSE{tWsz z^sUr)-IoG;V(ujDi7tC)=fY9ioT5#)4NbT7)sR66+cMh%cm%KH8G3hhW6A*a_p&<9 z+9!TI+o|-e%C<|NYV@6b-cY*!26P?geV0M+YmfDHo>94Mle&+GZPI=)#vsC@MZXzi zIN==_G=>b&{)WIeI3#|8`0=`L)jqoNUBte`Vs2uhCo?P3zPx+XSM?+ zgToCa1F<=fxnt{1J5SpwHr}x9+#9(mC*42kK7u^5Z=hdc_s!BLh%IP0sa|anWPG*N z3HF|IiyWu0e)!5 zZU?}o-K2WylTq$@R)-l|9qegyr~|;>9^)wdiMa>-PapAC^syP6BTvlS97W1IMbJmVh8VS*?5l5TwV!gl)qd(q8CTKI;n+m`mSS@< zvo|j-+-APKOY5Bb;&mIlq2X3yV9#0EevxB~FlR*UV23iAHwE_1 zkY2I6<6SJr-E3Rz^Of#9{g($jD3d8kk>Go!_FW{t5@$1Z|5rXN3783mA$&)PBk^FKqz2Uubojxl!hqY@2WQ?`&Ia zXA18CY=-cHV>ajRGUl${l`;O#v&flup7j2H1U-d;_ z(sE%7#Aaz*#IZ8%5HTc{z!bVYJXISIyWeT%I=jzRa|?V&C3=sxMIS@yyXXdFpsz_+ zCg+KsyKmwrS@p%Yz4Vboi(hnlE{7P=51=jLxBz|&+kngw5!;T_llBA9{SH<)yla-Z z9p!5}cBkR+KVjJN$UB?&XFM+3_2rT1IiueeyXYCW4D~0jd=r1W4b;vN5!((^J0xoZ zfsWHBi?7dlHPhb_Kc{}|O|tgf{^1S8KP}s1W_}%eAO*2!Y_js}^}e=6^og_`;=3D; zqkwTD{Lwd@9_U(j5VlI@lhJ?Br=njk{Tb+u1P6*;#gXSogN*A;&r>}2>e4SV?u(Be zjYPUU`mTR;dgWJRME}I@3!J`KuqDPFr^l`N4RoA3Du{Q>vM!A4F(9`Pd? z_89_J4Vlipbyal7-ovI^r*-}=_5eHyu{ax&W~qx<5i??^+JLO(!axJOP>q<=eQPfB=%^_5thxPPUdx{=XE_d`X3*$4P@LGJc#+Zy6SNA=1YFG zmk{kXF~rzd<_=_j4)3jiwTXYA)3nn9-OBcJ&4{ecL7yAy58K5yG!XdHjuQXDF8qya z^MC1_*Oj;V^~)n~ZrqzO{s#^njeM8$b4qXOs?W`5R{5#7Vuv6D=?6GR4f_tPA%oMj z3}BCX28&>Oxc*D}-2`?ASd*5I(o#Oo=eOA=*(qncOfQ01Jh1hZP+1>iv?R^ z46Fm4ruTI4?=<6C_6@XAGS7g1SK5x<=Roh@iUtQScaA)FU+X;S-uL?4GxI*ns4td> zV7~_a`2*i*_ROR9E!YM)hLSNc##0=Rb)NAo@kcvo7U(j24d+bN9Gmz&+U84LLlpk% z$b07fY|cYQnW@fCyXo!7Cp!+MJuZ{Z0}bI*JbOCINL7cQdcB)LBHYnJG42#C>PSeKYx(bm;i04Vfi3-jvZ2Sq{NtG z0bm~JGV2uJe`TNu?^w_`%X=L9{Ep8PtLLWfWTW%uvu2rforj0M6`?IxHpNq0(E86K z`jE&^ep@ZJ)TE)_&PD%$q4tG%n?{yYPSpY(K6m*e%dbDnIUi4 zZPqyQo_Rl;bHO}~^1gG|x+vZ`iU`=h$_f4?JLmt+|NZNOebE-fSNKMMoxBsnLFhZ_ z3vrH=^EsTCBj&`u%dD|7=Az%jv4XZej@`ldWoT}t2d8c(q#6AvjnUZey+L>y0^Df3 ziKXp}+87gS_5odIj|Kh{WepMEuh{t8?S*Ye>m~3PIjf`lzWaaV&1;@?^6Gj{x^ZP> z!NKTX)*| zhX8+>3*@^(H5Mc0#J<~{jk2CpY(MeGyoI(sL(z7wM;pq~^!l4S2z5q3h#=bG(f8_~ zH)5!45V6I4g#A8??`42}pnKUD0luqbzd#?AzAD>`=s)_Yp(`T9-#qf%eXaAPdq30X zSkF1PuZd!=pX!{{7Nz%iNsyeK@Bt?S(!M?ar{3Ih|UJcsu&mhx;P zo{a1d@jrldcR#qVI+86NDfy-El1TrFeUV{*2EzW+{+Rd=gZ&|%#*ydlYn>$%|tc|VXb{*TSs3jS(&)aCN%%Qe9F)!MS@_VpPb7(m1PmZQD zezrRLIo7Xd`uh;;?Y@Tna_W^oVkkC9<}m5s+dh!kpLyti%Qgak+y3x>gW>(H9x}|LnATKJ$TRI?cM%KEGbsA7C5c_{z5b z_@29LBlLf%%!~HpyDs>rdd`PF=dhKLAovuFj3e)v_p>?IWv5%lbq~K6;gV@_e(;BF zAkjJRCHmitRl6L%(ycqK{KOD8h&F}T2I$}Gx#*&KOCf`ad^aZJ0vrEY`y+Pe?P%zR z+)Bq`Py1CLf0?P-iSEVvx(n&^)Hh$lfg3T5u|KtAFk+81HrQ9zxY=_Q9DAX^2=f`p zj_o{5I*lXG-Pbx#y7x1A?pNbK6;)59*KU1}yT(KJOvF2R7eATMQ6~<>o){YX&oP+2 zCIU7e*av#d9YOmKn*{u6|7Gl}@JG8b=rn0G?`Lyvmg(jE{5#i0_Z&z)<~8Nw#((W6 z81s%=6~k9H^rw7Q@CSy(pZ+IpJJ&Nf_J8h(K+)VQgMIMs(ZrwnFTM|+gMTf!7)RbS z?`Lza%l1-6c*NV0$t!jywDTtB=D8dHHk|YEk^ar^oe%uaF#J!Eo3=mHf5<@Vj>z4(HvgB-x!1g> z-m*Hf@pEr;{uB=NUjKN#ZO`Gz**{+!!-LnlHP311ckB=SPqlVx0PH;R2j;+Ez6*m4 zo(=V{#b5jX(spiG0sDh?G(zEzBhTH}LR$v+y*~HwtgQm4iLIA&TL#hSsFy$Xjb=~0 zi@jF%NBn;S|NfZ&EGm=rCtQOPESlRh*l!8eD`M@#Agq1hS|#XznERNs4UJ$zd%Cf4 z(k1u36Fqo1bGx(T{k_rV*c-CJ=oecJ%*6h1{L6O$z?9f>o&i_~i_2=)Ulo^~7A&pW zFXJ*j|IaxH_8Z#%pzjOch_O13I+@p*p4atU;mB2y`S0v;?1>khH}f2ZtMtPMH#l_H zRUditCC?2gN82CH|ET$Y`Z>fpP%_V1e<*FBqpa^a?Jp5n|_v8T(E8@x&6o z53v=DOUmNcpO?%#J=kaAL0K<72y(Lf0N5X+|Aqb6x-YVZEc7+&&g4ARbI&cz{QC;zjpX*E^l@ zI4B>B{vt3Vb|LJ=C|J_21KVKFc}c&&!~cSz-j(Y@gD^%AKNNC9`w>4BZK{Cyvme5m z0j7eVvMsaqVn{G!uSIlx9R0#?Ax9>Wd+hP3B{ zfIqMe_Nqww{i$jL!@~nH78@x2Fy<{V7Pi|!q=0h8+OupiH+ga6fBxO;BV1FG4BNWy zS8v`QIeGNzhQ(iO0sSNBsIB|3+t6EE_r>P{rWo^;6XQex30L zR6jf{JP7k~?1%MS5ZjaJzsvso6xT=vqhO0NI5(f`ZqKh}+~>aX<_^*kLrXTad45*g zVcKS3B=a_E&Vd*LOWJ);@izp2*eLAPxd!{f!*2b{Zb!-wLiuQCwhY8}WPrKJm&X5; zpRJC(k9cmG+WejP=Kbtnt&7tIv#xqOr~R?9w|zyKZ^nBRtoalSfn~6FWlFnG+|3_+ z)VAS)OOJ@(1YZ>2K4718H6P^IAjAGZ|A}R$>G!j@W@>h#>@041byLQ5XGF8My}+LK zSjJqO?;%ENyhZGYVX#+uTJe`ChhXoDW}#jcuVDXk$Wi9x?Ec)gL(&G|U&q>5PKS#eF;+!=5ddl6jKbrrW3@qw6FTnpRv3`xe zOz>ySpDP}IH^Oop{Ec(L)3{ESFYWs=_`~J{lNg@|wwu@i`;6l6X4+rALnqP9KYmArUOLtp%euxI@HW98?b>+(ud%6ytn@+@`-HplUKFxFB& z4>1e&F3&3dq?SRbSNU_{q1q1FV=|c+5*gUz0jDo^+n*G=*YcNZBbyPoEfbr=XFlC| zD0A_RH2(P3HjG4F;xW*d6HENbpOZsei*U#{(=v zz=YTkqfnnpU*m5(cBr)C>hQqD`%o9!8f49SYx`r;RqKDVFLvucg`v}Bxo+rAOEN>?FGkM0gOQ2KR9O{zoYs&nW_H~9knW56k zAq9iqI4tuUcuym99nRPcb40L3h42pw(PkwtAp*4Me`az!sd8`^1e!5=h3@4*B0$@Vd9dr!NIu@!I!@5CU~r(v=MIMzbv>ZI+q~zu`zt?<1kJb~>d-!h9AMx?&V=6y~ zJW}7lbCkRj1C2*x98v$(;Q@=b7F0G(iR{K?ITgg#3squ*=`fIENs6+>=JC1<2T{_X6-|HJXd>zbd2o`y7!&5H@L8c;Lc~;E6u7vP(FUXZjVkENt6E8^G~{j3IEO zZD682(n;e4`+6H2FBio4T!#U4E+XY-@E+*!jc~qNXyFwX` zZK3UB9rWRKdjt*2&R`Fph3jDBhm+Q1d^Gj@;4O$Y-2unR1%sCyWIs**`JOsP7Sc9g zr#QF7_96WS3_MByzn4>PioOOc+(jqh+L;CJTPr&!-;CmG8?EHcu zOZFFB5Bo%4gY&pDpUZP3wiQXVS9K&CYdb%m6?Q667>AIQr)TfYFW%*(UXd4%QPYhUc-`HA*-$rFlRjACV3YD4XiW5F{wt4Q503yuun%eTi$<8Zl^H|Bw$JFL& zmh8MX%UAm>89wIy%36q$VqR05C!Flul(l(o>V|o4>XLbG>Yk(iYye|45iY&0)c2v= zHToPn?+}63=~R4nY};`vLz{9c!>O)J+MudH8n!A_8oVm>Z01)ST*Y%$keu2rPuT={ zSlJ4x`CdB@GpQ`KHs?aH(hliyq-J^N;BQ{upkg^R+8?XcKf_k**Dtf$Oc;SZ&!4o~ zO_^b}pSH?sKYa`K;QrF;F!QLLaL#-DAM>=G{G>d?XIsA>ZZrn!p<=7m>t|c7C){kc zoiYo&@3A^#f7`CuXSmyZJ7G`wKyAW)W3|_^FSpLeXZS43Xos@epv=~ci*>T@HWNq@duhwENU9zKgQq%7hmc%cqpWeHYjv!sEvw3$%gJEIvC z2s&E9mR3{W9bKV}NjrH^m@rnc%`4&uF>YmoB=OY~+;{Ra_ABs{vJ_TQhoe_Yr)aB4 zQ=5scGGNJZ} zHVU+%?|T|JweW>$kA+xmCtirYcQyBS6`hfNxO9%eOP;v0 z1WP3gN2fpwX_7W4y0n@!dt@)GY^bNb)oJz)(V5~Z*#lJeI6?afma2W0xEBW#MH1dq{9oe0Q!nbs<29eqBr;PNk!I2k+bwbiPf6HK>$cTyQhTcl^pbso#)h_+ z*eG5Y-$tJ&JNlJJ@JY8m(n^{YM&w0!YBxD6djxDQv^ve%5sMA#GO!8sMcfCW7kl*J zTqnk>d9mvA$mhEc=@d22^}U$mb@#6a&b4JEJ5N-fJ8x|srPqQHXg4t;Z|x>$)UV;W z7FNfZtKpl%=U45C?OAjNJb-_lg8i_!vT?1?FSu=0WM6`qjQTqT_i^o}FQd)zU8#OP z*)dstX8yQ5f;+G>Z4!7Rk5>CBY5PZ3`>9V`JurS^AIW|o2{xpOab-=~%=2IWGk%`{ zH!k{`(=~U!`f=U(q^M8VoscFIBk}~^$RqYQjSjK;%#BtCngZ|C2NJ+2efqwl4vug*N=fxM6>@J1em*Y;C9 zjYWV>fz@edb;A9u8H0>5h|^4Mepk(JwaC7w-_*Buq+N%1nixk7iG42j)1n^sqZO->>ZoSdh1W%tAKk;r>_0>re zcoN=}f27*k9jV$G@H^S+GIyKWhhFqh{1(Ir7e28+ANTQary%w`cHeVfpE+t}bo<`J zn*D@F6Q3HyygV82{@vy9+Dm;Y^H9!=d#?R(*dp=_-am(aB%MFPUgh^lU(P;|{UP@k zMw?1Rqj`_<)y?Zf=gc$eg%1M?@F={3XYq@GK_VTp zI!tK}{5Fg3Q0Lf>QwP~sB+wc2o>zPd(09M_-d+ty+|iBCK?I8b{@rWiF!EZTNn@gO z;f=O~yn<(?Lz}G*)8gi8p*sU&b;q_jqNm)SC`A%_Q~Y0I%Twma^TT`BMfh!aqMsWd z=BavTx2zpE>rbaX^Evmq@Fs1IeH!~SM~4Qubtuqz_I#NWM!$!4Nxfk^M@%$(|0dAH zeF_pMz29!K|MR}igMC(>oMXpybz_tWALbeUTup4fx-2jCrOHFz$Rl}Wzt29ObKStg z#3#^U>e+$r6-S7lj+xHAQi+G`*O{`8^z-~1*0pWge$Wk1QXlCvj8R%*&8?P??=+ED z>CbE(5-d7ScitUgPwSiQy-YEegSIGm5);OBrmS#Gl`_fG_vwwE@nOclE%=rTCq*i2H6(bo)iww{sIYb z@gvV~CfR#l5RSU=VV+;Uwh3$G)Sd@^(nOw>9TGcDOghiF40|b$2=v4_l>HWMl;BgV zC-p*$!Pf~`XN>(6$+{Si3Lnc1Nq|2-kfv$6&l=%y` zEfX8P$1Y{1&XXO5mMrpg7>hkyJ>o+gCu97|yU2g`43{#t!#$9 z4=8nq{SpoPFEs$c}wSQrc%58oM2S!0{`-L-vg( z@=V@=fy^P%S0pw)=4}ytxWBK)$H0SoASI(g-}fue=$AkCj1RH;!;CG-@yUd4jSsL8 zOo&ZU<(D#_M*oQVCAwqpQ!4v&u^#lPsgmI1L!Ls!MaMq$T1}E3bw9^FDlh)^`h>Kl zO%wNHCGWsM=5jd4NNkGdALU#cY%}+u#JY0Ebzux&#t?7#^y3HKVf<-A;ORHNdJn!4 z@qteAOq@Wv+aGNY@j)y~*{_hitMg<>p~Z(h)PqIuCZ8YDm}q=BPx8b2 z-*IR)=}Vg?@=V@=fy@;VlVY~7iY;;Y7`$SikGk>kDbMg1K63OzW0B~n@p*FIX9=+p zxn@FB96rFNq;i$?_tJL(KiVm^|D=1rMP4ULf?<;X>HKx&Y5B7?5$>s)2oLu?;@1lM?$5*k#$* ziOzk<^Ed3tp!LFyM}q5U-#^zRrzaD-)qaB72NFzxO`vGrlh{{E#uk{vpg#zFkQ5H) z-hTR?`Yq2LO_e}7L&zv>8@%=OQlSw42Oz9@im^bvt=VnmYGR<>7;%e{4e1-nih0lg92O=jRzD%a| zxp{EPp)P2AI1j|J7BC4Em&vyt!CsZK_^ym`w`I>A9g~fF@i0blJ*CMjy~pgQdUbf> zegA18kLG?uz<^i~6FvVNES`5W_CK@7r|=;i`AL0H# zvS$!=jXEdzpncUv3*v#NljoGtYa$_Y+zlD3gFrEr@2<=kv(fhU}KOz^b<^Ztv@(5Cc7OI%A+G zDA!AUDf3X*$&=l$qrCth?upZH(dWP?>G~+qpsErPZ7qvWnx@JkLl{M??w6k-0N@N`aPb@_DX|8H}@v7?XvJD`$>{l z@Ql4p>Rg{G81=4b6(0P?5sB-LxH7Wm0Anu^cghX%#mpqKa^2Ja$NnUdxAETG)D6er zxK5adQbMk3vXN(2IcE*@h zNidSQ)=BXB`2_p@x>tN2n7Jh-Hsn85eGWc|8>i3ZXka6K7JWR%apb-S;8pYO^$0v$ zAco6o;A_XqQT=@)jFQoN9^Sp|QsYlA_`HXGq)x(qQZalbz&%-g=5sN5+}MB@@nOgt zc*I^SX@6fJ?X0xoLd37xK^>Gnkg>MtQ>%FTLJ!t;2`xD4c=;0B9^>p6@y>V;*2Ak5 zHO}?Dm}BRsH##x*&9dC*iRw$5hxiIUpquj$;6Zp{TruzlY%=;jMe7;j-nU1L_k)5F zV~GhyXln*WiReXq`^fRPu8y2go1(_KdEYrV_at!hYhHWF=je}v4g4HpLmr3?dFoS{ z)%Q&fMtv#=hleaX%Ds|UzVaKz)**JG^tYggdXzjNW@^g+^I0!>yyj!ys@oNHUdC9& z27Nhs0Z*DgKaQb33q}^8T#3~sei3vC^)kk~9kZXYHSIX;jNR)_jFauY=f3umhtDL< zr|UwAfg3G)dyiU#U5_W^RS0j)MR2NB!OSH02L=p!&{kGO6-P#@!_N*qzf znY81E3Ql&sb@Z=svDW>Hu`Y4cU9pYPo?_ysGp??Voeo+_Gild+)aMcPT!?sVtJVGh zhy%$ul6HHO_?^<`V)aSlW1HfZgtZlaC7=5w*Ge>7Y6Lp61 zKtU_?CaZobT|7}=i*UaMPb0SX5j)l++neb}+VRSbHV69yel(v6aA2(L7);DO`Z< z^wMrJiH%Z6!3^iF7}hqHuDp2GiJQ!E0?SshWF1_nGmM)Gnt)4MV}?Zh`Ed^zVNAQ= z4#ZUbJO(HF+{B8pbQv30#wWI3iC-YTf#TW5ji{MLkP^;`)+e8SlpX3f|7_5HGx(SVpMH;y9Si#@Z9?`8#JTw8 zoNu82&%}Ew5AzBhh#AW=SQg7fig^d)2fdA12KWghUB=k1=yRWh4V{6uxGFqw$(Hb- zrC%Z;uB)Uk?R)qi^9)!t&W;CL#b+Bdz8~ftYVr{7F$&;f|1v(~+3J1`&XGy-jKN<7 zc9e7c;bZ(`hj6?~&iN=G=6yTl8fgrYjzgMvEiM}PhuEWMB2Hj2VpKhV zn2+NTe{MNqj(&=GefzAAvyMvQIq&g*%+q%A1M&=>ZD4YOAU5fhZjvY{KSOWG6S zk~S6&(8qklM;wXvyUpR7u>m#Dh+iVSGA5oR#>n%S5F^fhM&o7UXJZkA17m)UgbeZ< z5(mVvyZ~_;7fZXQOk()f{Nm{57qJ={yQI$KeokXZ-Enk6;Xpe9T7bg^4S@mTFb+WM zuRUU?sFRMo72le6iH+!iS1)+5EM2a|STt}T4H9Q@&K|Uhxb-o9VPdrzZy~nV6Np)a z^`nq^$wGZDk}`Ewi7IoJAD%B7;QRroqEV`>zEj`sxgzMLu>@}K(`0M zU>fxxP!3uq{k^B9le{J2D`^ zIK=d0*w%GBrw*o|`84o^c#ar^)cr#o$dqNJswd(9NE_P)XqEVRq#Zl}gS7eu@O}Z} zWMUn>!MmgDb;p?-`sa;xGmjitrW0-TK(&7{e>{zvP9>KDipo-nxkwU7PdU*x0OC}<}Sh~pt+OvHdo`F6pptIQ;?9d)m7g-l3m} z1(%Sl$(dAr1qVw>*=*=xBY1{-29 zWR$u;{;ntfnW<#ldl}V{ls4+CV4z}Z!WNK6w1c?$1H`erUHW-b??vA;j&j-ABWV*H3K)19ww z^29&B-H%>+_qwd>bLb@xyIku4Vm*xwqK}Zi9ym!HHT20&8Cmkq za#{?1E4l`8s=fRcZ$Rt|#%=SVUb_tNqI5ubWPjLs)>w!Cu4UCSp4GC4-pg}7$Btth zIQ2PooiN<+w;KOq#t}#Iq;-IO0eB|=(1SArMGIt~6}A!473N1Qw)#lVS8c14cV^gz zm<26w#y4<&(LtVQ4{6)z8-r)WfIAbh&Wc6%+26C>6DP`@@nY&BodKT6o0lK4@{I=a zM0-vhAg|H}W)>rc;sdge3T*@0huAv#yfV;ls)2aF}_xPq%g`|{O(>)7|>?6814fd#8B&iZUf!R#v|6&a^auf zd29abiEk6j@87#V%D&Fa@6LaI;*T6O{~XKF26Qi99_Ug2smy0--6u}upK+7wIXyai zs~7ggcRKIqYqFzvmtyd}~}!{^{G%kKprq`Axz2NI#eH zQMCcF0UQ^C_dp5bgh3xf_qFVOh`Z%6##qVIA9!J3^7MFWW9G5Tt_M%l0mfKmKLDP| zd!S@Nqw?Pwu^YVp`aTPv$RnUY^V`MfMRy~1PB!$!@lXA) z{1j=q@XzrM#whsqS=9sIb9nuw@BF$tig*K__`h$;W*=yw?tnM&h&adgc$mDGRE`(F zU*wOqy8LbzI5Q4WeWnV0tLx?W?v?}56Zp-0w)BuE^3Ohi_K!Xgcn=hpKM>-$rvdpN z_?&nDx5MLzZIdnUeBQ-Bc_#0HqPfMifA|hw>|c!h83V`X^zXN>j&9lI?fuZJZ|?A? z{uur_4&rOxBX21^Z%zxZ{y#cUml5`{fo6e(Ewh+ z6TYXm`#6q)z%yho>+2BnsTwh8Z2wQbDN?-idv_nxPab;La|~3(p7F14%2D$VAD6t6 zXFdND>{&53ME={Lr?suQ_3I@Xv>JTVE6Cr`@kz=o;dTL!Q#2i zf_v$?b@v0~?^)0Kao&oE?4-nkE0Q{ma|?kIm>8{{8kfS=U!fFXaawz>C;E@&+D* zy~<OqmT|=zcW+{LAG91^CC-4T{Px}6wm_P?ch8foeZAHE>c61;;48HT5Y8khA(fbGS z&QD{O^-=LIx=-2D=OHiTsdr_{^$+A9_~heTsKpXrh2NpG?ZAdOHh?;S?|kb@W$V4I zf8qH5B>2R2mFMaI68nL&S9VYOJoIOFJe9?W9aHD}7r~%+#Rd3IY!9DfEFaoW!2t6W zj(k#J&_9j&ea24l#9NB`Wo(ea7e^A>Jq-$FSBnV?~u9B1{xu+taM)Uoal(M2jKhqJ%R!IM%aV`*hRqs zx_~$t3G_k5l+m)R>xj0*SSwCK`=XAqY_YxMo3@zth%}IQ(8SmRiD>rw9&9T7gGGz+ zZ9Q}jX&7h_T>uVF{{SpVH*sGBs9 zmQ=LX=l@VQ#!OWYfRE z$eeYdPS#zYyrok%XuRNc!`6y@NsAbJE zQ8w#fUFp!)kak9ILJlCbh8swGP?BrN}Du?pBno*tdAN8ebhK< z+%%5*C}N>t#;_pZUdbZ_CIvt)QUK(n<)-E6$Q2ny&bHhcC*oqHK}b4I#F+DOQ4hcT zYyQDl5PcEjVI1N;l;PX)_pMH|cO%ZhQ9I$B_xL~NX+5ED{t%;KUKlAb#K8rF~mXz7*pDq2Q)hsLjs^>!ZZuKsq z+y1tmG~4SnL931>zRVxA%{Wp$ai2j3hPWy6EjZpuwox5^{V_nMS_ zDfY`Cw%89;f0LfgxvZCCS(I5R%H=-3Tt|;@L|Hf5#Z8&;fbXtuvbxW=+a|fA zMDjDw-^5;|VXmn#o^zkOuXXuauY?BBLYjbU$~K*U&GR9v^PCmrna^=OyIme=;QD?f zxi*fE&DbpaOeCV8yxjDBP0Vv<8SXM;{)calr7qGy94QCTmfz}?q_)-SFzqh&tsQh6 z?TNT?A1y_@l&`M!{gW{s!uN3r@hAN|_8yLYebkDWa`dyhT=P2a95e`y)Ft~Hxw{gz zXT(HkX?3gEE4G+>4MR_4-(l>-T-UMZxc5kE(%;LAx7Arj$~s63X+l3i8u2}ETt97f zn)#%B56t!7)N$%9zKfB4fcbrlO8QzK)&0l}wKp4?aO)Cpwf#NzopqIw=-Mo|_T1Jb z`#U|f1<-2Bz-sf_saAJjPZ`n15OeO?n@0NP?W-dD4kq5`i9A(*xuRS$_OE z?FRcG`hlHhj@B~3H)G4l7u&0RONBiH^*4H0Q|g@ap1#&c^WJfIC%gWW{c-w3eWDD$ zJa{yEA-;DopEaMa^PKuen?gP5R z->g+T?L9d&DxZ5Xf=2djpgGWSD!)_fE8i%h&Fb%TxM!^XKF&OsHU8><=6(11KNf!> zO_)r?x8-QG^y!UG`DU4QpX2?Ly~Z{oacMfw>>KDhcbxb(uTs78lZW5I$T~6DNBT(J=8nU+GG*dF4uEX9 z*DCi6_aTjXG4m--NYZDq&GI`Ed`}_!rK@jV%yaX9=KWOXV`tgl!jStbE!=5?|>=KV{?R7do;jEU%9Ub8P+ zh~BS|r!D+6(2n)e<76*n$VBMJ+A{8C%d(_A%T4hBB|c+9|QmA z!_4X+ZKRR3g62T+yuN{M)6NX`TY6OXG6!$6rdjPH%yo`>PhtJNg1**Acb;n=c`u@4 zF(ksj9N#D58SF^_`KpwxthoEs4fYqHpEP5yahbyomR2p3{)^ujFpreab(;1*$L4x} z{Xcy)^Bjk@0IN17_n!~od%5ra2H(va<(tpfc~06uqx4^(S^NKJ1EaBq7(N61Bic>D z-(DXtbs6i#^|d~7O}0M&<%Ca@>O5uf;Eb)Z23+UY>$%PAgy+N&Gzo2_k^4^fs;C{m zbf11|sQ;3^qE~v~9NCYAdy;rd=ij|v_I@LsiGEnm898Y*_R8~EUR*xxqa*twi|#Mk z8!VpVj2|h7Q0e?9xyPowKgGTndVRL+|7om|*4O%|)>cRK-s+^CHikca&RbpoxKZ|@ zWq#MQG1un*yAF+TZ@*V0ZaGF9-)*0o^k4eUhDOxQYNnT zk@tK2=>vyqO|wb=c&?|!Ud8S*5?!0+$~pNZ{oF5&-++KdtOt*qKT$Hc3+qbZN1!j` zzDJZb_Hr@zIdq?&g8dUe*cQJxIr-p^`!W9K-Rn}8;V#p;X5FNLy2Je?Nn7dsg!5OV zKiGXrDAa$^3huQiYZ9>D!oh>DdHlnit&X0UnCtU9Fy~82x_>rl^`A*@Zi3wv$u zb@3rGCmos>y%x~fLyfut9+C7~PuY)=>){mLvVWYE-GAX-q26;-+Q_W<=u_De>#m** z4|#)p#P1JBENMFuCH~L7znkpIgt7)MdJ=XoVZWLUy~&=%8vauukI|6zGVXsG=DIQ_ z?wiFmcJ_KZyC3AazHjHlT4y}NXZtT49qLmNv60QsJm^)i^8MxlZdGq^psfby_FkBEw&^@7!{Nbv_A>o|SDyX!{;bK=HlIgjW{b5`#(YPV0H znY%SEpZhtvW|=5kWP!R&IpkmQYNixknid|q_cjuI?9+g=8u@PANy|I&+}a}brE&rJ;^rW zq2F=p_nTghqhJ2mtX|F;f9iO=yGi*x-}RwR)*a}z=m_Rb?fiVtltwDRoGRZHxqtUf zM*Oc-SW!4-mD*b$NcKuoYy1kU$!y6e#U7_SqAE2ova&a8{WIw`8h|NMjG`} zJpTMXp7Fm@mcw_Es8j0hz3^Yuy)f?S_a$ zYyMxP?eQH2zsvL(L+Yia@$GNI@|_-@N8Nna!#0Sz1Kp;UN}La=pL0`nr#6qhA7_7d z?>`d#PsFXM^Lg+m^)sec(d<&|NB5i%?7ws$-ye91pL6_8tE1JMx%meLj#wr5k27de4O(FraJZhYmLdaoPW0aaP$nsJkxoS9#Ix!Jg{um5xC~DIRA^Z z1iQR)erWI;pM^NzBlD7q2Fwk8=eO0-D;{|_CS5oNF<9(*%EU*~&$>|-%VgQX?h|5g zH!FgE}Hm0fFW6jd1Qc4oJ!KhRW-6d&Dr&w#^4GaE^1KaI84Ttdo0ET$;h7oyC~k_%yX&cq-J9apwGA*Zg$OQ+6Qbd8@Y7 zS})q1iSJEs&3O6Lf8<=Cf7q9k6fQA4ou1lPs6&fp)dK!@qVkeQmx%9Tjys~ z_v`@Y`mb=W2}VTTSnFPvtScUc7C_{&y^lv0z|eNFRW6;ds02hrmpPK`$^hgFawN zo8_YFg-ts{uu0o&!=ftCHhGGL3b-E!UYicc%v6TsN84)77fddabINyu}X zw&crzYr%RSnwZs+p8L8N-+^ku_njt0JJ0bO)Uy&hQ!C2EF*Y0%fHEwbp4Z@=Nh4a( z&MLh$JW78|jhhiam3mY9NYrwgNw#ww4E43BqZNNqm8uQx@wP^A2N;iI@&z;X+u1Jl zi>Qn5S2jAep%s5#g>U;A$3{Ik{K|Rj`l5SIj70ZFMq)_OZ9n`;AJ)*in`K1zuJSxD z*wB{lwVg+LUcDFJSr`=`kva|}!Rc7AF0vjOoJ~+t~(e!bkP#%i03)nM8+d=$WC_fmS#^&u^ggTo|>k6SY`{@13`m zi7zi0klOR74hEo^><~W)Hd6U;i{2iv&BNbEts3VGbMza5ap~v4wq70?=J6VuQ20O4 zk|xW4h)WW$Ib1;vflg~t(w5?b4 zj>|DnXGH%2YtiI6rF3DT_$Y1{gDpLPNMQo4wFlj5=ZLou;;4J(IP&e@~99@%##whAsWk%~t=$cHCtLtDq}@ zP6pSB@mJus+nf5Mt*8Hvw&A;CNYRg>ab#>=#s68x${Sc2nl1EQtG~DP&+0r7M{ESn zVXTp{Vu@2TR&CBaU*u3mg7SoC}87n8zS^ z6BrD2JZst=k6YPRYkl!1FYr_g&IQjEV&KO7E6@W0yQ@d=}nTELyxGtbS@@`6e5>0ytF|nMqs|XSeA#fB%FsT`ciD{;b zD$lyKL0hy*+r(iE4|$Ok4lB~};~I-={tNkdj|cUV9|!fa?lDQoYfHZhy6&nbtaDzn zax=9jT0_82ljLKKaje8ENW>t-SlFjMxi6G<<3c+k%(^dATXCdY@C|sxYU`j}$#^_42blkoId5rr;2(nrPQVMS<65vbUtiu9l)B)*$bqS};)4t9yX5cx zQs`G}yV@`B(Wegt-Q|(NWL)WfbKcy)I4>@qVH=N1z6o{Rl_$r9n{?m@eVB`o?-jZoJe2A29}QtHph?k( zSR8@!{n3YEpQoZTl%7zR_0^&++K}&g{%|m(_S7J>3}cdzhwY&Dz-!a~zgH2X4kM1; PjBbbNT8k>jI;{IY2~2bs diff --git a/app/assets/images/ci_favicons/canary/favicon_status_scheduled.ico b/app/assets/images/ci_favicons/canary/favicon_status_scheduled.ico new file mode 100644 index 0000000000000000000000000000000000000000..5444b8e41dce57740b6c3d90a4b7aa9495e9933e GIT binary patch literal 5430 zcmcIoe@v9;9e=e=x<8h#X_lqoj)TMFeGfFOgv?=M1X45|F=ksNWc|Y_Vr^sNuwpHD z2OgD8ojRMkObd<{aa3_*wJBv&Lus>UT`3erpe=+0kwB*?6zlcwh3~$f=XlQdeid2Q zC;8s<<9)v0&-d5!>l9^yvQT;C5rykS<=F=nMO73fF)`3T$>V?EF*}cOlV=3Rxte#e z+wDJ8Rdw&OWy?NWyLPQFH#c`g)3oC}mO7U(o|n2}#fqV`XV22~^fbwG_3BkBC@Ao< z?6t9Eq@|^OFEcaqv!S6O!RGaPskgV6CMG6=eaMKV2g@$1si_fjySln)-MV$OfB$|e zDk`GJ#zrv?Jzz_?4%#t#RL%K$`|$9vn73)uCK?|f7wwsu86iXJ0e=|z)l<_ev}XTE za^A1$&Ye4gr?9Y4)LXV}q3Y^tp$BY1flnpB);RsLcHU36Q(uv-;6>``=@E0bZ{JRx zot?C2&mOvY^QN!`{(w&r7p;cOYLov1?L9xKfA><-i3zgqXr#)@O2L2Q#tkYdDIw?q zTi_2A#09a{n)NRP@4G%qs+$!2rrn*C{A2;Ow6uu1;XjBAVk`3F9e=NSj(JafDR`5r z$0^Qzk>Z>!LI(7JE$|0?inwHEF6%>nz;^X*-DP|8PJownx4beC;D-#R_-9@tTmFlb z{Ihjr`^EF5?)ZyW`<46q(r<~o$5Ku1Psrr%5M(VIEVerO^NbX~tLK9JZm#J5tK^ln zsqAA9uuNrbx8<0zHao8r3!9*iv%ARn4Q*3-d!^tnZ#x?P4gI8-Jdefg`jFywUD9o3 zgF)Tp+=SBprQo-ee)>x}7G5((8`IhFaoIP!u5@R-Mhj&>yf(W!Ux;^fJrh1o4=Kvb z1B#+AVB(KnCODy$$<4%@jH2*&H_tn?hp8h#16&VKV4q3kDcMZekG`vtpP#=lB_$<~ zbM`3b-DPf%a}IgA_Hcck$6r{yc(LWXz-PSGjM2>)v}DN=eZz(gG;O;d|bcE^`rj}Jzz7-Isn-@IXU{7GiL~EKU(SU@2AqzQer)!#}NA>?c3_Wvar{a z%jF`U&le3gVyvU11N)NBI!t{V9T>X=*ar_DjKG*{;QICJQU~l&^Nt0dKW4`F;=G7f zu3ft(_Ro_iPf~k(d(@b~4~PZwVZL!-o}--C`rzPT2>jI46uI4QV*NsB<;sZ>efFApUFFhl(l^Ma@x0V zpX^d|b2FuhOZKrgjRku(uYt?q`1abFMy*9Bfz7rj*L@whgBhN4^ihc`>IU4OCIVD9 zLCH_#QSQoBA@^p;#hQQ)@B?Cje8Alne#hFx{$MOPFObXZ|2){wcg3x0vtJLveuw>j zd=?uz9OXOMhC_m{p`pPjPPDP^5mV#?auVwTYZLo}vEaPGnZ>!+EzU9c`@MVq7CP{K zchMX7?tbt=;{FkF?@NB-Df;o_=@IMDi1AHroLR7)W4{`<+naS?h~0>J?cIBRpY^}# zDS?T@m=u0fd~yx_}VP`wP@mYNY`>`{+jFmyZAU#V`rZ3v zdwr6$)B3QqH(G7|?bz?*-MtiztQEtT`OWgjIjLf#JsLm8td*m$M8lP136B3fW^!MO zYNypX81p?Y`BC${)kc2=+ZC56$vraX{oCTaG8*KWjjg$?J@^hbV#R8k%R0OfGF)9> z#F}p$PjGyGES%Tk=>4~G&RlJaAK=}Y>WB>flO6Ru?d literal 0 HcmV?d00001 diff --git a/app/assets/images/ci_favicons/favicon_status_manual_with_auto_play.png b/app/assets/images/ci_favicons/favicon_status_manual_with_auto_play.png deleted file mode 100644 index 3ca612a542dc4e5585ed2d829bbc651409bde3b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1338 zcmV-A1;zS_P)+}A6Ki|(o@Q{r>44E_Jm1~j?ftjA z=7AMpjFI*0*Y9GC6*0y(0xRznQ5661`Ft%povz~AwQHlRfjkg^$z*z(5Yj>^-53=W z<;%#(5EBv-U^E&L92|_v$w|nvj5BA>puN4FQc699kWyKetL}!lZveB|{4ycro1mZ| zeMw1)n4O&sK@je#^2o>tDk>^iPfrgagfv+!mR!5t?pt!WF94poAP5%?27|u7zFv%u zj#fQNs;jF}TU*Ovt7Tb!=e_~{srdz+P8ZkO+PZ4+si`T<&(9+)EDTz$b_Lvl0|(I3 z(!wFqWm!JWH>g&?Y&KgdrC%I6bO<>)IRP)9ot=eVuSZ5k2AobOs;a86b?a70k_3yz z;@|7>c(7&57N6Vgc3CWzSY8U%049^^LS$s*i|y_0;_{l8m6f5Zs|#^)amt~ilq%VW zhlgWqYz#FuHAqfQ_CuUHbqdAB#Sld?O_HQGRRFxt8Dpb1n~f9{6!>2@Gc$vfloW93 z?`21h98r{6GG3zi_;|XwxLD_Mx%Q|6Fq_R;l+xDb=4K_+<;H-P zQj{uy+SgxsJ2fn{MneeLy2nB1Zb9@c2Go^9ynk~J6Z%l}_4O%b;bynn16q-<@m3rj z9`1HJoyn>I+}5=32@w*xbMWRgXkXimndU~|=gT0B`9Utu%wga48v$U;&jx=mI5?Y+$&^r8uYvOfKiIsOFv=m%O(i!QRPOv?eih!azAV~8xj)} z;cz&xW53(oeqZj zmKA~dg4^xJ_U+q!)6>&q0A7$~S-HK`0a&+nfsiEb5cFyiSXemb+rEZy=XOA_H5+0P zy1#PO%=hly%Y)C0qL|=tIIb>jQUx$>O-W)z^DTfij0gbrfsjB~qS0X8yJcAO-1B|_ ze&Fir>iDI@2q8B_QQWpXxM~IX#&K&}AtSG9eSLktp`jr@ zxV)6oV}c-*FUh{#UzGw&4&&DJBnGx01n~e_3$QQ w6O_{HjIlPYR{Iq{?W-OPwP)a7&i}>wUoM&|Kb2`CjQ{`u07*qoM6N<$f>P0mY5)KL diff --git a/app/assets/images/ci_favicons/favicon_status_scheduled.png b/app/assets/images/ci_favicons/favicon_status_scheduled.png new file mode 100644 index 0000000000000000000000000000000000000000..d198c255fddc10600c27c9f0010b1371b0af19af GIT binary patch literal 1072 zcmV-01kd}4P)Px&>PbXFR9Fe^R!c}*K@`0)Rn!(ygILm1?Z&@Wv5KFKBBFxWm5}ZQH&W=Lg5W}^ zh?E8dE4B(!Q$$1r9UUFf+S(e8kB`&n=qS{D9_Tk??R_vz-RmhSDTVO-I6FIAaX1_T9wGq2 zWpi^=)YaAbl9G}H4DYM0tu^Z=3shv!nY^T=L~%Nu!C9}_vD@u}jerC8KwMqFfoa0w zuk5)|a?PIR=4Q=^Y9=*w2SEN3x$JLkZ6$>9lFV1sQNQ>tyJ-TCtjM3Qv$Io+raEqHY>52) zd{JLtFHTNQ#QglcXm4*<_dp_Yb8|&YOUtD*XJ=<3BO}8LyiXxF5o`PKn464=OHG!Z zo*uRAKuXck(KIvP7 z%mX74aPmoRud;Bd&CSiJEmvOWxuPi4(a}Ni@$rAR~I|~L0i)`a@n8_ zd%3sE;&QpvQbS`A@`oxAs0vwCoi{Z#aXa>;r>8&BMXV_S{4sLw=7;49Di*f3wglJy zz`%f*nVAu8xBH43BEt_7KTMq(uAvc!-{7#|hsCJu?Cglb!a@b!Z{f2TB9@h5L?0W$ z56k)axh^}{^FQP+g#MX~XL!!Dbt?lHh@*jaT!DW@FF;Vl?G+PR!?=;q(9l6Ns~Anchor-with-border \ No newline at end of file diff --git a/app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg b/app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg deleted file mode 100644 index a08c43b156f..00000000000 --- a/app/views/shared/icons/_icon_status_manual_with_auto_play_borderless.svg +++ /dev/null @@ -1 +0,0 @@ -Anchor-with-border \ No newline at end of file diff --git a/app/views/shared/icons/_icon_status_scheduled.svg b/app/views/shared/icons/_icon_status_scheduled.svg new file mode 100644 index 00000000000..ca6e4efce50 --- /dev/null +++ b/app/views/shared/icons/_icon_status_scheduled.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/views/shared/icons/_icon_status_scheduled_borderless.svg b/app/views/shared/icons/_icon_status_scheduled_borderless.svg new file mode 100644 index 00000000000..dc38c01d898 --- /dev/null +++ b/app/views/shared/icons/_icon_status_scheduled_borderless.svg @@ -0,0 +1 @@ + \ No newline at end of file -- GitLab From e265fc3e28dbfe53d96646588a1587d5626e92de Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 18 Sep 2018 15:36:03 +0900 Subject: [PATCH 005/118] Rename delayed to scheduled --- app/models/ci/build.rb | 23 ++++++++++++------- app/workers/build_finished_worker.rb | 1 - app/workers/ci/build_schedule_worker.rb | 6 ++++- lib/gitlab/ci/config/entry/job.rb | 2 +- lib/gitlab/ci/status/build/factory.rb | 2 +- .../status/build/{delayed.rb => scheduled.rb} | 22 ++++++++++-------- 6 files changed, 34 insertions(+), 22 deletions(-) rename lib/gitlab/ci/status/build/{delayed.rb => scheduled.rb} (53%) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index be4a6c553e1..6ea574ed8ec 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -186,9 +186,8 @@ def retry(build, current_user) end after_transition any => [:manual] do |build| - puts "#{self.class.name} - #{__callee__}: 1" build.run_after_commit do - build.schedule_delayed_execution + build.schedule end end @@ -237,22 +236,30 @@ def playable? action? && (manual? || retryable?) end - def delayed? + def schedulable? manual? && options[:start_in].present? end - def execute_at - ChronicDuration.parse(options[:start_in])&.seconds&.from_now + def scheduled? + build.build_schedule.exist? end - def schedule_delayed_execution - return unless delayed? + def schedule + return unless schedulable? create_build_schedule!(execute_at: execute_at) end + def unschedule + build&.build_schedule&.delete + end + + def execute_at + ChronicDuration.parse(options[:start_in])&.seconds&.from_now + end + def action? - self.when == 'manual' || self.when == 'delayed' + %w[manual delayed].include?(self.when) end # rubocop: disable CodeReuse/ServiceClass diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index 889384d6be8..51cbbe8882e 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -9,7 +9,6 @@ class BuildFinishedWorker # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) Ci::Build.find_by(id: build_id).try do |build| - build&.build_schedule&.delete # We execute that in sync as this access the files in order to access local file, and reduce IO BuildTraceSectionsWorker.new.perform(build.id) BuildCoverageWorker.new.perform(build.id) diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 448fb5bf41e..9f81aa3c71e 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -9,7 +9,11 @@ def perform(build_id) ::Ci::Build.preload(:build_schedule).find_by(id: build_id).try do |build| break unless build.build_schedule.present? - Ci::PlayBuildService.new(build.project, build.user).execute(build) + begin + Ci::PlayBuildService.new(build.project, build.user).execute(build) + ensure + build.unschedule + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index fa64041f7db..02589d147ef 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -119,7 +119,7 @@ def commands end def manual_action? - self.when == 'manual' || self.when == 'delayed' + %w[manual delayed].include?(self.when) end def ignored? diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 0fbab6e7673..3f762c42747 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -5,7 +5,7 @@ module Build class Factory < Status::Factory def self.extended_statuses [[Status::Build::Erased, - Status::Build::Delayed, + Status::Build::Scheduled, Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, diff --git a/lib/gitlab/ci/status/build/delayed.rb b/lib/gitlab/ci/status/build/scheduled.rb similarity index 53% rename from lib/gitlab/ci/status/build/delayed.rb rename to lib/gitlab/ci/status/build/scheduled.rb index 553d4cf8a71..93da8fb9538 100644 --- a/lib/gitlab/ci/status/build/delayed.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -2,11 +2,7 @@ module Gitlab module Ci module Status module Build - class Delayed < Status::Extended - ### - # TODO: Those are random values. We have to fix accoding to the UX review - ### - + class Scheduled < Status::Extended ### # Core override ### @@ -23,7 +19,7 @@ def icon end def favicon - 'favicon_status_manual_with_auto_play' + 'favicon_status_scheduled' end ### @@ -33,17 +29,23 @@ def illustration { image: 'illustrations/canceled-job_empty.svg', size: 'svg-394', - title: _('This job is a scheduled job with manual actions!'), - content: _('auto playyyyyyyyyyyyyy! This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + title: _("This is a scheduled to run in ") + " #{execute_in}", + content: _("This job will automatically run after it's timer finishes. Often they are used for incremental roll-out deploys to production environments. When unscheduled it converts into a manual action.") } end def status_tooltip - @status.status_tooltip + " (scheulded) : Execute in #{subject.build_schedule.execute_in.round} sec" + "scheduled manual action (#{execute_in})" end def self.matches?(build, user) - build.delayed? && !build.canceled? + build.schedulable? && !build.canceled? + end + + private + + def execute_in + Time.at(subject.build_schedule.execute_in).utc.strftime("%H:%M:%S") end end end -- GitLab From 22e00b08e89f72eb0fefea2d1d623667c4461773 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 18 Sep 2018 15:53:31 +0900 Subject: [PATCH 006/118] Make schedule and unschedule consistent --- app/models/ci/build.rb | 10 +++++----- app/workers/ci/build_schedule_worker.rb | 9 +++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 6ea574ed8ec..b608bfdb8b9 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -187,7 +187,7 @@ def retry(build, current_user) after_transition any => [:manual] do |build| build.run_after_commit do - build.schedule + build.schedule! end end @@ -241,17 +241,17 @@ def schedulable? end def scheduled? - build.build_schedule.exist? + build_schedule.present? end - def schedule + def schedule! return unless schedulable? create_build_schedule!(execute_at: execute_at) end - def unschedule - build&.build_schedule&.delete + def unschedule! + build_schedule.delete! end def execute_at diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 9f81aa3c71e..84ef2edb767 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -7,13 +7,10 @@ class BuildScheduleWorker def perform(build_id) ::Ci::Build.preload(:build_schedule).find_by(id: build_id).try do |build| - break unless build.build_schedule.present? + break unless build.scheduled? - begin - Ci::PlayBuildService.new(build.project, build.user).execute(build) - ensure - build.unschedule - end + build.unschedule! + Ci::PlayBuildService.new(build.project, build.user).execute(build) end end end -- GitLab From 1a6a59d6bc3ab2fc43cd3537ef9d8deea7398cc9 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 18 Sep 2018 16:12:13 +0900 Subject: [PATCH 007/118] Add unschedule endpont to job controller --- app/controllers/projects/jobs_controller.rb | 7 +++++++ config/routes/project.rb | 1 + 2 files changed, 8 insertions(+) diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index 3f85e442be9..d4a0af6f0f9 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -110,6 +110,13 @@ def cancel redirect_to build_path(@build) end + def unschedule + return respond_422 unless @build.scheduled? + + @build.unschedule + redirect_to build_path(@build) + end + def status render json: BuildSerializer .new(project: @project, current_user: @current_user) diff --git a/config/routes/project.rb b/config/routes/project.rb index 8a5310b5c23..04a270c5cc9 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -275,6 +275,7 @@ member do get :status post :cancel + post :unschedule post :retry post :play post :erase -- GitLab From f9877d715fd9ac152a8b7e88d4e740c95cb0b962 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 20 Sep 2018 08:18:49 +0200 Subject: [PATCH 008/118] Add placeholders to debug script example --- scheduled_job_fixture.rb | 104 ++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 50 deletions(-) diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb index 55f50e941a4..6a5000e57fc 100644 --- a/scheduled_job_fixture.rb +++ b/scheduled_job_fixture.rb @@ -8,52 +8,52 @@ # ### How to use ### # # ### Prerequisite -# 1. Create a project +# 1. Create a project (for example with path `incremental-rollout`) # 1. Create a .gitlab-ci.yml with the following content # -# ``` -# stages: -# - build -# - test -# - production -# - rollout 10% -# - rollout 50% -# - rollout 100% -# - cleanup -# -# build: -# stage: build -# script: sleep 1s -# -# test: -# stage: test -# script: sleep 3s -# -# rollout 10%: -# stage: rollout 10% -# script: date -# when: delayed -# start_in: 10 seconds -# allow_failure: false -# -# rollout 50%: -# stage: rollout 50% -# script: date -# when: delayed -# start_in: 10 seconds -# allow_failure: false -# -# rollout 100%: -# stage: rollout 100% -# script: date -# when: delayed -# start_in: 10 seconds -# allow_failure: false -# -# cleanup: -# stage: cleanup -# script: date -# ``` +=begin +stages: +- build +- test +- production +- rollout 10% +- rollout 50% +- rollout 100% +- cleanup + +build: + stage: build + script: sleep 1s + +test: + stage: test + script: sleep 3s + +rollout 10%: + stage: rollout 10% + script: date + when: delayed + start_in: 10 seconds + allow_failure: false + +rollout 50%: + stage: rollout 50% + script: date + when: delayed + start_in: 10 seconds + allow_failure: false + +rollout 100%: + stage: rollout 100% + script: date + when: delayed + start_in: 10 seconds + allow_failure: false + +cleanup: + stage: cleanup + script: date +=end # # ### How to load this script # @@ -65,12 +65,16 @@ # ### Reproduce the scenario A) ~ Succeccfull timed incremantal rollout ~ # # ```` -# ScheduledJobFixture.new(29, 1).create_pipeline('master') -# ScheduledJobFixture.new(29, 1).finish_stage_until('test') # Succeed 'build' and 'test' jobs. 'rollout 10%' job will be scheduled. See the pipeline page -# ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 10%') # Succeed `rollout 10%` job. 'rollout 50%' job will be scheduled. -# ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 50%') # Succeed `rollout 50%` job. 'rollout 100%' job will be scheduled. -# ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 100%') # Succeed `rollout 100%` job. 'cleanup' job will be scheduled. -# ScheduledJobFixture.new(29, 1).finish_stage_until('cleanup') # Succeed `cleanup` job. The pipeline becomes green. +# project = Project.find_by(path: 'incremental-rollout') +# user = User.find_by(username: 'root') +# fixture = ScheduledJobFixture.new(project.id, user.id) +# +# fixture.create_pipeline('master') +# fixture.finish_stage_until('test') # Succeed 'build' and 'test' jobs. 'rollout 10%' job will be scheduled. See the pipeline page +# fixture.finish_stage_until('rollout 10%') # Succeed `rollout 10%` job. 'rollout 50%' job will be scheduled. +# fixture.finish_stage_until('rollout 50%') # Succeed `rollout 50%` job. 'rollout 100%' job will be scheduled. +# fixture.finish_stage_until('rollout 100%') # Succeed `rollout 100%` job. 'cleanup' job will be scheduled. +# fixture.finish_stage_until('cleanup') # Succeed `cleanup` job. The pipeline becomes green. # ``` class ScheduledJobFixture attr_reader :project -- GitLab From 7fffe0fb37fae4944b1a154e04a44006cee528f2 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 20 Sep 2018 09:29:41 +0200 Subject: [PATCH 009/118] Ensure that execute_in of a build schedule is not negative --- app/models/ci/build_schedule.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/ci/build_schedule.rb b/app/models/ci/build_schedule.rb index 7f0a34b246d..d6378224fe0 100644 --- a/app/models/ci/build_schedule.rb +++ b/app/models/ci/build_schedule.rb @@ -11,7 +11,7 @@ class BuildSchedule < ActiveRecord::Base after_create :schedule, unless: :importing? def execute_in - self.execute_at - Time.now + [0, self.execute_at - Time.now].max end private -- GitLab From 33304b32573ac8215e24d17cff52c5a0c7bbed2e Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 20 Sep 2018 09:30:43 +0200 Subject: [PATCH 010/118] Use secondary color for SVG icons in CI buttons --- app/assets/stylesheets/framework/buttons.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index 686ce0c63a4..c4296c7a88a 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -360,6 +360,10 @@ i { color: $gl-text-color-secondary; } + + svg { + fill: $gl-text-color-secondary; + } } .clone-dropdown-btn a { -- GitLab From af51b95442aa867bd570b99d32f8b580f554675d Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 20 Sep 2018 09:31:08 +0200 Subject: [PATCH 011/118] Add button group for scheduled jobs --- app/views/projects/ci/builds/_build.html.haml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index 44c1453e239..1ba8b698fe2 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -101,6 +101,17 @@ - if job.active? = link_to cancel_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Cancel', class: 'btn btn-build' do = icon('remove', class: 'cred') + - elsif job.scheduled? + .btn-group + .btn.btn-default.has-tooltip{ disabled: true, + title: job.build_schedule.execute_at } + = sprite_icon('planning') + = duration_in_numbers(job.build_schedule.execute_in) + .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Start now') } + = sprite_icon('play') + .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Unschedule') } + = sprite_icon('cancel') + -# sprite_icon('status_scheduled_borderless') - elsif allow_retry - if job.playable? && !admin && can?(current_user, :update_build, job) = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Play', class: 'btn btn-build' do -- GitLab From a7c767f16446f71f6e35a5aa3d2fdc73037fcdf5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 21 Sep 2018 11:17:37 +0900 Subject: [PATCH 012/118] Add a new status to ci_builds.status --- app/controllers/projects/jobs_controller.rb | 2 +- app/models/ci/build.rb | 42 ++++++++++----------- app/models/ci/build_schedule.rb | 10 +++++ app/models/commit_status.rb | 8 ++-- app/models/concerns/has_status.rb | 16 ++++---- app/services/ci/process_pipeline_service.rb | 16 ++++++-- app/workers/ci/build_schedule_worker.rb | 3 +- lib/api/jobs.rb | 2 +- lib/gitlab/ci/config/entry/job.rb | 12 ++++-- lib/gitlab/ci/status/build/scheduled.rb | 2 +- 10 files changed, 66 insertions(+), 47 deletions(-) diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index d4a0af6f0f9..9c9bbe04947 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -113,7 +113,7 @@ def cancel def unschedule return respond_422 unless @build.scheduled? - @build.unschedule + @build.unschedule! redirect_to build_path(@build) end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b608bfdb8b9..3ef149f3632 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -160,6 +160,22 @@ def retry(build, current_user) transition created: :manual end + event :schedule do + transition created: :scheduled + end + + event :unschedule do + transition scheduled: :manual + end + + before_transition created: :scheduled do |build| + build_build_schedule(execute_at: execute_at) + end + + before_transition scheduled: any do |build| + build_schedule.delete! + end + after_transition any => [:pending] do |build| build.run_after_commit do BuildQueueWorker.perform_async(id) @@ -185,12 +201,6 @@ def retry(build, current_user) end end - after_transition any => [:manual] do |build| - build.run_after_commit do - build.schedule! - end - end - before_transition any => [:failed] do |build| next unless build.project next if build.retries_max.zero? @@ -233,25 +243,11 @@ def pages_generator? end def playable? - action? && (manual? || retryable?) + action? && (manual? || scheduled? || retryable?) end def schedulable? - manual? && options[:start_in].present? - end - - def scheduled? - build_schedule.present? - end - - def schedule! - return unless schedulable? - - create_build_schedule!(execute_at: execute_at) - end - - def unschedule! - build_schedule.delete! + self.when == 'delayed' && options[:start_in].present? end def execute_at @@ -259,7 +255,7 @@ def execute_at end def action? - %w[manual delayed].include?(self.when) + %w[manual scheduled].include?(self.when) end # rubocop: disable CodeReuse/ServiceClass diff --git a/app/models/ci/build_schedule.rb b/app/models/ci/build_schedule.rb index d6378224fe0..4128fade86c 100644 --- a/app/models/ci/build_schedule.rb +++ b/app/models/ci/build_schedule.rb @@ -8,14 +8,24 @@ class BuildSchedule < ActiveRecord::Base belongs_to :build + validate :schedule_at_future + after_create :schedule, unless: :importing? + scope :stale, -> { where("execute_at < ?", Time.now) } + def execute_in [0, self.execute_at - Time.now].max end private + def schedule_at_future + if self.execute_at < Time.now + errors.add(:execute_at, "Excute point must be somewhere in the future") + end + end + def schedule run_after_commit do Ci::BuildScheduleWorker.perform_at(self.execute_at, self.build_id) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index fe2f144ef03..6bf2888505e 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -26,7 +26,7 @@ class CommitStatus < ActiveRecord::Base scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) end - + scope :exclude_ignored, -> do # We want to ignore failed but allowed to fail jobs. # @@ -71,7 +71,7 @@ class CommitStatus < ActiveRecord::Base end event :enqueue do - transition [:created, :skipped, :manual] => :pending + transition [:created, :skipped, :manual, :scheduled] => :pending end event :run do @@ -91,10 +91,10 @@ class CommitStatus < ActiveRecord::Base end event :cancel do - transition [:created, :pending, :running, :manual] => :canceled + transition [:created, :pending, :running, :manual, :scheduled] => :canceled end - before_transition [:created, :skipped, :manual] => :pending do |commit_status| + before_transition [:created, :skipped, :manual, :scheduled] => :pending do |commit_status| commit_status.queued_at = Time.now end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index e2700db1438..0a97e4cdfc4 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -4,14 +4,15 @@ module HasStatus extend ActiveSupport::Concern DEFAULT_STATUS = 'created'.freeze - BLOCKED_STATUS = 'manual'.freeze - AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual].freeze - STARTED_STATUSES = %w[running success failed skipped manual].freeze + BLOCKED_STATUS = %w[manual scheduled].freeze + AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual scheduled].freeze + STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze ACTIVE_STATUSES = %w[pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze - ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze + ORDERED_STATUSES = %w[failed pending running manual scheduled canceled success skipped created].freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + failed: 4, canceled: 5, skipped: 6, manual: 7, + scheduled: 8 }.freeze UnknownStatusError = Class.new(StandardError) @@ -24,6 +25,7 @@ def status_sql created = scope_relevant.created.select('count(*)').to_sql success = scope_relevant.success.select('count(*)').to_sql manual = scope_relevant.manual.select('count(*)').to_sql + scheduled = scope_relevant.scheduled.select('count(*)').to_sql pending = scope_relevant.pending.select('count(*)').to_sql running = scope_relevant.running.select('count(*)').to_sql skipped = scope_relevant.skipped.select('count(*)').to_sql @@ -31,6 +33,7 @@ def status_sql warnings = scope_warnings.select('count(*) > 0').to_sql.presence || 'false' "(CASE + WHEN (#{scheduled})>0 THEN 'scheduled' WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' WHEN (#{builds})=(#{skipped}) THEN 'skipped' WHEN (#{builds})=(#{success}) THEN 'success' @@ -92,8 +95,7 @@ def all_state_names scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } scope :cancelable, -> do - where("status IN ('running', 'pending', 'created') OR " \ - "(status = 'manual' AND EXISTS (select 1 from ci_build_schedules where ci_builds.id = ci_build_schedules.build_id))") + where("status IN ('running', 'pending', 'created', 'scheduled')") end end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 323075d404b..0a13da198cd 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -37,7 +37,7 @@ def process_stage(index) def process_build(build, current_status) if valid_statuses_for_when(build.when).include?(current_status) - build.action? ? build.actionize : enqueue_build(build) + proceed_build(build) true else build.skip @@ -53,8 +53,10 @@ def valid_statuses_for_when(value) %w[failed] when 'always' %w[success failed skipped] - when 'manual', 'delayed' + when 'manual' %w[success skipped] + when 'delayed' + %w[success skipped] # This might be `success` only else [] end @@ -102,8 +104,14 @@ def update_retried end # rubocop: enable CodeReuse/ActiveRecord - def enqueue_build(build) - Ci::EnqueueBuildService.new(project, @user).execute(build) + def proceed_build(build) + if build.schedulable? + build.schedule! + elsif build.action? + build.actionize + else + Ci::EnqueueBuildService.new(project, @user).execute(build) + end end end end diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 84ef2edb767..0d17a960c00 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -6,10 +6,9 @@ class BuildScheduleWorker include PipelineQueue def perform(build_id) - ::Ci::Build.preload(:build_schedule).find_by(id: build_id).try do |build| + ::Ci::Build.find_by(id: build_id).try do |build| break unless build.scheduled? - build.unschedule! Ci::PlayBuildService.new(build.project, build.user).execute(build) end end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 63fab6b0abb..fa992b9a440 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -151,7 +151,7 @@ class Jobs < Grape::API present build, with: Entities::Job end - desc 'Trigger a manual job' do + desc 'Trigger a actionable job (manual, scheduled, etc)' do success Entities::Job detail 'This feature was added in GitLab 8.11' end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 02589d147ef..3ad048883af 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -35,11 +35,11 @@ class Job < Node validates :dependencies, array_of_strings: true validates :extends, type: String - with_options if: :manual_action? do - validates :start_in, duration: true, allow_nil: true + with_options if: :delayed? do + validates :start_in, duration: true, allow_nil: false end - with_options unless: :manual_action? do + with_options unless: :delayed? do validates :start_in, presence: false end end @@ -119,7 +119,11 @@ def commands end def manual_action? - %w[manual delayed].include?(self.when) + self.when == 'manual' + end + + def delayed? + self.when == 'delayed' end def ignored? diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 93da8fb9538..010d5e2142f 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -39,7 +39,7 @@ def status_tooltip end def self.matches?(build, user) - build.schedulable? && !build.canceled? + build.scheduled? end private -- GitLab From 1a4f497e6093c8d1005986467c8b752cc61c6629 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 21 Sep 2018 15:24:19 +0900 Subject: [PATCH 013/118] Update pipelines and stages status as well --- app/helpers/ci_status_helper.rb | 1 + app/models/ci/build.rb | 4 +- app/models/ci/pipeline.rb | 5 ++ app/models/ci/stage.rb | 5 ++ app/models/concerns/has_status.rb | 2 + lib/gitlab/ci/status/build/scheduled.rb | 24 +---- lib/gitlab/ci/status/scheduled.rb | 23 +++++ scheduled_job_fixture.rb | 114 ++++++++++++++++++++---- 8 files changed, 138 insertions(+), 40 deletions(-) create mode 100644 lib/gitlab/ci/status/scheduled.rb diff --git a/app/helpers/ci_status_helper.rb b/app/helpers/ci_status_helper.rb index 136772e1ec3..f343607a343 100644 --- a/app/helpers/ci_status_helper.rb +++ b/app/helpers/ci_status_helper.rb @@ -7,6 +7,7 @@ # # See 'detailed_status?` method and `Gitlab::Ci::Status` module. # +# TODO: DO I need to update this deprecated module? module CiStatusHelper def ci_label_for_status(status) if detailed_status?(status) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3ef149f3632..cf5df2ca354 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -169,11 +169,11 @@ def retry(build, current_user) end before_transition created: :scheduled do |build| - build_build_schedule(execute_at: execute_at) + build.build_build_schedule(execute_at: build.execute_at) end before_transition scheduled: any do |build| - build_schedule.delete! + build.build_schedule.delete end after_transition any => [:pending] do |build| diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 6dac577c514..2d90c9bfe50 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -108,6 +108,10 @@ class Pipeline < ActiveRecord::Base transition any - [:manual] => :manual end + event :schedule do + transition any - [:scheduled] => :scheduled + end + # IMPORTANT # Do not add any operations to this state_machine # Create a separate worker for each new operation @@ -544,6 +548,7 @@ def update_status when 'canceled' then cancel when 'skipped' then skip when 'manual' then block + when 'scheduled' then schedule else raise HasStatus::UnknownStatusError, "Unknown status `#{latest_builds_status}`" diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 511ded55dc3..811261a252e 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -65,6 +65,10 @@ class Stage < ActiveRecord::Base event :block do transition any - [:manual] => :manual end + + event :schedule do + transition any - [:scheduled] => :scheduled + end end def update_status @@ -77,6 +81,7 @@ def update_status when 'failed' then drop when 'canceled' then cancel when 'manual' then block + when 'scheduled' then schedule when 'skipped', nil then skip else raise HasStatus::UnknownStatusError, diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 0a97e4cdfc4..652f56f7f11 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -77,6 +77,7 @@ def all_state_names state :canceled, value: 'canceled' state :skipped, value: 'skipped' state :manual, value: 'manual' + state :scheduled, value: 'scheduled' end scope :created, -> { where(status: 'created') } @@ -88,6 +89,7 @@ def all_state_names scope :canceled, -> { where(status: 'canceled') } scope :skipped, -> { where(status: 'skipped') } scope :manual, -> { where(status: 'manual') } + scope :scheduled, -> { where(status: 'scheduled') } scope :alive, -> { where(status: [:created, :pending, :running]) } scope :created_or_pending, -> { where(status: [:created, :pending]) } scope :running_or_pending, -> { where(status: [:running, :pending]) } diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 010d5e2142f..05a97b1de47 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -3,31 +3,9 @@ module Ci module Status module Build class Scheduled < Status::Extended - ### - # Core override - ### - def text - s_('CiStatusText|scheduled') - end - - def label - s_('CiStatusLabel|scheduled') - end - - def icon - 'timer' - end - - def favicon - 'favicon_status_scheduled' - end - - ### - # Extension override - ### def illustration { - image: 'illustrations/canceled-job_empty.svg', + image: 'illustrations/scheduled-job_countdown.svg', size: 'svg-394', title: _("This is a scheduled to run in ") + " #{execute_in}", content: _("This job will automatically run after it's timer finishes. Often they are used for incremental roll-out deploys to production environments. When unscheduled it converts into a manual action.") diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb new file mode 100644 index 00000000000..f4464d69eb2 --- /dev/null +++ b/lib/gitlab/ci/status/scheduled.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + class Scheduled < Status::Core + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|scheduled') + end + + def icon + 'timer' # TODO: 'status_scheduled' + end + + def favicon + 'favicon_status_scheduled' + end + end + end + end +end diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb index 6a5000e57fc..7389e63a0da 100644 --- a/scheduled_job_fixture.rb +++ b/scheduled_job_fixture.rb @@ -62,31 +62,87 @@ # require '/path/to/scheduled_job_fixture.rb' # Load this script # ``` # -# ### Reproduce the scenario A) ~ Succeccfull timed incremantal rollout ~ -# -# ```` -# project = Project.find_by(path: 'incremental-rollout') -# user = User.find_by(username: 'root') -# fixture = ScheduledJobFixture.new(project.id, user.id) -# -# fixture.create_pipeline('master') -# fixture.finish_stage_until('test') # Succeed 'build' and 'test' jobs. 'rollout 10%' job will be scheduled. See the pipeline page -# fixture.finish_stage_until('rollout 10%') # Succeed `rollout 10%` job. 'rollout 50%' job will be scheduled. -# fixture.finish_stage_until('rollout 50%') # Succeed `rollout 50%` job. 'rollout 100%' job will be scheduled. -# fixture.finish_stage_until('rollout 100%') # Succeed `rollout 100%` job. 'cleanup' job will be scheduled. -# fixture.finish_stage_until('cleanup') # Succeed `cleanup` job. The pipeline becomes green. -# ``` +# ### Reproduce the scenario ~ when all stages succeeded ~ +# +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. Wait until rollout 10% job is triggered +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 10%') +# 1. Wait until rollout 50% job is triggered +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 50%') +# 1. Wait until rollout 100% job is triggered +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 100%') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('cleanup') +# +# Expectation: Users see a succeccful pipeline +# +# ### Reproduce the scenario ~ when rollout 10% jobs failed ~ +# +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. Wait until rollout 10% job is triggered +# 1. ScheduledJobFixture.new(29, 1).drop_jobs('rollout 10%') +# +# Expectation: Following stages should be skipped. +# +# ### Reproduce the scenario ~ when user clicked cancel button before build job finished ~ +# +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).cancel_pipeline +# +# Expectation: All stages should be canceled. +# +# ### Reproduce the scenario ~ when user canceled the pipeline after rollout 10% job is scheduled ~ +# +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. Run next command before rollout 10% job is triggered +# 1. ScheduledJobFixture.new(29, 1).cancel_pipeline +# +# Expectation: rollout 10% job will be canceled. Following stages will be skipped. +# +# ### Reproduce the scenario ~ when user canceled rollout 10% job after rollout 10% job is scheduled ~ +# +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. Run next command before rollout 10% job is triggered +# 1. ScheduledJobFixture.new(29, 1).cancel_jobs('rollout 10%') +# +# Expectation: rollout 10% job will be canceled. Following stages will be skipped. +# +# ### Reproduce the scenario ~ when user played rollout 10% job immidiately ~ +# +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. Play rollout 10% job before rollout 10% job is triggered +# +# Expectation: rollout 10% becomes pending immidiately +# +# ### Reproduce the scenario ~ when rollout 10% job is allowed to fail ~ +# +# 1. Set `allow_failure: true` to rollout 10% job +# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. Wait until rollout 10% job is triggered +# 1. ScheduledJobFixture.new(29, 1).drop_jobs('rollout 10%') +# +# Expectation: rollout 50% job should be triggered +# + class ScheduledJobFixture attr_reader :project attr_reader :user + include GitlabRoutingHelper + def initialize(project_id, user_id) @project = Project.find_by_id(project_id) @user = User.find_by_id(user_id) end def create_pipeline(ref) - Ci::CreatePipelineService.new(project, user, ref: ref).execute(:web) + pipeline = Ci::CreatePipelineService.new(project, user, ref: ref).execute(:web) + Rails.application.routes.url_helpers.namespace_project_pipeline_url(project.namespace, project, pipeline) end def finish_stage_until(stage_name) @@ -99,4 +155,32 @@ def finish_stage_until(stage_name) return if stage.name == stage_name end end + + def run_jobs(stage_name) + pipeline = Ci::Pipeline.last + stage = pipeline.stages.find_by_name(stage_name) + stage.builds.map(&:run) + stage.update_status + pipeline.update_status + end + + def drop_jobs(stage_name) + pipeline = Ci::Pipeline.last + stage = pipeline.stages.find_by_name(stage_name) + stage.builds.map(&:drop) + stage.update_status + pipeline.update_status + end + + def cancel_jobs(stage_name) + pipeline = Ci::Pipeline.last + stage = pipeline.stages.find_by_name(stage_name) + stage.builds.map(&:cancel) + stage.update_status + pipeline.update_status + end + + def cancel_pipeline + Ci::Pipeline.last.cancel_running + end end -- GitLab From 532be543a5e6d1f7ef402f41ecdf091ef9eee72a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 21 Sep 2018 15:40:24 +0900 Subject: [PATCH 014/118] Add changelog --- changelogs/unreleased/scheduled-manual-jobs.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/scheduled-manual-jobs.yml diff --git a/changelogs/unreleased/scheduled-manual-jobs.yml b/changelogs/unreleased/scheduled-manual-jobs.yml new file mode 100644 index 00000000000..fa3f5a6f461 --- /dev/null +++ b/changelogs/unreleased/scheduled-manual-jobs.yml @@ -0,0 +1,5 @@ +--- +title: Allow pipelines to schedule delayed job runs +merge_request: 21767 +author: +type: added -- GitLab From ffbc0b1c291233a05e6729bf6031ee43b61798e4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 21 Sep 2018 15:42:29 +0900 Subject: [PATCH 015/118] Remove whitespace --- app/models/commit_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 6bf2888505e..03a5522b4ba 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -26,7 +26,7 @@ class CommitStatus < ActiveRecord::Base scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) end - + scope :exclude_ignored, -> do # We want to ignore failed but allowed to fail jobs. # -- GitLab From f8e680b786377443471780d9a096dfb2b873de4a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 21 Sep 2018 17:44:15 +0900 Subject: [PATCH 016/118] Fix rubocop offence --- lib/gitlab/ci/yaml_processor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 1dc6c28d24a..a427aa30683 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -50,7 +50,7 @@ def build_attributes(name) after_script: job[:after_script], environment: job[:environment], retry: job[:retry], - start_in: job[:start_in], + start_in: job[:start_in] }.compact } end -- GitLab From 571a934f29bee7af9569176e62e5376b471e35fb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 24 Sep 2018 13:12:11 +0900 Subject: [PATCH 017/118] Fix spec. Create scheduled status entry for pipeline --- app/models/concerns/has_status.rb | 2 +- lib/gitlab/ci/status/pipeline/factory.rb | 1 + lib/gitlab/ci/status/pipeline/scheduled.rb | 21 +++++++++++ .../gitlab/ci/status/pipeline/factory_spec.rb | 35 ++++++------------- spec/models/concerns/has_status_spec.rb | 2 +- 5 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 lib/gitlab/ci/status/pipeline/scheduled.rb diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 652f56f7f11..a8044f2ab15 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -114,7 +114,7 @@ def complete? end def blocked? - BLOCKED_STATUS == status + BLOCKED_STATUS.include?(status) end private diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 17f9a75f436..00d8f01cbdc 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -5,6 +5,7 @@ module Pipeline class Factory < Status::Factory def self.extended_statuses [[Status::SuccessWarning, + Status::Pipeline::Scheduled, Status::Pipeline::Blocked]] end diff --git a/lib/gitlab/ci/status/pipeline/scheduled.rb b/lib/gitlab/ci/status/pipeline/scheduled.rb new file mode 100644 index 00000000000..5e8f99e58d7 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/scheduled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Scheduled < Status::Extended + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|waiting for scheduled job') + end + + def self.matches?(pipeline, user) + pipeline.scheduled? + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb b/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb index defb3fdc0df..8e3d4464898 100644 --- a/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb @@ -11,8 +11,7 @@ end context 'when pipeline has a core status' do - (HasStatus::AVAILABLE_STATUSES - [HasStatus::BLOCKED_STATUS]) - .each do |simple_status| + HasStatus::AVAILABLE_STATUSES.each do |simple_status| context "when core status is #{simple_status}" do let(:pipeline) { create(:ci_pipeline, status: simple_status) } @@ -24,8 +23,15 @@ expect(factory.core_status).to be_a expected_status end - it 'does not match extended statuses' do - expect(factory.extended_statuses).to be_empty + if HasStatus::BLOCKED_STATUS.include?(simple_status) + it 'matches a correct extended statuses' do + expect(factory.extended_statuses) + .to eq [Gitlab::Ci::Status::Pipeline::Blocked] + end + else + it 'does not match extended statuses' do + expect(factory.extended_statuses).to be_empty + end end it "fabricates a core status #{simple_status}" do @@ -40,27 +46,6 @@ end end end - - context "when core status is manual" do - let(:pipeline) { create(:ci_pipeline, status: :manual) } - - it "matches manual core status" do - expect(factory.core_status) - .to be_a Gitlab::Ci::Status::Manual - end - - it 'matches a correct extended statuses' do - expect(factory.extended_statuses) - .to eq [Gitlab::Ci::Status::Pipeline::Blocked] - end - - it 'extends core status with common pipeline methods' do - expect(status).to have_details - expect(status).not_to have_action - expect(status.details_path) - .to include "pipelines/#{pipeline.id}" - end - end end context 'when pipeline has warnings' do diff --git a/spec/models/concerns/has_status_spec.rb b/spec/models/concerns/has_status_spec.rb index 6866b43432c..fe9a89e8806 100644 --- a/spec/models/concerns/has_status_spec.rb +++ b/spec/models/concerns/has_status_spec.rb @@ -300,7 +300,7 @@ describe '::BLOCKED_STATUS' do it 'is a status manual' do - expect(described_class::BLOCKED_STATUS).to eq 'manual' + expect(described_class::BLOCKED_STATUS).to eq %w[manual scheduled] end end end -- GitLab From f97ec4b8f44152036a8f8242bcf1584cfbd56cec Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 24 Sep 2018 19:17:04 +0900 Subject: [PATCH 018/118] Add scheduled_at column to ci_builds, and add a partial index as well --- .../20180913102839_create_build_schedules.rb | 19 ------------------- ...924190739_add_scheduled_at_to_ci_builds.rb | 9 +++++++++ ...01039_add_partial_index_to_scheduled_at.rb | 18 ++++++++++++++++++ db/schema.rb | 5 ++++- 4 files changed, 31 insertions(+), 20 deletions(-) delete mode 100644 db/migrate/20180913102839_create_build_schedules.rb create mode 100644 db/migrate/20180924190739_add_scheduled_at_to_ci_builds.rb create mode 100644 db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb diff --git a/db/migrate/20180913102839_create_build_schedules.rb b/db/migrate/20180913102839_create_build_schedules.rb deleted file mode 100644 index 1e9d9a70b0f..00000000000 --- a/db/migrate/20180913102839_create_build_schedules.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -class CreateBuildSchedules < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def change - create_table :ci_build_schedules, id: :bigserial do |t| - t.integer :build_id, null: false - t.datetime :execute_at, null: false - - t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade - t.index :build_id, unique: true - end - end -end diff --git a/db/migrate/20180924190739_add_scheduled_at_to_ci_builds.rb b/db/migrate/20180924190739_add_scheduled_at_to_ci_builds.rb new file mode 100644 index 00000000000..c163fbb1fd6 --- /dev/null +++ b/db/migrate/20180924190739_add_scheduled_at_to_ci_builds.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddScheduledAtToCiBuilds < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :ci_builds, :scheduled_at, :datetime_with_timezone + end +end diff --git a/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb b/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb new file mode 100644 index 00000000000..1b79365a0f6 --- /dev/null +++ b/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddPartialIndexToScheduledAt < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'partial_index_ci_builds_on_id_with_scheduled_jobs'.freeze + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_builds, :id, where: "scheduled_at <> NULL", name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(:ci_builds, INDEX_NAME) + end +end diff --git a/db/schema.rb b/db/schema.rb index 581496d78ce..ff45c9effc6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180924141949) do +ActiveRecord::Schema.define(version: 20180924201039) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -341,6 +341,7 @@ t.integer "artifacts_metadata_store" t.boolean "protected" t.integer "failure_reason" + t.datetime_with_timezone "scheduled_at" end add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree @@ -350,6 +351,7 @@ add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree + add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_scheduled_jobs", where: "(scheduled_at <> NULL::timestamp with time zone)", using: :btree add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree @@ -1799,6 +1801,7 @@ end add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree + add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree create_table "releases", force: :cascade do |t| -- GitLab From 703a41f8862c7278559ff13f2aa4f39ffd660c4e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 24 Sep 2018 20:02:26 +0900 Subject: [PATCH 019/118] Introduce enqueue_scheduled event --- app/models/ci/build.rb | 23 +++++++++--- app/models/ci/build_schedule.rb | 35 ------------------- app/presenters/ci/build_presenter.rb | 4 +++ .../ci/run_scheduled_build_service.rb | 13 +++++++ app/views/projects/ci/builds/_build.html.haml | 4 +-- app/workers/ci/build_schedule_worker.rb | 7 ++-- lib/gitlab/ci/status/build/scheduled.rb | 2 +- scheduled_job_fixture.rb | 12 +++---- 8 files changed, 47 insertions(+), 53 deletions(-) delete mode 100644 app/models/ci/build_schedule.rb create mode 100644 app/services/ci/run_scheduled_build_service.rb diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index cf5df2ca354..3f2630798f3 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -22,7 +22,6 @@ class Build < CommitStatus }.freeze has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' - has_one :build_schedule, class_name: 'Ci::BuildSchedule', foreign_key: :build_id has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id @@ -168,12 +167,26 @@ def retry(build, current_user) transition scheduled: :manual end - before_transition created: :scheduled do |build| - build.build_build_schedule(execute_at: build.execute_at) + event :enqueue_scheduled do + transition scheduled: :pending do + validate do |build| + build.scheduled_at && build.scheduled_at < Time.now + end + end end before_transition scheduled: any do |build| - build.build_schedule.delete + build.scheduled_at = nil + end + + before_transition created: :scheduled do |build| + build.scheduled_at = build.get_scheduled_at + end + + after_transition created: :scheduled do |build| + build.run_after_commit do + Ci::BuildScheduleWorker.perform_at(build.scheduled_at, build.id) + end end after_transition any => [:pending] do |build| @@ -250,7 +263,7 @@ def schedulable? self.when == 'delayed' && options[:start_in].present? end - def execute_at + def get_scheduled_at ChronicDuration.parse(options[:start_in])&.seconds&.from_now end diff --git a/app/models/ci/build_schedule.rb b/app/models/ci/build_schedule.rb deleted file mode 100644 index 4128fade86c..00000000000 --- a/app/models/ci/build_schedule.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Ci - class BuildSchedule < ActiveRecord::Base - extend Gitlab::Ci::Model - include Importable - include AfterCommitQueue - - belongs_to :build - - validate :schedule_at_future - - after_create :schedule, unless: :importing? - - scope :stale, -> { where("execute_at < ?", Time.now) } - - def execute_in - [0, self.execute_at - Time.now].max - end - - private - - def schedule_at_future - if self.execute_at < Time.now - errors.add(:execute_at, "Excute point must be somewhere in the future") - end - end - - def schedule - run_after_commit do - Ci::BuildScheduleWorker.perform_at(self.execute_at, self.build_id) - end - end - end -end diff --git a/app/presenters/ci/build_presenter.rb b/app/presenters/ci/build_presenter.rb index 5331cdf632b..4005840ce58 100644 --- a/app/presenters/ci/build_presenter.rb +++ b/app/presenters/ci/build_presenter.rb @@ -35,6 +35,10 @@ def tooltip_message "#{subject.name} - #{detailed_status.status_tooltip}" end + def execute_in + [0, scheduled_at - Time.now].max + end + private def tooltip_for_badge diff --git a/app/services/ci/run_scheduled_build_service.rb b/app/services/ci/run_scheduled_build_service.rb new file mode 100644 index 00000000000..8e4a628296f --- /dev/null +++ b/app/services/ci/run_scheduled_build_service.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Ci + class RunScheduledBuildService < ::BaseService + def execute(build) + unless can?(current_user, :update_build, build) + raise Gitlab::Access::AccessDeniedError + end + + build.enqueue_scheduled! + end + end +end diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index 1ba8b698fe2..c706703ae6f 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -104,9 +104,9 @@ - elsif job.scheduled? .btn-group .btn.btn-default.has-tooltip{ disabled: true, - title: job.build_schedule.execute_at } + title: job.scheduled_at } = sprite_icon('planning') - = duration_in_numbers(job.build_schedule.execute_in) + = duration_in_numbers(job.execute_in) .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Start now') } = sprite_icon('play') .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Unschedule') } diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 0d17a960c00..2a2d2bff282 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -6,10 +6,9 @@ class BuildScheduleWorker include PipelineQueue def perform(build_id) - ::Ci::Build.find_by(id: build_id).try do |build| - break unless build.scheduled? - - Ci::PlayBuildService.new(build.project, build.user).execute(build) + ::Ci::Build.find_by_id(build_id).try do |build| + Ci::RunScheduledBuildService + .new(build.project, build.user).execute(build) end end end diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 05a97b1de47..270a2706c87 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -23,7 +23,7 @@ def self.matches?(build, user) private def execute_in - Time.at(subject.build_schedule.execute_in).utc.strftime("%H:%M:%S") + Time.at(subject.scheduled_at).utc.strftime("%H:%M:%S") end end end diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb index 7389e63a0da..9ed59d337f7 100644 --- a/scheduled_job_fixture.rb +++ b/scheduled_job_fixture.rb @@ -64,15 +64,15 @@ # # ### Reproduce the scenario ~ when all stages succeeded ~ # -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') # 1. Wait until rollout 10% job is triggered -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 10%') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('rollout 10%') # 1. Wait until rollout 50% job is triggered -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 50%') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('rollout 50%') # 1. Wait until rollout 100% job is triggered -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('rollout 100%') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('cleanup') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('rollout 100%') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('cleanup') # # Expectation: Users see a succeccful pipeline # -- GitLab From 422970c93eb0ff445da5c3351cdfd70bb387e57c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 24 Sep 2018 20:07:18 +0900 Subject: [PATCH 020/118] Remove unnecessary table --- db/schema.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index ff45c9effc6..b55a9badc6b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -260,13 +260,6 @@ add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree - create_table "ci_build_schedules", id: :bigserial, force: :cascade do |t| - t.integer "build_id", null: false - t.datetime "execute_at", null: false - end - - add_index "ci_build_schedules", ["build_id"], name: "index_ci_build_schedules_on_build_id", unique: true, using: :btree - create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t| t.integer "build_id", null: false t.integer "chunk_index", null: false @@ -2298,7 +2291,6 @@ add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade add_foreign_key "chat_teams", "namespaces", on_delete: :cascade - add_foreign_key "ci_build_schedules", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade -- GitLab From 2f03c503fb299a4a821d74f75c31aa1189fcbccb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 14:21:41 +0900 Subject: [PATCH 021/118] Introduce ProceedBuildService --- app/models/ci/build.rb | 4 ++-- app/presenters/ci/build_presenter.rb | 2 +- app/services/ci/enqueue_build_service.rb | 8 -------- app/services/ci/proceed_build_service.rb | 15 +++++++++++++++ app/services/ci/process_pipeline_service.rb | 12 +----------- 5 files changed, 19 insertions(+), 22 deletions(-) delete mode 100644 app/services/ci/enqueue_build_service.rb create mode 100644 app/services/ci/proceed_build_service.rb diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3f2630798f3..6c72bce75e3 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -180,7 +180,7 @@ def retry(build, current_user) end before_transition created: :scheduled do |build| - build.scheduled_at = build.get_scheduled_at + build.scheduled_at = build.options_scheduled_at end after_transition created: :scheduled do |build| @@ -263,7 +263,7 @@ def schedulable? self.when == 'delayed' && options[:start_in].present? end - def get_scheduled_at + def options_scheduled_at ChronicDuration.parse(options[:start_in])&.seconds&.from_now end diff --git a/app/presenters/ci/build_presenter.rb b/app/presenters/ci/build_presenter.rb index 4005840ce58..33056a809b7 100644 --- a/app/presenters/ci/build_presenter.rb +++ b/app/presenters/ci/build_presenter.rb @@ -36,7 +36,7 @@ def tooltip_message end def execute_in - [0, scheduled_at - Time.now].max + scheduled? && scheduled_at && [0, scheduled_at - Time.now].max end private diff --git a/app/services/ci/enqueue_build_service.rb b/app/services/ci/enqueue_build_service.rb deleted file mode 100644 index 8140651d980..00000000000 --- a/app/services/ci/enqueue_build_service.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true -module Ci - class EnqueueBuildService < BaseService - def execute(build) - build.enqueue - end - end -end diff --git a/app/services/ci/proceed_build_service.rb b/app/services/ci/proceed_build_service.rb new file mode 100644 index 00000000000..c8119c2b468 --- /dev/null +++ b/app/services/ci/proceed_build_service.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Ci + class ProceedBuildService < BaseService + def execute(build) + if build.schedulable? + build.schedule! + elsif build.action? + build.actionize + else + build.enqueue + end + end + end +end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 0a13da198cd..f122fc8798c 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -37,7 +37,7 @@ def process_stage(index) def process_build(build, current_status) if valid_statuses_for_when(build.when).include?(current_status) - proceed_build(build) + Ci::ProceedBuildService.new(project, @user).execute(build) true else build.skip @@ -103,15 +103,5 @@ def update_retried .update_all(retried: true) if latest_statuses.any? end # rubocop: enable CodeReuse/ActiveRecord - - def proceed_build(build) - if build.schedulable? - build.schedule! - elsif build.action? - build.actionize - else - Ci::EnqueueBuildService.new(project, @user).execute(build) - end - end end end -- GitLab From 27a4c03502c4eca0be03a81a27c6a9d5f5e4967e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 14:24:08 +0900 Subject: [PATCH 022/118] Rename to process build service --- .../ci/{proceed_build_service.rb => process_build_service.rb} | 2 +- app/services/ci/process_pipeline_service.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename app/services/ci/{proceed_build_service.rb => process_build_service.rb} (84%) diff --git a/app/services/ci/proceed_build_service.rb b/app/services/ci/process_build_service.rb similarity index 84% rename from app/services/ci/proceed_build_service.rb rename to app/services/ci/process_build_service.rb index c8119c2b468..0f06683587d 100644 --- a/app/services/ci/proceed_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class ProceedBuildService < BaseService + class ProcessBuildService < BaseService def execute(build) if build.schedulable? build.schedule! diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index f122fc8798c..07371acb3c7 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -37,7 +37,7 @@ def process_stage(index) def process_build(build, current_status) if valid_statuses_for_when(build.when).include?(current_status) - Ci::ProceedBuildService.new(project, @user).execute(build) + Ci::ProcessBuildService.new(project, @user).execute(build) true else build.skip -- GitLab From c6e4b6a7d9afd3c8561bc1cddebf8dd05f5534ae Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 14:27:00 +0900 Subject: [PATCH 023/118] Optimize query format --- app/models/concerns/has_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index a8044f2ab15..88a0c9919e7 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -97,7 +97,7 @@ def all_state_names scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } scope :cancelable, -> do - where("status IN ('running', 'pending', 'created', 'scheduled')") + where(status: [:running, :pending, :created, :scheduled]) end end -- GitLab From e0cfa9279cc8620f64bfceebc743a8c245be215c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 14:30:11 +0900 Subject: [PATCH 024/118] Execute the worker in pipeline_processing queue --- app/workers/ci/build_schedule_worker.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb index 2a2d2bff282..da219adffc6 100644 --- a/app/workers/ci/build_schedule_worker.rb +++ b/app/workers/ci/build_schedule_worker.rb @@ -5,8 +5,12 @@ class BuildScheduleWorker include ApplicationWorker include PipelineQueue + queue_namespace :pipeline_processing + def perform(build_id) ::Ci::Build.find_by_id(build_id).try do |build| + break unless build.scheduled? + Ci::RunScheduledBuildService .new(build.project, build.user).execute(build) end -- GitLab From f76f6df19377945609c7e68103077ef04b0702fb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 14:39:32 +0900 Subject: [PATCH 025/118] Add feature flag to schedulable? method --- app/models/ci/build.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 6c72bce75e3..1209e7ef696 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -260,7 +260,8 @@ def playable? end def schedulable? - self.when == 'delayed' && options[:start_in].present? + Feature.enabled?('ci_enable_scheduled_build') && + self.when == 'delayed' && options[:start_in].present? end def options_scheduled_at -- GitLab From c9077a0efdca065f848e3698c3afd5251a135049 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 15:18:02 +0900 Subject: [PATCH 026/118] Add cleanup mechanizm for stale scheduled jobs --- app/models/commit_status.rb | 3 ++- app/workers/stuck_ci_jobs_worker.rb | 30 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 03a5522b4ba..2fd3365098a 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -49,7 +49,8 @@ class CommitStatus < ActiveRecord::Base stuck_or_timeout_failure: 3, runner_system_failure: 4, missing_dependency_failure: 5, - runner_unsupported: 6 + runner_unsupported: 6, + schedule_expired: 7 } ## diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index f6bca1176d1..884843e4465 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -8,6 +8,7 @@ class StuckCiJobsWorker BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour BUILD_PENDING_OUTDATED_TIMEOUT = 1.day + BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour BUILD_PENDING_STUCK_TIMEOUT = 1.hour def perform @@ -15,9 +16,10 @@ def perform Rails.logger.info "#{self.class}: Cleaning stuck builds" - drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT - drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT - drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT + drop :running, :updated_at, BUILD_RUNNING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure + drop :pending, :updated_at, BUILD_PENDING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure + drop :scheduled, :scheduled_at, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired + drop_stuck :pending, :updated_at, BUILD_PENDING_STUCK_TIMEOUT, :stuck_or_timeout_failure remove_lease end @@ -32,25 +34,27 @@ def remove_lease Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) end - def drop(status, timeout) - search(status, timeout) do |build| - drop_build :outdated, build, status, timeout + def drop(status, column, timeout, reason) + search(status, column, timeout) do |build| + drop_build :outdated, build, status, timeout, reason end end - def drop_stuck(status, timeout) - search(status, timeout) do |build| + def drop_stuck(status, column, timeout, reason) + search(status, column, timeout) do |build| break unless build.stuck? - drop_build :stuck, build, status, timeout + drop_build :stuck, build, status, timeout, reason end end # rubocop: disable CodeReuse/ActiveRecord - def search(status, timeout) + def search(status, column, timeout) + quoted_column = ActiveRecord::Base.connection.quote_column_name(column) + loop do jobs = Ci::Build.where(status: status) - .where('ci_builds.updated_at < ?', timeout.ago) + .where("#{quoted_column} < ?", timeout.ago) .includes(:tags, :runner, project: :namespace) .limit(100) .to_a @@ -63,10 +67,10 @@ def search(status, timeout) end # rubocop: enable CodeReuse/ActiveRecord - def drop_build(type, build, status, timeout) + def drop_build(type, build, status, timeout, reason) Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| - b.drop(:stuck_or_timeout_failure) + b.drop(reason) end end end -- GitLab From b1d24c0d14afdf3312e8f0745cc5ba87e41004b4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 25 Sep 2018 18:44:08 +0900 Subject: [PATCH 027/118] Fix stuck job worker. Fix sidekiq queue namespace --- app/workers/all_queues.yml | 2 +- app/workers/stuck_ci_jobs_worker.rb | 46 ++++++++++++++++--------- lib/gitlab/ci/status/build/scheduled.rb | 3 +- scheduled_job_fixture.rb | 38 +++++++++++--------- 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index b5a492122a3..f21789de37d 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -60,7 +60,6 @@ - pipeline_default:build_trace_sections - pipeline_default:pipeline_metrics - pipeline_default:pipeline_notification -- pipeline_default:ci_build_schedule - pipeline_hooks:build_hooks - pipeline_hooks:pipeline_hooks - pipeline_processing:build_finished @@ -71,6 +70,7 @@ - pipeline_processing:pipeline_update - pipeline_processing:stage_update - pipeline_processing:update_head_pipeline_for_merge_request +- pipeline_processing:ci_build_schedule - repository_check:repository_check_clear - repository_check:repository_check_batch diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 884843e4465..67d88c75f91 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -16,10 +16,10 @@ def perform Rails.logger.info "#{self.class}: Cleaning stuck builds" - drop :running, :updated_at, BUILD_RUNNING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure - drop :pending, :updated_at, BUILD_PENDING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure - drop :scheduled, :scheduled_at, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired - drop_stuck :pending, :updated_at, BUILD_PENDING_STUCK_TIMEOUT, :stuck_or_timeout_failure + drop :running, condition_for_outdated_running, :stuck_or_timeout_failure + drop :pending, condition_for_outdated_pending, :stuck_or_timeout_failure + drop :scheduled, condition_for_outdated_scheduled, :schedule_expired + drop_stuck :pending, condition_for_outdated_pending_stuck, :stuck_or_timeout_failure remove_lease end @@ -34,27 +34,41 @@ def remove_lease Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) end - def drop(status, column, timeout, reason) - search(status, column, timeout) do |build| - drop_build :outdated, build, status, timeout, reason + def drop(status, condition, reason) + search(status, condition) do |build| + drop_build :outdated, build, status, reason end end - def drop_stuck(status, column, timeout, reason) - search(status, column, timeout) do |build| + def drop_stuck(status, condition, reason) + search(status, condition) do |build| break unless build.stuck? - drop_build :stuck, build, status, timeout, reason + drop_build :stuck, build, status, reason end end - # rubocop: disable CodeReuse/ActiveRecord - def search(status, column, timeout) - quoted_column = ActiveRecord::Base.connection.quote_column_name(column) + def condition_for_outdated_running + ["updated_at < ?", BUILD_RUNNING_OUTDATED_TIMEOUT.ago] + end + def condition_for_outdated_pending + ["updated_at < ?", BUILD_PENDING_OUTDATED_TIMEOUT.ago] + end + + def condition_for_outdated_scheduled + ["scheduled_at <> '' && scheduled_at < ?", BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago] + end + + def condition_for_outdated_pending_stuck + ["updated_at < ?", BUILD_PENDING_STUCK_TIMEOUT.ago] + end + + # rubocop: disable CodeReuse/ActiveRecord + def search(status, condition) loop do jobs = Ci::Build.where(status: status) - .where("#{quoted_column} < ?", timeout.ago) + .where(*condition) .includes(:tags, :runner, project: :namespace) .limit(100) .to_a @@ -67,8 +81,8 @@ def search(status, column, timeout) end # rubocop: enable CodeReuse/ActiveRecord - def drop_build(type, build, status, timeout, reason) - Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" + def drop_build(type, build, status, reason) + Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status})" Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| b.drop(reason) end diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 270a2706c87..c6713f0d633 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -23,7 +23,8 @@ def self.matches?(build, user) private def execute_in - Time.at(subject.scheduled_at).utc.strftime("%H:%M:%S") + diff = [0, subject.scheduled_at - Time.now].max + Time.at(diff).utc.strftime("%H:%M:%S") end end end diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb index 9ed59d337f7..ae33c6be6ad 100644 --- a/scheduled_job_fixture.rb +++ b/scheduled_job_fixture.rb @@ -1,4 +1,10 @@ ## +# ### +# IMPORTANT +# - Enable the feature flag `ci_enable_scheduled_build` on rails console! You can do `Feature.enable('ci_enable_scheduled_build')` +# This feature is off by default! +# +# # This is a debug script to reproduce specific scenarios for scheduled jobs (https://gitlab.com/gitlab-org/gitlab-ce/issues/51352) # By using this script, you don't need to setup GitLab runner. # This script is specifically made for FE/UX engineers. They can quickly check how scheduled jobs behave. @@ -78,42 +84,42 @@ # # ### Reproduce the scenario ~ when rollout 10% jobs failed ~ # -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') # 1. Wait until rollout 10% job is triggered -# 1. ScheduledJobFixture.new(29, 1).drop_jobs('rollout 10%') +# 1. ScheduledJobFixture.new(16, 1).drop_jobs('rollout 10%') # # Expectation: Following stages should be skipped. # # ### Reproduce the scenario ~ when user clicked cancel button before build job finished ~ # -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).cancel_pipeline +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).cancel_pipeline # # Expectation: All stages should be canceled. # # ### Reproduce the scenario ~ when user canceled the pipeline after rollout 10% job is scheduled ~ # -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') # 1. Run next command before rollout 10% job is triggered -# 1. ScheduledJobFixture.new(29, 1).cancel_pipeline +# 1. ScheduledJobFixture.new(16, 1).cancel_pipeline # # Expectation: rollout 10% job will be canceled. Following stages will be skipped. # # ### Reproduce the scenario ~ when user canceled rollout 10% job after rollout 10% job is scheduled ~ # -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') # 1. Run next command before rollout 10% job is triggered -# 1. ScheduledJobFixture.new(29, 1).cancel_jobs('rollout 10%') +# 1. ScheduledJobFixture.new(16, 1).cancel_jobs('rollout 10%') # # Expectation: rollout 10% job will be canceled. Following stages will be skipped. # # ### Reproduce the scenario ~ when user played rollout 10% job immidiately ~ # -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') # 1. Play rollout 10% job before rollout 10% job is triggered # # Expectation: rollout 10% becomes pending immidiately @@ -121,10 +127,10 @@ # ### Reproduce the scenario ~ when rollout 10% job is allowed to fail ~ # # 1. Set `allow_failure: true` to rollout 10% job -# 1. ScheduledJobFixture.new(29, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(29, 1).finish_stage_until('test') +# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') +# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') # 1. Wait until rollout 10% job is triggered -# 1. ScheduledJobFixture.new(29, 1).drop_jobs('rollout 10%') +# 1. ScheduledJobFixture.new(16, 1).drop_jobs('rollout 10%') # # Expectation: rollout 50% job should be triggered # -- GitLab From c514636a83987152b2a95e442f49b3ee61dcbeb8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 14:28:22 +0900 Subject: [PATCH 028/118] Simplify StuckCiJobsWorker --- app/services/ci/process_pipeline_service.rb | 2 +- app/workers/stuck_ci_jobs_worker.rb | 55 ++++++++++----------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 07371acb3c7..1d1e39232fe 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -56,7 +56,7 @@ def valid_statuses_for_when(value) when 'manual' %w[success skipped] when 'delayed' - %w[success skipped] # This might be `success` only + %w[success skipped] else [] end diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 67d88c75f91..8979596c581 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -16,10 +16,10 @@ def perform Rails.logger.info "#{self.class}: Cleaning stuck builds" - drop :running, condition_for_outdated_running, :stuck_or_timeout_failure - drop :pending, condition_for_outdated_pending, :stuck_or_timeout_failure - drop :scheduled, condition_for_outdated_scheduled, :schedule_expired - drop_stuck :pending, condition_for_outdated_pending_stuck, :stuck_or_timeout_failure + drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT + drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT + drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT + drop_stale_scheduled_builds remove_lease end @@ -34,41 +34,25 @@ def remove_lease Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) end - def drop(status, condition, reason) - search(status, condition) do |build| - drop_build :outdated, build, status, reason + def drop(status, timeout) + search(status, timeout) do |build| + drop_build :outdated, build, status, timeout, :stuck_or_timeout_failure end end - def drop_stuck(status, condition, reason) - search(status, condition) do |build| + def drop_stuck(status, timeout) + search(status, timeout) do |build| break unless build.stuck? - drop_build :stuck, build, status, reason + drop_build :stuck, build, status, timeout, :stuck_or_timeout_failure end end - def condition_for_outdated_running - ["updated_at < ?", BUILD_RUNNING_OUTDATED_TIMEOUT.ago] - end - - def condition_for_outdated_pending - ["updated_at < ?", BUILD_PENDING_OUTDATED_TIMEOUT.ago] - end - - def condition_for_outdated_scheduled - ["scheduled_at <> '' && scheduled_at < ?", BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago] - end - - def condition_for_outdated_pending_stuck - ["updated_at < ?", BUILD_PENDING_STUCK_TIMEOUT.ago] - end - # rubocop: disable CodeReuse/ActiveRecord - def search(status, condition) + def search(status, timeout) loop do jobs = Ci::Build.where(status: status) - .where(*condition) + .where('ci_builds.updated_at < ?', timeout.ago) .includes(:tags, :runner, project: :namespace) .limit(100) .to_a @@ -81,10 +65,21 @@ def search(status, condition) end # rubocop: enable CodeReuse/ActiveRecord - def drop_build(type, build, status, reason) - Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status})" + def drop_build(type, build, status, timeout, reason) + Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| b.drop(reason) end end + + def drop_stale_scheduled_builds + # `ci_builds` table has a partial index on `id` with `scheduled_at <> NULL` condition. + # Therefore this query's first step uses Index Search, and the following expensive + # filter `scheduled_at < ?` will only perform on a small subset (max: 100 rows) + Ci::Build.include(EachBach).where('scheduled_at <> NULL').each_batch(of: 100) do |relation| + relation.where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago).find_each do |build| + drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired) + end + end + end end -- GitLab From 173300370a77f84b06755498b488cb68547eb1b3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 14:37:11 +0900 Subject: [PATCH 029/118] Fix retry_build_service_spec --- spec/services/ci/retry_build_service_spec.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index 951c0b16a68..a945a769d4e 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -26,7 +26,8 @@ erased_at auto_canceled_by job_artifacts job_artifacts_archive job_artifacts_metadata job_artifacts_trace job_artifacts_junit job_artifacts_sast job_artifacts_dependency_scanning - job_artifacts_container_scanning job_artifacts_dast].freeze + job_artifacts_container_scanning job_artifacts_dast + scheduled_at].freeze IGNORE_ACCESSORS = %i[type lock_version target_url base_tags trace_sections @@ -43,7 +44,8 @@ create(:ci_build, :failed, :expired, :erased, :queued, :coverage, :tags, :allowed_to_fail, :on_tag, :triggered, :teardown_environment, description: 'my-job', stage: 'test', stage_id: stage.id, - pipeline: pipeline, auto_canceled_by: another_pipeline) + pipeline: pipeline, auto_canceled_by: another_pipeline, + scheduled_at: 10.seconds.since) end before do -- GitLab From 44491012828af1dbda8e807b74cd14f87be34bbd Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 14:39:05 +0900 Subject: [PATCH 030/118] Remove unnecessary change from schema.rb --- db/schema.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index b55a9badc6b..aec446a2f2e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1794,7 +1794,6 @@ end add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree - add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree create_table "releases", force: :cascade do |t| -- GitLab From af4b85cef57c00f4cccdac7fde15d4c69d9e94fb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 14:43:03 +0900 Subject: [PATCH 031/118] Fix commit status presenter spec --- app/presenters/commit_status_presenter.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/presenters/commit_status_presenter.rb b/app/presenters/commit_status_presenter.rb index 65e77ea3f92..b2b9fb55cba 100644 --- a/app/presenters/commit_status_presenter.rb +++ b/app/presenters/commit_status_presenter.rb @@ -8,7 +8,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated stuck_or_timeout_failure: 'There has been a timeout failure or the job got stuck. Check your timeout limits or try again', runner_system_failure: 'There has been a runner system failure, please try again', missing_dependency_failure: 'There has been a missing dependency failure', - runner_unsupported: 'Your runner is outdated, please upgrade your runner' + runner_unsupported: 'Your runner is outdated, please upgrade your runner', + schedule_expired: 'Scheduled job could not be executed by some reason, please try again' }.freeze private_constant :CALLOUT_FAILURE_MESSAGES -- GitLab From 6eee8d2d53a327051515ec18953726fd5606c000 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 15:13:39 +0900 Subject: [PATCH 032/118] Fix process build service spec --- spec/factories/ci/builds.rb | 9 +++ .../services/ci/enqueue_build_service_spec.rb | 16 ----- .../services/ci/process_build_service_spec.rb | 61 +++++++++++++++++++ 3 files changed, 70 insertions(+), 16 deletions(-) delete mode 100644 spec/services/ci/enqueue_build_service_spec.rb create mode 100644 spec/services/ci/process_build_service_spec.rb diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index 9813190925b..aea6b5d6b2f 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -98,6 +98,15 @@ success end + trait :schedulable do + self.when 'delayed' + options start_in: '1 minute' + end + + trait :actionable do + self.when 'manual' + end + trait :retried do retried true end diff --git a/spec/services/ci/enqueue_build_service_spec.rb b/spec/services/ci/enqueue_build_service_spec.rb deleted file mode 100644 index e41b8e4800b..00000000000 --- a/spec/services/ci/enqueue_build_service_spec.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -describe Ci::EnqueueBuildService, '#execute' do - let(:user) { create(:user) } - let(:project) { create(:project) } - let(:ci_build) { create(:ci_build, :created) } - - subject { described_class.new(project, user).execute(ci_build) } - - it 'enqueues the build' do - subject - - expect(ci_build.pending?).to be_truthy - end -end diff --git a/spec/services/ci/process_build_service_spec.rb b/spec/services/ci/process_build_service_spec.rb new file mode 100644 index 00000000000..74a81af75f2 --- /dev/null +++ b/spec/services/ci/process_build_service_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Ci::ProcessBuildService, '#execute' do + let(:user) { create(:user) } + let(:project) { create(:project) } + + subject { described_class.new(project, user).execute(build) } + + before do + project.add_maintainer(user) + end + + context 'when build is schedulable' do + let(:build) { create(:ci_build, :created, :schedulable, user: user, project: project) } + + context 'when ci_enable_scheduled_build feature flag is enabled' do + before do + stub_feature_flags(ci_enable_scheduled_build: true) + end + + it 'schedules the build' do + subject + + expect(build).to be_scheduled + end + end + + context 'when ci_enable_scheduled_build feature flag is disabled' do + before do + stub_feature_flags(ci_enable_scheduled_build: false) + end + + it 'enqueues the build' do + subject + + expect(build).to be_pending + end + end + end + + context 'when build is actionable' do + let(:build) { create(:ci_build, :created, :actionable, user: user, project: project) } + + it 'actionizes the build' do + subject + + expect(build).to be_manual + end + end + + context 'when build does not have any actions' do + let(:build) { create(:ci_build, :created, user: user, project: project) } + + it 'enqueues the build' do + subject + + expect(build).to be_pending + end + end +end -- GitLab From 20de2480d2431bc4afcd264fbb4aa73baa74a2b4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 15:15:52 +0900 Subject: [PATCH 033/118] Fix stuck ci jobs worker --- app/workers/stuck_ci_jobs_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 8979596c581..c146db1086b 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -76,7 +76,7 @@ def drop_stale_scheduled_builds # `ci_builds` table has a partial index on `id` with `scheduled_at <> NULL` condition. # Therefore this query's first step uses Index Search, and the following expensive # filter `scheduled_at < ?` will only perform on a small subset (max: 100 rows) - Ci::Build.include(EachBach).where('scheduled_at <> NULL').each_batch(of: 100) do |relation| + Ci::Build.include(EachBatch).where('scheduled_at <> NULL').each_batch(of: 100) do |relation| relation.where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago).find_each do |build| drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired) end -- GitLab From 4b0aa573498dda340bc24a63164433e1de670c03 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 15:19:56 +0900 Subject: [PATCH 034/118] Check the precense of scheduled_at in Status::Build --- lib/gitlab/ci/status/build/scheduled.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index c6713f0d633..7b46c81fb5d 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -17,7 +17,7 @@ def status_tooltip end def self.matches?(build, user) - build.scheduled? + build.scheduled? && build.scheduled_at end private -- GitLab From 9266cd5e8b543ab356df3fba78bf9e01536a180d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 19:12:48 +0900 Subject: [PATCH 035/118] Add unit tests for Ci::Build. Fix validation on state transition --- app/models/ci/build.rb | 8 +- spec/factories/ci/builds.rb | 12 +++ spec/models/ci/build_spec.rb | 153 +++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 5 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1209e7ef696..f83dfa5d1c4 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -168,10 +168,8 @@ def retry(build, current_user) end event :enqueue_scheduled do - transition scheduled: :pending do - validate do |build| - build.scheduled_at && build.scheduled_at < Time.now - end + transition scheduled: :pending, if: ->(build) do + build.scheduled_at && build.scheduled_at < Time.now end end @@ -269,7 +267,7 @@ def options_scheduled_at end def action? - %w[manual scheduled].include?(self.when) + %w[manual delayed].include?(self.when) end # rubocop: disable CodeReuse/ServiceClass diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index aea6b5d6b2f..73fa16fe6bf 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -70,6 +70,18 @@ status 'created' end + trait :scheduled do + schedulable + status 'scheduled' + scheduled_at 1.minute.since + end + + trait :expired_scheduled do + schedulable + status 'scheduled' + scheduled_at 1.minute.ago + end + trait :manual do status 'manual' self.when 'manual' diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index e82d93d5935..939017e7ee7 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -209,6 +209,147 @@ end end + describe '#schedulable?' do + subject { build.schedulable? } + + context 'when build is schedulable' do + let(:build) { create(:ci_build, :created, :schedulable, project: project) } + + it { expect(subject).to be_truthy } + + context 'when feature flag is diabled' do + before do + stub_feature_flags(ci_enable_scheduled_build: false) + end + + it { expect(subject).to be_falsy } + end + end + + context 'when build is not schedulable' do + let(:build) { create(:ci_build, :created, project: project) } + + it { expect(subject).to be_falsy } + end + end + + describe '#schedule' do + subject { build.schedule } + + before do + project.add_developer(user) + end + + let(:build) { create(:ci_build, :created, :schedulable, user: user, project: project) } + + it 'transits to scheduled' do + subject + + expect(build).to be_scheduled + end + + it 'updates scheduled_at column' do + subject + + expect(build.scheduled_at).not_to be_nil + end + + it 'schedules BuildScheduleWorker at the right time' do + Timecop.freeze do + expect(Ci::BuildScheduleWorker) + .to receive(:perform_at).with(1.minute.since, build.id) + + subject + end + end + end + + describe '#unschedule' do + subject { build.unschedule } + + context 'when build is scheduled' do + let(:build) { create(:ci_build, :scheduled, pipeline: pipeline) } + + it 'cleans scheduled_at column' do + subject + + expect(build.scheduled_at).to be_nil + end + + it 'transits to manual' do + subject + + expect(build).to be_manual + end + end + + context 'when build is not scheduled' do + let(:build) { create(:ci_build, :created, pipeline: pipeline) } + + it 'does not transit status' do + subject + + expect(build).to be_created + end + end + end + + describe '#options_scheduled_at' do + subject { build.options_scheduled_at } + + let(:build) { build_stubbed(:ci_build, options: option) } + + context 'when start_in is 1 day' do + let(:option) { { start_in: '1 day' } } + + it 'returns date after 1 day' do + Timecop.freeze do + is_expected.to eq(1.day.since) + end + end + end + + context 'when start_in is 1 week' do + let(:option) { { start_in: '1 week' } } + + it 'returns date after 1 week' do + Timecop.freeze do + is_expected.to eq(1.week.since) + end + end + end + end + + describe '#enqueue_scheduled' do + subject { build.enqueue_scheduled } + + context 'when build is scheduled and the right time has not come yet' do + let(:build) { create(:ci_build, :scheduled, pipeline: pipeline) } + + it 'does not transits the status' do + subject + + expect(build).to be_scheduled + end + end + + context 'when build is scheduled and the right time has already come' do + let(:build) { create(:ci_build, :expired_scheduled, pipeline: pipeline) } + + it 'cleans scheduled_at column' do + subject + + expect(build.scheduled_at).to be_nil + end + + it 'transits to pending' do + subject + + expect(build).to be_pending + end + end + end + describe '#any_runners_online?' do subject { build.any_runners_online? } @@ -1193,6 +1334,12 @@ it { is_expected.to be_truthy } end + context 'when is set to delayed' do + let(:value) { 'delayed' } + + it { is_expected.to be_truthy } + end + context 'when set to something else' do let(:value) { 'something else' } @@ -1463,6 +1610,12 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now) end end + context 'when build is scheduled' do + subject { build_stubbed(:ci_build, :scheduled) } + + it { is_expected.to be_playable } + end + context 'when build is not a manual action' do subject { build_stubbed(:ci_build, :success) } -- GitLab From cc8b8a60b7ac9df0008192a489da6446c7fd5f89 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 19:49:29 +0900 Subject: [PATCH 036/118] Add unit spec for Ci::Pipeline --- spec/models/ci/pipeline_spec.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 4755702c0e9..0a570394192 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -75,6 +75,18 @@ end end + describe '#schedule' do + subject { pipeline.schedule } + + let(:pipeline) { build(:ci_pipeline, status: :created) } + + it 'changes pipeline status to schedule' do + subject + + expect(pipeline).to be_scheduled + end + end + describe '#valid_commit_sha' do context 'commit.sha can not start with 00000000' do before do @@ -1288,6 +1300,19 @@ def create_pipeline(status, ref, sha, project) end end + context 'when updating status to scheduled' do + before do + allow(pipeline) + .to receive_message_chain(:statuses, :latest, :status) + .and_return(:scheduled) + end + + it 'updates pipeline status to scheduled' do + expect { pipeline.update_status } + .to change { pipeline.reload.status }.to 'scheduled' + end + end + context 'when statuses status was not recognized' do before do allow(pipeline) -- GitLab From 8ed7b34066464758e5cab955abb7a06b44c8e677 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 26 Sep 2018 20:21:36 +0900 Subject: [PATCH 037/118] Add unit tests for CommitStatus and Ci::Stage --- spec/factories/commit_statuses.rb | 4 ++ spec/models/ci/stage_spec.rb | 24 ++++++++++++ spec/models/commit_status_spec.rb | 20 ++++++++++ spec/models/concerns/has_status_spec.rb | 50 ++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 2 deletions(-) diff --git a/spec/factories/commit_statuses.rb b/spec/factories/commit_statuses.rb index 53368c64e10..381bf07f6a0 100644 --- a/spec/factories/commit_statuses.rb +++ b/spec/factories/commit_statuses.rb @@ -41,6 +41,10 @@ status 'manual' end + trait :scheduled do + status 'scheduled' + end + after(:build) do |build, evaluator| build.project = build.pipeline.project end diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb index 22a4556c10c..060a1d95293 100644 --- a/spec/models/ci/stage_spec.rb +++ b/spec/models/ci/stage_spec.rb @@ -89,6 +89,18 @@ end end + context 'when stage is scheduled because of scheduled builds' do + before do + create(:ci_build, :scheduled, stage_id: stage.id) + end + + it 'updates status to scheduled' do + expect { stage.update_status } + .to change { stage.reload.status } + .to 'scheduled' + end + end + context 'when stage is skipped because is empty' do it 'updates status to skipped' do expect { stage.update_status } @@ -188,6 +200,18 @@ end end + describe '#schedule' do + subject { stage.schedule } + + let(:stage) { create(:ci_stage_entity, status: :created) } + + it 'updates stage status' do + subject + + expect(stage).to be_scheduled + end + end + describe '#position' do context 'when stage has been imported and does not have position index set' do before do diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index f3f2bc28d2c..917685399d4 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -129,6 +129,20 @@ def create_status(**opts) end end + describe '#cancel' do + subject { job.cancel } + + context 'when status is scheduled' do + let(:job) { build(:commit_status, :scheduled) } + + it 'updates the status' do + subject + + expect(job).to be_canceled + end + end + end + describe '#auto_canceled?' do subject { commit_status.auto_canceled? } @@ -564,6 +578,12 @@ def create_status(**opts) it_behaves_like 'commit status enqueued' end + + context 'when initial state is :scheduled' do + let(:commit_status) { create(:commit_status, :scheduled) } + + it_behaves_like 'commit status enqueued' + end end describe '#present' do diff --git a/spec/models/concerns/has_status_spec.rb b/spec/models/concerns/has_status_spec.rb index fe9a89e8806..6b1038cb8fd 100644 --- a/spec/models/concerns/has_status_spec.rb +++ b/spec/models/concerns/has_status_spec.rb @@ -270,11 +270,11 @@ describe '.cancelable' do subject { CommitStatus.cancelable } - %i[running pending created].each do |status| + %i[running pending created scheduled].each do |status| it_behaves_like 'containing the job', status end - %i[failed success skipped canceled].each do |status| + %i[failed success skipped canceled manual].each do |status| it_behaves_like 'not containing the job', status end end @@ -290,6 +290,18 @@ it_behaves_like 'not containing the job', status end end + + describe '.scheduled' do + subject { CommitStatus.scheduled } + + %i[scheduled].each do |status| + it_behaves_like 'containing the job', status + end + + %i[failed success skipped canceled].each do |status| + it_behaves_like 'not containing the job', status + end + end end describe '::DEFAULT_STATUS' do @@ -303,4 +315,38 @@ expect(described_class::BLOCKED_STATUS).to eq %w[manual scheduled] end end + + describe 'blocked?' do + subject { object.blocked? } + + %w[ci_pipeline ci_stage ci_build generic_commit_status].each do |type| + let(:object) { build(type, status: status) } + + context 'when status is scheduled' do + let(:status) { :scheduled } + + it { is_expected.to be_truthy } + end + + context 'when status is manual' do + let(:status) { :manual } + + it { is_expected.to be_truthy } + end + + context 'when status is created' do + let(:status) { :created } + + it { is_expected.to be_falsy } + end + end + end + + describe '.status_sql' do + subject { Ci::Build.status_sql } + + it 'returns SQL' do + puts subject + end + end end -- GitLab From f228f23a3243ee64cd4bdcf54d010a5eedf0b8a7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 13:26:08 +0900 Subject: [PATCH 038/118] Fix process build service spec --- spec/services/ci/process_build_service_spec.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/spec/services/ci/process_build_service_spec.rb b/spec/services/ci/process_build_service_spec.rb index 74a81af75f2..962d07e185b 100644 --- a/spec/services/ci/process_build_service_spec.rb +++ b/spec/services/ci/process_build_service_spec.rb @@ -20,9 +20,14 @@ end it 'schedules the build' do - subject + Timecop.freeze do + expect(Ci::BuildScheduleWorker) + .to receive(:perform_at).with(1.minute.since, build.id) + + subject - expect(build).to be_scheduled + expect(build).to be_scheduled + end end end @@ -34,7 +39,7 @@ it 'enqueues the build' do subject - expect(build).to be_pending + expect(build).to be_manual end end end -- GitLab From 6d712148c9595c4c87247757100ca80198cdd889 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 13:30:16 +0900 Subject: [PATCH 039/118] Fix build_spec --- spec/models/ci/build_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 939017e7ee7..83add15fab7 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -243,12 +243,16 @@ let(:build) { create(:ci_build, :created, :schedulable, user: user, project: project) } it 'transits to scheduled' do + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + subject expect(build).to be_scheduled end it 'updates scheduled_at column' do + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + subject expect(build.scheduled_at).not_to be_nil -- GitLab From ddb313aebf2c499264b32567d3509f152c242d7a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 13:52:15 +0900 Subject: [PATCH 040/118] Remove Scheduled Status class from pipeline --- app/services/ci/process_pipeline_service.rb | 2 +- lib/gitlab/ci/status/pipeline/blocked.rb | 2 +- lib/gitlab/ci/status/pipeline/factory.rb | 1 - lib/gitlab/ci/status/pipeline/scheduled.rb | 21 --------------------- 4 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 lib/gitlab/ci/status/pipeline/scheduled.rb diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 1d1e39232fe..5e8a3976cc0 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -24,7 +24,7 @@ def execute(pipeline) def process_stage(index) current_status = status_for_prior_stages(index) - return if HasStatus::BLOCKED_STATUS == current_status + return if HasStatus::BLOCKED_STATUS.include?(current_status) if HasStatus::COMPLETED_STATUSES.include?(current_status) created_builds_in_stage(index).select do |build| diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index bf7e484ee9b..59fcd8ad7ff 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -8,7 +8,7 @@ def text end def label - s_('CiStatusLabel|waiting for manual action') + s_('CiStatusLabel|waiting for manual action or delayed job') end def self.matches?(pipeline, user) diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 00d8f01cbdc..17f9a75f436 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -5,7 +5,6 @@ module Pipeline class Factory < Status::Factory def self.extended_statuses [[Status::SuccessWarning, - Status::Pipeline::Scheduled, Status::Pipeline::Blocked]] end diff --git a/lib/gitlab/ci/status/pipeline/scheduled.rb b/lib/gitlab/ci/status/pipeline/scheduled.rb deleted file mode 100644 index 5e8f99e58d7..00000000000 --- a/lib/gitlab/ci/status/pipeline/scheduled.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Gitlab - module Ci - module Status - module Pipeline - class Scheduled < Status::Extended - def text - s_('CiStatusText|scheduled') - end - - def label - s_('CiStatusLabel|waiting for scheduled job') - end - - def self.matches?(pipeline, user) - pipeline.scheduled? - end - end - end - end - end -end -- GitLab From b98be35b61aaf71a263ec03807ab8e5d07a9d637 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 13:53:54 +0900 Subject: [PATCH 041/118] Fix safe model attributes --- spec/lib/gitlab/import_export/safe_model_attributes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index e9f1be172b0..1d59cff7ba8 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -300,6 +300,7 @@ CommitStatus: - retried - protected - failure_reason +- scheduled_at Ci::Variable: - id - project_id -- GitLab From 174fd391f07439f94e0c29502384cfb0cb6582da Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 13:57:43 +0900 Subject: [PATCH 042/118] Add schedule_expired to failed status --- lib/gitlab/ci/status/build/failed.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 2fa9a0d4541..cdbcb7a47cd 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -10,7 +10,8 @@ class Failed < Status::Extended stuck_or_timeout_failure: 'stuck or timeout failure', runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', - runner_unsupported: 'unsupported runner' + runner_unsupported: 'unsupported runner', + schedule_expired: 'schedule expired', }.freeze private_constant :REASONS -- GitLab From c6bb038c006f4cfff68666661d3a8fe257c51ed7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 13:59:12 +0900 Subject: [PATCH 043/118] Fix favicon spec --- spec/lib/gitlab/favicon_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/lib/gitlab/favicon_spec.rb b/spec/lib/gitlab/favicon_spec.rb index 68abcb3520a..49a423191bb 100644 --- a/spec/lib/gitlab/favicon_spec.rb +++ b/spec/lib/gitlab/favicon_spec.rb @@ -58,6 +58,7 @@ favicon_status_not_found favicon_status_pending favicon_status_running + favicon_status_scheduled favicon_status_skipped favicon_status_success favicon_status_warning -- GitLab From f3348951a80f11b376d5d21843f57102630e1d5d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 14:02:24 +0900 Subject: [PATCH 044/118] Fix coding style offence --- app/workers/stuck_ci_jobs_worker.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index c146db1086b..ab090b96c52 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -63,14 +63,6 @@ def search(status, timeout) end end end - # rubocop: enable CodeReuse/ActiveRecord - - def drop_build(type, build, status, timeout, reason) - Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" - Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| - b.drop(reason) - end - end def drop_stale_scheduled_builds # `ci_builds` table has a partial index on `id` with `scheduled_at <> NULL` condition. @@ -82,4 +74,12 @@ def drop_stale_scheduled_builds end end end + # rubocop: enable CodeReuse/ActiveRecord + + def drop_build(type, build, status, timeout, reason) + Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" + Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| + b.drop(reason) + end + end end -- GitLab From 80a92650faf9d7ca7a706b6a74019d869598fe41 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 16:10:55 +0900 Subject: [PATCH 045/118] Add Spec for ProcessPipelineService --- app/models/ci/pipeline.rb | 2 +- app/services/ci/process_build_service.rb | 37 +++- app/services/ci/process_pipeline_service.rb | 30 +-- .../ci/process_pipeline_service_spec.rb | 203 +++++++++++++++++- 4 files changed, 236 insertions(+), 36 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2d90c9bfe50..f936115014d 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -80,7 +80,7 @@ class Pipeline < ActiveRecord::Base state_machine :status, initial: :created do event :enqueue do - transition [:created, :skipped] => :pending + transition [:created, :skipped, :scheduled] => :pending transition [:success, :failed, :canceled] => :running end diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 0f06683587d..41ea62b4e4a 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -2,13 +2,38 @@ module Ci class ProcessBuildService < BaseService - def execute(build) - if build.schedulable? - build.schedule! - elsif build.action? - build.actionize + def execute(build, current_status) + if valid_statuses_for_when(build.when).include?(current_status) + if build.schedulable? + build.schedule! + elsif build.action? + build.actionize + else + build.enqueue + end + true else - build.enqueue + build.skip + false + end + end + + private + + def valid_statuses_for_when(value) + case value + when 'on_success' + %w[success skipped] + when 'on_failure' + %w[failed] + when 'always' + %w[success failed skipped] + when 'manual' + %w[success skipped] + when 'delayed' + %w[success skipped] + else + [] end end end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 5e8a3976cc0..446188347df 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -29,39 +29,13 @@ def process_stage(index) if HasStatus::COMPLETED_STATUSES.include?(current_status) created_builds_in_stage(index).select do |build| Gitlab::OptimisticLocking.retry_lock(build) do |subject| - process_build(subject, current_status) + Ci::ProcessBuildService.new(project, @user) + .execute(build, current_status) end end end end - def process_build(build, current_status) - if valid_statuses_for_when(build.when).include?(current_status) - Ci::ProcessBuildService.new(project, @user).execute(build) - true - else - build.skip - false - end - end - - def valid_statuses_for_when(value) - case value - when 'on_success' - %w[success skipped] - when 'on_failure' - %w[failed] - when 'always' - %w[success failed skipped] - when 'manual' - %w[success skipped] - when 'delayed' - %w[success skipped] - else - [] - end - end - # rubocop: disable CodeReuse/ActiveRecord def status_for_prior_stages(index) pipeline.builds.where('stage_idx < ?', index).latest.status || 'success' diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb index feb5120bc68..d314d774be4 100644 --- a/spec/services/ci/process_pipeline_service_spec.rb +++ b/spec/services/ci/process_pipeline_service_spec.rb @@ -242,6 +242,187 @@ end end + context 'when delayed jobs are defined' do + context 'when the scene is timed incremental rollout' do + before do + create_build('build', stage_idx: 0) + create_build('rollout10%', **delayed_options, stage_idx: 1) + create_build('rollout100%', **delayed_options, stage_idx: 2) + create_build('cleanup', stage_idx: 3) + + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + end + + context 'when builds are successful' do + it 'properly processes the pipeline' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) + + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) + + enqueue_scheduled('rollout10%') + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'scheduled' }) + + enqueue_scheduled('rollout100%') + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'success', 'cleanup': 'pending' }) + + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'success', 'cleanup': 'success' }) + expect(pipeline.reload.status).to eq 'success' + end + end + + context 'when build job fails' do + it 'properly processes the pipeline' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) + + fail_running_or_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'failed' }) + expect(pipeline.reload.status).to eq 'failed' + end + end + + context 'when rollout 10% is unscheduled' do + it 'properly processes the pipeline' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) + + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) + + unschedule + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'manual' }) + expect(pipeline.reload.status).to eq 'manual' + end + + context 'when user plays rollout 10%' do + it 'schedules rollout100%' do + process_pipeline + succeed_pending + unschedule + play_manual_action('rollout10%') + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'scheduled' }) + expect(pipeline.reload.status).to eq 'scheduled' + end + end + end + + context 'when rollout 10% fails' do + it 'properly processes the pipeline' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) + + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) + + enqueue_scheduled('rollout10%') + fail_running_or_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'failed' }) + expect(pipeline.reload.status).to eq 'failed' + end + + context 'when user retries rollout 10%' do + it 'does not schedule rollout10% again' do + process_pipeline + succeed_pending + enqueue_scheduled('rollout10%') + fail_running_or_pending + retry_build('rollout10%') + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' }) + expect(pipeline.reload.status).to eq 'running' + end + end + end + + context 'when rollout 10% is played immidiately' do + it 'properly processes the pipeline' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) + + succeed_pending + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) + + play_manual_action('rollout10%') + + expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' }) + expect(pipeline.reload.status).to eq 'running' + end + end + end + + context 'when only one scheduled job exists in a pipeline' do + before do + create_build('delayed', **delayed_options, stage_idx: 0) + + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + end + + it 'properly processes the pipeline' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'delayed': 'scheduled' }) + + expect(pipeline.reload.status).to eq 'scheduled' + end + end + + context 'when there are two delayed jobs in a stage' do + before do + create_build('delayed1', **delayed_options, stage_idx: 0) + create_build('delayed2', **delayed_options, stage_idx: 0) + create_build('job', stage_idx: 1) + + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + end + + it 'blocks the stage until all scheduled jobs finished' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'delayed1': 'scheduled', 'delayed2': 'scheduled' }) + + enqueue_scheduled('delayed1') + + expect(builds_names_and_statuses).to eq({ 'delayed1': 'pending', 'delayed2': 'scheduled' }) + expect(pipeline.reload.status).to eq 'scheduled' + end + end + + context 'when a delayed job is allowed to fail' do + before do + create_build('delayed', **delayed_options, allow_failure: true, stage_idx: 0) + create_build('job', stage_idx: 1) + + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + end + + it 'blocks the stage and continues after it failed' do + expect(process_pipeline).to be_truthy + expect(builds_names_and_statuses).to eq({ 'delayed': 'scheduled' }) + + enqueue_scheduled('delayed') + fail_running_or_pending + + expect(builds_names_and_statuses).to eq({ 'delayed': 'failed', 'job': 'pending' }) + expect(pipeline.reload.status).to eq 'pending' + end + end + end + context 'when there are manual action in earlier stages' do context 'when first stage has only optional manual actions' do before do @@ -536,6 +717,10 @@ def builds_names builds.pluck(:name) end + def builds_names_and_statuses + builds.inject({}) { |h, b| h[b.name.to_sym] = b.status; h } + end + def all_builds_names all_builds.pluck(:name) end @@ -549,7 +734,7 @@ def all_builds_statuses end def succeed_pending - builds.pending.update_all(status: 'success') + builds.pending.map(&:success) end def succeed_running_or_pending @@ -568,6 +753,14 @@ def play_manual_action(name) builds.find_by(name: name).play(user) end + def enqueue_scheduled(name) + builds.scheduled.find_by(name: name).enqueue + end + + def retry_build(name) + Ci::Build.retry(builds.find_by(name: name), user) + end + def manual_actions pipeline.manual_actions(true) end @@ -575,4 +768,12 @@ def manual_actions def create_build(name, **opts) create(:ci_build, :created, pipeline: pipeline, name: name, **opts) end + + def delayed_options + { when: 'delayed', options: { start_in: '1 minute' } } + end + + def unschedule + pipeline.builds.scheduled.map(&:unschedule) + end end -- GitLab From 71fc37c9cf8aee5de3d4d43ec6ea97a56e34537d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 17:50:50 +0900 Subject: [PATCH 046/118] Add spec for BuildScheduleWorker and RunScheduledBuildService --- .../ci/run_scheduled_build_service_spec.rb | 62 +++++++++++++++++++ spec/workers/ci/build_schedule_worker_spec.rb | 40 ++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 spec/services/ci/run_scheduled_build_service_spec.rb create mode 100644 spec/workers/ci/build_schedule_worker_spec.rb diff --git a/spec/services/ci/run_scheduled_build_service_spec.rb b/spec/services/ci/run_scheduled_build_service_spec.rb new file mode 100644 index 00000000000..145905db0cf --- /dev/null +++ b/spec/services/ci/run_scheduled_build_service_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' + +describe Ci::RunScheduledBuildService do + let(:user) { create(:user) } + let(:project) { create(:project) } + let(:pipeline) { create(:ci_pipeline, project: project) } + + subject { described_class.new(project, user).execute(build) } + + context 'when user can update build' do + before do + project.add_developer(user) + + create(:protected_branch, :developers_can_merge, + name: pipeline.ref, project: project) + end + + context 'when build is scheduled' do + context 'when scheduled_at is expired' do + let(:build) { create(:ci_build, :expired_scheduled, user: user, project: project, pipeline: pipeline) } + + it 'can run the build' do + expect { subject }.not_to raise_error + + expect(build).to be_pending + end + end + + context 'when scheduled_at is not expired' do + let(:build) { create(:ci_build, :scheduled, user: user, project: project, pipeline: pipeline) } + + it 'can run the build' do + expect { subject }.to raise_error(StateMachines::InvalidTransition) + + expect(build).to be_scheduled + end + end + end + + context 'when build is not scheduled' do + let(:build) { create(:ci_build, :created, user: user, project: project, pipeline: pipeline) } + + it 'can not run the build' do + expect { subject }.to raise_error(StateMachines::InvalidTransition) + + expect(build).to be_created + end + end + end + + context 'when user can not update build' do + context 'when build is scheduled' do + let(:build) { create(:ci_build, :scheduled, user: user, project: project, pipeline: pipeline) } + + it 'can not run the build' do + expect { subject }.to raise_error(Gitlab::Access::AccessDeniedError) + + expect(build).to be_scheduled + end + end + end +end diff --git a/spec/workers/ci/build_schedule_worker_spec.rb b/spec/workers/ci/build_schedule_worker_spec.rb new file mode 100644 index 00000000000..c76537b9233 --- /dev/null +++ b/spec/workers/ci/build_schedule_worker_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Ci::BuildScheduleWorker do + subject { described_class.new.perform(build.id) } + + context 'when build is found' do + context 'when build is scheduled' do + let(:build) { create(:ci_build, :scheduled) } + + it 'executes RunScheduledBuildService' do + expect_any_instance_of(Ci::RunScheduledBuildService) + .to receive(:execute).once + + subject + end + end + + context 'when build is not scheduled' do + let(:build) { create(:ci_build, :created) } + + it 'executes RunScheduledBuildService' do + expect_any_instance_of(Ci::RunScheduledBuildService) + .not_to receive(:execute) + + subject + end + end + end + + context 'when build is not found' do + let(:build) { build_stubbed(:ci_build, :scheduled) } + + it 'does nothing' do + expect_any_instance_of(Ci::RunScheduledBuildService) + .not_to receive(:execute) + + subject + end + end +end -- GitLab From 587560757faaedb6c61ef7aa77f11934cb76084b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 18:17:43 +0900 Subject: [PATCH 047/118] Fix StuckCiJobsWorker and added tests --- app/models/commit_status.rb | 2 +- app/workers/stuck_ci_jobs_worker.rb | 2 +- spec/workers/stuck_ci_jobs_worker_spec.rb | 41 +++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 2fd3365098a..2020dd637a0 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -84,7 +84,7 @@ class CommitStatus < ActiveRecord::Base end event :drop do - transition [:created, :pending, :running] => :failed + transition [:created, :pending, :running, :scheduled] => :failed end event :success do diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index ab090b96c52..821ea75703f 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -68,7 +68,7 @@ def drop_stale_scheduled_builds # `ci_builds` table has a partial index on `id` with `scheduled_at <> NULL` condition. # Therefore this query's first step uses Index Search, and the following expensive # filter `scheduled_at < ?` will only perform on a small subset (max: 100 rows) - Ci::Build.include(EachBatch).where('scheduled_at <> NULL').each_batch(of: 100) do |relation| + Ci::Build.include(EachBatch).where('scheduled_at IS NOT NULL').each_batch(of: 100) do |relation| relation.where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago).find_each do |build| drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired) end diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb index 856886e3df5..2f8ba4859d7 100644 --- a/spec/workers/stuck_ci_jobs_worker_spec.rb +++ b/spec/workers/stuck_ci_jobs_worker_spec.rb @@ -127,6 +127,47 @@ end end + describe 'drop_stale_scheduled_builds' do + let(:status) { 'scheduled' } + let(:updated_at) { } + + context 'when scheduled at 2 hours ago but it is not executed yet' do + let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) } + + it 'drops the stale scheduled build' do + expect(Ci::Build.scheduled.count).to eq(1) + expect(job).to be_scheduled + + worker.perform + job.reload + + expect(Ci::Build.scheduled.count).to eq(0) + expect(job).to be_failed + expect(job).to be_schedule_expired + end + end + + context 'when scheduled at 30 minutes ago but it is not executed yet' do + let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) } + + it 'does not drop the stale scheduled build yet' do + expect(Ci::Build.scheduled.count).to eq(1) + expect(job).to be_scheduled + + worker.perform + + expect(Ci::Build.scheduled.count).to eq(1) + expect(job).to be_scheduled + end + end + + context 'when there are no stale scheduled builds' do + it 'does not drop the stale scheduled build yet' do + expect { worker.perform }.not_to raise_error + end + end + end + describe 'exclusive lease' do let(:status) { 'running' } let(:updated_at) { 2.days.ago } -- GitLab From 6fb1e0d8884fb7e64cb3f98ccd84c9b4db5096ac Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 18:23:02 +0900 Subject: [PATCH 048/118] Fix partial index for scheduled_at --- db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb | 2 +- db/schema.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb b/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb index 1b79365a0f6..19326e63839 100644 --- a/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb +++ b/db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb @@ -9,7 +9,7 @@ class AddPartialIndexToScheduledAt < ActiveRecord::Migration disable_ddl_transaction! def up - add_concurrent_index(:ci_builds, :id, where: "scheduled_at <> NULL", name: INDEX_NAME) + add_concurrent_index(:ci_builds, :id, where: "scheduled_at IS NOT NULL", name: INDEX_NAME) end def down diff --git a/db/schema.rb b/db/schema.rb index aec446a2f2e..f19fab6a26b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -344,7 +344,7 @@ add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree - add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_scheduled_jobs", where: "(scheduled_at <> NULL::timestamp with time zone)", using: :btree + add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_scheduled_jobs", where: "(scheduled_at IS NOT NULL)", using: :btree add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree -- GitLab From eee454e142fb99646649f8b8c9ccd8626c9bd70a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 19:08:11 +0900 Subject: [PATCH 049/118] Fix validation methods in Config::Entry::Job. Added spec for that --- lib/gitlab/ci/config/entry/job.rb | 13 ++--- spec/lib/gitlab/ci/config/entry/job_spec.rb | 65 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 3ad048883af..03971254310 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -30,19 +30,14 @@ class Job < Node validates :when, inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ - 'always or manual' } + 'always, manual or delayed' } validates :dependencies, array_of_strings: true validates :extends, type: String - - with_options if: :delayed? do - validates :start_in, duration: true, allow_nil: false - end - - with_options unless: :delayed? do - validates :start_in, presence: false - end end + + validates :start_in, duration: true, if: :delayed? + validates :start_in, absence: true, unless: :delayed? end entry :before_script, Entry::Script, diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 2c9758401b7..d745c4ca2ad 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -39,6 +39,16 @@ expect(entry.errors).to include "job name can't be blank" end end + + context 'when delayed job' do + context 'when start_in is specified' do + let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } } + + it 'returns error about invalid type' do + expect(entry).to be_valid + end + end + end end context 'when entry value is not correct' do @@ -129,6 +139,43 @@ end end end + + context 'when delayed job' do + context 'when start_in is specified' do + let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } } + + it 'returns error about invalid type' do + expect(entry).to be_valid + end + end + + context 'when start_in is empty' do + let(:config) { { when: 'delayed', start_in: nil } } + + it 'returns error about invalid type' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job start in should be a duration' + end + end + + context 'when start_in is not formateed ad a duration' do + let(:config) { { when: 'delayed', start_in: 'test' } } + + it 'returns error about invalid type' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job start in should be a duration' + end + end + end + + context 'when start_in specified without delayed specification' do + let(:config) { { start_in: '1 day' } } + + it 'returns error about invalid type' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job start in must be blank' + end + end end end @@ -238,6 +285,24 @@ end end + describe '#delayed?' do + context 'when job is a delayed' do + let(:config) { { script: 'deploy', when: 'delayed' } } + + it 'is a delayed' do + expect(entry).to be_delayed + end + end + + context 'when job is not a delayed' do + let(:config) { { script: 'deploy' } } + + it 'is not a delayed' do + expect(entry).not_to be_delayed + end + end + end + describe '#ignored?' do context 'when job is a manual action' do context 'when it is not specified if job is allowed to fail' do -- GitLab From fcb77970b6ca26f031d5bcf935855a91bbb35158 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 19:32:26 +0900 Subject: [PATCH 050/118] Fix Status::Build::Scheduled. Add spec for the class. --- lib/gitlab/ci/status/build/scheduled.rb | 5 +- .../gitlab/ci/status/build/scheduled_spec.rb | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 spec/lib/gitlab/ci/status/build/scheduled_spec.rb diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 7b46c81fb5d..d7d762cdf7a 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -8,7 +8,10 @@ def illustration image: 'illustrations/scheduled-job_countdown.svg', size: 'svg-394', title: _("This is a scheduled to run in ") + " #{execute_in}", - content: _("This job will automatically run after it's timer finishes. Often they are used for incremental roll-out deploys to production environments. When unscheduled it converts into a manual action.") + content: _("This job will automatically run after it's timer finishes. " \ + "Often they are used for incremental roll-out deploys " \ + "to production environments. When unscheduled it converts " \ + "into a manual action.") } end diff --git a/spec/lib/gitlab/ci/status/build/scheduled_spec.rb b/spec/lib/gitlab/ci/status/build/scheduled_spec.rb new file mode 100644 index 00000000000..3098a17c50d --- /dev/null +++ b/spec/lib/gitlab/ci/status/build/scheduled_spec.rb @@ -0,0 +1,58 @@ +require 'spec_helper' + +describe Gitlab::Ci::Status::Build::Scheduled do + let(:user) { create(:user) } + let(:project) { create(:project, :stubbed_repository) } + let(:build) { create(:ci_build, :scheduled, project: project) } + let(:status) { Gitlab::Ci::Status::Core.new(build, user) } + + subject { described_class.new(status) } + + describe '#illustration' do + it { expect(subject.illustration).to include(:image, :size, :title) } + end + + describe '#status_tooltip' do + context 'when scheduled_at is not expired' do + let(:build) { create(:ci_build, scheduled_at: 1.minute.since, project: project) } + + it 'shows execute_in of the scheduled job' do + Timecop.freeze do + expect(subject.status_tooltip).to include('00:01:00') + end + end + end + + context 'when scheduled_at is expired' do + let(:build) { create(:ci_build, :expired_scheduled, project: project) } + + it 'shows 00:00:00' do + Timecop.freeze do + expect(subject.status_tooltip).to include('00:00:00') + end + end + end + end + + describe '.matches?' do + subject { described_class.matches?(build, user) } + + context 'when build is scheduled and scheduled_at is present' do + let(:build) { create(:ci_build, :expired_scheduled, project: project) } + + it { is_expected.to be_truthy } + end + + context 'when build is scheduled' do + let(:build) { create(:ci_build, status: :scheduled, project: project) } + + it { is_expected.to be_falsy } + end + + context 'when scheduled_at is present' do + let(:build) { create(:ci_build, scheduled_at: 1.minute.since, project: project) } + + it { is_expected.to be_falsy } + end + end +end -- GitLab From 5e4824d9ed8d164ff5a8ed5bfbf1e5a0f1bf1cff Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 19:33:41 +0900 Subject: [PATCH 051/118] Fix Status::Pipeline::Blocked spec --- spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb b/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb index 1a2b952d374..49a25b4a389 100644 --- a/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb +++ b/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb @@ -15,7 +15,7 @@ describe '#label' do it 'overrides status label' do - expect(subject.label).to eq 'waiting for manual action' + expect(subject.label).to eq 'waiting for manual action or delayed job' end end -- GitLab From 5e35e85acb5403ae1a992e2cd1c9618f6fc9273e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 19:36:09 +0900 Subject: [PATCH 052/118] Add spec for scheduled status --- spec/lib/gitlab/ci/status/scheduled_spec.rb | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 spec/lib/gitlab/ci/status/scheduled_spec.rb diff --git a/spec/lib/gitlab/ci/status/scheduled_spec.rb b/spec/lib/gitlab/ci/status/scheduled_spec.rb new file mode 100644 index 00000000000..c35a6f43d5d --- /dev/null +++ b/spec/lib/gitlab/ci/status/scheduled_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe Gitlab::Ci::Status::Scheduled do + subject do + described_class.new(double('subject'), double('user')) + end + + describe '#text' do + it { expect(subject.text).to eq 'scheduled' } + end + + describe '#label' do + it { expect(subject.label).to eq 'scheduled' } + end + + describe '#icon' do + it { expect(subject.icon).to eq 'status_scheduled' } + end + + describe '#favicon' do + it { expect(subject.favicon).to eq 'favicon_status_scheduled' } + end + + describe '#group' do + it { expect(subject.group).to eq 'scheduled' } + end +end -- GitLab From 9ceb61634e2cf6176bd733e1684b80f97e0bb086 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 27 Sep 2018 19:43:10 +0900 Subject: [PATCH 053/118] Add spec for YamlProcessor --- spec/lib/gitlab/ci/yaml_processor_spec.rb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index a2d429fa859..d75c473eb66 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -121,6 +121,21 @@ module Ci end end end + + describe 'delayed job entry' do + context 'when delayed is defined' do + let(:config) do + YAML.dump(rspec: { script: 'rollout 10%', + when: 'delayed', + start_in: '1 day' }) + end + + it 'has the attributes' do + expect(subject[:when]).to eq 'delayed' + expect(subject[:options][:start_in]).to eq '1 day' + end + end + end end describe '#stages_attributes' do @@ -1260,7 +1275,7 @@ module Ci config = YAML.dump({ rspec: { script: "test", when: 1 } }) expect do Gitlab::Ci::YamlProcessor.new(config) - end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec when should be on_success, on_failure, always or manual") + end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec when should be on_success, on_failure, always, manual or delayed") end it "returns errors if job artifacts:name is not an a string" do -- GitLab From 52c769526cbe749b4e884238a6530df268345e26 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 27 Sep 2018 13:08:13 +0200 Subject: [PATCH 054/118] Upgrade gitlab-svgs to include scheduled job assets --- yarn.lock | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 20b587d1fc5..56fc008a0ea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -180,11 +180,15 @@ lodash "^4.17.10" to-fast-properties "^2.0.0" -"@gitlab-org/gitlab-svgs@^1.23.0", "@gitlab-org/gitlab-svgs@^1.29.0": +"@gitlab-org/gitlab-svgs@^1.23.0": version "1.29.0" resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.29.0.tgz#03b65b513f9099bbda6ecf94d673a2952f8c6c70" integrity sha512-sCl6nP3ph36+8P3nrw9VanAR648rgOUEBlEoLPHkhKm79xB1dUkXGBtI0uaSJVgbJx40M1/Ts8HSdMv+PF3EIg== +"@gitlab-org/gitlab-svgs@^1.29.0": + version "1.30.0" + resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.30.0.tgz#72dca2fb67bafb3c975a322dc6406aaa29aed86c" + "@gitlab-org/gitlab-ui@^1.7.1": version "1.7.1" resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-ui/-/gitlab-ui-1.7.1.tgz#e9cce86cb7e34311405e705c1de674276b453f17" -- GitLab From f976418d128e9f00d6bc6e7eb2862e553e82934c Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 27 Sep 2018 13:11:24 +0200 Subject: [PATCH 055/118] Fix URL to empty state graphic of scheduled jobs --- lib/gitlab/ci/status/build/scheduled.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index d7d762cdf7a..493c71718a2 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -5,7 +5,7 @@ module Build class Scheduled < Status::Extended def illustration { - image: 'illustrations/scheduled-job_countdown.svg', + image: 'illustrations/illustrations_scheduled-job_countdown.svg', size: 'svg-394', title: _("This is a scheduled to run in ") + " #{execute_in}", content: _("This job will automatically run after it's timer finishes. " \ -- GitLab From ea38e832f0f197c5121504d7e193227d3d9c7867 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 27 Sep 2018 13:44:03 +0200 Subject: [PATCH 056/118] Allow remaining time of scheduled jobs to overflow one day --- app/helpers/time_helper.rb | 18 +++++++-- app/views/projects/ci/builds/_build.html.haml | 2 +- lib/gitlab/ci/status/build/scheduled.rb | 6 ++- spec/helpers/time_helper_spec.rb | 38 ++++++++++++++----- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/app/helpers/time_helper.rb b/app/helpers/time_helper.rb index 94044d7b85e..737ec33b2dd 100644 --- a/app/helpers/time_helper.rb +++ b/app/helpers/time_helper.rb @@ -21,9 +21,21 @@ def date_from_to(from, to) "#{from.to_s(:short)} - #{to.to_s(:short)}" end - def duration_in_numbers(duration) - time_format = duration < 1.hour ? "%M:%S" : "%H:%M:%S" + def duration_in_numbers(duration_in_seconds, allow_overflow = false) + if allow_overflow + seconds = duration_in_seconds % 1.minute + minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute) + hours = duration_in_seconds / 1.hour - Time.at(duration).utc.strftime(time_format) + if hours == 0 + "%02d:%02d" % [minutes, seconds] + else + "%02d:%02d:%02d" % [hours, minutes, seconds] + end + else + time_format = duration_in_seconds < 1.hour ? "%M:%S" : "%H:%M:%S" + + Time.at(duration_in_seconds).utc.strftime(time_format) + end end end diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index c706703ae6f..0a9a36e089a 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -106,7 +106,7 @@ .btn.btn-default.has-tooltip{ disabled: true, title: job.scheduled_at } = sprite_icon('planning') - = duration_in_numbers(job.execute_in) + = duration_in_numbers(job.execute_in, true) .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Start now') } = sprite_icon('play') .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Unschedule') } diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 493c71718a2..eebb3f761c5 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -25,9 +25,11 @@ def self.matches?(build, user) private + include TimeHelper + def execute_in - diff = [0, subject.scheduled_at - Time.now].max - Time.at(diff).utc.strftime("%H:%M:%S") + remaining_seconds = [0, subject.scheduled_at - Time.now].max + duration_in_numbers(remaining_seconds, true) end end end diff --git a/spec/helpers/time_helper_spec.rb b/spec/helpers/time_helper_spec.rb index 0b371d69ecf..37455c3e491 100644 --- a/spec/helpers/time_helper_spec.rb +++ b/spec/helpers/time_helper_spec.rb @@ -20,17 +20,35 @@ end describe "#duration_in_numbers" do - it "returns minutes and seconds" do - durations_and_expectations = { - 100 => "01:40", - 121 => "02:01", - 3721 => "01:02:01", - 0 => "00:00", - 42 => "00:42" - } + using RSpec::Parameterized::TableSyntax + + context "without passing allow_overflow" do + where(:duration, :formatted_string) do + 0 | "00:00" + 1.second | "00:01" + 42.seconds | "00:42" + 2.minutes + 1.second | "02:01" + 3.hours + 2.minutes + 1.second | "03:02:01" + 30.hours | "06:00:00" + end + + with_them do + it { expect(duration_in_numbers(duration)).to eq formatted_string } + end + end + + context "with allow_overflow = true" do + where(:duration, :formatted_string) do + 0 | "00:00" + 1.second | "00:01" + 42.seconds | "00:42" + 2.minutes + 1.second | "02:01" + 3.hours + 2.minutes + 1.second | "03:02:01" + 30.hours | "30:00:00" + end - durations_and_expectations.each do |duration, expectation| - expect(duration_in_numbers(duration)).to eq(expectation) + with_them do + it { expect(duration_in_numbers(duration, true)).to eq formatted_string } end end end -- GitLab From 308d11f4bbf48ae2ff539772c3b565a8db0bc276 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 27 Sep 2018 14:19:44 +0200 Subject: [PATCH 057/118] Use correct icon for scheduled jobs in pipeline graph --- lib/gitlab/ci/status/scheduled.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index f4464d69eb2..542100e41da 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -11,7 +11,7 @@ def label end def icon - 'timer' # TODO: 'status_scheduled' + 'status_scheduled' end def favicon -- GitLab From b5a591d8c2c8fff0136037f6c2a8b635f101e07e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 28 Sep 2018 14:30:43 +0900 Subject: [PATCH 058/118] Include delayed jobs action into manual actions --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index f83dfa5d1c4..b3916b67bd6 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -92,7 +92,7 @@ def persisted_environment scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } - scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } + scope :manual_actions, ->() { where(when: %i[manual delayed], status: COMPLETED_STATUSES + %i[manual scheduled]) } scope :ref_protected, -> { where(protected: true) } scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) } -- GitLab From 54263dc1d9719f871fb2f23ecac4e0cfcabebe77 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 28 Sep 2018 14:41:39 +0900 Subject: [PATCH 059/118] Fix coding style offence --- app/services/ci/process_build_service.rb | 1 + lib/gitlab/ci/status/build/failed.rb | 2 +- .../ci/process_pipeline_service_spec.rb | 21 +++++++++++-------- spec/workers/ci/build_schedule_worker_spec.rb | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 41ea62b4e4a..8a1a524429e 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -11,6 +11,7 @@ def execute(build, current_status) else build.enqueue end + true else build.skip diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index cdbcb7a47cd..014eb66a26b 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -11,7 +11,7 @@ class Failed < Status::Extended runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', runner_unsupported: 'unsupported runner', - schedule_expired: 'schedule expired', + schedule_expired: 'schedule expired' }.freeze private_constant :REASONS diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb index d314d774be4..3ce3785b162 100644 --- a/spec/services/ci/process_pipeline_service_spec.rb +++ b/spec/services/ci/process_pipeline_service_spec.rb @@ -257,7 +257,7 @@ it 'properly processes the pipeline' do expect(process_pipeline).to be_truthy expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) - + succeed_pending expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) @@ -283,7 +283,7 @@ it 'properly processes the pipeline' do expect(process_pipeline).to be_truthy expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) - + fail_running_or_pending expect(builds_names_and_statuses).to eq({ 'build': 'failed' }) @@ -295,11 +295,11 @@ it 'properly processes the pipeline' do expect(process_pipeline).to be_truthy expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) - + succeed_pending expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) - + unschedule expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'manual' }) @@ -324,11 +324,11 @@ it 'properly processes the pipeline' do expect(process_pipeline).to be_truthy expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) - + succeed_pending expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) - + enqueue_scheduled('rollout10%') fail_running_or_pending @@ -354,11 +354,11 @@ it 'properly processes the pipeline' do expect(process_pipeline).to be_truthy expect(builds_names_and_statuses).to eq({ 'build': 'pending' }) - + succeed_pending expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' }) - + play_manual_action('rollout10%') expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' }) @@ -718,7 +718,10 @@ def builds_names end def builds_names_and_statuses - builds.inject({}) { |h, b| h[b.name.to_sym] = b.status; h } + builds.each_with_object({}) do |h, b| + h[b.name.to_sym] = b.status + h + end end def all_builds_names diff --git a/spec/workers/ci/build_schedule_worker_spec.rb b/spec/workers/ci/build_schedule_worker_spec.rb index c76537b9233..4a3fe84d7f7 100644 --- a/spec/workers/ci/build_schedule_worker_spec.rb +++ b/spec/workers/ci/build_schedule_worker_spec.rb @@ -10,7 +10,7 @@ it 'executes RunScheduledBuildService' do expect_any_instance_of(Ci::RunScheduledBuildService) .to receive(:execute).once - + subject end end -- GitLab From 97c556bf1d49d9b5362fdd18255244d2da349a17 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 28 Sep 2018 15:55:13 +0900 Subject: [PATCH 060/118] Fix spec failure --- app/services/ci/process_build_service.rb | 2 +- spec/services/ci/process_pipeline_service_spec.rb | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 8a1a524429e..0fbe93130d0 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -5,7 +5,7 @@ class ProcessBuildService < BaseService def execute(build, current_status) if valid_statuses_for_when(build.when).include?(current_status) if build.schedulable? - build.schedule! + build.schedule elsif build.action? build.actionize else diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb index 3ce3785b162..4f1e487197e 100644 --- a/spec/services/ci/process_pipeline_service_spec.rb +++ b/spec/services/ci/process_pipeline_service_spec.rb @@ -31,17 +31,14 @@ succeed_pending expect(builds.success.count).to eq(2) - expect(process_pipeline).to be_truthy succeed_pending expect(builds.success.count).to eq(4) - expect(process_pipeline).to be_truthy succeed_pending expect(builds.success.count).to eq(5) - expect(process_pipeline).to be_falsey end it 'does not process pipeline if existing stage is running' do @@ -718,7 +715,7 @@ def builds_names end def builds_names_and_statuses - builds.each_with_object({}) do |h, b| + builds.each_with_object({}) do |b, h| h[b.name.to_sym] = b.status h end -- GitLab From 6d4511135d08aa2dc2ffa8aec236cea3ff77e1e8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 28 Sep 2018 16:17:01 +0900 Subject: [PATCH 061/118] Add spec for ProcessBuildService --- .../services/ci/process_build_service_spec.rb | 199 +++++++++++++++--- 1 file changed, 171 insertions(+), 28 deletions(-) diff --git a/spec/services/ci/process_build_service_spec.rb b/spec/services/ci/process_build_service_spec.rb index 962d07e185b..692f28c80cb 100644 --- a/spec/services/ci/process_build_service_spec.rb +++ b/spec/services/ci/process_build_service_spec.rb @@ -5,62 +5,205 @@ let(:user) { create(:user) } let(:project) { create(:project) } - subject { described_class.new(project, user).execute(build) } + subject { described_class.new(project, user).execute(build, current_status) } before do project.add_maintainer(user) end - context 'when build is schedulable' do - let(:build) { create(:ci_build, :created, :schedulable, user: user, project: project) } + shared_examples_for 'Enqueuing properly' do |valid_statuses_for_when| + valid_statuses_for_when.each do |status_for_prior_stages| + context "when status for prior stages is #{status_for_prior_stages}" do + let(:current_status) { status_for_prior_stages } - context 'when ci_enable_scheduled_build feature flag is enabled' do - before do - stub_feature_flags(ci_enable_scheduled_build: true) + %w[created skipped manual scheduled].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) } + + it 'enqueues the build' do + expect { subject }.to change { build.status }.to('pending') + end + end + end + + %w[pending running success failed canceled].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) } + + it 'does not change the build status' do + expect { subject }.not_to change { build.status } + end + end + end end + end - it 'schedules the build' do - Timecop.freeze do - expect(Ci::BuildScheduleWorker) - .to receive(:perform_at).with(1.minute.since, build.id) + (HasStatus::AVAILABLE_STATUSES - valid_statuses_for_when).each do |status_for_prior_stages| + let(:current_status) { status_for_prior_stages } - subject + context "when status for prior stages is #{status_for_prior_stages}" do + %w[created pending].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) } - expect(build).to be_scheduled + it 'skips the build' do + expect { subject }.to change { build.status }.to('skipped') + end + end + end + + (HasStatus::AVAILABLE_STATUSES - %w[created pending]).each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) } + + it 'does not change build status' do + expect { subject }.not_to change { build.status } + end + end end end end + end + + shared_examples_for 'Actionizing properly' do |valid_statuses_for_when| + valid_statuses_for_when.each do |status_for_prior_stages| + context "when status for prior stages is #{status_for_prior_stages}" do + let(:current_status) { status_for_prior_stages } + + %w[created].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) } - context 'when ci_enable_scheduled_build feature flag is disabled' do - before do - stub_feature_flags(ci_enable_scheduled_build: false) + it 'enqueues the build' do + expect { subject }.to change { build.status }.to('manual') + end + end + end + + %w[manual skipped pending running success failed canceled scheduled].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) } + + it 'does not change the build status' do + expect { subject }.not_to change { build.status } + end + end + end end + end + + (HasStatus::AVAILABLE_STATUSES - valid_statuses_for_when).each do |status_for_prior_stages| + let(:current_status) { status_for_prior_stages } - it 'enqueues the build' do - subject + context "when status for prior stages is #{status_for_prior_stages}" do + %w[created pending].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) } - expect(build).to be_manual + it 'skips the build' do + expect { subject }.to change { build.status }.to('skipped') + end + end + end + + (HasStatus::AVAILABLE_STATUSES - %w[created pending]).each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) } + + it 'does not change build status' do + expect { subject }.not_to change { build.status } + end + end + end end end end - context 'when build is actionable' do - let(:build) { create(:ci_build, :created, :actionable, user: user, project: project) } + shared_examples_for 'Scheduling properly' do |valid_statuses_for_when| + valid_statuses_for_when.each do |status_for_prior_stages| + context "when status for prior stages is #{status_for_prior_stages}" do + let(:current_status) { status_for_prior_stages } + + %w[created].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) } + + it 'enqueues the build' do + expect { subject }.to change { build.status }.to('scheduled') + end + end + end + + %w[manual skipped pending running success failed canceled scheduled].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) } + + it 'does not change the build status' do + expect { subject }.not_to change { build.status } + end + end + end + end + end - it 'actionizes the build' do - subject + (HasStatus::AVAILABLE_STATUSES - valid_statuses_for_when).each do |status_for_prior_stages| + let(:current_status) { status_for_prior_stages } - expect(build).to be_manual + context "when status for prior stages is #{status_for_prior_stages}" do + %w[created pending].each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) } + + it 'skips the build' do + expect { subject }.to change { build.status }.to('skipped') + end + end + end + + (HasStatus::AVAILABLE_STATUSES - %w[created pending]).each do |status| + context "when build status is #{status}" do + let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) } + + it 'does not change build status' do + expect { subject }.not_to change { build.status } + end + end + end + end end end - context 'when build does not have any actions' do - let(:build) { create(:ci_build, :created, user: user, project: project) } + context 'when build has on_success option' do + let(:when_option) { :on_success } - it 'enqueues the build' do - subject + it_behaves_like 'Enqueuing properly', %w[success skipped] + end + + context 'when build has on_failure option' do + let(:when_option) { :on_failure } + + it_behaves_like 'Enqueuing properly', %w[failed] + end + + context 'when build has always option' do + let(:when_option) { :always } + + it_behaves_like 'Enqueuing properly', %w[success failed skipped] + end + + context 'when build has manual option' do + let(:when_option) { :manual } + + it_behaves_like 'Actionizing properly', %w[success skipped] + end + + context 'when build has delayed option' do + let(:when_option) { :delayed } - expect(build).to be_pending + before do + allow(Ci::BuildScheduleWorker).to receive(:perform_at) { } end + + it_behaves_like 'Scheduling properly', %w[success skipped] end end -- GitLab From 19fb42b5ca917d94b44544e596b9e474e50e0907 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 28 Sep 2018 17:26:45 +0900 Subject: [PATCH 062/118] Add spec for Build::Factory --- .../gitlab/ci/status/build/factory_spec.rb | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/spec/lib/gitlab/ci/status/build/factory_spec.rb b/spec/lib/gitlab/ci/status/build/factory_spec.rb index 8b92088902b..1073c4b7ccd 100644 --- a/spec/lib/gitlab/ci/status/build/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/build/factory_spec.rb @@ -319,4 +319,53 @@ end end end + + context 'when build is a delayed action' do + let(:build) { create(:ci_build, :scheduled) } + + it 'matches correct core status' do + expect(factory.core_status).to be_a Gitlab::Ci::Status::Scheduled + end + + it 'matches correct extended statuses' do + expect(factory.extended_statuses) + .to eq [Gitlab::Ci::Status::Build::Scheduled, + Gitlab::Ci::Status::Build::Play, + Gitlab::Ci::Status::Build::Action] + end + + it 'fabricates action detailed status' do + expect(status).to be_a Gitlab::Ci::Status::Build::Action + end + + it 'fabricates status with correct details' do + expect(status.text).to eq 'scheduled' + expect(status.group).to eq 'scheduled' + expect(status.icon).to eq 'status_scheduled' + expect(status.favicon).to eq 'favicon_status_scheduled' + expect(status.illustration).to include(:image, :size, :title, :content) + expect(status.label).to include 'manual play action' + expect(status).to have_details + expect(status.action_path).to include 'play' + end + + context 'when user has ability to play action' do + it 'fabricates status that has action' do + expect(status).to have_action + end + end + + context 'when user does not have ability to play action' do + before do + allow(build.project).to receive(:empty_repo?).and_return(false) + + create(:protected_branch, :no_one_can_push, + name: build.ref, project: build.project) + end + + it 'fabricates status that has no action' do + expect(status).not_to have_action + end + end + end end -- GitLab From ef9c8520dc8d572928a2a382ab3c92214356efc3 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Fri, 28 Sep 2018 11:28:43 +0200 Subject: [PATCH 063/118] Upgrade gitlab-svgs to include new scheduled job assets --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 56fc008a0ea..f2e948b0dc2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -186,8 +186,8 @@ integrity sha512-sCl6nP3ph36+8P3nrw9VanAR648rgOUEBlEoLPHkhKm79xB1dUkXGBtI0uaSJVgbJx40M1/Ts8HSdMv+PF3EIg== "@gitlab-org/gitlab-svgs@^1.29.0": - version "1.30.0" - resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.30.0.tgz#72dca2fb67bafb3c975a322dc6406aaa29aed86c" + version "1.31.0" + resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.31.0.tgz#495b074669f93af40e34f9978ce887773dea470a" "@gitlab-org/gitlab-ui@^1.7.1": version "1.7.1" -- GitLab From 93cd5edbc1fe0023d82227ab135314d068263441 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Fri, 28 Sep 2018 11:47:55 +0200 Subject: [PATCH 064/118] Add black border to scheduled icon in pipeline graphs --- app/assets/stylesheets/pages/pipelines.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/stylesheets/pages/pipelines.scss b/app/assets/stylesheets/pages/pipelines.scss index 8bb8b83dc5e..14395cc59b0 100644 --- a/app/assets/stylesheets/pages/pipelines.scss +++ b/app/assets/stylesheets/pages/pipelines.scss @@ -760,6 +760,7 @@ } &.ci-status-icon-canceled, + &.ci-status-icon-scheduled, &.ci-status-icon-disabled, &.ci-status-icon-not-found, &.ci-status-icon-manual { -- GitLab From fb4d06d1df731c5e13e0f464f451d66cbe389571 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Fri, 28 Sep 2018 11:50:00 +0200 Subject: [PATCH 065/118] Use time-out icon for unschedule job action --- app/views/projects/ci/builds/_build.html.haml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index 0a9a36e089a..4bef4ba2d7e 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -110,8 +110,7 @@ .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Start now') } = sprite_icon('play') .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Unschedule') } - = sprite_icon('cancel') - -# sprite_icon('status_scheduled_borderless') + = sprite_icon('time-out') - elsif allow_retry - if job.playable? && !admin && can?(current_user, :update_build, job) = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Play', class: 'btn btn-build' do -- GitLab From a220e72c7027ecf48da73daf0d2de5315b4bc675 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Fri, 28 Sep 2018 11:52:30 +0200 Subject: [PATCH 066/118] Add black border to scheduled job badge --- app/assets/stylesheets/pages/status.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/stylesheets/pages/status.scss b/app/assets/stylesheets/pages/status.scss index 620297e589d..7d59dd3b5d1 100644 --- a/app/assets/stylesheets/pages/status.scss +++ b/app/assets/stylesheets/pages/status.scss @@ -27,6 +27,7 @@ &.ci-canceled, &.ci-disabled, + &.ci-scheduled, &.ci-manual { color: $gl-text-color; border-color: $gl-text-color; -- GitLab From 70d015d1ba5adde82c6f38567ad51cfb85dae5f6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 13:10:30 +0900 Subject: [PATCH 067/118] Cleanup drop_stale_scheduled_builds code --- app/workers/stuck_ci_jobs_worker.rb | 9 +++++++-- scheduled_job_fixture.rb | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 821ea75703f..5028965862f 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -10,6 +10,7 @@ class StuckCiJobsWorker BUILD_PENDING_OUTDATED_TIMEOUT = 1.day BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour BUILD_PENDING_STUCK_TIMEOUT = 1.hour + BUILD_SCHEDULED_OUTDATED_BATCH_SIZE = 100 def perform return unless try_obtain_lease @@ -68,8 +69,12 @@ def drop_stale_scheduled_builds # `ci_builds` table has a partial index on `id` with `scheduled_at <> NULL` condition. # Therefore this query's first step uses Index Search, and the following expensive # filter `scheduled_at < ?` will only perform on a small subset (max: 100 rows) - Ci::Build.include(EachBatch).where('scheduled_at IS NOT NULL').each_batch(of: 100) do |relation| - relation.where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago).find_each do |build| + Ci::Build.include(EachBatch) + .where('scheduled_at IS NOT NULL') + .each_batch(of: BUILD_SCHEDULED_OUTDATED_BATCH_SIZE) do |relation| + relation + .where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago) + .find_each(batch_size: BUILD_SCHEDULED_OUTDATED_BATCH_SIZE) do |build| drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired) end end diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb index ae33c6be6ad..ed205bf656c 100644 --- a/scheduled_job_fixture.rb +++ b/scheduled_job_fixture.rb @@ -189,4 +189,26 @@ def cancel_jobs(stage_name) def cancel_pipeline Ci::Pipeline.last.cancel_running end + + def create_stale_scheduled_builds + count = 100 + rows = [] + last_pipeline = Ci::Pipeline.last + last_stage = last_pipeline.stages.last + + count.times do |i| + rows << { + name: "delayed-job-bulk-#{i}", + project_id: project.id, + commit_id: last_pipeline.id, + status: 'scheduled', + scheduled_at: 1.day.ago, + user_id: user.id, + stage_id: last_stage.id, + type: 'Ci::Build' + } + end + + Gitlab::Database.bulk_insert('ci_builds', rows) + end end -- GitLab From efaa3669c182e68d43b68a1f4257b483aad55f01 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 13:38:37 +0900 Subject: [PATCH 068/118] Change the order of status_sql to avoid the query for scheduled status at the earlier step --- app/models/concerns/has_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 88a0c9919e7..b92643f87f8 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -33,7 +33,6 @@ def status_sql warnings = scope_warnings.select('count(*) > 0').to_sql.presence || 'false' "(CASE - WHEN (#{scheduled})>0 THEN 'scheduled' WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' WHEN (#{builds})=(#{skipped}) THEN 'skipped' WHEN (#{builds})=(#{success}) THEN 'success' @@ -43,6 +42,7 @@ def status_sql WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' WHEN (#{running})+(#{pending})>0 THEN 'running' WHEN (#{manual})>0 THEN 'manual' + WHEN (#{scheduled})>0 THEN 'scheduled' WHEN (#{created})>0 THEN 'running' ELSE 'failed' END)" -- GitLab From 384da9279eefd7ebcdfd684ff234540d935ededd Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 13:57:11 +0900 Subject: [PATCH 069/118] Fix spec --- spec/lib/gitlab/ci/status/pipeline/factory_spec.rb | 6 +++--- spec/services/ci/run_scheduled_build_service_spec.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb b/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb index 8e3d4464898..d83f260927f 100644 --- a/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb @@ -32,10 +32,10 @@ it 'does not match extended statuses' do expect(factory.extended_statuses).to be_empty end - end - it "fabricates a core status #{simple_status}" do - expect(status).to be_a expected_status + it "fabricates a core status #{simple_status}" do + expect(status).to be_a expected_status + end end it 'extends core status with common pipeline methods' do diff --git a/spec/services/ci/run_scheduled_build_service_spec.rb b/spec/services/ci/run_scheduled_build_service_spec.rb index 145905db0cf..be2aad33ef4 100644 --- a/spec/services/ci/run_scheduled_build_service_spec.rb +++ b/spec/services/ci/run_scheduled_build_service_spec.rb @@ -29,7 +29,7 @@ context 'when scheduled_at is not expired' do let(:build) { create(:ci_build, :scheduled, user: user, project: project, pipeline: pipeline) } - it 'can run the build' do + it 'can not run the build' do expect { subject }.to raise_error(StateMachines::InvalidTransition) expect(build).to be_scheduled -- GitLab From 7fc74818a32a713d971582f8730b163dade8df3d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 15:27:34 +0900 Subject: [PATCH 070/118] Add scheduled_actions as an explicit group of actions --- app/models/ci/build.rb | 3 ++- app/models/ci/pipeline.rb | 1 + app/serializers/build_action_entity.rb | 1 + app/serializers/job_entity.rb | 1 + app/serializers/pipeline_details_entity.rb | 1 + app/serializers/pipeline_serializer.rb | 1 + 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b3916b67bd6..4f1f5d047a3 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -92,7 +92,8 @@ def persisted_environment scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } - scope :manual_actions, ->() { where(when: %i[manual delayed], status: COMPLETED_STATUSES + %i[manual scheduled]) } + scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } + scope :scheduled_actions, ->() { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) } scope :ref_protected, -> { where(protected: true) } scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) } diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index f936115014d..b74c65c2627 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -35,6 +35,7 @@ class Pipeline < ActiveRecord::Base has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus' has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' + has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' diff --git a/app/serializers/build_action_entity.rb b/app/serializers/build_action_entity.rb index f9da3f63911..3e81f8f0218 100644 --- a/app/serializers/build_action_entity.rb +++ b/app/serializers/build_action_entity.rb @@ -12,6 +12,7 @@ class BuildActionEntity < Grape::Entity end expose :playable?, as: :playable + expose :scheduled_at private diff --git a/app/serializers/job_entity.rb b/app/serializers/job_entity.rb index 26b29993fec..dd6c2fa1a6d 100644 --- a/app/serializers/job_entity.rb +++ b/app/serializers/job_entity.rb @@ -25,6 +25,7 @@ class JobEntity < Grape::Entity end expose :playable?, as: :playable + expose :scheduled_at expose :created_at expose :updated_at expose :detailed_status, as: :status, with: DetailedStatusEntity diff --git a/app/serializers/pipeline_details_entity.rb b/app/serializers/pipeline_details_entity.rb index 3b56767f774..d78ad4af4dc 100644 --- a/app/serializers/pipeline_details_entity.rb +++ b/app/serializers/pipeline_details_entity.rb @@ -5,5 +5,6 @@ class PipelineDetailsEntity < PipelineEntity expose :ordered_stages, as: :stages, using: StageEntity expose :artifacts, using: BuildArtifactEntity expose :manual_actions, using: BuildActionEntity + expose :scheduled_actions, using: BuildActionEntity end end diff --git a/app/serializers/pipeline_serializer.rb b/app/serializers/pipeline_serializer.rb index 4f31af3c46d..7451433a841 100644 --- a/app/serializers/pipeline_serializer.rb +++ b/app/serializers/pipeline_serializer.rb @@ -13,6 +13,7 @@ def represent(resource, opts = {}) :cancelable_statuses, :trigger_requests, :manual_actions, + :scheduled_actions, :artifacts, { pending_builds: :project, -- GitLab From bc5d649a4c456aab4ec7eddc52d13d172a5adace Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 16:09:41 +0900 Subject: [PATCH 071/118] Add unschedule action to status build --- app/serializers/build_action_entity.rb | 6 +++- lib/gitlab/ci/status/build/factory.rb | 1 + lib/gitlab/ci/status/build/unschedule.rb | 41 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/ci/status/build/unschedule.rb diff --git a/app/serializers/build_action_entity.rb b/app/serializers/build_action_entity.rb index 3e81f8f0218..0db7875aa87 100644 --- a/app/serializers/build_action_entity.rb +++ b/app/serializers/build_action_entity.rb @@ -12,7 +12,11 @@ class BuildActionEntity < Grape::Entity end expose :playable?, as: :playable - expose :scheduled_at + expose :scheduled_at, if: -> (build) { build.scheduled? } + + expose :unschedule_path, if: -> (build) { build.scheduled? } do |build| + unschedule_project_job_path(build.project, build) + end private diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 3f762c42747..4a74d6d6ed1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -15,6 +15,7 @@ def self.extended_statuses Status::Build::Retryable], [Status::Build::Failed], [Status::Build::FailedAllowed, + Status::Build::Unschedule, Status::Build::Play, Status::Build::Stop], [Status::Build::Action], diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb new file mode 100644 index 00000000000..e1b7b83428c --- /dev/null +++ b/lib/gitlab/ci/status/build/unschedule.rb @@ -0,0 +1,41 @@ +module Gitlab + module Ci + module Status + module Build + class Unschedule < Status::Extended + def label + 'unschedule action' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'time-out' + end + + def action_title + 'Unschedule' + end + + def action_button_title + _('Unschedule job') + end + + def action_path + unschedule_project_job_path(subject.project, subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.scheduled? + end + end + end + end + end +end -- GitLab From 533f5ca4c9f899910f9cdc4f0e0b43d619a9c7df Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 17:09:07 +0900 Subject: [PATCH 072/118] Fix spec --- spec/services/ci/process_pipeline_service_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb index 4f1e487197e..8c7258c42ad 100644 --- a/spec/services/ci/process_pipeline_service_spec.rb +++ b/spec/services/ci/process_pipeline_service_spec.rb @@ -395,7 +395,7 @@ enqueue_scheduled('delayed1') expect(builds_names_and_statuses).to eq({ 'delayed1': 'pending', 'delayed2': 'scheduled' }) - expect(pipeline.reload.status).to eq 'scheduled' + expect(pipeline.reload.status).to eq 'running' end end -- GitLab From eb238ec16004d9d22c20193eb8a8ee6cf0df4c8b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 1 Oct 2018 19:00:34 +0900 Subject: [PATCH 073/118] Fix scheduled icon for stages --- app/assets/stylesheets/framework/icons.scss | 1 + app/helpers/ci_status_helper.rb | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/assets/stylesheets/framework/icons.scss b/app/assets/stylesheets/framework/icons.scss index f002edced8a..abd26e38d18 100644 --- a/app/assets/stylesheets/framework/icons.scss +++ b/app/assets/stylesheets/framework/icons.scss @@ -64,6 +64,7 @@ } } +.ci-status-icon-scheduled, .ci-status-icon-manual { svg { fill: $gl-text-color; diff --git a/app/helpers/ci_status_helper.rb b/app/helpers/ci_status_helper.rb index f343607a343..6f9e2ef78cd 100644 --- a/app/helpers/ci_status_helper.rb +++ b/app/helpers/ci_status_helper.rb @@ -7,7 +7,6 @@ # # See 'detailed_status?` method and `Gitlab::Ci::Status` module. # -# TODO: DO I need to update this deprecated module? module CiStatusHelper def ci_label_for_status(status) if detailed_status?(status) @@ -21,6 +20,8 @@ def ci_label_for_status(status) 'passed with warnings' when 'manual' 'waiting for manual action' + when 'scheduled' + 'waiting for delayed job' else status end @@ -40,6 +41,8 @@ def ci_text_for_status(status) s_('CiStatusText|passed') when 'manual' s_('CiStatusText|blocked') + when 'scheduled' + s_('CiStatusText|scheduled') else # All states are already being translated inside the detailed statuses: # :running => Gitlab::Ci::Status::Running @@ -84,6 +87,8 @@ def ci_icon_for_status(status, size: 16) 'status_skipped' when 'manual' 'status_manual' + when 'scheduled' + 'status_scheduled' else 'status_canceled' end -- GitLab From 6369ff1ce172cefc84574f2dde055399b64bf7b5 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Mon, 1 Oct 2018 12:47:07 +0200 Subject: [PATCH 074/118] Add actions to scheduled job buttons on job list page --- app/views/projects/ci/builds/_build.html.haml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index 4bef4ba2d7e..c09137ec085 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -107,9 +107,15 @@ title: job.scheduled_at } = sprite_icon('planning') = duration_in_numbers(job.execute_in, true) - .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Start now') } + = link_to play_project_job_path(job.project, job, return_to: request.original_url), + method: :post, + title: s_('DelayedJobs|Start now'), + class: 'btn btn-default btn-build has-tooltip' do = sprite_icon('play') - .btn.btn-default.btn-build.has-tooltip{ title: s_('DelayedJobs|Unschedule') } + = link_to unschedule_project_job_path(job.project, job, return_to: request.original_url), + method: :post, + title: s_('DelayedJobs|Unschedule'), + class: 'btn btn-default btn-build has-tooltip' do = sprite_icon('time-out') - elsif allow_retry - if job.playable? && !admin && can?(current_user, :update_build, job) -- GitLab From 786ae683679d90a0e55bfe844ac694aeb7d68ce6 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Mon, 1 Oct 2018 12:55:09 +0200 Subject: [PATCH 075/118] Do not omit leading zeros in duration_in_numbers helper --- app/helpers/time_helper.rb | 6 +----- spec/helpers/time_helper_spec.rb | 8 ++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/app/helpers/time_helper.rb b/app/helpers/time_helper.rb index 737ec33b2dd..3e6a301b77d 100644 --- a/app/helpers/time_helper.rb +++ b/app/helpers/time_helper.rb @@ -27,11 +27,7 @@ def duration_in_numbers(duration_in_seconds, allow_overflow = false) minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute) hours = duration_in_seconds / 1.hour - if hours == 0 - "%02d:%02d" % [minutes, seconds] - else - "%02d:%02d:%02d" % [hours, minutes, seconds] - end + "%02d:%02d:%02d" % [hours, minutes, seconds] else time_format = duration_in_seconds < 1.hour ? "%M:%S" : "%H:%M:%S" diff --git a/spec/helpers/time_helper_spec.rb b/spec/helpers/time_helper_spec.rb index 37455c3e491..cc310766433 100644 --- a/spec/helpers/time_helper_spec.rb +++ b/spec/helpers/time_helper_spec.rb @@ -39,10 +39,10 @@ context "with allow_overflow = true" do where(:duration, :formatted_string) do - 0 | "00:00" - 1.second | "00:01" - 42.seconds | "00:42" - 2.minutes + 1.second | "02:01" + 0 | "00:00:00" + 1.second | "00:00:01" + 42.seconds | "00:00:42" + 2.minutes + 1.second | "00:02:01" 3.hours + 2.minutes + 1.second | "03:02:01" 30.hours | "30:00:00" end -- GitLab From fc8d4c7046807ac65e36ce65df982ff5c4ff4a9a Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Mon, 1 Oct 2018 15:21:33 +0200 Subject: [PATCH 076/118] Add scheduled job dropdown to pipelines list --- .../javascripts/lib/utils/datetime_utility.js | 20 +++++++++++++++ .../components/pipelines_actions.vue | 17 +++++++++++-- .../components/pipelines_table_row.vue | 7 ++++-- spec/javascripts/datetime_utility_spec.js | 25 ++++++++++++++----- 4 files changed, 59 insertions(+), 10 deletions(-) diff --git a/app/assets/javascripts/lib/utils/datetime_utility.js b/app/assets/javascripts/lib/utils/datetime_utility.js index 1f66fa811ea..abfcf1eaf3f 100644 --- a/app/assets/javascripts/lib/utils/datetime_utility.js +++ b/app/assets/javascripts/lib/utils/datetime_utility.js @@ -370,3 +370,23 @@ window.gl.utils = { getTimeago, localTimeAgo, }; + +/** + * Formats milliseconds as timestamp (e.g. 01:02:03). + * + * @param milliseconds + * @returns {string} + */ +export const formatTime = milliseconds => { + const remainingSeconds = Math.floor(milliseconds / 1000) % 60; + const remainingMinutes = Math.floor(milliseconds / 1000 / 60) % 60; + const remainingHours = Math.floor(milliseconds / 1000 / 60 / 60); + let formattedTime = ''; + if (remainingHours < 10) formattedTime += '0'; + formattedTime += `${remainingHours}:`; + if (remainingMinutes < 10) formattedTime += '0'; + formattedTime += `${remainingMinutes}:`; + if (remainingSeconds < 10) formattedTime += '0'; + formattedTime += remainingSeconds; + return formattedTime; +}; diff --git a/app/assets/javascripts/pipelines/components/pipelines_actions.vue b/app/assets/javascripts/pipelines/components/pipelines_actions.vue index 017dd560621..fa5ad62f438 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_actions.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_actions.vue @@ -1,4 +1,5 @@ @@ -63,8 +69,8 @@ export default {
diff --git a/app/assets/javascripts/pipelines/components/pipelines_table_row.vue b/app/assets/javascripts/pipelines/components/pipelines_table_row.vue index a39cc265601..bae6ff43ee4 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_table_row.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_table_row.vue @@ -59,6 +59,9 @@ export default { }; }, computed: { + actions() { + return [...this.pipeline.details.manual_actions, ...this.pipeline.details.scheduled_actions]; + }, /** * If provided, returns the commit tag. * Needed to render the commit component column. @@ -321,8 +324,8 @@ export default { >
{ const date = new Date(); date.setFullYear(date.getFullYear() - 1); - expect( - datetimeUtility.timeFor(date), - ).toBe('Past due'); + expect(datetimeUtility.timeFor(date)).toBe('Past due'); }); it('returns remaining time when in the future', () => { @@ -19,9 +17,7 @@ describe('Date time utils', () => { // short of a full year, timeFor will return '11 months remaining' date.setDate(date.getDate() + 1); - expect( - datetimeUtility.timeFor(date), - ).toBe('1 year remaining'); + expect(datetimeUtility.timeFor(date)).toBe('1 year remaining'); }); }); @@ -168,3 +164,20 @@ describe('getTimeframeWindowFrom', () => { }); }); }); + +describe('formatTime', () => { + const expectedTimestamps = [ + [0, '00:00:00'], + [1000, '00:00:01'], + [42000, '00:00:42'], + [121000, '00:02:01'], + [10921000, '03:02:01'], + [108000000, '30:00:00'], + ]; + + expectedTimestamps.forEach(([milliseconds, expectedTimestamp]) => { + it(`formats ${milliseconds}ms as ${expectedTimestamp}`, () => { + expect(datetimeUtility.formatTime(milliseconds)).toBe(expectedTimestamp); + }); + }); +}); -- GitLab From 51a8177658b2a90e78b762667b37807a72d7a982 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Mon, 1 Oct 2018 18:12:30 +0200 Subject: [PATCH 077/118] Add badge for scheduled jobs --- app/views/projects/ci/builds/_build.html.haml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index c09137ec085..be24e7dadaf 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -49,6 +49,8 @@ %span.badge.badge-danger allowed to fail - if job.action? %span.badge.badge-info manual + - if job.scheduled? + %span.badge.badge-info= s_('DelayedJobs|scheduled') - if pipeline_link %td -- GitLab From 336affe911885a84c2f14193e3fa43f0320c0cb2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 09:44:22 +0900 Subject: [PATCH 078/118] Add scheduled status --- lib/gitlab/ci/status/pipeline/blocked.rb | 2 +- lib/gitlab/ci/status/pipeline/factory.rb | 1 + lib/gitlab/ci/status/pipeline/scheduled.rb | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/ci/status/pipeline/scheduled.rb diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index 59fcd8ad7ff..bf7e484ee9b 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -8,7 +8,7 @@ def text end def label - s_('CiStatusLabel|waiting for manual action or delayed job') + s_('CiStatusLabel|waiting for manual action') end def self.matches?(pipeline, user) diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 17f9a75f436..00d8f01cbdc 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -5,6 +5,7 @@ module Pipeline class Factory < Status::Factory def self.extended_statuses [[Status::SuccessWarning, + Status::Pipeline::Scheduled, Status::Pipeline::Blocked]] end diff --git a/lib/gitlab/ci/status/pipeline/scheduled.rb b/lib/gitlab/ci/status/pipeline/scheduled.rb new file mode 100644 index 00000000000..9ec6994bd2f --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/scheduled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Scheduled < Status::Extended + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|waiting for delayed job') + end + + def self.matches?(pipeline, user) + pipeline.scheduled? + end + end + end + end + end +end -- GitLab From 8bc065e02dcc4582aebbc7d28a30f7468a15ccc0 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 13:32:35 +0900 Subject: [PATCH 079/118] Rename failure reason to stale_schedule --- app/models/commit_status.rb | 2 +- app/presenters/commit_status_presenter.rb | 2 +- app/workers/stuck_ci_jobs_worker.rb | 2 +- lib/gitlab/ci/status/build/failed.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 2020dd637a0..06507345fe8 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -50,7 +50,7 @@ class CommitStatus < ActiveRecord::Base runner_system_failure: 4, missing_dependency_failure: 5, runner_unsupported: 6, - schedule_expired: 7 + stale_schedule: 7 } ## diff --git a/app/presenters/commit_status_presenter.rb b/app/presenters/commit_status_presenter.rb index b2b9fb55cba..29eaad759bb 100644 --- a/app/presenters/commit_status_presenter.rb +++ b/app/presenters/commit_status_presenter.rb @@ -9,7 +9,7 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated runner_system_failure: 'There has been a runner system failure, please try again', missing_dependency_failure: 'There has been a missing dependency failure', runner_unsupported: 'Your runner is outdated, please upgrade your runner', - schedule_expired: 'Scheduled job could not be executed by some reason, please try again' + stale_schedule: 'Delayed job could not be executed by some reason, please try again' }.freeze private_constant :CALLOUT_FAILURE_MESSAGES diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 5028965862f..cc5762156d7 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -75,7 +75,7 @@ def drop_stale_scheduled_builds relation .where('scheduled_at < ?', BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago) .find_each(batch_size: BUILD_SCHEDULED_OUTDATED_BATCH_SIZE) do |build| - drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired) + drop_build(:outdated, build, :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :stale_schedule) end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 014eb66a26b..50b0d044265 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -11,7 +11,7 @@ class Failed < Status::Extended runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', runner_unsupported: 'unsupported runner', - schedule_expired: 'schedule expired' + stale_schedule: 'stale schedule' }.freeze private_constant :REASONS -- GitLab From 46fc55993a295b3832f82359d9a6ac4cd1ee8aa7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 13:43:45 +0900 Subject: [PATCH 080/118] Add feature flag spec for process_build_service --- spec/services/ci/process_build_service_spec.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/spec/services/ci/process_build_service_spec.rb b/spec/services/ci/process_build_service_spec.rb index 692f28c80cb..9f47439dc4a 100644 --- a/spec/services/ci/process_build_service_spec.rb +++ b/spec/services/ci/process_build_service_spec.rb @@ -204,6 +204,20 @@ allow(Ci::BuildScheduleWorker).to receive(:perform_at) { } end - it_behaves_like 'Scheduling properly', %w[success skipped] + context 'when ci_enable_scheduled_build is enabled' do + before do + stub_feature_flags(ci_enable_scheduled_build: true) + end + + it_behaves_like 'Scheduling properly', %w[success skipped] + end + + context 'when ci_enable_scheduled_build is enabled' do + before do + stub_feature_flags(ci_enable_scheduled_build: false) + end + + it_behaves_like 'Actionizing properly', %w[success skipped] + end end end -- GitLab From 63829ae4e81171a7fd6bee801e90708136da573d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 16:04:31 +0900 Subject: [PATCH 081/118] Show sheculed label for present and past jobs --- app/views/projects/ci/builds/_build.html.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index be24e7dadaf..2b699770998 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -47,10 +47,10 @@ %span.badge.badge-info triggered - if job.try(:allow_failure) %span.badge.badge-danger allowed to fail - - if job.action? - %span.badge.badge-info manual - - if job.scheduled? + - if job.schedulable? %span.badge.badge-info= s_('DelayedJobs|scheduled') + - elsif job.action? + %span.badge.badge-info manual - if pipeline_link %td -- GitLab From f1226cd5a9240c03438ef5f24a57bce4ad3ec554 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 17:32:49 +0900 Subject: [PATCH 082/118] Remove scheduled job fixture --- scheduled_job_fixture.rb | 214 --------------------------------------- 1 file changed, 214 deletions(-) delete mode 100644 scheduled_job_fixture.rb diff --git a/scheduled_job_fixture.rb b/scheduled_job_fixture.rb deleted file mode 100644 index ed205bf656c..00000000000 --- a/scheduled_job_fixture.rb +++ /dev/null @@ -1,214 +0,0 @@ -## -# ### -# IMPORTANT -# - Enable the feature flag `ci_enable_scheduled_build` on rails console! You can do `Feature.enable('ci_enable_scheduled_build')` -# This feature is off by default! -# -# -# This is a debug script to reproduce specific scenarios for scheduled jobs (https://gitlab.com/gitlab-org/gitlab-ce/issues/51352) -# By using this script, you don't need to setup GitLab runner. -# This script is specifically made for FE/UX engineers. They can quickly check how scheduled jobs behave. -# -# *** THIS IS NOT TO BE MERGED *** -# -# ### How to use ### -# -# ### Prerequisite -# 1. Create a project (for example with path `incremental-rollout`) -# 1. Create a .gitlab-ci.yml with the following content -# -=begin -stages: -- build -- test -- production -- rollout 10% -- rollout 50% -- rollout 100% -- cleanup - -build: - stage: build - script: sleep 1s - -test: - stage: test - script: sleep 3s - -rollout 10%: - stage: rollout 10% - script: date - when: delayed - start_in: 10 seconds - allow_failure: false - -rollout 50%: - stage: rollout 50% - script: date - when: delayed - start_in: 10 seconds - allow_failure: false - -rollout 100%: - stage: rollout 100% - script: date - when: delayed - start_in: 10 seconds - allow_failure: false - -cleanup: - stage: cleanup - script: date -=end -# -# ### How to load this script -# -# ``` -# bundle exec rails console # Login to rails console -# require '/path/to/scheduled_job_fixture.rb' # Load this script -# ``` -# -# ### Reproduce the scenario ~ when all stages succeeded ~ -# -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') -# 1. Wait until rollout 10% job is triggered -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('rollout 10%') -# 1. Wait until rollout 50% job is triggered -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('rollout 50%') -# 1. Wait until rollout 100% job is triggered -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('rollout 100%') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('cleanup') -# -# Expectation: Users see a succeccful pipeline -# -# ### Reproduce the scenario ~ when rollout 10% jobs failed ~ -# -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') -# 1. Wait until rollout 10% job is triggered -# 1. ScheduledJobFixture.new(16, 1).drop_jobs('rollout 10%') -# -# Expectation: Following stages should be skipped. -# -# ### Reproduce the scenario ~ when user clicked cancel button before build job finished ~ -# -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).cancel_pipeline -# -# Expectation: All stages should be canceled. -# -# ### Reproduce the scenario ~ when user canceled the pipeline after rollout 10% job is scheduled ~ -# -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') -# 1. Run next command before rollout 10% job is triggered -# 1. ScheduledJobFixture.new(16, 1).cancel_pipeline -# -# Expectation: rollout 10% job will be canceled. Following stages will be skipped. -# -# ### Reproduce the scenario ~ when user canceled rollout 10% job after rollout 10% job is scheduled ~ -# -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') -# 1. Run next command before rollout 10% job is triggered -# 1. ScheduledJobFixture.new(16, 1).cancel_jobs('rollout 10%') -# -# Expectation: rollout 10% job will be canceled. Following stages will be skipped. -# -# ### Reproduce the scenario ~ when user played rollout 10% job immidiately ~ -# -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') -# 1. Play rollout 10% job before rollout 10% job is triggered -# -# Expectation: rollout 10% becomes pending immidiately -# -# ### Reproduce the scenario ~ when rollout 10% job is allowed to fail ~ -# -# 1. Set `allow_failure: true` to rollout 10% job -# 1. ScheduledJobFixture.new(16, 1).create_pipeline('master') -# 1. ScheduledJobFixture.new(16, 1).finish_stage_until('test') -# 1. Wait until rollout 10% job is triggered -# 1. ScheduledJobFixture.new(16, 1).drop_jobs('rollout 10%') -# -# Expectation: rollout 50% job should be triggered -# - -class ScheduledJobFixture - attr_reader :project - attr_reader :user - - include GitlabRoutingHelper - - def initialize(project_id, user_id) - @project = Project.find_by_id(project_id) - @user = User.find_by_id(user_id) - end - - def create_pipeline(ref) - pipeline = Ci::CreatePipelineService.new(project, user, ref: ref).execute(:web) - Rails.application.routes.url_helpers.namespace_project_pipeline_url(project.namespace, project, pipeline) - end - - def finish_stage_until(stage_name) - pipeline = Ci::Pipeline.last - pipeline.stages.order(:id).each do |stage| - stage.builds.map(&:success) - stage.update_status - pipeline.update_status - - return if stage.name == stage_name - end - end - - def run_jobs(stage_name) - pipeline = Ci::Pipeline.last - stage = pipeline.stages.find_by_name(stage_name) - stage.builds.map(&:run) - stage.update_status - pipeline.update_status - end - - def drop_jobs(stage_name) - pipeline = Ci::Pipeline.last - stage = pipeline.stages.find_by_name(stage_name) - stage.builds.map(&:drop) - stage.update_status - pipeline.update_status - end - - def cancel_jobs(stage_name) - pipeline = Ci::Pipeline.last - stage = pipeline.stages.find_by_name(stage_name) - stage.builds.map(&:cancel) - stage.update_status - pipeline.update_status - end - - def cancel_pipeline - Ci::Pipeline.last.cancel_running - end - - def create_stale_scheduled_builds - count = 100 - rows = [] - last_pipeline = Ci::Pipeline.last - last_stage = last_pipeline.stages.last - - count.times do |i| - rows << { - name: "delayed-job-bulk-#{i}", - project_id: project.id, - commit_id: last_pipeline.id, - status: 'scheduled', - scheduled_at: 1.day.ago, - user_id: user.id, - stage_id: last_stage.id, - type: 'Ci::Build' - } - end - - Gitlab::Database.bulk_insert('ci_builds', rows) - end -end -- GitLab From 4dd70b9e1e67b50d846b4fb48c0ff942104119a5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 18:10:58 +0900 Subject: [PATCH 083/118] Fix spec --- spec/lib/gitlab/ci/status/build/factory_spec.rb | 6 +++--- spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb | 2 +- spec/lib/gitlab/ci/status/pipeline/factory_spec.rb | 7 ++++++- spec/lib/gitlab/import_export/all_models.yml | 1 + spec/workers/stuck_ci_jobs_worker_spec.rb | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/spec/lib/gitlab/ci/status/build/factory_spec.rb b/spec/lib/gitlab/ci/status/build/factory_spec.rb index 1073c4b7ccd..aa53ecd5967 100644 --- a/spec/lib/gitlab/ci/status/build/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/build/factory_spec.rb @@ -330,7 +330,7 @@ it 'matches correct extended statuses' do expect(factory.extended_statuses) .to eq [Gitlab::Ci::Status::Build::Scheduled, - Gitlab::Ci::Status::Build::Play, + Gitlab::Ci::Status::Build::Unschedule, Gitlab::Ci::Status::Build::Action] end @@ -344,9 +344,9 @@ expect(status.icon).to eq 'status_scheduled' expect(status.favicon).to eq 'favicon_status_scheduled' expect(status.illustration).to include(:image, :size, :title, :content) - expect(status.label).to include 'manual play action' + expect(status.label).to include 'unschedule action' expect(status).to have_details - expect(status.action_path).to include 'play' + expect(status.action_path).to include 'unschedule' end context 'when user has ability to play action' do diff --git a/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb b/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb index 49a25b4a389..1a2b952d374 100644 --- a/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb +++ b/spec/lib/gitlab/ci/status/pipeline/blocked_spec.rb @@ -15,7 +15,7 @@ describe '#label' do it 'overrides status label' do - expect(subject.label).to eq 'waiting for manual action or delayed job' + expect(subject.label).to eq 'waiting for manual action' end end diff --git a/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb b/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb index d83f260927f..694d4ce160a 100644 --- a/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/pipeline/factory_spec.rb @@ -23,11 +23,16 @@ expect(factory.core_status).to be_a expected_status end - if HasStatus::BLOCKED_STATUS.include?(simple_status) + if simple_status == 'manual' it 'matches a correct extended statuses' do expect(factory.extended_statuses) .to eq [Gitlab::Ci::Status::Pipeline::Blocked] end + elsif simple_status == 'scheduled' + it 'matches a correct extended statuses' do + expect(factory.extended_statuses) + .to eq [Gitlab::Ci::Status::Pipeline::Scheduled] + end else it 'does not match extended statuses' do expect(factory.extended_statuses).to be_empty diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index ec2bdbe22e1..fe167033941 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -117,6 +117,7 @@ pipelines: - retryable_builds - cancelable_statuses - manual_actions +- scheduled_actions - artifacts - pipeline_schedule - merge_requests diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb index 2f8ba4859d7..f38bc11f7f0 100644 --- a/spec/workers/stuck_ci_jobs_worker_spec.rb +++ b/spec/workers/stuck_ci_jobs_worker_spec.rb @@ -143,7 +143,7 @@ expect(Ci::Build.scheduled.count).to eq(0) expect(job).to be_failed - expect(job).to be_schedule_expired + expect(job).to be_stale_schedule end end -- GitLab From 49af84c2b0df199f1d8cda16a126ce6701e07743 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 18:35:34 +0900 Subject: [PATCH 084/118] enqueue in process_build_service --- app/services/ci/process_build_service.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 0fbe93130d0..d9f8e7cb452 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -9,7 +9,7 @@ def execute(build, current_status) elsif build.action? build.actionize else - build.enqueue + enqueue(build) end true @@ -21,6 +21,10 @@ def execute(build, current_status) private + def enqueue(build) + build.enqueue + end + def valid_statuses_for_when(value) case value when 'on_success' -- GitLab From 7fe14c42a913c683bd6f7c4f8791fff63db3c499 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 18:47:41 +0900 Subject: [PATCH 085/118] Add integration spec --- .../projects/jobs_controller_spec.rb | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/spec/controllers/projects/jobs_controller_spec.rb b/spec/controllers/projects/jobs_controller_spec.rb index fd11cb31a2a..f1eefdbaaaf 100644 --- a/spec/controllers/projects/jobs_controller_spec.rb +++ b/spec/controllers/projects/jobs_controller_spec.rb @@ -632,6 +632,46 @@ def post_cancel end end + describe 'POST unschedule' do + before do + project.add_developer(user) + + create(:protected_branch, :developers_can_merge, + name: 'master', project: project) + + sign_in(user) + + post_unschedule + end + + context 'when job is scheduled' do + let(:job) { create(:ci_build, :scheduled, pipeline: pipeline) } + + it 'redirects to the unscheduled job page' do + expect(response).to have_gitlab_http_status(:found) + expect(response).to redirect_to(namespace_project_job_path(id: job.id)) + end + + it 'transits to manual' do + expect(job.reload).to be_manual + end + end + + context 'when job is not scheduled' do + let(:job) { create(:ci_build, pipeline: pipeline) } + + it 'renders unprocessable_entity' do + expect(response).to have_gitlab_http_status(:unprocessable_entity) + end + end + + def post_unschedule + post :unschedule, namespace_id: project.namespace, + project_id: project, + id: job.id + end + end + describe 'POST cancel_all' do before do project.add_developer(user) -- GitLab From 9613f7e2fb2d5b4190adff29107b1bca6993fd41 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 2 Oct 2018 19:45:39 +0900 Subject: [PATCH 086/118] Stub feature flag for spec --- spec/models/ci/build_spec.rb | 4 ++++ spec/services/ci/run_scheduled_build_service_spec.rb | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 83add15fab7..ffad82c7820 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -327,6 +327,10 @@ describe '#enqueue_scheduled' do subject { build.enqueue_scheduled } + before do + stub_feature_flags(ci_enable_scheduled_build: true) + end + context 'when build is scheduled and the right time has not come yet' do let(:build) { create(:ci_build, :scheduled, pipeline: pipeline) } diff --git a/spec/services/ci/run_scheduled_build_service_spec.rb b/spec/services/ci/run_scheduled_build_service_spec.rb index be2aad33ef4..2c921dac238 100644 --- a/spec/services/ci/run_scheduled_build_service_spec.rb +++ b/spec/services/ci/run_scheduled_build_service_spec.rb @@ -7,6 +7,10 @@ subject { described_class.new(project, user).execute(build) } + before do + stub_feature_flags(ci_enable_scheduled_build: true) + end + context 'when user can update build' do before do project.add_developer(user) -- GitLab From c9e8b59c8540f7be198463db4477130046a8d655 Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Tue, 2 Oct 2018 12:58:18 +0200 Subject: [PATCH 087/118] Add missing translations from scheduled job UI --- locale/gitlab.pot | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index cc11577b624..29ab9f7d6eb 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -1221,9 +1221,15 @@ msgstr "" msgid "CiStatusLabel|pending" msgstr "" +msgid "CiStatusLabel|scheduled" +msgstr "" + msgid "CiStatusLabel|skipped" msgstr "" +msgid "CiStatusLabel|waiting for delayed job" +msgstr "" + msgid "CiStatusLabel|waiting for manual action" msgstr "" @@ -1248,6 +1254,9 @@ msgstr "" msgid "CiStatusText|pending" msgstr "" +msgid "CiStatusText|scheduled" +msgstr "" + msgid "CiStatusText|skipped" msgstr "" @@ -2134,6 +2143,21 @@ msgstr "" msgid "Define a custom pattern with cron syntax" msgstr "" +msgid "DelayedJobs|Are you sure you want to run %{jobName} immediately? This job will run automatically after it's timer finishes." +msgstr "" + +msgid "DelayedJobs|Are you sure you want to run %{job_name} immediately? This job will run automatically after it's timer finishes." +msgstr "" + +msgid "DelayedJobs|Start now" +msgstr "" + +msgid "DelayedJobs|Unschedule" +msgstr "" + +msgid "DelayedJobs|scheduled" +msgstr "" + msgid "Delete" msgstr "" @@ -6051,6 +6075,9 @@ msgstr "" msgid "This is a confidential issue." msgstr "" +msgid "This is a scheduled to run in " +msgstr "" + msgid "This is the author's first Merge Request to this project." msgstr "" @@ -6111,6 +6138,9 @@ msgstr "" msgid "This job requires a manual action" msgstr "" +msgid "This job will automatically run after it's timer finishes. Often they are used for incremental roll-out deploys to production environments. When unscheduled it converts into a manual action." +msgstr "" + msgid "This means you can not push code until you create an empty repository or import existing one." msgstr "" @@ -6463,6 +6493,9 @@ msgstr "" msgid "Unresolve discussion" msgstr "" +msgid "Unschedule job" +msgstr "" + msgid "Unstage" msgstr "" -- GitLab From 64ad6458486dc53da0a646419e4702319df6dace Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Tue, 2 Oct 2018 14:34:02 +0200 Subject: [PATCH 088/118] Add confirmation for immediately starting scheduled jobs --- .../pipelines/components/pipelines_actions.vue | 14 +++++++++++--- app/views/projects/ci/builds/_build.html.haml | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/pipelines/components/pipelines_actions.vue b/app/assets/javascripts/pipelines/components/pipelines_actions.vue index fa5ad62f438..1768c7dc2b3 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_actions.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_actions.vue @@ -1,4 +1,5 @@