From c9c4c081fc01b9bc555fb8c0c789c0010f4602b8 Mon Sep 17 00:00:00 2001 From: Siddharth VP Date: Thu, 13 Jun 2024 01:49:43 +0530 Subject: [PATCH 1/4] Simplify ToolsDb detection in replica.py --- quarry/web/replica.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/quarry/web/replica.py b/quarry/web/replica.py index d2db9ef..1c80da0 100644 --- a/quarry/web/replica.py +++ b/quarry/web/replica.py @@ -10,9 +10,9 @@ class Replica: def __init__(self, config): self.config = config self.dbname = "" - self.is_tools_db = False def _db_name_mangler(self): + self.is_tools_db = False if self.dbname == "": raise ReplicaConnectionException( "Attempting connection before a database is selected" @@ -21,15 +21,12 @@ def _db_name_mangler(self): self.is_tools_db = True self.database_p = self.dbname elif self.dbname == "meta" or self.dbname == "meta_p": - self.is_tools_db = False self.database_name = "s7" self.database_p = "meta_p" elif self.dbname == "centralauth" or self.dbname == "centralauth_p": - self.is_tools_db = False self.database_name = "s7" self.database_p = "centralauth_p" else: - self.is_tools_db = False self.database_name = ( self.dbname if not self.dbname.endswith("_p") From 11ee6b39880fce831671d1e746b47037df3d9194 Mon Sep 17 00:00:00 2001 From: Siddharth VP Date: Thu, 13 Jun 2024 01:54:17 +0530 Subject: [PATCH 2/4] Remove unused REPLICA_HOST config propery --- helm-quarry/dev-config.yaml | 1 - helm-quarry/prod-config.yaml | Bin 1514 -> 1460 bytes 2 files changed, 1 deletion(-) diff --git a/helm-quarry/dev-config.yaml b/helm-quarry/dev-config.yaml index 2b02faf..9e66efa 100644 --- a/helm-quarry/dev-config.yaml +++ b/helm-quarry/dev-config.yaml @@ -16,7 +16,6 @@ task_track_started: True worker_prefetch_multiplier: 1 # Tasks can run for a long time # Just query the quarry database itself. REPLICA_DOMAIN: '' -REPLICA_HOST: 'mywiki' REPLICA_DB: 'mywiki_p' REPLICA_USER: 'repl' REPLICA_PASSWORD: 'repl' diff --git a/helm-quarry/prod-config.yaml b/helm-quarry/prod-config.yaml index c33a0b353f8cdaf9d9ee1a46d273ad33a62d820f..41ab78c9797dde0f768e1c09e1ae69c31e7246c8 100644 GIT binary patch literal 1460 zcmV;l1xxw>M@dveQdv+`0A`FxRUz2L@ju?X*@u4h66srsQm8>}b<;NN=-mBrG$jPM z3~lo0#6RCeb}}NwWeeE$Gf2^m?xVh+2xBC+T<{*?A>ogG=Y&0O{ ziEoilHw&WT#OBAIwAr0O+p#uP`}UiVhA*4=V~;&*P_D-5wjZ-KvTlEVKTw7Fj)IC5 zxW>x5a`9EZb_f(iD`zq>d>O8!{5O4WKP(P&2Naa6sQ(YF;qmm#^_pL7-p@)TPUq^q zjyt;6a6~<}Y{g#nBQBw+qom}^9)?F#2=zc#Ni?SSWA1Bp`dUL>*ZYChsy_hX%r@>_ z6Tegg5N$&yB7#?rZ6QP$Rd))qmY%U_3(nM1@oJ+9o;2|Gh!*=)CB7|r?tOl7K+D(U zhb>18Pn5+P_PejsF0x&|j}yb#EN?Nlp7rhT^kb1RkSF`;b6o=EzFmeU^V};=1u?_* z5VkB-{M**d9%hnKCUR}%&AGHjC~K4~TM%~7Rv~ZhR!x=-L)%_|=FHNVnf19_6i=T$ zq8UdQ1j?C?G=Rj$Mc(<)BJ(e*1d4z5bo+ zq!i+^y)wph(orLiX}xjZ>*N3M#b;tGQ|%k4^}^Y5`ta&jhxmGTxJ4GymhY1O#+r1XS{LP$Hyyo;}n9dJfA!?6&gjOvVamdAs)E---7 zSB9bYyu4~NL5NuLxGCf0G zpZTnX2q2J^(VI`RC)>S&)z-49St8`(6uCuQWi->&TR_-D@r16ek) z^@_^VJfko<0QSPGgA+Vrl5BB^wcff@zLE@ZdP>~-($ zLv)MF1BrMwL_Alk`1IGisL~r?;h#JIaBHO$D#$@DOQ)TQ5rj9x6Ez`T9{cQtoQ}Z0 zhAn<>i02W|-bdvQGXb0wO4PIE5g#jGMDp5E<_Idz>JC4y`qYcE;eV4u50p8)Tqk?g z02D~~Rx1Sn4zXgaf?zYjY7b=^%sAokK8%yf>`|u6!=_C~IwLIc;`${Elc*lya(WUB zgnC!(HhHr~|KBmC>mD${Td`)fPn#+`m$~Et>>@?YNtzO9DWFKfarYlkAh zt@qI#=X8SHubOHuT1FU4nKZd3c<7?;TDT!Rxyp;-yp8_?T4XgPB(+mXN$d%eNWLBV z;tCX!jlfHW0r6G7wZl9x;ChWyLO~>UPiuGAEThrbPv@fw7GGtw;|ythgXX9MPSoCn z;W`#xIOCvRwI4YWn5XeIbuaVv;kj=x+5wnFbEOP?Lss1!5k!G)EMv#0Nk`u%WKf)u z4Rur|Hfo{x9Sr47>$4HQ6r^q5#eD*bK{xdp;`eU<8Jv_!zPRDpk87AB5NA;UVB@5tP z&*+t?KyYIn3EID;J3CrtOZ6d!yt5QyaJr-pGjtMhP%^7hVy6Y|g&oO8+jaa;voi*k OeGY_&AaNt;)Obt=3fIyA literal 1514 zcmVTd67xv@=G@B`l#ZKTpc~s9fL^TeXuJ{&%@yJhe~5(Ol_OUg?n0t<@zMAA zCyP6)UoUsHiG?#I10%Bpw99F+szLk|S0bfSTixSC9E)5|hTesqZ|@ zoE?QI{a|Q&;tmO%btGMNWgG2>ICT52@*mB+9!=@A$XVs%-Qa#7`~zLWHnM7>n&!;= z9C!n4<_^c$yQ%z_p8(c=g2eeQYgc{TU@JiH*(elBeaH*OH~Ox3{|pjw&pSTN1XM30 zI6Jfk=?Gp)>_Ml`1kmgEdPw0`F^6T%klUciM64^gHj~|;(_QjM4E@y;J7>sfj2s*e z)=DzCYuUNNf);!C`hKJ@*2ObymnsB^5BM<9oEud|J+^ho{2idl&1+&280l?PlLy>c zJuoJElEvkTYEvg|6n_y#d6YDl$MCwiB8^_vv>6^d4}cSJvaq#FI}p z)o`0KkOGS(+RG)SAS7WY8rEM=VFY) z-rJF0CbfSR%0_+3f^Sm-#WE_^vc0o)UQb;oI$DTyk*}ogoZbPHu<9$lv}r zAy0nQ1l(<7AGXsCeJ`b))Kv5?Sw!k9v%JXT#@F$#w{(IH}@Y{xR`gA80LQsq&DH*jgOHyt4EGfGE zS+29q;IZ)M%jh9LdD|8s1Z7q=4SPc zxA8FQqX<$VgSnGwg}5?P2wax5`)pV-C-nRuYR&hOd_z0Xp!TZ_3=RKiCc!1Ze)tw% zhlvf{mf1tBX&Rn=hYdF`FbD;8vtL%4)=kM1Pmha_r^JqK+ z=bT&A<;KwandQ)ga8li112nfnC-xPS&E7WHydbfw4}&agHK$BC6~wJ#(HH>n<=S^h zx@3`sPcUC6W;)2r9_3XYD!7}=Jjo8K$;ySbk0^^Ni^aS2%%%1g&f~XK-;}Jif!izh zwQ%Qmx@8imH_ITOjdrwUiS)RF=@*sG`HGSx5ld Date: Thu, 13 Jun 2024 02:14:47 +0530 Subject: [PATCH 3/4] Update query killer to work with multi-instance replicas Bug: T367363 --- helm-quarry/prod-config.yaml | Bin 1460 -> 1502 bytes quarry/default_config.yaml | 1 + quarry/web/killer.py | 61 +++++++++++++++++++---------------- quarry/web/replica.py | 4 +++ 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/helm-quarry/prod-config.yaml b/helm-quarry/prod-config.yaml index 41ab78c9797dde0f768e1c09e1ae69c31e7246c8..ca2b47a027b3fa12bf9cafc116a87c4bdbcefce7 100644 GIT binary patch literal 1502 zcmV<41tIzXM@dveQdv+`0NUnN?bTCw}gwwFuSVKsH65gR&>l6#yGKF-JetIx85 z`(v&uIKnecN-wu)<5nr;*>h|h1tFhTFTj>D{X15<2H*pazS;A&c#Xs|TDxeN5A5v8Z}fYS!Oog^V#{P%(k4#*0bnMs$J&mTpml+x28h3$7pW^G+Kr>tx z8oGkpOc`nDl0}2R3q}B|5N^j96<_AN#Y@SovG3S9p2hPdbpPnK*?Cqsb3&PJ^I2g( zAf)Q)Y1Hkj^S{^}Qd|s}^aaT+8jT2K6BOts`!!@QZb=>!xykWoUqrCC6O3VneZkqq zGMwh20U>upqoneF^#c-$OE@;TK$98#K#!S3$C($mUsGb5p5fKY{Xpg_xn|sx}XeemR|_Nn-MD5iA@bGAsP1EjcX|E zo#Vll-oLB3ytuRasm+pa-2^Hu+M9Opwm;X%hc6@ zqG59IDo$j(^fEh$pI1Ix!-O?x&`v$v1MPnF6qjM|5N}Ea36SFUUiL}->lVaG#ed{{ zU*aS351{|=u*n^{#5-V-`1}7L{P}LE;&mO_k9A@tYeWhOCAKVM$(<;A4dGwXUPH6i zH}7?A{?SW`Rj+S!7WF>D$g}il$&fs8V!y3nwIuat$hf}uU{R5z1r?s;1_J9;^xE=^ z3ucx66TMv%8r^hUm#WP*BhghYSM zCeHd4d=s;|asmVp{(vMIIQ~RXL=@CdeM%h@BaqHNe?U=5b8WYUuq7K@S-|w&Zg?`6Ww$FvGo^Q34D(k7+ZoYx&E4J7>4@V(n{<%q9binVo~ zY}P(vHbRDfD35o?2TFk_umQREs)K&W1NX3cPmBvO7Z!ozdcUAU$IZ`V?yuT_;B&F2 zs*eVLUbpeA(bla&BF2)0v#I$Tj?3cg&o7$6N0rBY6QgC~0 zPNTZUdlk{AmBBh+BUzGU;JWEqj^sB5f2tcUrenSv0?Q+_KPK}Y!}n9do=*q4u))K_Q*W_8L@j!8+c;A?1l%I z(P;#bNz+G{eLz{IYI|xx>Vv2F7MbjO(-h(4=>0>7cYk)@spXYsvIpg6R7ZkSPWJ*( zmmty)kRG93`H?Le0SU6YX|!%go|1+4_$2LU8a}qzXq^$6D5&=C{J3S?nP4kn-n@Hp zhf@!h4eVy9i1TpLczgpdnv(uhy`ssNCsLDxino|3q&i$4x`WegI|C6;ylqyhp|W5T(H3)l%H1@v9GYFJj%D6#hp0_TZdC&29w1gt3df)f!3bJ^qtk+BqGbdssNxh zTiR5il)*L*!^MQRHco)kglhnqtbFVf%HGzbz17g_M@dveQdv+`0A`FxRUz2L@ju?X*@u4h66srsQm8>}b<;NN=-mBrG$jPM z3~lo0#6RCeb}}NwWeeE$Gf2^m?xVh+2xBC+T<{*?A>ogG=Y&0O{ ziEoilHw&WT#OBAIwAr0O+p#uP`}UiVhA*4=V~;&*P_D-5wjZ-KvTlEVKTw7Fj)IC5 zxW>x5a`9EZb_f(iD`zq>d>O8!{5O4WKP(P&2Naa6sQ(YF;qmm#^_pL7-p@)TPUq^q zjyt;6a6~<}Y{g#nBQBw+qom}^9)?F#2=zc#Ni?SSWA1Bp`dUL>*ZYChsy_hX%r@>_ z6Tegg5N$&yB7#?rZ6QP$Rd))qmY%U_3(nM1@oJ+9o;2|Gh!*=)CB7|r?tOl7K+D(U zhb>18Pn5+P_PejsF0x&|j}yb#EN?Nlp7rhT^kb1RkSF`;b6o=EzFmeU^V};=1u?_* z5VkB-{M**d9%hnKCUR}%&AGHjC~K4~TM%~7Rv~ZhR!x=-L)%_|=FHNVnf19_6i=T$ zq8UdQ1j?C?G=Rj$Mc(<)BJ(e*1d4z5bo+ zq!i+^y)wph(orLiX}xjZ>*N3M#b;tGQ|%k4^}^Y5`ta&jhxmGTxJ4GymhY1O#+r1XS{LP$Hyyo;}n9dJfA!?6&gjOvVamdAs)E---7 zSB9bYyu4~NL5NuLxGCf0G zpZTnX2q2J^(VI`RC)>S&)z-49St8`(6uCuQWi->&TR_-D@r16ek) z^@_^VJfko<0QSPGgA+Vrl5BB^wcff@zLE@ZdP>~-($ zLv)MF1BrMwL_Alk`1IGisL~r?;h#JIaBHO$D#$@DOQ)TQ5rj9x6Ez`T9{cQtoQ}Z0 zhAn<>i02W|-bdvQGXb0wO4PIE5g#jGMDp5E<_Idz>JC4y`qYcE;eV4u50p8)Tqk?g z02D~~Rx1Sn4zXgaf?zYjY7b=^%sAokK8%yf>`|u6!=_C~IwLIc;`${Elc*lya(WUB zgnC!(HhHr~|KBmC>mD${Td`)fPn#+`m$~Et>>@?YNtzO9DWFKfarYlkAh zt@qI#=X8SHubOHuT1FU4nKZd3c<7?;TDT!Rxyp;-yp8_?T4XgPB(+mXN$d%eNWLBV z;tCX!jlfHW0r6G7wZl9x;ChWyLO~>UPiuGAEThrbPv@fw7GGtw;|ythgXX9MPSoCn z;W`#xIOCvRwI4YWn5XeIbuaVv;kj=x+5wnFbEOP?Lss1!5k!G)EMv#0Nk`u%WKf)u z4Rur|Hfo{x9Sr47>$4HQ6r^q5#eD*bK{xdp;`eU<8Jv_!zPRDpk87AB5NA;UVB@5tP z&*+t?KyYIn3EID;J3CrtOZ6d!yt5QyaJr-pGjtMhP%^7hVy6Y|g&oO8+jaa;voi*k OeGY_&AaNt;)Obt=3fIyA diff --git a/quarry/default_config.yaml b/quarry/default_config.yaml index 88f3412..22ae8fa 100644 --- a/quarry/default_config.yaml +++ b/quarry/default_config.yaml @@ -31,6 +31,7 @@ REPLICA_DOMAIN: '' # Change to `analytics.db.svc.wikimedia.cloud` for live repl REPLICA_USER: 'repl' # For live replicas, your replica.my.cnf username REPLICA_PASSWORD: 'repl' # For live replicas, your replica.my.cnf password REPLICA_PORT: 3306 +REPLICA_SLICES: '' TOOLS_DB_HOST: 'tools-readonly.db.svc.wikimedia.cloud' TOOLS_DB_PORT: 3306 diff --git a/quarry/web/killer.py b/quarry/web/killer.py index 248879a..3438a23 100755 --- a/quarry/web/killer.py +++ b/quarry/web/killer.py @@ -3,7 +3,7 @@ import yaml import logging import pymysql -from connections import Connections +from quarry.web.replica import Replica __dir__ = os.path.dirname(__file__) config = yaml.safe_load(open(os.path.join(__dir__, "../default_config.yaml"))) @@ -21,31 +21,36 @@ logging.info( "Started killer process, with limit %s", config["QUERY_TIME_LIMIT"] ) -conn = Connections(config) -cur = conn.replica.cursor() -try: - cur.execute("SHOW PROCESSLIST") - queries = cur.fetchall() - logging.info("Found %s queries running", len(queries)) - to_kill = [ - q - for q in queries - if q[5] > config["QUERY_TIME_LIMIT"] and q[4] != "Sleep" - ] - logging.info("Found %s queries to kill", len(to_kill)) - for q in to_kill: - try: - cur.execute("KILL QUERY %s", q[0]) - logging.info("Killed query with thread_id:%s" % q[0]) - except pymysql.InternalError as e: - if e.args[0] == 1094: # Error code for 'no such thread' - logging.info( - "Query with thread_id:%s dead before it could be killed" - ) - else: - raise -finally: - logging.info("Finished killer process") - cur.close() - conn.close_all() +hosts = [db_slice + '.' + config.get['REPLICA_DOMAIN'] for db_slice in config.get['REPLICA_SLICES'].split(',')] +hosts.append(config.get['TOOLS_DB_HOST']) + +for host in hosts: + conn = Replica(config) + + cur = conn.connection.cursor() + try: + cur.execute("SHOW PROCESSLIST") + queries = cur.fetchall() + logging.info("Found %s queries running", len(queries)) + to_kill = [ + q + for q in queries + if q[5] > config["QUERY_TIME_LIMIT"] and q[4] != "Sleep" + ] + logging.info("Found %s queries to kill", len(to_kill)) + for q in to_kill: + try: + cur.execute("KILL QUERY %s", q[0]) + logging.info("Killed query with thread_id:%s" % q[0]) + except pymysql.InternalError as e: + if e.args[0] == 1094: # Error code for 'no such thread' + logging.info( + "Query with thread_id:%s dead before it could be killed" + ) + else: + raise + finally: + logging.info("Finished killer process") + cur.close() + del conn.connection diff --git a/quarry/web/replica.py b/quarry/web/replica.py index 1c80da0..9c1316b 100644 --- a/quarry/web/replica.py +++ b/quarry/web/replica.py @@ -26,6 +26,10 @@ def _db_name_mangler(self): elif self.dbname == "centralauth" or self.dbname == "centralauth_p": self.database_name = "s7" self.database_p = "centralauth_p" + elif len(self.dbname) == 2: + # slice, eg. s1, s2 + self.database_name = self.dbname + self.database_p = self.dbname else: self.database_name = ( self.dbname From 51ceb5537e14e197a38f7fdee2f549066bbfe6a2 Mon Sep 17 00:00:00 2001 From: Github Action Date: Wed, 12 Jun 2024 20:48:07 +0000 Subject: [PATCH 4/4] auto update of tag --- helm-quarry/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-quarry/values.yaml b/helm-quarry/values.yaml index 69d7796..15cbe90 100644 --- a/helm-quarry/values.yaml +++ b/helm-quarry/values.yaml @@ -1,7 +1,7 @@ web: repository: 'quay.io/wikimedia-quarry/quarry' - tag: pr-47 # web tag managed by github actions + tag: pr-48 # web tag managed by github actions worker: repository: 'quay.io/wikimedia-quarry/quarry' - tag: pr-47 # worker tag managed by github actions + tag: pr-48 # worker tag managed by github actions