Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: mtr run that runs last N failed tests #178

Merged
merged 9 commits into from
Dec 28, 2023
+170 −142
Merged

WIP: mtr run that runs last N failed tests #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
582af3c
WIP: mtr run that runs last N failed tests
vuvova Sep 26, 2023
0edcba4
WIP: change to custom step to fetch the last N failed tests
vladbogo Sep 26, 2023
f8291b9
in last N failed tests, look only for failures in the main branch
vuvova Sep 27, 2023
b939ac1
Use runQueryWithRetry
vladbogo Sep 28, 2023
0e5834a
append all changed tests to the run set
vuvova Oct 22, 2023
06ffa7b
use ./mtr --skip-not-found
vuvova Dec 27, 2023
e1accc6
typo
vuvova Dec 28, 2023
ee0255c
don't do FetchTestData step for builds that don't use it
vuvova Dec 27, 2023
a429eee
Enable running last N failed tests
vladbogo Dec 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
307 changes: 167 additions & 140 deletions common_factories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from buildbot.plugins import *
from buildbot.process import results
from buildbot.process.properties import Property, Properties
from buildbot.steps.package.rpm.rpmlint import RpmLint
from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand
Expand All @@ -10,8 +11,130 @@
from utils import *
from constants import *

class FetchTestData(MTR):
def __init__(self, mtrDbPool, **kwargs):
self.mtrDbPool = mtrDbPool
super().__init__(dbpool=mtrDbPool, **kwargs)

def getQuickBuildFactory(mtrDbPool):
@defer.inlineCallbacks
def run(self):
master_branch = self.getProperty('master_branch')
typ = 'nm'
limit = 50
overlimit = 1000
test_re = r'^(?:.+/)?mysql-test/(?:suite/)?(.+?)/(?:[rt]/)?([^/]+)\.(?:test|result|rdiff)$'

if master_branch:
query = """
select concat(test_name,',',test_variant) from (select id, test_name,test_variant from test_failure,test_run where branch='%s' and test_run_id=id order by test_run_id desc limit %d) x group by test_name,test_variant order by max(id) desc limit %d
Copy link
Member

@cvicentiu cvicentiu Sep 27, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For better readability, I suggest using Python's format:

query = f"""
            select concat(test_name, ',', test_variant)
            from
              (select id, test_name, test_variant
               from test_failure, test_run
               where
                 branch = '{master_branch}' and test_run_id = id
               order by test_run_id desc
               limit {overlimit}) x
            group by test_name, test_variant
            order by max(id) desc
            limit {limit}
"""
tests = yield self.mtrDbPool.runQuery(query)

Copy link
Member

@cvicentiu cvicentiu Sep 27, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, consider the following changes to make it easier for new contributors to understand the code:

  1. using JOIN instead of the equality in the WHERE clause for readability (I don't know by looking at that query which table has column test_run_id and which has id, I can guess, but being explicit is only 6 more characters 😄 )
  2. table aliases for test_failure (such as tf) and test_run (such as tr), and specify columns in the select list with:
    tf.id, tf.test_name, tf.test_variant.

"""
tests = yield self.runQueryWithRetry(query % (master_branch, overlimit, limit))
tests = list(t[0] for t in tests)

tests += (m.expand(r'\1.\2') for m in (re.search(test_re, f) for f in self.build.allFiles()) if m)

if tests:
test_args = ' '.join(tests)
self.setProperty('tests_to_run', '--skip-not-found ' + test_args)

return results.SUCCESS

def addPostTests(f_quick_build):
f_quick_build.addStep(
steps.DirectoryUpload(
name="save log files",
compress="bz2",
alwaysRun=True,
workersrc="/buildbot/logs/",
masterdest=util.Interpolate(
"/srv/buildbot/packages/"
+ "%(prop:tarbuildnum)s"
+ "/logs/"
+ "%(prop:buildername)s"
),
)
)

## trigger packages
f_quick_build.addStep(
steps.Trigger(
schedulerNames=["s_packages"],
waitForFinish=False,
updateSourceStamp=False,
alwaysRun=True,
set_properties={
"parentbuildername": Property("buildername"),
"tarbuildnum": Property("tarbuildnum"),
"mariadb_version": Property("mariadb_version"),
"master_branch": Property("master_branch"),
},
doStepIf=hasAutobake,
)
)
## trigger bigtest
f_quick_build.addStep(
steps.Trigger(
schedulerNames=["s_bigtest"],
waitForFinish=False,
updateSourceStamp=False,
set_properties={
"parentbuildername": Property("buildername"),
"tarbuildnum": Property("tarbuildnum"),
"mariadb_version": Property("mariadb_version"),
"master_branch": Property("master_branch"),
},
doStepIf=hasBigtest,
)
)
# create package and upload to master
f_quick_build.addStep(
steps.SetPropertyFromCommand(
command="basename mariadb-*-linux-*.tar.gz",
property="mariadb_binary",
doStepIf=savePackage,
)
)
f_quick_build.addStep(
steps.ShellCommand(
name="save_packages",
timeout=7200,
haltOnFailure=True,
command=util.Interpolate(
"""
mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s \
&& sha256sum %(prop:mariadb_binary)s >> sha256sums.txt \
&& cp %(prop:mariadb_binary)s sha256sums.txt /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ \
&& sync /packages/%(prop:tarbuildnum)s
"""
),
doStepIf=savePackage,
)
)
f_quick_build.addStep(
steps.Trigger(
name="eco",
schedulerNames=["s_eco"],
waitForFinish=False,
updateSourceStamp=False,
set_properties={
"parentbuildername": Property("buildername"),
"tarbuildnum": Property("tarbuildnum"),
"mariadb_binary": Property("mariadb_binary"),
"mariadb_version": Property("mariadb_version"),
"master_branch": Property("master_branch"),
"parentbuildername": Property("buildername"),
},
doStepIf=lambda step: savePackage(step) and hasEco(step),
)
)
f_quick_build.addStep(
steps.ShellCommand(
name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
)
)
return f_quick_build

def getBuildFactoryPreTest():
f_quick_build = util.BuildFactory()
f_quick_build.addStep(
steps.ShellCommand(
Expand Down Expand Up @@ -69,6 +192,9 @@ def getQuickBuildFactory(mtrDbPool):
haltOnFailure="true",
)
)
return f_quick_build

def addTests(f_quick_build, mtrDbPool, mtrArgs):
f_quick_build.addStep(
steps.MTR(
logfiles={"mysqld*": "/buildbot/mysql_logs.html"},
Expand All @@ -79,53 +205,26 @@ def getQuickBuildFactory(mtrDbPool):
"""
cd mysql-test &&
exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=1 --max-save-datadir=10 --max-test-fail=20 --mem --parallel=$(expr %(kw:jobs)s \* 2) %(kw:mtr_additional_args)s
""",
mtr_additional_args=util.Property(
"mtr_additional_args", default=""
),
jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"),
),
],
timeout=950,
haltOnFailure="true",
parallel=mtrJobsMultiplier,
dbpool=mtrDbPool,
autoCreateTables=True,
env=MTR_ENV,
)
)
f_quick_build.addStep(
steps.ShellCommand(
name="move mariadb log files",
alwaysRun=True,
command=[
"bash",
"-c",
util.Interpolate(
moveMTRLogs(),
jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN"),
),
],
)
)
f_quick_build.addStep(
steps.ShellCommand(
name="create var archive",
alwaysRun=True,
command=["bash", "-c", util.Interpolate(createVar())],
doStepIf=hasFailed,
)
)
f_quick_build.addStep(
steps.MTR(
description="testing galera",
descriptionDone="test galera",
logfiles={"mysqld*": "/buildbot/mysql_logs.html"},
command=[
"sh",
"-c",
util.Interpolate(
"""
""", mtr_additional_args=mtrArgs,
jobs=util.Property('jobs', default='$(getconf _NPROCESSORS_ONLN)'),
)],
timeout=950,
haltOnFailure="true",
parallel=mtrJobsMultiplier,
dbpool=mtrDbPool,
autoCreateTables=True,
env=MTR_ENV,
))
f_quick_build.addStep(steps.ShellCommand(name="move mariadb log files", alwaysRun=True, command=['bash', '-c', util.Interpolate(moveMTRLogs(), jobs=util.Property('jobs', default='$(getconf _NPROCESSORS_ONLN'))]))
f_quick_build.addStep(steps.ShellCommand(name="create var archive", alwaysRun=True, command=['bash', '-c', util.Interpolate(createVar())], doStepIf=hasFailed))
return f_quick_build

def addGaleraTests(f_quick_build, mtrDbPool):
f_quick_build.addStep(steps.MTR(
description="testing galera",
descriptionDone="test galera",
logfiles={"mysqld*": "/buildbot/mysql_logs.html"},
command=["sh", "-c", util.Interpolate("""
cd mysql-test &&
if [ -f "$WSREP_PROVIDER" ]; then exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=1 --max-save-datadir=10 --max-test-fail=20 --mem --big-test --parallel=$(expr %(kw:jobs)s \* 2) %(kw:mtr_additional_args)s --suite=wsrep,galera,galera_3nodes,galera_3nodes_sr; fi
""",
Expand Down Expand Up @@ -161,100 +260,28 @@ def getQuickBuildFactory(mtrDbPool):
doStepIf=hasGalera,
)
)
f_quick_build.addStep(
steps.DirectoryUpload(
name="save log files",
compress="bz2",
alwaysRun=True,
workersrc="/buildbot/logs/",
masterdest=util.Interpolate(
"/srv/buildbot/packages/"
+ "%(prop:tarbuildnum)s"
+ "/logs/"
+ "%(prop:buildername)s"
),
)
)

## trigger packages
f_quick_build.addStep(
steps.Trigger(
schedulerNames=["s_packages"],
waitForFinish=False,
updateSourceStamp=False,
alwaysRun=True,
set_properties={
"parentbuildername": Property("buildername"),
"tarbuildnum": Property("tarbuildnum"),
"mariadb_version": Property("mariadb_version"),
"master_branch": Property("master_branch"),
},
doStepIf=hasAutobake,
)
)
## trigger bigtest
f_quick_build.addStep(
steps.Trigger(
schedulerNames=["s_bigtest"],
waitForFinish=False,
updateSourceStamp=False,
set_properties={
"parentbuildername": Property("buildername"),
"tarbuildnum": Property("tarbuildnum"),
"mariadb_version": Property("mariadb_version"),
"master_branch": Property("master_branch"),
},
doStepIf=hasBigtest,
)
)
# create package and upload to master
f_quick_build.addStep(
steps.SetPropertyFromCommand(
command="basename mariadb-*-linux-*.tar.gz",
property="mariadb_binary",
doStepIf=savePackage,
)
)
f_quick_build.addStep(
steps.ShellCommand(
name="save_packages",
timeout=7200,
haltOnFailure=True,
command=util.Interpolate(
"""
mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s \
&& sha256sum %(prop:mariadb_binary)s >> sha256sums.txt \
&& cp %(prop:mariadb_binary)s sha256sums.txt /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ \
&& sync /packages/%(prop:tarbuildnum)s
"""
),
doStepIf=savePackage,
)
)
f_quick_build.addStep(
steps.Trigger(
name="eco",
schedulerNames=["s_eco"],
waitForFinish=False,
updateSourceStamp=False,
set_properties={
"parentbuildername": Property("buildername"),
"tarbuildnum": Property("tarbuildnum"),
"mariadb_binary": Property("mariadb_binary"),
"mariadb_version": Property("mariadb_version"),
"master_branch": Property("master_branch"),
"parentbuildername": Property("buildername"),
},
doStepIf=lambda step: savePackage(step) and hasEco(step),
)
)
f_quick_build.addStep(
steps.ShellCommand(
name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
)
)
return f_quick_build

def getQuickBuildFactory(mtrDbPool):
f = getBuildFactoryPreTest()
addTests(f, mtrDbPool, util.Property( "mtr_additional_args", default=""))
addGaleraTests(f, mtrDbPool)
return addPostTests(f)

def getLastNFailedBuildsFactory(mtrDbPool):
@util.renderer
def getTests(props):
mtr_additional_args = props.getProperty('mtr_additional_args', '--suite=main')
tests_to_run = props.getProperty('tests_to_run', None)
if tests_to_run:
mtr_additional_args = mtr_additional_args.replace('--suite=main', tests_to_run)

return mtr_additional_args

f = getBuildFactoryPreTest()
f.addStep(FetchTestData(name="Get last N failed tests", mtrDbPool=mtrDbPool))
addTests(f, mtrDbPool, getTests)
return addPostTests(f)

def getRpmAutobakeFactory(mtrDbPool):
## f_rpm_autobake
Expand Down
5 changes: 3 additions & 2 deletions master-protected-branches/master.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ for w_name in ["hz-bbw"]:
####### FACTORY CODE

f_quick_build = getQuickBuildFactory(mtrDbPool)
f_last_n_failed = getLastNFailedBuildsFactory(mtrDbPool)

## f_tarball - create source tarball
f_tarball = util.BuildFactory()
Expand Down Expand Up @@ -522,7 +523,7 @@ c["builders"].append(
canStartBuild=canStartBuild,
locks=getLocks,
properties={"mtr_additional_args": protected_branches_mtr_additional_args},
factory=f_quick_build,
factory=f_last_n_failed,
)
)

Expand All @@ -536,7 +537,7 @@ c["builders"].append(
canStartBuild=canStartBuild,
locks=getLocks,
properties={"mtr_additional_args": protected_branches_mtr_additional_args},
factory=f_quick_build,
factory=f_last_n_failed,
)
)

Expand Down