From 8a82aced5a291dc94dbd2f3f5e659830c6a15c01 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Wed, 11 Oct 2023 11:44:59 -0500 Subject: [PATCH 1/8] Generate core dumps for tests that segfault Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/docker/lib/docker_run.bash | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jenkins-scripts/docker/lib/docker_run.bash b/jenkins-scripts/docker/lib/docker_run.bash index 5c96eaef5..1751ce483 100644 --- a/jenkins-scripts/docker/lib/docker_run.bash +++ b/jenkins-scripts/docker/lib/docker_run.bash @@ -12,6 +12,11 @@ mkdir -p ${PACKAGE_DIR} sudo rm -fr ${WORKSPACE}/build mkdir -p ${WORKSPACE}/build +# Remove old core dumps to conserve space +CORE_DUMPS_DIR="${WORKSPACE}/core_dumps" +sudo rm -rf ${CORE_DUMPS_DIR} +mkdir -p ${CORE_DUMPS_DIR} + [[ -z ${DOCKER_DO_NOT_CACHE} ]] && DOCKER_DO_NOT_CACHE=false [[ -z ${USE_DOCKER_IN_DOCKER} ]] && export USE_DOCKER_IN_DOCKER=false @@ -81,6 +86,8 @@ if [[ -d /dev/snd ]]; then DEVICE_SND="--device /dev/snd" fi +echo "${CORE_DUMPS_DIR}/core.%e.%p" | sudo tee /proc/sys/kernel/core_pattern + # DOCKER_FIX is for workaround https://github.com/docker/docker/issues/14203 sudo ${docker_cmd} run ${PLAFTORM_PARAM} $EXTRA_PARAMS_STR \ -e DOCKER_FIX='' \ @@ -93,6 +100,8 @@ sudo ${docker_cmd} run ${PLAFTORM_PARAM} $EXTRA_PARAMS_STR \ ${DEVICE_SND} \ --tty \ --rm \ + --init \ + --ulimit core=-1 \ ${DOCKER_TAG} \ /bin/bash build.sh From 8b52776cc9e6c06e2b7c33a817d650c81f56e581 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 13 Oct 2023 15:26:00 -0500 Subject: [PATCH 2/8] Generate back traces Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 9 ++++++++- jenkins-scripts/lib/dependencies_archive.sh | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 9d63185d9..9272afc67 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -79,7 +79,7 @@ cd $WORKSPACE [[ ! -d $WORKSPACE/build ]] && mkdir -p $WORKSPACE/build cd $WORKSPACE/build cmake $WORKSPACE/${SOFTWARE_DIR} ${BUILDING_EXTRA_CMAKE_PARAMS} \ - -DCMAKE_INSTALL_PREFIX=/usr + -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr echo '# END SECTION' echo '# BEGIN SECTION: compiling' @@ -97,6 +97,13 @@ if $GENERIC_ENABLE_TESTS; then init_stopwatch TEST mkdir -p \$HOME make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS}" || true + if [[ -d $WORKSPACE/core_dumps ]]; then + cd $WORKSPACE/core_dumps + for corefile in core.* + do + gdb --batch -ex "thread apply all bt" --core $corefile > ${corefile}_backtrace.txt + done + fi stop_stopwatch TEST echo '# END SECTION' else diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index d52c69bba..b7dd95a04 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -36,6 +36,7 @@ fi # gnupg apt-key requires gnupg, gnupg2 or gnupg1 BASE_DEPENDENCIES="build-essential \\ cmake \\ + gdb \\ debhelper \\ mesa-utils \\ x11-utils \\ From f7b730dd45aeb7a5939c09ce04050a44ea6ea2a8 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 13 Oct 2023 16:47:36 -0500 Subject: [PATCH 3/8] Fix command Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 9272afc67..efe5a80e3 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -101,7 +101,7 @@ if $GENERIC_ENABLE_TESTS; then cd $WORKSPACE/core_dumps for corefile in core.* do - gdb --batch -ex "thread apply all bt" --core $corefile > ${corefile}_backtrace.txt + gdb --batch -ex "thread apply all bt" --core \$corefile > \${corefile}_backtrace.txt done fi stop_stopwatch TEST From 6bfdfa93b1263afd34186ab29cc060eadaf219d2 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 13 Oct 2023 22:14:45 -0500 Subject: [PATCH 4/8] Call sharedlibrary Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index efe5a80e3..558e9b69b 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -101,7 +101,7 @@ if $GENERIC_ENABLE_TESTS; then cd $WORKSPACE/core_dumps for corefile in core.* do - gdb --batch -ex "thread apply all bt" --core \$corefile > \${corefile}_backtrace.txt + gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile > \${corefile}_backtrace.txt done fi stop_stopwatch TEST From 5d3496cb2a72e7bb3ff37ce4fdcebf224f59b57b Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 13 Oct 2023 23:10:41 -0500 Subject: [PATCH 5/8] Print backtraces to log Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 558e9b69b..1508e78b1 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -101,7 +101,7 @@ if $GENERIC_ENABLE_TESTS; then cd $WORKSPACE/core_dumps for corefile in core.* do - gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile > \${corefile}_backtrace.txt + gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee \${corefile}_backtrace.txt done fi stop_stopwatch TEST From 73ea7324c64bc8065352ddd849e4349778beebb6 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 13 Oct 2023 23:20:05 -0500 Subject: [PATCH 6/8] Run gdb from test dir Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 1508e78b1..65595f074 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -98,10 +98,9 @@ if $GENERIC_ENABLE_TESTS; then mkdir -p \$HOME make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS}" || true if [[ -d $WORKSPACE/core_dumps ]]; then - cd $WORKSPACE/core_dumps - for corefile in core.* + for corefile in core_dumps/core.* do - gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee \${corefile}_backtrace.txt + gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee core_dumps/\${corefile}_backtrace.txt done fi stop_stopwatch TEST From 1e24c98b776e9e055d637044fb948d803c86f911 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 13 Oct 2023 23:39:58 -0500 Subject: [PATCH 7/8] Fix path Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 65595f074..506694e5b 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -98,9 +98,9 @@ if $GENERIC_ENABLE_TESTS; then mkdir -p \$HOME make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS}" || true if [[ -d $WORKSPACE/core_dumps ]]; then - for corefile in core_dumps/core.* + for corefile in $WORKSPACE/core_dumps/core.* do - gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee core_dumps/\${corefile}_backtrace.txt + gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee $WORKSPACE/core_dumps/\${corefile}_backtrace.txt done fi stop_stopwatch TEST From 607a94900f882b78005a4af86a3f1cb1eff1581a Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Sat, 14 Oct 2023 00:29:13 -0500 Subject: [PATCH 8/8] Fix path Signed-off-by: Addisu Z. Taddese --- .../docker/lib/_generic_linux_compilation_build.sh.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 506694e5b..9a045a622 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -100,7 +100,7 @@ if $GENERIC_ENABLE_TESTS; then if [[ -d $WORKSPACE/core_dumps ]]; then for corefile in $WORKSPACE/core_dumps/core.* do - gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee $WORKSPACE/core_dumps/\${corefile}_backtrace.txt + gdb --batch -ex "sharedlibrary; thread apply all bt" --core \$corefile | tee \${corefile}_backtrace.txt done fi stop_stopwatch TEST