diff --git a/.github/workflows/github-actions-build-deb-package.yml b/.github/workflows/github-actions-build-deb-package.yml deleted file mode 100644 index 741c832fe8b..00000000000 --- a/.github/workflows/github-actions-build-deb-package.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Build deb packages - -on: - push: - branches: - - master - -jobs: - builder: - name: 'Build .deb package' - strategy: - fail-fast: false - matrix: - os: ['debian11', 'ubuntu20.04', 'ubuntu22.04'] - runs-on: ubuntu-latest - container: - image: openroad/${{ matrix.os }}-dev - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set release version - id: set_version - run: | - git config --global --add safe.directory '*' - echo "RELEASE_VERSION=$(git describe | sed 's/v//')" >> $GITHUB_ENV - echo "RELEASE_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - - - name: Build package - run: | - ./debian/create-changelog.sh ${{ env.RELEASE_VERSION }} - debuild --preserve-env --preserve-envvar=PATH -B - mkdir -p artifacts - mv ../openroad_* artifacts - - - name: Rename artifact - id: artifact - run: | - cd artifacts - file=$(realpath openroad_*.deb) - artifact=$(echo $file | sed 's/\.\([^.]*\)$/-${{ matrix.os }}.\1/') - mv $file $artifact - echo "file=$(realpath $artifact)" >> $GITHUB_OUTPUT - name=$(basename $artifact) - name=${name%.deb} - echo "name=$name" >> $GITHUB_OUTPUT - - - name: Upload workflow artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact.outputs.name }} - path: ${{ steps.artifact.outputs.file }} - - - name: Get release info - uses: cardinalby/git-get-release-action@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.RELEASE_VERSION }} - release_name: ${{ env.RELEASE_VERSION }} - doNotFailIfNotFound: true - id: release_info - - - name: Create release if not exists - if: steps.release_info.outputs.name == '' - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.RELEASE_VERSION }} - release_name: ${{ env.RELEASE_VERSION }} - body: "Release ${{ env.RELEASE_VERSION }} on ${{ env.RELEASE_DATE }}" - id: create_release - - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.release_info.outputs.upload_url || steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.artifact.outputs.file }} - asset_name: ${{ steps.artifact.outputs.name }}-${{ env.RELEASE_VERSION }}.deb - asset_content_type: application/octet-stream - diff --git a/Jenkinsfile b/Jenkinsfile index 701d4d243ed..8f47f404aab 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ def baseTests(String image) { Map base_tests = [failFast: false]; base_tests['Unit Tests CTest'] = { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage('Setup CTest') { echo 'Nothing to be done.'; } @@ -27,7 +27,7 @@ def baseTests(String image) { base_tests['Unit Tests Tcl'] = { node { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage('Setup Tcl Tests') { sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; checkout scm; @@ -68,7 +68,7 @@ def baseTests(String image) { flow_tests.each { current_test -> base_tests["Flow Test - ${current_test}"] = { node { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage("Setup ${current_test}") { sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; checkout scm; @@ -104,7 +104,7 @@ def getParallelTests(String image) { 'Build without GUI': { node { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage('Setup no-GUI Build') { echo "Build without GUI"; sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; @@ -121,7 +121,7 @@ def getParallelTests(String image) { 'Build without Test': { node { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage('Setup no-test Build') { echo "Build without Tests"; sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; @@ -148,7 +148,7 @@ def getParallelTests(String image) { 'Unit Tests Ninja': { node { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage('Setup Ninja Tests') { sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; checkout scm; @@ -174,7 +174,7 @@ def getParallelTests(String image) { 'Compile with C++20': { node { - docker.image(image).inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { + withDockerContainer(args: '-u root', image: image) { stage('Setup C++20 Compile') { sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; checkout scm; @@ -188,6 +188,42 @@ def getParallelTests(String image) { ]; + if (env.BRANCH_NAME == 'master') { + deb_os = [ + [name: 'Ubuntu 20.04' , artifact_name: 'ubuntu-20.04', image: 'openroad/ubuntu20.04-dev'], + [name: 'Ubuntu 22.04' , artifact_name: 'ubuntu-22.04', image: 'openroad/ubuntu22.04-dev'], + [name: 'Debian 11' , artifact_name: 'debian11', image: 'openroad/debian11-dev'] + ]; + deb_os.each { os -> + ret["Build .deb - ${os.name}"] = { + node { + stage('Setup and Build') { + sh label: 'Pull latest image', script: "docker pull ${os.image}:latest"; + withDockerContainer(args: '-u root', image: "${os.image}") { + sh label: 'Configure git', script: "git config --system --add safe.directory '*'"; + checkout([ + $class: 'GitSCM', + branches: [[name: scm.branches[0].name]], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CloneOption', noTags: false], + [$class: 'SubmoduleOption', recursiveSubmodules: true] + ], + submoduleCfg: [], + userRemoteConfigs: scm.userRemoteConfigs + ]); + def version = sh(script: 'git describe | sed s,^v,,', returnStdout: true).trim(); + sh label: 'Create Changelog', script: "./debian/create-changelog.sh ${version}"; + sh label: 'Run debuild', script: 'debuild --preserve-env --preserve-envvar=PATH -B -j$(nproc)'; + sh label: 'Move generated files', script: "./debian/move-artifacts.sh ${version} ${os.artifact_name}"; + archiveArtifacts artifacts: '*' + "${version}" + '*'; + } + } + } + } + } + } + return ret; } @@ -196,7 +232,26 @@ node { checkout scm; } def DOCKER_IMAGE; - stage('Build and Push Docker Image') { + stage('Build, Test and Push Docker Image') { + Map build_docker_images = [failFast: false]; + test_os = [ + [name: 'Ubuntu 20.04', base: 'ubuntu:20.04', image: 'ubuntu20.04'], + [name: 'Ubuntu 22.04', base: 'ubuntu:22.04', image: 'ubuntu22.04'], + [name: 'Ubuntu 24.04', base: 'ubuntu:24.04', image: 'ubuntu24.04'], + [name: 'RockyLinux 9', base: 'rockylinux:9', image: 'rockylinux9'], + [name: 'Debian 11', base: 'debian:11', image: 'debian11'] + ]; + test_os.each { os -> + build_docker_images["Test Installer - ${os.name}"] = { + node { + checkout scm; + sh label: 'Build Docker image', script: "./etc/DockerHelper.sh create -target=builder -os=${os.image}"; + sh label: 'Test Docker image', script: "./etc/DockerHelper.sh test -target=builder -os=${os.image}"; + dockerPush("${os.image}", 'openroad'); + } + } + } + parallel(build_docker_images); DOCKER_IMAGE = dockerPush('ubuntu22.04', 'openroad'); echo "Docker image is ${DOCKER_IMAGE}"; } diff --git a/debian/create-changelog.sh b/debian/create-changelog.sh index 320f475f707..901e4541cd5 100755 --- a/debian/create-changelog.sh +++ b/debian/create-changelog.sh @@ -8,5 +8,7 @@ fi cat > debian/changelog < on %ad") + * Automated release + +$(git log --date=rfc -1 --pretty=format:" -- %an <%ae> %ad") EOF diff --git a/debian/move-artifacts.sh b/debian/move-artifacts.sh new file mode 100755 index 00000000000..d2329ab9b9a --- /dev/null +++ b/debian/move-artifacts.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +if [[ $# -ne 2 ]]; then + echo "Error: This script requires exactly one argument." + echo "usage: $0 " + exit 1 +fi + +for file in ../*${1}*; do + base_name=$(basename "$file") + name="${base_name%.*}" + ext="${base_name##*.}" + mv "$file" "${name}-${2}.${ext}" +done diff --git a/debian/rules b/debian/rules index 4512dc18b7b..b62ff4b8fc9 100755 --- a/debian/rules +++ b/debian/rules @@ -18,4 +18,5 @@ override_dh_shlibdeps: override_dh_install: dh_install --sourcedir=/opt +override_dh_auto_test: override_dh_dwz: diff --git a/etc/DependencyInstaller.sh b/etc/DependencyInstaller.sh index 7d0d3ac1f01..a9878c8bfe4 100755 --- a/etc/DependencyInstaller.sh +++ b/etc/DependencyInstaller.sh @@ -16,9 +16,6 @@ _equivalenceDeps() { # yosys yosysPrefix=${PREFIX:-"/usr/local"} if [[ ! $(command -v yosys) || ! $(command -v yosys-config) ]]; then ( - if [[ -f /opt/rh/llvm-toolset-7.0/enable ]]; then - source /opt/rh/llvm-toolset-7.0/enable - fi cd "${baseDir}" git clone --depth=1 -b "${yosysVersion}" --recursive https://github.com/YosysHQ/yosys cd yosys @@ -31,9 +28,6 @@ _equivalenceDeps() { # eqy eqyPrefix=${PREFIX:-"/usr/local"} if ! command -v eqy &> /dev/null; then ( - if [[ -f /opt/rh/llvm-toolset-7.0/enable ]]; then - source /opt/rh/llvm-toolset-7.0/enable - fi cd "${baseDir}" git clone --depth=1 -b "${yosysVersion}" https://github.com/YosysHQ/eqy cd eqy @@ -46,9 +40,6 @@ _equivalenceDeps() { # sby sbyPrefix=${PREFIX:-"/usr/local"} if ! command -v sby &> /dev/null; then ( - if [[ -f /opt/rh/llvm-toolset-7.0/enable ]]; then - source /opt/rh/llvm-toolset-7.0/enable - fi cd "${baseDir}" git clone --depth=1 -b "${yosysVersion}" --recursive https://github.com/YosysHQ/sby cd sby @@ -262,10 +253,10 @@ EOF _installOrTools() { os=$1 - version=$2 + osVersion=$2 arch=$3 - orToolsVersionBig=9.10 - orToolsVersionSmall=${orToolsVersionBig}.4067 + orToolsVersionBig=9.11 + orToolsVersionSmall=${orToolsVersionBig}.4210 rm -rf "${baseDir}" mkdir -p "${baseDir}" @@ -274,7 +265,7 @@ _installOrTools() { # Disable exit on error for 'find' command, as it might return non zero set +euo pipefail - LIST=($(find / -type f -name "libortools.so*" 2>/dev/null)) + LIST=($(find /local* /opt* /lib* /usr* /bin* -type f -name "libortools.so*" 2>/dev/null)) # Bring back exit on error set -euo pipefail # Return if right version of or-tools is installed @@ -295,10 +286,10 @@ _installOrTools() { ${cmakePrefix}/bin/cmake -S. -Bbuild -DBUILD_DEPS:BOOL=ON -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_SAMPLES:BOOL=OFF -DBUILD_TESTING:BOOL=OFF -DCMAKE_INSTALL_PREFIX=${orToolsPath} -DCMAKE_CXX_FLAGS="-w" -DCMAKE_C_FLAGS="-w" ${cmakePrefix}/bin/cmake --build build --config Release --target install -v -j $(nproc) else - if [[ $version == rodete ]]; then - version=11 + if [[ $osVersion == rodete ]]; then + osVersion=11 fi - orToolsFile=or-tools_${arch}_${os}-${version}_cpp_v${orToolsVersionSmall}.tar.gz + orToolsFile=or-tools_${arch}_${os}-${osVersion}_cpp_v${orToolsVersionSmall}.tar.gz eval wget https://github.com/google/or-tools/releases/download/v${orToolsVersionBig}/${orToolsFile} if command -v brew &> /dev/null; then orToolsPath="$(brew --prefix or-tools)" @@ -388,17 +379,12 @@ _installRHELCleanUp() { _installRHELPackages() { arch=amd64 - version=3.1.11.1 + pandocVersion=3.1.11.1 yum -y update - if [[ $(yum repolist | egrep -c "rhel-8-for-x86_64-appstream-rpms") -eq 0 ]]; then - yum -y install http://mirror.centos.org/centos/8-stream/BaseOS/x86_64/os/Packages/centos-gpg-keys-8-6.el8.noarch.rpm - yum -y install http://mirror.centos.org/centos/8-stream/BaseOS/x86_64/os/Packages/centos-stream-repos-8-6.el8.noarch.rpm - rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - fi yum -y install tzdata yum -y install redhat-rpm-config rpm-build - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm yum -y install \ autoconf \ automake \ @@ -409,11 +395,11 @@ _installRHELPackages() { gdb \ git \ glibc-devel \ - libtool \ libffi-devel \ - llvm7.0 \ - llvm7.0-devel \ - llvm7.0-libs \ + libtool \ + llvm \ + llvm-devel \ + llvm-libs \ make \ pcre-devel \ pcre2-devel \ @@ -424,10 +410,9 @@ _installRHELPackages() { python3-devel \ python3-pip \ qt5-qtbase-devel \ + qt5-qtcharts-devel \ qt5-qtimageformats \ readline \ - readline-devel \ - tcl-devel \ tcl-tclreadline \ tcl-tclreadline-devel \ tcl-thread-devel \ @@ -436,65 +421,14 @@ _installRHELPackages() { zlib-devel yum install -y \ - http://repo.okay.com.mx/centos/8/x86_64/release/bison-3.0.4-10.el8.x86_64.rpm \ - https://forensics.cert.org/centos/cert/7/x86_64/flex-2.6.1-9.el7.x86_64.rpm - - eval wget https://github.com/jgm/pandoc/releases/download/${version}/pandoc-${version}-linux-${arch}.tar.gz - tar xvzf pandoc-${version}-linux-${arch}.tar.gz --strip-components 1 -C /usr/local/ - rm -rf pandoc-${version}-linux-${arch}.tar.gz -} - -_installCentosCleanUp() { - yum clean -y all - rm -rf /var/lib/apt/lists/* -} - -_installCentosPackages() { - yum update -y - yum install -y tzdata - yum groupinstall -y "Development Tools" - if ! command -v lcov &> /dev/null; then - yum install -y http://downloads.sourceforge.net/ltp/lcov-1.14-1.noarch.rpm - fi - if ! command -v yum list installed ius-release &> /dev/null; then - yum install -y https://repo.ius.io/ius-release-el7.rpm - fi - if ! command -v yum list installed epel-release &> /dev/null; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - fi - yum install -y centos-release-scl - yum install -y \ - devtoolset-8 \ - devtoolset-8-libatomic-devel \ - groff \ - libffi-devel \ - libgomp \ - libstdc++ \ - llvm-toolset-7.0 \ - llvm-toolset-7.0-libomp-devel \ - pandoc \ - pcre-devel \ - pcre2-devel \ - python-devel \ - python36 \ - python36-devel \ - python36-libs \ - python36-pip \ - qt5-qtbase-devel \ - qt5-qtimageformats \ - readline-devel \ - rh-python38-python \ - rh-python38-python-libs \ - rh-python38-python-pip \ - rh-python38-scldevel \ - tcl \ - tcl-devel \ - tcl-tclreadline \ - tcl-tclreadline-devel \ - tcllib \ - wget \ - ccache \ - zlib-devel + https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/bison-3.7.4-5.el9.x86_64.rpm \ + https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/flex-2.6.4-9.el9.x86_64.rpm \ + https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/readline-devel-8.1-4.el9.x86_64.rpm \ + https://rpmfind.net/linux/centos-stream/9-stream/AppStream/x86_64/os/Packages/tcl-devel-8.6.10-7.el9.x86_64.rpm + + eval wget https://github.com/jgm/pandoc/releases/download/${pandocVersion}/pandoc-${pandocVersion}-linux-${arch}.tar.gz + tar xvzf pandoc-${pandocVersion}-linux-${arch}.tar.gz --strip-components 1 -C /usr/local/ + rm -rf pandoc-${pandocVersion}-linux-${arch}.tar.gz } _installOpenSuseCleanUp() { @@ -699,10 +633,12 @@ _installCI() { containerd.io \ docker-buildx-plugin - # Install clang for C++20 support - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - ./llvm.sh 16 all + if _versionCompare ${1} -lt 24.04; then + # Install clang for C++20 support + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + ./llvm.sh 16 all + fi } @@ -873,50 +809,39 @@ case "${platform}" in esac case "${os}" in - "CentOS Linux" ) - if [[ ${CI} == "yes" ]]; then - echo "WARNING: Installing CI dependencies is only supported on Ubuntu 22.04" >&2 - fi - if [[ "${option}" == "base" || "${option}" == "all" ]]; then - _checkIsLocal - _installCentosPackages - _installCentosCleanUp - fi - if [[ "${option}" == "common" || "${option}" == "all" ]]; then - _installCommonDev - _installOrTools "centos" "7" "amd64" - fi - cat <&2 fi @@ -927,7 +852,7 @@ EOF fi if [[ "${option}" == "common" || "${option}" == "all" ]]; then _installCommonDev - _installOrTools "centos" "8" "amd64" + _installOrTools "rockylinux" "9" "amd64" fi ;; "Darwin" ) @@ -962,21 +887,21 @@ To enable GCC-11 you need to run: EOF ;; "Debian GNU/Linux" | "Debian GNU/Linux rodete" ) - version=$(awk -F= '/^VERSION_ID/{print $2}' /etc/os-release | sed 's/"//g') - if [[ -z ${version} ]]; then - version=$(awk -F= '/^VERSION_CODENAME/{print $2}' /etc/os-release | sed 's/"//g') + debianVersion=$(awk -F= '/^VERSION_ID/{print $2}' /etc/os-release | sed 's/"//g') + if [[ -z ${debianVersion} ]]; then + debianVersion=$(awk -F= '/^VERSION_CODENAME/{print $2}' /etc/os-release | sed 's/"//g') fi if [[ ${CI} == "yes" ]]; then echo "WARNING: Installing CI dependencies is only supported on Ubuntu 22.04" >&2 fi if [[ "${option}" == "base" || "${option}" == "all" ]]; then _checkIsLocal - _installDebianPackages "${version}" + _installDebianPackages "${debianVersion}" _installDebianCleanUp fi if [[ "${option}" == "common" || "${option}" == "all" ]]; then _installCommonDev - _installOrTools "debian" "${version}" "amd64" + _installOrTools "debian" "${debianVersion}" "amd64" fi ;; *) diff --git a/etc/DockerHelper.sh b/etc/DockerHelper.sh index de32db17aff..e66938fe772 100755 --- a/etc/DockerHelper.sh +++ b/etc/DockerHelper.sh @@ -8,6 +8,7 @@ baseDir="$(pwd)" # docker hub organization/user from where to pull/push images org=openroad depsPrefixesFile="/etc/openroad_deps_prefixes.txt" +args=("${@}") _help() { cat <&2 @@ -90,7 +96,7 @@ _setup() { imageName="${IMAGE_NAME_OVERRIDE:-"${imageName}-${compiler}"}" ;; "dev" ) - fromImage="${FROM_IMAGE_OVERRIDE:-$osBaseImage}" + fromImage="${FROM_IMAGE_OVERRIDE:-${osBaseImage}}" context="etc" buildArgs="-save-deps-prefixes=${depsPrefixesFile}" if [[ "${isLocal}" == "yes" ]]; then @@ -99,7 +105,7 @@ _setup() { if [[ "${equivalenceDeps}" == "yes" ]]; then buildArgs="${buildArgs} -eqy" fi - if [[ "$CI" == "yes" ]]; then + if [[ "${CI}" == "yes" ]]; then buildArgs="${buildArgs} -ci" fi if [[ "${buildArgs}" != "" ]]; then @@ -139,7 +145,56 @@ _test() { docker run --rm "${imagePath}" "./docker/test_wrapper.sh" "${compiler}" "ctest --test-dir build -j ${numThreads}" } +_checkFromImage() { + set +e + # Check if the image exists locally + if docker image inspect "${fromImage}" > /dev/null 2>&1; then + echo "Image '${fromImage}' exists locally." + else + echo "Image '${fromImage}' does not exist locally. Attempting to pull..." + # Try to pull the image + if docker pull "${fromImage}"; then + echo "Successfully pulled '${fromImage}'." + else + echo "Unable to pull '${fromImage}'. Attempting to build..." + # Build the image using the createImage command + newArgs="" + newTarget="" + for arg in "${args[@]}"; do + # Check if the argument matches -target=builder + if [[ "${arg}" == "-target=builder" ]]; then + newTarget="dev" + elif [[ "${arg}" == "-target=binary" ]]; then + newTarget="builder" + else + newArgs+=" ${arg}" + fi + done + if [[ "${newTarget}" == "" ]]; then + echo "Error" + exit 1 + fi + newArgs+=" -target=${newTarget}" + createImage="$0 ${newArgs}" + echo "Running: ${createImage}" + if ${createImage}; then + echo "Successfully built '${newTarget}' image." + else + echo "Failed to build '${newTarget}' needed for '${target}' target." + return 1 + fi + fi + fi + set -e +} + _create() { + if [[ "${target}" == "binary" ]]; then + _checkFromImage "builder" + fi + if [[ "${target}" == "builder" ]]; then + _checkFromImage "dev" + fi echo "Create docker image ${imagePath} using ${file}" eval docker buildx build \ --file "${file}" \ @@ -260,9 +315,9 @@ while [ "$#" -gt 0 ]; do done if [[ "${numThreads}" == "-1" ]]; then - if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then numThreads=$(nproc --all) - elif [[ "$OSTYPE" == "darwin"* ]]; then + elif [[ "${OSTYPE}" == "darwin"* ]]; then numThreads=$(sysctl -n hw.ncpu) else numThreads=2