From 5cb9bf40a5c903ebb55c3c6fe26f140c407b18dc Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Thu, 25 Apr 2024 15:15:32 +0100 Subject: [PATCH 1/7] First crack at a parallelisable build process --- .github/workflows/{ => deprecated}/ci.yml | 0 .github/workflows/improvedci.yml | 208 ++++++++++++++++++++++ 2 files changed, 208 insertions(+) rename .github/workflows/{ => deprecated}/ci.yml (100%) create mode 100644 .github/workflows/improvedci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/deprecated/ci.yml similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/deprecated/ci.yml diff --git a/.github/workflows/improvedci.yml b/.github/workflows/improvedci.yml new file mode 100644 index 00000000000..6d48c664d46 --- /dev/null +++ b/.github/workflows/improvedci.yml @@ -0,0 +1,208 @@ +name: Facia-Tool CI +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + build_client_v1: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + cache-dependency-path: package-lock.json + - name: Set up grunt + run: npm i -g grunt-cli + - name: Install NPM dependencies + run: npm install + - name: Set up JSPM + env: + JSPM_GITHUB_AUTH_SECRET: ${{ secrets.GITHUB_TOKEN }} + run: | + npm run jspm config registries.github.auth ${JSPM_GITHUB_AUTH_SECRET} + npm run jspm registry export github + + - name: Install JSPM dependencies + run: npm run jspm install + + - name: Build frontend v1 + run: | + grunt --stack validate + grunt --stack bundle + - uses: actions/upload-artifact@v2 + with: + name: fronts-client.dist + path: public/fronts-client.dist + + test_client_v1: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + cache-dependency-path: package-lock.json + - name: Set up grunt + run: npm i -g grunt-cli + - name: Install NPM dependencies + run: npm install + - name: Set up JSPM + env: + JSPM_GITHUB_AUTH_SECRET: ${{ secrets.GITHUB_TOKEN }} + run: | + npm run jspm config registries.github.auth ${JSPM_GITHUB_AUTH_SECRET} + npm run jspm registry export github + + - name: Install JSPM dependencies + run: npm run jspm install + + - name: Test frontend v1 + run: | + grunt --stack validate + grunt --stack test + + build_client_v2: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: fronts-client/.nvmrc + cache: yarn + cache-dependency-path: fronts-client/yarn.lock + - run: yarn install --frozen-lockfile + working-directory: fronts-client + - run: yarn build + working-directory: fronts-client + - uses: actions/upload-artifact@v2 + with: + name: fronts-client-v2 + path: public/fronts-client-v2 + + test_client_v2: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: fronts-client/.nvmrc + cache: yarn + cache-dependency-path: fronts-client/yarn.lock + - run: yarn install --frozen-lockfile + working-directory: fronts-client + - run: yarn test + working-directory: fronts-client + - run: test-integration-ci + working-directory: fronts-client + + test_backend: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + services: + # See https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers + postgres: + image: postgres:10.7-alpine + env: + POSTGRES_USER: faciatool + POSTGRES_PASSWORD: faciatool + POSTGRES_DB: faciatool + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 4724:5432 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: "11" + distribution: "corretto" + cache: "sbt" + - name: Run tests + run: sbt 'test; database-int:test' + + build_backend: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + needs: + - build_client_v1 + - build_client_v2 + services: + # See https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers + postgres: + image: postgres:10.7-alpine + env: + POSTGRES_USER: faciatool + POSTGRES_PASSWORD: faciatool + POSTGRES_DB: faciatool + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 4724:5432 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: "11" + distribution: "corretto" + cache: "sbt" + - name: Bundle + run: sbt debian:packageBin + - uses: actions/upload-artifact@v2 + with: + name: build-package + path: target/facia-tool_1.0_all.deb + + upload: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + needs: + - test_client_v1 + - test_client_v2 + - test_backend + - build_backend + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v2 + with: + path: target/facia-tool_1.0_all.deb + name: build-package + - uses: guardian/actions-riff-raff@v4 + with: + app: cms-fronts::facia-tool + roleArn: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }} + githubToken: ${{ secrets.GITHUB_TOKEN }} + configPath: riff-raff.yaml + contentDirectories: | + facia-tool: + - target/facia-tool_1.0_all.deb From f47c71b49e9b93ac3556ff6c6cc42ad02b2c5d24 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Thu, 25 Apr 2024 15:37:18 +0100 Subject: [PATCH 2/7] set build number properly, add path fix --- .github/workflows/improvedci.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/improvedci.yml b/.github/workflows/improvedci.yml index 6d48c664d46..2e4197a4261 100644 --- a/.github/workflows/improvedci.yml +++ b/.github/workflows/improvedci.yml @@ -6,6 +6,18 @@ on: - main jobs: + generate_build_number: + name: Generate build number + runs-on: ubuntu-latest + outputs: + BUILD_NUMBER: ${{ steps.create-build-number.outputs.BUILD_NUMBER }} + steps: + - id: create-build-number + name: Create build number + run: | + LAST_TEAMCITY_BUILD=7962 + echo "BUILD_NUMBER=$(( $GITHUB_RUN_NUMBER + $LAST_TEAMCITY_BUILD ))" >> "$GITHUB_OUTPUT" + build_client_v1: runs-on: ubuntu-latest permissions: @@ -34,12 +46,11 @@ jobs: - name: Build frontend v1 run: | - grunt --stack validate grunt --stack bundle - uses: actions/upload-artifact@v2 with: - name: fronts-client.dist - path: public/fronts-client.dist + name: fronts-client-v1 + path: public/fronts-client-v1 test_client_v1: runs-on: ubuntu-latest @@ -109,7 +120,7 @@ jobs: working-directory: fronts-client - run: yarn test working-directory: fronts-client - - run: test-integration-ci + - run: yarn test-integration-ci working-directory: fronts-client test_backend: @@ -151,6 +162,7 @@ jobs: needs: - build_client_v1 - build_client_v2 + - generate_build_number services: # See https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers postgres: @@ -169,6 +181,17 @@ jobs: - 4724:5432 steps: - uses: actions/checkout@v4 + - uses: actions/download-artifact@v2 + with: + path: public/fronts-client-v1 + name: fronts-client-v1 + - uses: actions/download-artifact@v2 + with: + path: public/fronts-client-v2 + name: fronts-client-v2 + - name: Set GITHUB_RUN_NUMBER environment variable + run: | + echo "GITHUB_RUN_NUMBER=${{ needs.generate_build_number.outputs.BUILD_NUMBER }}" >> $GITHUB_ENV - uses: actions/setup-java@v4 with: java-version: "11" From f0c7f0d053aef968048fb658743a8f7ee13f46f3 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Thu, 25 Apr 2024 15:46:19 +0100 Subject: [PATCH 3/7] fix frontend integration test, versions & build number --- .github/workflows/improvedci.yml | 49 +++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/.github/workflows/improvedci.yml b/.github/workflows/improvedci.yml index 2e4197a4261..d6abc4d796c 100644 --- a/.github/workflows/improvedci.yml +++ b/.github/workflows/improvedci.yml @@ -47,11 +47,11 @@ jobs: - name: Build frontend v1 run: | grunt --stack bundle - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: fronts-client-v1 path: public/fronts-client-v1 - + if-no-files-found: error test_client_v1: runs-on: ubuntu-latest permissions: @@ -99,11 +99,11 @@ jobs: working-directory: fronts-client - run: yarn build working-directory: fronts-client - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: fronts-client-v2 path: public/fronts-client-v2 - + if-no-files-found: error test_client_v2: runs-on: ubuntu-latest permissions: @@ -120,6 +120,27 @@ jobs: working-directory: fronts-client - run: yarn test working-directory: fronts-client + + integrationtest_client_v2: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + needs: + - build_client_v2 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: fronts-client/.nvmrc + cache: yarn + cache-dependency-path: fronts-client/yarn.lock + - uses: actions/download-artifact@v4 #integration test requires built frontend (of course!) + with: + path: public/fronts-client-v2 + name: fronts-client-v2 + - run: yarn install --frozen-lockfile + working-directory: fronts-client - run: yarn test-integration-ci working-directory: fronts-client @@ -181,11 +202,11 @@ jobs: - 4724:5432 steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: path: public/fronts-client-v1 name: fronts-client-v1 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: path: public/fronts-client-v2 name: fronts-client-v2 @@ -199,31 +220,39 @@ jobs: cache: "sbt" - name: Bundle run: sbt debian:packageBin - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: build-package path: target/facia-tool_1.0_all.deb - + if-no-files-found: error + compression-level: '0' #no point, it's already compressed upload: runs-on: ubuntu-latest permissions: id-token: write contents: read + pull-requests: write needs: + - generate_build_number - test_client_v1 - test_client_v2 - test_backend - build_backend + - integrationtest_client_v2 steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: path: target/facia-tool_1.0_all.deb name: build-package + - name: Set GITHUB_RUN_NUMBER environment variable + run: | + echo "GITHUB_RUN_NUMBER=${{ needs.generate_build_number.outputs.BUILD_NUMBER }}" >> $GITHUB_ENV - uses: guardian/actions-riff-raff@v4 with: - app: cms-fronts::facia-tool + app: facia-tool roleArn: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }} + buildNumber: ${{ env.GITHUB_RUN_NUMBER }} githubToken: ${{ secrets.GITHUB_TOKEN }} configPath: riff-raff.yaml contentDirectories: | From d8cc86e22ab8572a47e4f7d925cde3ee407850de Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Thu, 25 Apr 2024 16:23:46 +0100 Subject: [PATCH 4/7] explicitly bump backend to java11 --- build.sbt | 6 +++--- riff-raff.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 3e2a2bb44aa..c044e614b82 100644 --- a/build.sbt +++ b/build.sbt @@ -12,8 +12,8 @@ ThisBuild / scalaVersion := "2.13.5" import com.gu.riffraff.artifact.BuildInfo import sbt.Resolver - -debianPackageDependencies := Seq("openjdk-8-jre-headless") +// +//debianPackageDependencies := Seq("openjdk-11-jre-headless") def env(key: String): Option[String] = Option(System.getenv(key)) @@ -43,7 +43,7 @@ Universal / javaOptions ++= Seq( routesGenerator := InjectedRoutesGenerator -scalacOptions := Seq("-unchecked", "-deprecation", "-target:jvm-1.8", "-Xcheckinit", "-encoding", "utf8", "-feature") +scalacOptions := Seq("-unchecked", "-deprecation", "-target:jvm-11", "-Xcheckinit", "-encoding", "utf8", "-feature") Compile / doc / sources := Seq.empty diff --git a/riff-raff.yaml b/riff-raff.yaml index 1953b7f3dbe..fd7726bd0d4 100644 --- a/riff-raff.yaml +++ b/riff-raff.yaml @@ -11,7 +11,7 @@ deployments: app: facia-tool parameters: amiTags: - Recipe: editorial-tools-focal-java8-ARM-WITH-cdk-base + Recipe: editorial-tools-focal-java11-ARM-WITH-cdk-base AmigoStage: PROD BuiltBy: amigo amiEncrypted: true From 7278bcd0c05650b62005f6a36173cd6687e78d27 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Thu, 25 Apr 2024 17:01:58 +0100 Subject: [PATCH 5/7] Remove sbt-riffraff plugin and deprecated jvm-11 options --- .github/workflows/deprecated/ci.yml | 30 ----------------- build.sbt | 25 +++----------- project/plugins.sbt | 2 -- scripts/ci.sh | 52 ----------------------------- 4 files changed, 5 insertions(+), 104 deletions(-) delete mode 100644 .github/workflows/deprecated/ci.yml delete mode 100755 scripts/ci.sh diff --git a/.github/workflows/deprecated/ci.yml b/.github/workflows/deprecated/ci.yml deleted file mode 100644 index 4d9b42dcae7..00000000000 --- a/.github/workflows/deprecated/ci.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Facia-tool CI -on: - workflow_dispatch: - push: - -jobs: - CI: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v2 - - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }} - aws-region: eu-west-1 - - uses: actions/setup-java@v3 - with: - java-version: "8" - distribution: "corretto" - - name: build - env: - JSPM_GITHUB_AUTH_SECRET: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - # Ensure we don't overwrite existing (Teamcity) builds. - LAST_TEAMCITY_BUILD=7360 - export GITHUB_RUN_NUMBER=$(( $GITHUB_RUN_NUMBER + $LAST_TEAMCITY_BUILD )) - scripts/ci.sh diff --git a/build.sbt b/build.sbt index c044e614b82..0091256e8b9 100644 --- a/build.sbt +++ b/build.sbt @@ -10,23 +10,11 @@ packageDescription := "Guardian front pages editor" ThisBuild / scalaVersion := "2.13.5" -import com.gu.riffraff.artifact.BuildInfo import sbt.Resolver -// -//debianPackageDependencies := Seq("openjdk-11-jre-headless") -def env(key: String): Option[String] = Option(System.getenv(key)) +debianPackageDependencies := Seq("java11-runtime-headless") -riffRaffPackageName := s"cms-fronts::${name.value}" -riffRaffManifestProjectName := riffRaffPackageName.value -riffRaffUploadArtifactBucket := Option("riffraff-artifact") -riffRaffUploadManifestBucket := Option("riffraff-builds") -riffRaffArtifactResources := { - Seq( - (Debian / packageBin).value -> s"${name.value}/${name.value}_1.0_all.deb", - baseDirectory.value / "riff-raff.yaml" -> "riff-raff.yaml" - ) -} +def env(key: String): Option[String] = Option(System.getenv(key)) ThisBuild / javacOptions := Seq("-g","-encoding", "utf8") @@ -35,8 +23,6 @@ Universal / javaOptions ++= Seq( "-J-XX:MaxRAMFraction=2", "-J-XX:InitialRAMFraction=2", "-J-XX:MaxMetaspaceSize=500m", - "-J-XX:+PrintGCDetails", - "-J-XX:+PrintGCDateStamps", s"-J-Xloggc:/var/log/${packageName.value}/gc.log", "-Dcom.amazonaws.sdk.disableCbor" ) @@ -128,19 +114,18 @@ libraryDependencies ++= Seq( val UsesDatabaseTest = config("database-int") extend Test lazy val root = (project in file(".")) - .enablePlugins(PlayScala, RiffRaffArtifact, JDebPackaging, SystemdPlugin, BuildInfoPlugin) + .enablePlugins(PlayScala, JDebPackaging, SystemdPlugin, BuildInfoPlugin) .configs(UsesDatabaseTest) .settings( buildInfoPackage := "facia", buildInfoKeys := { - lazy val buildInfo = BuildInfo(baseDirectory.value) Seq[BuildInfoKey]( - BuildInfoKey.constant("buildNumber", buildInfo.buildIdentifier), + BuildInfoKey.constant("buildNumber", env("GITHUB_BUILD_NUMBER").getOrElse("unknown")), // so this next one is constant to avoid it always recompiling on dev machines. // we only really care about build time on teamcity, when a constant based on when // it was loaded is just fine BuildInfoKey.constant("buildTime", System.currentTimeMillis), - BuildInfoKey.constant("gitCommitId", buildInfo.revision) + BuildInfoKey.constant("gitCommitId", env("GITHUB_SHA").getOrElse("unknown")) ) } ) diff --git a/project/plugins.sbt b/project/plugins.sbt index 66f50aaee9e..ff7bfa72582 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -16,6 +16,4 @@ addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.11") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.9.3") -addSbtPlugin("com.gu" % "sbt-riffraff-artifact" % "1.1.18") - addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") diff --git a/scripts/ci.sh b/scripts/ci.sh deleted file mode 100755 index 670c8d2f49b..00000000000 --- a/scripts/ci.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -set -e - -setupNvm() { - export NVM_DIR="$HOME/.nvm" - [[ -s "$NVM_DIR/nvm.sh" ]] && . "$NVM_DIR/nvm.sh" # This loads nvm - nvm install - nvm use -} - -globalJsDependencies() { - npm install -g grunt-cli - npm install -g yarn -} - -javascriptV1() { - npm install - npm run jspm config registries.github.auth ${JSPM_GITHUB_AUTH_SECRET} - npm run jspm registry export github - npm run jspm install - grunt --stack validate - grunt --stack test - grunt --stack bundle -} - -javascriptV2() { - pushd fronts-client - setupNvm - yarn install - yarn lint - yarn test - yarn run build - yarn test-integration-ci - - popd -} - -riffRaffUpload() { - sbt clean compile test database-int:test riffRaffUpload -} - -main() { - docker-compose up -d - setupNvm - globalJsDependencies - javascriptV1 - javascriptV2 - riffRaffUpload -} - -main From 1497134f768107b7734f63b9cab858e54674d34e Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Fri, 26 Apr 2024 12:09:10 +0100 Subject: [PATCH 6/7] Pick up JSPM packages from client v1 build too --- .github/workflows/improvedci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/improvedci.yml b/.github/workflows/improvedci.yml index d6abc4d796c..973eaa36a90 100644 --- a/.github/workflows/improvedci.yml +++ b/.github/workflows/improvedci.yml @@ -52,6 +52,11 @@ jobs: name: fronts-client-v1 path: public/fronts-client-v1 if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: jspm-packages + path: public/src/jspm_packages + if-no-files-found: error test_client_v1: runs-on: ubuntu-latest permissions: @@ -210,6 +215,10 @@ jobs: with: path: public/fronts-client-v2 name: fronts-client-v2 + - uses: actions/download-artifact@v4 + with: + name: jspm-packages + path: public/src/jspm_packages - name: Set GITHUB_RUN_NUMBER environment variable run: | echo "GITHUB_RUN_NUMBER=${{ needs.generate_build_number.outputs.BUILD_NUMBER }}" >> $GITHUB_ENV From 34587a32eae7180063842278c8b0503baf6773a3 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Mon, 29 Apr 2024 11:54:51 +0100 Subject: [PATCH 7/7] We shouldn't need DB service during backend build, only test --- .github/workflows/improvedci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/improvedci.yml b/.github/workflows/improvedci.yml index 973eaa36a90..99419fdb59c 100644 --- a/.github/workflows/improvedci.yml +++ b/.github/workflows/improvedci.yml @@ -189,22 +189,6 @@ jobs: - build_client_v1 - build_client_v2 - generate_build_number - services: - # See https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers - postgres: - image: postgres:10.7-alpine - env: - POSTGRES_USER: faciatool - POSTGRES_PASSWORD: faciatool - POSTGRES_DB: faciatool - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 4724:5432 steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4