From be991f5a6266052481e396879f882b3800b6de1a Mon Sep 17 00:00:00 2001 From: Kyle Meyer Date: Fri, 3 May 2024 13:50:10 -0400 Subject: [PATCH] ci: switch to GitHub Actions The goal is for the new setup to largely follow the Drone setup, but there are some notable differences: * The Drone pipelines used custom images built to track Metworx's Ubuntu version (currently 18.04, soon to be 20.04). The GitHub Actions use GitHub-hosted runners and test with the two most recent Ubuntu releases available (20.04 and 22.04). r-lib/actions is used to install R and dependencies at the time of the run, with the built-in caching of setup-r-dependencies. * There is a build for the latest R version in addition to the R versions that were tested by Drone (3.6, 4.0, and 4.1). * For a PR, the Drone setup had two builds, one triggered by the branch push (testing on branch tip) and another by the PR (testing on merge with base). For the new setup, there is just one build triggered by the PR. It's done on the merge of the PR branch and its base. This means that the tip of the PR branch isn't tested. In general, that's fine because the merge is what ultimately will land. And for the cases where testing on the tip of the branch is useful (e.g., debugging a discrepancy due to a semantic merge conflict), a 'scratch/*' branch pointing to the same spot can be pushed. --- .Rbuildignore | 3 +- .drone.jsonnet | 386 ------------------------------------ .drone.yml | 138 ------------- .github/workflows/main.yaml | 91 +++++++++ 4 files changed, 92 insertions(+), 526 deletions(-) delete mode 100644 .drone.jsonnet delete mode 100644 .drone.yml create mode 100644 .github/workflows/main.yaml diff --git a/.Rbuildignore b/.Rbuildignore index f57ea72d..4f64d87d 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -17,6 +17,5 @@ Makefile ^\.Rproj\.user$ dev ^docs$ -\.drone\.yml -^\.drone\.jsonnet$ +^\.github$ vignettes/*\.pdf diff --git a/.drone.jsonnet b/.drone.jsonnet deleted file mode 100644 index 9c567964..00000000 --- a/.drone.jsonnet +++ /dev/null @@ -1,386 +0,0 @@ -######################################## -## START CONFIGURE BLOCK ## -######################################## - -# project name -local name = "yspec"; - -# where to mount a temporary volume; useful for persisting files across steps -local temp_volume_dir = "/ephemeral"; - -# environment variables available to every R call, including devtools::check(); -# R_LIBS_USER will be set separately; use the empty object {} if there are no -# variables to pass -local r_env_vars = { - "NOT_CRAN": "true", # you almost certainly want this -}; - -# images specified as repo:tag; first element will be used to build release -# https://console.aws.amazon.com/ecr/repositories/mpn-dev/?region=us-east-1 -# generally, only the second element should be modified -local ci_images = [ - "mpn-dev:latest", # latest MPN snapshot - "mpn-dev:2020-03-24", # oldest compatible snapshot - "cran-latest:latest", # latest MPN snapshot for MRG packages + current CRAN -]; - -# events that should not trigger Drone; recommended value is "promote", see -# https://discourse.drone.io/t/github-pages-triggering-builds-incorrectly/6370 -local exclude_events = [ - "promote", -]; - -######################################## -## END CONFIGURE BLOCK ## -######################################## - -# there should generally be no need to modify anything below here - -######################################## -## START SETTINGS BLOCK ## -######################################## - -# R major.minor versions to test against -# - first element will be used to lint package and build release -# - must be updated when versions in images are updated -local r_versions = [ - "4.0", - "3.6", -]; - -local ecr_repo_base = "906087756158.dkr.ecr.us-east-1.amazonaws.com"; - -local s3_bucket = "mpn.metworx.dev"; -local s3_target = "/releases"; - -local default_git_user = "Drony"; -local default_git_email = "drone@metrumrg.com"; -local default_volume_name = "docker.sock"; -local default_volume_path = "/var/run/docker.sock"; - -######################################## -## END SETTINGS BLOCK ## -######################################## - -######################################## -## START UTILITIES BLOCK ## -######################################## - -# Convert a key-value pair to a string "key" = "value" -# -# key a key -# value a value -local kv_to_string(key, value) = '"' + key + '" = "' + value + '"'; - -# Concatenate key-value pairs from an object -# -# obj an object with key-value pairs -local concat_kvs(obj) = - local kvs = std.mapWithKey(kv_to_string, obj); - - std.join(", ", [kvs[x] for x in std.objectFields(kvs)]); - -# Create a build tag -# -# name name of the application -# image image name as repo:tag -local create_build_tag(name, image) = std.join("-", [name, image]); - -# Create a CI image name -# -# repo the repository base -# tag image name as repo:tag -local create_ci_image(repo, image) = std.join("/", [repo, image]); - -# Get the name of the environment variable holding the path to the R executable -# r_major_minor R version to use, as major.minor, e.g., "4.0" -local get_r_exe_var(r_major_minor) = - std.join("_", ["R", "EXE"] + std.split(r_major_minor, ".")); - -######################################## -## END UTILITIES BLOCK ## -######################################## - -######################################## -## START PIPELINE BLOCK ## -######################################## - -# Add a block to control whether Drone executes for certain events -# -# include array of events that should trigger Drone -# exclude array of events that should not trigger Drone -local add_trigger(include=[], exclude=[]) = { - "trigger": { - "event": { - "include": include, - "exclude": exclude, - } - } -}; - -# Create a volume "object" -# -# name volume name -# path volume path (mount point) -local volume(name, path) = { - "name": name, - "path": path, -}; - -# Add host volume to pipeline volumes -# Intended to be called from add_volumes() -# -# volume volume object -local add_host_volume(volume) = { - "name": volume.name, - "host": { - "path": volume.path, - }, -}; - -# Add temporary volume to pipeline volumes -# Intended to be called from add_volumes() -# -# volume volume object -local add_temp_volume(volume) = { - "name": volume.name, - "temp": {}, -}; - -# Add volumes to a pipeline -# -# host array of volume objects to be added as host volumes -# temp array of volume objects to be added as temporary volumes -local add_volumes(host=[], temp=[]) = { - "volumes": - [add_host_volume(v) for v in host] + - [add_temp_volume(v) for v in temp], -}; - -# Add volume to a step -# -# volume volume object -local add_step_volume(volume) = { - "name": volume.name, - "path": volume.path, -}; - -# Drone step to pull a Docker image -# -# image image to pull -# volumes array of volume objects -local pull_image(image, volumes=[]) = { - "name": "Pull image", - "image": "omerxx/drone-ecr-auth", - "volumes": [add_step_volume(v) for v in volumes], - "commands": [ - "$(aws ecr get-login --no-include-email --region us-east-1)", - "docker pull " + image, - ], -}; - -# Run an R expression -# -# r_path path to R executable -# expr expression to run -local run_r_expression(r_path, expr) = - std.join(" ", [r_path, "-e", std.escapeStringBash(expr)]); - -# Drone step to copy a tagged release to S3 -# -# source_tag source tag name -# target_tag target "tag" name (allows for renaming tag) -# temp volume object -local s3_publish_tag(source_tag, target_tag, temp) = { - local strip_prefix = std.join("/", [temp.path, source_tag]), - - "name": "Publish package: " + target_tag, - # s3-sync plugin does not appear to support volumes (/tmp appears as - # /drone/src/tmp), and may not be under active development - # https://github.com/drone-plugins/drone-s3-sync/issues/17#issuecomment-374286940 - "image": "plugins/s3", - "pull": "if-not-exists", - "volumes": [add_step_volume(temp)], - "settings": { - "bucket": s3_bucket, - "source": std.join("/", [strip_prefix, "**/*"]), - "target": std.join( - "/", - [ - s3_target, - "${DRONE_REPO_NAME}", - target_tag - ] - ), - "strip_prefix": strip_prefix + "/", - }, -}; - -# Set up a Docker pipeline -# -# name pipeline name -local setup_docker_pipeline(name) = { - "kind": "pipeline", - "type": "docker", - "name": name, -}; - -# Shell command to source /etc/environment -local source_env() = ". /etc/environment"; - -# Drone step to check an R package -# -# r_major_minor R version to use, as major.minor, e.g., "4.0" -# image_uri URI of the CI image -# volumes array of volume objects -local check_step(r_major_minor, image, volumes=[]) = { - local r_bin_var = "$${" + get_r_exe_var(r_major_minor) + "}", - - "name": "Check package: R " + r_major_minor, - "image": image, - "pull": "never", - "volumes": [add_step_volume(v) for v in volumes], - "environment": r_env_vars + { - "R_LIBS_USER": "/opt/rpkgs/" + r_major_minor, - }, - "commands": [ - # can't evaluate shell expressions in environment - # https://docs.drone.io/pipeline/environment/syntax/#common-problems - "export PATH=" + volumes[0].path + ":$PATH", - source_env(), - run_r_expression( - r_bin_var, - "devtools::install_deps(upgrade = 'never')" - ), - run_r_expression( - r_bin_var, - "devtools::check(env_vars = c(" + concat_kvs(r_env_vars) + "))" - ), - ], -}; - -# Drone pipeline to check an R package -# -# name name of the application -# image CI image, as repo:tag -local check(name, image) = - local build_tag = create_build_tag(name, image); - local image_uri = create_ci_image(ecr_repo_base, image); - - local host_volume = volume(default_volume_name, default_volume_path); - local temp_volume = volume("cache", temp_volume_dir); - - setup_docker_pipeline(build_tag) + - add_volumes([host_volume], [temp_volume]) + - add_trigger(exclude=exclude_events) + - { - "steps": [ - pull_image(image_uri, [host_volume]), - ] + [ - check_step(r_ver, image_uri, [temp_volume]) - for r_ver in r_versions - ], - }; - -# Drone pipeline to lint an R package -# arguments are the same as for check() -local lint(name, r_major_minor, image) = - local r_bin_var = "$${" + get_r_exe_var(r_major_minor) + "}"; - - local image_uri = create_ci_image(ecr_repo_base, image); - - local host_volume = volume(default_volume_name, default_volume_path); - - setup_docker_pipeline(name + "-lint") + - add_volumes([host_volume]) + - add_trigger(exclude=exclude_events) + - { - "steps": [ - pull_image(image_uri, [host_volume]), - { - "name": "Lint package", - "image": image_uri, - "pull": "never", - "environment": r_env_vars + { - "R_LIBS_USER": "/opt/rpkgs/" + r_major_minor, - }, - "commands": [ - source_env(), - # need NOT_CRAN = "true" to run this - run_r_expression(r_bin_var, "lintr::expect_lint_free()"), - ], - }, - ], - }; - -# Drone pipeline to build and deploy an R package -# arguments are the same as for check() -local release(name, r_major_minor, image) = - local r_bin_var = "$${" + get_r_exe_var(r_major_minor) + "}"; - - local image_uri = create_ci_image(ecr_repo_base, image); - - local host_volume = volume(default_volume_name, default_volume_path); - local temp_volume = volume("cache", temp_volume_dir); - - setup_docker_pipeline(name + '-release') + - add_volumes([host_volume], [temp_volume]) + - add_trigger(include=["tag"]) + - { - "steps": [ - pull_image(image_uri, [host_volume]), - { - "name": "Build package", - "image": image_uri, - "pull": "never", - "volumes": [add_step_volume(v) for v in [temp_volume]], - "environment": r_env_vars + { - "R_LIBS_USER": "/opt/rpkgs/" + r_major_minor, - }, - "commands": [ - # git config needs to sit next to pkgpub - "git config --global user.email " + default_git_email, - "git config --global user.name " + default_git_user, - "git fetch --tags", - "export PATH=" + temp_volume.path + ":$PATH", - source_env(), - run_r_expression( - r_bin_var, - std.format( - "pkgpub::create_tagged_repo(.dir = '%s')", - temp_volume.path - ) - ), - ], - }, - s3_publish_tag("${DRONE_TAG}", "${DRONE_TAG}", temp_volume), - s3_publish_tag("${DRONE_TAG}", "latest_tag", temp_volume), - ], - }; - -######################################## -## END PIPELINE BLOCK ## -######################################## - -######################################## -## START DRONE CONFIG BLOCK ## -######################################## - -[ - check(name, image) - for image in ci_images -] + [ - # lint(name, r_versions[0], ci_images[0]), - # release step requires all check steps to pass - release(name, r_versions[0], ci_images[0]) + - { - "depends_on": [ - create_build_tag(name, image) - for image in ci_images - ], - }, -] - -######################################## -## END DRONE CONFIG BLOCK ## -######################################## diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 31d358ef..00000000 --- a/.drone.yml +++ /dev/null @@ -1,138 +0,0 @@ ---- -kind: pipeline -type: docker -name: cran-latest - -platform: - os: linux - arch: amd64 - -steps: -- name: pull - image: omerxx/drone-ecr-auth - commands: - - $(aws ecr get-login --no-include-email --region us-east-1) - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-3.6:cran-latest - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.0:cran-latest - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:cran-latest - volumes: - - name: docker.sock - path: /var/run/docker.sock - -- name: "Check package: R 3.6" - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-3.6:cran-latest - commands: - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'devtools::check(env_vars = c("NOT_CRAN" = "true"))' - environment: - USER: drone - volumes: - - name: cache - path: /ephemeral - -- name: "Check package: R 4.0" - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.0:cran-latest - commands: - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'devtools::check(env_vars = c("NOT_CRAN" = "true"))' - environment: - USER: drone - volumes: - - name: cache - path: /ephemeral - -- name: "Check package: R 4.1" - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:cran-latest - commands: - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'devtools::check(env_vars = c("NOT_CRAN" = "true"))' - environment: - USER: drone - volumes: - - name: cache - path: /ephemeral - -volumes: -- name: docker.sock - host: - path: /var/run/docker.sock -- name: cache - temp: {} - -trigger: - event: - exclude: - - promote - ---- -kind: pipeline -type: docker -name: yspec-release - -platform: - os: linux - arch: amd64 - -steps: -- name: pull - image: omerxx/drone-ecr-auth - commands: - - $(aws ecr get-login --no-include-email --region us-east-1) - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:latest - volumes: - - name: docker.sock - path: /var/run/docker.sock - -- name: Build package - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:latest - commands: - - git config --global user.email drone@metrumrg.com - - git config --global user.name Drony - - git fetch --tags - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'pkgpub::create_tagged_repo(.dir = '"'"'/ephemeral'"'"')' - volumes: - - name: cache - path: /ephemeral - -- name: "Publish package: ${DRONE_TAG}" - pull: if-not-exists - image: plugins/s3 - settings: - bucket: mpn.metworx.dev - source: /ephemeral/${DRONE_TAG}/**/* - strip_prefix: /ephemeral/${DRONE_TAG}/ - target: /releases/${DRONE_REPO_NAME}/${DRONE_TAG} - volumes: - - name: cache - path: /ephemeral - -- name: "Publish package: latest_tag" - pull: if-not-exists - image: plugins/s3 - settings: - bucket: mpn.metworx.dev - source: /ephemeral/${DRONE_TAG}/**/* - strip_prefix: /ephemeral/${DRONE_TAG}/ - target: /releases/${DRONE_REPO_NAME}/latest_tag - volumes: - - name: cache - path: /ephemeral - -volumes: -- name: docker.sock - host: - path: /var/run/docker.sock -- name: cache - temp: {} - -trigger: - event: - - tag - -depends_on: -- cran-latest \ No newline at end of file diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..1b47117f --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,91 @@ +name: CI +on: + push: + branches: + - 'main' + - 'scratch/**' + tags: + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.[0-9]+' + pull_request: + +jobs: + check: + runs-on: ${{ matrix.config.os }} + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + strategy: + fail-fast: false + matrix: + config: + - os: ubuntu-20.04 + r: 3.6.3 + - os: ubuntu-20.04 + r: 4.0.5 + - os: ubuntu-20.04 + r: 4.1.3 + - os: ubuntu-latest + r: release + env: + R_KEEP_PKG_SOURCE: yes + steps: + - uses: actions/checkout@v4 + - uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + use-public-rspm: true + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + upgrade: 'TRUE' + - name: Install pdflatex + shell: bash + run: sudo apt-get install texlive-latex-base texlive-fonts-extra + - uses: r-lib/actions/check-r-package@v2 + release: + if: github.ref_type == 'tag' + name: Upload release + needs: check + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + use-public-rspm: true + # For pkgpub. + extra-repositories: 'https://mpn.metworx.com/snapshots/stable/2024-03-01' + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgpub + - name: Install pdflatex + shell: bash + run: sudo apt-get install texlive-latex-base texlive-fonts-extra + - name: Configure Git + shell: bash + run: | + git config --global user.name CI + git config --global user.email ci@metrumrg + - name: Create output directory + shell: bash + run: echo "REPO_DIR=$(mktemp -d)" >>$GITHUB_ENV + - name: Create CRAN-like repo for release + run: pkgpub::create_tagged_repo("${{ env.REPO_DIR }}") + shell: Rscript {0} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::906087756158:role/github-actions-mpn-s3-publish + aws-region: us-east-1 + - name: 'Publish package: ${{ github.ref_name }}' + run: | + aws s3 sync \ + '${{ env.REPO_DIR }}/${{ github.ref_name }}/' \ + 's3://mpn.metworx.dev/releases/${{ github.event.repository.name }}/${{ github.ref_name }}/' + - name: 'Publish package: latest_tag' + run: | + aws s3 sync \ + '${{ env.REPO_DIR }}/${{ github.ref_name }}/' \ + 's3://mpn.metworx.dev/releases/${{ github.event.repository.name }}/latest_tag/'