From 08770f9ee8dfebce11e128838a7248a3e5e12471 Mon Sep 17 00:00:00 2001 From: Kyle Meyer Date: Fri, 3 May 2024 12:47:34 -0400 Subject: [PATCH] wip --- .Rbuildignore | 3 +- .drone.jsonnet | 386 ------------------------------------ .drone.yml | 138 ------------- .github/workflows/main.yaml | 86 ++++++++ 4 files changed, 87 insertions(+), 526 deletions(-) delete mode 100644 .drone.jsonnet delete mode 100644 .drone.yml create mode 100644 .github/workflows/main.yaml diff --git a/.Rbuildignore b/.Rbuildignore index f57ea72d..4f64d87d 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -17,6 +17,5 @@ Makefile ^\.Rproj\.user$ dev ^docs$ -\.drone\.yml -^\.drone\.jsonnet$ +^\.github$ vignettes/*\.pdf diff --git a/.drone.jsonnet b/.drone.jsonnet deleted file mode 100644 index 9c567964..00000000 --- a/.drone.jsonnet +++ /dev/null @@ -1,386 +0,0 @@ -######################################## -## START CONFIGURE BLOCK ## -######################################## - -# project name -local name = "yspec"; - -# where to mount a temporary volume; useful for persisting files across steps -local temp_volume_dir = "/ephemeral"; - -# environment variables available to every R call, including devtools::check(); -# R_LIBS_USER will be set separately; use the empty object {} if there are no -# variables to pass -local r_env_vars = { - "NOT_CRAN": "true", # you almost certainly want this -}; - -# images specified as repo:tag; first element will be used to build release -# https://console.aws.amazon.com/ecr/repositories/mpn-dev/?region=us-east-1 -# generally, only the second element should be modified -local ci_images = [ - "mpn-dev:latest", # latest MPN snapshot - "mpn-dev:2020-03-24", # oldest compatible snapshot - "cran-latest:latest", # latest MPN snapshot for MRG packages + current CRAN -]; - -# events that should not trigger Drone; recommended value is "promote", see -# https://discourse.drone.io/t/github-pages-triggering-builds-incorrectly/6370 -local exclude_events = [ - "promote", -]; - -######################################## -## END CONFIGURE BLOCK ## -######################################## - -# there should generally be no need to modify anything below here - -######################################## -## START SETTINGS BLOCK ## -######################################## - -# R major.minor versions to test against -# - first element will be used to lint package and build release -# - must be updated when versions in images are updated -local r_versions = [ - "4.0", - "3.6", -]; - -local ecr_repo_base = "906087756158.dkr.ecr.us-east-1.amazonaws.com"; - -local s3_bucket = "mpn.metworx.dev"; -local s3_target = "/releases"; - -local default_git_user = "Drony"; -local default_git_email = "drone@metrumrg.com"; -local default_volume_name = "docker.sock"; -local default_volume_path = "/var/run/docker.sock"; - -######################################## -## END SETTINGS BLOCK ## -######################################## - -######################################## -## START UTILITIES BLOCK ## -######################################## - -# Convert a key-value pair to a string "key" = "value" -# -# key a key -# value a value -local kv_to_string(key, value) = '"' + key + '" = "' + value + '"'; - -# Concatenate key-value pairs from an object -# -# obj an object with key-value pairs -local concat_kvs(obj) = - local kvs = std.mapWithKey(kv_to_string, obj); - - std.join(", ", [kvs[x] for x in std.objectFields(kvs)]); - -# Create a build tag -# -# name name of the application -# image image name as repo:tag -local create_build_tag(name, image) = std.join("-", [name, image]); - -# Create a CI image name -# -# repo the repository base -# tag image name as repo:tag -local create_ci_image(repo, image) = std.join("/", [repo, image]); - -# Get the name of the environment variable holding the path to the R executable -# r_major_minor R version to use, as major.minor, e.g., "4.0" -local get_r_exe_var(r_major_minor) = - std.join("_", ["R", "EXE"] + std.split(r_major_minor, ".")); - -######################################## -## END UTILITIES BLOCK ## -######################################## - -######################################## -## START PIPELINE BLOCK ## -######################################## - -# Add a block to control whether Drone executes for certain events -# -# include array of events that should trigger Drone -# exclude array of events that should not trigger Drone -local add_trigger(include=[], exclude=[]) = { - "trigger": { - "event": { - "include": include, - "exclude": exclude, - } - } -}; - -# Create a volume "object" -# -# name volume name -# path volume path (mount point) -local volume(name, path) = { - "name": name, - "path": path, -}; - -# Add host volume to pipeline volumes -# Intended to be called from add_volumes() -# -# volume volume object -local add_host_volume(volume) = { - "name": volume.name, - "host": { - "path": volume.path, - }, -}; - -# Add temporary volume to pipeline volumes -# Intended to be called from add_volumes() -# -# volume volume object -local add_temp_volume(volume) = { - "name": volume.name, - "temp": {}, -}; - -# Add volumes to a pipeline -# -# host array of volume objects to be added as host volumes -# temp array of volume objects to be added as temporary volumes -local add_volumes(host=[], temp=[]) = { - "volumes": - [add_host_volume(v) for v in host] + - [add_temp_volume(v) for v in temp], -}; - -# Add volume to a step -# -# volume volume object -local add_step_volume(volume) = { - "name": volume.name, - "path": volume.path, -}; - -# Drone step to pull a Docker image -# -# image image to pull -# volumes array of volume objects -local pull_image(image, volumes=[]) = { - "name": "Pull image", - "image": "omerxx/drone-ecr-auth", - "volumes": [add_step_volume(v) for v in volumes], - "commands": [ - "$(aws ecr get-login --no-include-email --region us-east-1)", - "docker pull " + image, - ], -}; - -# Run an R expression -# -# r_path path to R executable -# expr expression to run -local run_r_expression(r_path, expr) = - std.join(" ", [r_path, "-e", std.escapeStringBash(expr)]); - -# Drone step to copy a tagged release to S3 -# -# source_tag source tag name -# target_tag target "tag" name (allows for renaming tag) -# temp volume object -local s3_publish_tag(source_tag, target_tag, temp) = { - local strip_prefix = std.join("/", [temp.path, source_tag]), - - "name": "Publish package: " + target_tag, - # s3-sync plugin does not appear to support volumes (/tmp appears as - # /drone/src/tmp), and may not be under active development - # https://github.com/drone-plugins/drone-s3-sync/issues/17#issuecomment-374286940 - "image": "plugins/s3", - "pull": "if-not-exists", - "volumes": [add_step_volume(temp)], - "settings": { - "bucket": s3_bucket, - "source": std.join("/", [strip_prefix, "**/*"]), - "target": std.join( - "/", - [ - s3_target, - "${DRONE_REPO_NAME}", - target_tag - ] - ), - "strip_prefix": strip_prefix + "/", - }, -}; - -# Set up a Docker pipeline -# -# name pipeline name -local setup_docker_pipeline(name) = { - "kind": "pipeline", - "type": "docker", - "name": name, -}; - -# Shell command to source /etc/environment -local source_env() = ". /etc/environment"; - -# Drone step to check an R package -# -# r_major_minor R version to use, as major.minor, e.g., "4.0" -# image_uri URI of the CI image -# volumes array of volume objects -local check_step(r_major_minor, image, volumes=[]) = { - local r_bin_var = "$${" + get_r_exe_var(r_major_minor) + "}", - - "name": "Check package: R " + r_major_minor, - "image": image, - "pull": "never", - "volumes": [add_step_volume(v) for v in volumes], - "environment": r_env_vars + { - "R_LIBS_USER": "/opt/rpkgs/" + r_major_minor, - }, - "commands": [ - # can't evaluate shell expressions in environment - # https://docs.drone.io/pipeline/environment/syntax/#common-problems - "export PATH=" + volumes[0].path + ":$PATH", - source_env(), - run_r_expression( - r_bin_var, - "devtools::install_deps(upgrade = 'never')" - ), - run_r_expression( - r_bin_var, - "devtools::check(env_vars = c(" + concat_kvs(r_env_vars) + "))" - ), - ], -}; - -# Drone pipeline to check an R package -# -# name name of the application -# image CI image, as repo:tag -local check(name, image) = - local build_tag = create_build_tag(name, image); - local image_uri = create_ci_image(ecr_repo_base, image); - - local host_volume = volume(default_volume_name, default_volume_path); - local temp_volume = volume("cache", temp_volume_dir); - - setup_docker_pipeline(build_tag) + - add_volumes([host_volume], [temp_volume]) + - add_trigger(exclude=exclude_events) + - { - "steps": [ - pull_image(image_uri, [host_volume]), - ] + [ - check_step(r_ver, image_uri, [temp_volume]) - for r_ver in r_versions - ], - }; - -# Drone pipeline to lint an R package -# arguments are the same as for check() -local lint(name, r_major_minor, image) = - local r_bin_var = "$${" + get_r_exe_var(r_major_minor) + "}"; - - local image_uri = create_ci_image(ecr_repo_base, image); - - local host_volume = volume(default_volume_name, default_volume_path); - - setup_docker_pipeline(name + "-lint") + - add_volumes([host_volume]) + - add_trigger(exclude=exclude_events) + - { - "steps": [ - pull_image(image_uri, [host_volume]), - { - "name": "Lint package", - "image": image_uri, - "pull": "never", - "environment": r_env_vars + { - "R_LIBS_USER": "/opt/rpkgs/" + r_major_minor, - }, - "commands": [ - source_env(), - # need NOT_CRAN = "true" to run this - run_r_expression(r_bin_var, "lintr::expect_lint_free()"), - ], - }, - ], - }; - -# Drone pipeline to build and deploy an R package -# arguments are the same as for check() -local release(name, r_major_minor, image) = - local r_bin_var = "$${" + get_r_exe_var(r_major_minor) + "}"; - - local image_uri = create_ci_image(ecr_repo_base, image); - - local host_volume = volume(default_volume_name, default_volume_path); - local temp_volume = volume("cache", temp_volume_dir); - - setup_docker_pipeline(name + '-release') + - add_volumes([host_volume], [temp_volume]) + - add_trigger(include=["tag"]) + - { - "steps": [ - pull_image(image_uri, [host_volume]), - { - "name": "Build package", - "image": image_uri, - "pull": "never", - "volumes": [add_step_volume(v) for v in [temp_volume]], - "environment": r_env_vars + { - "R_LIBS_USER": "/opt/rpkgs/" + r_major_minor, - }, - "commands": [ - # git config needs to sit next to pkgpub - "git config --global user.email " + default_git_email, - "git config --global user.name " + default_git_user, - "git fetch --tags", - "export PATH=" + temp_volume.path + ":$PATH", - source_env(), - run_r_expression( - r_bin_var, - std.format( - "pkgpub::create_tagged_repo(.dir = '%s')", - temp_volume.path - ) - ), - ], - }, - s3_publish_tag("${DRONE_TAG}", "${DRONE_TAG}", temp_volume), - s3_publish_tag("${DRONE_TAG}", "latest_tag", temp_volume), - ], - }; - -######################################## -## END PIPELINE BLOCK ## -######################################## - -######################################## -## START DRONE CONFIG BLOCK ## -######################################## - -[ - check(name, image) - for image in ci_images -] + [ - # lint(name, r_versions[0], ci_images[0]), - # release step requires all check steps to pass - release(name, r_versions[0], ci_images[0]) + - { - "depends_on": [ - create_build_tag(name, image) - for image in ci_images - ], - }, -] - -######################################## -## END DRONE CONFIG BLOCK ## -######################################## diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 31d358ef..00000000 --- a/.drone.yml +++ /dev/null @@ -1,138 +0,0 @@ ---- -kind: pipeline -type: docker -name: cran-latest - -platform: - os: linux - arch: amd64 - -steps: -- name: pull - image: omerxx/drone-ecr-auth - commands: - - $(aws ecr get-login --no-include-email --region us-east-1) - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-3.6:cran-latest - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.0:cran-latest - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:cran-latest - volumes: - - name: docker.sock - path: /var/run/docker.sock - -- name: "Check package: R 3.6" - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-3.6:cran-latest - commands: - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'devtools::check(env_vars = c("NOT_CRAN" = "true"))' - environment: - USER: drone - volumes: - - name: cache - path: /ephemeral - -- name: "Check package: R 4.0" - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.0:cran-latest - commands: - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'devtools::check(env_vars = c("NOT_CRAN" = "true"))' - environment: - USER: drone - volumes: - - name: cache - path: /ephemeral - -- name: "Check package: R 4.1" - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:cran-latest - commands: - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'devtools::check(env_vars = c("NOT_CRAN" = "true"))' - environment: - USER: drone - volumes: - - name: cache - path: /ephemeral - -volumes: -- name: docker.sock - host: - path: /var/run/docker.sock -- name: cache - temp: {} - -trigger: - event: - exclude: - - promote - ---- -kind: pipeline -type: docker -name: yspec-release - -platform: - os: linux - arch: amd64 - -steps: -- name: pull - image: omerxx/drone-ecr-auth - commands: - - $(aws ecr get-login --no-include-email --region us-east-1) - - docker pull 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:latest - volumes: - - name: docker.sock - path: /var/run/docker.sock - -- name: Build package - pull: never - image: 906087756158.dkr.ecr.us-east-1.amazonaws.com/r-dev-ci-mpn-4.1:latest - commands: - - git config --global user.email drone@metrumrg.com - - git config --global user.name Drony - - git fetch --tags - - R -s -e 'devtools::install_deps(upgrade = '"'"'always'"'"')' - - R -s -e 'pkgpub::create_tagged_repo(.dir = '"'"'/ephemeral'"'"')' - volumes: - - name: cache - path: /ephemeral - -- name: "Publish package: ${DRONE_TAG}" - pull: if-not-exists - image: plugins/s3 - settings: - bucket: mpn.metworx.dev - source: /ephemeral/${DRONE_TAG}/**/* - strip_prefix: /ephemeral/${DRONE_TAG}/ - target: /releases/${DRONE_REPO_NAME}/${DRONE_TAG} - volumes: - - name: cache - path: /ephemeral - -- name: "Publish package: latest_tag" - pull: if-not-exists - image: plugins/s3 - settings: - bucket: mpn.metworx.dev - source: /ephemeral/${DRONE_TAG}/**/* - strip_prefix: /ephemeral/${DRONE_TAG}/ - target: /releases/${DRONE_REPO_NAME}/latest_tag - volumes: - - name: cache - path: /ephemeral - -volumes: -- name: docker.sock - host: - path: /var/run/docker.sock -- name: cache - temp: {} - -trigger: - event: - - tag - -depends_on: -- cran-latest \ No newline at end of file diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..eea1897d --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,86 @@ +name: CI +on: + push: + branches: + - 'main' + - 'scratch/**' + tags: + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.[0-9]+' + pull_request: + +jobs: + check: + runs-on: ${{ matrix.config.os }} + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + strategy: + fail-fast: false + matrix: + config: + - os: ubuntu-20.04 + r: 3.6.3 + - os: ubuntu-20.04 + r: 4.0.5 + - os: ubuntu-20.04 + r: 4.1.3 + - os: ubuntu-latest + r: release + env: + R_KEEP_PKG_SOURCE: yes + steps: + - uses: actions/checkout@v4 + - uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + use-public-rspm: true + env: + RSPM: ${{ matrix.config.rspm }} + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + - uses: r-lib/actions/check-r-package@v2 + release: + if: github.ref_type == 'tag' + name: Upload release + needs: check + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + use-public-rspm: true + # For pkgpub. + extra-repositories: 'https://mpn.metworx.com/snapshots/stable/2024-03-01' + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgpub + - name: Configure Git + shell: bash + run: | + git config --global user.email CI + git config --global user.name ci@metrumrg + - name: Create output directory + shell: bash + run: echo "REPO_DIR=$(mktemp -d)" >>$GITHUB_ENV + - name: Create CRAN-like repo for release + run: pkgpub::create_tagged_repo("${{ env.REPO_DIR }}") + shell: Rscript {0} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::906087756158:role/github-actions-mpn-s3-publish + aws-region: us-east-1 + - name: 'Publish package: ${{ github.ref_name }}' + run: | + aws s3 sync \ + '${{ env.REPO_DIR }}/${{ github.ref_name }}/' \ + 's3://mpn.metworx.dev/releases/${{ github.event.repository.name }}/${{ github.ref_name }}/' + - name: 'Publish package: latest_tag' + run: | + aws s3 sync \ + '${{ env.REPO_DIR }}/${{ github.ref_name }}/' \ + 's3://mpn.metworx.dev/releases/${{ github.event.repository.name }}/latest_tag/'