From 4e1c1e7afdacadbdb248e9464eaf1ee06d5faa72 Mon Sep 17 00:00:00 2001 From: Yousaf Nabi Date: Mon, 15 Jul 2024 22:15:59 +0100 Subject: [PATCH] feat: create os/arch specific npm optional deps for pact_ffi lib --- .github/workflows/build-and-test.yml | 11 ++-- .gitignore | 3 ++ .npmignore | 7 +++ DEVELOPER.md | 48 +++-------------- Makefile | 78 ++++++++++++++++++++++++++++ package-lock.json | 8 ++- package.json | 8 ++- package.json.tmpl | 18 +++++++ script/ci/build-and-test.sh | 5 +- script/ci/lib/publish.sh | 7 +++ script/ci/prebuild.sh | 2 +- script/ci/unpack-and-test.sh | 4 +- src/ffi/index.ts | 32 +++++++++++- 13 files changed, 180 insertions(+), 51 deletions(-) create mode 100644 Makefile create mode 100644 package.json.tmpl diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b042a1e7..4dfc1954 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -90,9 +90,10 @@ jobs: if: ${{ matrix.docker != true }} - name: Upload prebuild for ${{ runner.os }}-${{ runner.arch }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: prebuilds/*.tar.gz + name: artifact-${{ matrix.docker == true && matrix.alpine == true && 'linux-musl' || matrix.docker == true && matrix.alpine == false && 'linux' || matrix.os }}-${{ matrix.arch }} - run: GH_PRE_RELEASE_UPLOAD=true ./script/ci/release.sh if: github.ref == 'refs/heads/master' && env.ACT != true @@ -165,7 +166,7 @@ jobs: fetch-depth: 0 - name: Download prebuilds - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Use Node.js ${{ env.NODE_VERSION }} uses: actions/setup-node@v4 @@ -186,15 +187,15 @@ jobs: - if: ${{ matrix.docker == true && matrix.alpine != true && matrix.arch == 'arm64' }} name: test arm64 - run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }} bin/bash -c 'cd /home && /home/script/ci/unpack-and-test.sh' + run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }} bin/bash -c 'apt update -y && apt-get install -y gettext-base make jq && cd /home && /home/script/ci/unpack-and-test.sh' - if: ${{ matrix.docker == true && matrix.alpine == true && matrix.arch == 'amd64' }} name: test linux amd64 musl - run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add bash curl gcompat file && cd /home && /home/script/ci/unpack-and-test.sh' + run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add bash curl gcompat file make gettext jq && cd /home && /home/script/ci/unpack-and-test.sh' - if: ${{ matrix.docker == true && matrix.alpine == true && matrix.arch == 'arm64' }} name: test linux arm64 musl - run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add bash curl file protoc protobuf-dev && cd /home && /home/script/ci/unpack-and-test.sh' + run: docker run -v $PWD:/home --platform linux/${{ matrix.arch }} --rm node:${{ matrix.node-version }}-alpine bin/sh -c 'apk add bash curl file protoc protobuf-dev make gettext jq && cd /home && /home/script/ci/unpack-and-test.sh' release_dry_run: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index b428edab..b80859c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# platform-arch specific packages +@pact-foundation/* + # Logs logs npm-debug.log diff --git a/.npmignore b/.npmignore index aa54c8e6..157f6bd7 100644 --- a/.npmignore +++ b/.npmignore @@ -1,3 +1,10 @@ +# Standalone Binaries - Published as seperate packages +@pact-foundation/ + +# Cross packaging files +Makefile +package.json.tmpl + # Logs logs npm-debug.log diff --git a/DEVELOPER.md b/DEVELOPER.md index 4a90971f..78d17adc 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -6,7 +6,15 @@ Do this and you should be 👌👌👌: ``` bash script/ci/prebuild.sh +# Alternatively, you can download the latest github release prebuilds from the pact-js-core project +FETCH_ASSETS=true ./script/ci/check-release-libs.sh --fetch-assets -t v15.1.0 +# Make os/arch specific npm packages, with newly created prebuilds +make all npm ci --ignore-scripts +# Link os/arch specific npm package, for running os/arch system +make link +# Update main package.json optional dependencies versions, with those created earlier +make update_opt_deps npm run build npm test ``` @@ -50,43 +58,3 @@ npm test ```sh act --container-architecture linux/amd64 -W .github/workflows/build-and-test.yml --artifact-server-path tmp ``` - -### MacOS ARM64 Task - -#### Pre Reqs - -1. Arm64 Mac -2. Cirrus-Cli -3. Tart.run - - -```sh -cirrus run --output github-actions macos_arm --artifacts-dir tmp -``` - -### Linux ARM64 Task - -#### Pre Reqs - -1. Arm64 Machine - -### CI Locally - -1. Arm64 Machine -2. Docker / Podman -3. Cirrus-Cli - - -```sh -cirrus run --output github-actions linux_arm --artifacts-dir tmp -``` - -#### Publishing Assets - -MacOS ARM64 - -`cirrus run --output github-actions macos_arm --artifacts-dir tmp --environment GITHUB_TOKEN=$GITHUB_TOKEN --environment CIRRUS_RELEASE=test --environment CIRRUS_REPO_FULL_NAME=pact-foundation/pact-js-core;` - -Linux ARM64 - -`cirrus run --output github-actions linux_arm --artifacts-dir tmp --environment GITHUB_TOKEN=$GITHUB_TOKEN --environment CIRRUS_RELEASE=test --environment CIRRUS_REPO_FULL_NAME=pact-foundation/pact-js-core;` \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..891159c9 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +SHELL:=/bin/bash +export bin=@pact-foundation/pact-core +export pkg_version=$(shell cat package.json | jq -r .version) +supported_platforms = "linux-x64" "linux-arm64" "darwin-x64" "darwin-arm64" "windows-x64" + +# https://github.com/npm/npm/issues/17722 +# https://github.com/npm/cli/issues/4828 +# https://github.com/orhun/packaging-rust-for-npm +# https://blog.orhun.dev/packaging-rust-for-npm/ + +clean: + rm -rf @pact-foundation + npm run clean-libs +libs: + FETCH_ASSETS=true ./script/ci/check-release-libs.sh --fetch-assets -t v$(pkg_version) + +all: + for supported_platform in $(supported_platforms); do \ + IFS='-' read -r node_os node_arch <<< "$$supported_platform"; \ + export node_os=$$node_os; \ + export node_arch=$$node_arch; \ + export node_pkg=$(bin)-$$node_os-$$node_arch; \ + if [ "$$node_os" = "windows" ]; then \ + export node_os="win32"; \ + fi; \ + export standalone_package=prebuilds/$$node_os-$$node_arch; \ + echo "Building for $$node_os-$$node_arch"; \ + echo "Building $$node_pkg"; \ + mkdir -p "$$node_pkg/prebuilds"; \ + mv "$$standalone_package" "$$node_pkg/prebuilds"; \ + envsubst < package.json.tmpl > "$$node_pkg/package.json"; \ + (cd $$node_pkg && npm publish --access public --dry-run;)\ + done + +update_opt_deps: + @for supported_platform in $(supported_platforms); do \ + IFS='-' read -r node_os node_arch <<< "$$supported_platform"; \ + export node_pkg=$(bin)-$$node_os-$$node_arch; \ + jq '.optionalDependencies."'$$node_pkg'" = "$(pkg_version)"' package.json > package-new.json; \ + mv package-new.json package.json; \ + done + +dry_run: + set -eu; for supported_platform in $(supported_platforms); do \ + IFS='-' read -r node_os node_arch <<< "$$supported_platform"; \ + export node_os=$$node_os; \ + export node_pkg=$(bin)-$$node_os-$$node_arch; \ + export node_arch=$$node_arch; \ + if [ "$$node_os" = "windows" ]; then \ + export node_os="win32"; \ + fi; \ + export standalone_package=prebuilds/$$node_os-$$node_arch; \ + echo "Building $$node_pkg for $$node_os-$$node_arch (dry-run)"; \ + (cd $$node_pkg && npm publish --access public --dry-run;)\ + done +publish: + set -eu; for supported_platform in $(supported_platforms); do \ + IFS='-' read -r node_os node_arch <<< "$$supported_platform"; \ + export node_os=$$node_os; \ + export node_arch=$$node_arch; \ + export node_pkg=$(bin)-$$node_os-$$node_arch; \ + if [ "$$node_os" = "windows" ]; then \ + export node_os="win32"; \ + fi; \ + export standalone_package=prebuilds/$$node_os-$$node_arch; \ + echo "Building for $$node_os-$$node_arch"; \ + echo "Building $$node_pkg for $$node_os-$$node_arch"; \ + (cd $$node_pkg && npm publish --access public;)\ + done +link: + set -eu; for supported_platform in $(supported_platforms); do \ + IFS='-' read -r node_os node_arch <<< "$$supported_platform"; \ + export node_os=$$node_os; \ + export node_arch=$$node_arch; \ + export node_pkg=$(bin)-$$node_os-$$node_arch; \ + (cd $$node_pkg && npm link || echo "cannot link for platform";);\ + npm link $$node_pkg || echo "cannot link for platform";\ + done diff --git a/package-lock.json b/package-lock.json index c79b81ca..ca67b447 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "15.1.0", "cpu": [ "x64", - "ia32", "arm64" ], "license": "MIT", @@ -81,6 +80,13 @@ }, "engines": { "node": ">=16" + }, + "optionalDependencies": { + "@pact-foundation/pact-core-darwin-arm64": "15.1.0", + "@pact-foundation/pact-core-darwin-x64": "15.1.0", + "@pact-foundation/pact-core-linux-arm64": "15.1.0", + "@pact-foundation/pact-core-linux-x64": "15.1.0", + "@pact-foundation/pact-core-windows-x64": "15.1.0" } }, "node_modules/@aashutoshrathi/word-wrap": { diff --git a/package.json b/package.json index 950f152e..889c65ab 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ ], "cpu": [ "x64", - "ia32", "arm64" ], "engines": { @@ -42,6 +41,13 @@ "publishConfig": { "access": "public" }, + "optionalDependencies": { + "@pact-foundation/pact-core-darwin-arm64": "15.1.0", + "@pact-foundation/pact-core-darwin-x64": "15.1.0", + "@pact-foundation/pact-core-windows-x64": "15.1.0", + "@pact-foundation/pact-core-linux-x64": "15.1.0", + "@pact-foundation/pact-core-linux-arm64": "15.1.0" + }, "dependencies": { "check-types": "7.3.0", "node-gyp-build": "^4.6.0", diff --git a/package.json.tmpl b/package.json.tmpl new file mode 100644 index 00000000..c1af56b5 --- /dev/null +++ b/package.json.tmpl @@ -0,0 +1,18 @@ +{ + "name": "${node_pkg}", + "version": "${pkg_version}", + "description": "${node_os}-${node_arch} Platform specific Core of @pact-foundation/pact. You almost certainly don't want to depend on this directly.", + "repository": { + "type": "git", + "url": "git://github.com/pact-foundation/pact-js-cli.git" + }, + "scripts": {}, + "author": "Yousaf Nabi ", + "homepage": "https://github.com/pact-foundation/pact-js-cli#readme", + "os": [ + "${node_os}" + ], + "cpu": [ + "${node_arch}" + ] +} \ No newline at end of file diff --git a/script/ci/build-and-test.sh b/script/ci/build-and-test.sh index abc3c01b..7f3a53e6 100755 --- a/script/ci/build-and-test.sh +++ b/script/ci/build-and-test.sh @@ -21,7 +21,10 @@ node --version npm --version npm ci --ignore-scripts - +# Link os/arch specific npm package, for running os/arch system +make link +# Update main package.json optional dependencies versions, with those created earlier +make update_opt_deps npm run format:check npm run lint npm run build diff --git a/script/ci/lib/publish.sh b/script/ci/lib/publish.sh index 0020219a..acd06083 100755 --- a/script/ci/lib/publish.sh +++ b/script/ci/lib/publish.sh @@ -14,12 +14,19 @@ echo "--> Releasing artifacts" echo " Publishing pact-core@${VERSION}..." if [[ ${DRY_RUN:-} == 'true' ]]; then echo "publishing in dry run mode" + # Dry-run Publish os/arch specific npm packages + make dry_run + # Update main package.json optional dependencies versions, with those created earlier + make update_opt_deps npm publish --access-public --dry-run else echo "--> Preparing npmrc file" "$SCRIPT_DIR"/create_npmrc_file.sh echo "--> Removing binding.gyp to prevent rebuild. See https://github.com/npm/cli/issues/5234#issuecomment-1291139150" rm "${SCRIPT_DIR}/../../../binding.gyp" + # Update main package.json optional dependencies versions, with those created earlier + make publish + make update_opt_deps npm publish --access public --tag latest fi echo " done!" diff --git a/script/ci/prebuild.sh b/script/ci/prebuild.sh index c3c5b0a3..b5df85e7 100755 --- a/script/ci/prebuild.sh +++ b/script/ci/prebuild.sh @@ -1,4 +1,4 @@ -# !/bin/bash -eu +#!/bin/bash -eu set -e # This needs to be here for windows bash, which doesn't read the #! line above set -u diff --git a/script/ci/unpack-and-test.sh b/script/ci/unpack-and-test.sh index b21ad2c3..ca226854 100755 --- a/script/ci/unpack-and-test.sh +++ b/script/ci/unpack-and-test.sh @@ -6,9 +6,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)" # Figure out where the . "$SCRIPT_DIR"/../lib/robust-bash.sh ls -1 -ls -1 artifact +ls -1 artifact* mkdir -p prebuilds mv artifact*/*.tar.gz . || echo "no mac prebuilds" ls *.gz |xargs -n1 tar -xzf +# Make os/arch specific npm packages, with newly created prebuilds +make all "$SCRIPT_DIR"/../download-plugins.sh "$SCRIPT_DIR"/build-and-test.sh \ No newline at end of file diff --git a/src/ffi/index.ts b/src/ffi/index.ts index ac217c2f..2d7f0b20 100644 --- a/src/ffi/index.ts +++ b/src/ffi/index.ts @@ -6,6 +6,30 @@ import { Ffi } from './types'; export const PACT_FFI_VERSION = '0.4.21'; +/** + * Returns the library path which is located inside `node_modules` + * The naming convention is @pact-foundation/pact-core-${os}-${arch} + * @see https://nodejs.org/api/os.html#osarch + * @see https://nodejs.org/api/os.html#osplatform + * @example "x/xx/node_modules/@pact-foundation/pact-core-darwin-arm64" + */ +function getPlatformArchSpecificPackage() { + const { arch } = process; + let os = process.platform as string; + if (['win32', 'cygwin'].includes(process.platform)) { + os = 'windows'; + } + const platformArchSpecificPackage = `@pact-foundation/pact-core-${os}-${arch}`; + try { + require.resolve(`${platformArchSpecificPackage}/package.json`); + return platformArchSpecificPackage; + } catch (e) { + throw new Error( + `Couldn't find application binary for ${os}-${arch}:\n 💡 npm install --save-dev ${platformArchSpecificPackage}` + ); + } +} + // supported prebuilds // darwin-arm64 // darwin-x64 @@ -54,7 +78,13 @@ const bindingsResolver = (bindingsPath: string | undefined) => bindings(bindingsPath); const bindingPaths = [ - path.resolve(__dirname, '..', '..'), + path.resolve( + __dirname, + '..', + '..', + 'node_modules', + getPlatformArchSpecificPackage() + ), process.env['PACT_PREBUILD_LOCATION']?.toString() ?? path.resolve(), ]; let ffiLib: Ffi;