chore: release pipeline (#9)

Signed-off-by: usamoi <[email protected]>
tensorchord · Jun 5, 2024 · 43dd5cd · 43dd5cd
1 parent d270f23
commit 43dd5cd
Show file tree

Hide file tree

Showing 30 changed files with 486,150 additions and 12 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,117 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        description: Version
+        required: true
+      tag:
+        type: string
+        description: Tag
+        required: true
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  SCCACHE_GHA_ENABLED: true
+  RUSTC_WRAPPER: sccache
+  RUSTFLAGS: "-Dwarnings"
+
+permissions:
+  contents: write
+  pull-requests: read
+
+jobs:
+  semver:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const r = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/;
+            if (!r.test("${{ github.event.inputs.version }}")) {
+              core.setFailed(`Action failed with an invalid semver.`);
+            }
+  binary:
+    strategy:
+      matrix:
+        version: [12, 13, 14, 15, 16]
+        arch: ["x86_64", "aarch64"]
+    runs-on: ubuntu-20.04
+    env:
+      SEMVER: ${{ github.event.inputs.version }}
+      VERSION: ${{ matrix.version }}
+      ARCH: ${{ matrix.arch }}
+    needs: ["semver"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Environment
+        run: |
+          sudo apt-get remove -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*'
+          sudo apt-get purge -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*'
+          sudo apt-get update
+          sudo apt-get install -y build-essential crossbuild-essential-arm64
+          sudo apt-get install -y qemu-user-static
+          echo 'target.aarch64-unknown-linux-gnu.linker = "aarch64-linux-gnu-gcc"' | tee ~/.cargo/config.toml
+      - name: Set up Sccache
+        uses: mozilla-actions/[email protected]
+      - name: Set up Cache
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ github.job }}-${{ hashFiles('./Cargo.lock') }}-${{ matrix.version }}-${{ matrix.arch }}
+      - name: Set up Clang-16
+        run: |
+          sudo sh -c 'echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-16 main" >> /etc/apt/sources.list'
+          wget --quiet -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+          sudo apt-get update
+          sudo apt-get install -y clang-16
+      - name: Set up Pgrx
+        run: |
+          # pg_config
+          mkdir -p ~/.pg_config
+          touch ~/.pg_config/pg_config
+          chmod 777 ~/.pg_config/pg_config
+          echo "#!/usr/bin/env bash" >> ~/.pg_config/pg_config
+          echo "$(pwd)/tools/pg_config.sh \"\$@\" < $(pwd)/vendor/pg_config/pg${VERSION}_${ARCH}-unknown-linux-gnu.txt" >> ~/.pg_config/pg_config
+          mkdir -p ~/.pgrx && echo "configs.pg$VERSION=\"$HOME/.pg_config/pg_config\"" > ~/.pgrx/config.toml
+          # pgrx_binding
+          mkdir -p ~/.pgrx_binding
+          cp ./vendor/pgrx_binding/pg${VERSION}_$(uname --machine)-unknown-linux-gnu.rs ~/.pgrx_binding/pg${VERSION}_raw_bindings.rs
+          echo PGRX_TARGET_INFO_PATH_PG$VERSION=$HOME/.pgrx_binding >> "$GITHUB_ENV"
+      - name: Build
+        run: |
+          cargo build --lib --no-default-features --features pg$VERSION --release --target $ARCH-unknown-linux-gnu
+          ./tools/schema.sh --no-default-features --features pg$VERSION --release --target $ARCH-unknown-linux-gnu | expand -t 4 > ./target/pg_bestmatch--$SEMVER.sql
+      - name: Package
+        run: |
+          export PLATFORM=$(echo $ARCH | sed 's/aarch64/arm64/; s/x86_64/amd64/')
+          ./scripts/package.sh
+      - name: Upload
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          export TAG=${{ github.event.inputs.tag }}
+          export PLATFORM=$(echo $ARCH | sed 's/aarch64/arm64/; s/x86_64/amd64/')
+          gh release upload --clobber $TAG ./build/pg_bestmatch-pg${VERSION}_${ARCH}-unknown-linux-gnu_${SEMVER}.zip
+          gh release upload --clobber $TAG ./build/pg_bestmatch-pg${VERSION}_${SEMVER}_${PLATFORM}.deb
+      - name: Post Set up Cache
+        uses: actions/cache/save@v4
+        if: ${{ !steps.cache.outputs.cache-hit }}
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ github.job }}-${{ hashFiles('./Cargo.lock') }}-${{ matrix.version }}-${{ matrix.arch }}
diff --git a/.github/workflows/release_nightly.yml b/.github/workflows/release_nightly.yml
@@ -0,0 +1,25 @@
+name: Release Nightly
+
+on:
+  schedule:
+    # 00:00 UTC+8 -> 16:00
+    - cron: "0 16 * * *"
+  workflow_dispatch:
+
+permissions:
+  actions: write
+
+jobs:
+  trigger:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Trigger
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          commit_date=$(git log -1 --since="24 hours ago" --pretty=format:"%cI")
+          if [[ -n "$commit_date" ]]; then
+            gh workflow run release.yml -f version=$(TZ='Asia/Shanghai' date +"0.0.0-nightly.%Y%m%d") -f tag=v0.0.0-nightly
+          fi
diff --git a/.github/workflows/release_officially.yml b/.github/workflows/release_officially.yml
@@ -0,0 +1,24 @@
+name: Release Officially
+
+on:
+  release:
+    types:
+      - created
+      - edited
+
+permissions:
+  actions: write
+
+jobs:
+  trigger:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Trigger
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          R=${{ github.event.release.tag_name }}
+          V=${R:1}
+          gh workflow run release.yml -r $R -f version=$V -f tag=$R
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -10,6 +10,7 @@ on:
       - "Cargo.lock"
       - "Cargo.toml"
       - "pg_bestmatch.control"
+      - "vendor/**"
   pull_request:
     branches: ["main"]
     paths:
@@ -19,6 +20,7 @@ on:
       - "Cargo.lock"
       - "Cargo.toml"
       - "pg_bestmatch.control"
+      - "vendor/**"
   merge_group:
   workflow_dispatch:
 
@@ -38,7 +40,12 @@ jobs:
     strategy:
       matrix:
         version: [12, 13, 14, 15, 16]
+        arch: ["x86_64"]
     runs-on: ubuntu-latest
+    env:
+      SEMVER: "0.0.0"
+      VERSION: ${{ matrix.version }}
+      ARCH: ${{ matrix.arch }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -47,20 +54,45 @@ jobs:
           sudo apt-get remove -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*'
           sudo apt-get purge -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*'
           sudo apt-get update
-          sudo apt-get install -y build-essential
+          sudo apt-get install -y build-essential crossbuild-essential-arm64
+          sudo apt-get install -y qemu-user-static
+          touch ~/.cargo/config.toml
+          echo 'target.aarch64-unknown-linux-gnu.linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config.toml
+          echo 'target.aarch64-unknown-linux-gnu.runner = ["qemu-aarch64-static", "-L", "/usr/aarch64-linux-gnu"]' >> ~/.cargo/config.toml
       - name: Set up Sccache
         uses: mozilla-actions/[email protected]
-      - name: Set up PostgreSQL
-        run: |
-          sudo apt-get install -y postgresql-common
-          echo | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh
-          sudo apt update
-          sudo apt-get install -y postgresql-${{ matrix.version }} postgresql-server-dev-${{ matrix.version }}
+      - name: Set up Cache
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ github.job }}-${{ hashFiles('./Cargo.lock') }}-${{ matrix.version }}-${{ matrix.arch }}
       - name: Set up Pgrx
         run: |
-          cargo install cargo-pgrx --debug --branch v0.12.0-alpha.1-patch2 --git https://github.com/tensorchord/pgrx.git
-          cargo pgrx init --pg${{ matrix.version }}=$(which pg_config)
+          # pg_config
+          mkdir -p ~/.pg_config
+          touch ~/.pg_config/pg_config
+          chmod 777 ~/.pg_config/pg_config
+          echo "#!/usr/bin/env bash" >> ~/.pg_config/pg_config
+          echo "$(pwd)/tools/pg_config.sh \"\$@\" < $(pwd)/vendor/pg_config/pg${VERSION}_${ARCH}-unknown-linux-gnu.txt" >> ~/.pg_config/pg_config
+          mkdir -p ~/.pgrx && echo "configs.pg$VERSION=\"$HOME/.pg_config/pg_config\"" > ~/.pgrx/config.toml
+          # pgrx_binding
+          mkdir -p ~/.pgrx_binding
+          cp ./vendor/pgrx_binding/pg${VERSION}_$(uname --machine)-unknown-linux-gnu.rs ~/.pgrx_binding/pg${VERSION}_raw_bindings.rs
+          echo PGRX_TARGET_INFO_PATH_PG$VERSION=$HOME/.pgrx_binding >> "$GITHUB_ENV"
       - name: Clippy
-        run: cargo clippy --lib --features "pg${{ matrix.version }}"
+        run: cargo clippy --features "pg$VERSION" --target $ARCH-unknown-linux-gnu
       - name: Build
-        run: cargo build --lib --features "pg${{ matrix.version }}"
+        run: cargo build --lib --features "pg$VERSION" --target $ARCH-unknown-linux-gnu
+      - name: Post Set up Cache
+        uses: actions/cache/save@v4
+        if: ${{ !steps.cache.outputs.cache-hit }}
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ github.job }}-${{ hashFiles('./Cargo.lock') }}-${{ matrix.version }}-${{ matrix.arch }}
diff --git a/.github/workflows/update_vendor.yml b/.github/workflows/update_vendor.yml
@@ -0,0 +1,56 @@
+name: Update Vendor
+
+on:
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  SCCACHE_GHA_ENABLED: true
+  RUSTC_WRAPPER: sccache
+  RUSTFLAGS: "-Dwarnings"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  generate:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Generate
+        run: |
+          export BRANCH=$(grep -o 'pgrx = { git = "https://github.com/tensorchord/pgrx.git", branch = "[^"]*' Cargo.toml | cut -d '"' -f 4)
+          docker run --rm --platform linux/amd64 -v ./:/mnt/build \
+            -e "VERSION=12" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/amd64 -v ./:/mnt/build \
+            -e "VERSION=13" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/amd64 -v ./:/mnt/build \
+            -e "VERSION=14" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/amd64 -v ./:/mnt/build \
+            -e "VERSION=15" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/amd64 -v ./:/mnt/build \
+            -e "VERSION=16" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/arm64 -v ./:/mnt/build \
+            -e "VERSION=12" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/arm64 -v ./:/mnt/build \
+            -e "VERSION=13" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/arm64 -v ./:/mnt/build \
+            -e "VERSION=14" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/arm64 -v ./:/mnt/build \
+            -e "VERSION=15" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          docker run --rm --platform linux/arm64 -v ./:/mnt/build \
+            -e "VERSION=16" -e "BRANCH=$BRANCH" debian:buster bash /mnt/build/scripts/update_vendor.sh &
+          wait
+          sudo chown -R $USER ./vendor
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: 'chore: update vendor'
+          title: 'chore: update vendor'
+          body: 'Update vendor: `pg_config` contents and pgrx bindings.'
+          branch: update-vendor
diff --git a/.typos.toml b/.typos.toml
@@ -5,4 +5,8 @@ extend-ignore-re = [
 ]
 
 [files]
-extend-exclude = ["tokenizer/*.json"]
+extend-exclude = [
+    "tokenizer/*.json",
+    "vendor/pg_config/*.txt",
+    "vendor/pgrx_binding/*.rs",
+]
diff --git a/scripts/update_vendor.sh b/scripts/update_vendor.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e
+
+printf "VERSION = ${VERSION}\n"
+printf "BRANCH = ${BRANCH}\n"
+
+apt-get update
+apt-get install -y --no-install-recommends ca-certificates curl build-essential gnupg lsb-release wget git
+
+echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee -a /etc/apt/sources.list.d/pgdg.list
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+apt-get update
+apt-get install -y --no-install-recommends postgresql-${VERSION} postgresql-server-dev-${VERSION}
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+source ~/.cargo/env
+
+cd $(mktemp -d)
+
+cargo init --lib --name vectors
+cargo add pgrx-pg-sys --git https://github.com/tensorchord/pgrx.git --branch $BRANCH --no-default-features --features pg$VERSION
+PGRX_PG_CONFIG_PATH=$(which pg_config) PGRX_PG_SYS_EXTRA_OUTPUT_PATH=$(pwd)/pgrx-binding.rs cargo build
+rustfmt ./pgrx-binding.rs
+
+cp ./pgrx-binding.rs /mnt/build/vendor/pgrx_binding/pg${VERSION}_$(uname --machine)-unknown-linux-gnu.rs
+pg_config > /mnt/build/vendor/pg_config/pg${VERSION}_$(uname --machine)-unknown-linux-gnu.txt