H-3614: Introduce chonky PDF Embeddings

.dockerignore

@@ -89,6 +89,9 @@ blocks/**/.env
 .env.local
 .env.*.local
 
+# Generated credentials from google-github-actions/auth
+gha-creds-*.json
+
 # macOS directory file
 **/.DS_Store
 

.github/workflows/test.yml

@@ -105,6 +105,11 @@ jobs:
       fail-fast: false
     if: needs.setup.outputs.unit-tests != '{"package":[],"include":[]}'
     runs-on: ubuntu-24.04
+    permissions:
+      # Required to fetch an OIDC token, used to auth with Google Cloud Platform for @rust/chonky tests
+      id-token: "write"
+      # Maintain permission to read repo contents
+      contents: "read"
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -148,6 +153,15 @@ jobs:
           rm -rf $temp_dir
           echo "PDFIUM_DYNAMIC_LIB_PATH=$(pwd)/${{ matrix.directory }}/libs/" >> $GITHUB_ENV
 
+      # Sets GOOGLE_APPLICATION_CREDENTIALS in the environment, to be consumed by gcloud or client libraries
+      - name: Generate Google Cloud credential configuration
+        if: matrix.package == '@rust/chonky'
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          project_id: ${{ secrets.GOOGLE_CLOUD_HASH_PROJECT_ID }}
+          service_account: ${{ secrets.GOOGLE_CLOUD_VERTEX_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ secrets.GOOGLE_CLOUD_IDENTITY_PROVIDER }}
+
       - name: Install Rust toolchain
         if: always() && steps.tests.outputs.has-rust == 'true'
         uses: ./.github/actions/install-rust-toolchain
@@ -178,6 +192,10 @@ jobs:
         continue-on-error: ${{ steps.tests.outputs.allow-failure == 'true' }}
         env:
           TEST_COVERAGE: ${{ github.event_name != 'merge_group' }}
+          # Variables needed for chonky tests
+          GOOGLE_PROJECT_ID: ${{ secrets.GOOGLE_CLOUD_HASH_PROJECT_ID }}
+          GOOGLE_DEFAULT_CREDENTIALS: ${{ env.GOOGLE_DEFAULT_CREDENTIALS }} # set by google-github-actions/auth
+          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
         run: |
           turbo run test:unit --env-mode=loose --filter "${{ matrix.package }}"
           echo "TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')" >> $GITHUB_ENV
@@ -225,6 +243,11 @@ jobs:
       fail-fast: false
     if: needs.setup.outputs.integration-tests != '{"package":[],"include":[]}'
     runs-on: ubuntu-24.04
+    permissions:
+      # Required to fetch an OIDC token, used to auth with Google Cloud Platform for @rust/chonky tests
+      id-token: "write"
+      # Maintain permission to read repo contents
+      contents: "read"
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -293,6 +316,15 @@ jobs:
           rm -rf $temp_dir
           echo "PDFIUM_DYNAMIC_LIB_PATH=$(pwd)/${{ matrix.directory }}/libs/" >> $GITHUB_ENV
 
+      # Sets GOOGLE_APPLICATION_CREDENTIALS in the environment, to be consumed by gcloud or client libraries
+      - name: Generate Google Cloud credential configuration
+        if: matrix.package == '@rust/chonky'
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          project_id: ${{ secrets.GOOGLE_CLOUD_HASH_PROJECT_ID }}
+          service_account: ${{ secrets.GOOGLE_CLOUD_VERTEX_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ secrets.GOOGLE_CLOUD_IDENTITY_PROVIDER }}
+
       - name: Install playwright
         if: matrix.package == '@tests/hash-playwright'
         uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
@@ -350,6 +382,11 @@ jobs:
 
       - name: Run tests
         continue-on-error: ${{ steps.tests.outputs.allow-failure == 'true' }}
+        env:
+          # Variables needed for chonky tests
+          GOOGLE_PROJECT_ID: ${{ secrets.GOOGLE_CLOUD_HASH_PROJECT_ID }}
+          GOOGLE_DEFAULT_CREDENTIALS: ${{ env.GOOGLE_DEFAULT_CREDENTIALS }} # set by google-github-actions/auth
+          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
         run: |
           turbo run test:integration --env-mode=loose --filter "${{ matrix.package }}"
           echo "TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')" >> $GITHUB_ENV

.gitignore

@@ -73,6 +73,9 @@ blocks/**/.env
 .env.local
 .env.*.local
 
+# Generated credentials from google-github-actions/auth
+gha-creds-*.json
+
 # macOS directory file
 **/.DS_Store
 

.markdownlintignore

@@ -82,6 +82,9 @@ blocks/**/.env
 .env.local
 .env.*.local
 
+# Generated credentials from google-github-actions/auth
+gha-creds-*.json
+
 # macOS directory file
 **/.DS_Store
 

Cargo.toml

@@ -93,6 +93,7 @@ ariadne            = { version = "=0.5.0", default-features = false }
 aws-types          = { version = "=1.3.3", default-features = false }
 axum               = { version = "0.7.5" }
 axum-core          = { version = "0.4.3" }
+base64             = { version = "=0.22.1" }
 bumpalo            = { version = "=3.16.0", default-features = false }
 bytes              = { version = "1.6.0" }
 clap_builder       = { version = "=4.5.21", default-features = false, features = ["std"] }

libs/chonky/Cargo.toml

@@ -17,8 +17,16 @@ edition.workspace = true
 error-stack = { workspace = true, public = true }
 
 # Public third-party dependencies
+base64        = { workspace = true }
+clap          = { workspace = true, features = ["derive"] }
+futures       = { workspace = true, features = ["alloc"] }
 image         = { workspace = true, public = true, features = ["png", "bmp"] }
-pdfium-render = { workspace = true, public = true }
+num-traits    = { workspace = true }
+pdfium-render = { workspace = true, features = ["thread_safe"], public = true }
+reqwest       = { workspace = true, features = ["json"] }
+serde         = { workspace = true, features = ["derive"] }
+serde_json    = { workspace = true }
+tokio         = { workspace = true, features = ["macros", "rt-multi-thread", "fs"] }
 
 # Private workspace dependencies