pr-chromatic-tests.yml

name: Run Chromatic Tests
concurrency:
  group: Run-Chromatic-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
  cancel-in-progress: true

on: push

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

jobs:
  playwright-tests:
    name: Playwright Tests

    # See https://runs-on.com/runners/linux/
    runs-on:
      [
        runs-on,
        runner=32cpu-linux-x64,
        disk=large,
        "run-id=${{ github.run_id }}",
      ]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: |
            backend/requirements/default.txt
            backend/requirements/dev.txt
            backend/requirements/model_server.txt
      - run: |
          python -m pip install --upgrade pip
          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
          pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt

      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install node dependencies
        working-directory: ./web
        run: npm ci

      - name: Install playwright browsers
        working-directory: ./web
        run: npx playwright install --with-deps

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}

      # tag every docker image with "test" so that we can spin up the correct set
      # of images during testing

      # we use the runs-on cache for docker builds
      # in conjunction with runs-on runners, it has better speed and unlimited caching
      # https://runs-on.com/caching/s3-cache-for-github-actions/
      # https://runs-on.com/caching/docker/
      # https://github.com/moby/buildkit#s3-cache-experimental

      # images are built and run locally for testing purposes. Not pushed.

      - name: Build Web Docker image
        uses: ./.github/actions/custom-build-and-push
        with:
          context: ./web
          file: ./web/Dockerfile
          platforms: linux/amd64
          tags: onyxdotapp/onyx-web-server:test
          push: false
          load: true
          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max

      - name: Build Backend Docker image
        uses: ./.github/actions/custom-build-and-push
        with:
          context: ./backend
          file: ./backend/Dockerfile
          platforms: linux/amd64
          tags: onyxdotapp/onyx-backend:test
          push: false
          load: true
          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max

      - name: Build Model Server Docker image
        uses: ./.github/actions/custom-build-and-push
        with:
          context: ./backend
          file: ./backend/Dockerfile.model_server
          platforms: linux/amd64
          tags: onyxdotapp/onyx-model-server:test
          push: false
          load: true
          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max

      - name: Start Docker containers
        run: |
          cd deployment/docker_compose
          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
          AUTH_TYPE=basic \
          GEN_AI_API_KEY=${{ secrets.OPENAI_API_KEY }} \
          REQUIRE_EMAIL_VERIFICATION=false \
          DISABLE_TELEMETRY=true \
          IMAGE_TAG=test \
          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
        id: start_docker

      - name: Wait for service to be ready
        run: |
          echo "Starting wait-for-service script..."

          docker logs -f danswer-stack-api_server-1 &

          start_time=$(date +%s)
          timeout=300  # 5 minutes in seconds

          while true; do
            current_time=$(date +%s)
            elapsed_time=$((current_time - start_time))
            
            if [ $elapsed_time -ge $timeout ]; then
              echo "Timeout reached. Service did not become ready in 5 minutes."
              exit 1
            fi
            
            # Use curl with error handling to ignore specific exit code 56
            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || echo "curl_error")
            
            if [ "$response" = "200" ]; then
              echo "Service is ready!"
              break
            elif [ "$response" = "curl_error" ]; then
              echo "Curl encountered an error, possibly exit code 56. Continuing to retry..."
            else
              echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
            fi
            
            sleep 5
          done
          echo "Finished waiting for service."

      - name: Run pytest playwright test init
        working-directory: ./backend
        env:
          PYTEST_IGNORE_SKIP: true
        run: pytest -s tests/integration/tests/playwright/test_playwright.py

      - name: Run Playwright tests
        working-directory: ./web
        run: npx playwright test

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          # Chromatic automatically defaults to the test-results directory.
          # Replace with the path to your custom directory and adjust the CHROMATIC_ARCHIVE_LOCATION environment variable accordingly.
          name: test-results
          path: ./web/test-results
          retention-days: 30

      # save before stopping the containers so the logs can be captured
      - name: Save Docker logs
        if: success() || failure()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
          mv docker-compose.log ${{ github.workspace }}/docker-compose.log

      - name: Upload logs
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: docker-logs
          path: ${{ github.workspace }}/docker-compose.log

      - name: Stop Docker containers
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack down -v

  chromatic-tests:
    name: Chromatic Tests

    needs: playwright-tests
    runs-on:
      [
        runs-on,
        runner=32cpu-linux-x64,
        disk=large,
        "run-id=${{ github.run_id }}",
      ]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install node dependencies
        working-directory: ./web
        run: npm ci

      - name: Download Playwright test results
        uses: actions/download-artifact@v4
        with:
          name: test-results
          path: ./web/test-results

      - name: Run Chromatic
        uses: chromaui/action@latest
        with:
          playwright: true
          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
          workingDir: ./web
        env:
          CHROMATIC_ARCHIVE_LOCATION: ./test-results