Fix CTAS for non-hdfs storages, also fixes multi storage cases #1008
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Gradle Build, Tag, and Publish OpenHouse | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| jobs: | |
| build-gradle-project: | |
| name: Build tagged commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout project sources | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.merge_commit_sha }} | |
| fetch-depth: 0 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v1 | |
| with: | |
| distribution: 'microsoft' | |
| java-version: '17' | |
| - name: Validate Gradle Wrapper | |
| uses: gradle/wrapper-validation-action@v1 | |
| - name: Build with Gradle | |
| run: ./gradlew clean build | |
| - name: Start Docker Containers | |
| run: docker compose -f infra/recipes/docker-compose/oh-only/docker-compose.yml up -d --build | |
| - name: Wait for Docker Containers to start | |
| run: sleep 30 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| run: pip install -r scripts/python/requirements.txt | |
| - name: Run Integration Tests | |
| run: python scripts/python/integration_test.py ./tables-test-fixtures/tables-test-fixtures-iceberg-1.2/src/main/resources/dummy.token | |
| - name: Stop Docker Containers | |
| run: docker compose -f infra/recipes/docker-compose/oh-only/docker-compose.yml down | |
| - name: Bump version and push tag | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| uses: anothrNick/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| WITH_V: true # prefix for tag "v" | |
| DEFAULT_BUMP: patch # major, minor, patch | |
| DRY_RUN: false # if true, will not push tag | |
| INITIAL_VERSION: 0.5.0 # if no tags, will use this version | |
| - name: Get the latest tag | |
| id: get_tag | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| run: | | |
| latest_tag=$(git describe --tags --abbrev=0 main) | |
| semVer=${latest_tag:1} # Remove the first character ('v') | |
| echo "semVer=${semVer}" >> "$GITHUB_OUTPUT" | |
| - name: Publish with Gradle | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| env: | |
| JFROG_USERNAME: ${{ secrets.JFROG_USERNAME }} | |
| JFROG_PASSWORD: ${{ secrets.JFROG_PASSWORD }} | |
| run: ./gradlew publish -Pversion=${{ steps.get_tag.outputs.semVer }} | |
| - name: Login to LinkedIn Jfrog Docker artifactory | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: linkedin-openhouse-docker.jfrog.io | |
| username: ${{ secrets.JFROG_DOCKER_USERNAME }} | |
| password: ${{ secrets.JFROG_DOCKER_PASSWORD }} | |
| - name: Build and push Docker image for Tables Service | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| run: | | |
| docker build -t linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/tables-service:${{ steps.get_tag.outputs.semVer }} -f tables-service.Dockerfile . | |
| docker push linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/tables-service:${{ steps.get_tag.outputs.semVer }} | |
| - name: Build and push Docker image for House Tables Service | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| run: | | |
| docker build -t linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/housetables-service:${{ steps.get_tag.outputs.semVer }} -f housetables-service.Dockerfile . | |
| docker push linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/housetables-service:${{ steps.get_tag.outputs.semVer }} | |
| - name: Build and push Docker image for Jobs Service | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| run: | | |
| docker build -t linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/jobs-service:${{ steps.get_tag.outputs.semVer }} -f jobs-service.Dockerfile . | |
| docker push linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/jobs-service:${{ steps.get_tag.outputs.semVer }} | |
| - name: Build and push Docker image for Jobs Scheduler | |
| if: ${{ success() && github.ref == 'refs/heads/main' && github.repository == 'linkedin/openhouse' }} | |
| run: | | |
| docker build -t linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/jobs-scheduler:${{ steps.get_tag.outputs.semVer }} -f jobs-scheduler.Dockerfile . | |
| docker push linkedin-openhouse-docker.jfrog.io/linkedin/openhouse/jobs-scheduler:${{ steps.get_tag.outputs.semVer }} |