release-integrations.yml

name: Release integrations
on:
  push:
    branches:
      - main
  workflow_dispatch:

jobs:
  prepare-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.prepare-matrix.outputs.INTEGRATIONS_MATRIX }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Prepare matrix
        id: prepare-matrix
        run: |
          integrations_to_release=()
          # Get the list of integrations
          files=$(find integrations/*/.port -name "spec.yaml")
          for file in $files; do
            folder=$(dirname "$file")
            type=$(grep -E '^name = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2)

            # Get the version from pyproject.toml
            version=$(grep -E '^version = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2)

            # Check if the version exists in the ghcr.io registry
            rc=0
            docker manifest inspect ghcr.io/port-labs/port-ocean-$type:$version > /dev/null 2>&1 || rc=$?

            if [ $rc -eq 0 ]; then
              echo "Image already exists in $repository: port-ocean-$type:$version"
            else
              integrations_to_release+=($file)
            fi
          done
          echo $(echo ${integrations_to_release[@]} | jq -R -c 'split(" ")')
          echo "INTEGRATIONS_MATRIX=$(echo ${integrations_to_release[@]} | jq -R -c 'split(" ")')" >> $GITHUB_OUTPUT


  release-integration:
    runs-on: ubuntu-latest
    if: needs.prepare-matrix.outputs.matrix != '[]'
    outputs:
      is_dev_version: ${{ steps.prepare_tags.outputs.is_dev_version }}
    permissions:
      packages: write
      contents: read
    needs: [prepare-matrix]
    strategy:
      # limit the number of parallel jobs to avoid hitting the ghcr.io rate limit
      max-parallel: 5
      matrix:
        integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Prepare Docker images tags
        id: prepare_tags
        run: |
          current_integration_spec=${{ matrix.integration }}

          folder=$(dirname "$current_integration_spec")
          context_dir=$(dirname "$folder")
          echo "context_dir=$context_dir" >> $GITHUB_OUTPUT

          version=$(grep -E '^version = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2)
          type=$(grep -E '^name = ".*"' "$folder/../pyproject.toml" | cut -d'"' -f2)
          echo "version=$version" >> $GITHUB_OUTPUT

          dockerfile_path=integrations/_infra/Dockerfile
          if test -e $folder/../Dockerfile; then
            dockerfile_path=$folder/../Dockerfile
          fi
          echo "dockerfile_path=$dockerfile_path" >> $GITHUB_OUTPUT

          # Check if the 'version' variable contains any character other than digits and "."
          if [[ ! "$version" =~ ^[0-9.]+$ ]]; then
            # If 'version' contains non-numeric and non-dot characters, skip building 'latest' tag
            tags="ghcr.io/port-labs/port-ocean-$type:$version"
            echo "tags=$tags" >> $GITHUB_OUTPUT
            echo "is_dev_version=true" >> $GITHUB_OUTPUT
            echo "Version contains non-numeric characters. Building without 'latest' tag."
          else
            # If 'version' contains only digits and dots, build with both 'latest' and version tags
            tags="ghcr.io/port-labs/port-ocean-$type:$version,ghcr.io/port-labs/port-ocean-$type:latest"
            echo "tags=$tags" >> $GITHUB_OUTPUT
            echo "is_dev_version=false" >> $GITHUB_OUTPUT
          fi

      - name: Build Docker Image
        uses: ./.github/workflows/actions/build-docker-image
        with:
          dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.prepare_tags.outputs.tags }}
          build-args: |
            BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
            INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}

  upload-specs:
    runs-on: ubuntu-latest
    permissions:
      packages: write
      contents: read
    needs: release-integration
    steps:
      - name: Check out code
        uses: actions/checkout@v4
      - name: Configure AWS Credentials 🔒
        id: aws-credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}
      - name: Upload specifications to s3
        run: |
          # Temporary file
          temp_file="temp.yaml"

          # Output file name
          index_file="index.json"

          # AWS S3 bucket details
          aws_s3_bucket="ocean-registry"

          # Fetch existing index file or create empty one of not exists
          if aws s3 ls "s3://$aws_s3_bucket/$index_file" >/dev/null 2>&1; then
              aws s3 cp "s3://$aws_s3_bucket/$index_file" $index_file
              echo "Successfully fetched global index file from s3 bucket."
          else
              echo "Index file does not exist in the S3 bucket, Creating new one..."
              echo "[]" >"$index_file"
          fi

          # Find all ocean-spec.yaml files under the specified directory
          find integrations/*/.port -type f -name "spec.yaml" >file_list.txt

          while IFS= read -r file; do
              integration_dir=$(dirname "$file")
              integration_name=$(echo "$integration_dir" | awk -F'/' '{print $2}')

              # Extract the type for pyproject.toml
              type=$(grep -E '^name = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2)

              # Extract the version from pyproject.toml
              version=$(grep -E '^version = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2)

              integration_spec="$integration_name-$version.json"
              integration_dest="$integration_name/$integration_spec"
              integration_legacy_dest="$integration_name.json" # TODO: legacy support, remove once not in use

              # Convert YAML to JSON
              yq -o json "$file" >"$temp_file"

              # Add version attribute
              jq --arg type "$type" --arg version "$version" '. + {type: $type, version: $version}' "$temp_file" >"$integration_spec"

              if [[ ! ${version} =~ ^.+-dev$ ]]; then
                  # Upload integration's version manifest to s3
                  aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_dest"
                  aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_legacy_dest" # TODO: legacy support, remove once not in use
                  echo "Successfully uploaded $integration_spec to s3 bucket."

                  # Get the latest version of the current integration
                  latest_version=$(jq --arg type "$type" -r '.[] | select(.type == $type) | .version' "$index_file")

                  # Add integration's spec to global index file
                  regexp="^[0-9.]+$"
                  if [[ ! "$latest_version" ]]; then
                      # Add new integration spec if latest tag doesn't exist
                      jq --argjson new_spec "[$(cat "$integration_spec")]" '. += $new_spec' "$index_file" >"$temp_file"
                      mv "$temp_file" "$index_file"
                  elif [[ ! "$latest_version" =~ $regexp ]] || [[ "$version" =~ $regexp ]]; then
                      # Override global index file if released non-dev version or integration doesn't have non-dev latest tag
                      jq --argjson updated_spec "$(cat "$integration_spec")" --arg type "$type" \
                          'map(if .type == $type then $updated_spec else . end)' "$index_file" >"$temp_file"
                      mv "$temp_file" "$index_file"
                  fi

                  # Upload global index file to s3
                  aws s3 cp "$index_file" "s3://$aws_s3_bucket/$index_file"
                  echo "Successfully uploaded $index_file to s3 bucket."
              else
                  echo "Did not upload integration spec since the version was in development ('${version}')"
              fi

          done <file_list.txt