cloudbuild.yaml

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,

steps:
- name: 'gcr.io/cloud-builders/docker'
  id: Pull docker cache
  entrypoint: 'bash'
  args:
  - '-c'
  - |
   docker pull gcr.io/$PROJECT_ID/airflow-dags-builder:latest || exit 0
- name: gcr.io/cloud-builders/docker
  id: Build Airflow DAGs Builder
  args: [
      'build',
      '-t', 'gcr.io/$PROJECT_ID/airflow-dags-builder',
      '--cache-from', 'gcr.io/$PROJECT_ID/airflow-dags-builder:latest',
      './'
    ]
  dir: ${_DIRECTORY}
- name: 'gcr.io/$PROJECT_ID/airflow-dags-builder'
  id: Validation Test
  # Validate the integrity of the DAG files.
  entrypoint: python
  env:
  - AIRFLOW__CORE__DAGS_FOLDER=/workspace/${_DIRECTORY}/dags
  args:
  - -m
  - unittest
  - tests/dag_integrity_test.py
  dir: ${_DIRECTORY}
- name: 'gcr.io/$PROJECT_ID/airflow-dags-builder'
  # Actually run some DAGs with test data, this type of test is less common and requires
  # more setup, creating test data sets, ensuring your task IO is configured
  # via airflow Variables, etc.
  # To succeed, these need to complete before the cloudbuild timeout.
  id: End to End Test
  entrypoint: bash
  env:
  - AIRFLOW__CORE__DAGS_FOLDER=/workspace/${_DIRECTORY}/dags
  dir: ${_DIRECTORY}
  # dir: examples/cloud-composer-cicd
  args:
  - -c
  - |
    pwd
    ls -l
    # Init an empty airflow metadata db
    airflow initdb
    # Initialise variables to configure airflow DAGs
    airflow variables -s input gs://test-input-bucket/
    airflow variables -s output gs://test-output-bucket/
    # Run a backfill of our DAG for a specific date, add more dags here if required.
    airflow backfill echo_and_wait --start_date=2020-01-03 --end_date=2020-01-03
    ret=$?
    # Cat the task logs so we can see when there was any error
    find /workspace/airflow/logs/ -type f -name *.log | sort | xargs cat
    # Fail if the backfill failed.
    exit $ret
- name: gcr.io/cloud-builders/gsutil
  # Deploy the DAGs to your composer environment DAGs GCS folder
  id: Deploy DAGs
  args:
  - -m
  - rsync
  - -r
  - -c
  - -x
  - .*\.pyc|airflow_monitoring.py
  - /workspace/${_DIRECTORY}/dags/
  - ${_DEPLOY_DAGS_LOCATION}
  dir: ${_DIRECTORY}
images: ['gcr.io/$PROJECT_ID/airflow-dags-builder:latest']