From c0c6921e8c3d49165dfdac8e5866815ffbcfb082 Mon Sep 17 00:00:00 2001 From: Bitnami Bot Date: Thu, 31 Oct 2024 16:54:05 +0000 Subject: [PATCH] [bitnami/airflow] Release airflow-2.10.2-debian-12-r5 Signed-off-by: Bitnami Bot --- bitnami/airflow/2/debian-12/Dockerfile | 8 +- .../opt/bitnami/.bitnami_components.json | 4 +- .../rootfs/opt/bitnami/scripts/airflow-env.sh | 2 + .../rootfs/opt/bitnami/scripts/libairflow.sh | 15 ++- bitnami/airflow/README.md | 95 ++++++++++--------- 5 files changed, 67 insertions(+), 57 deletions(-) diff --git a/bitnami/airflow/2/debian-12/Dockerfile b/bitnami/airflow/2/debian-12/Dockerfile index ed679a52fa4af..383350dae9392 100644 --- a/bitnami/airflow/2/debian-12/Dockerfile +++ b/bitnami/airflow/2/debian-12/Dockerfile @@ -8,11 +8,11 @@ ARG TARGETARCH LABEL com.vmware.cp.artifact.flavor="sha256:c50c90cfd9d12b445b011e6ad529f1ad3daea45c26d20b00732fae3cd71f6a83" \ org.opencontainers.image.base.name="docker.io/bitnami/minideb:bookworm" \ - org.opencontainers.image.created="2024-10-30T14:53:31Z" \ + org.opencontainers.image.created="2024-10-31T15:31:42Z" \ org.opencontainers.image.description="Application packaged by Broadcom, Inc." \ org.opencontainers.image.documentation="https://github.com/bitnami/containers/tree/main/bitnami/airflow/README.md" \ org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.ref.name="2.10.2-debian-12-r4" \ + org.opencontainers.image.ref.name="2.10.2-debian-12-r5" \ org.opencontainers.image.source="https://github.com/bitnami/containers/tree/main/bitnami/airflow" \ org.opencontainers.image.title="airflow" \ org.opencontainers.image.vendor="Broadcom, Inc." \ @@ -30,9 +30,9 @@ RUN install_packages ca-certificates curl git krb5-user libbz2-1.0 libcom-err2 l RUN mkdir -p /tmp/bitnami/pkg/cache/ ; cd /tmp/bitnami/pkg/cache/ ; \ COMPONENTS=( \ "wait-for-port-1.0.8-6-linux-${OS_ARCH}-debian-12" \ - "python-3.12.7-3-linux-${OS_ARCH}-debian-12" \ + "python-3.12.7-4-linux-${OS_ARCH}-debian-12" \ "ini-file-1.4.7-6-linux-${OS_ARCH}-debian-12" \ - "airflow-2.10.2-1-linux-${OS_ARCH}-debian-12" \ + "airflow-2.10.2-2-linux-${OS_ARCH}-debian-12" \ ) ; \ for COMPONENT in "${COMPONENTS[@]}"; do \ if [ ! -f "${COMPONENT}.tar.gz" ]; then \ diff --git a/bitnami/airflow/2/debian-12/prebuildfs/opt/bitnami/.bitnami_components.json b/bitnami/airflow/2/debian-12/prebuildfs/opt/bitnami/.bitnami_components.json index 539d30c4530e5..3c045df3000bb 100644 --- a/bitnami/airflow/2/debian-12/prebuildfs/opt/bitnami/.bitnami_components.json +++ b/bitnami/airflow/2/debian-12/prebuildfs/opt/bitnami/.bitnami_components.json @@ -3,7 +3,7 @@ "arch": "amd64", "distro": "debian-12", "type": "NAMI", - "version": "2.10.2-1" + "version": "2.10.2-2" }, "ini-file": { "arch": "amd64", @@ -15,7 +15,7 @@ "arch": "amd64", "distro": "debian-12", "type": "NAMI", - "version": "3.12.7-3" + "version": "3.12.7-4" }, "wait-for-port": { "arch": "amd64", diff --git a/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/airflow-env.sh b/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/airflow-env.sh index acb7479eae09d..d8783b87f8794 100644 --- a/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/airflow-env.sh +++ b/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/airflow-env.sh @@ -43,6 +43,7 @@ airflow_env_vars=( AIRFLOW_POOL_NAME AIRFLOW_POOL_SIZE AIRFLOW_POOL_DESC + AIRFLOW_STANDALONE_DAG_PROCESSOR AIRFLOW_WORKER_QUEUE AIRFLOW_DATABASE_HOST AIRFLOW_DATABASE_PORT_NUMBER @@ -124,6 +125,7 @@ export AIRFLOW_HOSTNAME_CALLABLE="${AIRFLOW_HOSTNAME_CALLABLE:-}" export AIRFLOW_POOL_NAME="${AIRFLOW_POOL_NAME:-}" export AIRFLOW_POOL_SIZE="${AIRFLOW_POOL_SIZE:-}" export AIRFLOW_POOL_DESC="${AIRFLOW_POOL_DESC:-}" +export AIRFLOW_STANDALONE_DAG_PROCESSOR="${AIRFLOW_STANDALONE_DAG_PROCESSOR:-no}" AIRFLOW_WORKER_QUEUE="${AIRFLOW_WORKER_QUEUE:-"${AIRFLOW_QUEUE:-}"}" export AIRFLOW_WORKER_QUEUE="${AIRFLOW_WORKER_QUEUE:-}" diff --git a/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/libairflow.sh b/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/libairflow.sh index 4357ccd7ed93c..8e39ecc4d581a 100644 --- a/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/libairflow.sh +++ b/bitnami/airflow/2/debian-12/rootfs/opt/bitnami/scripts/libairflow.sh @@ -76,8 +76,9 @@ airflow_validate() { # Check component type & executor check_empty_value "AIRFLOW_COMPONENT_TYPE" - check_multi_value "AIRFLOW_COMPONENT_TYPE" "webserver scheduler worker" + check_multi_value "AIRFLOW_COMPONENT_TYPE" "webserver scheduler worker dag-processor" check_empty_value "AIRFLOW_EXECUTOR" + check_yes_no_value "AIRFLOW_STANDALONE_DAG_PROCESSOR" # Check cryptography parameters if [[ -n "$AIRFLOW_RAW_FERNET_KEY" && -z "$AIRFLOW_FERNET_KEY" ]]; then @@ -127,7 +128,7 @@ airflow_validate() { done fi ;; - scheduler|worker) + *) # Check webserver host and port number check_empty_value "AIRFLOW_WEBSERVER_HOST" check_resolved_hostname "$AIRFLOW_WEBSERVER_HOST" @@ -189,7 +190,7 @@ airflow_initialize() { true # Avoid return false when I am not root fi ;; - scheduler|worker) + *) info "Waiting for Airflow Webserver to be up" airflow_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER" if [[ "$AIRFLOW_EXECUTOR" == "CeleryExecutor" || "$AIRFLOW_EXECUTOR" == "CeleryKubernetesExecutor" ]]; then @@ -250,7 +251,7 @@ airflow_generate_config() { # Configure Airflow webserver authentication airflow_configure_webserver_authentication ;; - scheduler|worker) + *) # Generate Airflow default files debug_execute airflow version ;; @@ -276,6 +277,12 @@ airflow_generate_config() { else airflow_conf_set "core" "load_examples" "False" fi + # Configure Dag Processor mode + if is_boolean_yes "$AIRFLOW_STANDALONE_DAG_PROCESSOR"; then + airflow_conf_set "scheduler" "standalone_dag_processor" "True" + else + airflow_conf_set "scheduler" "standalone_dag_processor" "False" + fi fi # Configure Airflow executor diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 7f25d54b9f113..bcd919b45d962 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -301,53 +301,54 @@ This container supports the installation of additional python modules at start-u #### Customizable environment variables -| Name | Description | Default Value | -|---------------------------------------|-----------------------------------------------------------------------------|----------------------| -| `AIRFLOW_USERNAME` | Airflow username | `user` | -| `AIRFLOW_PASSWORD` | Airflow password | `bitnami` | -| `AIRFLOW_FIRSTNAME` | Airflow firstname | `Firstname` | -| `AIRFLOW_LASTNAME` | Airflow lastname | `Lastname` | -| `AIRFLOW_EMAIL` | Airflow email | `user@example.com` | -| `AIRFLOW_COMPONENT_TYPE` | Airflow component type. Allowed values: *webserver*, *scheduler*. *worker*. | `webserver` | -| `AIRFLOW_EXECUTOR` | Airflow executor. | `SequentialExecutor` | -| `AIRFLOW_RAW_FERNET_KEY` | Airflow raw/unencoded Fernet key | `nil` | -| `AIRFLOW_FORCE_OVERWRITE_CONF_FILE` | Force the airflow.cfg config file generation. | `no` | -| `AIRFLOW_FERNET_KEY` | Airflow Fernet key | `nil` | -| `AIRFLOW_SECRET_KEY` | Airflow Secret key | `nil` | -| `AIRFLOW_WEBSERVER_BASE_URL` | Airflow Webserver base URL. | `nil` | -| `AIRFLOW_WEBSERVER_HOST` | Airflow Webserver host | `127.0.0.1` | -| `AIRFLOW_WEBSERVER_PORT_NUMBER` | Airflow Webserver port. | `8080` | -| `AIRFLOW_LOAD_EXAMPLES` | To load example tasks into the application. | `yes` | -| `AIRFLOW_HOSTNAME_CALLABLE` | Method to obtain the hostname. | `nil` | -| `AIRFLOW_POOL_NAME` | Pool name. | `nil` | -| `AIRFLOW_POOL_SIZE` | Pool size, required with AIRFLOW_POOL_NAME. | `nil` | -| `AIRFLOW_POOL_DESC` | Pool description, required with AIRFLOW_POOL_NAME. | `nil` | -| `AIRFLOW_WORKER_QUEUE` | A queue for the worker to pull tasks from. | `nil` | -| `AIRFLOW_DATABASE_HOST` | Hostname for PostgreSQL server. | `postgresql` | -| `AIRFLOW_DATABASE_PORT_NUMBER` | Port used by PostgreSQL server. | `5432` | -| `AIRFLOW_DATABASE_NAME` | Database name that Airflow will use to connect with the database. | `bitnami_airflow` | -| `AIRFLOW_DATABASE_USERNAME` | Database user that Airflow will use to connect with the database. | `bn_airflow` | -| `AIRFLOW_DATABASE_PASSWORD` | Database password that Airflow will use to connect with the database. | `nil` | -| `AIRFLOW_DATABASE_USE_SSL` | Set to yes if the database is using SSL. | `no` | -| `AIRFLOW_REDIS_USE_SSL` | Set to yes if Redis(R) uses SSL. | `no` | -| `REDIS_HOST` | Hostname for Redis(R) server. | `redis` | -| `REDIS_PORT_NUMBER` | Port used by Redis(R) server. | `6379` | -| `REDIS_USER` | User that Airflow will use to connect with Redis(R). | `nil` | -| `REDIS_PASSWORD` | Password that Airflow will use to connect with Redis(R). | `nil` | -| `REDIS_DATABASE` | Name of the Redis(R) database. | `1` | -| `AIRFLOW_LDAP_ENABLE` | Enable LDAP authentication. | `no` | -| `AIRFLOW_LDAP_URI` | LDAP server URI. | `nil` | -| `AIRFLOW_LDAP_SEARCH` | LDAP search base. | `nil` | -| `AIRFLOW_LDAP_UID_FIELD` | LDAP field used for uid. | `nil` | -| `AIRFLOW_LDAP_BIND_USER` | LDAP user name. | `nil` | -| `AIRFLOW_LDAP_BIND_PASSWORD` | LDAP user password. | `nil` | -| `AIRFLOW_LDAP_USER_REGISTRATION` | User self registration. | `True` | -| `AIRFLOW_LDAP_USER_REGISTRATION_ROLE` | Role name to be assign when a user registers himself. | `nil` | -| `AIRFLOW_LDAP_ROLES_MAPPING` | Mapping from LDAP DN to a list of Airflow roles. | `nil` | -| `AIRFLOW_LDAP_ROLES_SYNC_AT_LOGIN` | Replace ALL the user roles each login, or only on registration. | `True` | -| `AIRFLOW_LDAP_USE_TLS` | Use LDAP SSL. | `False` | -| `AIRFLOW_LDAP_ALLOW_SELF_SIGNED` | Allow self signed certicates in LDAP ssl. | `True` | -| `AIRFLOW_LDAP_TLS_CA_CERTIFICATE` | File that store the CA for LDAP ssl. | `nil` | +| Name | Description | Default Value | +|---------------------------------------|----------------------------------------------------------------------------------------------|----------------------| +| `AIRFLOW_USERNAME` | Airflow username | `user` | +| `AIRFLOW_PASSWORD` | Airflow password | `bitnami` | +| `AIRFLOW_FIRSTNAME` | Airflow firstname | `Firstname` | +| `AIRFLOW_LASTNAME` | Airflow lastname | `Lastname` | +| `AIRFLOW_EMAIL` | Airflow email | `user@example.com` | +| `AIRFLOW_COMPONENT_TYPE` | Airflow component type. Allowed values: *webserver*, *scheduler*, *dag-processor*, *worker*. | `webserver` | +| `AIRFLOW_EXECUTOR` | Airflow executor. | `SequentialExecutor` | +| `AIRFLOW_RAW_FERNET_KEY` | Airflow raw/unencoded Fernet key | `nil` | +| `AIRFLOW_FORCE_OVERWRITE_CONF_FILE` | Force the airflow.cfg config file generation. | `no` | +| `AIRFLOW_FERNET_KEY` | Airflow Fernet key | `nil` | +| `AIRFLOW_SECRET_KEY` | Airflow Secret key | `nil` | +| `AIRFLOW_WEBSERVER_BASE_URL` | Airflow Webserver base URL. | `nil` | +| `AIRFLOW_WEBSERVER_HOST` | Airflow Webserver host | `127.0.0.1` | +| `AIRFLOW_WEBSERVER_PORT_NUMBER` | Airflow Webserver port. | `8080` | +| `AIRFLOW_LOAD_EXAMPLES` | To load example tasks into the application. | `yes` | +| `AIRFLOW_HOSTNAME_CALLABLE` | Method to obtain the hostname. | `nil` | +| `AIRFLOW_POOL_NAME` | Pool name. | `nil` | +| `AIRFLOW_POOL_SIZE` | Pool size, required with AIRFLOW_POOL_NAME. | `nil` | +| `AIRFLOW_POOL_DESC` | Pool description, required with AIRFLOW_POOL_NAME. | `nil` | +| `AIRFLOW_STANDALONE_DAG_PROCESSOR` | Enable running Dag Processor in standalone mode | `no` | +| `AIRFLOW_WORKER_QUEUE` | A queue for the worker to pull tasks from. | `nil` | +| `AIRFLOW_DATABASE_HOST` | Hostname for PostgreSQL server. | `postgresql` | +| `AIRFLOW_DATABASE_PORT_NUMBER` | Port used by PostgreSQL server. | `5432` | +| `AIRFLOW_DATABASE_NAME` | Database name that Airflow will use to connect with the database. | `bitnami_airflow` | +| `AIRFLOW_DATABASE_USERNAME` | Database user that Airflow will use to connect with the database. | `bn_airflow` | +| `AIRFLOW_DATABASE_PASSWORD` | Database password that Airflow will use to connect with the database. | `nil` | +| `AIRFLOW_DATABASE_USE_SSL` | Set to yes if the database is using SSL. | `no` | +| `AIRFLOW_REDIS_USE_SSL` | Set to yes if Redis(R) uses SSL. | `no` | +| `REDIS_HOST` | Hostname for Redis(R) server. | `redis` | +| `REDIS_PORT_NUMBER` | Port used by Redis(R) server. | `6379` | +| `REDIS_USER` | User that Airflow will use to connect with Redis(R). | `nil` | +| `REDIS_PASSWORD` | Password that Airflow will use to connect with Redis(R). | `nil` | +| `REDIS_DATABASE` | Name of the Redis(R) database. | `1` | +| `AIRFLOW_LDAP_ENABLE` | Enable LDAP authentication. | `no` | +| `AIRFLOW_LDAP_URI` | LDAP server URI. | `nil` | +| `AIRFLOW_LDAP_SEARCH` | LDAP search base. | `nil` | +| `AIRFLOW_LDAP_UID_FIELD` | LDAP field used for uid. | `nil` | +| `AIRFLOW_LDAP_BIND_USER` | LDAP user name. | `nil` | +| `AIRFLOW_LDAP_BIND_PASSWORD` | LDAP user password. | `nil` | +| `AIRFLOW_LDAP_USER_REGISTRATION` | User self registration. | `True` | +| `AIRFLOW_LDAP_USER_REGISTRATION_ROLE` | Role name to be assign when a user registers himself. | `nil` | +| `AIRFLOW_LDAP_ROLES_MAPPING` | Mapping from LDAP DN to a list of Airflow roles. | `nil` | +| `AIRFLOW_LDAP_ROLES_SYNC_AT_LOGIN` | Replace ALL the user roles each login, or only on registration. | `True` | +| `AIRFLOW_LDAP_USE_TLS` | Use LDAP SSL. | `False` | +| `AIRFLOW_LDAP_ALLOW_SELF_SIGNED` | Allow self signed certicates in LDAP ssl. | `True` | +| `AIRFLOW_LDAP_TLS_CA_CERTIFICATE` | File that store the CA for LDAP ssl. | `nil` | #### Read-only environment variables