Merge pull request #23 from tulibraries/make-airflow-home-envvar

replacing airflow_home from config with envvar, per airflow deprecation warning
tulibraries · Sep 30, 2019 · b2848e4 · b2848e4
2 parents 9c4cd72 + 0df889a
commit b2848e4
Show file tree

Hide file tree

Showing 13 changed files with 553 additions and 333 deletions.
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.6.6
diff --git a/CHANGELOG.md b/CHANGELOG.md
diff --git a/Pipfile b/Pipfile
@@ -7,9 +7,10 @@ verify_ssl = true
 pytest = "*"
 pylint = "*"
 docker = "*"
+ansible = "*"
 
 [packages]
 molecule = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.6"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -126,7 +126,6 @@ airflow_do_upgrade_db: True
 # Default configuration
 airflow_defaults_config:
   core:
-    airflow_home: "{{ airflow_user_home_path ~ '/airflow' }}"
     airflow_home_mode: "0700"
     dags_folder: "{{ airflow_user_home_path ~ '/airflow/dags' }}"
     dags_folder_mode: "0755"

diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py
@@ -65,7 +65,7 @@ def test_debian_ubuntu_prerequisites_packages(host, name, codenames):
 
 
 @pytest.mark.parametrize('name,codenames', [
-    ('python36-devel', None),
+    ('python3-devel', None),
     ('openssl-devel', None),
     ('libffi-devel', None),
     ('libxml2', None),

diff --git a/tasks/main.yml b/tasks/main.yml
@@ -14,7 +14,7 @@
 - name: 'INSTALL | Include installation tasks'
   import_tasks: "manage_installation.yml"
   environment:
-    AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}'
+    AIRFLOW_HOME: "{{ airflow_user_home_path ~ '/airflow' }}"
   become: True
   tags:
     - 'role::airflow'
@@ -25,7 +25,7 @@
 - name: 'CONFIG | Include configuration tasks'
   import_tasks: "manage_configuration.yml"
   environment:
-    AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}'
+    AIRFLOW_HOME: "{{ airflow_user_home_path ~ '/airflow' }}"
   become: True
   tags:
     - 'role::airflow'
@@ -46,7 +46,7 @@
 - name: 'CONFIG | Include connections tasks'
   import_tasks: "manage_connections.yml"
   environment:
-    AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}'
+    AIRFLOW_HOME: "{{ airflow_user_home_path ~ '/airflow' }}"
   become: True
   tags:
     - 'role::airflow'
@@ -57,7 +57,7 @@
 - name: 'CONFIG | Include connections tasks'
   import_tasks: "manage_airflow_variables.yml"
   environment:
-    AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}'
+    AIRFLOW_HOME: "{{ airflow_user_home_path ~ '/airflow' }}"
   become: True
   tags:
     - 'role::airflow'
@@ -67,7 +67,7 @@
 - name: 'CONFIG | Include DAG codebases'
   import_tasks: "manage_dags.yml"
   environment:
-    AIRFLOW_HOME: '{{ airflow_config.core.airflow_home }}'
+    AIRFLOW_HOME: "{{ airflow_user_home_path ~ '/airflow' }}"
   become: True
   tags:
     - 'role::airflow'

diff --git a/tasks/manage_configuration.yml b/tasks/manage_configuration.yml
@@ -8,7 +8,7 @@
   become: true
   template:
     src: "templates/airflow.cfg.j2"
-    dest: "{{ airflow_config.core.airflow_home }}/airflow.cfg"
+    dest: "{{ airflow_user_home_path ~ '/airflow' }}/airflow.cfg"
     owner: "{{ airflow_user_name }}"
     group: "{{ airflow_user_group }}"
     mode: '0400'
@@ -52,7 +52,7 @@
 - name: 'CONFIG | Generate Webserver Config if RBAC is enabled'
   template:
     src: "templates/webserver_config.py"
-    dest: "{{ airflow_config.core.airflow_home }}/webserver_config.py"
+    dest: "{{ airflow_user_home_path ~ '/airflow' }}/webserver_config.py"
     owner: "{{ airflow_user_name }}"
     group: "{{ airflow_user_group }}"
     mode: '0400'

diff --git a/tasks/manage_installation.yml b/tasks/manage_installation.yml
@@ -24,7 +24,7 @@
 # This has to happen before the install, because the install creates a default file in $AIRFLOW_HOME
 - name: 'INSTALL | Check if configuration file exists'
   stat:
-    path: "{{ airflow_config.core.airflow_home }}/airflow.cfg"
+    path: "{{ airflow_user_home_path ~ '/airflow' }}/airflow.cfg"
   register: 'airflow_config_stat'
   changed_when: False
 
@@ -48,7 +48,7 @@
 
 - name: 'INSTALL | Manage permissions on airflow home directory'
   file:
-    path: "{{ airflow_config.core.airflow_home }}"
+    path: "{{ airflow_user_home_path ~ '/airflow' }}"
     owner: "{{ airflow_user_name }}"
     group: "{{ airflow_user_name }}"
     mode: "{{ airflow_config.core.airflow_home_mode }}"

diff --git a/templates/airflow.env.j2 b/templates/airflow.env.j2
@@ -3,8 +3,8 @@
 PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ airflow_virtualenv ~ '/bin' }}:{{ airflow_user_home_path }}.local/bin:$PATH"
 
 # General
-AIRFLOW_HOME="{{ airflow_config.core.airflow_home }}"
-AIRFLOW_CONFIG="{{ airflow_config.core.airflow_home }}/airflow.cfg"
+AIRFLOW_HOME="{{ airflow_user_home_path ~ '/airflow' }}"
+AIRFLOW_CONFIG="{{ airflow_user_home_path ~ '/airflow' }}/airflow.cfg"
 AIRFLOW_BINARY_PATH="{{ airflow_virtualenv ~ '/bin' }}"
 
 # Webserver specific

diff --git a/templates/airflow.profile.j2 b/templates/airflow.profile.j2
@@ -1,5 +1,5 @@
 # {{ ansible_managed }}
 
 # General
-export AIRFLOW_HOME="{{ airflow_config.core.airflow_home }}"
+export AIRFLOW_HOME="{{ airflow_user_home_path ~ '/airflow' }}"
 export PATH="$PATH:{{ airflow_virtualenv ~ '/bin' }}"
diff --git a/vars/os_distribution/centos.yml b/vars/os_distribution/centos.yml
@@ -3,7 +3,7 @@
 # Centos 7 specific vars
 
 # Python
-_airflow_python_version: 'python36'
+_airflow_python_version: 'python3'
 
 # Services management
 _is_systemd_managed_system: True
diff --git a/vars/os_family/redhat.yml b/vars/os_family/redhat.yml
@@ -3,7 +3,7 @@
 # RedHat CentOS 7 family specific vars
 
 _airflow_system_dependencies:
- - name: 'python36-devel'
+ - name: 'python3-devel'
  - name: 'postgresql-devel'
  - name: 'openssl-devel'
  - name: 'libffi-devel'