Merge pull request #90 from tulibraries/DEVO-1000-update-python-version

Devo 1000 update python version

.circleci/config.yml

@@ -4,16 +4,16 @@ jobs:
   build:
     working_directory: ~/ansible-role-airflow
     docker:
-      - image: cimg/python:3.9.10
+      - image: cimg/python:3.12.3
         environment:
           PIPENV_VENV_IN_PROJECT: true
     steps:
       - checkout
+      - setup_remote_docker
       - run:
           name: Install
           command: pipenv install --dev
-      - setup_remote_docker:
-          version: 20.10.11
+      - run: sudo apt update; sudo apt install rsync
       - run:
           name: Molecule
           command: pipenv run molecule test

.python-version

@@ -1 +1 @@
-3.9.10
+3.12.3

Pipfile

@@ -5,18 +5,19 @@ verify_ssl = true
 
 [dev-packages]
 pytest = "*"
-yamllint = "*"
-flake8 = "*"
 pytest-testinfra = "*"
 
 [packages]
-ansible = "==2.9.27"
-ansible-lint = "==4.2.0"
+ansible = "==9.6.0"
+ansible-lint = "*"
 cryptography = "*"
-docker = "==4.4.4"
-molecule = "==3.0.8"
+docker = "*"
+molecule = "*"
+molecule-docker = "*"
 pylint = "*"
-requests = "==2.28.1"
+
+# Remove once this is fixed: https://github.com/ansible-collections/community.docker/issues/868#issuecomment-2124979976
+requests = "==2.31.0"
 
 [requires]
-python_version = "3.9"
+python_version = "3.12"

defaults/main.yml

@@ -33,21 +33,21 @@ airflow_pid_owner: "{{ airflow_user_name }}"
 airflow_pid_group: "{{ airflow_user_group }}"
 airflow_pid_mode: '0700'
 
-airflow_virtualenv: "{{ airflow_user_home_path }}/venv3.9"
+airflow_virtualenv: "{{ airflow_user_home_path }}/venv3.11"
 airflow_virtualenv_mode: "0755"
 
 airflow_python_version: "{{ _airflow_python_version }}"
-python_version: "3.9.10"
+python_version: "3.11.5"
 # Update if python version or apache airflow version changes.
-airflow_python_constraint: "--constraint=https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-3.9.txt"
+airflow_python_constraint: "--constraint=https://raw.githubusercontent.com/apache/airflow/constraints-2.8.3/constraints-3.11.txt"
 dags_repos_folder: "{{ airflow_user_home_path ~ '/airflow/dags_repos' }}"
 dags_folder: "{{ airflow_user_home_path ~ '/airflow/dags' }}"
 
 airflow_packages:
   - name: 'GitPython'
   - name: 'Cython'
   - name: 'apache-airflow'
-    version: '2.7.1'
+    version: '2.8.3'
   - name: 'pipenv'
   - name: 'numpy'
 airflow_extra_packages:

meta/main.yml

@@ -20,10 +20,11 @@ galaxy_info:
     - name: 'Centos'
       versions:
         - '7'
+    - name: rockylinux
+      versions:
+        - 8
   galaxy_tags:
     - 'system'
 dependencies:
-  - src: 'diodonfrost.python'
-    version: '1.0.0'
   - src: 'geerlingguy.git'
     version: '2.0.4'

molecule/default/Dockerfile.j2

@@ -7,7 +7,7 @@ FROM {{ item.image }}
 {% endif %}
 
 RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
-    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python3.11 sudo python3.11-devel python*-dnf iproute procps bash && dnf clean all; \
     elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
     elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
     elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \

molecule/default/converge.yml

@@ -16,4 +16,5 @@
         - name: 'werkzeug'
   vars:
     airflow_fernet_key: '46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho='
-    python_version: '3.9.10'
+    python_version: '3.11.5'
+    is_systemd_managed_system: True

molecule/default/molecule.yml

@@ -2,7 +2,6 @@
 scenario:
   name: default
   test_sequence:
-    - lint
     - cleanup
     - destroy
     - dependency
@@ -20,19 +19,19 @@ dependency:
   role-file: requirements.yml
 driver:
   name: docker
-lint: |
-  yamllint .
-  ansible-lint
-  flake8 .
 platforms:
   - name: instance
-    image: centos:7
-    command: /sbin/init
+    image: rockylinux:8
+    command: /lib/systemd/systemd
     privileged: true
+    cgroupns_mode: host
     capabilities:
       - SYS_ADMIN
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    tmpfs:
+      - /run
+      - /tmp
 provisioner:
   name: ansible
 verifier:

molecule/default/requirements.yml

@@ -2,12 +2,8 @@
 
 - src: geerlingguy.git
   name: geerlingguy.git
-  version: 2.0.4
+  version: 3.0.1
 
-- src: https://github.com/tulibraries/ansible-role-server-bootstrap.git
+- src: tulibraries.ansible_role_server_bootstrap
   name: tulibraries.server-bootstrap
-  version: main
-
-- src: diodonfrost.python
-  name: diodonfrost.python
-  version: 1.0.0
+  version: v1.0.0

molecule/default/tests/test_default.py

@@ -123,15 +123,15 @@ def test_pip_libraries(host, library):
     Test pip libraries installed
     """
 
-    pip_path = "/var/lib/airflow/venv3.9/bin/pip3.9"
+    pip_path = "/var/lib/airflow/venv3.11/bin/pip3.11"
     libraries = host.pip.get_packages(pip_path=pip_path)
     assert libraries.get(library)
 
 
 @pytest.mark.parametrize('path,path_type,user,group,mode', [
     ('/var/run/airflow', 'directory', 'airflow', 'airflow', 0o700),
     ('/var/lib/airflow/airflow', 'directory', 'airflow', 'airflow', 0o700),
-    ('/var/lib/airflow/venv3.9', 'directory', 'airflow', 'airflow', 0o755),
+    ('/var/lib/airflow/venv3.11', 'directory', 'airflow', 'airflow', 0o755),
     ('/var/lib/airflow/airflow/airflow.cfg', 'file', 'airflow', 'airflow',
      0o400)
     ])
@@ -208,7 +208,7 @@ def test_dags_pip_libraries(host, library):
     Test DAGs' required pip libraries installed
     """
 
-    pip_path = "/var/lib/airflow/venv3.9/bin/pip3.9"
+    pip_path = "/var/lib/airflow/venv3.11/bin/pip3.11"
     libraries = host.pip.get_packages(pip_path=pip_path)
     print(libraries)
     assert libraries.get(library)

tasks/main.yml

@@ -20,18 +20,6 @@
     - 'role::airflow'
     - 'role::airflow::install'
 
-# Sqlite installation tasks management
-- name: 'INSTALL | Include sqlite installation tasks'
-  import_tasks: "manage_sqlite_install.yml"
-  environment:
-    - CFLAGS: -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_SOUNDEX -DSQLITE_TEMP_STORE=3 -DSQLITE_USE_URI -O2 -fPIC
-    - PREFIX: /usr/local
-    - LIBS: "-lm"
-  become: true
-  tags:
-    - 'role::airflow'
-    - 'role::airflow::config'
-
 # Configuration tasks management
 - name: 'CONFIG | Include configuration tasks'
   import_tasks: "manage_configuration.yml"

tasks/manage_installation.yml

@@ -9,32 +9,24 @@
     state: "{{ item.state | default('present') }}"
   with_items: "{{ airflow_system_dependencies }}"
 
-# Install Python
-- name: 'INSTALL | Base python'
-  include_role:
-    name: diodonfrost.python
-  tags:
-    - python
-
-- name: 'INSTALL | Add python3 symlink to python3.9'
-  ansible.builtin.file:
-    force: true
-    dest: "/usr/local/bin/python3"
-    src: "/usr/local/python3.9/bin/python3.9"
-    state: link
+- name: 'CONFIG | Use Python 3.11 as default selection'
+  community.general.alternatives:
+    name: python3
+    path: /usr/bin/python3.11
+    link: /usr/bin/python3
 
-- name: 'INSTALL | Add python symlink to python3.9'
+- name: 'INSTALL | Add python symlink to python3'
   ansible.builtin.file:
     force: true
-    dest: "/usr/local/bin/python"
-    src: "/usr/local/python3.9/bin/python3.9"
+    dest: "/usr/bin/python"
+    src: "/usr/bin/python3"
     state: link
 
-- name: 'INSTALL | Add pip symlink to pip3.9'
+- name: 'INSTALL | Add pip symlink to pip3'
   ansible.builtin.file:
     force: true
-    dest: "/usr/local/bin/pip"
-    src: "/usr/local/python3.9/bin/pip3.9"
+    dest: "/usr/bin/pip"
+    src: "/usr/bin/pip3"
     state: link
 
 # Create user
@@ -95,7 +87,7 @@
     version: "{{ item.version | default(omit) }}"
     state: "{{ item.state | default('present') }}"
     virtualenv: "{{ airflow_virtualenv }}"
-    virtualenv_command: "/usr/local/bin/python3 -m venv"
+    virtualenv_command: "python3 -m venv"
     extra_args: "{{ airflow_python_constraint }}"
   changed_when: "airflow_pip_changed_when"
   with_items: "{{ airflow_packages }}"
@@ -114,7 +106,7 @@
     version: "{{ item.version | default(omit) }}"
     state: "{{ item.state | default('present') }}"
     virtualenv: "{{ airflow_virtualenv }}"
-    virtualenv_command: "/usr/local/bin/python3 -m venv"
+    virtualenv_command: "/usr/bin/python3 -m venv"
   changed_when: "airflow_pip_changed_when"
   with_items: "{{ dag_packages }}"
   environment:
@@ -144,7 +136,7 @@
     name: "{{ item.name }}"
     state: "{{ item.state | default('present') }}"
     virtualenv: "{{ airflow_virtualenv }}"
-    virtualenv_command: "/usr/local/bin/python3 -m venv"
+    virtualenv_command: "/usr/bin/python3 -m venv"
     extra_args: "{{ airflow_python_constraint }}"
   changed_when: "airflow_pip_changed_when"
   with_items: "{{ airflow_extra_packages }}"

vars/os_family/redhat.yml

@@ -9,6 +9,6 @@ _airflow_system_dependencies:
  - name: 'libxml2-devel'
  - name: 'libxslt-devel'
  - name: 'gcc'
- # These are both necessary for the python venv setup
- - name: 'python-virtualenv.noarch'
- - name: 'python3-setuptools.noarch'
+ - name: 'python3.11-devel'
+ - name: 'python3.11-pip'
+ - name: 'sqlite'