Merge pull request #2080 from vrk-kpa/AV-2069_optimize-ckan-dockerfile

AV-2069: Optimize CKAN Docker build for cacheability

ckan/ckan-requirements.in

@@ -0,0 +1,64 @@
+# This file defines the used CKAN version and any overrides
+# Compile with pip-compile to produce the ckan-requirements.txt used in build
+
+# -e git+https://github.com/ckan/[email protected]#egg=ckan[requirements]
+-e git+https://github.com/ckan/[email protected]#egg=ckan
+
+# Runtime requirements
+uWSGI==2.0.22
+gevent==21.12.0
+
+# Copied from CKAN 2.9.9, remove after pip-compile works with [requirements] again
+alembic==1.0.0            # via -r requirements.in
+babel==2.7.0              # via -r requirements.in, flask-babel
+beaker==1.11.0            # via -r requirements.in
+bleach==3.1.4             # via -r requirements.in
+certifi==2021.5.30        # via requests
+chardet==4.0.0            # via requests
+click==7.1.2              # via -r requirements.in, flask, rq
+dominate==2.4.0           # via -r requirements.in
+fanstatic==1.1            # via -r requirements.in
+feedgen==0.9.0            # via -r requirements.in
+flask-babel==1.0.0        # via -r requirements.in
+flask-multistatic==1.0    # via -r requirements.in
+flask==1.1.1              # via -r requirements.in, flask-babel, flask-multistatic
+funcsigs==1.0.2           # via beaker
+idna==2.10                # via requests
+itsdangerous==1.1.0       # via flask
+jinja2==2.10.1            # via -r requirements.in, flask, flask-babel
+lxml==4.6.3               # via feedgen
+mako==1.1.5               # via alembic
+markdown==3.4.1           # via -r requirements.in
+markupsafe==1.1.1         # via jinja2, mako
+nose==1.3.7               # via pyutilib
+passlib==1.6.5            # via -r requirements.in
+polib==1.0.7              # via -r requirements.in
+psycopg2==2.9.3           # via -r requirements.in
+pyjwt==1.7.1              # via -r requirements.in
+pysolr==3.6.0             # via -r requirements.in
+python-dateutil==2.8.2    # via -r requirements.in, alembic, feedgen
+python-editor==1.0.4      # via alembic
+python-magic==0.4.15      # via -r requirements.in
+pytz==2016.7              # via -r requirements.in, babel, flask-babel, tzlocal
+pyutilib==5.7.1           # via -r requirements.in
+pyyaml==5.4.1             # via -r requirements.in
+redis==3.5.3              # via rq
+repoze.lru==0.7           # via routes
+repoze.who==2.3           # via -r requirements.in
+requests==2.25.1          # via -r requirements.in, pysolr
+routes==1.13              # via -r requirements.in
+rq==1.0                   # via -r requirements.in
+shutilwhich==1.1.0        # via fanstatic
+simplejson==3.10.0        # via -r requirements.in
+six==1.16.0               # via bleach, python-dateutil, pyutilib
+sqlalchemy==1.3.5         # via -r requirements.in, alembic
+sqlparse==0.3.0           # via -r requirements.in
+tzlocal==1.3              # via -r requirements.in
+unicodecsv==0.14.1        # via -r requirements.in
+urllib3==1.26.6           # via requests
+watchdog==2.1.5           # via -r requirements.in, werkzeug
+webassets==0.12.1         # via -r requirements.in
+webencodings==0.5.1       # via bleach
+webob==1.8.7              # via fanstatic, repoze.who
+werkzeug[watchdog]==1.0.0  # via -r requirements.in, flask
+zope.interface==4.3.2     # via -r requirements.in, repoze.who

ckan/ckan-requirements.txt

@@ -0,0 +1,196 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile ckan-requirements.in
+#
+--extra-index-url https://alpine-wheels.github.io/index
+
+-e git+https://github.com/ckan/[email protected]#egg=ckan
+    # via -r ckan-requirements.in
+alembic==1.0.0
+    # via -r ckan-requirements.in
+babel==2.7.0
+    # via
+    #   -r ckan-requirements.in
+    #   flask-babel
+beaker==1.11.0
+    # via -r ckan-requirements.in
+bleach==3.1.4
+    # via -r ckan-requirements.in
+certifi==2021.5.30
+    # via
+    #   -r ckan-requirements.in
+    #   requests
+chardet==4.0.0
+    # via
+    #   -r ckan-requirements.in
+    #   requests
+click==7.1.2
+    # via
+    #   -r ckan-requirements.in
+    #   flask
+    #   rq
+dominate==2.4.0
+    # via -r ckan-requirements.in
+fanstatic==1.1
+    # via -r ckan-requirements.in
+feedgen==0.9.0
+    # via -r ckan-requirements.in
+flask==1.1.1
+    # via
+    #   -r ckan-requirements.in
+    #   flask-babel
+    #   flask-multistatic
+flask-babel==1.0.0
+    # via -r ckan-requirements.in
+flask-multistatic==1.0
+    # via -r ckan-requirements.in
+funcsigs==1.0.2
+    # via -r ckan-requirements.in
+gevent==21.12.0
+    # via -r ckan-requirements.in
+greenlet==1.1.3.post0
+    # via gevent
+idna==2.10
+    # via
+    #   -r ckan-requirements.in
+    #   requests
+importlib-metadata==6.8.0
+    # via markdown
+itsdangerous==1.1.0
+    # via
+    #   -r ckan-requirements.in
+    #   flask
+jinja2==2.10.1
+    # via
+    #   -r ckan-requirements.in
+    #   flask
+    #   flask-babel
+lxml==4.6.3
+    # via
+    #   -r ckan-requirements.in
+    #   feedgen
+mako==1.1.5
+    # via
+    #   -r ckan-requirements.in
+    #   alembic
+markdown==3.4.1
+    # via -r ckan-requirements.in
+markupsafe==1.1.1
+    # via
+    #   -r ckan-requirements.in
+    #   jinja2
+    #   mako
+nose==1.3.7
+    # via
+    #   -r ckan-requirements.in
+    #   pyutilib
+passlib==1.6.5
+    # via -r ckan-requirements.in
+polib==1.0.7
+    # via -r ckan-requirements.in
+psycopg2==2.9.3
+    # via -r ckan-requirements.in
+pyjwt==1.7.1
+    # via -r ckan-requirements.in
+pysolr==3.6.0
+    # via -r ckan-requirements.in
+python-dateutil==2.8.2
+    # via
+    #   -r ckan-requirements.in
+    #   alembic
+    #   feedgen
+python-editor==1.0.4
+    # via
+    #   -r ckan-requirements.in
+    #   alembic
+python-magic==0.4.15
+    # via -r ckan-requirements.in
+pytz==2016.7
+    # via
+    #   -r ckan-requirements.in
+    #   babel
+    #   flask-babel
+    #   tzlocal
+pyutilib==5.7.1
+    # via -r ckan-requirements.in
+pyyaml==5.4.1
+    # via -r ckan-requirements.in
+redis==3.5.3
+    # via
+    #   -r ckan-requirements.in
+    #   rq
+repoze-lru==0.7
+    # via
+    #   -r ckan-requirements.in
+    #   routes
+repoze-who==2.3
+    # via -r ckan-requirements.in
+requests==2.25.1
+    # via
+    #   -r ckan-requirements.in
+    #   pysolr
+routes==1.13
+    # via -r ckan-requirements.in
+rq==1.0
+    # via -r ckan-requirements.in
+shutilwhich==1.1.0
+    # via
+    #   -r ckan-requirements.in
+    #   fanstatic
+simplejson==3.10.0
+    # via -r ckan-requirements.in
+six==1.16.0
+    # via
+    #   -r ckan-requirements.in
+    #   bleach
+    #   python-dateutil
+    #   pyutilib
+sqlalchemy==1.3.5
+    # via
+    #   -r ckan-requirements.in
+    #   alembic
+sqlparse==0.3.0
+    # via -r ckan-requirements.in
+tzlocal==1.3
+    # via -r ckan-requirements.in
+unicodecsv==0.14.1
+    # via -r ckan-requirements.in
+urllib3==1.26.6
+    # via
+    #   -r ckan-requirements.in
+    #   requests
+uwsgi==2.0.22
+    # via -r ckan-requirements.in
+watchdog==2.1.5
+    # via
+    #   -r ckan-requirements.in
+    #   werkzeug
+webassets==0.12.1
+    # via -r ckan-requirements.in
+webencodings==0.5.1
+    # via
+    #   -r ckan-requirements.in
+    #   bleach
+webob==1.8.7
+    # via
+    #   -r ckan-requirements.in
+    #   fanstatic
+    #   repoze-who
+werkzeug[watchdog]==1.0.0
+    # via
+    #   -r ckan-requirements.in
+    #   flask
+zipp==3.17.0
+    # via importlib-metadata
+zope-event==5.0
+    # via gevent
+zope-interface==4.3.2
+    # via
+    #   -r ckan-requirements.in
+    #   gevent
+    #   repoze-who
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

ckan/pip.conf

@@ -1,3 +1,4 @@
 [global]
 extra-index-url = https://alpine-wheels.github.io/index
-cache-dir=/tmp/pip-cache
+no-cache-dir = True
+root-user-action = ignore

ckan/scripts/entrypoint_ckan.sh

@@ -13,6 +13,7 @@ export CKAN_SYSADMIN_EMAIL="${SYSADMIN_EMAIL}"
 # install extensions (DEV_MODE)
 if [[ "${DEV_MODE}" == "true" ]]; then
     echo "entrypoint_ckan - installing extensions because DEV_MODE = 'true' ..."
+    sudo -E ${SCRIPT_DIR}/install_extension_requirements.sh
     sudo -E ${SCRIPT_DIR}/install_extensions.sh
 fi
 

ckan/scripts/entrypoint_cron.sh

@@ -13,6 +13,7 @@ done
 if [[ "${DEV_MODE}" == "true" ]]; then
   echo "entrypoint_cron - installing extensions because DEV_MODE = 'true' ..."
   sudo -E ${SCRIPT_DIR}/install_extensions.sh
+  sudo -E ${SCRIPT_DIR}/install_extension_requirements.sh
 fi
 
 # apply templates

ckan/scripts/install_extension_requirements.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -e
+
+echo "install_extension_requirements ..."
+
+pip_install() {
+  if [[ ! -f "$1" ]]; then
+    echo "pip_install: skipping $1 because file does not exist!"
+    return 0
+  fi
+
+  echo "pip_install: installing $1 ..."
+
+  pip install -r "$1"
+}
+
+# install extension requirements
+pip_install "${EXT_DIR}/ckanext-forcetranslation/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-archiver/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-dcat/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-drupal8/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-hierarchy/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-prh_tools/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-sentry/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-spatial/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-statistics/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-matomo/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-fluent/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-qa/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-showcase/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-report/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-sixodp_showcase/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-sixodp_showcasesubmit/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-ytp_recommendation/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-ytp_tasks/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-harvest/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-cloudstorage/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-organizationapproval/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-advancedsearch/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-apis/requirements.txt"
+pip_install "${EXT_DIR}/ckanext-ytp_main/requirements.txt"