Add Github Actions, compose workflow (#258)

* Updated instructions. * Update requirements and migration. * Upgrade to newer kubernetes version. * Add compose file. * Fix ports for Docker. * Add CI. * Add CI. * Fix CI * Add coverage * Install test requirements. * Format. * Fix CI. * Fix CI. * Add postgres to Actions. * Add postgres to Actions. * Drop 3.11. * Fix codecov. * Cleanup. * Fix new template upload. * Update templates. * Be specific when creating new Docker Versions. * update template * Fix template. * Fix template. * Improve template. * Add name and text in info; add new keys * Added pattern to templates. * Update template with patterns. * Use patterns in Python code. * Update fix typos at default template * Export all Delft3D files. * Update default_template.json * Set default names and templates correctly. * Add migration. * Add manual migration for legacy scenes. * Add manual migration for legacy scenes. * Add manual migration for legacy scenes. * Add manual migration for legacy scenes. * Updates by Bjorn to the input template. * Update the migrations. * Don't move into deleting workflow. * Fix requirements. * Fix pip version, add cache. * Fix STATICFILES_DIRS. --------- Co-authored-by: IoannaMi <[email protected]>
openearth · Nov 29, 2024 · 93dd2ff · 93dd2ff
1 parent deaee6a
commit 93dd2ff
Show file tree

Hide file tree

Showing 162 changed files with 2,262 additions and 4,128 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,45 @@
+name: Django CI
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: djangodb_test
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 1s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip==24.0
+        pip install -r test_requirements.txt
+    - name: Run Tests
+      run: |
+        coverage run --source='.' manage.py test --timing
+    - uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -72,3 +72,6 @@ target/
 
 # Idea editors
 .idea
+static/
+workflows/
+workflow_templates/
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,13 @@
+FROM python:3.10-slim-buster
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+COPY . /app
+WORKDIR /app
+
+# RUN apt-get update && apt-get install -y \
+#     kubectl && \
+#     rm -rf /var/lib/apt/lists/*
+# RUN wget https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/aws-iam-authenticator
+
+RUN pip install -r requirements.txt
+# RUN pip install awscli
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ Delft3D Geological Tool web application server
 ## Install
 Make sure to have postgres 12 container up and running:
 ```bash
-docker run -e "POSTGRES_DB=djangodb" -e "POSTGRES_HOST_AUTH_METHOD=trust" -e "POSTGRES_USER=django" postgres:12
+docker run -e "POSTGRES_DB=djangodb_test" -e "POSTGRES_HOST_AUTH_METHOD=trust" -e "POSTGRES_USER=postgres" -p5432:5432 postgres:12
 ```
 
 Install delft3d-gt-ui next to this repo and do

diff --git a/delft3dcontainermanager/__init__.py b/delft3dcontainermanager/__init__.py
@@ -1,5 +1,5 @@
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
-from celery import app as celery_app  # noqa
+from .celery import app as celery_app  # noqa
 
 __all__ = ["celery_app"]
diff --git a/delft3dcontainermanager/celery.py b/delft3dcontainermanager/celery.py
@@ -1,12 +1,7 @@
-from __future__ import absolute_import, print_function
-
 import os
 
-from ddtrace import patch_all
-
-patch_all()
 from celery import Celery
-from django.conf import settings  # noqa
+from django.conf import settings
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "delft3dcontainermanager.settings")

diff --git a/delft3dcontainermanager/settings.py b/delft3dcontainermanager/settings.py
@@ -17,8 +17,6 @@
 import sys
 from os import environ
 
-from kubernetes import config
-
 DEBUG = False
 SECRET_KEY = "notneeded"
 
@@ -43,7 +41,6 @@
 
 USE_I18N = True
 
-USE_L10N = True
 
 USE_TZ = True
 
@@ -75,7 +72,8 @@
 
 # import provisioned settings
 try:
-    from .provisionedsettings import *
+    AWS_IAM_PATH = ""
+    from .provisionedsettings import *  # noqa
 
     environ["PATH"] += ":{}".format(AWS_IAM_PATH)
 except ImportError:

diff --git a/delft3dcontainermanager/tasks.py b/delft3dcontainermanager/tasks.py
@@ -1,18 +1,14 @@
 from __future__ import absolute_import
 
-import logging
 from json import dumps
-from shutil import rmtree
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
 from celery_once import QueueOnce
-from django.conf import settings
 from django.core.management import call_command
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 from requests.exceptions import HTTPError
-from six.moves import configparser
 
 logger = get_task_logger(__name__)
 
@@ -131,6 +127,6 @@ def do_argo_remove(self, workflow_id):
     crd = client.CustomObjectsApi(client_api)
     status = crd.delete_namespaced_custom_object(
         "argoproj.io", "v1alpha1", "default", "workflows", workflow_id
-        )
+    )
 
     return {"do_argo_remove": status}
diff --git a/delft3dcontainermanager/test_tasks.py b/delft3dcontainermanager/test_tasks.py
@@ -1,12 +1,10 @@
 from __future__ import absolute_import
 
-import os
-import sys
 from time import time
 
 from django.test import TestCase
 from fakeredis import FakeStrictRedis
-from mock import MagicMock, Mock, patch
+from mock import Mock, patch
 
 from delft3dcontainermanager.tasks import (
     delft3dgt_kube_pulse,

diff --git a/delft3dgtfrontend/admin.py b/delft3dgtfrontend/admin.py
@@ -1,5 +1,3 @@
 from __future__ import absolute_import
 
-from django.contrib import admin
-
 # Register your models here.
diff --git a/delft3dgtfrontend/models.py b/delft3dgtfrontend/models.py
@@ -1,5 +1,3 @@
 from __future__ import absolute_import, unicode_literals
 
-from django.db import models
-
 # Create your models here.
diff --git a/delft3dgtfrontend/tests.py b/delft3dgtfrontend/tests.py
@@ -1,5 +1,3 @@
 from __future__ import absolute_import
 
-from django.test import TestCase
-
 # Create your tests here.
diff --git a/delft3dgtfrontend/urls.py b/delft3dgtfrontend/urls.py
@@ -1,7 +1,6 @@
 from __future__ import absolute_import
 
 from django.conf import settings  # noqa
-from django.conf.urls import url  # noqa
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.views import (
     LoginView,
@@ -11,43 +10,45 @@
     PasswordResetDoneView,
     PasswordResetView,
 )
+from django.urls import re_path  # noqa
+from django.urls import path
 from django.views.static import serve
 
 urlpatterns = (
     # Login
-    url(r"^login/$", LoginView.as_view(template_name="login.html"), name="login"),
+    path("login/", LoginView.as_view(template_name="login.html"), name="login"),
     # Logout
-    url(r"^logout/$", LogoutView.as_view(template_name="login.html"), name="logout"),
+    path("logout/", LogoutView.as_view(template_name="login.html"), name="logout"),
     # Password Reset
-    url(
-        r"^forgot/$",
+    path(
+        "forgot/",
         PasswordResetView.as_view(template_name="forgot.html"),
         name="password_reset",
     ),
     # Password Reset Done
-    url(
-        r"^done/$",
+    path(
+        "done/",
         PasswordResetDoneView.as_view(template_name="done.html"),
         name="password_reset_done",
     ),
     # Password Reset Done
-    url(
+    re_path(
         r"^reset/(?P<uidb64>[0-9A-Za-z]+)-(?P<token>.+)/$",
         PasswordResetConfirmView.as_view(template_name="confirm.html"),
         name="password_reset_confirm",
     ),
     # Password Reset Complete
-    url(
-        r"^complete/$",
+    path(
+        "complete/",
         PasswordResetCompleteView.as_view(template_name="complete.html"),
         name="password_reset_complete",
     ),
     # Index
-    url(
-        r"^$",
+    path(
+        "",
         login_required(serve),
         {"document_root": settings.STATIC_ROOT, "path": "index.html"},
     ),
     # static files/
-    url(r"^(?P<path>.*)$", serve, {"document_root": settings.STATIC_ROOT}),
+    re_path(r"^(?P<path>.*)$", serve, {"document_root": settings.STATIC_ROOT}),
 )
diff --git a/delft3dgtmain/__init__.py b/delft3dgtmain/__init__.py
@@ -1,5 +1,5 @@
-from __future__ import absolute_import
-
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
-from .celery import app as celery_app  # noqa
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)
diff --git a/delft3dgtmain/celery.py b/delft3dgtmain/celery.py
@@ -2,11 +2,11 @@
 
 import os
 
+from celery import Celery
 from ddtrace import patch_all
+from django.conf import settings  # noqa
 
 patch_all()
-from celery import Celery
-from django.conf import settings  # noqa
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "delft3dgtmain.settings")

diff --git a/delft3dgtmain/provisionedsettings.sample.py b/delft3dgtmain/provisionedsettings.sample.py
@@ -68,5 +68,5 @@
 OIDC_OP_TOKEN_ENDPOINT = ""
 OIDC_OP_USER_ENDPOINT = ""
 OIDC_OP_JWKS_ENDPOINT = ""
-LOGIN_REDIRECT_URL = ""
-LOGOUT_REDIRECT_URL = ""
+LOGIN_REDIRECT_URL = "/"
+LOGOUT_REDIRECT_URL = "/"
diff --git a/delft3dgtmain/settings.py b/delft3dgtmain/settings.py
@@ -13,6 +13,7 @@
 import os
 import sys
 from datetime import timedelta
+
 from ddtrace import config
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -123,13 +124,11 @@
 
 USE_I18N = True
 
-USE_L10N = True
 
 USE_TZ = True
 
 # Login
 LOGIN_URL = "/login/"
-LOGIN_REDIRECT_URL = "/"
 
 # Max form size for large scenarios with logs
 DATA_UPLOAD_MAX_MEMORY_SIZE = 20971520  # 20MB
@@ -199,7 +198,7 @@
 
 # import provisioned settings
 try:
-    from .provisionedsettings import *
+    from .provisionedsettings import *  # noqa
 except ImportError:
     print("Failed to import provisioned settings!")
     SECRET_KEY = "test"
@@ -222,25 +221,24 @@
 
 # TESTING
 if "test" in sys.argv:
-
     import logging
 
     from .celery import Celery
 
     logging.disable(logging.CRITICAL)
 
-    if "TRAVIS" in os.environ:
-        DATABASES = {
-            "default": {
-                "ENGINE": "django.db.backends.postgresql_psycopg2",
-                "NAME": "travis_ci_test",
-                "USER": "postgres",
-                "HOST": "localhost",
-                "PORT": "5432",
-            }
+    STATICFILES_DIRS = []
+
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.postgresql",
+            "NAME": "djangodb_test",
+            "USER": "postgres",
+            "PASSWORD": "postgres",
+            "HOST": "localhost",
+            "PORT": "5432",
         }
-    else:
-        DATABASES["default"].update({"NAME": "djangodb_test"})
+    }
 
     PASSWORD_HASHERS = [
         "django.contrib.auth.hashers.MD5PasswordHasher",
@@ -300,3 +298,14 @@
 
     # Docker URL this setting is from the delf3dcontainermanger app
     DOCKER_URL = "unix:///var/run/docker.sock"
+
+    # OIDC
+    OIDC_RP_CLIENT_ID = ""
+    OIDC_RP_CLIENT_SECRET = ""
+    OIDC_RP_SIGN_ALGO = ""
+    OIDC_OP_AUTHORIZATION_ENDPOINT = ""
+    OIDC_OP_TOKEN_ENDPOINT = ""
+    OIDC_OP_USER_ENDPOINT = ""
+    OIDC_OP_JWKS_ENDPOINT = ""
+    LOGIN_REDIRECT_URL = "/"
+    LOGOUT_REDIRECT_URL = "/"
diff --git a/delft3dgtmain/urls.py b/delft3dgtmain/urls.py
@@ -14,17 +14,17 @@
     2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
 """
 from django.conf import settings
-from django.conf.urls import include, url
 from django.conf.urls.static import static
 from django.contrib import admin
+from django.urls import include, path, re_path
 
 urlpatterns = [
     # Django Admin
-    url(r"^admin/", admin.site.urls),
+    re_path(r"^admin/", admin.site.urls),
     # Delft3D-GT Worker API
-    url(r"^", include("delft3dworker.urls")),
+    path("", include("delft3dworker.urls")),
     # Delft3D-GT Protected Services
-    url(r"^", include("delft3dgtprotectedservices.urls")),
+    path("", include("delft3dgtprotectedservices.urls")),
     # Delft3D-GT Frontend
-    url(r"^", include("delft3dgtfrontend.urls")),
+    path("", include("delft3dgtfrontend.urls")),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)