WIP batch mode and migration

.devcontainer/.dev_config.yaml

@@ -0,0 +1,14 @@
+service_name: dins
+
+service_instance_id: '1'
+kafka_servers: ["kafka:9092"]
+
+db_connection_str: "mongodb://mongodb:27017"
+db_name: "dev_db"
+
+dataset_change_event_topic: metadata_datasets
+dataset_deletion_event_type: dataset_deleted
+dataset_upsertion_event_type: dataset_created
+files_to_delete_topic: file_deletions
+file_registered_event_topic: internal_file_registry
+file_registered_event_type: file_registered

.devcontainer/Dockerfile

@@ -0,0 +1,19 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.12-bookworm
+
+ENV PYTHONUNBUFFERED 1
+
+# Update args in docker-compose.yaml to set the UID/GID of the "vscode" user.
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN if [ "$USER_GID" != "1000" ] || [ "$USER_UID" != "1000" ]; then groupmod --gid $USER_GID vscode && usermod --uid $USER_UID --gid $USER_GID vscode; fi
+
+# [Option] Install Node.js
+ARG INSTALL_NODE="false"
+ARG NODE_VERSION="lts/*"
+RUN if [ "${INSTALL_NODE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# Copy install and launcher script to bin:
+COPY ./dev_install /bin
+COPY ./dev_launcher /bin
+
+CMD ["sleep", "infinity"]

.devcontainer/dev_install

@@ -0,0 +1,16 @@
+#!/bin/bash
+# install service in dev container
+
+cd /workspace
+
+# upgrade pip
+python -m pip install --upgrade pip
+
+# install or upgrade dependencies for development and testing
+pip install --no-deps -r ./lock/requirements-dev.txt
+
+# install the package itself in edit mode:
+pip install --no-deps -e .
+
+# install pre-commit hooks to git
+pre-commit install

.devcontainer/dev_launcher

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+dins

.devcontainer/devcontainer.json

@@ -0,0 +1,75 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/python-3-postgres
+{
+	"name": "${localWorkspaceFolderBasename}",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "app",
+	"workspaceFolder": "/workspace",
+	"customizations": {
+		"vscode": {
+			// Set *default* container specific settings.json values on container create.
+			"settings": {
+				"files.eol": "\n",
+				"terminal.integrated.profiles.linux": {
+					"bash": {
+						"path": "/bin/bash"
+					}
+				},
+				"python.pythonPath": "/usr/local/bin/python",
+				"python.languageServer": "Pylance",
+				"python.analysis.typeCheckingMode": "basic",
+				"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest",
+				"python.testing.unittestEnabled": false,
+				"python.testing.pytestEnabled": true,
+				"editor.codeActionsOnSave": {
+					"source.organizeImports": "explicit"
+				},
+				"editor.formatOnSave": true,
+				"editor.renderWhitespace": "all",
+				"editor.rulers": [
+					88
+				],
+				"ruff.organizeImports": true,
+				"editor.defaultFormatter": "charliermarsh.ruff",
+				"licenser.license": "Custom",
+				"licenser.customHeaderFile": "/workspace/.devcontainer/license_header.txt"
+			},
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+				"mikestead.dotenv",
+				"ms-azuretools.vscode-docker",
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"ms-toolsai.jupyter",
+				"njpwerner.autodocstring",
+				"redhat.vscode-yaml",
+				"42crunch.vscode-openapi",
+				"arjun.swagger-viewer",
+				"eamodio.gitlens",
+				"github.vscode-pull-request-github",
+				"streetsidesoftware.code-spell-checker",
+				"yzhang.markdown-all-in-one",
+				"visualstudioexptteam.vscodeintellicode",
+				"ymotongpoo.licenser",
+				"charliermarsh.ruff",
+				"ms-python.mypy-type-checker",
+				"-ms-python.autopep8"
+			]
+		}
+	},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [5000, 5432],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "dev_install",
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"containerEnv": {
+		// for testcontainers to connect to the docker host:
+		"TC_HOST": "host.docker.internal",
+		"DOCKER_HOST": "unix:///var/run/docker.sock"
+	},
+	"features": {
+		// details can be found here: https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
+	}
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,43 @@
+services:
+  app:
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+      args:
+        # [Choice] Install Node.js
+        INSTALL_NODE: "false"
+        NODE_VERSION: "lts/*"
+        # Please adapt to package name:
+        PACKAGE_NAME: "dataset_information_service"
+        # On Linux, you may need to update USER_UID and USER_GID below if not your local UID is not 1000.
+        USER_UID: 1000
+        USER_GID: 1000
+
+    init: true
+
+    # Makes testcontainers work on linux based hosts
+    extra_hosts:
+      - host.docker.internal:host-gateway
+
+    volumes:
+      - ..:/workspace:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Uncomment the next line to use a non-root user for all processes.
+    user: vscode
+
+    # define environment variables
+    environment:
+      # Please adapt to package name:
+      DINS_CONFIG_YAML: /workspace/.devcontainer/.dev_config.yaml
+
+  mongodb:
+    image: mongo:latest
+    restart: unless-stopped
+    volumes:
+      - mongo_fs:/data/db
+
+volumes:
+  mongo_fs: {}

.devcontainer/license_header.txt

@@ -0,0 +1,14 @@
+Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+for the German Human Genome-Phenome Archive (GHGA)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

.gitattributes

@@ -0,0 +1,22 @@
+# CRLF line endings cause problems in Docker, so we force git
+# to check in and out using only LF line endings.
+
+*.cfg					text eol=lf
+*.ini					text eol=lf
+*.json					text eol=lf
+*.md					text eol=lf
+*.py					text eol=lf
+*.txt					text eol=lf
+*.yaml					text eol=lf
+*.yml					text eol=lf
+
+.editorconfig			text eol=lf
+.flake8					text eol=lf
+.pylintrc				text eol=lf
+
+.git*					text eol=lf
+*_files					text eol=lf
+*_files_ignore			text eol=lf
+
+**/dev_*				text eol=lf
+**/Dockerfile			text eol=lf

.github/workflows/check_config_docs.yaml

@@ -0,0 +1,25 @@
+name: Check if the config schema and the example are up to date
+
+on: push
+
+jobs:
+  static-code-analysis:
+    name: Check config schema and example
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Common steps
+        id: common
+        uses: ghga-de/gh-action-common@v6
+
+      - name: Check config docs
+        id: check-config-docs
+        run: |
+          export ${{ steps.common.outputs.CONFIG_YAML_ENV_VAR_NAME }}="${{ steps.common.outputs.CONFIG_YAML }}"
+
+          ./scripts/update_config_docs.py --check

.github/workflows/check_openapi_spec.yaml

@@ -0,0 +1,26 @@
+# This file is only needed if your repository uses FastAPI
+name: Check if OpenAPI spec is up to date
+
+on: push
+
+jobs:
+  static-code-analysis:
+    name: Check OpenAPI spec
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Common steps
+        id: common
+        uses: ghga-de/gh-action-common@v6
+
+      - name: Check openapi.yaml
+        id: check-openapi-docs
+        run: |
+          export ${{ steps.common.outputs.CONFIG_YAML_ENV_VAR_NAME }}="${{ steps.common.outputs.CONFIG_YAML }}"
+
+          ./scripts/update_openapi_docs.py --check

.github/workflows/check_pyproject.yaml

@@ -0,0 +1,23 @@
+name: Check if pyproject.toml file is up to date
+
+on: push
+
+jobs:
+  static-code-analysis:
+    name: Check pyproject file
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Common steps
+        id: common
+        uses: ghga-de/gh-action-common@v6
+
+      - name: Check pyproject.toml
+        id: check-pyproject
+        run: |
+          ./scripts/update_pyproject.py --check

.github/workflows/check_readme.yaml

@@ -0,0 +1,23 @@
+name: Check if the README file is up to date
+
+on: push
+
+jobs:
+  static-code-analysis:
+    name: Check README file
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Common steps
+        id: common
+        uses: ghga-de/gh-action-common@v6
+
+      - name: Check README
+        id: check-readme
+        run: |
+          ./scripts/update_readme.py --check

.github/workflows/check_template_files.yaml

@@ -0,0 +1,30 @@
+name: Check template files
+
+on: push
+
+jobs:
+  check-template-files:
+    name: Check template files
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Check template files
+        id: check-template-files
+        run: |
+          if [ "${{ github.event.repository.name }}" == "microservice-repository-template" ]
+          then
+            echo "Skipping this test as operating on the template repo."
+          else
+            ./scripts/update_template_files.py --check
+          fi

.github/workflows/ci_release.yaml

@@ -0,0 +1,23 @@
+name: CI on release
+
+on:
+  release:
+      types: [published]
+
+jobs:
+  push_to_docker_hub:
+    name: Push to Docker Hub
+
+    strategy:
+      matrix:
+        flavor: ["", "debian"]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: ghga-de/gh-action-ci@v1
+        with:
+          tag: ${{ github.event.release.tag_name }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+          flavor: ${{ matrix.flavor }}