Merge branch 'master' into circle-to-gha

# Conflicts:
#	.circleci/config.yml

.dockerignore

@@ -7,3 +7,6 @@
 !LICENSE
 !README.md
 !.git
+!tests
+!requirements_test.txt
+!.gitmodules

.github/workflows/integration-test.yml

@@ -0,0 +1,58 @@
+name: Run ocrd network integration tests
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          # - macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+            sudo apt-get -y update
+            sudo make deps-ubuntu
+          else
+            HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+            HOMEBREW_NO_AUTO_UPDATE=1 \
+            brew install imagemagick geos bash  # opencv
+          fi
+          make install deps-test
+      - name: Install Docker on macOS
+        if: runner.os == 'macos'
+        run: |
+          brew install docker docker-compose
+          colima start
+      - name: Test network integration with pytest
+        run: |
+          if [[ "${{ matrix.os }}" == "macos"* ]];then
+            make integration-test DOCKER_COMPOSE=docker-compose
+          else
+            make integration-test
+          fi

CHANGELOG.md

@@ -5,6 +5,14 @@ Versioned according to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+Added:
+
+  * Basic integration test for `ocrd_network`, #1164
+
+Fixed:
+
+  * METS Server: UDS sockets are removed on process exit, #117
+
 ## [2.61.2] - 2024-01-24
 
 Fixed:

Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE
-FROM $BASE_IMAGE
+FROM $BASE_IMAGE as ocrd_core_base
 ARG FIXUP=echo
 MAINTAINER OCR-D
 ENV DEBIAN_FRONTEND noninteractive
@@ -33,13 +33,25 @@ RUN apt-get update && apt-get -y install software-properties-common \
         curl \
         sudo \
         git \
-    && make deps-ubuntu \
-    && python3 -m venv /usr/local \
+    && make deps-ubuntu
+RUN python3 -m venv /usr/local \
     && hash -r \
     && make install \
-    && eval $FIXUP \
-    && rm -rf /build-ocrd
+    && eval $FIXUP
 
 WORKDIR /data
 
 CMD ["/usr/local/bin/ocrd", "--help"]
+
+FROM ocrd_core_base as ocrd_core_test
+WORKDIR /build-ocrd
+COPY Makefile .
+RUN make assets
+COPY tests ./tests
+COPY .gitmodules .
+COPY requirements_test.txt .
+RUN pip install -r requirements_test.txt
+RUN mkdir /ocrd-data && chmod 777 /ocrd-data
+
+CMD ["yes"]
+# CMD ["make", "test", "integration-test"]

Makefile

@@ -9,6 +9,8 @@ TESTDIR = $(CURDIR)/tests
 PYTEST_ARGS = --continue-on-collection-errors
 VERSION = $(shell cat VERSION)
 
+DOCKER_COMPOSE = docker compose
+
 SPHINX_APIDOC =
 
 BUILD_ORDER = ocrd_utils ocrd_models ocrd_modelfactory ocrd_validators ocrd_network ocrd
@@ -213,9 +215,16 @@ test: assets
 	$(PYTHON) \
 		-m pytest $(PYTEST_ARGS) --durations=10\
 		--ignore-glob="$(TESTDIR)/**/*bench*.py" \
+		--ignore-glob="$(TESTDIR)/network/*.py" \
 		$(TESTDIR)
 	cd ocrd_utils ; $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging -k TestDecorators $(TESTDIR)
 
+INTEGRATION_TEST_IN_DOCKER = docker exec core_test
+integration-test:
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d
+	-$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down --remove-orphans
+
 benchmark:
 	$(PYTHON) -m pytest $(TESTDIR)/model/test_ocrd_mets_bench.py
 
@@ -296,7 +305,7 @@ docker-cuda: DOCKER_FILE = Dockerfile.cuda
 docker-cuda: docker
 
 docker docker-cuda: 
-	docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) .
+	docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --target ocrd_core_base --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) .
 
 # Build wheels and source dist and twine upload them
 pypi: build

src/ocrd/mets_server.py

@@ -2,24 +2,24 @@
 # METS server functionality
 """
 import re
-from os import environ, _exit, chmod
-from io import BytesIO
-from typing import Any, Dict, Optional, Union, List, Tuple
+from os import _exit, chmod
+from typing import Dict, Optional, Union, List, Tuple
 from pathlib import Path
 from urllib.parse import urlparse
 import socket
+import atexit
 
-from fastapi import FastAPI, Request, File, Form, Response
+from fastapi import FastAPI, Request, Form, Response
 from fastapi.responses import JSONResponse
-from requests import request, Session as requests_session
+from requests import Session as requests_session
 from requests.exceptions import ConnectionError
 from requests_unixsocket import Session as requests_unixsocket_session
 from pydantic import BaseModel, Field, ValidationError
 
 import uvicorn
 
-from ocrd_models import OcrdMets, OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
-from ocrd_utils import initLogging, getLogger, deprecated_alias
+from ocrd_models import OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
+from ocrd_utils import getLogger, deprecated_alias
 
 #
 # Models
@@ -197,9 +197,10 @@ def __init__(self, workspace, url):
         self.log = getLogger(f'ocrd.mets_server[{self.url}]')
 
     def shutdown(self):
-        self.log.info("Shutting down METS server")
         if self.is_uds:
-            Path(self.url).unlink()
+            if Path(self.url).exists():
+                self.log.warning(f'UDS socket {self.url} still exists, removing it')
+                Path(self.url).unlink()
         # os._exit because uvicorn catches SystemExit raised by sys.exit
         _exit(0)
 
@@ -296,7 +297,7 @@ async def stop():
             """
             Stop the server
             """
-            getLogger('ocrd.models.ocrd_mets').info('Shutting down')
+            getLogger('ocrd.models.ocrd_mets').info(f'Shutting down METS Server {self.url}')
             workspace.save_mets()
             self.shutdown()
 
@@ -308,6 +309,7 @@ async def stop():
             self.log.debug(f"chmod 0o677 {self.url}")
             server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
             server.bind(self.url)  # creates the socket file
+            atexit.register(self.shutdown)
             server.close()
             chmod(self.url, 0o666)
             uvicorn_kwargs = {'uds': self.url}

src/ocrd/workspace.py

@@ -82,7 +82,7 @@ def __init__(self, resolver, directory, mets=None, mets_basename=DEFAULT_METS_BA
                 mets = ClientSideOcrdMets(mets_server_url)
                 if mets.workspace_path != self.directory:
                     raise ValueError(f"METS server {mets_server_url} workspace directory {mets.workspace_path} differs "
-                            "from local workspace directory {self.directory}. These are not the same workspaces.")
+                            f"from local workspace directory {self.directory}. These are not the same workspaces.")
             else:
                 mets = OcrdMets(filename=self.mets_target)
         self.mets = mets

src/ocrd_network/database.py

@@ -28,17 +28,17 @@
 from .utils import call_sync
 
 
-async def initiate_database(db_url: str):
+async def initiate_database(db_url: str, db_name: str = 'ocrd'):
     client = AsyncIOMotorClient(db_url)
     await init_beanie(
-        database=client.get_default_database(default='ocrd'),
+        database=client.get_default_database(default=db_name),
         document_models=[DBProcessorJob, DBWorkflowJob, DBWorkspace, DBWorkflowScript]
     )
 
 
 @call_sync
-async def sync_initiate_database(db_url: str):
-    await initiate_database(db_url)
+async def sync_initiate_database(db_url: str, db_name: str = 'ocrd'):
+    await initiate_database(db_url, db_name)
 
 
 async def db_create_workspace(mets_path: str) -> DBWorkspace:
@@ -60,6 +60,11 @@ async def db_create_workspace(mets_path: str) -> DBWorkspace:
         return workspace_db
 
 
+@call_sync
+async def sync_db_create_workspace(mets_path: str) -> DBWorkspace:
+    return await db_create_workspace(mets_path=mets_path)
+
+
 async def db_get_workspace(workspace_id: str = None, workspace_mets_path: str = None) -> DBWorkspace:
     workspace = None
     if not workspace_id and not workspace_mets_path:
@@ -134,6 +139,15 @@ async def sync_db_update_workspace(workspace_id: str = None, workspace_mets_path
     return await db_update_workspace(workspace_id=workspace_id, workspace_mets_path=workspace_mets_path, **kwargs)
 
 
+async def db_create_processing_job(db_processing_job: DBProcessorJob) -> DBProcessorJob:
+    return await db_processing_job.insert()
+
+
+@call_sync
+async def sync_db_create_processing_job(db_processing_job: DBProcessorJob) -> DBProcessorJob:
+    return await db_create_processing_job(db_processing_job=db_processing_job)
+
+
 async def db_get_processing_job(job_id: str) -> DBProcessorJob:
     job = await DBProcessorJob.find_one(
         DBProcessorJob.job_id == job_id)
@@ -180,6 +194,15 @@ async def sync_db_update_processing_job(job_id: str, **kwargs) -> DBProcessorJob
     return await db_update_processing_job(job_id=job_id, **kwargs)
 
 
+async def db_create_workflow_job(db_workflow_job: DBWorkflowJob) -> DBWorkflowJob:
+    return await db_workflow_job.insert()
+
+
+@call_sync
+async def sync_db_create_workflow_job(db_workflow_job: DBWorkflowJob) -> DBWorkflowJob:
+    return await db_create_workflow_job(db_workflow_job=db_workflow_job)
+
+
 async def db_get_workflow_job(job_id: str) -> DBWorkflowJob:
     job = await DBWorkflowJob.find_one(DBWorkflowJob.job_id == job_id)
     if not job:
@@ -202,6 +225,15 @@ async def sync_db_get_processing_jobs(job_ids: List[str]) -> [DBProcessorJob]:
     return await db_get_processing_jobs(job_ids)
 
 
+async def db_create_workflow_script(db_workflow_script: DBWorkflowScript) -> DBWorkflowScript:
+    return await db_workflow_script.insert()
+
+
+@call_sync
+async def sync_db_create_workflow_script(db_workflow_script: DBWorkflowScript) -> DBWorkflowScript:
+    return await db_create_workflow_script(db_workflow_script=db_workflow_script)
+
+
 async def db_get_workflow_script(workflow_id: str) -> DBWorkflowScript:
     workflow = await DBWorkflowScript.find_one(DBWorkflowScript.workflow_id == workflow_id)
     if not workflow:
@@ -221,6 +253,7 @@ async def db_find_first_workflow_script_by_content(content_hash: str) -> DBWorkf
     return workflow
 
 
+# TODO: Resolve the inconsistency between the async and sync versions of the same method
 @call_sync
 async def sync_db_find_first_workflow_script_by_content(workflow_id: str) -> DBWorkflowScript:
     return await db_get_workflow_script(workflow_id)

src/ocrd_network/models/job.py

@@ -20,6 +20,8 @@ class StateEnum(str, Enum):
     success = 'SUCCESS'
     # Processing job failed
     failed = 'FAILED'
+    # Processing job has not been assigned yet
+    unset = 'UNSET'
 
 
 class PYJobInput(BaseModel):