Merge branch 'master' into drop-pkg_resources

# Conflicts:
#	requirements.txt

.circleci/config.yml

@@ -1,7 +1,7 @@
-version: 2
+version: 2.1
 
 orbs:
-    python: circleci/python@2.0.3
+    python: circleci/python@2.1.1
 
 jobs:
 

.dockerignore

@@ -1,8 +1,12 @@
 *
 !src*
 !pyproject.toml
+!requirements.txt
 !Makefile
 !VERSION
 !LICENSE
 !README.md
 !.git
+!tests
+!requirements_test.txt
+!.gitmodules

.github/workflows/docker-image.yml

@@ -28,6 +28,10 @@ jobs:
     - name: Build the Docker image with GPU support
       # default tag uses docker.io, so override on command-line
       run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }}
+    - name: Smoke Test that ocrd --help works
+      run: |
+        docker run --rm ${{ env.DOCKER_TAG }} ocrd --version
+        docker run --rm ${{ env.DOCKER_TAG }}-cuda ocrd --version
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
       with:

.github/workflows/integration-test.yml

@@ -0,0 +1,58 @@
+name: Run ocrd network integration tests
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          # - macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+            sudo apt-get -y update
+            sudo make deps-ubuntu
+          else
+            HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+            HOMEBREW_NO_AUTO_UPDATE=1 \
+            brew install imagemagick geos bash  # opencv
+          fi
+          make install deps-test
+      - name: Install Docker on macOS
+        if: runner.os == 'macos'
+        run: |
+          brew install docker docker-compose
+          colima start
+      - name: Test network integration with pytest
+        run: |
+          if [[ "${{ matrix.os }}" == "macos"* ]];then
+            make integration-test DOCKER_COMPOSE=docker-compose
+          else
+            make integration-test
+          fi

CHANGELOG.md

@@ -5,6 +5,24 @@ Versioned according to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+Added:
+
+  * Basic integration test for `ocrd_network`, #1164
+
+Fixed:
+
+  * METS Server: UDS sockets are removed on process exit, #117
+
+Changed:
+
+  * replace license-incompatible sparkline library with a simpler implementation, #1176
+
+## [2.61.2] - 2024-01-24
+
+Fixed:
+
+  * another regression to docker deployment (requirements.txt missing), #1173
+
 ## [2.61.1] - 2024-01-23
 
 Fixed:
@@ -1977,6 +1995,7 @@ Fixed
 Initial Release
 
 <!-- link-labels -->
+[2.61.2]: ../../compare/v2.61.2..v2.61.1
 [2.61.1]: ../../compare/v2.61.1..v2.61.1
 [2.61.0]: ../../compare/v2.61.0..v2.60.3
 [2.60.3]: ../../compare/v2.60.3..v2.60.2

Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE
-FROM $BASE_IMAGE
+FROM $BASE_IMAGE as ocrd_core_base
 ARG FIXUP=echo
 MAINTAINER OCR-D
 ENV DEBIAN_FRONTEND noninteractive
@@ -13,6 +13,7 @@ WORKDIR /build-ocrd
 COPY src ./src
 COPY pyproject.toml .
 COPY VERSION ./VERSION
+COPY requirements.txt ./requirements.txt
 RUN mv ./src/ocrd_utils/ocrd_logging.conf /etc
 COPY Makefile .
 COPY README.md .
@@ -32,13 +33,25 @@ RUN apt-get update && apt-get -y install software-properties-common \
         curl \
         sudo \
         git \
-    && make deps-ubuntu \
-    && python3 -m venv /usr/local \
+    && make deps-ubuntu
+RUN python3 -m venv /usr/local \
     && hash -r \
     && make install \
-    && eval $FIXUP \
-    && rm -rf /build-ocrd
+    && eval $FIXUP
 
 WORKDIR /data
 
 CMD ["/usr/local/bin/ocrd", "--help"]
+
+FROM ocrd_core_base as ocrd_core_test
+WORKDIR /build-ocrd
+COPY Makefile .
+RUN make assets
+COPY tests ./tests
+COPY .gitmodules .
+COPY requirements_test.txt .
+RUN pip install -r requirements_test.txt
+RUN mkdir /ocrd-data && chmod 777 /ocrd-data
+
+CMD ["yes"]
+# CMD ["make", "test", "integration-test"]

Dockerfile.cuda

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE
-FROM $BASE_IMAGE
+FROM $BASE_IMAGE AS ocrd_core_base
 
 ENV MAMBA_EXE=/usr/local/bin/conda
 ENV MAMBA_ROOT_PREFIX=/conda

Makefile

@@ -9,6 +9,8 @@ TESTDIR = $(CURDIR)/tests
 PYTEST_ARGS = --continue-on-collection-errors
 VERSION = $(shell cat VERSION)
 
+DOCKER_COMPOSE = docker compose
+
 SPHINX_APIDOC =
 
 BUILD_ORDER = ocrd_utils ocrd_models ocrd_modelfactory ocrd_validators ocrd_network ocrd
@@ -213,9 +215,16 @@ test: assets
 	$(PYTHON) \
 		-m pytest $(PYTEST_ARGS) --durations=10\
 		--ignore-glob="$(TESTDIR)/**/*bench*.py" \
+		--ignore-glob="$(TESTDIR)/network/*.py" \
 		$(TESTDIR)
 	cd ocrd_utils ; $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging -k TestDecorators $(TESTDIR)
 
+INTEGRATION_TEST_IN_DOCKER = docker exec core_test
+integration-test:
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d
+	-$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v
+	$(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down --remove-orphans
+
 benchmark:
 	$(PYTHON) -m pytest $(TESTDIR)/model/test_ocrd_mets_bench.py
 
@@ -297,7 +306,7 @@ docker-cuda: DOCKER_FILE = Dockerfile.cuda
 docker-cuda: docker
 
 docker docker-cuda: 
-	docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) .
+	docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --target ocrd_core_base --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) .
 
 # Build wheels and source dist and twine upload them
 pypi: build

VERSION

@@ -1 +1 @@
-2.61.1
+2.61.2

requirements.txt

@@ -30,7 +30,6 @@ pyyaml
 requests < 2.30
 requests_unixsocket
 shapely
-sparklines >= 0.4.2
 uvicorn
 uvicorn>=0.17.6
 

src/ocrd/mets_server.py

@@ -2,24 +2,24 @@
 # METS server functionality
 """
 import re
-from os import environ, _exit, chmod
-from io import BytesIO
-from typing import Any, Dict, Optional, Union, List, Tuple
+from os import _exit, chmod
+from typing import Dict, Optional, Union, List, Tuple
 from pathlib import Path
 from urllib.parse import urlparse
 import socket
+import atexit
 
-from fastapi import FastAPI, Request, File, Form, Response
+from fastapi import FastAPI, Request, Form, Response
 from fastapi.responses import JSONResponse
-from requests import request, Session as requests_session
+from requests import Session as requests_session
 from requests.exceptions import ConnectionError
 from requests_unixsocket import Session as requests_unixsocket_session
 from pydantic import BaseModel, Field, ValidationError
 
 import uvicorn
 
-from ocrd_models import OcrdMets, OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
-from ocrd_utils import initLogging, getLogger, deprecated_alias
+from ocrd_models import OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
+from ocrd_utils import getLogger, deprecated_alias
 
 #
 # Models
@@ -197,9 +197,10 @@ def __init__(self, workspace, url):
         self.log = getLogger(f'ocrd.mets_server[{self.url}]')
 
     def shutdown(self):
-        self.log.info("Shutting down METS server")
         if self.is_uds:
-            Path(self.url).unlink()
+            if Path(self.url).exists():
+                self.log.warning(f'UDS socket {self.url} still exists, removing it')
+                Path(self.url).unlink()
         # os._exit because uvicorn catches SystemExit raised by sys.exit
         _exit(0)
 
@@ -296,7 +297,7 @@ async def stop():
             """
             Stop the server
             """
-            getLogger('ocrd.models.ocrd_mets').info('Shutting down')
+            getLogger('ocrd.models.ocrd_mets').info(f'Shutting down METS Server {self.url}')
             workspace.save_mets()
             self.shutdown()
 
@@ -308,6 +309,7 @@ async def stop():
             self.log.debug(f"chmod 0o677 {self.url}")
             server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
             server.bind(self.url)  # creates the socket file
+            atexit.register(self.shutdown)
             server.close()
             chmod(self.url, 0o666)
             uvicorn_kwargs = {'uds': self.url}

src/ocrd/processor/helpers.py

@@ -11,7 +11,7 @@
 
 from click import wrap_text
 from ocrd.workspace import Workspace
-from ocrd_utils import freeze_args, getLogger, config, setOverrideLogLevel, getLevelName
+from ocrd_utils import freeze_args, getLogger, config, setOverrideLogLevel, getLevelName, sparkline
 
 
 __all__ = [
@@ -106,7 +106,6 @@ def run_processor(
     if any(x in config.OCRD_PROFILE for x in ['RSS', 'PSS']):
         backend = 'psutil_pss' if 'PSS' in config.OCRD_PROFILE else 'psutil'
         from memory_profiler import memory_usage
-        from sparklines import sparklines
         try:
             mem_usage = memory_usage(proc=processor.process,
                                      # only run process once
@@ -123,7 +122,7 @@ def run_processor(
             chdir(old_cwd)
         mem_usage_values = [mem for mem, _ in mem_usage]
         mem_output = 'memory consumption: '
-        mem_output += ''.join(sparklines(mem_usage_values))
+        mem_output += sparkline(mem_usage_values)
         mem_output += ' max: %.2f MiB min: %.2f MiB' % (max(mem_usage_values), min(mem_usage_values))
         logProfile.info(mem_output)
     else:

src/ocrd/workspace.py

@@ -82,7 +82,7 @@ def __init__(self, resolver, directory, mets=None, mets_basename=DEFAULT_METS_BA
                 mets = ClientSideOcrdMets(mets_server_url)
                 if mets.workspace_path != self.directory:
                     raise ValueError(f"METS server {mets_server_url} workspace directory {mets.workspace_path} differs "
-                            "from local workspace directory {self.directory}. These are not the same workspaces.")
+                            f"from local workspace directory {self.directory}. These are not the same workspaces.")
             else:
                 mets = OcrdMets(filename=self.mets_target)
         self.mets = mets