Merge branch 'master' into ocrd-mets-get-pages-for-pageids

# Conflicts:
#	src/ocrd_models/constants.py

.circleci/config.yml

@@ -1,7 +1,7 @@
-version: 2
+version: 2.1
 
 orbs:
-    python: circleci/python@2.0.3
+    python: circleci/python@2.1.1
 
 jobs:
 
@@ -16,9 +16,10 @@ jobs:
       PIP_INSTALL: pip3 install --user
     steps:
       - checkout
-      - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash
+      - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash opencv
+      - run: $PIP_INSTALL -U pip setuptools
       - run: make install
-      - run: PATH="/Users/distiller/Library/Python/3.9/bin:$PATH" make deps-test test benchmark
+      - run: export PATH="/Users/distiller/Library/Python/3.9/bin:$PATH"; make deps-test test benchmark
 
   test-python37:
     docker:
@@ -121,4 +122,4 @@ workflows:
       - test-python39
       - test-python310
       - test-python311
-      - test-macos
+      # - test-macos

.dockerignore

@@ -1,6 +1,12 @@
 *
-!ocrd*
+!src*
+!pyproject.toml
+!requirements.txt
 !Makefile
+!VERSION
 !LICENSE
 !README.md
 !.git
+!tests
+!requirements_test.txt
+!.gitmodules

.github/workflows/docker-image.yml

@@ -28,6 +28,10 @@ jobs:
     - name: Build the Docker image with GPU support
       # default tag uses docker.io, so override on command-line
       run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }}
+    - name: Smoke Test that ocrd --help works
+      run: |
+        docker run --rm ${{ env.DOCKER_TAG }} ocrd --version
+        docker run --rm ${{ env.DOCKER_TAG }}-cuda ocrd --version
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
       with:

.github/workflows/integration-test.yml

@@ -0,0 +1,58 @@
+name: Run ocrd network integration tests
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          # - macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+            sudo apt-get -y update
+            sudo make deps-ubuntu
+          else
+            HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+            HOMEBREW_NO_AUTO_UPDATE=1 \
+            brew install imagemagick geos bash  # opencv
+          fi
+          make install deps-test
+      - name: Install Docker on macOS
+        if: runner.os == 'macos'
+        run: |
+          brew install docker docker-compose
+          colima start
+      - name: Test network integration with pytest
+        run: |
+          if [[ "${{ matrix.os }}" == "macos"* ]];then
+            make integration-test DOCKER_COMPOSE=docker-compose
+          else
+            make integration-test
+          fi

.github/workflows/unit-test.yml

@@ -0,0 +1,63 @@
+name: Test core installation and run tests
+
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          - ubuntu-20.04
+          - macos-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+          sudo apt-get -y update
+          sudo make deps-ubuntu
+        else
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+          HOMEBREW_NO_AUTO_UPDATE=1 \
+          brew install imagemagick geos bash  # opencv
+        fi
+        make install deps-test
+    - name: Test with pytest
+      run: |
+        make test benchmark
+    - name: test to ensure that --editable install works
+      run: |
+        make install-dev; ocrd --version
+    - name: Lint with flake8
+      run: |
+        python -m pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 src --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

.gitignore

@@ -109,7 +109,6 @@ env2/
 
 ocrd.egg-info
 .pytest_cache
-/src
 /profile
 .~lock*
 .pynative
@@ -125,5 +124,5 @@ tests/assets/test.ocrd.zip
 /foo
 sanders*
 ws1
-*.doctree
+*.doctrees
 .vscode

CHANGELOG.md

@@ -5,6 +5,46 @@ Versioned according to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+Added:
+
+  * Basic integration test for `ocrd_network`, #1164
+  * `ocrd-tool.json` for `ocrd-dummy` now in repo root, for ocrd_all's `make ocrd-all-tool.json`, #1168
+
+Fixed:
+
+  * METS Server: UDS sockets are removed on process exit, #117
+
+Changed:
+
+  * replace license-incompatible sparkline library with a simpler implementation, #1176
+  * remove all pkg_resources calls with modern alternatives, no more run-time setuptools dependency, #1174
+
+## [2.61.2] - 2024-01-24
+
+Fixed:
+
+  * another regression to docker deployment (requirements.txt missing), #1173
+
+## [2.61.1] - 2024-01-23
+
+Fixed:
+
+  * deps-cuda: add workaround for keras-team/tf-keras#62, #1169
+  * fix regression docker deployment, #1172
+
+
+## [2.61.0] - 2024-01-23
+
+Changed:
+
+  * :fire: simplify the project layout and distribution policy, #1166
+    * In the future there will be only one distribution `ocrd`
+    * The previous separate distributions of the `ocrd_utils`, `ocrd_models`, `ocrd_modelfactory`, `ocrd_validators` and `ocrd_network` are all part of `ocrd` now
+    * Nothing needs to be changed in code using OCR-D/core, the package structure and API is the same as before
+    * Until the next major release, we will continue to provide distributions for `ocrd_utils` etc. that contain the same code as `ocrd`
+    * Using `ocrd_utils` etc. as distributions in `requirements.txt` or `install_requires` is now deprecated
+    * Once we release v3.0.0, these distributions will be depublished
+
 ## [2.60.3] - 2024-01-10
 
 Fixed:
@@ -1957,6 +1997,9 @@ Fixed
 Initial Release
 
 <!-- link-labels -->
+[2.61.2]: ../../compare/v2.61.2..v2.61.1
+[2.61.1]: ../../compare/v2.61.1..v2.61.1
+[2.61.0]: ../../compare/v2.61.0..v2.60.3
 [2.60.3]: ../../compare/v2.60.3..v2.60.2
 [2.60.2]: ../../compare/v2.60.2..v2.60.1
 [2.60.1]: ../../compare/v2.60.1..v2.60.0

Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE
-FROM $BASE_IMAGE
+FROM $BASE_IMAGE as ocrd_core_base
 ARG FIXUP=echo
 MAINTAINER OCR-D
 ENV DEBIAN_FRONTEND noninteractive
@@ -9,17 +9,17 @@ ENV LANG=C.UTF-8
 ENV PIP=pip
 
 WORKDIR /build-ocrd
-COPY ocrd ./ocrd
-COPY ocrd_modelfactory ./ocrd_modelfactory/
-COPY ocrd_models ./ocrd_models
-COPY ocrd_utils ./ocrd_utils
-RUN mv ./ocrd_utils/ocrd_logging.conf /etc
-COPY ocrd_validators/ ./ocrd_validators
-COPY ocrd_network/ ./ocrd_network
+
+COPY src ./src
+COPY pyproject.toml .
+COPY VERSION ./VERSION
+COPY requirements.txt ./requirements.txt
+RUN mv ./src/ocrd_utils/ocrd_logging.conf /etc
 COPY Makefile .
 COPY README.md .
 COPY LICENSE .
 COPY .git ./.git
+
 RUN echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0";' >/etc/apt/apt.conf.d/ocr-d.conf
 RUN apt-get update && apt-get -y install software-properties-common \
     && apt-get update && apt-get -y install \
@@ -33,14 +33,25 @@ RUN apt-get update && apt-get -y install software-properties-common \
         curl \
         sudo \
         git \
-    && make deps-ubuntu \
-    && python3 -m venv /usr/local \
+    && make deps-ubuntu
+RUN python3 -m venv /usr/local \
     && hash -r \
-    && pip install --upgrade pip setuptools wheel \
     && make install \
-    && eval $FIXUP \
-    && rm -rf /build-ocrd
+    && eval $FIXUP
 
 WORKDIR /data
 
 CMD ["/usr/local/bin/ocrd", "--help"]
+
+FROM ocrd_core_base as ocrd_core_test
+WORKDIR /build-ocrd
+COPY Makefile .
+RUN make assets
+COPY tests ./tests
+COPY .gitmodules .
+COPY requirements_test.txt .
+RUN pip install -r requirements_test.txt
+RUN mkdir /ocrd-data && chmod 777 /ocrd-data
+
+CMD ["yes"]
+# CMD ["make", "test", "integration-test"]

Dockerfile.cuda

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE
-FROM $BASE_IMAGE
+FROM $BASE_IMAGE AS ocrd_core_base
 
 ENV MAMBA_EXE=/usr/local/bin/conda
 ENV MAMBA_ROOT_PREFIX=/conda

MANIFEST.in

@@ -0,0 +1,3 @@
+include README*.md
+include requirements.txt
+include VERSION