Merge branch 'master' into ocrd-tool-json-root

# Conflicts:
#	tests/cli/test_bashlib.py

.circleci/config.yml

@@ -16,9 +16,10 @@ jobs:
       PIP_INSTALL: pip3 install --user
     steps:
       - checkout
-      - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash
+      - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash opencv
+      - run: $PIP_INSTALL -U pip setuptools
       - run: make install
-      - run: PATH="/Users/distiller/Library/Python/3.9/bin:$PATH" make deps-test test benchmark
+      - run: export PATH="/Users/distiller/Library/Python/3.9/bin:$PATH"; make deps-test test benchmark
 
   test-python37:
     docker:
@@ -121,4 +122,4 @@ workflows:
       - test-python39
       - test-python310
       - test-python311
-      - test-macos
+      # - test-macos

.dockerignore

@@ -1,6 +1,9 @@
 *
-!ocrd*
+!src*
+!pyproject.toml
+!requirements.txt
 !Makefile
+!VERSION
 !LICENSE
 !README.md
 !.git

.github/workflows/docker-image.yml

@@ -28,6 +28,10 @@ jobs:
     - name: Build the Docker image with GPU support
       # default tag uses docker.io, so override on command-line
       run: make docker-cuda DOCKER_TAG=${{ env.DOCKER_TAG }}-cuda DOCKER_BASE_IMAGE=${{ env.DOCKER_TAG }}
+    - name: Smoke Test that ocrd --help works
+      run: |
+        docker run --rm ${{ env.DOCKER_TAG }} ocrd --version
+        docker run --rm ${{ env.DOCKER_TAG }}-cuda ocrd --version
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
       with:

.github/workflows/unit-test.yml

@@ -0,0 +1,63 @@
+name: Test core installation and run tests
+
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+        os:
+          - ubuntu-22.04
+          - ubuntu-20.04
+          - macos-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        if [[ "${{ matrix.os }}" == "ubuntu"* ]];then
+          sudo apt-get -y update
+          sudo make deps-ubuntu
+        else
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \
+          HOMEBREW_NO_AUTO_UPDATE=1 \
+          brew install imagemagick geos bash  # opencv
+        fi
+        make install deps-test
+    - name: Test with pytest
+      run: |
+        make test benchmark
+    - name: test to ensure that --editable install works
+      run: |
+        make install-dev; ocrd --version
+    - name: Lint with flake8
+      run: |
+        python -m pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 src --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

.gitignore

@@ -109,7 +109,6 @@ env2/
 
 ocrd.egg-info
 .pytest_cache
-/src
 /profile
 .~lock*
 .pynative
@@ -125,5 +124,5 @@ tests/assets/test.ocrd.zip
 /foo
 sanders*
 ws1
-*.doctree
+*.doctrees
 .vscode

CHANGELOG.md

@@ -5,6 +5,32 @@ Versioned according to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## [2.61.2] - 2024-01-24
+
+Fixed:
+
+  * another regression to docker deployment (requirements.txt missing), #1173
+
+## [2.61.1] - 2024-01-23
+
+Fixed:
+
+  * deps-cuda: add workaround for keras-team/tf-keras#62, #1169
+  * fix regression docker deployment, #1172
+
+
+## [2.61.0] - 2024-01-23
+
+Changed:
+
+  * :fire: simplify the project layout and distribution policy, #1166
+    * In the future there will be only one distribution `ocrd`
+    * The previous separate distributions of the `ocrd_utils`, `ocrd_models`, `ocrd_modelfactory`, `ocrd_validators` and `ocrd_network` are all part of `ocrd` now
+    * Nothing needs to be changed in code using OCR-D/core, the package structure and API is the same as before
+    * Until the next major release, we will continue to provide distributions for `ocrd_utils` etc. that contain the same code as `ocrd`
+    * Using `ocrd_utils` etc. as distributions in `requirements.txt` or `install_requires` is now deprecated
+    * Once we release v3.0.0, these distributions will be depublished
+
 ## [2.60.3] - 2024-01-10
 
 Fixed:
@@ -1957,6 +1983,9 @@ Fixed
 Initial Release
 
 <!-- link-labels -->
+[2.61.2]: ../../compare/v2.61.2..v2.61.1
+[2.61.1]: ../../compare/v2.61.1..v2.61.1
+[2.61.0]: ../../compare/v2.61.0..v2.60.3
 [2.60.3]: ../../compare/v2.60.3..v2.60.2
 [2.60.2]: ../../compare/v2.60.2..v2.60.1
 [2.60.1]: ../../compare/v2.60.1..v2.60.0

Dockerfile

@@ -9,17 +9,17 @@ ENV LANG=C.UTF-8
 ENV PIP=pip
 
 WORKDIR /build-ocrd
-COPY ocrd ./ocrd
-COPY ocrd_modelfactory ./ocrd_modelfactory/
-COPY ocrd_models ./ocrd_models
-COPY ocrd_utils ./ocrd_utils
-RUN mv ./ocrd_utils/ocrd_logging.conf /etc
-COPY ocrd_validators/ ./ocrd_validators
-COPY ocrd_network/ ./ocrd_network
+
+COPY src ./src
+COPY pyproject.toml .
+COPY VERSION ./VERSION
+COPY requirements.txt ./requirements.txt
+RUN mv ./src/ocrd_utils/ocrd_logging.conf /etc
 COPY Makefile .
 COPY README.md .
 COPY LICENSE .
 COPY .git ./.git
+
 RUN echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0";' >/etc/apt/apt.conf.d/ocr-d.conf
 RUN apt-get update && apt-get -y install software-properties-common \
     && apt-get update && apt-get -y install \
@@ -36,7 +36,6 @@ RUN apt-get update && apt-get -y install software-properties-common \
     && make deps-ubuntu \
     && python3 -m venv /usr/local \
     && hash -r \
-    && pip install --upgrade pip setuptools wheel \
     && make install \
     && eval $FIXUP \
     && rm -rf /build-ocrd

MANIFEST.in

@@ -0,0 +1,3 @@
+include README*.md
+include requirements.txt
+include VERSION

Makefile

@@ -7,6 +7,7 @@ LOG_LEVEL = INFO
 PYTHONIOENCODING=utf8
 TESTDIR = $(CURDIR)/tests
 PYTEST_ARGS = --continue-on-collection-errors
+VERSION = $(shell cat VERSION)
 
 SPHINX_APIDOC =
 
@@ -61,6 +62,8 @@ deps-cuda:
 	mv bin/micromamba $(CONDA_EXE)
 # Install Conda system-wide (for interactive / login shells)
 	echo 'export MAMBA_EXE=$(CONDA_EXE) MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) CONDA_PREFIX=$(CONDA_PREFIX) PATH=$(CONDA_PREFIX)/bin:$$PATH' >> /etc/profile.d/98-conda.sh
+# workaround for tf-keras#62
+	echo 'export XLA_FLAGS=--xla_gpu_cuda_data_dir=$(CONDA_PREFIX)/' >> /etc/profile.d/98-conda.sh
 	mkdir -p $(CONDA_PREFIX)/lib $(CONDA_PREFIX)/include
 	echo $(CONDA_PREFIX)/lib >> /etc/ld.so.conf.d/conda.conf
 # Get CUDA toolkit, including compiler and libraries with dev,
@@ -119,14 +122,15 @@ deps-test:
 .PHONY: build install install-dev uninstall
 
 build:
-	$(PIP) install build setuptools_scm
-	$(foreach MODULE,$(BUILD_ORDER),$(PYTHON) -m build ./$(MODULE) &&) echo done
+	$(PIP) install build
+	$(PYTHON) -m build .
 # or use -n ?
 
 # (Re)install the tool
 install: #build
-#	$(PIP_INSTALL) $(BUILD_ORDER:%=./%/dist/ocrd*`$(PYTHON) -m setuptools_scm 2>/dev/null`*.whl)
-	$(foreach MODULE,$(BUILD_ORDER),$(PIP_INSTALL) ./$(MODULE) $(PIP_INSTALL_CONFIG_OPTION) &&) echo done
+	# not stricttly necessary but a precaution against outdated python build tools, https://github.com/OCR-D/core/pull/1166
+	$(PIP) install -U pip wheel setuptools
+	$(PIP_INSTALL) . $(PIP_INSTALL_CONFIG_OPTION)
 	@# workaround for shapely#1598
 	$(PIP) config set global.no-binary shapely
 
@@ -138,11 +142,11 @@ install-dev: uninstall
 
 # Uninstall the tool
 uninstall:
-	$(PIP) uninstall -y $(call reverse,$(BUILD_ORDER))
+	$(PIP) uninstall --yes ocrd
 
 # Regenerate python code from PAGE XSD
-generate-page: GDS_PAGE = ocrd_models/ocrd_models/ocrd_page_generateds.py
-generate-page: GDS_PAGE_USER = ocrd_models/ocrd_page_user_methods.py
+generate-page: GDS_PAGE = src/ocrd_models/ocrd_page_generateds.py
+generate-page: GDS_PAGE_USER = src/ocrd_page_user_methods.py
 generate-page: repo/assets
 	generateDS \
 		-f \
@@ -152,7 +156,7 @@ generate-page: repo/assets
 		--export "write etree" \
 		--disable-generatedssuper-lookup \
 		--user-methods=$(GDS_PAGE_USER) \
-		ocrd_validators/ocrd_validators/page.xsd
+		src/ocrd_validators/page.xsd
 	# hack to prevent #451: enum keys will be strings
 	sed -i 's/(Enum):$$/(str, Enum):/' $(GDS_PAGE)
 	# hack to ensure output has pc: prefix
@@ -236,8 +240,8 @@ coverage: assets
 # Build documentation
 docs:
 	for mod in $(BUILD_ORDER);do sphinx-apidoc -f -M -e \
-		-o docs/api/$$mod $$mod/$$mod \
-		'ocrd_models/ocrd_models/ocrd_page_generateds.py' \
+		-o docs/api/$$mod src/$$mod \
+		'src/ocrd_models/ocrd_page_generateds.py' \
 		;done
 	cd docs ; $(MAKE) html
 
@@ -263,8 +267,12 @@ gh-pages:
 #
 
 pyclean:
+	rm -rf ./build
+	rm -rf ./dist
+	rm -rf htmlcov
+	rm -rf .benchmarks
 	rm -f **/*.pyc
-	find . -name '__pycache__' -exec rm -rf '{}' \;
+	-find . -name '__pycache__' -exec rm -rf '{}' \;
 	rm -rf .pytest_cache
 
 #
@@ -292,4 +300,38 @@ docker docker-cuda:
 
 # Build wheels and source dist and twine upload them
 pypi: build
-	twine upload ocrd*/dist/ocrd*`$(PYTHON) -m setuptools_scm 2>/dev/null`*{tar.gz,whl}
+	twine upload dist/ocrd-$(VERSION)*{tar.gz,whl}
+
+pypi-workaround: build-workaround
+	for dist in $(BUILD_ORDER);do twine upload dist/$$dist-$(VERSION)*{tar.gz,whl};done
+
+# Only in place until v3 so we don't break existing installations
+build-workaround: pyclean
+	cp pyproject.toml pyproject.toml.BAK
+	cp src/ocrd_utils/constants.py src/ocrd_utils/constants.py.BAK
+	cp src/ocrd/cli/__init__.py src/ocrd/cli/__init__.py.BAK
+	for dist in $(BUILD_ORDER);do \
+		cat pyproject.toml.BAK | sed "s,^name =.*,name = \"$$dist\"," > pyproject.toml; \
+		cat src/ocrd_utils/constants.py.BAK | sed "s,get_distribution('ocrd'),get_distribution('$$dist')," > src/ocrd_utils/constants.py; \
+		cat src/ocrd/cli/__init__.py.BAK | sed "s,package_name='ocrd',package_name='$$dist'," > src/ocrd/cli/__init__.py; \
+		$(MAKE) build; \
+	done
+	rm pyproject.toml.BAK
+	rm src/ocrd_utils/constants.py.BAK
+	rm src/ocrd/cli/__init__.py.BAK
+
+# test that the aliased packages work in isolation and combined
+test-workaround: build-workaround
+	for dist in $(BUILD_ORDER);do pip uninstall --yes $$dist;done
+	for dist in $(BUILD_ORDER);do \
+		pip install dist/$$dist-*.whl ;\
+		ocrd --version ;\
+		make test ;\
+		pip uninstall --yes $$dist ;\
+	done
+	for dist in $(BUILD_ORDER);do \
+		pip install dist/$$dist-*.whl ;\
+	done
+	ocrd --version ;\
+	make test ;\
+	for dist in $(BUILD_ORDER);do pip uninstall --yes $$dist;done