From 67d4b17f792d9db1a1789dda0dfdb4151dc3fcfa Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 10 Jan 2024 17:38:37 +0100 Subject: [PATCH 01/93] initial configs --- tests/network/__init__.py | 0 tests/network/conftest.py | 4 ++ tests/network/constants.py | 18 ++++++ tests/network/docker-compose.yml | 18 ++++++ tests/network/fixtures_mongodb.py | 46 ++++++++++++++ tests/network/fixtures_rabbitmq.py | 99 ++++++++++++++++++++++++++++++ tests/network/test_rabbitmq.py | 80 ++++++++++++++++++++++++ tests/network/utils.py | 13 ++++ 8 files changed, 278 insertions(+) create mode 100644 tests/network/__init__.py create mode 100644 tests/network/conftest.py create mode 100644 tests/network/constants.py create mode 100644 tests/network/docker-compose.yml create mode 100644 tests/network/fixtures_mongodb.py create mode 100644 tests/network/fixtures_rabbitmq.py create mode 100644 tests/network/test_rabbitmq.py create mode 100644 tests/network/utils.py diff --git a/tests/network/__init__.py b/tests/network/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/network/conftest.py b/tests/network/conftest.py new file mode 100644 index 0000000000..da47f3f8b5 --- /dev/null +++ b/tests/network/conftest.py @@ -0,0 +1,4 @@ +pytest_plugins = [ + "tests.network.fixtures_mongodb", + "tests.network.fixtures_rabbitmq" +] diff --git a/tests/network/constants.py b/tests/network/constants.py new file mode 100644 index 0000000000..d769ef51fb --- /dev/null +++ b/tests/network/constants.py @@ -0,0 +1,18 @@ +__all__ = [ + "DB_NAME", + "DB_URL", + "DEFAULT_EXCHANGER_NAME", + "DEFAULT_QUEUE", + "PROCESSING_SERVER_URL", + "RABBITMQ_URL" +] + +# mongodb://localhost:6701/ocrd_network_test +DB_NAME: str = "ocrd_network_test" +DB_URL: str = "mongodb://localhost:6701" + +DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" +DEFAULT_QUEUE: str = "ocrd-network-default" + +PROCESSING_SERVER_URL: str = "http://localhost:8000" +RABBITMQ_URL: str = "amqp://network_test:network_test@localhost:6672/" diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml new file mode 100644 index 0000000000..2300dc6f09 --- /dev/null +++ b/tests/network/docker-compose.yml @@ -0,0 +1,18 @@ +services: + ocrd_network_mongo_db: + image: "mongo" + ports: + - "6701:27017" + environment: + - MONGO_INITDB_ROOT_USERNAME="network_test" + - MONGO_INITDB_ROOT_PASSWORD="network_test" + ocrd_network_rabbit_mq: + image: "rabbitmq:3.12-management" + ports: + - "6672:5672" + - "16672:15672" + - "26672:25672" + environment: + - RABBITMQ_DEFAULT_USER="network_test" + - RABBITMQ_DEFAULT_PASS="network_test" + - RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py new file mode 100644 index 0000000000..80d7b4c8ff --- /dev/null +++ b/tests/network/fixtures_mongodb.py @@ -0,0 +1,46 @@ +from pymongo import MongoClient, uri_parser as mongo_uri_parser +from pytest import fixture +from .constants import DB_NAME, DB_URL +from .utils import is_url_responsive + + +def verify_database_uri(mongodb_address: str) -> str: + try: + # perform validation check + mongo_uri_parser.parse_uri(uri=mongodb_address, validate=True) + except Exception as error: + raise ValueError(f"The MongoDB address '{mongodb_address}' is in wrong format, {error}") + return mongodb_address + + +@fixture(scope="package", name="mongo_db") +def fixture_mongo_db(docker_ip, docker_services): + port = docker_services.port_for("ocrd_network_mongo_db", 27017) + mongo_db_url = f"http://{docker_ip}:{port}" + docker_services.wait_until_responsive( + timeout=10.0, + pause=0.1, + check=lambda: is_url_responsive(mongo_db_url, retries=10) + ) + + +@fixture(scope="package", name="mongo_client") +def fixture_mongo_client(mongo_db): + mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) + yield mongo_client + + +@fixture(scope="package", name="mongo_processor_jobs") +def fixture_mongo_processor_jobs(mongo_client): + mydb = mongo_client[DB_NAME] + processor_jobs_collection = mydb["DBProcessorJob"] + yield processor_jobs_collection + processor_jobs_collection.drop() + + +@fixture(scope="package", name="mongo_workflow_jobs") +def fixture_mongo_workflow_jobs(mongo_client): + mydb = mongo_client[DB_NAME] + workflow_jobs_collection = mydb["DBWorkflowJob"] + yield workflow_jobs_collection + workflow_jobs_collection.drop() diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py new file mode 100644 index 0000000000..28c3b7dc83 --- /dev/null +++ b/tests/network/fixtures_rabbitmq.py @@ -0,0 +1,99 @@ +from pika import URLParameters +from pika.credentials import PlainCredentials +from pytest import fixture +from re import match as re_match +from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher +from .constants import RABBITMQ_URL, DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE +from .utils import is_url_responsive + + +def verify_and_parse_mq_uri(rabbitmq_address: str): + """ + Check the full list of available parameters in the docs here: + https://pika.readthedocs.io/en/stable/_modules/pika/connection.html#URLParameters + """ + + uri_pattern = r"^(?:([^:\/?#\s]+):\/{2})?(?:([^@\/?#\s]+)@)?([^\/?#\s]+)?(?:\/([^?#\s]*))?(?:[?]([^#\s]+))?\S*$" + match = re_match(pattern=uri_pattern, string=rabbitmq_address) + if not match: + raise ValueError(f"The RabbitMQ server address is in wrong format: '{rabbitmq_address}'") + url_params = URLParameters(rabbitmq_address) + + parsed_data = { + "username": url_params.credentials.username, + "password": url_params.credentials.password, + "host": url_params.host, + "port": url_params.port, + "vhost": url_params.virtual_host + } + return parsed_data + + +@fixture(scope="package", name="rabbit_mq") +def fixture_rabbit_mq(docker_ip, docker_services): + port = docker_services.port_for("ocrd_network_rabbit_mq", 15672) + rabbit_mq_management_url = f"http://{docker_ip}:{port}" + docker_services.wait_until_responsive( + timeout=10.0, + pause=0.1, + check=lambda: is_url_responsive(rabbit_mq_management_url, retries=10) + ) + + +@fixture(scope="package", name="rabbitmq_defaults") +def fixture_rabbitmq_defaults(): + rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) + rmq_username = rmq_data["username"] + rmq_password = rmq_data["password"] + rmq_host = rmq_data["host"] + rmq_port = rmq_data["port"] + rmq_vhost = rmq_data["vhost"] + + test_connection = RMQConnector.open_blocking_connection( + credentials=PlainCredentials(rmq_username, rmq_password), + host=rmq_host, + port=rmq_port, + vhost=rmq_vhost + ) + test_channel = RMQConnector.open_blocking_channel(test_connection) + RMQConnector.exchange_declare(channel=test_channel, exchange_name=DEFAULT_EXCHANGER_NAME, durable=False) + RMQConnector.queue_declare(channel=test_channel, queue_name=DEFAULT_QUEUE, durable=False) + RMQConnector.queue_bind( + channel=test_channel, + exchange_name=DEFAULT_EXCHANGER_NAME, + queue_name=DEFAULT_QUEUE, + routing_key=DEFAULT_QUEUE + ) + # Clean all messages inside if any from previous tests + RMQConnector.queue_purge(channel=test_channel, queue_name=DEFAULT_QUEUE) + + +@fixture(scope="package", name="rabbitmq_publisher") +def fixture_rabbitmq_publisher(rabbitmq_defaults): + rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) + rmq_publisher = RMQPublisher( + host=rmq_data["host"], + port=rmq_data["port"], + vhost=rmq_data["vhost"] + ) + rmq_publisher.authenticate_and_connect( + username=rmq_data["username"], + password=rmq_data["password"] + ) + rmq_publisher.enable_delivery_confirmations() + yield rmq_publisher + + +@fixture(scope="package", name="rabbitmq_consumer") +def fixture_rabbitmq_consumer(rabbitmq_defaults): + rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) + rmq_consumer = RMQConsumer( + host=rmq_data["host"], + port=rmq_data["port"], + vhost=rmq_data["vhost"] + ) + rmq_consumer.authenticate_and_connect( + username=rmq_data["username"], + password=rmq_data["password"] + ) + yield rmq_consumer diff --git a/tests/network/test_rabbitmq.py b/tests/network/test_rabbitmq.py new file mode 100644 index 0000000000..f9a2e89419 --- /dev/null +++ b/tests/network/test_rabbitmq.py @@ -0,0 +1,80 @@ +from pika import BasicProperties +from pickle import dumps, loads +from .constants import DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE + + +def test_publish_2_messages_to_rabbitmq(rabbitmq_publisher): + test_headers = {"Test Header": "Test Value"} + test_properties = BasicProperties( + app_id='webapi-processing-broker', + content_type='application/json', + headers=test_headers + ) + rabbitmq_publisher.publish_to_queue( + queue_name=DEFAULT_QUEUE, + message="RabbitMQ test 123", + exchange_name=DEFAULT_EXCHANGER_NAME, + properties=test_properties + ) + rabbitmq_publisher.publish_to_queue( + queue_name=DEFAULT_QUEUE, + message="RabbitMQ test 456", + exchange_name=DEFAULT_EXCHANGER_NAME, + properties=test_properties + ) + assert rabbitmq_publisher.message_counter == 2 + + +def test_consume_2_messages_from_rabbitmq(rabbitmq_consumer): + # Consume the 1st message + method_frame, header_frame, message = rabbitmq_consumer.get_one_message( + queue_name=DEFAULT_QUEUE, + auto_ack=True + ) + assert method_frame.delivery_tag == 1 # 1st delivered message to this queue + assert method_frame.message_count == 1 # messages left in the queue + assert method_frame.redelivered is False + assert method_frame.exchange == DEFAULT_EXCHANGER_NAME + assert method_frame.routing_key == DEFAULT_QUEUE + # It's possible to assert header_frame the same way + assert message.decode() == "RabbitMQ test 123" + + # Consume the 2nd message + method_frame, header_frame, message = rabbitmq_consumer.get_one_message( + queue_name=DEFAULT_QUEUE, + auto_ack=True + ) + assert method_frame.delivery_tag == 2 # 2nd delivered message to this queue + assert method_frame.message_count == 0 # messages left in the queue + assert method_frame.redelivered is False + assert method_frame.exchange == DEFAULT_EXCHANGER_NAME + assert method_frame.routing_key == DEFAULT_QUEUE + # It's possible to assert header_frame the same way + assert message.decode() == "RabbitMQ test 456" + + +def test_publish_ocrd_message_to_rabbitmq(rabbitmq_publisher): + ocrd_processing_message = { + "job_id": "Test_job_id", + "workflow_id": "Test_workflow_id", + "workspace_id": "Test_workspace_id" + } + message_bytes = dumps(ocrd_processing_message) + rabbitmq_publisher.publish_to_queue( + queue_name=DEFAULT_QUEUE, + message=message_bytes, + exchange_name=DEFAULT_EXCHANGER_NAME, + properties=None + ) + + +def test_consume_ocrd_message_from_rabbitmq(rabbitmq_consumer): + method_frame, header_frame, message = rabbitmq_consumer.get_one_message( + queue_name=DEFAULT_QUEUE, + auto_ack=True + ) + assert method_frame.message_count == 0 # messages left in the queue + decoded_message = loads(message) + assert decoded_message["job_id"] == "Test_job_id" + assert decoded_message["workflow_id"] == "Test_workflow_id" + assert decoded_message["workspace_id"] == "Test_workspace_id" diff --git a/tests/network/utils.py b/tests/network/utils.py new file mode 100644 index 0000000000..0d33fbc30d --- /dev/null +++ b/tests/network/utils.py @@ -0,0 +1,13 @@ +from requests import get + + +def is_url_responsive(url: str, retries: int = 0) -> bool: + while True: + try: + response = get(url) + if response.status_code == 200: + return True + except Exception: + if retries <= 0: + return False + retries -= 1 From ed13c4bc588ca07a0f43d8d24753ef9db5bb34e1 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 10 Jan 2024 18:29:08 +0100 Subject: [PATCH 02/93] fix rabbitmq tests --- tests/network/conftest.py | 8 ++++++++ tests/network/docker-compose.yml | 8 ++++---- tests/network/fixtures_rabbitmq.py | 9 +++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/network/conftest.py b/tests/network/conftest.py index da47f3f8b5..67eb0b4399 100644 --- a/tests/network/conftest.py +++ b/tests/network/conftest.py @@ -1,4 +1,12 @@ +from pytest import fixture +from os.path import join + pytest_plugins = [ "tests.network.fixtures_mongodb", "tests.network.fixtures_rabbitmq" ] + + +@fixture(scope="session") +def docker_compose_file(pytestconfig): + return join(str(pytestconfig.rootdir), "docker-compose.yml") diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 2300dc6f09..2805e2ebcd 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -4,8 +4,8 @@ services: ports: - "6701:27017" environment: - - MONGO_INITDB_ROOT_USERNAME="network_test" - - MONGO_INITDB_ROOT_PASSWORD="network_test" + - MONGO_INITDB_ROOT_USERNAME=network_test + - MONGO_INITDB_ROOT_PASSWORD=network_test ocrd_network_rabbit_mq: image: "rabbitmq:3.12-management" ports: @@ -13,6 +13,6 @@ services: - "16672:15672" - "26672:25672" environment: - - RABBITMQ_DEFAULT_USER="network_test" - - RABBITMQ_DEFAULT_PASS="network_test" + - RABBITMQ_DEFAULT_USER=network_test + - RABBITMQ_DEFAULT_PASS=network_test - RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 28c3b7dc83..6bf5ae9b3a 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -41,7 +41,7 @@ def fixture_rabbit_mq(docker_ip, docker_services): @fixture(scope="package", name="rabbitmq_defaults") -def fixture_rabbitmq_defaults(): +def fixture_rabbitmq_defaults(rabbit_mq): rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_username = rmq_data["username"] rmq_password = rmq_data["password"] @@ -56,7 +56,12 @@ def fixture_rabbitmq_defaults(): vhost=rmq_vhost ) test_channel = RMQConnector.open_blocking_channel(test_connection) - RMQConnector.exchange_declare(channel=test_channel, exchange_name=DEFAULT_EXCHANGER_NAME, durable=False) + RMQConnector.exchange_declare( + channel=test_channel, + exchange_name=DEFAULT_EXCHANGER_NAME, + exchange_type='direct', + durable=False + ) RMQConnector.queue_declare(channel=test_channel, queue_name=DEFAULT_QUEUE, durable=False) RMQConnector.queue_bind( channel=test_channel, From b8bab25dc78d49450891597e4326768234cbad81 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 10 Jan 2024 18:29:24 +0100 Subject: [PATCH 03/93] add pytest-docker to req --- requirements_test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_test.txt b/requirements_test.txt index 8b7997b1a0..254ccbd891 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,6 +2,7 @@ autopep8 pytest >= 4.0.0 generateDS == 2.35.20 pytest-benchmark >= 3.2.3 +pytest-docker>=1.0.0 coverage >= 4.5.2 sphinx sphinx_click From c18916e017d42b2a1b7320f856d10a8b765febed Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 13:47:49 +0100 Subject: [PATCH 04/93] fix conftest - move to top level --- tests/{network => }/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename tests/{network => }/conftest.py (63%) diff --git a/tests/network/conftest.py b/tests/conftest.py similarity index 63% rename from tests/network/conftest.py rename to tests/conftest.py index 67eb0b4399..f638928858 100644 --- a/tests/network/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ from pytest import fixture -from os.path import join +from pathlib import Path pytest_plugins = [ "tests.network.fixtures_mongodb", @@ -9,4 +9,4 @@ @fixture(scope="session") def docker_compose_file(pytestconfig): - return join(str(pytestconfig.rootdir), "docker-compose.yml") + return Path(str(pytestconfig.rootdir), "tests", "network", "docker-compose.yml") From f4c653b7dbab7b4f1be9da567f999406472fcb83 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 14:02:15 +0100 Subject: [PATCH 05/93] fix docker file path --- tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f638928858..49cdff50da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ -from pytest import fixture from pathlib import Path +from pkg_resources import resource_filename +from pytest import fixture pytest_plugins = [ "tests.network.fixtures_mongodb", @@ -8,5 +9,5 @@ @fixture(scope="session") -def docker_compose_file(pytestconfig): - return Path(str(pytestconfig.rootdir), "tests", "network", "docker-compose.yml") +def docker_compose_file(): + return Path(resource_filename("tests", "network"), "docker-compose.yml") From 9e341ba674b27c6766727a096e057c14a22337d4 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 14:48:19 +0100 Subject: [PATCH 06/93] experiment with circle ci --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 047780b810..c48fdc436f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,8 @@ jobs: working_directory: ~/ocrd-core steps: - checkout + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test From 9f799e166e9e1ba3f76b64ac9e963246a433c17f Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 15:41:59 +0100 Subject: [PATCH 07/93] increase timeout secs --- tests/network/fixtures_mongodb.py | 2 +- tests/network/fixtures_rabbitmq.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 80d7b4c8ff..409fba9d93 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -20,7 +20,7 @@ def fixture_mongo_db(docker_ip, docker_services): docker_services.wait_until_responsive( timeout=10.0, pause=0.1, - check=lambda: is_url_responsive(mongo_db_url, retries=10) + check=lambda: is_url_responsive(mongo_db_url, retries=30) ) diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 6bf5ae9b3a..073f8ddbba 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -36,7 +36,7 @@ def fixture_rabbit_mq(docker_ip, docker_services): docker_services.wait_until_responsive( timeout=10.0, pause=0.1, - check=lambda: is_url_responsive(rabbit_mq_management_url, retries=10) + check=lambda: is_url_responsive(rabbit_mq_management_url, retries=30) ) From 70b05d090605aa87f13c88174bec9c67ccc43206 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 15:57:59 +0100 Subject: [PATCH 08/93] extend circle ci config --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c48fdc436f..f87ef0c957 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,8 @@ jobs: working_directory: ~/ocrd-core steps: - checkout + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -54,6 +56,8 @@ jobs: working_directory: ~/ocrd-core steps: - checkout + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -67,6 +71,8 @@ jobs: working_directory: ~/ocrd-core steps: - checkout + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -80,6 +86,8 @@ jobs: working_directory: ~/ocrd-core steps: - checkout + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test From 29a624741d9b3613f5e1eec19be371d0366ceba9 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 16:16:32 +0100 Subject: [PATCH 09/93] try different host resolution --- tests/network/constants.py | 11 +---------- tests/network/fixtures_mongodb.py | 12 +++++++----- tests/network/fixtures_rabbitmq.py | 17 ++++++++++------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/tests/network/constants.py b/tests/network/constants.py index d769ef51fb..1c84a44236 100644 --- a/tests/network/constants.py +++ b/tests/network/constants.py @@ -1,18 +1,9 @@ __all__ = [ "DB_NAME", - "DB_URL", "DEFAULT_EXCHANGER_NAME", - "DEFAULT_QUEUE", - "PROCESSING_SERVER_URL", - "RABBITMQ_URL" + "DEFAULT_QUEUE" ] -# mongodb://localhost:6701/ocrd_network_test DB_NAME: str = "ocrd_network_test" -DB_URL: str = "mongodb://localhost:6701" - DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" DEFAULT_QUEUE: str = "ocrd-network-default" - -PROCESSING_SERVER_URL: str = "http://localhost:8000" -RABBITMQ_URL: str = "amqp://network_test:network_test@localhost:6672/" diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 409fba9d93..770ddb5404 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,6 +1,6 @@ from pymongo import MongoClient, uri_parser as mongo_uri_parser from pytest import fixture -from .constants import DB_NAME, DB_URL +from .constants import DB_NAME from .utils import is_url_responsive @@ -16,17 +16,19 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_db") def fixture_mongo_db(docker_ip, docker_services): port = docker_services.port_for("ocrd_network_mongo_db", 27017) - mongo_db_url = f"http://{docker_ip}:{port}" + test_url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( timeout=10.0, pause=0.1, - check=lambda: is_url_responsive(mongo_db_url, retries=30) + check=lambda: is_url_responsive(test_url, retries=30) ) @fixture(scope="package", name="mongo_client") -def fixture_mongo_client(mongo_db): - mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) +def fixture_mongo_client(docker_ip, docker_services, mongo_db): + mongodb_port = docker_services.port_for("ocrd_network_mongo_db", 27017) + mongodb_url = f"mongodb://network_test:network_test@{docker_ip}:{mongodb_port}" + mongo_client = MongoClient(mongodb_url, serverSelectionTimeoutMS=3000) yield mongo_client diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 073f8ddbba..72eddeaec3 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -3,7 +3,7 @@ from pytest import fixture from re import match as re_match from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher -from .constants import RABBITMQ_URL, DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE +from .constants import DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE from .utils import is_url_responsive @@ -32,17 +32,19 @@ def verify_and_parse_mq_uri(rabbitmq_address: str): @fixture(scope="package", name="rabbit_mq") def fixture_rabbit_mq(docker_ip, docker_services): port = docker_services.port_for("ocrd_network_rabbit_mq", 15672) - rabbit_mq_management_url = f"http://{docker_ip}:{port}" + test_url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( timeout=10.0, pause=0.1, - check=lambda: is_url_responsive(rabbit_mq_management_url, retries=30) + check=lambda: is_url_responsive(test_url, retries=30) ) @fixture(scope="package", name="rabbitmq_defaults") -def fixture_rabbitmq_defaults(rabbit_mq): - rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) +def fixture_rabbitmq_defaults(docker_ip, docker_services, rabbit_mq): + rabbitmq_port = docker_services.port_for("ocrd_network_rabbit_mq", 5672) + rabbitmq_url = f"amqp://network_test:network_test@{docker_ip}:{rabbitmq_port}/" + rmq_data = verify_and_parse_mq_uri(rabbitmq_url) rmq_username = rmq_data["username"] rmq_password = rmq_data["password"] rmq_host = rmq_data["host"] @@ -71,11 +73,12 @@ def fixture_rabbitmq_defaults(rabbit_mq): ) # Clean all messages inside if any from previous tests RMQConnector.queue_purge(channel=test_channel, queue_name=DEFAULT_QUEUE) + yield rabbitmq_url @fixture(scope="package", name="rabbitmq_publisher") def fixture_rabbitmq_publisher(rabbitmq_defaults): - rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) + rmq_data = verify_and_parse_mq_uri(rabbitmq_defaults) rmq_publisher = RMQPublisher( host=rmq_data["host"], port=rmq_data["port"], @@ -91,7 +94,7 @@ def fixture_rabbitmq_publisher(rabbitmq_defaults): @fixture(scope="package", name="rabbitmq_consumer") def fixture_rabbitmq_consumer(rabbitmq_defaults): - rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) + rmq_data = verify_and_parse_mq_uri(rabbitmq_defaults) rmq_consumer = RMQConsumer( host=rmq_data["host"], port=rmq_data["port"], From d4ce2b88217f735f3cd621d47f743867f3be9df1 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 16:25:26 +0100 Subject: [PATCH 10/93] revert host resolution --- tests/network/constants.py | 11 ++++++++++- tests/network/fixtures_mongodb.py | 12 +++++------- tests/network/fixtures_rabbitmq.py | 17 +++++++---------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/network/constants.py b/tests/network/constants.py index 1c84a44236..d769ef51fb 100644 --- a/tests/network/constants.py +++ b/tests/network/constants.py @@ -1,9 +1,18 @@ __all__ = [ "DB_NAME", + "DB_URL", "DEFAULT_EXCHANGER_NAME", - "DEFAULT_QUEUE" + "DEFAULT_QUEUE", + "PROCESSING_SERVER_URL", + "RABBITMQ_URL" ] +# mongodb://localhost:6701/ocrd_network_test DB_NAME: str = "ocrd_network_test" +DB_URL: str = "mongodb://localhost:6701" + DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" DEFAULT_QUEUE: str = "ocrd-network-default" + +PROCESSING_SERVER_URL: str = "http://localhost:8000" +RABBITMQ_URL: str = "amqp://network_test:network_test@localhost:6672/" diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 770ddb5404..409fba9d93 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,6 +1,6 @@ from pymongo import MongoClient, uri_parser as mongo_uri_parser from pytest import fixture -from .constants import DB_NAME +from .constants import DB_NAME, DB_URL from .utils import is_url_responsive @@ -16,19 +16,17 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_db") def fixture_mongo_db(docker_ip, docker_services): port = docker_services.port_for("ocrd_network_mongo_db", 27017) - test_url = f"http://{docker_ip}:{port}" + mongo_db_url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( timeout=10.0, pause=0.1, - check=lambda: is_url_responsive(test_url, retries=30) + check=lambda: is_url_responsive(mongo_db_url, retries=30) ) @fixture(scope="package", name="mongo_client") -def fixture_mongo_client(docker_ip, docker_services, mongo_db): - mongodb_port = docker_services.port_for("ocrd_network_mongo_db", 27017) - mongodb_url = f"mongodb://network_test:network_test@{docker_ip}:{mongodb_port}" - mongo_client = MongoClient(mongodb_url, serverSelectionTimeoutMS=3000) +def fixture_mongo_client(mongo_db): + mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) yield mongo_client diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 72eddeaec3..073f8ddbba 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -3,7 +3,7 @@ from pytest import fixture from re import match as re_match from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher -from .constants import DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE +from .constants import RABBITMQ_URL, DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE from .utils import is_url_responsive @@ -32,19 +32,17 @@ def verify_and_parse_mq_uri(rabbitmq_address: str): @fixture(scope="package", name="rabbit_mq") def fixture_rabbit_mq(docker_ip, docker_services): port = docker_services.port_for("ocrd_network_rabbit_mq", 15672) - test_url = f"http://{docker_ip}:{port}" + rabbit_mq_management_url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( timeout=10.0, pause=0.1, - check=lambda: is_url_responsive(test_url, retries=30) + check=lambda: is_url_responsive(rabbit_mq_management_url, retries=30) ) @fixture(scope="package", name="rabbitmq_defaults") -def fixture_rabbitmq_defaults(docker_ip, docker_services, rabbit_mq): - rabbitmq_port = docker_services.port_for("ocrd_network_rabbit_mq", 5672) - rabbitmq_url = f"amqp://network_test:network_test@{docker_ip}:{rabbitmq_port}/" - rmq_data = verify_and_parse_mq_uri(rabbitmq_url) +def fixture_rabbitmq_defaults(rabbit_mq): + rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_username = rmq_data["username"] rmq_password = rmq_data["password"] rmq_host = rmq_data["host"] @@ -73,12 +71,11 @@ def fixture_rabbitmq_defaults(docker_ip, docker_services, rabbit_mq): ) # Clean all messages inside if any from previous tests RMQConnector.queue_purge(channel=test_channel, queue_name=DEFAULT_QUEUE) - yield rabbitmq_url @fixture(scope="package", name="rabbitmq_publisher") def fixture_rabbitmq_publisher(rabbitmq_defaults): - rmq_data = verify_and_parse_mq_uri(rabbitmq_defaults) + rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_publisher = RMQPublisher( host=rmq_data["host"], port=rmq_data["port"], @@ -94,7 +91,7 @@ def fixture_rabbitmq_publisher(rabbitmq_defaults): @fixture(scope="package", name="rabbitmq_consumer") def fixture_rabbitmq_consumer(rabbitmq_defaults): - rmq_data = verify_and_parse_mq_uri(rabbitmq_defaults) + rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_consumer = RMQConsumer( host=rmq_data["host"], port=rmq_data["port"], From 55ff89a57f98e98a5b76dd3b20bc68b5bacf5b8f Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 11 Jan 2024 17:00:06 +0100 Subject: [PATCH 11/93] add docker network --- tests/network/docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 2805e2ebcd..7e1a2ca86e 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -1,6 +1,15 @@ +networks: + ocrd_network_test: + name: ocrd_network_test + driver: bridge + driver_opts: + com.docker.network.driver.mtu: 1450 services: ocrd_network_mongo_db: image: "mongo" + hostname: mongodb-docker-host + networks: + - ocrd_network_test ports: - "6701:27017" environment: @@ -8,6 +17,9 @@ services: - MONGO_INITDB_ROOT_PASSWORD=network_test ocrd_network_rabbit_mq: image: "rabbitmq:3.12-management" + hostname: rabbitmq-docker-host + networks: + - ocrd_network_test ports: - "6672:5672" - "16672:15672" From d383d2bb0ed364cf9d0e91f3ea0767c515d064df Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Fri, 12 Jan 2024 14:56:25 +0100 Subject: [PATCH 12/93] update python orb to 2.1.1 and config file to 2.1 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f87ef0c957..94536bf491 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ -version: 2 +version: 2.1 orbs: - python: circleci/python@2.0.3 + python: circleci/python@2.1.1 jobs: From abe5d1ee494ab238104ffab3bfbf6950adc9a001 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Fri, 12 Jan 2024 15:05:58 +0100 Subject: [PATCH 13/93] increase timeouts in wait_until_responsive --- tests/network/fixtures_mongodb.py | 4 ++-- tests/network/fixtures_rabbitmq.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 409fba9d93..8fdc6cdfcc 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -18,8 +18,8 @@ def fixture_mongo_db(docker_ip, docker_services): port = docker_services.port_for("ocrd_network_mongo_db", 27017) mongo_db_url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( - timeout=10.0, - pause=0.1, + timeout=120.0, + pause=1, check=lambda: is_url_responsive(mongo_db_url, retries=30) ) diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 073f8ddbba..a7968ba637 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -34,8 +34,8 @@ def fixture_rabbit_mq(docker_ip, docker_services): port = docker_services.port_for("ocrd_network_rabbit_mq", 15672) rabbit_mq_management_url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( - timeout=10.0, - pause=0.1, + timeout=120.0, + pause=1, check=lambda: is_url_responsive(rabbit_mq_management_url, retries=30) ) From 1df014ae889a412f0509c404a2cf4839760a05d1 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 14 Jan 2024 17:52:54 +0100 Subject: [PATCH 14/93] implement make integration-test w/o pytest_docker --- Makefile | 7 +++++++ requirements_test.txt | 1 - tests/conftest.py | 9 --------- tests/network/fixtures_mongodb.py | 14 +------------- tests/network/fixtures_rabbitmq.py | 17 ++++------------- 5 files changed, 12 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index f2adfdca59..9147eb9c9a 100644 --- a/Makefile +++ b/Makefile @@ -209,9 +209,16 @@ test: assets $(PYTHON) \ -m pytest $(PYTEST_ARGS) --durations=10\ --ignore-glob="$(TESTDIR)/**/*bench*.py" \ + --ignore-glob="$(TESTDIR)/network/*.py" \ $(TESTDIR) cd ocrd_utils ; $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging -k TestDecorators $(TESTDIR) +integration-test: + docker compose -f tests/network/docker-compose.yml up -d + sleep 10 + pytest -k 'test_consume or test_publish' + docker compose -f tests/network/docker-compose.yml down + benchmark: $(PYTHON) -m pytest $(TESTDIR)/model/test_ocrd_mets_bench.py diff --git a/requirements_test.txt b/requirements_test.txt index 254ccbd891..8b7997b1a0 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,6 @@ autopep8 pytest >= 4.0.0 generateDS == 2.35.20 pytest-benchmark >= 3.2.3 -pytest-docker>=1.0.0 coverage >= 4.5.2 sphinx sphinx_click diff --git a/tests/conftest.py b/tests/conftest.py index 49cdff50da..da47f3f8b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,4 @@ -from pathlib import Path -from pkg_resources import resource_filename -from pytest import fixture - pytest_plugins = [ "tests.network.fixtures_mongodb", "tests.network.fixtures_rabbitmq" ] - - -@fixture(scope="session") -def docker_compose_file(): - return Path(resource_filename("tests", "network"), "docker-compose.yml") diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 8fdc6cdfcc..1d8826aeaa 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,7 +1,6 @@ from pymongo import MongoClient, uri_parser as mongo_uri_parser from pytest import fixture from .constants import DB_NAME, DB_URL -from .utils import is_url_responsive def verify_database_uri(mongodb_address: str) -> str: @@ -13,19 +12,8 @@ def verify_database_uri(mongodb_address: str) -> str: return mongodb_address -@fixture(scope="package", name="mongo_db") -def fixture_mongo_db(docker_ip, docker_services): - port = docker_services.port_for("ocrd_network_mongo_db", 27017) - mongo_db_url = f"http://{docker_ip}:{port}" - docker_services.wait_until_responsive( - timeout=120.0, - pause=1, - check=lambda: is_url_responsive(mongo_db_url, retries=30) - ) - - @fixture(scope="package", name="mongo_client") -def fixture_mongo_client(mongo_db): +def fixture_mongo_client(): mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) yield mongo_client diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index a7968ba637..d2540b6418 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -4,7 +4,6 @@ from re import match as re_match from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher from .constants import RABBITMQ_URL, DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE -from .utils import is_url_responsive def verify_and_parse_mq_uri(rabbitmq_address: str): @@ -29,19 +28,8 @@ def verify_and_parse_mq_uri(rabbitmq_address: str): return parsed_data -@fixture(scope="package", name="rabbit_mq") -def fixture_rabbit_mq(docker_ip, docker_services): - port = docker_services.port_for("ocrd_network_rabbit_mq", 15672) - rabbit_mq_management_url = f"http://{docker_ip}:{port}" - docker_services.wait_until_responsive( - timeout=120.0, - pause=1, - check=lambda: is_url_responsive(rabbit_mq_management_url, retries=30) - ) - - @fixture(scope="package", name="rabbitmq_defaults") -def fixture_rabbitmq_defaults(rabbit_mq): +def fixture_rabbitmq_defaults(): rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_username = rmq_data["username"] rmq_password = rmq_data["password"] @@ -56,6 +44,7 @@ def fixture_rabbitmq_defaults(rabbit_mq): vhost=rmq_vhost ) test_channel = RMQConnector.open_blocking_channel(test_connection) + assert(test_channel) RMQConnector.exchange_declare( channel=test_channel, exchange_name=DEFAULT_EXCHANGER_NAME, @@ -75,6 +64,7 @@ def fixture_rabbitmq_defaults(rabbit_mq): @fixture(scope="package", name="rabbitmq_publisher") def fixture_rabbitmq_publisher(rabbitmq_defaults): + assert rabbitmq_defaults rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_publisher = RMQPublisher( host=rmq_data["host"], @@ -91,6 +81,7 @@ def fixture_rabbitmq_publisher(rabbitmq_defaults): @fixture(scope="package", name="rabbitmq_consumer") def fixture_rabbitmq_consumer(rabbitmq_defaults): + assert rabbitmq_defaults rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_consumer = RMQConsumer( host=rmq_data["host"], From fd13145f3742a58ab2f0ed892ef5cc069465ddde Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 14 Jan 2024 18:00:12 +0100 Subject: [PATCH 15/93] ci: make integration-test --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 94536bf491..74276b0a34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: - checkout - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos bash - run: make install - - run: PATH="/Users/distiller/Library/Python/3.9/bin:$PATH" make deps-test test benchmark + - run: PATH="/Users/distiller/Library/Python/3.9/bin:$PATH" make deps-test test benchmark integration-test test-python37: docker: @@ -31,7 +31,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark + - run: make test benchmark integration-test # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -46,7 +46,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark + - run: make test benchmark integration-test # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -61,7 +61,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark + - run: make test benchmark integration-test # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -76,7 +76,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark + - run: make test benchmark integration-test # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -91,7 +91,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark + - run: make test benchmark integration-test # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version From 55c3e3b66630788f7b19ef9ac9cb72afb3e8153e Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 14 Jan 2024 18:10:42 +0100 Subject: [PATCH 16/93] network: try 0.0.0.0 over localhost --- tests/network/constants.py | 6 +++--- tests/network/fixtures_rabbitmq.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/network/constants.py b/tests/network/constants.py index d769ef51fb..f7ef5a75bf 100644 --- a/tests/network/constants.py +++ b/tests/network/constants.py @@ -9,10 +9,10 @@ # mongodb://localhost:6701/ocrd_network_test DB_NAME: str = "ocrd_network_test" -DB_URL: str = "mongodb://localhost:6701" +DB_URL: str = "mongodb://0.0.0.0:6701" DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" DEFAULT_QUEUE: str = "ocrd-network-default" -PROCESSING_SERVER_URL: str = "http://localhost:8000" -RABBITMQ_URL: str = "amqp://network_test:network_test@localhost:6672/" +PROCESSING_SERVER_URL: str = "http://0.0.0.0:8000" +RABBITMQ_URL: str = "amqp://network_test:network_test@0.0.0.0:6672/" diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index d2540b6418..a99e759a2a 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -64,7 +64,6 @@ def fixture_rabbitmq_defaults(): @fixture(scope="package", name="rabbitmq_publisher") def fixture_rabbitmq_publisher(rabbitmq_defaults): - assert rabbitmq_defaults rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_publisher = RMQPublisher( host=rmq_data["host"], @@ -81,7 +80,6 @@ def fixture_rabbitmq_publisher(rabbitmq_defaults): @fixture(scope="package", name="rabbitmq_consumer") def fixture_rabbitmq_consumer(rabbitmq_defaults): - assert rabbitmq_defaults rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) rmq_consumer = RMQConsumer( host=rmq_data["host"], From b73879b2f8f89e6656c4e9a7b95713b6874b06e8 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 11:24:41 +0100 Subject: [PATCH 17/93] improve local exec of integration tests --- Makefile | 2 +- tests/network/docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9147eb9c9a..257710461e 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ integration-test: docker compose -f tests/network/docker-compose.yml up -d sleep 10 pytest -k 'test_consume or test_publish' - docker compose -f tests/network/docker-compose.yml down + docker compose -f tests/network/docker-compose.yml down --remove-orphans benchmark: $(PYTHON) -m pytest $(TESTDIR)/model/test_ocrd_mets_bench.py diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 7e1a2ca86e..f2ecd197df 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -4,6 +4,7 @@ networks: driver: bridge driver_opts: com.docker.network.driver.mtu: 1450 + external: true services: ocrd_network_mongo_db: image: "mongo" From 87e21e4aa714130f24bcc0c27757245c9169e972 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 11:34:06 +0100 Subject: [PATCH 18/93] remove external expectation --- tests/network/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index f2ecd197df..7e1a2ca86e 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -4,7 +4,6 @@ networks: driver: bridge driver_opts: com.docker.network.driver.mtu: 1450 - external: true services: ocrd_network_mongo_db: image: "mongo" From 0c296108e303c5fa7dfab709981af63ae9b7c8da Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 12:13:40 +0100 Subject: [PATCH 19/93] try machine executor for pyt37 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74276b0a34..17f5f34f33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ + - machine: # https://circleci.com/docs/2.0/building-docker-images/ docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu From 68a5c6ff16d4683ca292aa61de8ab07566118ded Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 12:20:17 +0100 Subject: [PATCH 20/93] revert to remote docker --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17f5f34f33..74276b0a34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - machine: # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ docker_layer_caching: true - run: sudo apt-get -y update - run: sudo make deps-ubuntu From 12836a83a59323955a6247fd280b8aaf5a133c3d Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Mon, 15 Jan 2024 12:24:08 +0100 Subject: [PATCH 21/93] proof-of-concept of in-docker integration tests --- .dockerignore | 3 +++ Dockerfile | 20 +++++++++++++++----- Makefile | 7 ++++--- tests/conftest.py | 12 ++++++++++++ tests/network/docker-compose.yml | 19 +++++++++++++++++++ tests/network/fixtures_mongodb.py | 4 +++- tests/network/fixtures_rabbitmq.py | 6 +++++- tests/network/test_rabbitmq.py | 5 ++++- 8 files changed, 65 insertions(+), 11 deletions(-) diff --git a/.dockerignore b/.dockerignore index aefc504c60..705eb6dd36 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,6 @@ !LICENSE !README.md !.git +!tests +!requirements_test.txt +!.gitmodules diff --git a/Dockerfile b/Dockerfile index 00bd8bb30a..f1cf5cb009 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE -FROM $BASE_IMAGE +FROM $BASE_IMAGE as ocrd_core_base ARG FIXUP=echo MAINTAINER OCR-D ENV DEBIAN_FRONTEND noninteractive @@ -33,14 +33,24 @@ RUN apt-get update && apt-get -y install software-properties-common \ curl \ sudo \ git \ - && make deps-ubuntu \ - && python3 -m venv /usr/local \ + && make deps-ubuntu +RUN python3 -m venv /usr/local \ && hash -r \ && pip install --upgrade pip setuptools wheel \ && make install \ - && eval $FIXUP \ - && rm -rf /build-ocrd + && eval $FIXUP WORKDIR /data CMD ["/usr/local/bin/ocrd", "--help"] + +FROM ocrd_core_base as ocrd_core_test +WORKDIR /build-ocrd +COPY tests ./tests +COPY .gitmodules . +COPY Makefile . +COPY requirements_test.txt . +RUN pip install -r requirements_test.txt + +CMD ["yes"] +# CMD ["make", "test", "integration-test"] diff --git a/Makefile b/Makefile index 9147eb9c9a..9997f968ec 100644 --- a/Makefile +++ b/Makefile @@ -213,10 +213,11 @@ test: assets $(TESTDIR) cd ocrd_utils ; $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging -k TestDecorators $(TESTDIR) +INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d - sleep 10 - pytest -k 'test_consume or test_publish' + sleep 5 + $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_consume or test_publish' docker compose -f tests/network/docker-compose.yml down benchmark: @@ -295,7 +296,7 @@ docker-cuda: DOCKER_FILE = Dockerfile.cuda docker-cuda: docker docker docker-cuda: - docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . + docker build --progress=plain -f $(DOCKER_FILE) -t $(DOCKER_TAG) --target ocrd_core_base --build-arg BASE_IMAGE=$(DOCKER_BASE_IMAGE) $(DOCKER_ARGS) . # Build wheels and source dist and twine upload them pypi: build diff --git a/tests/conftest.py b/tests/conftest.py index da47f3f8b5..5bf13863aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,16 @@ +from ocrd_utils.config import config + pytest_plugins = [ "tests.network.fixtures_mongodb", "tests.network.fixtures_rabbitmq" ] + +config.add("DB_NAME", description="...", default=(True, 'ocrd_network_test')) +# mongodb://localhost:6701/ocrd_network_test +config.add("DB_URL", description="...", default=(True, 'mongodb://0.0.0.0:6701')) + +config.add('DEFAULT_EXCHANGER_NAME', description="...", default=(True, 'ocrd-network-default')) +config.add('DEFAULT_QUEUE', description="...", default=(True, 'ocrd-network-default')) + +config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) +config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:6672/")) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 7e1a2ca86e..7019629530 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -4,7 +4,9 @@ networks: driver: bridge driver_opts: com.docker.network.driver.mtu: 1450 + services: + ocrd_network_mongo_db: image: "mongo" hostname: mongodb-docker-host @@ -15,6 +17,7 @@ services: environment: - MONGO_INITDB_ROOT_USERNAME=network_test - MONGO_INITDB_ROOT_PASSWORD=network_test + ocrd_network_rabbit_mq: image: "rabbitmq:3.12-management" hostname: rabbitmq-docker-host @@ -28,3 +31,19 @@ services: - RABBITMQ_DEFAULT_USER=network_test - RABBITMQ_DEFAULT_PASS=network_test - RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version + + ocrd_network_core_test: + build: + context: ../../ + dockerfile: Dockerfile + args: + BASE_IMAGE: 'ubuntu:20.04' + target: ocrd_core_test + container_name: core_test + networks: + - ocrd_network_test + environment: + DB_NAME: ocrd_network_test + DB_URL: mongodb://mongodb-docker-host:27017 + # PROCESSING_SERVER_URL : http://... + RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 1d8826aeaa..deecf61da8 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,6 +1,8 @@ from pymongo import MongoClient, uri_parser as mongo_uri_parser from pytest import fixture -from .constants import DB_NAME, DB_URL +from ocrd_utils.config import config +DB_NAME = config.DB_NAME +DB_URL = config.DB_URL def verify_database_uri(mongodb_address: str) -> str: diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index a99e759a2a..06e99a98c1 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -3,7 +3,11 @@ from pytest import fixture from re import match as re_match from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher -from .constants import RABBITMQ_URL, DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE +from ocrd_utils.config import config + +RABBITMQ_URL = config.RABBITMQ_URL +DEFAULT_EXCHANGER_NAME = config.DEFAULT_EXCHANGER_NAME +DEFAULT_QUEUE = config.DEFAULT_QUEUE def verify_and_parse_mq_uri(rabbitmq_address: str): diff --git a/tests/network/test_rabbitmq.py b/tests/network/test_rabbitmq.py index f9a2e89419..644f86adca 100644 --- a/tests/network/test_rabbitmq.py +++ b/tests/network/test_rabbitmq.py @@ -1,6 +1,9 @@ from pika import BasicProperties from pickle import dumps, loads -from .constants import DEFAULT_EXCHANGER_NAME, DEFAULT_QUEUE +from ocrd_utils.config import config + +DEFAULT_EXCHANGER_NAME = config.DEFAULT_EXCHANGER_NAME +DEFAULT_QUEUE = config.DEFAULT_QUEUE def test_publish_2_messages_to_rabbitmq(rabbitmq_publisher): From bea14b7bd56c6d6598b06567fe93c1f61f3879f1 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Mon, 15 Jan 2024 12:35:03 +0100 Subject: [PATCH 22/93] increase sleep time again X( --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9997f968ec..67b87cc71e 100644 --- a/Makefile +++ b/Makefile @@ -216,7 +216,7 @@ test: assets INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d - sleep 5 + sleep 10 $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_consume or test_publish' docker compose -f tests/network/docker-compose.yml down From 52064a57349c477acc45d637f5492dccc522194a Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 13:06:10 +0100 Subject: [PATCH 23/93] change core test envs --- tests/network/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 7019629530..3c2ef56852 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -44,6 +44,6 @@ services: - ocrd_network_test environment: DB_NAME: ocrd_network_test - DB_URL: mongodb://mongodb-docker-host:27017 + DB_URL: mongodb://network_test:network_test@mongodb-docker-host:27017 # PROCESSING_SERVER_URL : http://... - RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host:5672 From c96fd2ed5cb24ac83ad2ce97eccae3e204a48a97 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 13:08:08 +0100 Subject: [PATCH 24/93] remove constants --- tests/network/constants.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tests/network/constants.py diff --git a/tests/network/constants.py b/tests/network/constants.py deleted file mode 100644 index f7ef5a75bf..0000000000 --- a/tests/network/constants.py +++ /dev/null @@ -1,18 +0,0 @@ -__all__ = [ - "DB_NAME", - "DB_URL", - "DEFAULT_EXCHANGER_NAME", - "DEFAULT_QUEUE", - "PROCESSING_SERVER_URL", - "RABBITMQ_URL" -] - -# mongodb://localhost:6701/ocrd_network_test -DB_NAME: str = "ocrd_network_test" -DB_URL: str = "mongodb://0.0.0.0:6701" - -DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" -DEFAULT_QUEUE: str = "ocrd-network-default" - -PROCESSING_SERVER_URL: str = "http://0.0.0.0:8000" -RABBITMQ_URL: str = "amqp://network_test:network_test@0.0.0.0:6672/" From 0e5cf7e1b3831decf2065b02f6d6cb8b213f4315 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 14:46:26 +0100 Subject: [PATCH 25/93] add 2 base db tests --- ocrd_network/ocrd_network/database.py | 8 +++---- tests/conftest.py | 9 +++++--- tests/network/test_db.py | 33 +++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 tests/network/test_db.py diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 4945c13353..7844df991f 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -28,17 +28,17 @@ from .utils import call_sync -async def initiate_database(db_url: str): +async def initiate_database(db_url: str, db_name: str = 'ocrd'): client = AsyncIOMotorClient(db_url) await init_beanie( - database=client.get_default_database(default='ocrd'), + database=client.get_default_database(default=db_name), document_models=[DBProcessorJob, DBWorkflowJob, DBWorkspace, DBWorkflowScript] ) @call_sync -async def sync_initiate_database(db_url: str): - await initiate_database(db_url) +async def sync_initiate_database(db_url: str, db_name: str = 'ocrd'): + await initiate_database(db_url, db_name) async def db_create_workspace(mets_path: str) -> DBWorkspace: diff --git a/tests/conftest.py b/tests/conftest.py index 5bf13863aa..d6c0850c2d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,19 @@ +from ocrd_network.database import sync_initiate_database from ocrd_utils.config import config + pytest_plugins = [ "tests.network.fixtures_mongodb", "tests.network.fixtures_rabbitmq" ] -config.add("DB_NAME", description="...", default=(True, 'ocrd_network_test')) -# mongodb://localhost:6701/ocrd_network_test -config.add("DB_URL", description="...", default=(True, 'mongodb://0.0.0.0:6701')) +config.add("DB_NAME", description="...", default=(True, 'ocrd')) +config.add("DB_URL", description="...", default=(True, 'mongodb://network_test:network_test@0.0.0.0:6701')) config.add('DEFAULT_EXCHANGER_NAME', description="...", default=(True, 'ocrd-network-default')) config.add('DEFAULT_QUEUE', description="...", default=(True, 'ocrd-network-default')) config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:6672/")) + +sync_initiate_database(config.DB_URL, config.DB_NAME) diff --git a/tests/network/test_db.py b/tests/network/test_db.py new file mode 100644 index 0000000000..41b02a4cbd --- /dev/null +++ b/tests/network/test_db.py @@ -0,0 +1,33 @@ +from ocrd_network.models import DBProcessorJob, StateEnum +from ocrd_network.database import sync_db_get_processing_job + + +def test_db_write_processing_job(mongo_processor_jobs): + job_id = 'test_id_1234' + db_processing_job = DBProcessorJob( + job_id=job_id, + processor_name='ocrd-dummy', + state=StateEnum.cached, + path_to_mets='/ocrd/dummy/path', + input_file_grps=['DEFAULT'], + output_file_grps=['OCR-D-DUMMY'] + ) + inserted_job = db_processing_job.insert() + assert inserted_job + found_job = mongo_processor_jobs.find_one({"job_id": job_id}) + assert found_job + + +def test_db_read_processing_job(): + job_id = 'test_id_1234' + db_processing_job = sync_db_get_processing_job(job_id=job_id) + assert db_processing_job.job_id == job_id + assert db_processing_job.processor_name == 'ocrd-dummy' + assert db_processing_job.state == StateEnum.cached + assert db_processing_job.path_to_mets == '/ocrd/dummy/path' + assert db_processing_job.input_file_grps == ['DEFAULT'] + assert db_processing_job.output_file_grps == ['OCR-D-DUMMY'] + + +def test_db_update_processing_job(): + pass From 780621159ebe42669b4a6af1a0ca5fe2047732a0 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 14:46:43 +0100 Subject: [PATCH 26/93] refactor name rabbitmq tests --- Makefile | 2 +- tests/network/test_rabbitmq.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f4a7cd4855..5432df523a 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d sleep 10 - $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_consume or test_publish' + $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db' docker compose -f tests/network/docker-compose.yml down --remove-orphans benchmark: diff --git a/tests/network/test_rabbitmq.py b/tests/network/test_rabbitmq.py index 644f86adca..0b208718f3 100644 --- a/tests/network/test_rabbitmq.py +++ b/tests/network/test_rabbitmq.py @@ -6,7 +6,7 @@ DEFAULT_QUEUE = config.DEFAULT_QUEUE -def test_publish_2_messages_to_rabbitmq(rabbitmq_publisher): +def test_rmq_publish_2_messages(rabbitmq_publisher): test_headers = {"Test Header": "Test Value"} test_properties = BasicProperties( app_id='webapi-processing-broker', @@ -28,7 +28,7 @@ def test_publish_2_messages_to_rabbitmq(rabbitmq_publisher): assert rabbitmq_publisher.message_counter == 2 -def test_consume_2_messages_from_rabbitmq(rabbitmq_consumer): +def test_rmq_consume_2_messages(rabbitmq_consumer): # Consume the 1st message method_frame, header_frame, message = rabbitmq_consumer.get_one_message( queue_name=DEFAULT_QUEUE, @@ -56,7 +56,7 @@ def test_consume_2_messages_from_rabbitmq(rabbitmq_consumer): assert message.decode() == "RabbitMQ test 456" -def test_publish_ocrd_message_to_rabbitmq(rabbitmq_publisher): +def test_rmq_publish_ocrd_message(rabbitmq_publisher): ocrd_processing_message = { "job_id": "Test_job_id", "workflow_id": "Test_workflow_id", @@ -71,7 +71,7 @@ def test_publish_ocrd_message_to_rabbitmq(rabbitmq_publisher): ) -def test_consume_ocrd_message_from_rabbitmq(rabbitmq_consumer): +def test_rmq_consume_ocrd_message(rabbitmq_consumer): method_frame, header_frame, message = rabbitmq_consumer.get_one_message( queue_name=DEFAULT_QUEUE, auto_ack=True From 18d7c7ef644916b8f86bf0108fd7f7e22b448e10 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 15:51:01 +0100 Subject: [PATCH 27/93] fix DB tests --- tests/conftest.py | 5 +--- tests/network/fixtures_mongodb.py | 4 +++ tests/network/test_db.py | 50 ++++++++++++++++++------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d6c0850c2d..de39d7874a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -from ocrd_network.database import sync_initiate_database from ocrd_utils.config import config @@ -7,7 +6,7 @@ "tests.network.fixtures_rabbitmq" ] -config.add("DB_NAME", description="...", default=(True, 'ocrd')) +config.add("DB_NAME", description="...", default=(True, 'ocrd_network_test')) config.add("DB_URL", description="...", default=(True, 'mongodb://network_test:network_test@0.0.0.0:6701')) config.add('DEFAULT_EXCHANGER_NAME', description="...", default=(True, 'ocrd-network-default')) @@ -15,5 +14,3 @@ config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:6672/")) - -sync_initiate_database(config.DB_URL, config.DB_NAME) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index deecf61da8..6930962194 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,9 +1,13 @@ from pymongo import MongoClient, uri_parser as mongo_uri_parser from pytest import fixture from ocrd_utils.config import config +from ocrd_network.database import sync_initiate_database + DB_NAME = config.DB_NAME DB_URL = config.DB_URL +sync_initiate_database(DB_URL, DB_NAME) + def verify_database_uri(mongodb_address: str) -> str: try: diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 41b02a4cbd..5c0944ff89 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,33 +1,43 @@ -from ocrd_network.models import DBProcessorJob, StateEnum -from ocrd_network.database import sync_db_get_processing_job +from ocrd_network.models import StateEnum def test_db_write_processing_job(mongo_processor_jobs): job_id = 'test_id_1234' - db_processing_job = DBProcessorJob( - job_id=job_id, - processor_name='ocrd-dummy', - state=StateEnum.cached, - path_to_mets='/ocrd/dummy/path', - input_file_grps=['DEFAULT'], - output_file_grps=['OCR-D-DUMMY'] + inserted_job = mongo_processor_jobs.insert_one( + document={ + "job_id": job_id, + "processor_name": 'ocrd-dummy', + "state": StateEnum.cached, + "path_to_mets": '/ocrd/dummy/path', + "input_file_grps": ['DEFAULT'], + "output_file_grps": ['OCR-D-DUMMY'] + } ) - inserted_job = db_processing_job.insert() assert inserted_job found_job = mongo_processor_jobs.find_one({"job_id": job_id}) assert found_job -def test_db_read_processing_job(): +def test_db_read_processing_job(mongo_processor_jobs): job_id = 'test_id_1234' - db_processing_job = sync_db_get_processing_job(job_id=job_id) - assert db_processing_job.job_id == job_id - assert db_processing_job.processor_name == 'ocrd-dummy' - assert db_processing_job.state == StateEnum.cached - assert db_processing_job.path_to_mets == '/ocrd/dummy/path' - assert db_processing_job.input_file_grps == ['DEFAULT'] - assert db_processing_job.output_file_grps == ['OCR-D-DUMMY'] + found_job = mongo_processor_jobs.find_one({"job_id": job_id}) + assert found_job + assert found_job['job_id'] == job_id + assert found_job['processor_name'] == 'ocrd-dummy' + assert found_job['state'] == StateEnum.cached + assert found_job['path_to_mets'] == '/ocrd/dummy/path' + assert found_job['input_file_grps'] == ['DEFAULT'] + assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] -def test_db_update_processing_job(): - pass +def test_db_update_processing_job(mongo_processor_jobs): + job_id = 'test_id_1234' + found_job = mongo_processor_jobs.find_one({"job_id": job_id}) + assert found_job + found_job['state'] = StateEnum.running + assert found_job['job_id'] == job_id + assert found_job['processor_name'] == 'ocrd-dummy' + assert found_job['state'] == StateEnum.running + assert found_job['path_to_mets'] == '/ocrd/dummy/path' + assert found_job['input_file_grps'] == ['DEFAULT'] + assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] From 28debcf732b7d6c469d5f4baac60c64553945a36 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 16:01:40 +0100 Subject: [PATCH 28/93] do a proper update --- tests/network/test_db.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 5c0944ff89..a128aa1856 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -32,12 +32,11 @@ def test_db_read_processing_job(mongo_processor_jobs): def test_db_update_processing_job(mongo_processor_jobs): job_id = 'test_id_1234' - found_job = mongo_processor_jobs.find_one({"job_id": job_id}) - assert found_job - found_job['state'] = StateEnum.running - assert found_job['job_id'] == job_id - assert found_job['processor_name'] == 'ocrd-dummy' - assert found_job['state'] == StateEnum.running - assert found_job['path_to_mets'] == '/ocrd/dummy/path' - assert found_job['input_file_grps'] == ['DEFAULT'] - assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] + updated_job = mongo_processor_jobs.update_one(filter={"job_id": job_id}, update={"state": StateEnum.running}) + assert updated_job + assert updated_job['job_id'] == job_id + assert updated_job['processor_name'] == 'ocrd-dummy' + assert updated_job['state'] == StateEnum.running + assert updated_job['path_to_mets'] == '/ocrd/dummy/path' + assert updated_job['input_file_grps'] == ['DEFAULT'] + assert updated_job['output_file_grps'] == ['OCR-D-DUMMY'] From 425c4c24e8baef28e0468dbf22868f57ae00aee9 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 16:02:06 +0100 Subject: [PATCH 29/93] fix host for circle ci --- tests/network/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 3c2ef56852..5c20d29b94 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -44,6 +44,6 @@ services: - ocrd_network_test environment: DB_NAME: ocrd_network_test - DB_URL: mongodb://network_test:network_test@mongodb-docker-host:27017 + DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 # PROCESSING_SERVER_URL : http://... - RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host:5672 + RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 From 38a073779ca73a16d5296d58a6d8ec9ea74e0884 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 16:32:17 +0100 Subject: [PATCH 30/93] fix db update test --- tests/network/test_db.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/network/test_db.py b/tests/network/test_db.py index a128aa1856..435afbdf5a 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -14,13 +14,13 @@ def test_db_write_processing_job(mongo_processor_jobs): } ) assert inserted_job - found_job = mongo_processor_jobs.find_one({"job_id": job_id}) + found_job = mongo_processor_jobs.find_one(filter={"job_id": job_id}) assert found_job def test_db_read_processing_job(mongo_processor_jobs): job_id = 'test_id_1234' - found_job = mongo_processor_jobs.find_one({"job_id": job_id}) + found_job = mongo_processor_jobs.find_one(filter={"job_id": job_id}) assert found_job assert found_job['job_id'] == job_id assert found_job['processor_name'] == 'ocrd-dummy' @@ -32,11 +32,15 @@ def test_db_read_processing_job(mongo_processor_jobs): def test_db_update_processing_job(mongo_processor_jobs): job_id = 'test_id_1234' - updated_job = mongo_processor_jobs.update_one(filter={"job_id": job_id}, update={"state": StateEnum.running}) - assert updated_job - assert updated_job['job_id'] == job_id - assert updated_job['processor_name'] == 'ocrd-dummy' - assert updated_job['state'] == StateEnum.running - assert updated_job['path_to_mets'] == '/ocrd/dummy/path' - assert updated_job['input_file_grps'] == ['DEFAULT'] - assert updated_job['output_file_grps'] == ['OCR-D-DUMMY'] + mongo_processor_jobs.update_one( + filter={"job_id": job_id}, + update={"$set": {"state": StateEnum.running}} + ) + found_job = mongo_processor_jobs.find_one(filter={"job_id": job_id}) + assert found_job + assert found_job['job_id'] == job_id + assert found_job['processor_name'] == 'ocrd-dummy' + assert found_job['state'] == StateEnum.running + assert found_job['path_to_mets'] == '/ocrd/dummy/path' + assert found_job['input_file_grps'] == ['DEFAULT'] + assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] From 84656ef7dc9ca60cd706499e4648b68bd43fe137 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 16:42:46 +0100 Subject: [PATCH 31/93] wait longer time --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5432df523a..81b97ae800 100644 --- a/Makefile +++ b/Makefile @@ -216,7 +216,7 @@ test: assets INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d - sleep 10 + sleep 20 $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db' docker compose -f tests/network/docker-compose.yml down --remove-orphans From 9afea111610df9434175388c9393b497c0debb6c Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 16:51:26 +0100 Subject: [PATCH 32/93] move init db inside fixture --- tests/network/fixtures_mongodb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 6930962194..4c02897e8d 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -6,8 +6,6 @@ DB_NAME = config.DB_NAME DB_URL = config.DB_URL -sync_initiate_database(DB_URL, DB_NAME) - def verify_database_uri(mongodb_address: str) -> str: try: @@ -26,6 +24,7 @@ def fixture_mongo_client(): @fixture(scope="package", name="mongo_processor_jobs") def fixture_mongo_processor_jobs(mongo_client): + sync_initiate_database(DB_URL, DB_NAME) mydb = mongo_client[DB_NAME] processor_jobs_collection = mydb["DBProcessorJob"] yield processor_jobs_collection From c9a8cd37d0c227eb2fb66d1407c710b816abc643 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 16:57:47 +0100 Subject: [PATCH 33/93] put init_db inside the fixture --- tests/network/fixtures_mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 4c02897e8d..ec4fb23211 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -18,13 +18,13 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_client") def fixture_mongo_client(): + sync_initiate_database(DB_URL, DB_NAME) mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) yield mongo_client @fixture(scope="package", name="mongo_processor_jobs") def fixture_mongo_processor_jobs(mongo_client): - sync_initiate_database(DB_URL, DB_NAME) mydb = mongo_client[DB_NAME] processor_jobs_collection = mydb["DBProcessorJob"] yield processor_jobs_collection From be359f8244dade73654e2c943ef09d9fe49a7666 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Mon, 15 Jan 2024 17:10:48 +0100 Subject: [PATCH 34/93] comment out init db --- tests/network/fixtures_mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index ec4fb23211..2fd7e3f680 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -18,7 +18,7 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_client") def fixture_mongo_client(): - sync_initiate_database(DB_URL, DB_NAME) + # sync_initiate_database(DB_URL, DB_NAME) mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) yield mongo_client From cdf7c4afc0ab189e494dcd06d8064fa8be8fc29b Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Mon, 15 Jan 2024 18:34:50 +0100 Subject: [PATCH 35/93] ci: disable layer caching until we fix the outdated core_test image problem --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74276b0a34..7da320a83d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: true + docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -42,7 +42,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: true + docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -57,7 +57,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: true + docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -72,7 +72,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: true + docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -87,7 +87,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: true + docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -101,7 +101,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: true + docker_layer_caching: false - run: make docker - run: make docker-cuda - run: From 6182a004fa45564ae70c94403d07c9965b25515e Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 11:32:45 +0100 Subject: [PATCH 36/93] add sync version of db_create_workspace --- ocrd_network/ocrd_network/database.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 7844df991f..73e2699f23 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -60,6 +60,11 @@ async def db_create_workspace(mets_path: str) -> DBWorkspace: return workspace_db +@call_sync +async def sync_db_create_workspace(mets_path: str) -> DBWorkspace: + return await db_create_workspace(mets_path=mets_path) + + async def db_get_workspace(workspace_id: str = None, workspace_mets_path: str = None) -> DBWorkspace: workspace = None if not workspace_id and not workspace_mets_path: From a23fe9ef06b31ec9837ef5ebc2ace0342576c3ea Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 11:34:42 +0100 Subject: [PATCH 37/93] add db workspace tests --- tests/network/fixtures_mongodb.py | 11 ++++++++- tests/network/test_db.py | 39 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 2fd7e3f680..b3f8b8c2b8 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -18,7 +18,8 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_client") def fixture_mongo_client(): - # sync_initiate_database(DB_URL, DB_NAME) + # This extra initialization is needed to test the db wrapper methods + sync_initiate_database(DB_URL, DB_NAME) mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) yield mongo_client @@ -37,3 +38,11 @@ def fixture_mongo_workflow_jobs(mongo_client): workflow_jobs_collection = mydb["DBWorkflowJob"] yield workflow_jobs_collection workflow_jobs_collection.drop() + + +@fixture(scope="package", name="mongo_workspaces") +def fixture_mongo_workspaces(mongo_client): + mydb = mongo_client[DB_NAME] + workspaces_collection = mydb["DBWorkspace"] + yield workspaces_collection + workspaces_collection.drop() diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 435afbdf5a..a3fb87317a 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,4 +1,10 @@ +from tests.base import assets from ocrd_network.models import StateEnum +from ocrd_network.database import ( + sync_db_create_workspace, + sync_db_get_workspace, + sync_db_update_workspace +) def test_db_write_processing_job(mongo_processor_jobs): @@ -44,3 +50,36 @@ def test_db_update_processing_job(mongo_processor_jobs): assert found_job['path_to_mets'] == '/ocrd/dummy/path' assert found_job['input_file_grps'] == ['DEFAULT'] assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] + + +def test_db_create_workspace(mongo_workspaces): + mets_path = assets.path_to('kant_aufklaerung_1784/data/mets.xml') + db_created_workspace = sync_db_create_workspace(mets_path=mets_path) + assert db_created_workspace + assert db_created_workspace.workspace_mets_path == mets_path + db_found_workspace = sync_db_get_workspace(workspace_mets_path=mets_path) + assert db_found_workspace + assert db_found_workspace.workspace_mets_path == mets_path + + +def test_db_update_workspace(mongo_workspaces): + mets_path = assets.path_to('kant_aufklaerung_1784-binarized/data/mets.xml') + dummy_mets_server_url = '/tmp/dummy.sock' + + db_created_workspace = sync_db_create_workspace(mets_path=mets_path) + assert db_created_workspace + db_found_workspace = sync_db_get_workspace(workspace_mets_path=mets_path) + assert db_found_workspace + assert db_created_workspace == db_found_workspace + + db_updated_workspace = sync_db_update_workspace( + workspace_mets_path=mets_path, + mets_server_url=dummy_mets_server_url + ) + assert db_updated_workspace + assert db_updated_workspace != db_created_workspace + + db_found_updated_workspace = sync_db_get_workspace(workspace_mets_path=mets_path) + assert db_found_updated_workspace + assert db_found_updated_workspace.workspace_mets_path == mets_path + assert db_found_updated_workspace.mets_server_url == dummy_mets_server_url From 39c7e05e06b9c137fecf7bf80965196d1aeaec88 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 12:28:22 +0100 Subject: [PATCH 38/93] refactor db processing job tests --- tests/network/test_db.py | 75 +++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/tests/network/test_db.py b/tests/network/test_db.py index a3fb87317a..371b0dbe10 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,15 +1,19 @@ from tests.base import assets from ocrd_network.models import StateEnum from ocrd_network.database import ( + sync_db_get_processing_job, + sync_db_update_processing_job, sync_db_create_workspace, sync_db_get_workspace, sync_db_update_workspace ) -def test_db_write_processing_job(mongo_processor_jobs): +def test_db_processing_job_create(mongo_processor_jobs): job_id = 'test_id_1234' - inserted_job = mongo_processor_jobs.insert_one( + # TODO: There is no db wrapper to create processing job + # Hence, for now, use a low level method to insert a job + db_created_processing_job = mongo_processor_jobs.insert_one( document={ "job_id": job_id, "processor_name": 'ocrd-dummy', @@ -19,50 +23,56 @@ def test_db_write_processing_job(mongo_processor_jobs): "output_file_grps": ['OCR-D-DUMMY'] } ) - assert inserted_job - found_job = mongo_processor_jobs.find_one(filter={"job_id": job_id}) - assert found_job + assert db_created_processing_job + db_found_processing_job = sync_db_get_processing_job(job_id=job_id) + assert db_found_processing_job + assert db_found_processing_job.job_id == job_id + assert db_found_processing_job.processor_name == 'ocrd-dummy' + assert db_found_processing_job.state == StateEnum.cached + assert db_found_processing_job.path_to_mets == '/ocrd/dummy/path' + assert db_found_processing_job.input_file_grps == ['DEFAULT'] + assert db_found_processing_job.output_file_grps == ['OCR-D-DUMMY'] -def test_db_read_processing_job(mongo_processor_jobs): - job_id = 'test_id_1234' - found_job = mongo_processor_jobs.find_one(filter={"job_id": job_id}) - assert found_job - assert found_job['job_id'] == job_id - assert found_job['processor_name'] == 'ocrd-dummy' - assert found_job['state'] == StateEnum.cached - assert found_job['path_to_mets'] == '/ocrd/dummy/path' - assert found_job['input_file_grps'] == ['DEFAULT'] - assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] +def test_db_processing_job_update(mongo_processor_jobs): + job_id = 'test_id_12345' + # TODO: There is no db wrapper to create processing job + # Hence, for now, use a low level method to insert a job + db_created_processing_job = mongo_processor_jobs.insert_one( + document={ + "job_id": job_id, + "processor_name": 'ocrd-dummy', + "state": StateEnum.cached, + "path_to_mets": '/ocrd/dummy/path', + "input_file_grps": ['DEFAULT'], + "output_file_grps": ['OCR-D-DUMMY'] + } + ) + assert db_created_processing_job + db_found_processing_job = sync_db_get_processing_job(job_id=job_id) + assert db_found_processing_job -def test_db_update_processing_job(mongo_processor_jobs): - job_id = 'test_id_1234' - mongo_processor_jobs.update_one( - filter={"job_id": job_id}, - update={"$set": {"state": StateEnum.running}} - ) - found_job = mongo_processor_jobs.find_one(filter={"job_id": job_id}) - assert found_job - assert found_job['job_id'] == job_id - assert found_job['processor_name'] == 'ocrd-dummy' - assert found_job['state'] == StateEnum.running - assert found_job['path_to_mets'] == '/ocrd/dummy/path' - assert found_job['input_file_grps'] == ['DEFAULT'] - assert found_job['output_file_grps'] == ['OCR-D-DUMMY'] + db_updated_processing_job = sync_db_update_processing_job(job_id=job_id, state=StateEnum.running) + assert db_found_processing_job != db_updated_processing_job + + db_found_updated_processing_job = sync_db_get_processing_job(job_id=job_id) + assert db_found_updated_processing_job + assert db_found_updated_processing_job == db_updated_processing_job + assert db_found_updated_processing_job.state == StateEnum.running -def test_db_create_workspace(mongo_workspaces): +def test_db_workspace_create(mongo_workspaces): mets_path = assets.path_to('kant_aufklaerung_1784/data/mets.xml') db_created_workspace = sync_db_create_workspace(mets_path=mets_path) assert db_created_workspace assert db_created_workspace.workspace_mets_path == mets_path db_found_workspace = sync_db_get_workspace(workspace_mets_path=mets_path) assert db_found_workspace - assert db_found_workspace.workspace_mets_path == mets_path + assert db_found_workspace == db_created_workspace -def test_db_update_workspace(mongo_workspaces): +def test_db_workspace_update(mongo_workspaces): mets_path = assets.path_to('kant_aufklaerung_1784-binarized/data/mets.xml') dummy_mets_server_url = '/tmp/dummy.sock' @@ -83,3 +93,4 @@ def test_db_update_workspace(mongo_workspaces): assert db_found_updated_workspace assert db_found_updated_workspace.workspace_mets_path == mets_path assert db_found_updated_workspace.mets_server_url == dummy_mets_server_url + assert db_found_updated_workspace == db_updated_workspace From 9ee09f9788094212030770809c9c4f44e8b84a83 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 13:01:41 +0100 Subject: [PATCH 39/93] refine tests, remove unnecessary --- tests/network/fixtures_mongodb.py | 16 ---------------- tests/network/test_db.py | 23 ++++++++++++++++++----- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index b3f8b8c2b8..647492986b 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -30,19 +30,3 @@ def fixture_mongo_processor_jobs(mongo_client): processor_jobs_collection = mydb["DBProcessorJob"] yield processor_jobs_collection processor_jobs_collection.drop() - - -@fixture(scope="package", name="mongo_workflow_jobs") -def fixture_mongo_workflow_jobs(mongo_client): - mydb = mongo_client[DB_NAME] - workflow_jobs_collection = mydb["DBWorkflowJob"] - yield workflow_jobs_collection - workflow_jobs_collection.drop() - - -@fixture(scope="package", name="mongo_workspaces") -def fixture_mongo_workspaces(mongo_client): - mydb = mongo_client[DB_NAME] - workspaces_collection = mydb["DBWorkspace"] - yield workspaces_collection - workspaces_collection.drop() diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 371b0dbe10..6ab7d9816b 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,3 +1,4 @@ +from pytest import raises from tests.base import assets from ocrd_network.models import StateEnum from ocrd_network.database import ( @@ -33,6 +34,9 @@ def test_db_processing_job_create(mongo_processor_jobs): assert db_found_processing_job.input_file_grps == ['DEFAULT'] assert db_found_processing_job.output_file_grps == ['OCR-D-DUMMY'] + with raises(ValueError) as value_error: + sync_db_get_processing_job(job_id='non-existing-id') + def test_db_processing_job_update(mongo_processor_jobs): job_id = 'test_id_12345' @@ -49,20 +53,22 @@ def test_db_processing_job_update(mongo_processor_jobs): } ) assert db_created_processing_job - db_found_processing_job = sync_db_get_processing_job(job_id=job_id) assert db_found_processing_job - db_updated_processing_job = sync_db_update_processing_job(job_id=job_id, state=StateEnum.running) assert db_found_processing_job != db_updated_processing_job - db_found_updated_processing_job = sync_db_get_processing_job(job_id=job_id) assert db_found_updated_processing_job assert db_found_updated_processing_job == db_updated_processing_job assert db_found_updated_processing_job.state == StateEnum.running + with raises(ValueError) as value_error: + sync_db_update_processing_job(job_id='non-existing', state=StateEnum.running) + sync_db_update_processing_job(job_id=job_id, non_existing_field='dummy_value') + sync_db_update_processing_job(job_id=job_id, processor_name='non-updatable-field') -def test_db_workspace_create(mongo_workspaces): + +def test_db_workspace_create(): mets_path = assets.path_to('kant_aufklaerung_1784/data/mets.xml') db_created_workspace = sync_db_create_workspace(mets_path=mets_path) assert db_created_workspace @@ -71,8 +77,15 @@ def test_db_workspace_create(mongo_workspaces): assert db_found_workspace assert db_found_workspace == db_created_workspace + with raises(ValueError) as value_error: + sync_db_get_workspace(workspace_id='non-existing-id') + sync_db_get_workspace(workspace_mets_path='non-existing-mets') + + with raises(FileNotFoundError) as io_error: + sync_db_create_workspace(mets_path='non-existing-mets') + -def test_db_workspace_update(mongo_workspaces): +def test_db_workspace_update(): mets_path = assets.path_to('kant_aufklaerung_1784-binarized/data/mets.xml') dummy_mets_server_url = '/tmp/dummy.sock' From 720e0ccaef18893a7af69e20c1ae1a17859b1107 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 13:27:58 +0100 Subject: [PATCH 40/93] add db create for jobs --- ocrd_network/ocrd_network/database.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 73e2699f23..72324a132d 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -139,6 +139,15 @@ async def sync_db_update_workspace(workspace_id: str = None, workspace_mets_path return await db_update_workspace(workspace_id=workspace_id, workspace_mets_path=workspace_mets_path, **kwargs) +async def db_create_processing_job(db_processing_job: DBProcessorJob) -> DBProcessorJob: + return await db_processing_job.insert() + + +@call_sync +async def sync_db_create_processing_job(db_processing_job: DBProcessorJob) -> DBProcessorJob: + return await db_create_processing_job(db_processing_job=db_processing_job) + + async def db_get_processing_job(job_id: str) -> DBProcessorJob: job = await DBProcessorJob.find_one( DBProcessorJob.job_id == job_id) From 0bf4bc9bc48cc85f8831ed4fa64530893b839176 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 13:28:19 +0100 Subject: [PATCH 41/93] simplify fixtures --- tests/network/fixtures_mongodb.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 647492986b..d146f4f0f9 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,11 +1,8 @@ -from pymongo import MongoClient, uri_parser as mongo_uri_parser +from pymongo import uri_parser as mongo_uri_parser from pytest import fixture from ocrd_utils.config import config from ocrd_network.database import sync_initiate_database -DB_NAME = config.DB_NAME -DB_URL = config.DB_URL - def verify_database_uri(mongodb_address: str) -> str: try: @@ -18,15 +15,4 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_client") def fixture_mongo_client(): - # This extra initialization is needed to test the db wrapper methods - sync_initiate_database(DB_URL, DB_NAME) - mongo_client = MongoClient(DB_URL, serverSelectionTimeoutMS=3000) - yield mongo_client - - -@fixture(scope="package", name="mongo_processor_jobs") -def fixture_mongo_processor_jobs(mongo_client): - mydb = mongo_client[DB_NAME] - processor_jobs_collection = mydb["DBProcessorJob"] - yield processor_jobs_collection - processor_jobs_collection.drop() + sync_initiate_database(config.DB_URL, config.DB_NAME) From afcbd3fadaf5e814aa60dd10877c73584c742fe1 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 13:28:32 +0100 Subject: [PATCH 42/93] simplify db tests --- tests/network/test_db.py | 51 +++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 6ab7d9816b..d138ad1c1b 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,7 +1,8 @@ from pytest import raises from tests.base import assets -from ocrd_network.models import StateEnum +from ocrd_network.models import DBProcessorJob, StateEnum from ocrd_network.database import ( + sync_db_create_processing_job, sync_db_get_processing_job, sync_db_update_processing_job, sync_db_create_workspace, @@ -10,19 +11,17 @@ ) -def test_db_processing_job_create(mongo_processor_jobs): +def test_db_processing_job_create(mongo_client): job_id = 'test_id_1234' - # TODO: There is no db wrapper to create processing job - # Hence, for now, use a low level method to insert a job - db_created_processing_job = mongo_processor_jobs.insert_one( - document={ - "job_id": job_id, - "processor_name": 'ocrd-dummy', - "state": StateEnum.cached, - "path_to_mets": '/ocrd/dummy/path', - "input_file_grps": ['DEFAULT'], - "output_file_grps": ['OCR-D-DUMMY'] - } + db_created_processing_job = sync_db_create_processing_job( + db_processing_job=DBProcessorJob( + job_id=job_id, + processor_name='ocrd-dummy', + state=StateEnum.cached, + path_to_mets='/ocrd/dummy/path', + input_file_grps=['DEFAULT'], + output_file_grps=['OCR-D-DUMMY'] + ) ) assert db_created_processing_job db_found_processing_job = sync_db_get_processing_job(job_id=job_id) @@ -38,19 +37,17 @@ def test_db_processing_job_create(mongo_processor_jobs): sync_db_get_processing_job(job_id='non-existing-id') -def test_db_processing_job_update(mongo_processor_jobs): +def test_db_processing_job_update(mongo_client): job_id = 'test_id_12345' - # TODO: There is no db wrapper to create processing job - # Hence, for now, use a low level method to insert a job - db_created_processing_job = mongo_processor_jobs.insert_one( - document={ - "job_id": job_id, - "processor_name": 'ocrd-dummy', - "state": StateEnum.cached, - "path_to_mets": '/ocrd/dummy/path', - "input_file_grps": ['DEFAULT'], - "output_file_grps": ['OCR-D-DUMMY'] - } + db_created_processing_job = sync_db_create_processing_job( + db_processing_job=DBProcessorJob( + job_id=job_id, + processor_name='ocrd-dummy', + state=StateEnum.cached, + path_to_mets='/ocrd/dummy/path', + input_file_grps=['DEFAULT'], + output_file_grps=['OCR-D-DUMMY'] + ) ) assert db_created_processing_job db_found_processing_job = sync_db_get_processing_job(job_id=job_id) @@ -68,7 +65,7 @@ def test_db_processing_job_update(mongo_processor_jobs): sync_db_update_processing_job(job_id=job_id, processor_name='non-updatable-field') -def test_db_workspace_create(): +def test_db_workspace_create(mongo_client): mets_path = assets.path_to('kant_aufklaerung_1784/data/mets.xml') db_created_workspace = sync_db_create_workspace(mets_path=mets_path) assert db_created_workspace @@ -85,7 +82,7 @@ def test_db_workspace_create(): sync_db_create_workspace(mets_path='non-existing-mets') -def test_db_workspace_update(): +def test_db_workspace_update(mongo_client): mets_path = assets.path_to('kant_aufklaerung_1784-binarized/data/mets.xml') dummy_mets_server_url = '/tmp/dummy.sock' From 1115e3d5782e0f8a3dd1dfa1b1b34c5206bec820 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 16 Jan 2024 15:12:36 +0100 Subject: [PATCH 43/93] extend db tests --- ocrd_network/ocrd_network/database.py | 19 ++++++++ tests/network/dummy-workflow.txt | 1 + tests/network/fixtures_mongodb.py | 1 + tests/network/test_db.py | 67 +++++++++++++++++++++++++-- 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 tests/network/dummy-workflow.txt diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 72324a132d..946ed95a5b 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -194,6 +194,15 @@ async def sync_db_update_processing_job(job_id: str, **kwargs) -> DBProcessorJob return await db_update_processing_job(job_id=job_id, **kwargs) +async def db_create_workflow_job(db_workflow_job: DBWorkflowJob) -> DBWorkflowJob: + return await db_workflow_job.insert() + + +@call_sync +async def sync_db_create_workflow_job(db_workflow_job: DBWorkflowJob) -> DBWorkflowJob: + return await db_create_workflow_job(db_workflow_job=db_workflow_job) + + async def db_get_workflow_job(job_id: str) -> DBWorkflowJob: job = await DBWorkflowJob.find_one(DBWorkflowJob.job_id == job_id) if not job: @@ -216,6 +225,15 @@ async def sync_db_get_processing_jobs(job_ids: List[str]) -> [DBProcessorJob]: return await db_get_processing_jobs(job_ids) +async def db_create_workflow_script(db_workflow_script: DBWorkflowScript) -> DBWorkflowScript: + return await db_workflow_script.insert() + + +@call_sync +async def sync_db_create_workflow_script(db_workflow_script: DBWorkflowScript) -> DBWorkflowScript: + return await db_create_workflow_script(db_workflow_script=db_workflow_script) + + async def db_get_workflow_script(workflow_id: str) -> DBWorkflowScript: workflow = await DBWorkflowScript.find_one(DBWorkflowScript.workflow_id == workflow_id) if not workflow: @@ -235,6 +253,7 @@ async def db_find_first_workflow_script_by_content(content_hash: str) -> DBWorkf return workflow +# TODO: Resolve the inconsistency between the async and sync versions of the same method @call_sync async def sync_db_find_first_workflow_script_by_content(workflow_id: str) -> DBWorkflowScript: return await db_get_workflow_script(workflow_id) diff --git a/tests/network/dummy-workflow.txt b/tests/network/dummy-workflow.txt new file mode 100644 index 0000000000..223c217678 --- /dev/null +++ b/tests/network/dummy-workflow.txt @@ -0,0 +1 @@ +dummy -I DEFAULT -O OCR-D-DUMMY \ No newline at end of file diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index d146f4f0f9..0365560dd2 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -15,4 +15,5 @@ def verify_database_uri(mongodb_address: str) -> str: @fixture(scope="package", name="mongo_client") def fixture_mongo_client(): + verify_database_uri(config.DB_URL) sync_initiate_database(config.DB_URL, config.DB_NAME) diff --git a/tests/network/test_db.py b/tests/network/test_db.py index d138ad1c1b..bc044812a2 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,18 +1,23 @@ +from hashlib import md5 +from pathlib import Path from pytest import raises from tests.base import assets -from ocrd_network.models import DBProcessorJob, StateEnum +from ocrd_network.models import DBProcessorJob, DBWorkflowScript, StateEnum from ocrd_network.database import ( sync_db_create_processing_job, sync_db_get_processing_job, sync_db_update_processing_job, sync_db_create_workspace, sync_db_get_workspace, - sync_db_update_workspace + sync_db_update_workspace, + sync_db_create_workflow_script, + sync_db_get_workflow_script, + sync_db_find_first_workflow_script_by_content ) def test_db_processing_job_create(mongo_client): - job_id = 'test_id_1234' + job_id = 'test_job_id_1' db_created_processing_job = sync_db_create_processing_job( db_processing_job=DBProcessorJob( job_id=job_id, @@ -38,7 +43,7 @@ def test_db_processing_job_create(mongo_client): def test_db_processing_job_update(mongo_client): - job_id = 'test_id_12345' + job_id = 'test_job_id_2' db_created_processing_job = sync_db_create_processing_job( db_processing_job=DBProcessorJob( job_id=job_id, @@ -104,3 +109,57 @@ def test_db_workspace_update(mongo_client): assert db_found_updated_workspace.workspace_mets_path == mets_path assert db_found_updated_workspace.mets_server_url == dummy_mets_server_url assert db_found_updated_workspace == db_updated_workspace + + +# TODO: There is no db wrapper implemented due to direct access in the processing server... +# TODO2: Should be refactored with proper asset access +def create_db_model_workflow_script( + workflow_id: str, + script_path: Path = Path(Path(__file__).parent, "dummy-workflow.txt") +) -> DBWorkflowScript: + workflow_id = workflow_id + with open(script_path, 'rb') as fp: + content = (fp.read()).decode("utf-8") + content_hash = md5(content.encode("utf-8")).hexdigest() + return DBWorkflowScript(workflow_id=workflow_id, content=content, content_hash=content_hash) + + +def test_db_workflow_script_create(mongo_client): + workflow_id = 'test_workflow_1' + db_model_workflow_script = create_db_model_workflow_script(workflow_id=workflow_id) + db_created_workflow_script = sync_db_create_workflow_script( + db_workflow_script=db_model_workflow_script + ) + assert db_created_workflow_script + db_found_workflow_script = sync_db_get_workflow_script(workflow_id=workflow_id) + assert db_found_workflow_script + assert db_found_workflow_script == db_created_workflow_script + + with raises(ValueError) as value_error: + sync_db_get_workflow_script(workflow_id='non-existing-id') + + +def test_db_find_workflow_script_by_content(mongo_client): + workflow_id = 'test_workflow_2' + db_model_workflow_script = create_db_model_workflow_script(workflow_id=workflow_id) + db_created_workflow_script = sync_db_create_workflow_script( + db_workflow_script=db_model_workflow_script + ) + assert db_created_workflow_script + db_found_workflow_script = sync_db_find_first_workflow_script_by_content( + workflow_id=db_model_workflow_script.workflow_id + ) + assert db_found_workflow_script + assert db_found_workflow_script == db_created_workflow_script + + +# TODO: hard to implement without some refactoring in the ocrd_network +# and providing proper db wrappers +def test_db_workflow_job_create(): + pass + + +# TODO: hard to implement without some refactoring in the ocrd_network +# and providing proper db wrappers +def test_db_workflow_job_update(): + pass From 2d4809ac0155b8fdc3aaff9b84fc522f65b04d8d Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 18 Jan 2024 14:22:00 +0100 Subject: [PATCH 44/93] add processing server test --- Makefile | 2 +- .../ocrd_network/processing_server.py | 17 +++++++ tests/network/docker-compose.yml | 47 ++++++++++++++++++- tests/network/ps_config.yml | 28 +++++++++++ tests/network/test_processing_server.py | 18 +++++++ 5 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 tests/network/ps_config.yml create mode 100644 tests/network/test_processing_server.py diff --git a/Makefile b/Makefile index 81b97ae800..14427a5a65 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d sleep 20 - $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db' + $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' docker compose -f tests/network/docker-compose.yml down --remove-orphans benchmark: diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index e6606f90f0..77c2dd9025 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -1,3 +1,4 @@ +from datetime import datetime from hashlib import md5 import httpx import json @@ -127,6 +128,14 @@ def __init__(self, config_path: str, host: str, port: int) -> None: else: self.internal_job_callback_url = f'http://{host}:{port}/result_callback' + self.router.add_api_route( + path='/', + endpoint=self.home_page, + methods=['GET'], + status_code=status.HTTP_200_OK, + summary='Get information about the processing server' + ) + # Create routes self.router.add_api_route( path='/stop', @@ -288,6 +297,14 @@ async def on_shutdown(self) -> None: """ await self.stop_deployed_agents() + async def home_page(self): + message = f"The home page of the {self.title}" + json_message = { + "message": message, + "time": datetime.now().strftime("%Y-%m-%d %H:%M") + } + return json_message + async def stop_deployed_agents(self) -> None: self.deployer.kill_all() diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 5c20d29b94..a17f306e38 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -10,6 +10,7 @@ services: ocrd_network_mongo_db: image: "mongo" hostname: mongodb-docker-host + container_name: ocrd_network_mongo_db networks: - ocrd_network_test ports: @@ -17,10 +18,16 @@ services: environment: - MONGO_INITDB_ROOT_USERNAME=network_test - MONGO_INITDB_ROOT_PASSWORD=network_test + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet + interval: 1s + timeout: 3s + retries: 30 ocrd_network_rabbit_mq: image: "rabbitmq:3.12-management" hostname: rabbitmq-docker-host + container_name: ocrd_network_rabbit_mq networks: - ocrd_network_test ports: @@ -31,6 +38,39 @@ services: - RABBITMQ_DEFAULT_USER=network_test - RABBITMQ_DEFAULT_PASS=network_test - RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version + healthcheck: + test: rabbitmq-diagnostics check_port_connectivity + interval: 1s + timeout: 3s + retries: 30 + + ocrd_network_processing_server: + build: + context: ../../ + dockerfile: Dockerfile + args: + BASE_IMAGE: 'ubuntu:20.04' + hostname: processing-server-host + container_name: ocrd_network_processing_server + depends_on: + ocrd_network_mongo_db: + condition: service_healthy + ocrd_network_rabbit_mq: + condition: service_healthy + networks: + - ocrd_network_test + ports: + - "8000:8000" + environment: + DB_NAME: ocrd_network_test + DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 + RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + volumes: + - "./ps_config.yml:/ps_config.yml" + - "/home/mm/.ssh/cloud.key:/ssh_key" + - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" + - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" + command: ocrd network processing-server /ps_config.yml --address 0.0.0.0:8000 ocrd_network_core_test: build: @@ -40,10 +80,15 @@ services: BASE_IMAGE: 'ubuntu:20.04' target: ocrd_core_test container_name: core_test + depends_on: + ocrd_network_mongo_db: + condition: service_healthy + ocrd_network_rabbit_mq: + condition: service_healthy networks: - ocrd_network_test environment: DB_NAME: ocrd_network_test DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 - # PROCESSING_SERVER_URL : http://... + PROCESSING_SERVER_URL: http://processing-server-host.ocrd_network_test:8000 RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 diff --git a/tests/network/ps_config.yml b/tests/network/ps_config.yml new file mode 100644 index 0000000000..63c21d0175 --- /dev/null +++ b/tests/network/ps_config.yml @@ -0,0 +1,28 @@ +process_queue: + address: rabbitmq-docker-host.ocrd_network_test + port: 5672 + credentials: + username: network_test + password: network_test + ssh: + username: root + path_to_privkey: /ssh_key + skip_deployment: true +database: + address: mongodb-docker-host.ocrd_network_test + port: 27017 + credentials: + username: network_test + password: network_test + ssh: + username: root + path_to_privkey: /ssh_key + skip_deployment: true +hosts: + - address: processing-server-host.ocrd_network_test + username: root + path_to_privkey: /ssh_key + workers: + - name: ocrd-dummy + number_of_instance: 1 + deploy_type: native diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py new file mode 100644 index 0000000000..f758012aa1 --- /dev/null +++ b/tests/network/test_processing_server.py @@ -0,0 +1,18 @@ +from requests import get +from ocrd_utils.config import config + +PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL + + +def test_processing_server_connectivity(): + test_url = f'{PROCESSING_SERVER_URL}/' + response = get(test_url) + assert response.status_code == 200, \ + f'Processing server is not reachable on: {test_url}, {response.status_code}' + + +def _test_processing_server_deployed_processors(): + test_url = f'{PROCESSING_SERVER_URL}/processor' + response = get(test_url) + assert response.status_code == 200, \ + f'Processing server is not reachable on: {test_url}, {response.status_code}' From c063a67f138424d3625547116b0f2027370bd805 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 19 Jan 2024 13:13:04 +0100 Subject: [PATCH 45/93] fix processing server tests --- tests/network/docker-compose.yml | 42 +++++++++++++++++++++++-- tests/network/test_processing_server.py | 9 +++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index a17f306e38..10c95594ba 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -50,6 +50,7 @@ services: dockerfile: Dockerfile args: BASE_IMAGE: 'ubuntu:20.04' + target: ocrd_core_test hostname: processing-server-host container_name: ocrd_network_processing_server depends_on: @@ -66,11 +67,46 @@ services: DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 volumes: - - "./ps_config.yml:/ps_config.yml" - - "/home/mm/.ssh/cloud.key:/ssh_key" - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - command: ocrd network processing-server /ps_config.yml --address 0.0.0.0:8000 + command: | + /bin/bash -c "echo -e \" + internal_callback_url: http://processing-server-host.ocrd_network_test:8000 + process_queue: + address: rabbitmq-docker-host.ocrd_network_test + port: 5672 + skip_deployment: true + credentials: + username: network_test + password: network_test + database: + address: mongodb-docker-host.ocrd_network_test + port: 27017 + skip_deployment: true + credentials: + username: network_test + password: network_test + hosts: []\" > ./ps_config.yaml && \ + ocrd network processing-server -a 0.0.0.0:8000 ./ps_config.yaml" + + ocrd_dummy_processing_worker: + build: + context: ../../ + dockerfile: Dockerfile + args: + BASE_IMAGE: 'ubuntu:20.04' + target: ocrd_core_test + depends_on: + ocrd_network_mongo_db: + condition: service_healthy + ocrd_network_rabbit_mq: + condition: service_healthy + networks: + - ocrd_network_test + volumes: + - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" + - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" + command: ocrd-dummy worker --database mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 --queue amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 ocrd_network_core_test: build: diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index f758012aa1..12057dadb9 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -9,10 +9,17 @@ def test_processing_server_connectivity(): response = get(test_url) assert response.status_code == 200, \ f'Processing server is not reachable on: {test_url}, {response.status_code}' + message = response.json()['message'] + assert message.startsWith('The home page of'), \ + f'Processing server home page message is corrupted' -def _test_processing_server_deployed_processors(): +# TODO: The processing workers are still not registered when deployed separately. +# Fix that by extending the processing server. +def test_processing_server_deployed_processors(): test_url = f'{PROCESSING_SERVER_URL}/processor' response = get(test_url) + processors = response.json() assert response.status_code == 200, \ f'Processing server is not reachable on: {test_url}, {response.status_code}' + assert processors == [], f'Mismatch in deployed processors' From f0654bf212e151d4023534f0dc67a8fc9ef3a236 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 19 Jan 2024 13:24:49 +0100 Subject: [PATCH 46/93] fix test typo --- tests/network/test_processing_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 12057dadb9..d1c58b87d2 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -10,7 +10,7 @@ def test_processing_server_connectivity(): assert response.status_code == 200, \ f'Processing server is not reachable on: {test_url}, {response.status_code}' message = response.json()['message'] - assert message.startsWith('The home page of'), \ + assert message.startswith('The home page of'), \ f'Processing server home page message is corrupted' From 14a163b494075cdca654081e3291a2ff8c30ff97 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 19 Jan 2024 17:19:50 +0100 Subject: [PATCH 47/93] add processing request test - failing --- .../ocrd_network/processing_server.py | 12 ++ tests/network/docker-compose.yml | 42 ++--- tests/network/dummy-workflow.txt | 4 +- tests/network/ocrd_logging.conf | 150 ++++++++++++++++++ tests/network/ps_config.yml | 28 ---- tests/network/test_processing_server.py | 19 ++- 6 files changed, 206 insertions(+), 49 deletions(-) create mode 100644 tests/network/ocrd_logging.conf delete mode 100644 tests/network/ps_config.yml diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 77c2dd9025..884ae8783d 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -339,6 +339,12 @@ def create_message_queues(self) -> None: unique_only=True ) + # TODO: Reconsider and refactor this. + # Added ocrd-dummy by default if not available for the integration tests. + # A proper Processing Worker / Processor Server registration endpoint is needed on the Processing Server side + if 'dummy' not in queue_names: + queue_names.append('dummy') + for queue_name in queue_names: # The existence/validity of the worker.name is not tested. # Even if an ocr-d processor does not exist, the queue is created @@ -395,6 +401,12 @@ def processing_agent_exists(self, processor_name: str, agent_type: str) -> bool: if agent_type not in [NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER]: return False if agent_type == NETWORK_AGENT_WORKER: + # TODO: Reconsider and refactor this. + # Added ocrd-dummy by default if not available for the integration tests. + # A proper Processing Worker / Processor Server registration endpoint + # is needed on the Processing Server side + if processor_name == 'dummy': + return True if not self.check_if_queue_exists(processor_name): return False if agent_type == NETWORK_AGENT_SERVER: diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 10c95594ba..02a984bce1 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -45,6 +45,7 @@ services: retries: 30 ocrd_network_processing_server: + image: "ocrd_core_test" build: context: ../../ dockerfile: Dockerfile @@ -66,9 +67,16 @@ services: DB_NAME: ocrd_network_test DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + healthcheck: + test: curl -f http://processing-server-host.ocrd_network_test:8000/ + interval: 1s + timeout: 3s + retries: 30 volumes: - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" + - "../assets:/tmp/assets" + - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: | /bin/bash -c "echo -e \" internal_callback_url: http://processing-server-host.ocrd_network_test:8000 @@ -90,39 +98,35 @@ services: ocrd network processing-server -a 0.0.0.0:8000 ./ps_config.yaml" ocrd_dummy_processing_worker: - build: - context: ../../ - dockerfile: Dockerfile - args: - BASE_IMAGE: 'ubuntu:20.04' - target: ocrd_core_test + image: "ocrd_core_test" depends_on: - ocrd_network_mongo_db: - condition: service_healthy - ocrd_network_rabbit_mq: + ocrd_network_processing_server: condition: service_healthy networks: - ocrd_network_test volumes: - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - command: ocrd-dummy worker --database mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 --queue amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + - "../assets:/tmp/assets" + - "./ocrd_logging.conf:/etc/ocrd_logging.conf" + command: > + ocrd-dummy worker + --database mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 + --queue amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 ocrd_network_core_test: - build: - context: ../../ - dockerfile: Dockerfile - args: - BASE_IMAGE: 'ubuntu:20.04' - target: ocrd_core_test + image: "ocrd_core_test" container_name: core_test depends_on: - ocrd_network_mongo_db: - condition: service_healthy - ocrd_network_rabbit_mq: + ocrd_network_processing_server: condition: service_healthy networks: - ocrd_network_test + volumes: + - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" + - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" + - "../assets:/tmp/assets" + - "./ocrd_logging.conf:/etc/ocrd_logging.conf" environment: DB_NAME: ocrd_network_test DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 diff --git a/tests/network/dummy-workflow.txt b/tests/network/dummy-workflow.txt index 223c217678..2de01dc67b 100644 --- a/tests/network/dummy-workflow.txt +++ b/tests/network/dummy-workflow.txt @@ -1 +1,3 @@ -dummy -I DEFAULT -O OCR-D-DUMMY \ No newline at end of file +dummy -I DEFAULT -O OCR-D-DUMMY1 +dummy -I OCR-D-DUMMY1 -O OCR-D-DUMMY2 +dummy -I OCR-D-DUMMY2 -O OCR-D-DUMMY3 diff --git a/tests/network/ocrd_logging.conf b/tests/network/ocrd_logging.conf new file mode 100644 index 0000000000..5ee5e015d8 --- /dev/null +++ b/tests/network/ocrd_logging.conf @@ -0,0 +1,150 @@ +# This is a template configuration file which allows customizing +# format and destination of log messages with OCR-D. +# It is meant as an example, and should be customized. +# To get into effect, you must put a copy (under the same name) +# into your CWD, HOME or /etc. These directories are searched +# in said order, and the first find wins. When no config file +# is found, the default logging configuration applies (cf. ocrd.logging.py). +# +# mandatory loggers section +# configure loggers with corresponding keys "root", "" +# each logger requires a corresponding configuration section below +# +[loggers] +keys=root,ocrd,ocrd_network,ocrd_tensorflow,ocrd_shapely_geos,ocrd_PIL,uvicorn,uvicorn_access,uvicorn_error,multipart + +# +# mandatory handlers section +# handle output for each logging "channel" +# i.e. console, file, smtp, syslog, http, ... +# each handler requires a corresponding configuration section below +# +[handlers] +keys=consoleHandler,fileHandler,processingServerHandler + +# +# optional custom formatters section +# format message fields, to be used differently by logging handlers +# each formatter requires a corresponding formatter section below +# +[formatters] +keys=defaultFormatter,detailedFormatter + +# +# default logger "root" using consoleHandler +# +[logger_root] +level=DEBUG +handlers=consoleHandler,fileHandler + + +# +# additional logger configurations can be added +# as separate configuration sections like below +# +# example logger "ocrd_workspace" uses fileHandler and overrides +# default log level "INFO" with custom level "DEBUG" +# "qualname" must match the logger label used in the corresponding +# ocrd module +# see in the module-of-interest (moi) +# +#[logger_ocrd_workspace] +#level=DEBUG +#handlers=fileHandler +#qualname=ocrd.workspace + +# ocrd loggers +[logger_ocrd] +level=DEBUG +handlers=consoleHandler,fileHandler +qualname=ocrd +propagate=0 + +[logger_ocrd_network] +level=DEBUG +handlers=consoleHandler,processingServerHandler +qualname=ocrd_network +propagate=0 + +# +# logger tensorflow +# +[logger_ocrd_tensorflow] +level=DEBUG +handlers=consoleHandler +qualname=tensorflow + +# +# logger shapely.geos +# +[logger_ocrd_shapely_geos] +level=DEBUG +handlers=consoleHandler +qualname=shapely.geos + + +# +# logger PIL +# +[logger_ocrd_PIL] +level=DEBUG +handlers=consoleHandler +qualname=PIL + +# +# uvicorn loggers +# +[logger_uvicorn] +level=INFO +handlers=consoleHandler +qualname=uvicorn +[logger_uvicorn_access] +level=DEBUG +handlers=consoleHandler +qualname=uvicorn.access +[logger_uvicorn_error] +level=DEBUG +handlers=consoleHandler +qualname=uvicorn.error +[logger_multipart] +level=DEBUG +handlers=consoleHandler +qualname=multipart + + + +# +# handle stderr output +# +[handler_consoleHandler] +class=StreamHandler +formatter=defaultFormatter +args=(sys.stderr,) + +# +# example logfile handler +# handle output with logfile +# +[handler_fileHandler] +class=FileHandler +formatter=defaultFormatter +args=('ocrd.log','a+') + +[handler_processingServerHandler] +class=FileHandler +formatter=defaultFormatter +args=('/tmp/ocrd_processing_server.log','a+') + +# +# default log format conforming to OCR-D (https://ocr-d.de/en/spec/cli#logging) +# +[formatter_defaultFormatter] +format=%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s +datefmt=%H:%M:%S + +# +# store more logging context information +# +[formatter_detailedFormatter] +format=%(asctime)s.%(msecs)03d %(levelname)-8s (%(name)s)[%(filename)s:%(lineno)d] - %(message)s +datefmt=%H:%M:%S diff --git a/tests/network/ps_config.yml b/tests/network/ps_config.yml deleted file mode 100644 index 63c21d0175..0000000000 --- a/tests/network/ps_config.yml +++ /dev/null @@ -1,28 +0,0 @@ -process_queue: - address: rabbitmq-docker-host.ocrd_network_test - port: 5672 - credentials: - username: network_test - password: network_test - ssh: - username: root - path_to_privkey: /ssh_key - skip_deployment: true -database: - address: mongodb-docker-host.ocrd_network_test - port: 27017 - credentials: - username: network_test - password: network_test - ssh: - username: root - path_to_privkey: /ssh_key - skip_deployment: true -hosts: - - address: processing-server-host.ocrd_network_test - username: root - path_to_privkey: /ssh_key - workers: - - name: ocrd-dummy - number_of_instance: 1 - deploy_type: native diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index d1c58b87d2..33cbbe6a69 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -1,5 +1,6 @@ -from requests import get +from requests import get, post from ocrd_utils.config import config +from ocrd_network import NETWORK_AGENT_WORKER PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL @@ -23,3 +24,19 @@ def test_processing_server_deployed_processors(): assert response.status_code == 200, \ f'Processing server is not reachable on: {test_url}, {response.status_code}' assert processors == [], f'Mismatch in deployed processors' + + +def test_processing_server_processing_request(): + # Note: the path_to_mets is volume mapped + test_processing_job_input = { + "path_to_mets": "/tmp/assets/kant_aufklaerung_1784/data/mets.xml", + "input_file_grps": ['OCR-D-IMG'], + "output_file_grps": ['OCR-D-DUMMY'], + "agent_type": NETWORK_AGENT_WORKER, + "parameters": {} + } + test_processor = 'ocrd-dummy' + test_url = f'{PROCESSING_SERVER_URL}/processor/run/{test_processor}' + response = post(url=test_url, data=test_processing_job_input) + assert response.status_code == 200, \ + f'Processing server is not reachable on: {test_url}, {response.status_code}' From cdb71eea7e61b35b8dc7ccde6138233b7e142346 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 19 Jan 2024 17:33:47 +0100 Subject: [PATCH 48/93] fix ocrd-dummy queue and processor name --- ocrd_network/ocrd_network/processing_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 884ae8783d..f60201311e 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -342,8 +342,8 @@ def create_message_queues(self) -> None: # TODO: Reconsider and refactor this. # Added ocrd-dummy by default if not available for the integration tests. # A proper Processing Worker / Processor Server registration endpoint is needed on the Processing Server side - if 'dummy' not in queue_names: - queue_names.append('dummy') + if 'ocrd-dummy' not in queue_names: + queue_names.append('ocrd-dummy') for queue_name in queue_names: # The existence/validity of the worker.name is not tested. @@ -405,7 +405,7 @@ def processing_agent_exists(self, processor_name: str, agent_type: str) -> bool: # Added ocrd-dummy by default if not available for the integration tests. # A proper Processing Worker / Processor Server registration endpoint # is needed on the Processing Server side - if processor_name == 'dummy': + if processor_name == 'ocrd-dummy': return True if not self.check_if_queue_exists(processor_name): return False From e7261f4cd0cbacadf24c521d37332b716bd87a55 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 12:12:14 +0100 Subject: [PATCH 49/93] rm integ tests from CCI --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a8c1ab1e60..086810af4b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark integration-test + - run: make test benchmark # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -47,7 +47,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark integration-test + - run: make test benchmark # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -62,7 +62,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark integration-test + - run: make test benchmark # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -77,7 +77,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark integration-test + - run: make test benchmark # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version @@ -92,7 +92,7 @@ jobs: - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test - - run: make test benchmark integration-test + - run: make test benchmark # smoke test to ensure that --editable install works - run: make install-dev; ocrd --version From 9640edb86c4e01a1c82af579f316b984e52bec6d Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 12:18:34 +0100 Subject: [PATCH 50/93] add integ test yml workflow --- .github/workflows/integration-test.yml | 60 ++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/integration-test.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000000..b58f8322dc --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,60 @@ +name: Test core installation and run ocrd network integration tests + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11' + os: + - ubuntu-22.04 + - ubuntu-20.04 + - macos-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + if [[ "${{ matrix.os }}" == "ubuntu"* ]];then + sudo apt-get -y update + sudo make deps-ubuntu + else + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \ + HOMEBREW_NO_AUTO_UPDATE=1 \ + brew install imagemagick geos bash # opencv + fi + make install deps-test + - name: Test with pytest + run: | + make test integration-test + - name: test to ensure that --editable install works + run: | + make install-dev; ocrd --version + - name: Lint with flake8 + run: | + python -m pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 src --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics From 6ba0b4ca3c84f1b3f9fe410e93e66a2421fff416 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 12:20:02 +0100 Subject: [PATCH 51/93] remove editable install check --- .github/workflows/integration-test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b58f8322dc..fed553dd74 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -48,9 +48,6 @@ jobs: - name: Test with pytest run: | make test integration-test - - name: test to ensure that --editable install works - run: | - make install-dev; ocrd --version - name: Lint with flake8 run: | python -m pip install flake8 From 4bb3a54f58a9b797d24889b48d367fbfa5749ae9 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 12:29:17 +0100 Subject: [PATCH 52/93] refine integration test yml --- .github/workflows/integration-test.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index fed553dd74..adba4879f2 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -22,7 +22,6 @@ jobs: - '3.11' os: - ubuntu-22.04 - - ubuntu-20.04 - macos-latest steps: @@ -45,13 +44,6 @@ jobs: brew install imagemagick geos bash # opencv fi make install deps-test - - name: Test with pytest + - name: Test network integration with pytest run: | make test integration-test - - name: Lint with flake8 - run: | - python -m pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 src --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics From 4bbd9dc25ac1aadb91d7d1706c1bb0e465742880 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 12:34:30 +0100 Subject: [PATCH 53/93] run only integration-test --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index adba4879f2..bba46dfcdd 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -46,4 +46,4 @@ jobs: make install deps-test - name: Test network integration with pytest run: | - make test integration-test + make integration-test From dd5bacd57d7e5540422668efc088f22f474c93f0 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 13:24:47 +0100 Subject: [PATCH 54/93] revert unnecessary changes --- .circleci/config.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 086810af4b..858d03901a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,8 +27,6 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -42,8 +40,6 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -57,8 +53,6 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -72,8 +66,6 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -87,8 +79,6 @@ jobs: working_directory: ~/ocrd-core steps: - checkout - - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: false - run: sudo apt-get -y update - run: sudo make deps-ubuntu - run: make install deps-test @@ -102,7 +92,7 @@ jobs: steps: - checkout - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ - docker_layer_caching: false + docker_layer_caching: true - run: make docker - run: make docker-cuda - run: From 721f0af6919fc0b470d42fe6c2612e8eef32ea84 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 13:50:58 +0100 Subject: [PATCH 55/93] shorten the GAct wf name --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index bba46dfcdd..1915a4f3d9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1,4 +1,4 @@ -name: Test core installation and run ocrd network integration tests +name: Run ocrd network integration tests on: push: From c69b334dba3f1ff8e5a0f639107fc3a1f81be784 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Wed, 24 Jan 2024 16:12:05 +0100 Subject: [PATCH 56/93] add PS run workflow test --- tests/network/docker-compose.yml | 11 +++++---- tests/network/test_processing_server.py | 31 +++++++++++++++++++++---- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 02a984bce1..a732026221 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -50,7 +50,7 @@ services: context: ../../ dockerfile: Dockerfile args: - BASE_IMAGE: 'ubuntu:20.04' + BASE_IMAGE: 'ubuntu:22.04' target: ocrd_core_test hostname: processing-server-host container_name: ocrd_network_processing_server @@ -75,7 +75,8 @@ services: volumes: - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - - "../assets:/tmp/assets" + - "../assets/kant_aufklaerung_1784:/tmp/assets/kant_aufklaerung_1784" + - "./dummy-workflow.txt:/tmp/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: | /bin/bash -c "echo -e \" @@ -107,7 +108,8 @@ services: volumes: - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - - "../assets:/tmp/assets" + - "../assets/kant_aufklaerung_1784:/tmp/assets/kant_aufklaerung_1784" + - "./dummy-workflow.txt:/tmp/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: > ocrd-dummy worker @@ -125,7 +127,8 @@ services: volumes: - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - - "../assets:/tmp/assets" + - "../assets/kant_aufklaerung_1784:/tmp/assets/kant_aufklaerung_1784" + - "./dummy-workflow.txt:/tmp/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" environment: DB_NAME: ocrd_network_test diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 33cbbe6a69..35e4d9aa45 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -22,12 +22,13 @@ def test_processing_server_deployed_processors(): response = get(test_url) processors = response.json() assert response.status_code == 200, \ - f'Processing server is not reachable on: {test_url}, {response.status_code}' + f'Processing server: {test_url}, {response.status_code}' assert processors == [], f'Mismatch in deployed processors' -def test_processing_server_processing_request(): - # Note: the path_to_mets is volume mapped +# TODO: Still failing test with internal error 500 +def _test_processing_server_processing_request(): + # Note: the used path is volume mapped test_processing_job_input = { "path_to_mets": "/tmp/assets/kant_aufklaerung_1784/data/mets.xml", "input_file_grps": ['OCR-D-IMG'], @@ -37,6 +38,26 @@ def test_processing_server_processing_request(): } test_processor = 'ocrd-dummy' test_url = f'{PROCESSING_SERVER_URL}/processor/run/{test_processor}' - response = post(url=test_url, data=test_processing_job_input) + response = post( + url=test_url, + headers={"accept": "application/json"}, + json=test_processing_job_input + ) assert response.status_code == 200, \ - f'Processing server is not reachable on: {test_url}, {response.status_code}' + f'Processing server: {test_url}, {response.status_code}' + + +def test_processing_server_workflow_request(): + # Note: the used paths are volume mapped + path_to_mets = "/tmp/assets/kant_aufklaerung_1784/data/mets.xml" + path_to_dummy_wf = "/tmp/assets/dummy-workflow.txt" + + test_url = f"{PROCESSING_SERVER_URL}/workflow?mets_path={path_to_mets}&page_wise=False" + response = post( + url=test_url, + files={"workflow": open(path_to_dummy_wf, 'rb')} + ) + assert response.status_code == 201, \ + f'Processing server: {test_url}, {response.status_code}' + + # TODO: Check workflow status here From 6b52c21671c7c0028bea18ea33316ad96cdf1e79 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 12:18:24 +0100 Subject: [PATCH 57/93] include assets in ocrd_core_test --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9178e65c66..e87800bf73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,9 +45,10 @@ CMD ["/usr/local/bin/ocrd", "--help"] FROM ocrd_core_base as ocrd_core_test WORKDIR /build-ocrd +COPY Makefile . +RUN make assets COPY tests ./tests COPY .gitmodules . -COPY Makefile . COPY requirements_test.txt . RUN pip install -r requirements_test.txt From cc2fca71b9575c54560154e4c41b9c391b03ce3b Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 12:32:16 +0100 Subject: [PATCH 58/93] revert and add make test again --- .github/workflows/integration-test.yml | 2 +- Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1915a4f3d9..8b0cf6e89e 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -46,4 +46,4 @@ jobs: make install deps-test - name: Test network integration with pytest run: | - make integration-test + make test integration-test diff --git a/Dockerfile b/Dockerfile index e87800bf73..9178e65c66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,10 +45,9 @@ CMD ["/usr/local/bin/ocrd", "--help"] FROM ocrd_core_base as ocrd_core_test WORKDIR /build-ocrd -COPY Makefile . -RUN make assets COPY tests ./tests COPY .gitmodules . +COPY Makefile . COPY requirements_test.txt . RUN pip install -r requirements_test.txt From da59c5f7eaf84f33e967629f44acf5cffedd5710 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 12:38:27 +0100 Subject: [PATCH 59/93] revert previous commit --- .github/workflows/integration-test.yml | 2 +- Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 8b0cf6e89e..1915a4f3d9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -46,4 +46,4 @@ jobs: make install deps-test - name: Test network integration with pytest run: | - make test integration-test + make integration-test diff --git a/Dockerfile b/Dockerfile index 9178e65c66..e87800bf73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,9 +45,10 @@ CMD ["/usr/local/bin/ocrd", "--help"] FROM ocrd_core_base as ocrd_core_test WORKDIR /build-ocrd +COPY Makefile . +RUN make assets COPY tests ./tests COPY .gitmodules . -COPY Makefile . COPY requirements_test.txt . RUN pip install -r requirements_test.txt From 1fe515555fb41a0de1abd91eb70277c1cc3810d9 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 12:44:52 +0100 Subject: [PATCH 60/93] temp disable mac-os --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1915a4f3d9..6b4791a37b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -22,7 +22,7 @@ jobs: - '3.11' os: - ubuntu-22.04 - - macos-latest + # - macos-latest steps: - uses: actions/checkout@v3 From a57b8ead1558641192fdeb75298db526e415b653 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 13:41:37 +0100 Subject: [PATCH 61/93] add simple wf status check --- src/ocrd_network/processing_server.py | 44 +++++++++++++++++++++++++ tests/network/test_processing_server.py | 26 ++++++++++++--- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/ocrd_network/processing_server.py b/src/ocrd_network/processing_server.py index f60201311e..d4a28e58ec 100644 --- a/src/ocrd_network/processing_server.py +++ b/src/ocrd_network/processing_server.py @@ -228,6 +228,15 @@ def __init__(self, config_path: str, host: str, port: int) -> None: summary='Get information about a workflow run', ) + self.router.add_api_route( + path='/workflow/job-simple/{workflow_job_id}', + endpoint=self.get_workflow_info_simple, + methods=['GET'], + tags=['workflow', 'processing'], + status_code=status.HTTP_200_OK, + summary='Get simplified overall job status', + ) + self.router.add_api_route( path='/workflow', endpoint=self.upload_workflow, @@ -911,6 +920,41 @@ async def get_workflow_info(self, workflow_job_id) -> Dict: }) return res + """ + Simplified version of the `get_workflow_info` that returns a single state for the entire workflow. + - If a single processing job fails, the entire workflow job status is set to FAILED. + - If there are any processing jobs running, regardless of other states, such as QUEUED and CACHED, + the entire workflow job status is set to RUNNING. + - If all processing jobs has finished successfully, only then the workflow job status is set to SUCCESS + """ + async def get_workflow_info_simple(self, workflow_job_id) -> Dict[str, StateEnum]: + """ Return list of a workflow's processor jobs + """ + try: + workflow_job = await db_get_workflow_job(workflow_job_id) + except ValueError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, + detail=f"Workflow-Job with id: {workflow_job_id} not found") + job_ids: List[str] = [job_id for lst in workflow_job.processing_job_ids.values() for job_id in lst] + jobs = await db_get_processing_jobs(job_ids) + + workflow_job_state = "UNSET" + success_jobs = 0 + for job in jobs: + if job.state == StateEnum.cached or job.state == StateEnum.queued: + continue + if job.state == StateEnum.failed or job.state == StateEnum.cancelled: + workflow_job_state = StateEnum.failed + break + if job.state == StateEnum.running: + workflow_job_state = StateEnum.running + if job.state == StateEnum.success: + success_jobs += 1 + # if all jobs succeeded + if len(job_ids) == success_jobs: + workflow_job_state = StateEnum.success + return {"wf_job_state": workflow_job_state} + async def upload_workflow(self, workflow: UploadFile) -> Dict: """ Store a script for a workflow in the database """ diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 35e4d9aa45..6716ddf503 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -1,6 +1,8 @@ +from time import sleep from requests import get, post from ocrd_utils.config import config from ocrd_network import NETWORK_AGENT_WORKER +from ocrd_network.models import StateEnum PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL @@ -52,12 +54,28 @@ def test_processing_server_workflow_request(): path_to_mets = "/tmp/assets/kant_aufklaerung_1784/data/mets.xml" path_to_dummy_wf = "/tmp/assets/dummy-workflow.txt" - test_url = f"{PROCESSING_SERVER_URL}/workflow?mets_path={path_to_mets}&page_wise=False" + # submit the workflow job + test_url = f"{PROCESSING_SERVER_URL}/workflow/run?mets_path={path_to_mets}&page_wise=False" response = post( url=test_url, files={"workflow": open(path_to_dummy_wf, 'rb')} ) - assert response.status_code == 201, \ - f'Processing server: {test_url}, {response.status_code}' + assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" + + wf_job_id = response.json()["job_id"] + assert wf_job_id - # TODO: Check workflow status here + # check simplified workflow status till timeout + tries = 50 + wait_between_tries = 30 + wf_job_state = None + test_url = f"{PROCESSING_SERVER_URL}/workflow/job-simple/{wf_job_id}" + while tries > 0: + sleep(wait_between_tries) + response = post(url=test_url) + assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" + wf_job_state = response.json()["wf_job_state"] + if wf_job_state == StateEnum.success or wf_job_state == StateEnum.failed: + break + tries -= 1 + assert wf_job_state == "SUCCESS" From ff5e41e4f8d708f0a8a998e4bd287bccd7c94001 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 14:05:55 +0100 Subject: [PATCH 62/93] fix post -> get --- tests/network/test_processing_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 6716ddf503..f91aa91caa 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -55,7 +55,7 @@ def test_processing_server_workflow_request(): path_to_dummy_wf = "/tmp/assets/dummy-workflow.txt" # submit the workflow job - test_url = f"{PROCESSING_SERVER_URL}/workflow/run?mets_path={path_to_mets}&page_wise=False" + test_url = f"{PROCESSING_SERVER_URL}/workflow/run?mets_path={path_to_mets}&page_wise=True" response = post( url=test_url, files={"workflow": open(path_to_dummy_wf, 'rb')} @@ -72,7 +72,7 @@ def test_processing_server_workflow_request(): test_url = f"{PROCESSING_SERVER_URL}/workflow/job-simple/{wf_job_id}" while tries > 0: sleep(wait_between_tries) - response = post(url=test_url) + response = get(url=test_url) assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" wf_job_state = response.json()["wf_job_state"] if wf_job_state == StateEnum.success or wf_job_state == StateEnum.failed: From 2c7bea135063678fdbae86a6939ad53fa44e4250 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 25 Jan 2024 14:36:26 +0100 Subject: [PATCH 63/93] add header to request --- tests/network/test_processing_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index f91aa91caa..4a18f6042f 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -58,6 +58,7 @@ def test_processing_server_workflow_request(): test_url = f"{PROCESSING_SERVER_URL}/workflow/run?mets_path={path_to_mets}&page_wise=True" response = post( url=test_url, + headers={"accept": "application/json"}, files={"workflow": open(path_to_dummy_wf, 'rb')} ) assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" From 6ed08e7c7ca79101cde5476c2b7490b3fb57f1fe Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 12:26:06 +0100 Subject: [PATCH 64/93] fix workflow run test --- Dockerfile | 1 + Makefile | 3 ++- src/ocrd_utils/ocrd_logging.conf | 2 +- tests/network/docker-compose.yml | 34 +++++++++++++++---------- tests/network/dummy-workflow.txt | 2 +- tests/network/test_processing_server.py | 13 +++++++--- 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index e87800bf73..95130fdd4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,7 @@ COPY tests ./tests COPY .gitmodules . COPY requirements_test.txt . RUN pip install -r requirements_test.txt +RUN mkdir /ocrd-data && chmod 777 /ocrd-data CMD ["yes"] # CMD ["make", "test", "integration-test"] diff --git a/Makefile b/Makefile index 8f58c614a6..158415d424 100644 --- a/Makefile +++ b/Makefile @@ -221,7 +221,8 @@ INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d sleep 20 - $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' + # TODO: Remove the `-s` flag before finalizing the PR + $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v -s docker compose -f tests/network/docker-compose.yml down --remove-orphans benchmark: diff --git a/src/ocrd_utils/ocrd_logging.conf b/src/ocrd_utils/ocrd_logging.conf index 6f1a28ace5..8650c7d473 100644 --- a/src/ocrd_utils/ocrd_logging.conf +++ b/src/ocrd_utils/ocrd_logging.conf @@ -133,7 +133,7 @@ args=('ocrd.log','a+') [handler_processingServerHandler] class=FileHandler formatter=defaultFormatter -args=('/tmp/ocrd_processing_server.log','a+') +args=('/ocrd-data/ocrd_processing_server.log','a+') # # default log format conforming to OCR-D (https://ocr-d.de/en/spec/cli#logging) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index a732026221..fcdd1e5f53 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -67,16 +67,18 @@ services: DB_NAME: ocrd_network_test DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs + OCRD_NETWORK_SOCKETS_ROOT_DIR: /ocrd-data/ocrd_network_sockets healthcheck: test: curl -f http://processing-server-host.ocrd_network_test:8000/ interval: 1s timeout: 3s retries: 30 volumes: - - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - - "../assets/kant_aufklaerung_1784:/tmp/assets/kant_aufklaerung_1784" - - "./dummy-workflow.txt:/tmp/assets/dummy-workflow.txt" + - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" + - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" + - "../assets/kant_aufklaerung_1784:/ocrd-data/assets/kant_aufklaerung_1784" + - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: | /bin/bash -c "echo -e \" @@ -105,11 +107,13 @@ services: condition: service_healthy networks: - ocrd_network_test + environment: + OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs volumes: - - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - - "../assets/kant_aufklaerung_1784:/tmp/assets/kant_aufklaerung_1784" - - "./dummy-workflow.txt:/tmp/assets/dummy-workflow.txt" + - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" + - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" + - "../assets/kant_aufklaerung_1784:/ocrd-data/assets/kant_aufklaerung_1784" + - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: > ocrd-dummy worker @@ -124,14 +128,16 @@ services: condition: service_healthy networks: - ocrd_network_test - volumes: - - "/tmp/ocrd_network_logs:/tmp/ocrd_network_logs" - - "/tmp/ocrd_network_sockets:/tmp/ocrd_network_sockets" - - "../assets/kant_aufklaerung_1784:/tmp/assets/kant_aufklaerung_1784" - - "./dummy-workflow.txt:/tmp/assets/dummy-workflow.txt" - - "./ocrd_logging.conf:/etc/ocrd_logging.conf" environment: DB_NAME: ocrd_network_test DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 PROCESSING_SERVER_URL: http://processing-server-host.ocrd_network_test:8000 RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs + OCRD_NETWORK_SOCKETS_ROOT_DIR: /ocrd-data/ocrd_network_sockets + volumes: + - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" + - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" + - "../assets/kant_aufklaerung_1784:/ocrd-data/assets/kant_aufklaerung_1784" + - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" + - "./ocrd_logging.conf:/etc/ocrd_logging.conf" diff --git a/tests/network/dummy-workflow.txt b/tests/network/dummy-workflow.txt index 2de01dc67b..8d4e04d651 100644 --- a/tests/network/dummy-workflow.txt +++ b/tests/network/dummy-workflow.txt @@ -1,3 +1,3 @@ -dummy -I DEFAULT -O OCR-D-DUMMY1 +dummy -I OCR-D-IMG -O OCR-D-DUMMY1 dummy -I OCR-D-DUMMY1 -O OCR-D-DUMMY2 dummy -I OCR-D-DUMMY2 -O OCR-D-DUMMY3 diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 4a18f6042f..cd05ae6823 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -32,7 +32,7 @@ def test_processing_server_deployed_processors(): def _test_processing_server_processing_request(): # Note: the used path is volume mapped test_processing_job_input = { - "path_to_mets": "/tmp/assets/kant_aufklaerung_1784/data/mets.xml", + "path_to_mets": "/ocrd-data/assets/kant_aufklaerung_1784/data/mets.xml", "input_file_grps": ['OCR-D-IMG'], "output_file_grps": ['OCR-D-DUMMY'], "agent_type": NETWORK_AGENT_WORKER, @@ -45,14 +45,16 @@ def _test_processing_server_processing_request(): headers={"accept": "application/json"}, json=test_processing_job_input ) + # TODO: Remove print before finalizing the PR + print(response.json()) assert response.status_code == 200, \ f'Processing server: {test_url}, {response.status_code}' -def test_processing_server_workflow_request(): +def _test_processing_server_workflow_request(): # Note: the used paths are volume mapped - path_to_mets = "/tmp/assets/kant_aufklaerung_1784/data/mets.xml" - path_to_dummy_wf = "/tmp/assets/dummy-workflow.txt" + path_to_mets = "/ocrd-data/assets/kant_aufklaerung_1784/data/mets.xml" + path_to_dummy_wf = "/ocrd-data/assets/dummy-workflow.txt" # submit the workflow job test_url = f"{PROCESSING_SERVER_URL}/workflow/run?mets_path={path_to_mets}&page_wise=True" @@ -61,6 +63,9 @@ def test_processing_server_workflow_request(): headers={"accept": "application/json"}, files={"workflow": open(path_to_dummy_wf, 'rb')} ) + + # TODO: Remove print before finalizing the PR + print(response.json()) assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" wf_job_id = response.json()["job_id"] From 0dcd789c01dc45a4cb3ce565a249747c09d7cb27 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 12:26:34 +0100 Subject: [PATCH 65/93] enable the test --- tests/network/test_processing_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index cd05ae6823..d2442e2244 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -51,7 +51,7 @@ def _test_processing_server_processing_request(): f'Processing server: {test_url}, {response.status_code}' -def _test_processing_server_workflow_request(): +def test_processing_server_workflow_request(): # Note: the used paths are volume mapped path_to_mets = "/ocrd-data/assets/kant_aufklaerung_1784/data/mets.xml" path_to_dummy_wf = "/ocrd-data/assets/dummy-workflow.txt" From bc160a6ffc5cd7089021983f7aaea6ed0c3f3a0c Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 12:39:48 +0100 Subject: [PATCH 66/93] fix the ocrd_logging.conf --- src/ocrd_utils/ocrd_logging.conf | 2 +- tests/network/ocrd_logging.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ocrd_utils/ocrd_logging.conf b/src/ocrd_utils/ocrd_logging.conf index 8650c7d473..6f1a28ace5 100644 --- a/src/ocrd_utils/ocrd_logging.conf +++ b/src/ocrd_utils/ocrd_logging.conf @@ -133,7 +133,7 @@ args=('ocrd.log','a+') [handler_processingServerHandler] class=FileHandler formatter=defaultFormatter -args=('/ocrd-data/ocrd_processing_server.log','a+') +args=('/tmp/ocrd_processing_server.log','a+') # # default log format conforming to OCR-D (https://ocr-d.de/en/spec/cli#logging) diff --git a/tests/network/ocrd_logging.conf b/tests/network/ocrd_logging.conf index 5ee5e015d8..49391334e4 100644 --- a/tests/network/ocrd_logging.conf +++ b/tests/network/ocrd_logging.conf @@ -133,7 +133,7 @@ args=('ocrd.log','a+') [handler_processingServerHandler] class=FileHandler formatter=defaultFormatter -args=('/tmp/ocrd_processing_server.log','a+') +args=('/ocrd-data/ocrd_processing_server.log','a+') # # default log format conforming to OCR-D (https://ocr-d.de/en/spec/cli#logging) From 00b983be882305e04f6f1d1ecce2cfc0a4c7b202 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 13:23:12 +0100 Subject: [PATCH 67/93] use internal assets for testing --- tests/network/docker-compose.yml | 3 --- tests/network/ocrd_logging.conf | 2 +- tests/network/test_processing_server.py | 9 +++++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index fcdd1e5f53..0d65cf6a45 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -77,7 +77,6 @@ services: volumes: - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" - - "../assets/kant_aufklaerung_1784:/ocrd-data/assets/kant_aufklaerung_1784" - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: | @@ -112,7 +111,6 @@ services: volumes: - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" - - "../assets/kant_aufklaerung_1784:/ocrd-data/assets/kant_aufklaerung_1784" - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" command: > @@ -138,6 +136,5 @@ services: volumes: - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" - - "../assets/kant_aufklaerung_1784:/ocrd-data/assets/kant_aufklaerung_1784" - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" diff --git a/tests/network/ocrd_logging.conf b/tests/network/ocrd_logging.conf index 49391334e4..ee081b7b56 100644 --- a/tests/network/ocrd_logging.conf +++ b/tests/network/ocrd_logging.conf @@ -133,7 +133,7 @@ args=('ocrd.log','a+') [handler_processingServerHandler] class=FileHandler formatter=defaultFormatter -args=('/ocrd-data/ocrd_processing_server.log','a+') +args=('/ocrd-data/ocrd_processing_server_conf.log','a+') # # default log format conforming to OCR-D (https://ocr-d.de/en/spec/cli#logging) diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index d2442e2244..c24e537ec6 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -3,6 +3,7 @@ from ocrd_utils.config import config from ocrd_network import NETWORK_AGENT_WORKER from ocrd_network.models import StateEnum +from tests.base import assets PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL @@ -30,9 +31,9 @@ def test_processing_server_deployed_processors(): # TODO: Still failing test with internal error 500 def _test_processing_server_processing_request(): - # Note: the used path is volume mapped + path_to_mets = assets.path_to('kant_aufklaerung_1784/data/mets.xml') test_processing_job_input = { - "path_to_mets": "/ocrd-data/assets/kant_aufklaerung_1784/data/mets.xml", + "path_to_mets": path_to_mets, "input_file_grps": ['OCR-D-IMG'], "output_file_grps": ['OCR-D-DUMMY'], "agent_type": NETWORK_AGENT_WORKER, @@ -52,9 +53,9 @@ def _test_processing_server_processing_request(): def test_processing_server_workflow_request(): - # Note: the used paths are volume mapped - path_to_mets = "/ocrd-data/assets/kant_aufklaerung_1784/data/mets.xml" + # Note: the used workflow path is volume mapped path_to_dummy_wf = "/ocrd-data/assets/dummy-workflow.txt" + path_to_mets = assets.path_to('kant_aufklaerung_1784/data/mets.xml') # submit the workflow job test_url = f"{PROCESSING_SERVER_URL}/workflow/run?mets_path={path_to_mets}&page_wise=True" From d4a46f658489d017c6ae93065c4f9d950c138ce8 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 13:47:26 +0100 Subject: [PATCH 68/93] temp disable macos and u20 wfs --- .github/workflows/integration-test.yml | 2 +- .github/workflows/unit-test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 6b4791a37b..d0834500f8 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -22,7 +22,7 @@ jobs: - '3.11' os: - ubuntu-22.04 - # - macos-latest + # - macos-latest TODO: Activate back before finalizing the PR steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index cfe282cd54..6aac7aa262 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -25,8 +25,8 @@ jobs: - '3.11' os: - ubuntu-22.04 - - ubuntu-20.04 - - macos-latest + # - ubuntu-20.04 TODO: Activate back before finalizing the PR + # - macos-latest TODO: Activate back before finalizing the PR steps: - uses: actions/checkout@v3 From ca6e7ebf6bac9fd569a4329f67f96c21a63dec85 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 14:51:37 +0100 Subject: [PATCH 69/93] fix and refactor processing test --- src/ocrd_network/processing_server.py | 30 +++++++++----- tests/network/test_processing_server.py | 54 ++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/ocrd_network/processing_server.py b/src/ocrd_network/processing_server.py index d4a28e58ec..a23423d326 100644 --- a/src/ocrd_network/processing_server.py +++ b/src/ocrd_network/processing_server.py @@ -472,15 +472,25 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ validate_job_input(self.log, data.processor_name, ocrd_tool, data) - db_workspace = await db_get_workspace( - workspace_id=data.workspace_id, - workspace_mets_path=data.path_to_mets - ) - if not db_workspace: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"Workspace with id: {data.workspace_id} or path: {data.path_to_mets} not found" - ) + if data.workspace_id: + try: + db_workspace = await db_get_workspace(workspace_id=data.workspace_id) + except ValueError as error: + self.log.exception(error) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Workspace with id `{data.workspace_id}` not found in the DB." + ) + else: # data.path_to_mets provided instead + try: + # TODO: Reconsider and refactor this. Core cannot create workspaces by api, but processing-server needs + # the workspace in the database. Here the workspace is created if the path is available locally and + # not existing in the DB - since it has not been uploaded through the Workspace Server. + await db_create_workspace(data.path_to_mets) + except FileNotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, + detail=f"Mets file not existing: {data.path_to_mets}") + workspace_key = data.path_to_mets if data.path_to_mets else data.workspace_id # initialize the request counter for the workspace_key self.cache_processing_requests.update_request_counter(workspace_key=workspace_key, by_value=0) @@ -953,7 +963,7 @@ async def get_workflow_info_simple(self, workflow_job_id) -> Dict[str, StateEnum # if all jobs succeeded if len(job_ids) == success_jobs: workflow_job_state = StateEnum.success - return {"wf_job_state": workflow_job_state} + return {"state": workflow_job_state} async def upload_workflow(self, workflow: UploadFile) -> Dict: """ Store a script for a workflow in the database diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index c24e537ec6..722a120adf 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -8,6 +8,18 @@ PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL +def poll_till_timeout_fail_or_success(test_url: str, tries: int, wait: int) -> StateEnum: + while tries > 0: + sleep(wait) + response = get(url=test_url) + assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" + job_state = response.json()["state"] + if job_state == StateEnum.success or job_state == StateEnum.failed: + break + tries -= 1 + return job_state + + def test_processing_server_connectivity(): test_url = f'{PROCESSING_SERVER_URL}/' response = get(test_url) @@ -29,8 +41,7 @@ def test_processing_server_deployed_processors(): assert processors == [], f'Mismatch in deployed processors' -# TODO: Still failing test with internal error 500 -def _test_processing_server_processing_request(): +def test_processing_server_processing_request(): path_to_mets = assets.path_to('kant_aufklaerung_1784/data/mets.xml') test_processing_job_input = { "path_to_mets": path_to_mets, @@ -50,6 +61,32 @@ def _test_processing_server_processing_request(): print(response.json()) assert response.status_code == 200, \ f'Processing server: {test_url}, {response.status_code}' + processing_job_id = response.json()["job_id"] + assert processing_job_id + + job_state = poll_till_timeout_fail_or_success( + test_url=f"{PROCESSING_SERVER_URL}/processor/job/{processing_job_id}", + tries=10, + wait=10 + ) + assert job_state == StateEnum.success + + """ + # check processing job status till timeout + tries = 50 + wait_between_tries = 30 + processing_job_state = None + test_url = f"{PROCESSING_SERVER_URL}/processor/job/{processing_job_id}" + while tries > 0: + sleep(wait_between_tries) + response = get(url=test_url) + assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" + processing_job_state = response.json()["state"] + if processing_job_state == StateEnum.success or processing_job_state == StateEnum.failed: + break + tries -= 1 + assert processing_job_state == StateEnum.success + """ def test_processing_server_workflow_request(): @@ -64,14 +101,20 @@ def test_processing_server_workflow_request(): headers={"accept": "application/json"}, files={"workflow": open(path_to_dummy_wf, 'rb')} ) - # TODO: Remove print before finalizing the PR print(response.json()) assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" - wf_job_id = response.json()["job_id"] assert wf_job_id + job_state = poll_till_timeout_fail_or_success( + test_url=f"{PROCESSING_SERVER_URL}/workflow/job-simple/{wf_job_id}", + tries=30, + wait=10 + ) + assert job_state == StateEnum.success + + """ # check simplified workflow status till timeout tries = 50 wait_between_tries = 30 @@ -85,4 +128,5 @@ def test_processing_server_workflow_request(): if wf_job_state == StateEnum.success or wf_job_state == StateEnum.failed: break tries -= 1 - assert wf_job_state == "SUCCESS" + assert wf_job_state == StateEnum.success + """ From 68dc06b2a0e0f1a73a4b3ed9d95c7aa705aadcb4 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 15:10:36 +0100 Subject: [PATCH 70/93] new job state - unset, refactor --- src/ocrd_network/models/job.py | 2 ++ src/ocrd_network/processing_server.py | 2 +- tests/network/test_processing_server.py | 35 +------------------------ 3 files changed, 4 insertions(+), 35 deletions(-) diff --git a/src/ocrd_network/models/job.py b/src/ocrd_network/models/job.py index e5230aa5fd..6cb31bfb92 100644 --- a/src/ocrd_network/models/job.py +++ b/src/ocrd_network/models/job.py @@ -20,6 +20,8 @@ class StateEnum(str, Enum): success = 'SUCCESS' # Processing job failed failed = 'FAILED' + # Processing job has not been assigned yet + unset = 'UNSET' class PYJobInput(BaseModel): diff --git a/src/ocrd_network/processing_server.py b/src/ocrd_network/processing_server.py index a23423d326..3c09e8a107 100644 --- a/src/ocrd_network/processing_server.py +++ b/src/ocrd_network/processing_server.py @@ -948,7 +948,7 @@ async def get_workflow_info_simple(self, workflow_job_id) -> Dict[str, StateEnum job_ids: List[str] = [job_id for lst in workflow_job.processing_job_ids.values() for job_id in lst] jobs = await db_get_processing_jobs(job_ids) - workflow_job_state = "UNSET" + workflow_job_state = StateEnum.unset success_jobs = 0 for job in jobs: if job.state == StateEnum.cached or job.state == StateEnum.queued: diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 722a120adf..749122700f 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -9,6 +9,7 @@ def poll_till_timeout_fail_or_success(test_url: str, tries: int, wait: int) -> StateEnum: + job_state = StateEnum.unset while tries > 0: sleep(wait) response = get(url=test_url) @@ -71,23 +72,6 @@ def test_processing_server_processing_request(): ) assert job_state == StateEnum.success - """ - # check processing job status till timeout - tries = 50 - wait_between_tries = 30 - processing_job_state = None - test_url = f"{PROCESSING_SERVER_URL}/processor/job/{processing_job_id}" - while tries > 0: - sleep(wait_between_tries) - response = get(url=test_url) - assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" - processing_job_state = response.json()["state"] - if processing_job_state == StateEnum.success or processing_job_state == StateEnum.failed: - break - tries -= 1 - assert processing_job_state == StateEnum.success - """ - def test_processing_server_workflow_request(): # Note: the used workflow path is volume mapped @@ -113,20 +97,3 @@ def test_processing_server_workflow_request(): wait=10 ) assert job_state == StateEnum.success - - """ - # check simplified workflow status till timeout - tries = 50 - wait_between_tries = 30 - wf_job_state = None - test_url = f"{PROCESSING_SERVER_URL}/workflow/job-simple/{wf_job_id}" - while tries > 0: - sleep(wait_between_tries) - response = get(url=test_url) - assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" - wf_job_state = response.json()["wf_job_state"] - if wf_job_state == StateEnum.success or wf_job_state == StateEnum.failed: - break - tries -= 1 - assert wf_job_state == StateEnum.success - """ From a34bf19eeac8151fd48b45ff70639bb3db3dee10 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 26 Jan 2024 15:34:27 +0100 Subject: [PATCH 71/93] randomize ids --- tests/network/test_db.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/network/test_db.py b/tests/network/test_db.py index bc044812a2..4dd8f26999 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -1,3 +1,4 @@ +from datetime import datetime from hashlib import md5 from pathlib import Path from pytest import raises @@ -17,7 +18,7 @@ def test_db_processing_job_create(mongo_client): - job_id = 'test_job_id_1' + job_id = f'test_job_id_{datetime.now()}' db_created_processing_job = sync_db_create_processing_job( db_processing_job=DBProcessorJob( job_id=job_id, @@ -43,7 +44,7 @@ def test_db_processing_job_create(mongo_client): def test_db_processing_job_update(mongo_client): - job_id = 'test_job_id_2' + job_id = f'test_job_id_{datetime.now()}' db_created_processing_job = sync_db_create_processing_job( db_processing_job=DBProcessorJob( job_id=job_id, @@ -125,7 +126,7 @@ def create_db_model_workflow_script( def test_db_workflow_script_create(mongo_client): - workflow_id = 'test_workflow_1' + workflow_id = f'test_workflow_{datetime.now()}' db_model_workflow_script = create_db_model_workflow_script(workflow_id=workflow_id) db_created_workflow_script = sync_db_create_workflow_script( db_workflow_script=db_model_workflow_script @@ -140,7 +141,7 @@ def test_db_workflow_script_create(mongo_client): def test_db_find_workflow_script_by_content(mongo_client): - workflow_id = 'test_workflow_2' + workflow_id = f'test_workflow_{datetime.now()}' db_model_workflow_script = create_db_model_workflow_script(workflow_id=workflow_id) db_created_workflow_script = sync_db_create_workflow_script( db_workflow_script=db_model_workflow_script From aba2263088648398d9e893461950ef5fef950277 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Fri, 26 Jan 2024 17:19:20 +0100 Subject: [PATCH 72/93] METS Server: try to remove UDS socket at sys.exit --- src/ocrd/mets_server.py | 22 ++++++++++++---------- src/ocrd/workspace.py | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/ocrd/mets_server.py b/src/ocrd/mets_server.py index 08011557b5..be2c1333e0 100644 --- a/src/ocrd/mets_server.py +++ b/src/ocrd/mets_server.py @@ -2,24 +2,24 @@ # METS server functionality """ import re -from os import environ, _exit, chmod -from io import BytesIO -from typing import Any, Dict, Optional, Union, List, Tuple +from os import _exit, chmod +from typing import Dict, Optional, Union, List, Tuple from pathlib import Path from urllib.parse import urlparse import socket +import atexit -from fastapi import FastAPI, Request, File, Form, Response +from fastapi import FastAPI, Request, Form, Response from fastapi.responses import JSONResponse -from requests import request, Session as requests_session +from requests import Session as requests_session from requests.exceptions import ConnectionError from requests_unixsocket import Session as requests_unixsocket_session from pydantic import BaseModel, Field, ValidationError import uvicorn -from ocrd_models import OcrdMets, OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent -from ocrd_utils import initLogging, getLogger, deprecated_alias +from ocrd_models import OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent +from ocrd_utils import getLogger, deprecated_alias # # Models @@ -197,9 +197,10 @@ def __init__(self, workspace, url): self.log = getLogger(f'ocrd.mets_server[{self.url}]') def shutdown(self): - self.log.info("Shutting down METS server") if self.is_uds: - Path(self.url).unlink() + if Path(self.url).exists(): + self.log.warning(f'UDS socket {self.url} still exists, removing it') + Path(self.url).unlink() # os._exit because uvicorn catches SystemExit raised by sys.exit _exit(0) @@ -296,7 +297,7 @@ async def stop(): """ Stop the server """ - getLogger('ocrd.models.ocrd_mets').info('Shutting down') + getLogger('ocrd.models.ocrd_mets').info(f'Shutting down METS Server {self.url}') workspace.save_mets() self.shutdown() @@ -308,6 +309,7 @@ async def stop(): self.log.debug(f"chmod 0o677 {self.url}") server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) server.bind(self.url) # creates the socket file + atexit.register(self.shutdown) server.close() chmod(self.url, 0o666) uvicorn_kwargs = {'uds': self.url} diff --git a/src/ocrd/workspace.py b/src/ocrd/workspace.py index 7772c54d79..633e45acf5 100644 --- a/src/ocrd/workspace.py +++ b/src/ocrd/workspace.py @@ -82,7 +82,7 @@ def __init__(self, resolver, directory, mets=None, mets_basename=DEFAULT_METS_BA mets = ClientSideOcrdMets(mets_server_url) if mets.workspace_path != self.directory: raise ValueError(f"METS server {mets_server_url} workspace directory {mets.workspace_path} differs " - "from local workspace directory {self.directory}. These are not the same workspaces.") + f"from local workspace directory {self.directory}. These are not the same workspaces.") else: mets = OcrdMets(filename=self.mets_target) self.mets = mets From f6ea344c95b0d736e657f1aa366895805af19a01 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 11:09:20 +0100 Subject: [PATCH 73/93] clean leftovers --- Makefile | 3 +-- tests/network/fixtures_rabbitmq.py | 2 +- tests/network/test_db.py | 10 +++++----- tests/network/test_processing_server.py | 6 ++---- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 158415d424..6be39d7a1c 100644 --- a/Makefile +++ b/Makefile @@ -221,8 +221,7 @@ INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: docker compose -f tests/network/docker-compose.yml up -d sleep 20 - # TODO: Remove the `-s` flag before finalizing the PR - $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v -s + $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v docker compose -f tests/network/docker-compose.yml down --remove-orphans benchmark: diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 06e99a98c1..de86386a1f 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -48,7 +48,7 @@ def fixture_rabbitmq_defaults(): vhost=rmq_vhost ) test_channel = RMQConnector.open_blocking_channel(test_connection) - assert(test_channel) + assert test_channel RMQConnector.exchange_declare( channel=test_channel, exchange_name=DEFAULT_EXCHANGER_NAME, diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 4dd8f26999..9401cba7c3 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -39,7 +39,7 @@ def test_db_processing_job_create(mongo_client): assert db_found_processing_job.input_file_grps == ['DEFAULT'] assert db_found_processing_job.output_file_grps == ['OCR-D-DUMMY'] - with raises(ValueError) as value_error: + with raises(ValueError): sync_db_get_processing_job(job_id='non-existing-id') @@ -65,7 +65,7 @@ def test_db_processing_job_update(mongo_client): assert db_found_updated_processing_job == db_updated_processing_job assert db_found_updated_processing_job.state == StateEnum.running - with raises(ValueError) as value_error: + with raises(ValueError): sync_db_update_processing_job(job_id='non-existing', state=StateEnum.running) sync_db_update_processing_job(job_id=job_id, non_existing_field='dummy_value') sync_db_update_processing_job(job_id=job_id, processor_name='non-updatable-field') @@ -80,11 +80,11 @@ def test_db_workspace_create(mongo_client): assert db_found_workspace assert db_found_workspace == db_created_workspace - with raises(ValueError) as value_error: + with raises(ValueError): sync_db_get_workspace(workspace_id='non-existing-id') sync_db_get_workspace(workspace_mets_path='non-existing-mets') - with raises(FileNotFoundError) as io_error: + with raises(FileNotFoundError): sync_db_create_workspace(mets_path='non-existing-mets') @@ -136,7 +136,7 @@ def test_db_workflow_script_create(mongo_client): assert db_found_workflow_script assert db_found_workflow_script == db_created_workflow_script - with raises(ValueError) as value_error: + with raises(ValueError): sync_db_get_workflow_script(workflow_id='non-existing-id') diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 749122700f..a666a21105 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -58,8 +58,7 @@ def test_processing_server_processing_request(): headers={"accept": "application/json"}, json=test_processing_job_input ) - # TODO: Remove print before finalizing the PR - print(response.json()) + # print(response.json()) assert response.status_code == 200, \ f'Processing server: {test_url}, {response.status_code}' processing_job_id = response.json()["job_id"] @@ -85,8 +84,7 @@ def test_processing_server_workflow_request(): headers={"accept": "application/json"}, files={"workflow": open(path_to_dummy_wf, 'rb')} ) - # TODO: Remove print before finalizing the PR - print(response.json()) + # print(response.json()) assert response.status_code == 200, f"Processing server: {test_url}, {response.status_code}" wf_job_id = response.json()["job_id"] assert wf_job_id From be43f47e499eb5d590964574adf7a0a90203a69a Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 11:14:41 +0100 Subject: [PATCH 74/93] transfer method to src --- src/ocrd_network/utils.py | 14 +++++++++++++- tests/network/utils.py | 13 ------------- 2 files changed, 13 insertions(+), 14 deletions(-) delete mode 100644 tests/network/utils.py diff --git a/src/ocrd_network/utils.py b/src/ocrd_network/utils.py index 8f92706f46..8b3764c718 100644 --- a/src/ocrd_network/utils.py +++ b/src/ocrd_network/utils.py @@ -3,7 +3,7 @@ from pika import URLParameters from pymongo import uri_parser as mongo_uri_parser from re import match as re_match -from requests import Session as Session_TCP +from requests import get, Session as Session_TCP from requests_unixsocket import Session as Session_UDS from typing import Dict, List from uuid import uuid4 @@ -49,6 +49,18 @@ def generate_id() -> str: return str(uuid4()) +def is_url_responsive(url: str, retries: int = 0) -> bool: + while True: + try: + response = get(url) + if response.status_code == 200: + return True + except Exception: + if retries <= 0: + return False + retries -= 1 + + def validate_and_load_config(config_path: str) -> Dict: # Load and validate the config with open(config_path) as fin: diff --git a/tests/network/utils.py b/tests/network/utils.py deleted file mode 100644 index 0d33fbc30d..0000000000 --- a/tests/network/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -from requests import get - - -def is_url_responsive(url: str, retries: int = 0) -> bool: - while True: - try: - response = get(url) - if response.status_code == 200: - return True - except Exception: - if retries <= 0: - return False - retries -= 1 From 466d2e6514e77f20f24a11823bde1949b4dfbeb7 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 11:33:50 +0100 Subject: [PATCH 75/93] reuse verifiers from network --- src/ocrd_network/utils.py | 2 +- tests/network/fixtures_mongodb.py | 13 ++----------- tests/network/fixtures_rabbitmq.py | 26 ++------------------------ 3 files changed, 5 insertions(+), 36 deletions(-) diff --git a/src/ocrd_network/utils.py b/src/ocrd_network/utils.py index 8b3764c718..4f66554bc3 100644 --- a/src/ocrd_network/utils.py +++ b/src/ocrd_network/utils.py @@ -76,7 +76,7 @@ def verify_database_uri(mongodb_address: str) -> str: # perform validation check mongo_uri_parser.parse_uri(uri=mongodb_address, validate=True) except Exception as error: - raise ValueError(f"The database address '{mongodb_address}' is in wrong format, {error}") + raise ValueError(f"The MongoDB address '{mongodb_address}' is in wrong format, {error}") return mongodb_address diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 0365560dd2..e99112b7e3 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,16 +1,7 @@ -from pymongo import uri_parser as mongo_uri_parser from pytest import fixture -from ocrd_utils.config import config from ocrd_network.database import sync_initiate_database - - -def verify_database_uri(mongodb_address: str) -> str: - try: - # perform validation check - mongo_uri_parser.parse_uri(uri=mongodb_address, validate=True) - except Exception as error: - raise ValueError(f"The MongoDB address '{mongodb_address}' is in wrong format, {error}") - return mongodb_address +from ocrd_network.utils import verify_database_uri +from ocrd_utils.config import config @fixture(scope="package", name="mongo_client") diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index de86386a1f..8d296f3370 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -1,37 +1,15 @@ -from pika import URLParameters from pika.credentials import PlainCredentials from pytest import fixture -from re import match as re_match from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher +from ocrd_network.utils import verify_and_parse_mq_uri from ocrd_utils.config import config + RABBITMQ_URL = config.RABBITMQ_URL DEFAULT_EXCHANGER_NAME = config.DEFAULT_EXCHANGER_NAME DEFAULT_QUEUE = config.DEFAULT_QUEUE -def verify_and_parse_mq_uri(rabbitmq_address: str): - """ - Check the full list of available parameters in the docs here: - https://pika.readthedocs.io/en/stable/_modules/pika/connection.html#URLParameters - """ - - uri_pattern = r"^(?:([^:\/?#\s]+):\/{2})?(?:([^@\/?#\s]+)@)?([^\/?#\s]+)?(?:\/([^?#\s]*))?(?:[?]([^#\s]+))?\S*$" - match = re_match(pattern=uri_pattern, string=rabbitmq_address) - if not match: - raise ValueError(f"The RabbitMQ server address is in wrong format: '{rabbitmq_address}'") - url_params = URLParameters(rabbitmq_address) - - parsed_data = { - "username": url_params.credentials.username, - "password": url_params.credentials.password, - "host": url_params.host, - "port": url_params.port, - "vhost": url_params.virtual_host - } - return parsed_data - - @fixture(scope="package", name="rabbitmq_defaults") def fixture_rabbitmq_defaults(): rmq_data = verify_and_parse_mq_uri(RABBITMQ_URL) From 477f37ca36ba6adfca025d0dd8272eac60284b6b Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 11:35:51 +0100 Subject: [PATCH 76/93] use prefix 'src.' for imports --- tests/network/fixtures_mongodb.py | 6 +++--- tests/network/fixtures_rabbitmq.py | 6 +++--- tests/network/test_db.py | 4 ++-- tests/network/test_processing_server.py | 6 +++--- tests/network/test_rabbitmq.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index e99112b7e3..34ded26fd2 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,7 +1,7 @@ from pytest import fixture -from ocrd_network.database import sync_initiate_database -from ocrd_network.utils import verify_database_uri -from ocrd_utils.config import config +from src.ocrd_network.database import sync_initiate_database +from src.ocrd_network.utils import verify_database_uri +from src.ocrd_utils.config import config @fixture(scope="package", name="mongo_client") diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 8d296f3370..33f2944b11 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -1,8 +1,8 @@ from pika.credentials import PlainCredentials from pytest import fixture -from ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher -from ocrd_network.utils import verify_and_parse_mq_uri -from ocrd_utils.config import config +from src.ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher +from src.ocrd_network.utils import verify_and_parse_mq_uri +from src.ocrd_utils.config import config RABBITMQ_URL = config.RABBITMQ_URL diff --git a/tests/network/test_db.py b/tests/network/test_db.py index 9401cba7c3..6a69822883 100644 --- a/tests/network/test_db.py +++ b/tests/network/test_db.py @@ -3,8 +3,8 @@ from pathlib import Path from pytest import raises from tests.base import assets -from ocrd_network.models import DBProcessorJob, DBWorkflowScript, StateEnum -from ocrd_network.database import ( +from src.ocrd_network.models import DBProcessorJob, DBWorkflowScript, StateEnum +from src.ocrd_network.database import ( sync_db_create_processing_job, sync_db_get_processing_job, sync_db_update_processing_job, diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index a666a21105..18c957e311 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -1,8 +1,8 @@ from time import sleep from requests import get, post -from ocrd_utils.config import config -from ocrd_network import NETWORK_AGENT_WORKER -from ocrd_network.models import StateEnum +from src.ocrd_utils.config import config +from src.ocrd_network import NETWORK_AGENT_WORKER +from src.ocrd_network.models import StateEnum from tests.base import assets PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL diff --git a/tests/network/test_rabbitmq.py b/tests/network/test_rabbitmq.py index 0b208718f3..8fddd12686 100644 --- a/tests/network/test_rabbitmq.py +++ b/tests/network/test_rabbitmq.py @@ -1,6 +1,6 @@ from pika import BasicProperties from pickle import dumps, loads -from ocrd_utils.config import config +from src.ocrd_utils.config import config DEFAULT_EXCHANGER_NAME = config.DEFAULT_EXCHANGER_NAME DEFAULT_QUEUE = config.DEFAULT_QUEUE From 8d54093703e25fe9b1100221b66cf96a46b0bc94 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 11:46:39 +0100 Subject: [PATCH 77/93] missed src. refactor --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index de39d7874a..62b0c4c863 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -from ocrd_utils.config import config +from src.ocrd_utils.config import config pytest_plugins = [ From b89990c66c33760b094d4a50670f7b04b835900b Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 11:49:34 +0100 Subject: [PATCH 78/93] activate back mac-os and u20 --- .github/workflows/integration-test.yml | 2 +- .github/workflows/unit-test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index d0834500f8..1915a4f3d9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -22,7 +22,7 @@ jobs: - '3.11' os: - ubuntu-22.04 - # - macos-latest TODO: Activate back before finalizing the PR + - macos-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 6aac7aa262..cfe282cd54 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -25,8 +25,8 @@ jobs: - '3.11' os: - ubuntu-22.04 - # - ubuntu-20.04 TODO: Activate back before finalizing the PR - # - macos-latest TODO: Activate back before finalizing the PR + - ubuntu-20.04 + - macos-latest steps: - uses: actions/checkout@v3 From 0d2682dd49b1900ab630e27fa34957b96b18c880 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 30 Jan 2024 12:59:07 +0100 Subject: [PATCH 79/93] ci/integration-test: install docker via brew for macos --- .github/workflows/integration-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1915a4f3d9..18c743d2cb 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -44,6 +44,11 @@ jobs: brew install imagemagick geos bash # opencv fi make install deps-test + - name: Install Docker on macOS + if: runner.os == 'macos' + run: | + brew install docker + colima start - name: Test network integration with pytest run: | make integration-test From 5b433e7edbaba17b43d606a3b0bebacf58fbbf6f Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 30 Jan 2024 12:59:07 +0100 Subject: [PATCH 80/93] ci/integration-test: install docker via brew for macos --- .github/workflows/integration-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1915a4f3d9..15d692e53e 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -44,6 +44,11 @@ jobs: brew install imagemagick geos bash # opencv fi make install deps-test + - name: Install Docker on macOS + if: runner.os == 'macos' + run: | + brew install docker docker-compose + colima start - name: Test network integration with pytest run: | make integration-test From 0f1992ed167bb3c3a572242e8e7dca836261c291 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 30 Jan 2024 13:49:25 +0100 Subject: [PATCH 81/93] make integration-test: DOCKER_COMPOSE overrideable --- .github/workflows/integration-test.yml | 6 +++++- Makefile | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 15d692e53e..858f6d47a7 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -51,4 +51,8 @@ jobs: colima start - name: Test network integration with pytest run: | - make integration-test + if [[ "${{ matrix.os }}" == "macos"* ]];then + make integration-test DOCKER_COMPOSE=docker-compose + else + make integration-test + fi diff --git a/Makefile b/Makefile index 6be39d7a1c..06e864d194 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ TESTDIR = $(CURDIR)/tests PYTEST_ARGS = --continue-on-collection-errors VERSION = $(shell cat VERSION) +DOCKER_COMPOSE = docker compose + SPHINX_APIDOC = BUILD_ORDER = ocrd_utils ocrd_models ocrd_modelfactory ocrd_validators ocrd_network ocrd @@ -219,10 +221,10 @@ test: assets INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: - docker compose -f tests/network/docker-compose.yml up -d + $(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d sleep 20 $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v - docker compose -f tests/network/docker-compose.yml down --remove-orphans + $(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down --remove-orphans benchmark: $(PYTHON) -m pytest $(TESTDIR)/model/test_ocrd_mets_bench.py From 1fdcf4cb2b94fd62bce30e2df4708fbe9e2c8643 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 14:23:31 +0100 Subject: [PATCH 82/93] add .env and adapt --- tests/network/docker-compose.yml | 94 +++++++++++++------------------- tests/network/ps_config.yml | 17 ++++++ 2 files changed, 54 insertions(+), 57 deletions(-) create mode 100644 tests/network/ps_config.yml diff --git a/tests/network/docker-compose.yml b/tests/network/docker-compose.yml index 0d65cf6a45..a5cef49e23 100644 --- a/tests/network/docker-compose.yml +++ b/tests/network/docker-compose.yml @@ -1,23 +1,23 @@ networks: ocrd_network_test: - name: ocrd_network_test + name: ${DOCKER_OCRD_NETWORK_NAME} driver: bridge driver_opts: - com.docker.network.driver.mtu: 1450 + com.docker.network.driver.mtu: ${DOCKER_OCRD_NETWORK_MTU} services: ocrd_network_mongo_db: image: "mongo" - hostname: mongodb-docker-host + hostname: ${MONGODB_HOST} container_name: ocrd_network_mongo_db networks: - - ocrd_network_test + - ${DOCKER_OCRD_NETWORK_NAME} ports: - - "6701:27017" + - ${MONGODB_PORT}:27017 environment: - - MONGO_INITDB_ROOT_USERNAME=network_test - - MONGO_INITDB_ROOT_PASSWORD=network_test + - MONGO_INITDB_ROOT_USERNAME=${MONGODB_USER} + - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_PASS} healthcheck: test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet interval: 1s @@ -26,18 +26,18 @@ services: ocrd_network_rabbit_mq: image: "rabbitmq:3.12-management" - hostname: rabbitmq-docker-host + hostname: ${RABBITMQ_HOST} container_name: ocrd_network_rabbit_mq networks: - - ocrd_network_test + - ${DOCKER_OCRD_NETWORK_NAME} ports: - - "6672:5672" - - "16672:15672" - - "26672:25672" + - ${RABBITMQ_PORT}:5672 + - 15672:15672 + - 25672:25672 environment: - - RABBITMQ_DEFAULT_USER=network_test - - RABBITMQ_DEFAULT_PASS=network_test - - RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version + - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASS} + - RABBITMQ_FEATURE_FLAGS=${RABBITMQ_FEATURE_FLAGS} healthcheck: test: rabbitmq-diagnostics check_port_connectivity interval: 1s @@ -52,7 +52,7 @@ services: args: BASE_IMAGE: 'ubuntu:22.04' target: ocrd_core_test - hostname: processing-server-host + hostname: ${OCRD_PS_HOST} container_name: ocrd_network_processing_server depends_on: ocrd_network_mongo_db: @@ -60,44 +60,27 @@ services: ocrd_network_rabbit_mq: condition: service_healthy networks: - - ocrd_network_test + - ${DOCKER_OCRD_NETWORK_NAME} ports: - - "8000:8000" + - ${OCRD_PS_PORT}:8000 environment: - DB_NAME: ocrd_network_test - DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 - RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + DB_NAME: ${MONGODB_NAME} + DB_URL: ${MONGODB_URL} + RABBITMQ_URL: ${RABBITMQ_URL} OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs OCRD_NETWORK_SOCKETS_ROOT_DIR: /ocrd-data/ocrd_network_sockets healthcheck: - test: curl -f http://processing-server-host.ocrd_network_test:8000/ + test: curl -f ${OCRD_PS_URL}/ interval: 1s timeout: 3s retries: 30 volumes: - - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" - - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" + - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs + - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" - command: | - /bin/bash -c "echo -e \" - internal_callback_url: http://processing-server-host.ocrd_network_test:8000 - process_queue: - address: rabbitmq-docker-host.ocrd_network_test - port: 5672 - skip_deployment: true - credentials: - username: network_test - password: network_test - database: - address: mongodb-docker-host.ocrd_network_test - port: 27017 - skip_deployment: true - credentials: - username: network_test - password: network_test - hosts: []\" > ./ps_config.yaml && \ - ocrd network processing-server -a 0.0.0.0:8000 ./ps_config.yaml" + - "./ps_config.yml:/ocrd-data/ps_config.yml" + command: ocrd network processing-server -a 0.0.0.0:8000 /ocrd-data/ps_config.yml ocrd_dummy_processing_worker: image: "ocrd_core_test" @@ -105,18 +88,15 @@ services: ocrd_network_processing_server: condition: service_healthy networks: - - ocrd_network_test + - ${DOCKER_OCRD_NETWORK_NAME} environment: OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs volumes: - - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" - - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" + - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs + - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" - command: > - ocrd-dummy worker - --database mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 - --queue amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + command: ocrd-dummy worker --database ${MONGODB_URL} --queue ${RABBITMQ_URL} ocrd_network_core_test: image: "ocrd_core_test" @@ -125,16 +105,16 @@ services: ocrd_network_processing_server: condition: service_healthy networks: - - ocrd_network_test + - ${DOCKER_OCRD_NETWORK_NAME} environment: - DB_NAME: ocrd_network_test - DB_URL: mongodb://network_test:network_test@mongodb-docker-host.ocrd_network_test:27017 - PROCESSING_SERVER_URL: http://processing-server-host.ocrd_network_test:8000 - RABBITMQ_URL: amqp://network_test:network_test@rabbitmq-docker-host.ocrd_network_test:5672 + DB_NAME: ${MONGODB_NAME} + DB_URL: ${MONGODB_URL} + PROCESSING_SERVER_URL: ${OCRD_PS_URL} + RABBITMQ_URL: ${RABBITMQ_URL} OCRD_NETWORK_LOGS_ROOT_DIR: /ocrd-data/ocrd_network_logs OCRD_NETWORK_SOCKETS_ROOT_DIR: /ocrd-data/ocrd_network_sockets volumes: - - "/tmp/ocrd_network_logs:/ocrd-data/ocrd_network_logs" - - "/tmp/ocrd_network_sockets:/ocrd-data/ocrd_network_sockets" + - ${OCRD_NETWORK_LOGS_ROOT}:/ocrd-data/ocrd_network_logs + - ${OCRD_NETWORK_SOCKETS_ROOT}:/ocrd-data/ocrd_network_sockets - "./dummy-workflow.txt:/ocrd-data/assets/dummy-workflow.txt" - "./ocrd_logging.conf:/etc/ocrd_logging.conf" diff --git a/tests/network/ps_config.yml b/tests/network/ps_config.yml new file mode 100644 index 0000000000..655a847b8f --- /dev/null +++ b/tests/network/ps_config.yml @@ -0,0 +1,17 @@ +# the content of this config file is based on the .env +internal_callback_url: http://ps-docker-host.ocrd_network_test:8000 +process_queue: + address: rabbitmq-docker-host.ocrd_network_test + port: 5672 + skip_deployment: true + credentials: + username: network_test + password: network_test +database: + address: mongodb-docker-host.ocrd_network_test + port: 27017 + skip_deployment: true + credentials: + username: network_test + password: network_test +hosts: [] From 27cbb8f1f5741fee1f2b6271b2e0c6b92241fda3 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 30 Jan 2024 14:42:26 +0100 Subject: [PATCH 83/93] ci/integration: disable macos --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 858f6d47a7..ab34859c55 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -22,7 +22,7 @@ jobs: - '3.11' os: - ubuntu-22.04 - - macos-latest + # - macos-latest steps: - uses: actions/checkout@v3 From dfbbca3444062cffa161a89216cbc370b21a3e00 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 14:47:00 +0100 Subject: [PATCH 84/93] add the .env file --- tests/network/.env | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/network/.env diff --git a/tests/network/.env b/tests/network/.env new file mode 100644 index 0000000000..7e96308c06 --- /dev/null +++ b/tests/network/.env @@ -0,0 +1,23 @@ +DOCKER_OCRD_NETWORK_NAME=ocrd_network_test +DOCKER_OCRD_NETWORK_MTU=1450 + +OCRD_NETWORK_LOGS_ROOT=/tmp/ocrd_network_logs +OCRD_NETWORK_SOCKETS_ROOT=/tmp/ocrd_network_sockets + +OCRD_PS_HOST=ps-docker-host +OCRD_PS_PORT=8000 +OCRD_PS_URL=http://${OCRD_PS_HOST}.${DOCKER_OCRD_NETWORK_NAME}:${OCRD_PS_PORT} + +MONGODB_NAME=ocrd_network_test +MONGODB_USER=network_test +MONGODB_PASS=network_test +MONGODB_HOST=mongodb-docker-host +MONGODB_PORT=27017 +MONGODB_URL=mongodb://${MONGODB_USER}:${MONGODB_PASS}@${MONGODB_HOST}.${DOCKER_OCRD_NETWORK_NAME}:${MONGODB_PORT} + +RABBITMQ_FEATURE_FLAGS=quorum_queue,implicit_default_bindings,classic_mirrored_queue_version +RABBITMQ_USER=network_test +RABBITMQ_PASS=network_test +RABBITMQ_HOST=rabbitmq-docker-host +RABBITMQ_PORT=5672 +RABBITMQ_URL=amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@${RABBITMQ_HOST}.${DOCKER_OCRD_NETWORK_NAME}:${RABBITMQ_PORT} From 4fb485b2bd0909663dd6c82f94d4edf719f892d9 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 15:34:54 +0100 Subject: [PATCH 85/93] add separate test config --- tests/conftest.py | 14 +-- tests/network/config.py | 130 ++++++++++++++++++++++++ tests/network/fixtures_mongodb.py | 6 +- tests/network/fixtures_rabbitmq.py | 8 +- tests/network/test_processing_server.py | 4 +- tests/network/test_rabbitmq.py | 6 +- 6 files changed, 149 insertions(+), 19 deletions(-) create mode 100644 tests/network/config.py diff --git a/tests/conftest.py b/tests/conftest.py index 62b0c4c863..b699a73bbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -from src.ocrd_utils.config import config +from tests.network.config import test_config pytest_plugins = [ @@ -6,11 +6,11 @@ "tests.network.fixtures_rabbitmq" ] -config.add("DB_NAME", description="...", default=(True, 'ocrd_network_test')) -config.add("DB_URL", description="...", default=(True, 'mongodb://network_test:network_test@0.0.0.0:6701')) +test_config.add("DB_NAME", description="...", default=(True, 'ocrd_network_test')) +test_config.add("DB_URL", description="...", default=(True, 'mongodb://network_test:network_test@0.0.0.0:27017')) -config.add('DEFAULT_EXCHANGER_NAME', description="...", default=(True, 'ocrd-network-default')) -config.add('DEFAULT_QUEUE', description="...", default=(True, 'ocrd-network-default')) +test_config.add('DEFAULT_EXCHANGER_NAME', description="...", default=(True, 'ocrd-network-default')) +test_config.add('DEFAULT_QUEUE', description="...", default=(True, 'ocrd-network-default')) -config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) -config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:6672/")) +test_config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) +test_config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:5672/")) diff --git a/tests/network/config.py b/tests/network/config.py new file mode 100644 index 0000000000..df6c4e83cc --- /dev/null +++ b/tests/network/config.py @@ -0,0 +1,130 @@ +from pathlib import Path +from tempfile import gettempdir +from src.ocrd_utils.config import OcrdEnvConfig +from src.ocrd_utils.config import _ocrd_download_timeout_parser + +test_config = OcrdEnvConfig() + +test_config.add( + name='OCRD_METS_CACHING', + description='If set to `true`, access to the METS file is cached, speeding in-memory search and modification.', + validator=lambda val: val in ('true', 'false', '0', '1'), + parser=lambda val: val in ('true', '1') +) + +test_config.add( + name='OCRD_MAX_PROCESSOR_CACHE', + description=""" + Maximum number of processor instances (for each set of parameters) to be kept in memory (including loaded models) + for processing workers or processor servers. + """, + parser=int, + default=(True, 128) +) + +test_config.add( + name='OCRD_PROFILE', + description=""" + Whether to enable gathering runtime statistics + on the `ocrd.profile` logger (comma-separated): + - `CPU`: yields CPU and wall-time, + - `RSS`: also yields peak memory (resident set size) + - `PSS`: also yields peak memory (proportional set size) + """, + validator=lambda val: all(t in ('', 'CPU', 'RSS', 'PSS') for t in val.split(',')), + default=(True, '') +) + +test_config.add( + name="OCRD_PROFILE_FILE", + description=""" + If set, then the CPU profile is written to this file for later peruse with a analysis tools like snakeviz + """ +) + +test_config.add( + name="OCRD_DOWNLOAD_RETRIES", + description="Number of times to retry failed attempts for downloads of workspace files.", + validator=int, + parser=int, +) + +test_config.add( + name="OCRD_DOWNLOAD_TIMEOUT", + description="Timeout in seconds for connecting or reading (comma-separated) when downloading.", + parser=_ocrd_download_timeout_parser +) + +test_config.add( + name="OCRD_NETWORK_SERVER_ADDR_PROCESSING", + description="Default address of Processing Server to connect to (for `ocrd network client processing`).", + default=(True, '') +) + +test_config.add( + name="OCRD_NETWORK_SERVER_ADDR_WORKFLOW", + description="Default address of Workflow Server to connect to (for `ocrd network client workflow`).", + default=(True, '') +) + +test_config.add( + name="OCRD_NETWORK_SERVER_ADDR_WORKSPACE", + description="Default address of Workspace Server to connect to (for `ocrd network client workspace`).", + default=(True, '') +) + +test_config.add( + name="OCRD_NETWORK_WORKER_QUEUE_CONNECT_ATTEMPTS", + description=""" + Number of attempts for a worker to create its queue. Helpful if the rabbitmq-server needs time to be fully started + """, + parser=int, + default=(True, 1) +) + +test_config.add( + name="OCRD_NETWORK_SOCKETS_ROOT_DIR", + description="The root directory where all mets server related socket files are created", + parser=lambda val: Path(val), + default=(True, Path(gettempdir(), "ocrd_network_sockets")) +) +test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR.mkdir(parents=True, exist_ok=True) + +test_config.add( + name="OCRD_NETWORK_LOGS_ROOT_DIR", + description="The root directory where all ocrd_network related file logs are stored", + parser=lambda val: Path(val), + default=(True, Path(gettempdir(), "ocrd_network_logs")) +) +test_config.OCRD_NETWORK_LOGS_ROOT_DIR.mkdir(parents=True, exist_ok=True) + +test_config.add( + name="HOME", + description="Directory to look for `ocrd_logging.conf`, fallback for unset XDG variables.", + # description="HOME directory, cf. https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html", + validator=lambda val: Path(val).is_dir(), + parser=lambda val: Path(val), + default=(True, lambda: Path.home()) +) + +test_config.add( + name="XDG_DATA_HOME", + description="Directory to look for `./ocrd/resources.yml` (i.e. `ocrd resmgr` user database)", + parser=lambda val: Path(val), + default=(True, lambda: Path(test_config.HOME, '.local/share')) +) + +test_config.add( + name="XDG_CONFIG_HOME", + description="Directory to look for `./ocrd-resources/*` (i.e. `ocrd resmgr` data location)", + parser=lambda val: Path(val), + default=(True, lambda: Path(test_config.HOME, '.config')) +) + +test_config.add( + name="OCRD_LOGGING_DEBUG", + description="Print information about the logging setup to STDERR", + default=(True, False), + validator=lambda val: isinstance(val, bool) or val in ('true', 'false', '0', '1'), + parser=lambda val: val in ('true', '1') +) diff --git a/tests/network/fixtures_mongodb.py b/tests/network/fixtures_mongodb.py index 34ded26fd2..1409bafe72 100644 --- a/tests/network/fixtures_mongodb.py +++ b/tests/network/fixtures_mongodb.py @@ -1,10 +1,10 @@ from pytest import fixture from src.ocrd_network.database import sync_initiate_database from src.ocrd_network.utils import verify_database_uri -from src.ocrd_utils.config import config +from tests.network.config import test_config @fixture(scope="package", name="mongo_client") def fixture_mongo_client(): - verify_database_uri(config.DB_URL) - sync_initiate_database(config.DB_URL, config.DB_NAME) + verify_database_uri(test_config.DB_URL) + sync_initiate_database(test_config.DB_URL, test_config.DB_NAME) diff --git a/tests/network/fixtures_rabbitmq.py b/tests/network/fixtures_rabbitmq.py index 33f2944b11..a3b1300cf2 100644 --- a/tests/network/fixtures_rabbitmq.py +++ b/tests/network/fixtures_rabbitmq.py @@ -2,12 +2,12 @@ from pytest import fixture from src.ocrd_network.rabbitmq_utils import RMQConnector, RMQConsumer, RMQPublisher from src.ocrd_network.utils import verify_and_parse_mq_uri -from src.ocrd_utils.config import config +from tests.network.config import test_config -RABBITMQ_URL = config.RABBITMQ_URL -DEFAULT_EXCHANGER_NAME = config.DEFAULT_EXCHANGER_NAME -DEFAULT_QUEUE = config.DEFAULT_QUEUE +RABBITMQ_URL = test_config.RABBITMQ_URL +DEFAULT_EXCHANGER_NAME = test_config.DEFAULT_EXCHANGER_NAME +DEFAULT_QUEUE = test_config.DEFAULT_QUEUE @fixture(scope="package", name="rabbitmq_defaults") diff --git a/tests/network/test_processing_server.py b/tests/network/test_processing_server.py index 18c957e311..6d039f5bc9 100644 --- a/tests/network/test_processing_server.py +++ b/tests/network/test_processing_server.py @@ -1,11 +1,11 @@ from time import sleep from requests import get, post -from src.ocrd_utils.config import config from src.ocrd_network import NETWORK_AGENT_WORKER from src.ocrd_network.models import StateEnum from tests.base import assets +from tests.network.config import test_config -PROCESSING_SERVER_URL = config.PROCESSING_SERVER_URL +PROCESSING_SERVER_URL = test_config.PROCESSING_SERVER_URL def poll_till_timeout_fail_or_success(test_url: str, tries: int, wait: int) -> StateEnum: diff --git a/tests/network/test_rabbitmq.py b/tests/network/test_rabbitmq.py index 8fddd12686..afe6870f11 100644 --- a/tests/network/test_rabbitmq.py +++ b/tests/network/test_rabbitmq.py @@ -1,9 +1,9 @@ from pika import BasicProperties from pickle import dumps, loads -from src.ocrd_utils.config import config +from tests.network.config import test_config -DEFAULT_EXCHANGER_NAME = config.DEFAULT_EXCHANGER_NAME -DEFAULT_QUEUE = config.DEFAULT_QUEUE +DEFAULT_EXCHANGER_NAME = test_config.DEFAULT_EXCHANGER_NAME +DEFAULT_QUEUE = test_config.DEFAULT_QUEUE def test_rmq_publish_2_messages(rabbitmq_publisher): From 9979b2849cc06169cc75265dfd949ddf13209cb7 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 15:35:36 +0100 Subject: [PATCH 86/93] add suggestion of Jonas --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 06e864d194..ed777d7785 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: $(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d sleep 20 - $(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v + -$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v $(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down --remove-orphans benchmark: From bbc2a8c1ece0fccda332a4f5af4d14f871d8798e Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 15:40:22 +0100 Subject: [PATCH 87/93] remove unnecessary sleep --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index ed777d7785..468b1b8116 100644 --- a/Makefile +++ b/Makefile @@ -222,7 +222,6 @@ test: assets INTEGRATION_TEST_IN_DOCKER = docker exec core_test integration-test: $(DOCKER_COMPOSE) --file tests/network/docker-compose.yml up -d - sleep 20 -$(INTEGRATION_TEST_IN_DOCKER) pytest -k 'test_rmq or test_db or test_processing_server' -v $(DOCKER_COMPOSE) --file tests/network/docker-compose.yml down --remove-orphans From 4bfdc3a271b991de40c41715bb4a61de78c32643 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 15:43:46 +0100 Subject: [PATCH 88/93] remove test deps --- tests/network/test_rabbitmq.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/network/test_rabbitmq.py b/tests/network/test_rabbitmq.py index afe6870f11..951266e5dc 100644 --- a/tests/network/test_rabbitmq.py +++ b/tests/network/test_rabbitmq.py @@ -6,7 +6,7 @@ DEFAULT_QUEUE = test_config.DEFAULT_QUEUE -def test_rmq_publish_2_messages(rabbitmq_publisher): +def test_rmq_publish_then_consume_2_messages(rabbitmq_publisher, rabbitmq_consumer): test_headers = {"Test Header": "Test Value"} test_properties = BasicProperties( app_id='webapi-processing-broker', @@ -27,8 +27,6 @@ def test_rmq_publish_2_messages(rabbitmq_publisher): ) assert rabbitmq_publisher.message_counter == 2 - -def test_rmq_consume_2_messages(rabbitmq_consumer): # Consume the 1st message method_frame, header_frame, message = rabbitmq_consumer.get_one_message( queue_name=DEFAULT_QUEUE, @@ -56,7 +54,7 @@ def test_rmq_consume_2_messages(rabbitmq_consumer): assert message.decode() == "RabbitMQ test 456" -def test_rmq_publish_ocrd_message(rabbitmq_publisher): +def test_rmq_publish_then_consume_ocrd_message(rabbitmq_publisher, rabbitmq_consumer): ocrd_processing_message = { "job_id": "Test_job_id", "workflow_id": "Test_workflow_id", @@ -70,8 +68,6 @@ def test_rmq_publish_ocrd_message(rabbitmq_publisher): properties=None ) - -def test_rmq_consume_ocrd_message(rabbitmq_consumer): method_frame, header_frame, message = rabbitmq_consumer.get_one_message( queue_name=DEFAULT_QUEUE, auto_ack=True From 1a6d2370321974c80a55846b9649a65c4c0be8c0 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 16:21:54 +0100 Subject: [PATCH 89/93] clean socket files before tests --- tests/network/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/network/config.py b/tests/network/config.py index df6c4e83cc..669ad3872b 100644 --- a/tests/network/config.py +++ b/tests/network/config.py @@ -1,3 +1,4 @@ +from glob import glob from pathlib import Path from tempfile import gettempdir from src.ocrd_utils.config import OcrdEnvConfig @@ -89,6 +90,10 @@ default=(True, Path(gettempdir(), "ocrd_network_sockets")) ) test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR.mkdir(parents=True, exist_ok=True) +# Remove socket files left from previous tests +socket_files = glob(pathname="*.sock", root_dir=test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR) +for file in socket_files: + Path(test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR, file).unlink() test_config.add( name="OCRD_NETWORK_LOGS_ROOT_DIR", From 453bed3ceae3ffae93f5a3a2f42a95c693785d46 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 16:35:01 +0100 Subject: [PATCH 90/93] replace glob call available after P3.10 --- tests/conftest.py | 11 ++++++++++- tests/network/config.py | 5 ----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b699a73bbe..cdcb600096 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ +from pathlib import Path +from pytest import fixture from tests.network.config import test_config - pytest_plugins = [ "tests.network.fixtures_mongodb", "tests.network.fixtures_rabbitmq" @@ -14,3 +15,11 @@ test_config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) test_config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:5672/")) + + +@fixture(scope="session", autouse=True) +def do_before_tests(): + # Remove socket files left from previous tests + for file in test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR: + if file.endswith('.sock'): + Path(file).unlink() diff --git a/tests/network/config.py b/tests/network/config.py index 669ad3872b..df6c4e83cc 100644 --- a/tests/network/config.py +++ b/tests/network/config.py @@ -1,4 +1,3 @@ -from glob import glob from pathlib import Path from tempfile import gettempdir from src.ocrd_utils.config import OcrdEnvConfig @@ -90,10 +89,6 @@ default=(True, Path(gettempdir(), "ocrd_network_sockets")) ) test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR.mkdir(parents=True, exist_ok=True) -# Remove socket files left from previous tests -socket_files = glob(pathname="*.sock", root_dir=test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR) -for file in socket_files: - Path(test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR, file).unlink() test_config.add( name="OCRD_NETWORK_LOGS_ROOT_DIR", From 824a9abc04a304acf0770a359ac69da18f23469b Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Tue, 30 Jan 2024 17:24:30 +0100 Subject: [PATCH 91/93] fix socket removal --- tests/conftest.py | 10 ---------- tests/network/config.py | 4 ++++ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cdcb600096..1f35071075 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,3 @@ -from pathlib import Path -from pytest import fixture from tests.network.config import test_config pytest_plugins = [ @@ -15,11 +13,3 @@ test_config.add('PROCESSING_SERVER_URL', description="...", default=(True, "http://0.0.0.0:8000")) test_config.add('RABBITMQ_URL', description="...", default=(True, "amqp://network_test:network_test@0.0.0.0:5672/")) - - -@fixture(scope="session", autouse=True) -def do_before_tests(): - # Remove socket files left from previous tests - for file in test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR: - if file.endswith('.sock'): - Path(file).unlink() diff --git a/tests/network/config.py b/tests/network/config.py index df6c4e83cc..646833aeea 100644 --- a/tests/network/config.py +++ b/tests/network/config.py @@ -89,6 +89,10 @@ default=(True, Path(gettempdir(), "ocrd_network_sockets")) ) test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR.mkdir(parents=True, exist_ok=True) +# Remove socket files left from previous integration tests +for path in test_config.OCRD_NETWORK_SOCKETS_ROOT_DIR.iterdir(): + if path.is_socket() and path.suffix == '.sock': + path.unlink() test_config.add( name="OCRD_NETWORK_LOGS_ROOT_DIR", From f8e47c9ce75ca3210c3d7636fe190eb42a6faa95 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 30 Jan 2024 17:43:56 +0100 Subject: [PATCH 92/93] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1888ac1a62..1a0d266740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Added: + + * Basic integration test for `ocrd_network`, #1164 + ## [2.61.2] - 2024-01-24 Fixed: From ae6efca76b62b8375347028fd8145b241b6fb85b Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 30 Jan 2024 17:47:20 +0100 Subject: [PATCH 93/93] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0d266740..32b5217f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Added: * Basic integration test for `ocrd_network`, #1164 +Fixed: + + * METS Server: UDS sockets are removed on process exit, #117 + ## [2.61.2] - 2024-01-24 Fixed: