diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5817bbbe..a98e2bcc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,6 +71,19 @@ and now you can substitute `gmake` every time the make command is mentioned in g Several options of docker engines are available for Mac. Having Docker installed is also helpful for Testcontainers. +### Podman + +[Podman](https://podman.io/) also supports Rosetta (Apple specific) emulation which is handy since the Google MLMD project dependency is x86 specific. + +We recommend setting up the Podman machine with root privileges, +and setting the environment variable + +```sh +export TESTCONTAINERS_RYUK_PRIVILEGED=true +``` + +when running TestContainer-based Model Registry Python tests (for more information, see [here](https://pypi.org/project/testcontainers/#:~:text=TESTCONTAINERS_RYUK_PRIVILEGED)). + ### Colima Colima offers Rosetta (Apple specific) emulation which is handy since the Google MLMD project dependency is x86 specific. @@ -202,3 +215,31 @@ Then with the given setup MLMD is already installed inside the DevContainer: At this point Poetry is already installed as well and can be used to build and run test of the Model Registry Python client. + +# FAQ + +## Error `docker.errors.NotFound: 404 Client Error for http+docker://localhost/v1.41/containers ...` + +This happens on Mac OSX when running Testcontainers-based Python tests, and [Ryuk](https://github.com/testcontainers/moby-ryuk) does not have the correct access priviledges. + +``` +-------------------------------------------- Captured stderr setup --------------------------------------------- +Pulling image testcontainers/ryuk:0.7.0 +Container started: 4af385c2d670 +---------------------------------------------- Captured log setup ---------------------------------------------- +WARNING root:config.py:59 DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566 +INFO testcontainers.core.container:container.py:88 Pulling image testcontainers/ryuk:0.7.0 +INFO testcontainers.core.container:container.py:101 Container started: 4af385c2d670 +=========================================== short test summary info ============================================ +ERROR tests/store/test_wrapper.py::test_get_undefined_artifact_type_id - docker.errors.NotFound: 404 Client Error for http+docker://localhost/v1.41/containers/4af385c2d67067875f6f0... +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +=============================================== 1 error in 0.50s =============================================== +``` + +Solution: +- Use `export TESTCONTAINERS_RYUK_PRIVILEGED=true` (we recommend this option) +- Use `export TESTCONTAINERS_RYUK_DISABLED=true` to disable Ryuk entirely; the tests are configured to close all container resources, but we do not recommend this option + +For more information: +- section [Docker engine](https://github.com/kubeflow/model-registry/blob/main/CONTRIBUTING.md#docker-engine) in this document +- https://pypi.org/project/testcontainers/#:~:text=the%20database%20version.-,Configuration,-Env%20Variable diff --git a/clients/python/README.md b/clients/python/README.md index 7e4ef91f..73e7575e 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -110,13 +110,14 @@ Use `nox -l` to list sessions and execute them using `nox -s [session]`. ### Running Locally on Mac M1 or M2 (arm64 architecture) -If you want run tests locally you will need to set up a colima develeopment environment using the instructions [here](https://github.com/kubeflow/model-registry/blob/main/CONTRIBUTING.md#colima) +If you want run tests locally you will need to set up a development environment, including docker engine; we recommend following the instructions [here](https://github.com/kubeflow/model-registry/blob/main/CONTRIBUTING.md#docker-engine). You will also have to change the package source to one compatible with ARM64 architecture. This can be actioned by uncommenting lines 14 or 15 in the pyproject.toml file. Run the following command after you have uncommented the line. ```sh poetry lock ``` + Use the following commands to directly run the tests with individual test output. Alternatively you can use the nox session commands above. ```sh diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock index 42496901..da17eb08 100644 --- a/clients/python/poetry.lock +++ b/clients/python/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "absl-py" @@ -295,40 +295,27 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] -[[package]] -name = "deprecation" -version = "2.1.0" -description = "A library to handle automated deprecations" -optional = false -python-versions = "*" -files = [ - {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, - {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, -] - -[package.dependencies] -packaging = "*" - [[package]] name = "docker" -version = "6.1.3" +version = "7.1.0" description = "A Python library for the Docker Engine API." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"}, - {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"}, + {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, + {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, ] [package.dependencies] -packaging = ">=14.0" pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} requests = ">=2.26.0" urllib3 = ">=1.26.0" -websocket-client = ">=0.32.0" [package.extras] +dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"] +docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "docutils" @@ -1053,13 +1040,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.3" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -1341,36 +1328,46 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7 [[package]] name = "testcontainers" -version = "3.7.1" -description = "Library provides lightweight, throwaway instances of common databases, Selenium web browsers, or anything else that can run in a Docker container" +version = "4.5.0" +description = "Python library for throwaway instances of anything that can run in a Docker container" optional = false -python-versions = ">=3.7" +python-versions = "<4.0,>=3.9" files = [ - {file = "testcontainers-3.7.1-py2.py3-none-any.whl", hash = "sha256:7f48cef4bf0ccd78f1a4534d4b701a003a3bace851f24eae58a32f9e3f0aeba0"}, + {file = "testcontainers-4.5.0-py3-none-any.whl", hash = "sha256:6833df7db4cf8403feb1e4daf69b02a091d9da37aa3e8e808c756abbddb41cf3"}, + {file = "testcontainers-4.5.0.tar.gz", hash = "sha256:e8610985cae2f0bc4bc3991896a47d5547f3ed896192970826151059a75f4ac2"}, ] [package.dependencies] -deprecation = "*" -docker = ">=4.0.0" +docker = "*" +typing-extensions = "*" +urllib3 = "*" wrapt = "*" [package.extras] -arangodb = ["python-arango"] -azurite = ["azure-storage-blob"] +arangodb = ["python-arango (>=7.8,<8.0)"] +azurite = ["azure-storage-blob (>=12.19,<13.0)"] +chroma = ["chromadb-client"] clickhouse = ["clickhouse-driver"] -docker-compose = ["docker-compose"] -google-cloud-pubsub = ["google-cloud-pubsub (<2)"] -kafka = ["kafka-python"] +google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"] +influxdb = ["influxdb", "influxdb-client"] +k3s = ["kubernetes", "pyyaml"] keycloak = ["python-keycloak"] -mongo = ["pymongo"] -mssqlserver = ["pymssql"] -mysql = ["pymysql", "sqlalchemy"] +localstack = ["boto3"] +minio = ["minio"] +mongodb = ["pymongo"] +mssql = ["pymssql", "sqlalchemy"] +mysql = ["pymysql[rsa]", "sqlalchemy"] +nats = ["nats-py"] neo4j = ["neo4j"] -oracle = ["cx-Oracle", "sqlalchemy"] -postgresql = ["psycopg2-binary", "sqlalchemy"] +opensearch = ["opensearch-py"] +oracle = ["oracledb", "sqlalchemy"] +oracle-free = ["oracledb", "sqlalchemy"] +qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] +registry = ["bcrypt"] selenium = ["selenium"] +weaviate = ["weaviate-client (>=4.5.4,<5.0.0)"] [[package]] name = "tomli" @@ -1550,22 +1547,6 @@ files = [ [package.dependencies] anyio = ">=3.0.0" -[[package]] -name = "websocket-client" -version = "1.7.0" -description = "WebSocket client for Python with low level API options" -optional = false -python-versions = ">=3.8" -files = [ - {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, - {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, -] - -[package.extras] -docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - [[package]] name = "websockets" version = "12.0" @@ -1747,4 +1728,4 @@ hf = ["huggingface-hub"] [metadata] lock-version = "2.0" python-versions = ">= 3.9, < 3.11" -content-hash = "c973d75abf0797917559b1fa67e20bec050fad676a6c7ad14c6a0ee73a9ecf9f" +content-hash = "240f192369491ba111bcafa4d6a61d32b8c6fdd6f7b32c00d131148801891dab" diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 02b2a197..bd4d3df6 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -13,12 +13,14 @@ homepage = "https://github.com/kubeflow/model-registry" [tool.poetry.dependencies] python = ">= 3.9, < 3.11" attrs = "^21.0" -ml-metadata = "^1.14.0" +ml-metadata = "==1.14.0" # you might consider using locally the following alternative, when developing on Apple-silicon/ARM-based computers: # ml-metadata = { url = "https://github.com/opendatahub-io/ml-metadata/releases/download/v1.14.0%2Bremote.1/ml_metadata-1.14.0+remote.1-py3-none-any.whl" } typing-extensions = "^4.8" huggingface-hub = { version = ">=0.20.1,<0.24.0", optional = true } +# pin docker+requests dependencies of testcontainers to docker ^7.1.0 for compatibility with requests ^2.32.2; ref: https://github.com/testcontainers/testcontainers-python/issues/577#issuecomment-2123324367 +requests = { version = "^2.32.2", optional = true } [tool.poetry.extras] hf = ["huggingface-hub"] @@ -33,7 +35,9 @@ pytest-cov = "^4.1.0" sphinx-autobuild = ">=2021.3.14,<2025.0.0" ruff = "^0.4.4" mypy = "^1.7.0" -testcontainers = "^3.7.1" +testcontainers = "^4.5.0" +# pin docker+requests dependencies of testcontainers to docker ^7.1.0 for compatibility with requests ^2.32.2; ref: https://github.com/testcontainers/testcontainers-python/issues/577#issuecomment-2123324367 +docker = "^7.1.0" [tool.coverage.run] branch = true