From f51e18f3655b30d5b8f6c2fdf35330ec37245409 Mon Sep 17 00:00:00 2001 From: Isabella Basso Date: Thu, 3 Oct 2024 04:58:41 -0300 Subject: [PATCH] enable large file upload (#13) * enable large file upload Signed-off-by: Isabella do Amaral * bump oras-py to 0.2.21 Signed-off-by: tarilabs * test chunked uploads Signed-off-by: Isabella do Amaral --------- Signed-off-by: Isabella do Amaral Signed-off-by: tarilabs Co-authored-by: tarilabs --- .gitignore | 1 + omlmd/helpers.py | 1 + poetry.lock | 8 ++++---- pyproject.toml | 2 +- tests/test_helpers.py | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 8e8d683..e59db4a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ site # E2E venv +temp diff --git a/omlmd/helpers.py b/omlmd/helpers.py index c85ea51..4e38e36 100644 --- a/omlmd/helpers.py +++ b/omlmd/helpers.py @@ -100,6 +100,7 @@ def push( files=files, manifest_annotations=model_metadata.to_annotations_dict(), manifest_config=manifest_cfg, + do_chunked=True, ) self.notify_listeners(PushEvent(target, model_metadata)) return result diff --git a/poetry.lock b/poetry.lock index 4454231..799914e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1456,13 +1456,13 @@ files = [ [[package]] name = "oras" -version = "0.1.30" +version = "0.2.21" description = "OCI Registry as Storage Python SDK" optional = false python-versions = "*" files = [ - {file = "oras-0.1.30-py3-none-any.whl", hash = "sha256:57671f8f619a213a9482e7511c3a5685519d734da2850d2a0ff3286fb82670af"}, - {file = "oras-0.1.30.tar.gz", hash = "sha256:e4d8d9023e0e0fa1bd2c6332b7a14da90cfda63674907868a9364dcf2bee3fa1"}, + {file = "oras-0.2.21-py3-none-any.whl", hash = "sha256:4564d55c417b2fd62d5c5578458d1bfbb69f5c527f1c673b803211c50cf284b4"}, + {file = "oras-0.2.21.tar.gz", hash = "sha256:ea0a4fe180cf825967dd00999c7dec0a2e34cb963c9aa4b270132a811684c1bd"}, ] [package.dependencies] @@ -2567,4 +2567,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "528edb40f371fdf53dcda30256f6a2d5bab38f7a994dd4029e253de0ad9dd8c4" +content-hash = "c3c3ce57a4d5970244462ebea539e02dc5175cfaf68b495a6dabb66909c342f0" diff --git a/pyproject.toml b/pyproject.toml index 3f6743c..8ca3068 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ Changelog = "https://github.com/containers/omlmd/releases" [tool.poetry.dependencies] python = "^3.9" -oras = "^0.1.30" +oras = "^0.2.21" pyyaml = "^6.0.1" click = "^8.1.7" cloup = "^3.0.5" diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 399d527..93f2011 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,4 +1,5 @@ import json +import subprocess import tempfile from pathlib import Path @@ -64,6 +65,39 @@ def update(self, _, event: Event) -> None: assert e0.metadata == ModelMetadata.from_dict(md) +@pytest.mark.e2e +def test_push_pull_chunked(tmp_path, target): + omlmd = Helper() + + md = { + "name": "mnist", + "description": "Lorem ipsum", + "author": "John Doe", + "accuracy": 0.987, + } + here = Path.cwd() + temp = here / "temp" + base_size = 16 * 1024 * 1024 * 3 # 48MB + try: + subprocess.run( + [ + "dd", + "if=/dev/null", + f"of={temp}", + "bs=1", + "count=0", + f"seek={base_size}", + ], + ) + + omlmd.push(target, temp, **md) + omlmd.pull(target, tmp_path) + assert len(list(tmp_path.iterdir())) == 3 + assert tmp_path.joinpath(temp.name).stat().st_size == base_size + finally: + temp.unlink() + + @pytest.mark.e2e def test_e2e_push_pull(tmp_path, target): omlmd = Helper()