From 8a1f796822d8bc5116b4bbb2adb87414a45ec283 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:11:45 +0000 Subject: [PATCH 1/2] Bump openfoodfacts from 0.3.0 to 0.4.0 Bumps [openfoodfacts](https://github.com/openfoodfacts/openfoodfacts-python) from 0.3.0 to 0.4.0. - [Release notes](https://github.com/openfoodfacts/openfoodfacts-python/releases) - [Changelog](https://github.com/openfoodfacts/openfoodfacts-python/blob/develop/CHANGELOG.md) - [Commits](https://github.com/openfoodfacts/openfoodfacts-python/compare/v0.3.0...v0.4.0) --- updated-dependencies: - dependency-name: openfoodfacts dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dc9fcc24f..a1fbed1fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ flower==2.0.1 fontawesomefree~=6.5.1 icalendar==5.0.13 invoke==2.2.0 -openfoodfacts==0.3.0 +openfoodfacts==0.4.0 pillow==10.3.0 reportlab==4.2.0 requests==2.32.3 From 5c05ac14d2df8110e469806e706b4f5704f2bd4e Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Tue, 9 Jul 2024 15:37:08 +0200 Subject: [PATCH 2/2] Download OFF images from S3 --- wger/nutrition/sync.py | 33 ++++++++++++++++++++---------- wger/nutrition/tests/test_tasks.py | 9 ++++++-- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py index d9f1d29b9..cd8cc255b 100644 --- a/wger/nutrition/sync.py +++ b/wger/nutrition/sync.py @@ -23,6 +23,10 @@ # Third Party import requests +from openfoodfacts.images import ( + AWS_S3_BASE_URL, + generate_image_path, +) # wger from wger.nutrition.api.endpoints import ( @@ -93,7 +97,7 @@ def fetch_image_from_wger_instance(ingredient): Image.from_json(ingredient, retrieved_image, image_data) -def fetch_image_from_off(ingredient): +def fetch_image_from_off(ingredient: Ingredient): """ See - https://openfoodfacts.github.io/openfoodfacts-server/api/how-to-download-images/ @@ -104,10 +108,16 @@ def fetch_image_from_off(ingredient): url = ingredient.source_url + '?fields=images,image_front_url' headers = wger_headers() try: - product_data = requests.get(url, headers=headers).json() + product_data = requests.get(url, headers=headers, timeout=3).json() except requests.JSONDecodeError: logger.warning(f'Could not decode JSON response from {url}') return + except requests.ConnectTimeout as e: + logger.warning(f'Connection timeout while trying to fetch {url}: {e}') + return + except requests.ReadTimeout as e: + logger.warning(f'Read timeout while trying to fetch {url}: {e}') + return try: image_url: Optional[str] = product_data['product'].get('image_front_url') @@ -120,17 +130,11 @@ def fetch_image_from_off(ingredient): return image_data = product_data['product']['images'] - # Download the image file - response = requests.get(image_url, headers=headers) - if response.status_code != 200: - logger.info(f'An error occurred! Status code: {response.status_code}') - return - - # Parse the file name, looks something like this: - # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg + # Extract the image key from the url: + # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg -> "front_en" image_id: str = image_url.rpartition('/')[2].partition('.')[0] - # Retrieve the uploader name + # Extract the uploader name try: image_id: str = image_data[image_id]['imgid'] uploader_name: str = image_data[image_id]['uploader'] @@ -138,6 +142,13 @@ def fetch_image_from_off(ingredient): logger.info('could not load all image information, skipping...', e) return + # Download image from amazon + image_s3_url = f'{AWS_S3_BASE_URL}{generate_image_path(ingredient.code, image_id)}' + response = requests.get(image_s3_url, headers=headers) + if not response.ok: + logger.info(f'Could not locate image on AWS! Status code: {response.status_code}') + return + # Save to DB url = ( f'https://world.openfoodfacts.org/cgi/product_image.pl?code={ingredient.code}&id={image_id}' diff --git a/wger/nutrition/tests/test_tasks.py b/wger/nutrition/tests/test_tasks.py index b35f2bb50..273a2e80a 100644 --- a/wger/nutrition/tests/test_tasks.py +++ b/wger/nutrition/tests/test_tasks.py @@ -15,7 +15,10 @@ # Standard Library # Standard Library -from unittest.mock import patch +from unittest.mock import ( + ANY, + patch, +) # wger from wger.core.tests.base_testcase import WgerTestCase @@ -34,6 +37,7 @@ class MockOffResponse: def __init__(self): self.status_code = 200 + self.ok = True self.content = b'2000' # yapf: disable @@ -159,9 +163,10 @@ def test_download_ingredient_off(self, mock_logger, mock_from_json, mock_request mock_request.assert_any_call( 'https://world.openfoodfacts.org/api/v2/product/5055365635003.json?fields=images,image_front_url', headers=wger_headers(), + timeout=ANY, ) mock_request.assert_any_call( - 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg', + 'https://openfoodfacts-images.s3.eu-west-3.amazonaws.com/data/123/456/789/0987654321/12345.jpg', headers=wger_headers(), ) mock_from_json.assert_called()