From e7b3192bccccde203907304d71625d0b8d2b2448 Mon Sep 17 00:00:00 2001 From: Isabella Smallcombe Date: Thu, 7 Sep 2023 19:04:42 -0400 Subject: [PATCH 1/5] feat: add custom adapters to async gen --- offchain/metadata/fetchers/base_fetcher.py | 7 ++++--- offchain/metadata/fetchers/metadata_fetcher.py | 12 +++++++++--- offchain/metadata/pipelines/metadata_pipeline.py | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/offchain/metadata/fetchers/base_fetcher.py b/offchain/metadata/fetchers/base_fetcher.py index ad9b579..95d1ebe 100644 --- a/offchain/metadata/fetchers/base_fetcher.py +++ b/offchain/metadata/fetchers/base_fetcher.py @@ -1,6 +1,6 @@ -from typing import Protocol, Union +from typing import Optional, Protocol, Union -from offchain.metadata.adapters.base_adapter import Adapter +from offchain.metadata.adapters.base_adapter import Adapter, AdapterConfig class BaseFetcher(Protocol): @@ -13,8 +13,9 @@ class BaseFetcher(Protocol): timeout: int max_retries: int + async_adapter_configs: Optional[list[AdapterConfig]] = None - def __init__(self, timeout: int, max_retries: int) -> None: + def __init__(self, timeout: int, max_retries: int, async_adapter_configs: Optional[list[AdapterConfig]] = None) -> None: pass def set_timeout(self, new_timeout: int): diff --git a/offchain/metadata/fetchers/metadata_fetcher.py b/offchain/metadata/fetchers/metadata_fetcher.py index cc9d7cb..1fe57a7 100644 --- a/offchain/metadata/fetchers/metadata_fetcher.py +++ b/offchain/metadata/fetchers/metadata_fetcher.py @@ -1,11 +1,11 @@ import cgi -from typing import Tuple, Union +from typing import Optional, Tuple, Union import httpx import requests from offchain.logger.logging import logger -from offchain.metadata.adapters.base_adapter import Adapter +from offchain.metadata.adapters.base_adapter import Adapter, AdapterConfig from offchain.metadata.fetchers.base_fetcher import BaseFetcher from offchain.metadata.registries.fetcher_registry import FetcherRegistry @@ -24,11 +24,13 @@ def __init__( self, timeout: int = 30, max_retries: int = 0, + async_adapter_configs: Optional[list[AdapterConfig]] = None, ) -> None: self.timeout = timeout self.max_retries = max_retries self.sess = requests.Session() self.async_sess = httpx.AsyncClient() + self.async_adapter_configs = async_adapter_configs def register_adapter(self, adapter: Adapter, url_prefix: str): """Register an adapter to a url prefix. @@ -65,8 +67,12 @@ async def _gen(self, uri: str) -> httpx.Response: from offchain.metadata.pipelines.metadata_pipeline import ( DEFAULT_ADAPTER_CONFIGS, ) + configs = DEFAULT_ADAPTER_CONFIGS - for adapter_config in DEFAULT_ADAPTER_CONFIGS: + if self.async_adapter_configs: + configs = self.async_adapter_configs + + for adapter_config in configs: if any(uri.startswith(prefix) for prefix in adapter_config.mount_prefixes): adapter = adapter_config.adapter_cls(**adapter_config.kwargs) return await adapter.gen_send(url=uri, timeout=self.timeout, sess=self.async_sess) diff --git a/offchain/metadata/pipelines/metadata_pipeline.py b/offchain/metadata/pipelines/metadata_pipeline.py index bafd95a..0a303ae 100644 --- a/offchain/metadata/pipelines/metadata_pipeline.py +++ b/offchain/metadata/pipelines/metadata_pipeline.py @@ -75,7 +75,7 @@ def __init__( adapter_configs: Optional[list[AdapterConfig]] = None, ) -> None: self.contract_caller = contract_caller or ContractCaller() - self.fetcher = fetcher or MetadataFetcher() + self.fetcher = fetcher or MetadataFetcher(async_adapter_configs=adapter_configs) if adapter_configs is None: adapter_configs = DEFAULT_ADAPTER_CONFIGS for adapter_config in adapter_configs: From de82019ed396f3536cabf901e20ef6c8f157ff48 Mon Sep 17 00:00:00 2001 From: Isabella Smallcombe Date: Fri, 8 Sep 2023 13:28:14 -0400 Subject: [PATCH 2/5] fix: host prefixes + add unit test --- .../metadata/fetchers/metadata_fetcher.py | 2 +- .../pipelines/test_metadata_pipeline.py | 29 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/offchain/metadata/fetchers/metadata_fetcher.py b/offchain/metadata/fetchers/metadata_fetcher.py index 1fe57a7..b4937cf 100644 --- a/offchain/metadata/fetchers/metadata_fetcher.py +++ b/offchain/metadata/fetchers/metadata_fetcher.py @@ -74,7 +74,7 @@ async def _gen(self, uri: str) -> httpx.Response: for adapter_config in configs: if any(uri.startswith(prefix) for prefix in adapter_config.mount_prefixes): - adapter = adapter_config.adapter_cls(**adapter_config.kwargs) + adapter = adapter_config.adapter_cls(host_prefixes=adapter_config.host_prefixes, **adapter_config.kwargs) return await adapter.gen_send(url=uri, timeout=self.timeout, sess=self.async_sess) return await self.async_sess.get(uri, timeout=self.timeout) diff --git a/tests/metadata/pipelines/test_metadata_pipeline.py b/tests/metadata/pipelines/test_metadata_pipeline.py index 78dec2e..5e57b0d 100644 --- a/tests/metadata/pipelines/test_metadata_pipeline.py +++ b/tests/metadata/pipelines/test_metadata_pipeline.py @@ -20,7 +20,6 @@ MetadataPipeline, ) - class TestMetadataPipeline: def test_metadata_pipeline_mounts_adapters(self): pipeline = MetadataPipeline( @@ -315,6 +314,34 @@ async def test_metadata_pipeline_async_run(self, raw_crypto_coven_metadata): ], ) ] + + + @pytest.mark.asyncio + async def test_metadata_pipeline_async_run_with_custom_adapters(self, raw_crypto_coven_metadata): + token = Token( + chain_identifier="ETHEREUM-MAINNET", + collection_address="0x5180db8f5c931aae63c74266b211f580155ecac8", + token_id="1", + uri="ipfs://QmSr3vdMuP2fSxWD7S26KzzBWcAN1eNhm4hk1qaR3x3vmj/1.json", + ) + + fetcher = MetadataFetcher() + fetcher.async_fetch_content = AsyncMock(return_value=raw_crypto_coven_metadata) + fetcher.fetch_mime_type_and_size = MagicMock(return_value=("application/json", "3095")) + + pipeline = MetadataPipeline(fetcher=fetcher, adapter_configs=[AdapterConfig( + adapter_cls=IPFSAdapter, + mount_prefixes=[ + "ipfs://", + "https://gateway.pinata.cloud/", + "https://ipfs.io/", + "https://ipfs.decentralized-content.com/", + ], + host_prefixes=["https://gateway.pinata.cloud/"], + kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0}, + )]) + metadata = await pipeline.async_run(tokens=[token]) + assert metadata[0].token == token def test_metadata_pipeline_errors_with_no_parser(self): token = Token( From 750259f3b414163d6702d2cd9fd7ad64b2c757b8 Mon Sep 17 00:00:00 2001 From: Isabella Smallcombe Date: Fri, 8 Sep 2023 13:30:54 -0400 Subject: [PATCH 3/5] fix: unit test --- tests/metadata/pipelines/test_metadata_pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/metadata/pipelines/test_metadata_pipeline.py b/tests/metadata/pipelines/test_metadata_pipeline.py index 5e57b0d..8141dcf 100644 --- a/tests/metadata/pipelines/test_metadata_pipeline.py +++ b/tests/metadata/pipelines/test_metadata_pipeline.py @@ -335,7 +335,6 @@ async def test_metadata_pipeline_async_run_with_custom_adapters(self, raw_crypto "ipfs://", "https://gateway.pinata.cloud/", "https://ipfs.io/", - "https://ipfs.decentralized-content.com/", ], host_prefixes=["https://gateway.pinata.cloud/"], kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0}, From 4476a05d93ee95e8c9299ffa6046738b13162b88 Mon Sep 17 00:00:00 2001 From: Isabella Smallcombe Date: Fri, 8 Sep 2023 13:33:26 -0400 Subject: [PATCH 4/5] fix: format files --- offchain/metadata/fetchers/base_fetcher.py | 7 ++++++- offchain/metadata/fetchers/metadata_fetcher.py | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/offchain/metadata/fetchers/base_fetcher.py b/offchain/metadata/fetchers/base_fetcher.py index 95d1ebe..019ff03 100644 --- a/offchain/metadata/fetchers/base_fetcher.py +++ b/offchain/metadata/fetchers/base_fetcher.py @@ -15,7 +15,12 @@ class BaseFetcher(Protocol): max_retries: int async_adapter_configs: Optional[list[AdapterConfig]] = None - def __init__(self, timeout: int, max_retries: int, async_adapter_configs: Optional[list[AdapterConfig]] = None) -> None: + def __init__( + self, + timeout: int, + max_retries: int, + async_adapter_configs: Optional[list[AdapterConfig]] = None, + ) -> None: pass def set_timeout(self, new_timeout: int): diff --git a/offchain/metadata/fetchers/metadata_fetcher.py b/offchain/metadata/fetchers/metadata_fetcher.py index b4937cf..717a66c 100644 --- a/offchain/metadata/fetchers/metadata_fetcher.py +++ b/offchain/metadata/fetchers/metadata_fetcher.py @@ -67,6 +67,7 @@ async def _gen(self, uri: str) -> httpx.Response: from offchain.metadata.pipelines.metadata_pipeline import ( DEFAULT_ADAPTER_CONFIGS, ) + configs = DEFAULT_ADAPTER_CONFIGS if self.async_adapter_configs: @@ -74,8 +75,12 @@ async def _gen(self, uri: str) -> httpx.Response: for adapter_config in configs: if any(uri.startswith(prefix) for prefix in adapter_config.mount_prefixes): - adapter = adapter_config.adapter_cls(host_prefixes=adapter_config.host_prefixes, **adapter_config.kwargs) - return await adapter.gen_send(url=uri, timeout=self.timeout, sess=self.async_sess) + adapter = adapter_config.adapter_cls( + host_prefixes=adapter_config.host_prefixes, **adapter_config.kwargs + ) + return await adapter.gen_send( + url=uri, timeout=self.timeout, sess=self.async_sess + ) return await self.async_sess.get(uri, timeout=self.timeout) def fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]: From 83a015aa6ff1cc8436931acd8aea548f2afab80d Mon Sep 17 00:00:00 2001 From: Isabella Smallcombe Date: Fri, 8 Sep 2023 13:42:47 -0400 Subject: [PATCH 5/5] feat: update changelog --- docs/changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 011981b..b4170e9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## v0.2.1 + +- Add async support for custom adapters + ## v0.2.0 - Add async support for MetadataPipeline