Skip to content

Commit

Permalink
AsyncGenerator should not yield None (#2464)
Browse files Browse the repository at this point in the history
The `None` was added in 0e035fb, let's try to remove it.
  • Loading branch information
DimitriPapadopoulos authored Nov 5, 2024
1 parent ca46bab commit f05413e
Show file tree
Hide file tree
Showing 6 changed files with 18 additions and 21 deletions.
9 changes: 3 additions & 6 deletions src/zarr/abc/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,17 +329,16 @@ def supports_listing(self) -> bool:
...

@abstractmethod
def list(self) -> AsyncGenerator[str, None]:
def list(self) -> AsyncGenerator[str]:
"""Retrieve all keys in the store.
Returns
-------
AsyncGenerator[str, None]
"""
...

@abstractmethod
def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
"""
Retrieve all keys in the store that begin with a given prefix. Keys are returned relative
to the root of the store.
Expand All @@ -352,10 +351,9 @@ def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
-------
AsyncGenerator[str, None]
"""
...

@abstractmethod
def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
def list_dir(self, prefix: str) -> AsyncGenerator[str]:
"""
Retrieve all keys and prefixes with a given prefix and which do not contain the character
“/” after the given prefix.
Expand All @@ -368,7 +366,6 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
-------
AsyncGenerator[str, None]
"""
...

async def delete_dir(self, prefix: str) -> None:
"""
Expand Down
6 changes: 3 additions & 3 deletions src/zarr/storage/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,22 +217,22 @@ async def exists(self, key: str) -> bool:
path = self.root / key
return await asyncio.to_thread(path.is_file)

async def list(self) -> AsyncGenerator[str, None]:
async def list(self) -> AsyncGenerator[str]:
# docstring inherited
to_strip = self.root.as_posix() + "/"
for p in list(self.root.rglob("*")):
if p.is_file():
yield p.as_posix().replace(to_strip, "")

async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
to_strip = self.root.as_posix() + "/"
prefix = prefix.rstrip("/")
for p in (self.root / prefix).rglob("*"):
if p.is_file():
yield p.as_posix().replace(to_strip, "")

async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
base = self.root / prefix
to_strip = str(base) + "/"
Expand Down
6 changes: 3 additions & 3 deletions src/zarr/storage/logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,19 +204,19 @@ async def set_partial_values(
with self.log(keys):
return await self._store.set_partial_values(key_start_values=key_start_values)

async def list(self) -> AsyncGenerator[str, None]:
async def list(self) -> AsyncGenerator[str]:
# docstring inherited
with self.log():
async for key in self._store.list():
yield key

async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
with self.log(prefix):
async for key in self._store.list_prefix(prefix=prefix):
yield key

async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
with self.log(prefix):
async for key in self._store.list_dir(prefix=prefix):
Expand Down
6 changes: 3 additions & 3 deletions src/zarr/storage/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,19 +147,19 @@ async def set_partial_values(self, key_start_values: Iterable[tuple[str, int, by
# docstring inherited
raise NotImplementedError

async def list(self) -> AsyncGenerator[str, None]:
async def list(self) -> AsyncGenerator[str]:
# docstring inherited
for key in self._store_dict:
yield key

async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
# note: we materialize all dict keys into a list here so we can mutate the dict in-place (e.g. in delete_prefix)
for key in list(self._store_dict):
if key.startswith(prefix):
yield key

async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
prefix = prefix.rstrip("/")

Expand Down
6 changes: 3 additions & 3 deletions src/zarr/storage/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,13 +322,13 @@ async def set_partial_values(
# docstring inherited
raise NotImplementedError

async def list(self) -> AsyncGenerator[str, None]:
async def list(self) -> AsyncGenerator[str]:
# docstring inherited
allfiles = await self.fs._find(self.path, detail=False, withdirs=False)
for onefile in (a.replace(self.path + "/", "") for a in allfiles):
yield onefile

async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
prefix = f"{self.path}/{prefix.rstrip('/')}"
try:
Expand All @@ -338,7 +338,7 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
for onefile in (a.replace(prefix + "/", "") for a in allfiles):
yield onefile.removeprefix(self.path).removeprefix("/")

async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
for onefile in await self.fs._find(
f"{self.path}/{prefix}", detail=False, maxdepth=None, withdirs=False
Expand Down
6 changes: 3 additions & 3 deletions src/zarr/storage/zip.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,19 +234,19 @@ async def exists(self, key: str) -> bool:
else:
return True

async def list(self) -> AsyncGenerator[str, None]:
async def list(self) -> AsyncGenerator[str]:
# docstring inherited
with self._lock:
for key in self._zf.namelist():
yield key

async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
async for key in self.list():
if key.startswith(prefix):
yield key

async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
async def list_dir(self, prefix: str) -> AsyncGenerator[str]:
# docstring inherited
prefix = prefix.rstrip("/")

Expand Down

0 comments on commit f05413e

Please sign in to comment.