Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add os.walk wrapper #193

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 105 additions & 11 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
env/
pyvenv/
build/
develop-eggs/
dist/
Expand All @@ -20,10 +19,12 @@ lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
.venv*
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
Expand All @@ -38,43 +39,136 @@ pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# IDEA IDE files
.idea/
*.iml
.pytest_cache
# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version

# VScode
.vscode/
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
#uv.lock

# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

# Visual Studio Code
.vscode

# ruff stuff
.ruff_cache
13 changes: 12 additions & 1 deletion src/aiofiles/base.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from asyncio import get_running_loop
from collections.abc import Awaitable
from contextlib import AbstractAsyncContextManager
from functools import partial, wraps


class AsyncBase:
Expand All @@ -14,7 +15,6 @@ def _loop(self):
return self._ref_loop or get_running_loop()

def __aiter__(self):
"""We are our own iterator."""
return self

def __repr__(self):
Expand Down Expand Up @@ -65,3 +65,14 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
None, self._obj._file.__exit__, exc_type, exc_val, exc_tb
)
self._obj = None


def wrap(func):
@wraps(func)
async def run(*args, loop=None, executor=None, **kwargs):
if loop is None:
loop = get_running_loop()
pfunc = partial(func, *args, **kwargs)
return await loop.run_in_executor(executor, pfunc)

return run
stankudrow marked this conversation as resolved.
Show resolved Hide resolved
55 changes: 36 additions & 19 deletions src/aiofiles/os.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

import os

from . import ospath as path
from .ospath import wrap
from aiofiles.base import wrap
from aiofiles import ospath as path


__all__ = [
"path",
Expand All @@ -25,34 +26,50 @@
"wrap",
"getcwd",
]
if hasattr(os, "link"):
__all__ += ["link"]
if hasattr(os, "sendfile"):
__all__ += ["sendfile"]
if hasattr(os, "statvfs"):
__all__ += ["statvfs"]


stat = wrap(os.stat)
access = wrap(os.access)

getcwd = wrap(os.getcwd)

listdir = wrap(os.listdir)

makedirs = wrap(os.makedirs)
mkdir = wrap(os.mkdir)

readlink = wrap(os.readlink)
remove = wrap(os.remove)
removedirs = wrap(os.removedirs)
rename = wrap(os.rename)
renames = wrap(os.renames)
replace = wrap(os.replace)
remove = wrap(os.remove)
unlink = wrap(os.unlink)
mkdir = wrap(os.mkdir)
makedirs = wrap(os.makedirs)
rmdir = wrap(os.rmdir)
removedirs = wrap(os.removedirs)
symlink = wrap(os.symlink)
readlink = wrap(os.readlink)
listdir = wrap(os.listdir)

scandir = wrap(os.scandir)
access = wrap(os.access)
getcwd = wrap(os.getcwd)
stat = wrap(os.stat)
symlink = wrap(os.symlink)

unlink = wrap(os.unlink)


if hasattr(os, "link"):
__all__ += ["link"]
link = wrap(os.link)
if hasattr(os, "sendfile"):
__all__ += ["sendfile"]
sendfile = wrap(os.sendfile)
if hasattr(os, "statvfs"):
__all__ += ["statvfs"]
statvfs = wrap(os.statvfs)


async def walk(top, topdown=True, onerror=None, followlinks=False):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, better, but this function isn't really async now. It doesn't handle file system operations in a thread, and so can block the event loop, which we're trying to avoid.

Instead of a simple for loop, try executing next() on the threadpool.

To get you started, maybe try something like

source = os.walk(...)

while True:
    f = await loop.run_in_executor(None, next, source)

"""Asynchronous directory tree generator.

Wraps the `os.walk` function.
"""

for content in os.walk(
top, topdown=topdown, onerror=onerror, followlinks=followlinks
):
yield content
25 changes: 9 additions & 16 deletions src/aiofiles/ospath.py
Original file line number Diff line number Diff line change
@@ -1,30 +1,23 @@
"""Async executor versions of file functions from the os.path module."""

import asyncio
from functools import partial, wraps
from os import path

from aiofiles.base import wrap
stankudrow marked this conversation as resolved.
Show resolved Hide resolved

def wrap(func):
@wraps(func)
async def run(*args, loop=None, executor=None, **kwargs):
if loop is None:
loop = asyncio.get_running_loop()
pfunc = partial(func, *args, **kwargs)
return await loop.run_in_executor(executor, pfunc)

return run

abspath = wrap(path.abspath)

exists = wrap(path.exists)
isfile = wrap(path.isfile)
isdir = wrap(path.isdir)
islink = wrap(path.islink)
ismount = wrap(path.ismount)

getsize = wrap(path.getsize)
getmtime = wrap(path.getmtime)
getatime = wrap(path.getatime)
getctime = wrap(path.getctime)

isfile = wrap(path.isfile)
isdir = wrap(path.isdir)
islink = wrap(path.islink)
ismount = wrap(path.ismount)

samefile = wrap(path.samefile)
sameopenfile = wrap(path.sameopenfile)
abspath = wrap(path.abspath)
12 changes: 11 additions & 1 deletion tests/test_os.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import asyncio
import os
import platform
from os import stat
from os import stat, walk
from os.path import dirname, exists, isdir, join
from pathlib import Path

Expand Down Expand Up @@ -498,3 +498,13 @@ async def test_abspath():
abs_filename = join(dirname(__file__), "resources", "test_file1.txt")
result = await aiofiles.os.path.abspath(relative_filename)
assert result == abs_filename


@pytest.mark.parametrize(("path",), [(".",), ("..",)])
async def test_walk(path: str):
"""Testing the `os.walk` wrapped function."""

os_walk_res = list(walk(top=path))
aio_walk_res = [r async for r in aiofiles.os.walk(top=path)]

assert aio_walk_res == os_walk_res
Loading