Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add async stdio files #154

Merged
merged 3 commits into from
Dec 21, 2022
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ and delegate to an executor:

In case of failure, one of the usual exceptions will be raised.

``aiofiles.stdin``, ``aiofiles.stdout``, ``aiofiles.stderr``,
``aiofiles.stdin_bytes``, ``aiofiles.stdout_bytes``, and
``aiofiles.stderr_bytes`` provide async access to ``sys.stdin``,
``sys.stdout``, ``sys.stderr``, and their corresponding ``.buffer`` properties.

The ``aiofiles.os`` module contains executor-enabled coroutine versions of
several useful ``os`` functions that deal with files:

Expand Down Expand Up @@ -180,6 +185,8 @@ History
`#146 <https://github.com/Tinche/aiofiles/pull/146>`_
* Removed ``aiofiles.tempfile.temptypes.AsyncSpooledTemporaryFile.softspace``.
`#151 <https://github.com/Tinche/aiofiles/pull/151>`_
* Added ``aiofiles.stdin``, ``aiofiles.stdin_bytes``, and other stdio streams.
`#154 <https://github.com/Tinche/aiofiles/pull/154>`_

22.1.0 (2022-09-04)
```````````````````
Expand Down
21 changes: 19 additions & 2 deletions src/aiofiles/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
"""Utilities for asyncio-friendly file handling."""
from .threadpool import open
from .threadpool import (
open,
stdin,
stdout,
stderr,
stdin_bytes,
stdout_bytes,
stderr_bytes,
)
from . import tempfile

__all__ = ["open", "tempfile"]
__all__ = [
"open",
"tempfile",
"stdin",
"stdout",
"stderr",
"stdin_bytes",
"stdout_bytes",
"stderr_bytes",
]
21 changes: 20 additions & 1 deletion src/aiofiles/base.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
"""Various base classes."""
from types import coroutine
from collections.abc import Coroutine
from asyncio import get_running_loop


class AsyncBase:
def __init__(self, file, loop, executor):
self._file = file
self._loop = loop
self._executor = executor
self._ref_loop = loop

@property
def _loop(self):
return self._ref_loop or get_running_loop()

def __aiter__(self):
"""We are our own iterator."""
Expand All @@ -25,6 +30,20 @@ async def __anext__(self):
raise StopAsyncIteration


class AsyncIndirectBase(AsyncBase):
def __init__(self, file, loop, executor, indirect):
self._indirect = indirect
super().__init__(file, loop, executor)

@property
def _file(self):
return self._indirect()

@_file.setter
def _file(self, v):
pass # discard writes


class _ContextManager(Coroutine):
__slots__ = ("_coro", "_obj")

Expand Down
86 changes: 74 additions & 12 deletions src/aiofiles/threadpool/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Handle files using a thread pool executor."""
import asyncio
import sys
from types import coroutine

from io import (
Expand All @@ -8,16 +9,32 @@
BufferedReader,
BufferedWriter,
BufferedRandom,
BufferedIOBase,
)
from functools import partial, singledispatch

from .binary import AsyncBufferedIOBase, AsyncBufferedReader, AsyncFileIO
from .text import AsyncTextIOWrapper
from .binary import (
AsyncBufferedIOBase,
AsyncBufferedReader,
AsyncFileIO,
AsyncIndirectBufferedIOBase,
AsyncIndirectBufferedReader,
AsyncIndirectFileIO,
)
from .text import AsyncTextIOWrapper, AsyncTextIndirectIOWrapper
from ..base import AiofilesContextManager

sync_open = open

__all__ = ("open",)
__all__ = (
"open",
"stdin",
"stdout",
"stderr",
"stdin_bytes",
"stdout_bytes",
"stderr_bytes",
)


def open(
Expand Down Expand Up @@ -83,26 +100,71 @@ def _open(


@singledispatch
def wrap(file, *, loop=None, executor=None):
def wrap(file, *, loop=None, executor=None, indirect=None):
raise TypeError("Unsupported io type: {}.".format(file))


@wrap.register(TextIOBase)
def _(file, *, loop=None, executor=None):
return AsyncTextIOWrapper(file, loop=loop, executor=executor)
def _(file, *, loop=None, executor=None, indirect=None):
if indirect is None:
return AsyncTextIOWrapper(file, loop=loop, executor=executor)
else:
return AsyncTextIndirectIOWrapper(
file, loop=loop, executor=executor, indirect=indirect
)


@wrap.register(BufferedWriter)
def _(file, *, loop=None, executor=None):
return AsyncBufferedIOBase(file, loop=loop, executor=executor)
@wrap.register(BufferedIOBase)
def _(file, *, loop=None, executor=None, indirect=None):
if indirect is None:
return AsyncBufferedIOBase(file, loop=loop, executor=executor)
else:
return AsyncIndirectBufferedIOBase(
file, loop=loop, executor=executor, indirect=indirect
)


@wrap.register(BufferedReader)
@wrap.register(BufferedRandom)
def _(file, *, loop=None, executor=None):
return AsyncBufferedReader(file, loop=loop, executor=executor)
def _(file, *, loop=None, executor=None, indirect=None):
if indirect is None:
return AsyncBufferedReader(file, loop=loop, executor=executor)
else:
return AsyncIndirectBufferedReader(
file, loop=loop, executor=executor, indirect=indirect
)


@wrap.register(FileIO)
def _(file, *, loop=None, executor=None):
return AsyncFileIO(file, loop, executor)
def _(file, *, loop=None, executor=None, indirect=None):
if indirect is None:
return AsyncFileIO(file, loop, executor)
else:
return AsyncIndirectFileIO(file, loop, executor, indirect=indirect)


try:
stdin = wrap(sys.stdin, indirect=lambda: sys.stdin)
except TypeError:
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where do the TypeErrors come from btw?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sys.stdin may have been patched to be an unusable base type prior to importing aiofiles. This happens specifically when using pytest. I figured I would add it to the rest of the stdio files just be safe.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, alright. Do we want to document this for people somewhere?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

imo the best documentation would be replacing the stdin = None with a custom class which raises a descriptive error when you try to use any of its methods. I was hoping I could convince you to do that 😅

lmk if you would like to do that or just add a note to the readme/docs

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Never mind, just thought of a much better solution. One sec.

stdin = None
try:
stdout = wrap(sys.stdout, indirect=lambda: sys.stdout)
except TypeError:
stdout = None
try:
stderr = wrap(sys.stderr, indirect=lambda: sys.stderr)
except TypeError:
stdout = None
try:
stdin_bytes = wrap(sys.stdin.buffer, indirect=lambda: sys.stdin.buffer)
except TypeError:
stdin_bytes = None
try:
stdout_bytes = wrap(sys.stdout.buffer, indirect=lambda: sys.stdout.buffer)
except TypeError:
stdout_bytes = None
try:
stderr_bytes = wrap(sys.stderr.buffer, indirect=lambda: sys.stderr.buffer)
except TypeError:
stderr_bytes = None
55 changes: 53 additions & 2 deletions src/aiofiles/threadpool/binary.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from ..base import AsyncBase
from ..base import AsyncBase, AsyncIndirectBase
from .utils import (
delegate_to_executor,
proxy_method_directly,
Expand Down Expand Up @@ -26,7 +26,7 @@
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly("closed", "raw", "name", "mode")
class AsyncBufferedIOBase(AsyncBase):
"""The asyncio executor version of io.BufferedWriter."""
"""The asyncio executor version of io.BufferedWriter and BufferedIOBase."""


@delegate_to_executor("peek")
Expand Down Expand Up @@ -55,3 +55,54 @@ class AsyncBufferedReader(AsyncBufferedIOBase):
@proxy_property_directly("closed", "name", "mode")
class AsyncFileIO(AsyncBase):
"""The asyncio executor version of io.FileIO."""


@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"read1",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly("closed", "raw", "name", "mode")
class AsyncIndirectBufferedIOBase(AsyncIndirectBase):
"""The indirect asyncio executor version of io.BufferedWriter and BufferedIOBase."""


@delegate_to_executor("peek")
class AsyncIndirectBufferedReader(AsyncIndirectBufferedIOBase):
"""The indirect asyncio executor version of io.BufferedReader and Random."""


@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readall",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("fileno", "readable")
@proxy_property_directly("closed", "name", "mode")
class AsyncIndirectFileIO(AsyncIndirectBase):
"""The indirect asyncio executor version of io.FileIO."""
33 changes: 32 additions & 1 deletion src/aiofiles/threadpool/text.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from ..base import AsyncBase
from ..base import AsyncBase, AsyncIndirectBase
from .utils import (
delegate_to_executor,
proxy_method_directly,
Expand Down Expand Up @@ -35,3 +35,34 @@
)
class AsyncTextIOWrapper(AsyncBase):
"""The asyncio executor version of io.TextIOWrapper."""


@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readable",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"write",
"writable",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly(
"buffer",
"closed",
"encoding",
"errors",
"line_buffering",
"newlines",
"name",
"mode",
)
class AsyncTextIndirectIOWrapper(AsyncIndirectBase):
"""The indirect asyncio executor version of io.TextIOWrapper."""
10 changes: 7 additions & 3 deletions tests/test_os.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ async def test_renames():
assert exists(old_filename) is False and exists(new_filename)
await aiofiles.os.renames(new_filename, old_filename)
assert (
exists(old_filename) and
exists(new_filename) is False and
exists(dirname(new_filename)) is False
exists(old_filename)
and exists(new_filename) is False
and exists(dirname(new_filename)) is False
)


Expand Down Expand Up @@ -323,6 +323,7 @@ async def test_listdir_dir_with_only_one_file():
await aiofiles.os.remove(some_file)
await aiofiles.os.rmdir(some_dir)


@pytest.mark.asyncio
async def test_listdir_dir_with_only_one_dir():
"""Test the listdir call when the dir has one dir."""
Expand All @@ -335,6 +336,7 @@ async def test_listdir_dir_with_only_one_dir():
await aiofiles.os.rmdir(other_dir)
await aiofiles.os.rmdir(some_dir)


@pytest.mark.asyncio
async def test_listdir_dir_with_multiple_files():
"""Test the listdir call when the dir has multiple files."""
Expand All @@ -353,6 +355,7 @@ async def test_listdir_dir_with_multiple_files():
await aiofiles.os.remove(other_file)
await aiofiles.os.rmdir(some_dir)


@pytest.mark.asyncio
async def test_listdir_dir_with_a_file_and_a_dir():
"""Test the listdir call when the dir has files and other dirs."""
Expand Down Expand Up @@ -406,6 +409,7 @@ async def test_scandir_dir_with_only_one_file():
await aiofiles.os.remove(some_file)
await aiofiles.os.rmdir(some_dir)


@pytest.mark.asyncio
async def test_scandir_dir_with_only_one_dir():
"""Test the scandir call when the dir has one dir."""
Expand Down
21 changes: 21 additions & 0 deletions tests/test_stdio.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import sys
import pytest
from aiofiles import stdout, stderr, stdout_bytes, stderr_bytes


@pytest.mark.asyncio
async def test_stdio(capsys):
await stdout.write("hello")
await stderr.write("world")
out, err = capsys.readouterr()
assert out == "hello"
assert err == "world"


@pytest.mark.asyncio
async def test_stdio_bytes(capsysbinary):
await stdout_bytes.write(b"hello")
await stderr_bytes.write(b"world")
out, err = capsysbinary.readouterr()
assert out == b"hello"
assert err == b"world"
Loading