Третий коммит, добавление share, share_kb, а также ADMIN_ID

This commit is contained in:
2025-07-22 13:50:14 +03:00
parent 849feb7beb
commit b98123f4dc
1479 changed files with 323549 additions and 11 deletions

View File

@@ -0,0 +1,22 @@
"""Utilities for asyncio-friendly file handling."""
from .threadpool import (
open,
stdin,
stdout,
stderr,
stdin_bytes,
stdout_bytes,
stderr_bytes,
)
from . import tempfile
__all__ = [
"open",
"tempfile",
"stdin",
"stdout",
"stderr",
"stdin_bytes",
"stdout_bytes",
"stderr_bytes",
]

View File

@@ -0,0 +1,69 @@
"""Various base classes."""
from collections.abc import Awaitable
from contextlib import AbstractAsyncContextManager
from asyncio import get_running_loop
class AsyncBase:
def __init__(self, file, loop, executor):
self._file = file
self._executor = executor
self._ref_loop = loop
@property
def _loop(self):
return self._ref_loop or get_running_loop()
def __aiter__(self):
"""We are our own iterator."""
return self
def __repr__(self):
return super().__repr__() + " wrapping " + repr(self._file)
async def __anext__(self):
"""Simulate normal file iteration."""
line = await self.readline()
if line:
return line
else:
raise StopAsyncIteration
class AsyncIndirectBase(AsyncBase):
def __init__(self, name, loop, executor, indirect):
self._indirect = indirect
self._name = name
super().__init__(None, loop, executor)
@property
def _file(self):
return self._indirect()
@_file.setter
def _file(self, v):
pass # discard writes
class AiofilesContextManager(Awaitable, AbstractAsyncContextManager):
"""An adjusted async context manager for aiofiles."""
__slots__ = ("_coro", "_obj")
def __init__(self, coro):
self._coro = coro
self._obj = None
def __await__(self):
if self._obj is None:
self._obj = yield from self._coro.__await__()
return self._obj
async def __aenter__(self):
return await self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await get_running_loop().run_in_executor(
None, self._obj._file.__exit__, exc_type, exc_val, exc_tb
)
self._obj = None

View File

@@ -0,0 +1,58 @@
"""Async executor versions of file functions from the os module."""
import os
from . import ospath as path
from .ospath import wrap
__all__ = [
"path",
"stat",
"rename",
"renames",
"replace",
"remove",
"unlink",
"mkdir",
"makedirs",
"rmdir",
"removedirs",
"symlink",
"readlink",
"listdir",
"scandir",
"access",
"wrap",
"getcwd",
]
if hasattr(os, "link"):
__all__ += ["link"]
if hasattr(os, "sendfile"):
__all__ += ["sendfile"]
if hasattr(os, "statvfs"):
__all__ += ["statvfs"]
stat = wrap(os.stat)
rename = wrap(os.rename)
renames = wrap(os.renames)
replace = wrap(os.replace)
remove = wrap(os.remove)
unlink = wrap(os.unlink)
mkdir = wrap(os.mkdir)
makedirs = wrap(os.makedirs)
rmdir = wrap(os.rmdir)
removedirs = wrap(os.removedirs)
symlink = wrap(os.symlink)
readlink = wrap(os.readlink)
listdir = wrap(os.listdir)
scandir = wrap(os.scandir)
access = wrap(os.access)
getcwd = wrap(os.getcwd)
if hasattr(os, "link"):
link = wrap(os.link)
if hasattr(os, "sendfile"):
sendfile = wrap(os.sendfile)
if hasattr(os, "statvfs"):
statvfs = wrap(os.statvfs)

View File

@@ -0,0 +1,30 @@
"""Async executor versions of file functions from the os.path module."""
import asyncio
from functools import partial, wraps
from os import path
def wrap(func):
@wraps(func)
async def run(*args, loop=None, executor=None, **kwargs):
if loop is None:
loop = asyncio.get_running_loop()
pfunc = partial(func, *args, **kwargs)
return await loop.run_in_executor(executor, pfunc)
return run
exists = wrap(path.exists)
isfile = wrap(path.isfile)
isdir = wrap(path.isdir)
islink = wrap(path.islink)
ismount = wrap(path.ismount)
getsize = wrap(path.getsize)
getmtime = wrap(path.getmtime)
getatime = wrap(path.getatime)
getctime = wrap(path.getctime)
samefile = wrap(path.samefile)
sameopenfile = wrap(path.sameopenfile)
abspath = wrap(path.abspath)

View File

@@ -0,0 +1,357 @@
import asyncio
from functools import partial, singledispatch
from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOBase
from tempfile import NamedTemporaryFile as syncNamedTemporaryFile
from tempfile import SpooledTemporaryFile as syncSpooledTemporaryFile
from tempfile import TemporaryDirectory as syncTemporaryDirectory
from tempfile import TemporaryFile as syncTemporaryFile
from tempfile import _TemporaryFileWrapper as syncTemporaryFileWrapper
from ..base import AiofilesContextManager
from ..threadpool.binary import AsyncBufferedIOBase, AsyncBufferedReader, AsyncFileIO
from ..threadpool.text import AsyncTextIOWrapper
from .temptypes import AsyncSpooledTemporaryFile, AsyncTemporaryDirectory
import sys
__all__ = [
"NamedTemporaryFile",
"TemporaryFile",
"SpooledTemporaryFile",
"TemporaryDirectory",
]
# ================================================================
# Public methods for async open and return of temp file/directory
# objects with async interface
# ================================================================
if sys.version_info >= (3, 12):
def NamedTemporaryFile(
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
delete=True,
delete_on_close=True,
loop=None,
executor=None,
):
"""Async open a named temporary file"""
return AiofilesContextManager(
_temporary_file(
named=True,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
delete=delete,
delete_on_close=delete_on_close,
loop=loop,
executor=executor,
)
)
else:
def NamedTemporaryFile(
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
delete=True,
loop=None,
executor=None,
):
"""Async open a named temporary file"""
return AiofilesContextManager(
_temporary_file(
named=True,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
delete=delete,
loop=loop,
executor=executor,
)
)
def TemporaryFile(
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
loop=None,
executor=None,
):
"""Async open an unnamed temporary file"""
return AiofilesContextManager(
_temporary_file(
named=False,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
loop=loop,
executor=executor,
)
)
def SpooledTemporaryFile(
max_size=0,
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
loop=None,
executor=None,
):
"""Async open a spooled temporary file"""
return AiofilesContextManager(
_spooled_temporary_file(
max_size=max_size,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
loop=loop,
executor=executor,
)
)
def TemporaryDirectory(suffix=None, prefix=None, dir=None, loop=None, executor=None):
"""Async open a temporary directory"""
return AiofilesContextManagerTempDir(
_temporary_directory(
suffix=suffix, prefix=prefix, dir=dir, loop=loop, executor=executor
)
)
# =========================================================
# Internal coroutines to open new temp files/directories
# =========================================================
if sys.version_info >= (3, 12):
async def _temporary_file(
named=True,
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
delete=True,
delete_on_close=True,
loop=None,
executor=None,
max_size=0,
):
"""Async method to open a temporary file with async interface"""
if loop is None:
loop = asyncio.get_running_loop()
if named:
cb = partial(
syncNamedTemporaryFile,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
delete=delete,
delete_on_close=delete_on_close,
)
else:
cb = partial(
syncTemporaryFile,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
)
f = await loop.run_in_executor(executor, cb)
# Wrap based on type of underlying IO object
if type(f) is syncTemporaryFileWrapper:
# _TemporaryFileWrapper was used (named files)
result = wrap(f.file, f, loop=loop, executor=executor)
result._closer = f._closer
return result
else:
# IO object was returned directly without wrapper
return wrap(f, f, loop=loop, executor=executor)
else:
async def _temporary_file(
named=True,
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
delete=True,
loop=None,
executor=None,
max_size=0,
):
"""Async method to open a temporary file with async interface"""
if loop is None:
loop = asyncio.get_running_loop()
if named:
cb = partial(
syncNamedTemporaryFile,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
delete=delete,
)
else:
cb = partial(
syncTemporaryFile,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
)
f = await loop.run_in_executor(executor, cb)
# Wrap based on type of underlying IO object
if type(f) is syncTemporaryFileWrapper:
# _TemporaryFileWrapper was used (named files)
result = wrap(f.file, f, loop=loop, executor=executor)
# add delete property
result.delete = f.delete
return result
else:
# IO object was returned directly without wrapper
return wrap(f, f, loop=loop, executor=executor)
async def _spooled_temporary_file(
max_size=0,
mode="w+b",
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
loop=None,
executor=None,
):
"""Open a spooled temporary file with async interface"""
if loop is None:
loop = asyncio.get_running_loop()
cb = partial(
syncSpooledTemporaryFile,
max_size=max_size,
mode=mode,
buffering=buffering,
encoding=encoding,
newline=newline,
suffix=suffix,
prefix=prefix,
dir=dir,
)
f = await loop.run_in_executor(executor, cb)
# Single interface provided by SpooledTemporaryFile for all modes
return AsyncSpooledTemporaryFile(f, loop=loop, executor=executor)
async def _temporary_directory(
suffix=None, prefix=None, dir=None, loop=None, executor=None
):
"""Async method to open a temporary directory with async interface"""
if loop is None:
loop = asyncio.get_running_loop()
cb = partial(syncTemporaryDirectory, suffix, prefix, dir)
f = await loop.run_in_executor(executor, cb)
return AsyncTemporaryDirectory(f, loop=loop, executor=executor)
class AiofilesContextManagerTempDir(AiofilesContextManager):
"""With returns the directory location, not the object (matching sync lib)"""
async def __aenter__(self):
self._obj = await self._coro
return self._obj.name
@singledispatch
def wrap(base_io_obj, file, *, loop=None, executor=None):
"""Wrap the object with interface based on type of underlying IO"""
raise TypeError("Unsupported IO type: {}".format(base_io_obj))
@wrap.register(TextIOBase)
def _(base_io_obj, file, *, loop=None, executor=None):
return AsyncTextIOWrapper(file, loop=loop, executor=executor)
@wrap.register(BufferedWriter)
def _(base_io_obj, file, *, loop=None, executor=None):
return AsyncBufferedIOBase(file, loop=loop, executor=executor)
@wrap.register(BufferedReader)
@wrap.register(BufferedRandom)
def _(base_io_obj, file, *, loop=None, executor=None):
return AsyncBufferedReader(file, loop=loop, executor=executor)
@wrap.register(FileIO)
def _(base_io_obj, file, *, loop=None, executor=None):
return AsyncFileIO(file, loop=loop, executor=executor)

View File

@@ -0,0 +1,69 @@
"""Async wrappers for spooled temp files and temp directory objects"""
from functools import partial
from ..base import AsyncBase
from ..threadpool.utils import (
cond_delegate_to_executor,
delegate_to_executor,
proxy_property_directly,
)
@delegate_to_executor("fileno", "rollover")
@cond_delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readline",
"readlines",
"seek",
"tell",
"truncate",
)
@proxy_property_directly("closed", "encoding", "mode", "name", "newlines")
class AsyncSpooledTemporaryFile(AsyncBase):
"""Async wrapper for SpooledTemporaryFile class"""
async def _check(self):
if self._file._rolled:
return
max_size = self._file._max_size
if max_size and self._file.tell() > max_size:
await self.rollover()
async def write(self, s):
"""Implementation to anticipate rollover"""
if self._file._rolled:
cb = partial(self._file.write, s)
return await self._loop.run_in_executor(self._executor, cb)
else:
file = self._file._file # reference underlying base IO object
rv = file.write(s)
await self._check()
return rv
async def writelines(self, iterable):
"""Implementation to anticipate rollover"""
if self._file._rolled:
cb = partial(self._file.writelines, iterable)
return await self._loop.run_in_executor(self._executor, cb)
else:
file = self._file._file # reference underlying base IO object
rv = file.writelines(iterable)
await self._check()
return rv
@delegate_to_executor("cleanup")
@proxy_property_directly("name")
class AsyncTemporaryDirectory:
"""Async wrapper for TemporaryDirectory class"""
def __init__(self, file, loop, executor):
self._file = file
self._loop = loop
self._executor = executor
async def close(self):
await self.cleanup()

View File

@@ -0,0 +1,139 @@
"""Handle files using a thread pool executor."""
import asyncio
import sys
from functools import partial, singledispatch
from io import (
BufferedIOBase,
BufferedRandom,
BufferedReader,
BufferedWriter,
FileIO,
TextIOBase,
)
from ..base import AiofilesContextManager
from .binary import (
AsyncBufferedIOBase,
AsyncBufferedReader,
AsyncFileIO,
AsyncIndirectBufferedIOBase,
)
from .text import AsyncTextIndirectIOWrapper, AsyncTextIOWrapper
sync_open = open
__all__ = (
"open",
"stdin",
"stdout",
"stderr",
"stdin_bytes",
"stdout_bytes",
"stderr_bytes",
)
def open(
file,
mode="r",
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None,
*,
loop=None,
executor=None,
):
return AiofilesContextManager(
_open(
file,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
closefd=closefd,
opener=opener,
loop=loop,
executor=executor,
)
)
async def _open(
file,
mode="r",
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None,
*,
loop=None,
executor=None,
):
"""Open an asyncio file."""
if loop is None:
loop = asyncio.get_running_loop()
cb = partial(
sync_open,
file,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
closefd=closefd,
opener=opener,
)
f = await loop.run_in_executor(executor, cb)
return wrap(f, loop=loop, executor=executor)
@singledispatch
def wrap(file, *, loop=None, executor=None):
raise TypeError("Unsupported io type: {}.".format(file))
@wrap.register(TextIOBase)
def _(file, *, loop=None, executor=None):
return AsyncTextIOWrapper(file, loop=loop, executor=executor)
@wrap.register(BufferedWriter)
@wrap.register(BufferedIOBase)
def _(file, *, loop=None, executor=None):
return AsyncBufferedIOBase(file, loop=loop, executor=executor)
@wrap.register(BufferedReader)
@wrap.register(BufferedRandom)
def _(file, *, loop=None, executor=None):
return AsyncBufferedReader(file, loop=loop, executor=executor)
@wrap.register(FileIO)
def _(file, *, loop=None, executor=None):
return AsyncFileIO(file, loop=loop, executor=executor)
stdin = AsyncTextIndirectIOWrapper("sys.stdin", None, None, indirect=lambda: sys.stdin)
stdout = AsyncTextIndirectIOWrapper(
"sys.stdout", None, None, indirect=lambda: sys.stdout
)
stderr = AsyncTextIndirectIOWrapper(
"sys.stderr", None, None, indirect=lambda: sys.stderr
)
stdin_bytes = AsyncIndirectBufferedIOBase(
"sys.stdin.buffer", None, None, indirect=lambda: sys.stdin.buffer
)
stdout_bytes = AsyncIndirectBufferedIOBase(
"sys.stdout.buffer", None, None, indirect=lambda: sys.stdout.buffer
)
stderr_bytes = AsyncIndirectBufferedIOBase(
"sys.stderr.buffer", None, None, indirect=lambda: sys.stderr.buffer
)

View File

@@ -0,0 +1,104 @@
from ..base import AsyncBase, AsyncIndirectBase
from .utils import delegate_to_executor, proxy_method_directly, proxy_property_directly
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"read1",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly("closed", "raw", "name", "mode")
class AsyncBufferedIOBase(AsyncBase):
"""The asyncio executor version of io.BufferedWriter and BufferedIOBase."""
@delegate_to_executor("peek")
class AsyncBufferedReader(AsyncBufferedIOBase):
"""The asyncio executor version of io.BufferedReader and Random."""
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readall",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("fileno", "readable")
@proxy_property_directly("closed", "name", "mode")
class AsyncFileIO(AsyncBase):
"""The asyncio executor version of io.FileIO."""
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"read1",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly("closed", "raw", "name", "mode")
class AsyncIndirectBufferedIOBase(AsyncIndirectBase):
"""The indirect asyncio executor version of io.BufferedWriter and BufferedIOBase."""
@delegate_to_executor("peek")
class AsyncIndirectBufferedReader(AsyncIndirectBufferedIOBase):
"""The indirect asyncio executor version of io.BufferedReader and Random."""
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readall",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("fileno", "readable")
@proxy_property_directly("closed", "name", "mode")
class AsyncIndirectFileIO(AsyncIndirectBase):
"""The indirect asyncio executor version of io.FileIO."""

View File

@@ -0,0 +1,64 @@
from ..base import AsyncBase, AsyncIndirectBase
from .utils import delegate_to_executor, proxy_method_directly, proxy_property_directly
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readable",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"write",
"writable",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly(
"buffer",
"closed",
"encoding",
"errors",
"line_buffering",
"newlines",
"name",
"mode",
)
class AsyncTextIOWrapper(AsyncBase):
"""The asyncio executor version of io.TextIOWrapper."""
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readable",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"write",
"writable",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly(
"buffer",
"closed",
"encoding",
"errors",
"line_buffering",
"newlines",
"name",
"mode",
)
class AsyncTextIndirectIOWrapper(AsyncIndirectBase):
"""The indirect asyncio executor version of io.TextIOWrapper."""

View File

@@ -0,0 +1,72 @@
import functools
def delegate_to_executor(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_delegate_method(attr_name))
return cls
return cls_builder
def proxy_method_directly(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_proxy_method(attr_name))
return cls
return cls_builder
def proxy_property_directly(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_proxy_property(attr_name))
return cls
return cls_builder
def cond_delegate_to_executor(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_cond_delegate_method(attr_name))
return cls
return cls_builder
def _make_delegate_method(attr_name):
async def method(self, *args, **kwargs):
cb = functools.partial(getattr(self._file, attr_name), *args, **kwargs)
return await self._loop.run_in_executor(self._executor, cb)
return method
def _make_proxy_method(attr_name):
def method(self, *args, **kwargs):
return getattr(self._file, attr_name)(*args, **kwargs)
return method
def _make_proxy_property(attr_name):
def proxy_property(self):
return getattr(self._file, attr_name)
return property(proxy_property)
def _make_cond_delegate_method(attr_name):
"""For spooled temp files, delegate only if rolled to file object"""
async def method(self, *args, **kwargs):
if self._file._rolled:
cb = functools.partial(getattr(self._file, attr_name), *args, **kwargs)
return await self._loop.run_in_executor(self._executor, cb)
else:
return getattr(self._file, attr_name)(*args, **kwargs)
return method