From 4c32cd36ac1c95e045c790afc39e333b1dfdcc01 Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sun, 4 Oct 2026 02:45:25 +0400 Subject: [PATCH] =?UTF-8?q?feat:=20S3Storage.get=5Fbytes/put=5Fjson=20?= =?UTF-8?q?=D0=B8=20=D0=BF=D1=80=D0=BE=D1=82=D0=BE=D0=BA=D0=BE=D0=BB=20Sto?= =?UTF-8?q?rage=20=D0=B2=D0=BC=D0=B5=D1=81=D1=82=D0=BE=20=D1=88=D0=B0?= =?UTF-8?q?=D0=B1=D0=BB=D0=BE=D0=BD=D0=BD=D1=8B=D1=85=20put=5Fobject/get?= =?UTF-8?q?=5Fobject?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- api/src/core/clients.py | 46 +++++++++----- api/tests/unit/test_clients.py | 108 +++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 api/tests/unit/test_clients.py diff --git a/api/src/core/clients.py b/api/src/core/clients.py index ceb87e5..80692b6 100644 --- a/api/src/core/clients.py +++ b/api/src/core/clients.py @@ -1,28 +1,44 @@ -"""Клиенты инфраструктуры. Каркас: оставлен S3Storage; outbox-паблишер (PollWorker) реализуется в Срезе 0 -по docs/context/03-architecture.md (референс — reference-old-arm/api/src/core/clients.py). +"""Клиент S3 (MinIO) сервиса: чтение исходника и запись файла результата (design D-4). + +`Storage` — протокол, от которого зависит воркер (design D-7): в unit-тестах его подменяет фейк, в приложении — +`S3Storage` поверх `miniopy_async.Minio`. Бакет передаётся в каждый вызов: исходник читается из `request.s3.bucket`, +результат пишется в тот же бакет. """ +import json from io import BytesIO +from typing import Any, Protocol, runtime_checkable from miniopy_async import Minio +JSON_CONTENT_TYPE = "application/json" + + +@runtime_checkable +class Storage(Protocol): + """Хранилище объектов, нужное воркеру: байты исходника и JSON результата.""" + + async def get_bytes(self, bucket: str, key: str) -> bytes: ... + + async def put_json(self, bucket: str, key: str, obj: Any) -> None: ... + class S3Storage: - def __init__(self, client: Minio, bucket_name: str): + """`Storage` поверх MinIO. `bucket` — бакет из настроек (проверяется при старте), методы принимают бакет явно.""" + + def __init__(self, client: Minio, bucket_name: str) -> None: self.client = client self.bucket = bucket_name - async def put_object(self, obj_name, file_data): + async def get_bytes(self, bucket: str, key: str) -> bytes: + """Содержимое объекта целиком; ошибки MinIO/сети пробрасываются (воркер превращает их в `FILE_READ_ERROR`).""" + response = await self.client.get_object(bucket, key) try: - stream = BytesIO(file_data) - await self.client.put_object(self.bucket, obj_name, stream, length=len(file_data)) - except: - raise + return await response.read() + finally: + response.release() - async def get_object(self, obj_name): - try: - response = await self.client.get_object(self.bucket, obj_name) - # file_bytes = await response.content.read() - return response - except: - raise + async def put_json(self, bucket: str, key: str, obj: Any) -> None: + """Записывает JSON-сериализуемый `obj` как UTF-8 (`application/json`, кириллица без экранирования).""" + body = json.dumps(obj, ensure_ascii=False).encode("utf-8") + await self.client.put_object(bucket, key, BytesIO(body), length=len(body), content_type=JSON_CONTENT_TYPE) diff --git a/api/tests/unit/test_clients.py b/api/tests/unit/test_clients.py new file mode 100644 index 0000000..2facb20 --- /dev/null +++ b/api/tests/unit/test_clients.py @@ -0,0 +1,108 @@ +"""Unit-тесты `S3Storage` (design D-4): чтение исходника байтами и запись JSON результата — с фейковым MinIO.""" + +import json +from typing import Any + +import pytest + +from src.core.clients import S3Storage, Storage + + +class _FakeResponse: + """Двойник `aiohttp.ClientResponse`: тело и признак освобождения соединения.""" + + def __init__(self, body: bytes) -> None: + self.body = body + self.released = False + + async def read(self) -> bytes: + return self.body + + def release(self) -> None: + self.released = True + + +class _FakeMinio: + """Двойник `miniopy_async.Minio`: объекты в словаре `(bucket, key) -> bytes`.""" + + def __init__(self) -> None: + self.objects: dict[tuple[str, str], bytes] = {} + self.put_calls: list[dict[str, Any]] = [] + self.responses: list[_FakeResponse] = [] + + async def get_object(self, bucket_name: str, object_name: str) -> _FakeResponse: + try: + body = self.objects[(bucket_name, object_name)] + except KeyError: + raise FileNotFoundError(f"{bucket_name}/{object_name}") from None + response = _FakeResponse(body) + self.responses.append(response) + return response + + async def put_object( + self, bucket_name: str, object_name: str, data: Any, length: int, content_type: str = "" + ) -> None: + body = data.read() + assert len(body) == length + self.objects[(bucket_name, object_name)] = body + self.put_calls.append( + {"bucket": bucket_name, "key": object_name, "length": length, "content_type": content_type} + ) + + +def _storage(client: _FakeMinio) -> S3Storage: + return S3Storage(client=client, bucket_name="arm-ka") # type: ignore[arg-type] + + +async def test_get_bytes_reads_object_and_releases_response(): + client = _FakeMinio() + client.objects[("arm-ka", "uploads/f/osv.xlsx")] = b"\x50\x4b\x03\x04 xlsx" + + data = await _storage(client).get_bytes("arm-ka", "uploads/f/osv.xlsx") + + assert data == b"\x50\x4b\x03\x04 xlsx" + assert client.responses[0].released + + +async def test_get_bytes_uses_bucket_argument(): + client = _FakeMinio() + client.objects[("other-bucket", "k")] = b"x" + + assert await _storage(client).get_bytes("other-bucket", "k") == b"x" + + +async def test_get_bytes_missing_object_raises(): + with pytest.raises(FileNotFoundError): + await _storage(_FakeMinio()).get_bytes("arm-ka", "uploads/нет.xlsx") + + +async def test_put_json_writes_utf8_json_with_content_type(): + client = _FakeMinio() + obj = {"job_id": "j-1", "сообщение": "разбор", "amount": "1.10"} + + await _storage(client).put_json("arm-ka", "results/f/j-1/result.json", obj) + + body = client.objects[("arm-ka", "results/f/j-1/result.json")] + assert json.loads(body.decode("utf-8")) == obj + assert "разбор".encode() in body # кириллица без \u-экранирования + assert client.put_calls == [ + { + "bucket": "arm-ka", + "key": "results/f/j-1/result.json", + "length": len(body), + "content_type": "application/json", + } + ] + + +async def test_put_json_then_get_bytes_round_trip(): + storage = _storage(_FakeMinio()) + + await storage.put_json("arm-ka", "results/x.json", {"a": [1, 2]}) + + assert json.loads(await storage.get_bytes("arm-ka", "results/x.json")) == {"a": [1, 2]} + + +def test_s3_storage_satisfies_storage_protocol(): + storage: Storage = _storage(_FakeMinio()) + assert isinstance(storage, Storage)