diff --git a/src/agent.py b/src/agent.py index e33c3b4..7a8d0fe 100644 --- a/src/agent.py +++ b/src/agent.py @@ -118,6 +118,35 @@ class HttpError(Exception): self.code, self.text, self.headers = code, text, headers or {} +class RedirectAway(Exception): + """Сервер перенаправил запрос на другой адрес (схема, хост или порт): учётные данные туда не отправляются.""" + def __init__(self, host: str): + super().__init__("перенаправление на %s" % host) + self.host = host + + +class TooBig(Exception): + """Ответ длиннее предела: чтение оборвано.""" + + +def _origin(url: str) -> Tuple[str, str, Optional[int]]: + u = urllib.parse.urlsplit(url) + scheme = (u.scheme or "").lower() + return scheme, (u.hostname or "").lower(), u.port or {"http": 80, "https": 443}.get(scheme) + + +class _SameOriginRedirect(urllib.request.HTTPRedirectHandler): + """Перенаправления — только в пределах адреса клиента: urllib переносит Authorization и на чужой хост.""" + def __init__(self, base: str): + self.origin = _origin(base) + + def redirect_request(self, req, fp, code, msg, headers, newurl): + if _origin(newurl) != self.origin: + fp.close() + raise RedirectAway(urllib.parse.urlsplit(newurl).hostname or newurl) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + class Http: def __init__(self, base: str, headers: Dict[str, str], timeout: float = 60, ctx=None): self.base, self.headers, self.timeout, self.ctx = base.rstrip("/"), headers, timeout, ctx @@ -164,6 +193,40 @@ class Http: except ValueError as e: # включая UnicodeDecodeError и JSONDecodeError raise HttpError(status, "ответ не JSON: %s; начало: %s" % (e, raw[:200].decode("utf-8", "replace"))) from None + def fetch_bytes(self, path: str, limit: int, timeout=None) -> bytes: + """GET с ответом-байтами (содержимое вложения): без разбора JSON, читает не больше limit + 1 байт — длиннее + limit — TooBig. Перенаправление на другой адрес — RedirectAway без перехода. Ошибки — как у send.""" + req = urllib.request.Request(self.base + path, method="GET") + for k, v in self.headers.items(): + req.add_header(k, v) + req.add_header("Accept", "*/*") + handlers = [_SameOriginRedirect(self.base)] # type: List[Any] + if self.ctx is not None: + handlers.append(urllib.request.HTTPSHandler(context=self.ctx)) + opener = urllib.request.build_opener(*handlers) + try: + with opener.open(req, timeout=timeout or self.timeout) as resp: + data = resp.read(limit + 1) + except RedirectAway: + self._touch() + raise + except urllib.error.HTTPError as e: + self._touch() + try: + headers = {k.lower(): v for k, v in e.headers.items()} + text = e.read().decode("utf-8", "replace") + except (OSError, ConnectionError, TimeoutError, http.client.HTTPException) as err: + raise ConnectionError("%s: обрыв при чтении ответа: %s" % (self.base, err)) from None + finally: + e.close() + raise HttpError(e.code, text, headers) from None + except (urllib.error.URLError, ConnectionError, TimeoutError, OSError, http.client.HTTPException) as e: + raise ConnectionError("%s недоступен: %s" % (self.base, e)) from None + self._touch() + if len(data) > limit: + raise TooBig("ответ длиннее %d байт" % limit) + return data + def _touch(self) -> None: """Вызывается на каждый полученный HTTP-ответ (любой код).""" @@ -236,6 +299,10 @@ class JiraClient(Http): "multipart/form-data; boundary=" + boundary, timeout=max(self.timeout, 120))) + def download(self, path: str, limit: int) -> bytes: + """Содержимое вложения (байты), таймаут не меньше 120 с, 429/503 — повтор как у call. Не логируется.""" + return self._retry(lambda: self.fetch_bytes(path, limit, max(self.timeout, 120))) + def get(self, path, **params): return self.call("GET", path, params=params or None) @@ -370,6 +437,10 @@ class HubClient(Http): """Файл вложения из хранилища хаба: {file, size, content (base64)}; нет файла — HttpError 404 no_file.""" return self.call("GET", "/agent/files/%s" % sha256) + def upload_file(self, data: bytes) -> dict: + """Файл вложения, скачанный из Jira, — в хранилище хаба: {file (sha256), size}.""" + return self.call("POST", "/agent/files", {"content": base64.b64encode(data).decode("ascii")}) + # --------------------------------------------------------------------------- # Журнал: только дозапись; started → finished → acked по id команды diff --git a/tests/fake_hub.py b/tests/fake_hub.py index ad9fb4d..85adeff 100644 --- a/tests/fake_hub.py +++ b/tests/fake_hub.py @@ -32,6 +32,8 @@ class FakeHub: self.histories = [] # тела POST /agent/history self.reject_history = None # (код, тело) — ответ на /agent/history ошибкой; "drop" — оборвать без ответа self.posts = [] # порядок POST /agent/state и /agent/history + self.uploaded = [] # содержимое, присланное агентом POST /agent/files + self.reject_files = None # (код, тело) — ответ на POST /agent/files ошибкой; "drop" — оборвать без ответа self.lock = threading.Lock() hub = self @@ -85,6 +87,10 @@ class FakeHub: if h.headers.get("Authorization") != "Bearer " + self.token: return self._send(h, 401, {"error": "unauthorized"}) path = h.path.split("?")[0] + if path == "/agent/files" and method == "POST" and self.reject_files == "drop": + h.close_connection = True + h.connection.shutdown(socket.SHUT_RDWR) + return if path == "/agent/history" and self.reject_history == "drop": h.close_connection = True h.connection.shutdown(socket.SHUT_RDWR) # хаб недоступен: ответа клиент не получит @@ -141,6 +147,12 @@ class FakeHub: self.posts.append(path) self.states.append(body) return 200, {"issues": len(body.get("issues") or []), "links": len(body.get("links") or [])} + if path == "/agent/files" and method == "POST": + if self.reject_files: + return self.reject_files + data = base64.b64decode(body.get("content") or "", validate=True) + self.uploaded.append(data) + return 200, {"file": self.put_file(data), "size": len(data)} m = re.match(r"^/agent/files/([^/]+)$", path) if m and method == "GET": self.file_requests += 1 diff --git a/tests/fake_jira.py b/tests/fake_jira.py index 38be673..c807bf2 100644 --- a/tests/fake_jira.py +++ b/tests/fake_jira.py @@ -87,6 +87,9 @@ class FakeJira: self.bad_attachment_response = False # ответить на загрузку пустым списком self.hide_size = set() # id вложений без size в JSON (Jira не отдала размер) self.hide_attachment_field = False # поле attachment скрыто настройками Jira + self.content_gets = 0 # GET /secure/attachment//<имя> — запросы содержимого + self.redirect_content = None # адрес для 302 на запрос содержимого (абсолютный или путь этого сервера) + self.short_content = set() # id вложений, содержимое которых отдаётся без последнего байта self.last_headers = {} # заголовки текущего запроса (имена в нижнем регистре) self.search_comment_cap = None # сколько комментариев задачи отдаёт search (None — все), как обрезка Jira self.changelogs = {} # ключ → записи журнала Jira [{id, author, created, items}] @@ -260,6 +263,8 @@ class FakeJira: code, payload = 404, {"errorMessages": ["Issue Does Not Exist: %s" % e]} except Exception as e: # некорректный запрос к фейку code, payload = 400, {"errorMessages": ["фейк: %s: %s" % (type(e).__name__, e)]} + if method == "GET" and (url.path.startswith("/secure/attachment/") or url.path.startswith("/secure/moved/")): + return self._content(h, url.path) if self.drop_next_post_issue and method == "POST" and url.path == "/rest/api/2/issue": self.drop_next_post_issue = False h.close_connection = True @@ -272,6 +277,34 @@ class FakeJira: return self._send(h, code, payload) + def _content(self, h, path): + """Содержимое вложения, как /secure/attachment//<имя> Jira: байты, не JSON. /secure/moved/… — цель 302.""" + with self.lock: + m = re.match(r"^/secure/(attachment|moved)/(\d+)/(.+)$", path) + found = None + if m: + for lst in self.attachments.values(): + for a in lst: + if a["id"] == m.group(2) and a["filename"] == unquote(m.group(3)): + found = a + redirect = self.redirect_content if m and m.group(1) == "attachment" else None + if found is not None and redirect is None: + self.content_gets += 1 + if redirect is not None: + h.send_response(302) + h.send_header("Location", redirect) + h.send_header("Content-Length", "0") + h.end_headers() + return + if found is None: + return self._send(h, 404, {"errorMessages": ["нет вложения"]}) + data = found["data"][:-1] if found["id"] in self.short_content else found["data"] + h.send_response(200) + h.send_header("Content-Type", found["mimeType"]) + h.send_header("Content-Length", str(len(data))) + h.end_headers() + h.wfile.write(data) + def _send(self, h, code, payload, headers=None): data = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else b"" h.send_response(code) diff --git a/tests/test_clients.py b/tests/test_clients.py index 568fd6f..8349677 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -90,6 +90,49 @@ class ClientsTest(unittest.TestCase): self.jira.upload("AURORA-1", "a.txt", b"abc") self.assertEqual(seen, [120]) # timeout в конфиге тестов — 5 с + def _content_path(self, a): + return "/secure/attachment/%s/%s" % (a["id"], urllib.parse.quote(a["filename"], safe="")) + + def test_download_bytes(self): + self.fake.add("AURORA-1") + a = self.fake.add_attachment("AURORA-1", "скрин 1.png", b"\x89PNG\x00\x01", mime="image/png") + data = self.jira.download(self._content_path(a), 100) + self.assertEqual(data, b"\x89PNG\x00\x01") + self.assertEqual(self.jira.calls, 1) + self.assertTrue(self.fake.last_headers.get("authorization", "").startswith("Basic ")) + self.assertEqual(self.fake.content_gets, 1) + + def test_download_redirect_same_host(self): + self.fake.add("AURORA-1") + a = self.fake.add_attachment("AURORA-1", "a.bin", b"abc") + self.fake.redirect_content = "%s/secure/moved/%s/a.bin" % (self.fake.base_url, a["id"]) + self.assertEqual(self.jira.download(self._content_path(a), 100), b"abc") + + def test_download_redirect_other_host(self): + self.fake.add("AURORA-1") + a = self.fake.add_attachment("AURORA-1", "a.bin", b"abc") + port = self.fake.server.server_port + self.fake.redirect_content = "http://localhost:%d/secure/moved/%s/a.bin" % (port, a["id"]) + with self.assertRaises(agent.RedirectAway) as e: + self.jira.download(self._content_path(a), 100) + self.assertEqual(e.exception.host, "localhost") + self.assertEqual(self.fake.content_gets, 0) + self.assertFalse(any(r[1].startswith("/secure/moved/") for r in self.fake.requests)) + + def test_download_limit(self): + self.fake.add("AURORA-1") + a = self.fake.add_attachment("AURORA-1", "a.bin", b"x" * 20) + with self.assertRaises(agent.TooBig): + self.jira.download(self._content_path(a), 10) + self.assertEqual(self.jira.download(self._content_path(a), 20), b"x" * 20) + + def test_download_429_retry(self): + self.fake.add("AURORA-1") + a = self.fake.add_attachment("AURORA-1", "a.bin", b"abc") + self.fake.fail_next.append((429, {"Retry-After": "0"})) + self.assertEqual(self.jira.download(self._content_path(a), 100), b"abc") + self.assertEqual(self.jira.rate_limited, 1) + def test_retry_on_429_with_retry_after(self): self.fake.add("AURORA-1") self.fake.fail_next.append((429, {"Retry-After": "0"})) diff --git a/tests/test_folder.py b/tests/test_folder.py index cc5f2ac..41a72f9 100644 --- a/tests/test_folder.py +++ b/tests/test_folder.py @@ -1,4 +1,5 @@ import base64 +import hashlib import io import json import os @@ -148,6 +149,19 @@ class FolderTransportTest(unittest.TestCase): self.assertEqual(self.relay.requests[0]["path"], "/agent/files/" + sha) self.assertEqual(self.relay.requests[0]["method"], "GET") + def test_upload_file_through_folder(self): + self.run_relay() + data = os.urandom(300 * 1024) + got = self.client().upload_file(data) + self.assertEqual(got, {"file": hashlib.sha256(data).hexdigest(), "size": len(data)}) + self.assertEqual(self.hub.uploaded, [data]) + self.assertEqual(self.relay.requests[0]["path"], "/agent/files") + + def test_upload_file_over_http(self): + hub = agent.HubClient({"hub_url": self.hub.base_url, "hub_token": self.hub.token, "timeout": 5}) + self.assertEqual(hub.upload_file(b""), {"file": hashlib.sha256(b"").hexdigest(), "size": 0}) + self.assertEqual(self.hub.uploaded, [b""]) + def test_file_missing_through_folder(self): self.run_relay() with self.assertRaises(agent.HttpError) as e: