Агент: скачивание байтов из Jira без чужих перенаправлений, файл хабу (#43)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
0d72369875
commit
711f62e1df
71
src/agent.py
71
src/agent.py
@ -118,6 +118,35 @@ class HttpError(Exception):
|
||||
self.code, self.text, self.headers = code, text, headers or {}
|
||||
|
||||
|
||||
class RedirectAway(Exception):
|
||||
"""Сервер перенаправил запрос на другой адрес (схема, хост или порт): учётные данные туда не отправляются."""
|
||||
def __init__(self, host: str):
|
||||
super().__init__("перенаправление на %s" % host)
|
||||
self.host = host
|
||||
|
||||
|
||||
class TooBig(Exception):
|
||||
"""Ответ длиннее предела: чтение оборвано."""
|
||||
|
||||
|
||||
def _origin(url: str) -> Tuple[str, str, Optional[int]]:
|
||||
u = urllib.parse.urlsplit(url)
|
||||
scheme = (u.scheme or "").lower()
|
||||
return scheme, (u.hostname or "").lower(), u.port or {"http": 80, "https": 443}.get(scheme)
|
||||
|
||||
|
||||
class _SameOriginRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Перенаправления — только в пределах адреса клиента: urllib переносит Authorization и на чужой хост."""
|
||||
def __init__(self, base: str):
|
||||
self.origin = _origin(base)
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
if _origin(newurl) != self.origin:
|
||||
fp.close()
|
||||
raise RedirectAway(urllib.parse.urlsplit(newurl).hostname or newurl)
|
||||
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
||||
|
||||
|
||||
class Http:
|
||||
def __init__(self, base: str, headers: Dict[str, str], timeout: float = 60, ctx=None):
|
||||
self.base, self.headers, self.timeout, self.ctx = base.rstrip("/"), headers, timeout, ctx
|
||||
@ -164,6 +193,40 @@ class Http:
|
||||
except ValueError as e: # включая UnicodeDecodeError и JSONDecodeError
|
||||
raise HttpError(status, "ответ не JSON: %s; начало: %s" % (e, raw[:200].decode("utf-8", "replace"))) from None
|
||||
|
||||
def fetch_bytes(self, path: str, limit: int, timeout=None) -> bytes:
|
||||
"""GET с ответом-байтами (содержимое вложения): без разбора JSON, читает не больше limit + 1 байт — длиннее
|
||||
limit — TooBig. Перенаправление на другой адрес — RedirectAway без перехода. Ошибки — как у send."""
|
||||
req = urllib.request.Request(self.base + path, method="GET")
|
||||
for k, v in self.headers.items():
|
||||
req.add_header(k, v)
|
||||
req.add_header("Accept", "*/*")
|
||||
handlers = [_SameOriginRedirect(self.base)] # type: List[Any]
|
||||
if self.ctx is not None:
|
||||
handlers.append(urllib.request.HTTPSHandler(context=self.ctx))
|
||||
opener = urllib.request.build_opener(*handlers)
|
||||
try:
|
||||
with opener.open(req, timeout=timeout or self.timeout) as resp:
|
||||
data = resp.read(limit + 1)
|
||||
except RedirectAway:
|
||||
self._touch()
|
||||
raise
|
||||
except urllib.error.HTTPError as e:
|
||||
self._touch()
|
||||
try:
|
||||
headers = {k.lower(): v for k, v in e.headers.items()}
|
||||
text = e.read().decode("utf-8", "replace")
|
||||
except (OSError, ConnectionError, TimeoutError, http.client.HTTPException) as err:
|
||||
raise ConnectionError("%s: обрыв при чтении ответа: %s" % (self.base, err)) from None
|
||||
finally:
|
||||
e.close()
|
||||
raise HttpError(e.code, text, headers) from None
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError, OSError, http.client.HTTPException) as e:
|
||||
raise ConnectionError("%s недоступен: %s" % (self.base, e)) from None
|
||||
self._touch()
|
||||
if len(data) > limit:
|
||||
raise TooBig("ответ длиннее %d байт" % limit)
|
||||
return data
|
||||
|
||||
def _touch(self) -> None:
|
||||
"""Вызывается на каждый полученный HTTP-ответ (любой код)."""
|
||||
|
||||
@ -236,6 +299,10 @@ class JiraClient(Http):
|
||||
"multipart/form-data; boundary=" + boundary,
|
||||
timeout=max(self.timeout, 120)))
|
||||
|
||||
def download(self, path: str, limit: int) -> bytes:
|
||||
"""Содержимое вложения (байты), таймаут не меньше 120 с, 429/503 — повтор как у call. Не логируется."""
|
||||
return self._retry(lambda: self.fetch_bytes(path, limit, max(self.timeout, 120)))
|
||||
|
||||
def get(self, path, **params):
|
||||
return self.call("GET", path, params=params or None)
|
||||
|
||||
@ -370,6 +437,10 @@ class HubClient(Http):
|
||||
"""Файл вложения из хранилища хаба: {file, size, content (base64)}; нет файла — HttpError 404 no_file."""
|
||||
return self.call("GET", "/agent/files/%s" % sha256)
|
||||
|
||||
def upload_file(self, data: bytes) -> dict:
|
||||
"""Файл вложения, скачанный из Jira, — в хранилище хаба: {file (sha256), size}."""
|
||||
return self.call("POST", "/agent/files", {"content": base64.b64encode(data).decode("ascii")})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Журнал: только дозапись; started → finished → acked по id команды
|
||||
|
||||
@ -32,6 +32,8 @@ class FakeHub:
|
||||
self.histories = [] # тела POST /agent/history
|
||||
self.reject_history = None # (код, тело) — ответ на /agent/history ошибкой; "drop" — оборвать без ответа
|
||||
self.posts = [] # порядок POST /agent/state и /agent/history
|
||||
self.uploaded = [] # содержимое, присланное агентом POST /agent/files
|
||||
self.reject_files = None # (код, тело) — ответ на POST /agent/files ошибкой; "drop" — оборвать без ответа
|
||||
self.lock = threading.Lock()
|
||||
hub = self
|
||||
|
||||
@ -85,6 +87,10 @@ class FakeHub:
|
||||
if h.headers.get("Authorization") != "Bearer " + self.token:
|
||||
return self._send(h, 401, {"error": "unauthorized"})
|
||||
path = h.path.split("?")[0]
|
||||
if path == "/agent/files" and method == "POST" and self.reject_files == "drop":
|
||||
h.close_connection = True
|
||||
h.connection.shutdown(socket.SHUT_RDWR)
|
||||
return
|
||||
if path == "/agent/history" and self.reject_history == "drop":
|
||||
h.close_connection = True
|
||||
h.connection.shutdown(socket.SHUT_RDWR) # хаб недоступен: ответа клиент не получит
|
||||
@ -141,6 +147,12 @@ class FakeHub:
|
||||
self.posts.append(path)
|
||||
self.states.append(body)
|
||||
return 200, {"issues": len(body.get("issues") or []), "links": len(body.get("links") or [])}
|
||||
if path == "/agent/files" and method == "POST":
|
||||
if self.reject_files:
|
||||
return self.reject_files
|
||||
data = base64.b64decode(body.get("content") or "", validate=True)
|
||||
self.uploaded.append(data)
|
||||
return 200, {"file": self.put_file(data), "size": len(data)}
|
||||
m = re.match(r"^/agent/files/([^/]+)$", path)
|
||||
if m and method == "GET":
|
||||
self.file_requests += 1
|
||||
|
||||
@ -87,6 +87,9 @@ class FakeJira:
|
||||
self.bad_attachment_response = False # ответить на загрузку пустым списком
|
||||
self.hide_size = set() # id вложений без size в JSON (Jira не отдала размер)
|
||||
self.hide_attachment_field = False # поле attachment скрыто настройками Jira
|
||||
self.content_gets = 0 # GET /secure/attachment/<id>/<имя> — запросы содержимого
|
||||
self.redirect_content = None # адрес для 302 на запрос содержимого (абсолютный или путь этого сервера)
|
||||
self.short_content = set() # id вложений, содержимое которых отдаётся без последнего байта
|
||||
self.last_headers = {} # заголовки текущего запроса (имена в нижнем регистре)
|
||||
self.search_comment_cap = None # сколько комментариев задачи отдаёт search (None — все), как обрезка Jira
|
||||
self.changelogs = {} # ключ → записи журнала Jira [{id, author, created, items}]
|
||||
@ -260,6 +263,8 @@ class FakeJira:
|
||||
code, payload = 404, {"errorMessages": ["Issue Does Not Exist: %s" % e]}
|
||||
except Exception as e: # некорректный запрос к фейку
|
||||
code, payload = 400, {"errorMessages": ["фейк: %s: %s" % (type(e).__name__, e)]}
|
||||
if method == "GET" and (url.path.startswith("/secure/attachment/") or url.path.startswith("/secure/moved/")):
|
||||
return self._content(h, url.path)
|
||||
if self.drop_next_post_issue and method == "POST" and url.path == "/rest/api/2/issue":
|
||||
self.drop_next_post_issue = False
|
||||
h.close_connection = True
|
||||
@ -272,6 +277,34 @@ class FakeJira:
|
||||
return
|
||||
self._send(h, code, payload)
|
||||
|
||||
def _content(self, h, path):
|
||||
"""Содержимое вложения, как /secure/attachment/<id>/<имя> Jira: байты, не JSON. /secure/moved/… — цель 302."""
|
||||
with self.lock:
|
||||
m = re.match(r"^/secure/(attachment|moved)/(\d+)/(.+)$", path)
|
||||
found = None
|
||||
if m:
|
||||
for lst in self.attachments.values():
|
||||
for a in lst:
|
||||
if a["id"] == m.group(2) and a["filename"] == unquote(m.group(3)):
|
||||
found = a
|
||||
redirect = self.redirect_content if m and m.group(1) == "attachment" else None
|
||||
if found is not None and redirect is None:
|
||||
self.content_gets += 1
|
||||
if redirect is not None:
|
||||
h.send_response(302)
|
||||
h.send_header("Location", redirect)
|
||||
h.send_header("Content-Length", "0")
|
||||
h.end_headers()
|
||||
return
|
||||
if found is None:
|
||||
return self._send(h, 404, {"errorMessages": ["нет вложения"]})
|
||||
data = found["data"][:-1] if found["id"] in self.short_content else found["data"]
|
||||
h.send_response(200)
|
||||
h.send_header("Content-Type", found["mimeType"])
|
||||
h.send_header("Content-Length", str(len(data)))
|
||||
h.end_headers()
|
||||
h.wfile.write(data)
|
||||
|
||||
def _send(self, h, code, payload, headers=None):
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else b""
|
||||
h.send_response(code)
|
||||
|
||||
@ -90,6 +90,49 @@ class ClientsTest(unittest.TestCase):
|
||||
self.jira.upload("AURORA-1", "a.txt", b"abc")
|
||||
self.assertEqual(seen, [120]) # timeout в конфиге тестов — 5 с
|
||||
|
||||
def _content_path(self, a):
|
||||
return "/secure/attachment/%s/%s" % (a["id"], urllib.parse.quote(a["filename"], safe=""))
|
||||
|
||||
def test_download_bytes(self):
|
||||
self.fake.add("AURORA-1")
|
||||
a = self.fake.add_attachment("AURORA-1", "скрин 1.png", b"\x89PNG\x00\x01", mime="image/png")
|
||||
data = self.jira.download(self._content_path(a), 100)
|
||||
self.assertEqual(data, b"\x89PNG\x00\x01")
|
||||
self.assertEqual(self.jira.calls, 1)
|
||||
self.assertTrue(self.fake.last_headers.get("authorization", "").startswith("Basic "))
|
||||
self.assertEqual(self.fake.content_gets, 1)
|
||||
|
||||
def test_download_redirect_same_host(self):
|
||||
self.fake.add("AURORA-1")
|
||||
a = self.fake.add_attachment("AURORA-1", "a.bin", b"abc")
|
||||
self.fake.redirect_content = "%s/secure/moved/%s/a.bin" % (self.fake.base_url, a["id"])
|
||||
self.assertEqual(self.jira.download(self._content_path(a), 100), b"abc")
|
||||
|
||||
def test_download_redirect_other_host(self):
|
||||
self.fake.add("AURORA-1")
|
||||
a = self.fake.add_attachment("AURORA-1", "a.bin", b"abc")
|
||||
port = self.fake.server.server_port
|
||||
self.fake.redirect_content = "http://localhost:%d/secure/moved/%s/a.bin" % (port, a["id"])
|
||||
with self.assertRaises(agent.RedirectAway) as e:
|
||||
self.jira.download(self._content_path(a), 100)
|
||||
self.assertEqual(e.exception.host, "localhost")
|
||||
self.assertEqual(self.fake.content_gets, 0)
|
||||
self.assertFalse(any(r[1].startswith("/secure/moved/") for r in self.fake.requests))
|
||||
|
||||
def test_download_limit(self):
|
||||
self.fake.add("AURORA-1")
|
||||
a = self.fake.add_attachment("AURORA-1", "a.bin", b"x" * 20)
|
||||
with self.assertRaises(agent.TooBig):
|
||||
self.jira.download(self._content_path(a), 10)
|
||||
self.assertEqual(self.jira.download(self._content_path(a), 20), b"x" * 20)
|
||||
|
||||
def test_download_429_retry(self):
|
||||
self.fake.add("AURORA-1")
|
||||
a = self.fake.add_attachment("AURORA-1", "a.bin", b"abc")
|
||||
self.fake.fail_next.append((429, {"Retry-After": "0"}))
|
||||
self.assertEqual(self.jira.download(self._content_path(a), 100), b"abc")
|
||||
self.assertEqual(self.jira.rate_limited, 1)
|
||||
|
||||
def test_retry_on_429_with_retry_after(self):
|
||||
self.fake.add("AURORA-1")
|
||||
self.fake.fail_next.append((429, {"Retry-After": "0"}))
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
@ -148,6 +149,19 @@ class FolderTransportTest(unittest.TestCase):
|
||||
self.assertEqual(self.relay.requests[0]["path"], "/agent/files/" + sha)
|
||||
self.assertEqual(self.relay.requests[0]["method"], "GET")
|
||||
|
||||
def test_upload_file_through_folder(self):
|
||||
self.run_relay()
|
||||
data = os.urandom(300 * 1024)
|
||||
got = self.client().upload_file(data)
|
||||
self.assertEqual(got, {"file": hashlib.sha256(data).hexdigest(), "size": len(data)})
|
||||
self.assertEqual(self.hub.uploaded, [data])
|
||||
self.assertEqual(self.relay.requests[0]["path"], "/agent/files")
|
||||
|
||||
def test_upload_file_over_http(self):
|
||||
hub = agent.HubClient({"hub_url": self.hub.base_url, "hub_token": self.hub.token, "timeout": 5})
|
||||
self.assertEqual(hub.upload_file(b""), {"file": hashlib.sha256(b"").hexdigest(), "size": 0})
|
||||
self.assertEqual(self.hub.uploaded, [b""])
|
||||
|
||||
def test_file_missing_through_folder(self):
|
||||
self.run_relay()
|
||||
with self.assertRaises(agent.HttpError) as e:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user