Схема v5: issue_change, время комментариев в UTC (#70)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
ba7ac9fb41
commit
dc9419fbb9
@ -8,7 +8,7 @@ from pathlib import Path
|
||||
|
||||
from . import backup, paths
|
||||
|
||||
SCHEMA_VERSION = 4
|
||||
SCHEMA_VERSION = 5
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS setting (
|
||||
@ -50,8 +50,28 @@ CREATE TABLE IF NOT EXISTS issue_comment (
|
||||
created TEXT,
|
||||
updated TEXT,
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT,
|
||||
PRIMARY KEY (key, comment_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS issue_change (
|
||||
history_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
at TEXT,
|
||||
created TEXT NOT NULL,
|
||||
author TEXT,
|
||||
field TEXT NOT NULL,
|
||||
field_id TEXT,
|
||||
field_name TEXT NOT NULL,
|
||||
from_value TEXT,
|
||||
from_string TEXT,
|
||||
to_value TEXT,
|
||||
to_string TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY (history_id, seq)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_issue_change_at ON issue_change(at);
|
||||
CREATE INDEX IF NOT EXISTS ix_issue_change_key ON issue_change(key, at);
|
||||
CREATE TABLE IF NOT EXISTS issue_note (
|
||||
key TEXT PRIMARY KEY,
|
||||
fact_status TEXT,
|
||||
@ -99,7 +119,8 @@ CREATE TABLE IF NOT EXISTS sync_run (
|
||||
links_count INTEGER NOT NULL DEFAULT 0,
|
||||
jira_calls INTEGER NOT NULL DEFAULT 0,
|
||||
rate_limited INTEGER NOT NULL DEFAULT 0,
|
||||
command_id INTEGER
|
||||
command_id INTEGER,
|
||||
history_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS project (
|
||||
label TEXT PRIMARY KEY,
|
||||
@ -123,12 +144,28 @@ def _migrate_projects(conn) -> None:
|
||||
projects.migrate_v3(conn)
|
||||
|
||||
|
||||
def _add_column(table: str, column: str):
|
||||
"""Шаг миграции: добавить колонку, если её нет. Базы v1/v2 получают issue_comment из SCHEMA уже с новой колонкой."""
|
||||
def step(conn) -> None:
|
||||
if column not in {r["name"] for r in conn.execute(f"PRAGMA table_info({table})")}:
|
||||
conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} TEXT")
|
||||
return step
|
||||
|
||||
|
||||
def _fill_comment_utc(conn) -> None:
|
||||
for r in conn.execute("SELECT key, comment_id, created FROM issue_comment").fetchall():
|
||||
conn.execute("UPDATE issue_comment SET created_at = ? WHERE key = ? AND comment_id = ?",
|
||||
(jira_utc(r["created"]), r["key"], r["comment_id"]))
|
||||
|
||||
|
||||
# Шаги миграции: версия → SQL, переводящий схему в версию + 1 (design.md bridge-reliability, решение 5)
|
||||
MIGRATIONS = {
|
||||
1: ["ALTER TABLE command ADD COLUMN deliveries INTEGER NOT NULL DEFAULT 0",
|
||||
"UPDATE command SET deliveries = 1 WHERE status = 'claimed'"], # уже выданные — одна выдача
|
||||
2: [], # v2 → v3: issue_comment создаётся SCHEMA (CREATE TABLE IF NOT EXISTS), данных переносить не нужно
|
||||
3: ["ALTER TABLE batch ADD COLUMN project TEXT", _migrate_projects], # v3 → v4: проекты-метки
|
||||
4: [_add_column("issue_comment", "created_at"), _add_column("sync_run", "history_at"), # v4 → v5: журнал изменений
|
||||
_fill_comment_utc],
|
||||
}
|
||||
|
||||
|
||||
@ -217,6 +254,22 @@ def transaction(conn: sqlite3.Connection):
|
||||
conn.execute("COMMIT")
|
||||
|
||||
|
||||
def jira_utc(value) -> str | None:
|
||||
"""Время Jira (`2026-10-22T13:05:11.000+0300`, `+03:00`, `Z`, с дробной частью или без) → ISO-8601 UTC в формате
|
||||
now(); без смещения — UTC; неразбираемое — None."""
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
s = value[:-1] + "+00:00" if value.endswith("Z") else value
|
||||
if len(s) > 5 and s[-5] in "+-" and s[-4:].isdigit(): # ±HHMM → ±HH:MM
|
||||
s = s[:-2] + ":" + s[-2:]
|
||||
try:
|
||||
d = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
return None
|
||||
d = d if d.tzinfo else d.replace(tzinfo=timezone.utc)
|
||||
return d.astimezone(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0)
|
||||
|
||||
|
||||
@ -58,10 +58,10 @@ def apply_state(conn, issues, links, neighbor_keys=(), fetched_at=None, authorit
|
||||
if isinstance(it.get("comments"), list): # без поля (агент до v10) комментарии не трогаем
|
||||
conn.execute("DELETE FROM issue_comment WHERE key = ?", (it["key"],))
|
||||
conn.executemany(
|
||||
"INSERT OR REPLACE INTO issue_comment(key, comment_id, author, created, updated, body) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
[(it["key"], c["id"], c.get("author"), c.get("created"), c.get("updated"), c.get("body") or "")
|
||||
for c in it["comments"]])
|
||||
"INSERT OR REPLACE INTO issue_comment(key, comment_id, author, created, updated, body, created_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(it["key"], c["id"], c.get("author"), c.get("created"), c.get("updated"), c.get("body") or "",
|
||||
db.jira_utc(c.get("created"))) for c in it["comments"]])
|
||||
conn.executemany("DELETE FROM issue_link WHERE source_key = ?", [(k,) for k in keys])
|
||||
for link in links:
|
||||
conn.execute("INSERT OR IGNORE INTO issue_link(source_key, target_key, link_type) VALUES (?, ?, ?)",
|
||||
|
||||
@ -5,6 +5,8 @@ import sqlite3
|
||||
import stat
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from jira_hub import backup, db, paths, projects, queue, settings
|
||||
|
||||
|
||||
@ -12,14 +14,23 @@ def test_connect_creates_schema(open_db):
|
||||
conn = open_db()
|
||||
names = {r["name"] for r in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")}
|
||||
assert {"setting", "issue", "issue_link", "issue_comment", "issue_note", "batch", "command", "sync_run",
|
||||
"agent_status", "project"} <= names
|
||||
"agent_status", "project", "issue_change"} <= names
|
||||
assert "created_at" in {r["name"] for r in conn.execute("PRAGMA table_info(issue_comment)")}
|
||||
assert "history_at" in {r["name"] for r in conn.execute("PRAGMA table_info(sync_run)")}
|
||||
assert "project" in {r["name"] for r in conn.execute("PRAGMA table_info(batch)")}
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "4"
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
assert conn.execute("SELECT COUNT(*) AS n FROM agent_status").fetchone()["n"] == 1
|
||||
cols = {r["name"] for r in conn.execute("PRAGMA table_info(command)")}
|
||||
assert "deliveries" in cols
|
||||
|
||||
|
||||
def _drop_v5(conn):
|
||||
"""Снять с базы текущей схемы то, что добавила v5: так строятся базы прежних версий."""
|
||||
conn.execute("DROP TABLE issue_change")
|
||||
conn.execute("ALTER TABLE issue_comment DROP COLUMN created_at")
|
||||
conn.execute("ALTER TABLE sync_run DROP COLUMN history_at")
|
||||
|
||||
|
||||
def _v1_database(path):
|
||||
"""База в состоянии схемы v1: без command.deliveries, schema_version = 1, с живыми данными."""
|
||||
conn = db.connect(path)
|
||||
@ -28,6 +39,7 @@ def _v1_database(path):
|
||||
conn.execute("INSERT INTO command(batch_id, seq, type, params, status, claimed_at) VALUES (1, 1, 'refresh', '{}', ?, ?)",
|
||||
(status, "2026-10-13T00:00:00+00:00" if status == "claimed" else None))
|
||||
conn.execute("INSERT INTO issue_note(key, fact_status) VALUES ('A-1', 'В работе')")
|
||||
_drop_v5(conn)
|
||||
conn.execute("ALTER TABLE command DROP COLUMN deliveries")
|
||||
conn.execute("ALTER TABLE batch DROP COLUMN project")
|
||||
conn.execute("DROP TABLE project")
|
||||
@ -39,7 +51,7 @@ def test_migrates_v1_with_backup(tmp_path):
|
||||
path = tmp_path / "hub.sqlite"
|
||||
_v1_database(path)
|
||||
conn = db.connect(path)
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "4"
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
rows = conn.execute("SELECT status, deliveries FROM command ORDER BY id").fetchall()
|
||||
assert [(r["status"], r["deliveries"]) for r in rows] == [("claimed", 1), ("done", 0), ("queued", 0)]
|
||||
assert conn.execute("SELECT fact_status FROM issue_note WHERE key = 'A-1'").fetchone()["fact_status"] == "В работе"
|
||||
@ -186,7 +198,7 @@ def test_parallel_connect_makes_one_backup(tmp_path):
|
||||
assert errors == []
|
||||
assert len(list((tmp_path / "backups").glob("*-schema-v1.sqlite"))) == 1
|
||||
conn = db.connect(path)
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "4"
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
conn.close()
|
||||
|
||||
|
||||
@ -196,13 +208,14 @@ def test_migrates_v2_with_backup(tmp_path):
|
||||
conn = db.connect(path)
|
||||
conn.execute("INSERT INTO issue(key, summary, fetched_at) VALUES ('A-1', 'Т', '2026-10-15T00:00:00+00:00')")
|
||||
conn.execute("INSERT INTO issue_note(key, fact_status) VALUES ('A-1', 'В работе')")
|
||||
_drop_v5(conn)
|
||||
conn.execute("DROP TABLE issue_comment")
|
||||
conn.execute("ALTER TABLE batch DROP COLUMN project")
|
||||
conn.execute("DROP TABLE project")
|
||||
conn.execute("UPDATE setting SET value = '2' WHERE key = 'schema_version'")
|
||||
conn.close()
|
||||
conn = db.connect(path)
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "4"
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
assert conn.execute("SELECT COUNT(*) AS n FROM issue_comment").fetchone()["n"] == 0
|
||||
assert conn.execute("SELECT summary FROM issue WHERE key = 'A-1'").fetchone()["summary"] == "Т"
|
||||
assert conn.execute("SELECT fact_status FROM issue_note WHERE key = 'A-1'").fetchone()["fact_status"] == "В работе"
|
||||
@ -247,6 +260,7 @@ def _v3_database(path, base_jql="project = AURORA AND labels = AWPCA", board_id=
|
||||
(ts, ts, base_jql, cid))
|
||||
conn.execute("INSERT INTO issue(key, summary, fetched_at) VALUES ('A-1', 'Т', ?)", (ts,))
|
||||
conn.execute("INSERT INTO issue_note(key, fact_status) VALUES ('A-1', 'В работе')")
|
||||
_drop_v5(conn)
|
||||
conn.execute("ALTER TABLE batch DROP COLUMN project")
|
||||
conn.execute("DROP TABLE project")
|
||||
conn.execute("UPDATE setting SET value = '3' WHERE key = 'schema_version'")
|
||||
@ -269,7 +283,7 @@ def test_migrates_v3_creates_awpca(tmp_path):
|
||||
queued = [json.loads(r["params"])["jql"] for r in conn.execute("SELECT params FROM command WHERE status = 'queued'")]
|
||||
assert sorted(queued) == sorted([new, "key = A-1"])
|
||||
assert conn.execute("SELECT project FROM batch").fetchall()[0]["project"] is None
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "4"
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
assert len(list((tmp_path / "backups").glob("*-schema-v3.sqlite"))) == 1
|
||||
conn.close()
|
||||
|
||||
@ -288,5 +302,43 @@ def test_migrates_v3_unparsed_jql_keeps_running(tmp_path, caplog):
|
||||
conn = db.connect(tmp_path / "hub.sqlite")
|
||||
assert projects.list_projects(conn) == []
|
||||
assert "ORDER BY key" in caplog.text
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "4"
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw, want", [
|
||||
("2026-10-22T13:05:11.000+0300", "2026-10-22T10:05:11+00:00"),
|
||||
("2026-10-22T13:05:11+03:00", "2026-10-22T10:05:11+00:00"),
|
||||
("2026-10-22T10:05:11Z", "2026-10-22T10:05:11+00:00"),
|
||||
("2026-10-22T10:05:11.123Z", "2026-10-22T10:05:11+00:00"),
|
||||
("2026-10-22T10:05:11.123", "2026-10-22T10:05:11+00:00"),
|
||||
("2026-10-22T02:05:11-0500", "2026-10-22T07:05:11+00:00"),
|
||||
("вчера", None), ("", None), (None, None), (42, None),
|
||||
])
|
||||
def test_jira_utc(raw, want):
|
||||
assert db.jira_utc(raw) == want
|
||||
|
||||
|
||||
def _v4_database(path):
|
||||
"""База v4: комментарий со временем Jira, полный прогон в журнале синхронизаций."""
|
||||
conn = db.connect(path)
|
||||
conn.execute("INSERT INTO issue_comment(key, comment_id, created, body) VALUES ('X', '1', '2026-10-22T13:05:11.000+0300', 'a')")
|
||||
conn.execute("INSERT INTO issue_comment(key, comment_id, created, body) VALUES ('X', '2', 'мусор', 'b')")
|
||||
conn.execute("INSERT INTO sync_run(started_at, finished_at, kind, query) VALUES "
|
||||
"('2026-10-22T09:00:00+00:00', '2026-10-22T09:01:00+00:00', 'jql', 'q')")
|
||||
_drop_v5(conn)
|
||||
conn.execute("UPDATE setting SET value = '4' WHERE key = 'schema_version'")
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_migrates_v4(tmp_path):
|
||||
path = tmp_path / "hub.sqlite"
|
||||
_v4_database(path)
|
||||
conn = db.connect(path)
|
||||
assert conn.execute("SELECT value FROM setting WHERE key = 'schema_version'").fetchone()["value"] == "5"
|
||||
rows = {r["comment_id"]: (r["created"], r["created_at"]) for r in conn.execute("SELECT * FROM issue_comment")}
|
||||
assert rows == {"1": ("2026-10-22T13:05:11.000+0300", "2026-10-22T10:05:11+00:00"), "2": ("мусор", None)}
|
||||
assert conn.execute("SELECT history_at FROM sync_run").fetchone()["history_at"] is None
|
||||
assert conn.execute("SELECT COUNT(*) AS n FROM issue_change").fetchone()["n"] == 0
|
||||
assert len(list((tmp_path / "backups").glob("*-schema-v4.sqlite"))) == 1
|
||||
conn.close()
|
||||
|
||||
@ -311,3 +311,11 @@ def test_labels_of(conn):
|
||||
_scene(conn)
|
||||
assert mirror.labels_of(conn, "AURORA-4") == ["STOCKS_DRRK", "kind-dev"]
|
||||
assert mirror.labels_of(conn, "AURORA-404") is None
|
||||
|
||||
|
||||
def test_apply_state_comment_created_at(conn):
|
||||
mirror.apply_state(conn, [{"key": "X", "comments": [
|
||||
{"id": "1", "body": "a", "created": "2026-10-22T13:05:11.000+0300"},
|
||||
{"id": "2", "body": "b", "created": "мусор"}]}], [])
|
||||
rows = {r["comment_id"]: r["created_at"] for r in conn.execute("SELECT * FROM issue_comment")}
|
||||
assert rows == {"1": "2026-10-22T10:05:11+00:00", "2": None}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user