c1c73e96f5
- funding_rate table and "Фондирование" page: rate from date, history, rate on date (D27)
- positions by issue: BUY - SELL per portfolio, "Остатки" page (D29)
- quotes are loaded only for securities with an open position (D10)
- business calendar from xmlcalendar.ru: yearly file upload, report date
and update job use previous working day (D30)
- xlsx download for positions, quotes and funding rate history
- manual operation type as radio buttons
- QUIK instruments with purchase marks ("МТС 2Р-18_1") match the issue,
file name kept in operation.instrument (D17)
- migrations d3a6f8b1c254, e7b2c9d4a813, f1c4a7e9b352
- project.md: decisions and questions after customer meetings
425 lines
15 KiB
Python
425 lines
15 KiB
Python
"""Предпросмотр загрузки QUIK (D19): чистая логика плана + проверка повтора
|
||
и сопоставления с бумагами на временном SQLite (схема — миграциями)."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import replace
|
||
from datetime import date, datetime
|
||
from decimal import Decimal
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from sqlalchemy import select
|
||
from sqlalchemy.exc import IntegrityError
|
||
|
||
from app.db.base import run_db
|
||
from app.db.models import Currency, Portfolio, Security
|
||
from app.imports.quik import QuikTrade, RowIssue, parse_quik_xlsx
|
||
from app.repositories.import_batches import ImportBatchRepository
|
||
from app.services import quik_import
|
||
from app.services.quik_import import (
|
||
EXCLUDE,
|
||
INCLUDE,
|
||
BatchInfo,
|
||
ImportContext,
|
||
PortfolioRef,
|
||
RowStatus,
|
||
SecurityMatch,
|
||
build_plan,
|
||
commission_totals,
|
||
match_security,
|
||
portfolio_hint,
|
||
split_mark,
|
||
suggest_portfolio,
|
||
totals,
|
||
)
|
||
|
||
USER = "analyst@rshb.ru"
|
||
PORTFOLIOS = [
|
||
PortfolioRef(1, "UW", "Андеррайтинг"),
|
||
PortfolioRef(2, "SALES", "Продажи"),
|
||
]
|
||
SHORT = "ЕврХол3P06"
|
||
|
||
|
||
@pytest.fixture
|
||
def fixture_trades(fixtures_dir: Path) -> list[QuikTrade]:
|
||
return parse_quik_xlsx(
|
||
(fixtures_dir / "quik_sample_300326_anon.xlsx").read_bytes()
|
||
).trades
|
||
|
||
|
||
def _mixed(trades: list[QuikTrade]) -> list[QuikTrade]:
|
||
"""6 строк фикстуры с разными комментариями: 2×/pp, /uw, /trde, пусто, /xyz."""
|
||
comments = ["/pp", "/pp", "/uw", "/trde", "", "/xyz"]
|
||
return [replace(t, comment=c) for t, c in zip(trades, comments, strict=True)]
|
||
|
||
|
||
def _context(**kwargs) -> ImportContext: # noqa: ANN003
|
||
kwargs.setdefault("securities", {SHORT: [SecurityMatch(10, "RU000A10XXXX")]})
|
||
return ImportContext(portfolios=PORTFOLIOS, **kwargs)
|
||
|
||
|
||
def _batch(code: str, loaded: set[int], excluded: int = 0) -> BatchInfo:
|
||
return BatchInfo(
|
||
portfolio_code=code,
|
||
file_name="сделки.xlsx",
|
||
date_from=date(2026, 3, 30),
|
||
date_to=date(2026, 3, 30),
|
||
rows_loaded=len(loaded),
|
||
rows_excluded=excluded,
|
||
loaded_rows=frozenset(loaded),
|
||
created_at=datetime(2026, 10, 15, 10, 0),
|
||
created_by=USER,
|
||
)
|
||
|
||
|
||
# --- комментарии --------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("comment", "expected"),
|
||
[
|
||
("/uw", "UW"),
|
||
("/pp", "SALES"),
|
||
(" /PP ", "SALES"),
|
||
("/trde", None), # D19: не относим никуда автоматически
|
||
("", None),
|
||
("/uw2", None),
|
||
],
|
||
)
|
||
def test_portfolio_hint(comment: str, expected: str | None) -> None:
|
||
assert portfolio_hint(comment) == expected
|
||
|
||
|
||
def test_suggest_portfolio(fixture_trades: list[QuikTrade]) -> None:
|
||
assert suggest_portfolio(fixture_trades) == "SALES"
|
||
assert suggest_portfolio(_mixed(fixture_trades)) == "SALES" # 2 /pp против 1 /uw
|
||
assert suggest_portfolio([replace(t, comment="") for t in fixture_trades]) is None
|
||
tie = [replace(fixture_trades[0], comment="/uw"), fixture_trades[1]]
|
||
assert suggest_portfolio(tie) is None
|
||
|
||
|
||
# --- план загрузки ----------------------------------------------------------
|
||
|
||
|
||
def test_plan_splits_rows_by_portfolio_and_waits_for_unknown(
|
||
fixture_trades: list[QuikTrade],
|
||
) -> None:
|
||
plan = build_plan(_mixed(fixture_trades), _context(), "SALES", decisions={})
|
||
|
||
assert [r.status for r in plan.rows] == [
|
||
RowStatus.OWN,
|
||
RowStatus.OWN,
|
||
RowStatus.OTHER,
|
||
RowStatus.UNKNOWN_PENDING, # /trde
|
||
RowStatus.UNKNOWN_PENDING, # пусто
|
||
RowStatus.UNKNOWN_PENDING, # /xyz
|
||
]
|
||
assert [r.trade.ordinal for r in plan.to_load] == [1, 2]
|
||
assert plan.pending == 3
|
||
assert not plan.ready # пока есть строки без решения
|
||
assert plan.duplicate is None
|
||
assert plan.summary.rows_total == 6
|
||
assert plan.summary.loaded == {"UW": 0, "SALES": 0}
|
||
assert plan.summary.nowhere == 6
|
||
|
||
|
||
def test_plan_applies_user_decisions(fixture_trades: list[QuikTrade]) -> None:
|
||
plan = build_plan(
|
||
_mixed(fixture_trades),
|
||
_context(),
|
||
"UW",
|
||
decisions={4: INCLUDE, 5: EXCLUDE, 6: EXCLUDE, 1: EXCLUDE},
|
||
)
|
||
|
||
statuses = {r.trade.ordinal: r.status for r in plan.rows}
|
||
assert statuses == {
|
||
1: RowStatus.OTHER, # решение для строки с портфелем игнорируется
|
||
2: RowStatus.OTHER,
|
||
3: RowStatus.OWN,
|
||
4: RowStatus.UNKNOWN_INCLUDED,
|
||
5: RowStatus.UNKNOWN_EXCLUDED,
|
||
6: RowStatus.UNKNOWN_EXCLUDED,
|
||
}
|
||
assert [r.trade.ordinal for r in plan.to_load] == [3, 4]
|
||
assert (plan.pending, plan.excluded) == (0, 2)
|
||
assert plan.ready
|
||
|
||
|
||
def test_same_file_same_portfolio_is_duplicate(
|
||
fixture_trades: list[QuikTrade],
|
||
) -> None:
|
||
loaded = _batch("SALES", {1, 2, 3, 4, 5, 6})
|
||
plan = build_plan(fixture_trades, _context(same_file=[loaded]), "SALES", {})
|
||
|
||
assert plan.duplicate == loaded
|
||
assert {r.status for r in plan.rows} == {RowStatus.LOADED_HERE}
|
||
assert not plan.ready
|
||
assert plan.summary.loaded == {"UW": 0, "SALES": 6}
|
||
assert plan.summary.nowhere == 0
|
||
|
||
|
||
def test_same_file_other_portfolio_takes_only_rest(
|
||
fixture_trades: list[QuikTrade],
|
||
) -> None:
|
||
"""Файл уже загружали в SALES (/pp + строка без комментария отнесена туда,
|
||
/trde исключена). Загрузка в UW берёт только /uw; строка без комментария
|
||
уже в SALES и второй раз не загружается."""
|
||
in_sales = _batch("SALES", {1, 2, 5}, excluded=1)
|
||
plan = build_plan(
|
||
_mixed(fixture_trades), _context(same_file=[in_sales]), "UW", {4: EXCLUDE}
|
||
)
|
||
|
||
statuses = {r.trade.ordinal: r.status for r in plan.rows}
|
||
assert statuses == {
|
||
1: RowStatus.LOADED_ELSEWHERE,
|
||
2: RowStatus.LOADED_ELSEWHERE,
|
||
3: RowStatus.OWN,
|
||
4: RowStatus.UNKNOWN_EXCLUDED,
|
||
5: RowStatus.LOADED_ELSEWHERE,
|
||
6: RowStatus.UNKNOWN_PENDING,
|
||
}
|
||
assert plan.duplicate is None
|
||
assert plan.rows[0].loaded_in == "SALES"
|
||
assert plan.summary.loaded == {"UW": 0, "SALES": 3}
|
||
assert plan.summary.nowhere == 3
|
||
|
||
|
||
def test_overlapping_period_only_for_selected_portfolio(
|
||
fixture_trades: list[QuikTrade],
|
||
) -> None:
|
||
other_uw, other_sales = _batch("UW", {1}), _batch("SALES", {1})
|
||
plan = build_plan(
|
||
fixture_trades, _context(overlapping=[other_uw, other_sales]), "SALES", {}
|
||
)
|
||
|
||
assert plan.overlapping == [other_sales]
|
||
assert plan.duplicate is None # пересечение периода — не отказ
|
||
|
||
|
||
def test_unmatched_and_ambiguous_instruments(fixture_trades: list[QuikTrade]) -> None:
|
||
trades = _mixed(fixture_trades)
|
||
trades[1] = replace(trades[1], instrument="МТС 2Р-18")
|
||
trades[2] = replace(trades[2], instrument="Нет такой") # /uw — не наша строка
|
||
trades[4] = replace(trades[4], instrument="Нет такой") # без портфеля, ждёт
|
||
context = _context(
|
||
securities={
|
||
SHORT: [SecurityMatch(10, "RU000A10XXXX")],
|
||
"МТС 2Р-18": [
|
||
SecurityMatch(11, "RU000A10ESM7"),
|
||
SecurityMatch(12, "RU000A10ESM8"),
|
||
],
|
||
}
|
||
)
|
||
|
||
plan = build_plan(trades, context, "SALES", {4: EXCLUDE, 6: EXCLUDE})
|
||
|
||
assert plan.unmatched_instruments == {"Нет такой": 1}
|
||
assert plan.ambiguous_instruments == {"МТС 2Р-18": ["RU000A10ESM7", "RU000A10ESM8"]}
|
||
assert not plan.ready
|
||
assert plan.blockers == [
|
||
"не решено строк без портфеля: 1",
|
||
"нет в справочнике бумаг: Нет такой",
|
||
"одному краткому имени соответствует несколько бумаг: МТС 2Р-18",
|
||
]
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("instrument", "expected"),
|
||
[
|
||
("МТС 2Р-18_1", ("МТС 2Р-18", "1")),
|
||
("РЖД 1Р-49R_Д", ("РЖД 1Р-49R", "Д")),
|
||
("Бумага_А_2", ("Бумага_А", "2")),
|
||
("МТС 2Р-18 _1", ("МТС 2Р-18", "1")),
|
||
(SHORT, (SHORT, None)),
|
||
("_1", ("_1", None)),
|
||
("МТС 2Р-18_", ("МТС 2Р-18_", None)),
|
||
],
|
||
)
|
||
def test_split_mark(instrument: str, expected: tuple[str, str | None]) -> None:
|
||
assert split_mark(instrument) == expected
|
||
|
||
|
||
def test_match_security_exact_name_wins() -> None:
|
||
issue = [SecurityMatch(10, "RU000A10XXXX")]
|
||
with_underscore = [SecurityMatch(11, "RU000A10YYYY")]
|
||
securities = {"Бумага": issue, "Бумага_1": with_underscore}
|
||
|
||
assert match_security("Бумага_1", securities) == (tuple(with_underscore), None)
|
||
assert match_security("Бумага_2", securities) == (tuple(issue), "Бумага")
|
||
assert match_security("Бумага", securities) == (tuple(issue), None)
|
||
assert match_security("Нет_1", securities) == ((), None)
|
||
|
||
|
||
def test_marked_instruments_go_to_issue(fixture_trades: list[QuikTrade]) -> None:
|
||
trades = _mixed(fixture_trades)
|
||
trades[0] = replace(trades[0], instrument=f"{SHORT}_1")
|
||
trades[1] = replace(trades[1], instrument="Нет такой_1")
|
||
trades[2] = replace(trades[2], instrument=f"{SHORT}_2") # /uw — не наша строка
|
||
|
||
plan = build_plan(trades, _context(), "SALES", {4: EXCLUDE, 5: EXCLUDE, 6: EXCLUDE})
|
||
|
||
first = plan.rows[0]
|
||
assert (first.securities, first.issue_name) == (
|
||
(SecurityMatch(10, "RU000A10XXXX"),),
|
||
SHORT,
|
||
)
|
||
assert plan.marked_instruments == {f"{SHORT}_1": (SHORT, 1)}
|
||
assert plan.unmatched_instruments == {"Нет такой_1": 1}
|
||
|
||
|
||
def test_blockers_explain_why_not_ready(fixture_trades: list[QuikTrade]) -> None:
|
||
pending = build_plan(_mixed(fixture_trades), _context(), "SALES", {})
|
||
assert pending.blockers == ["не решено строк без портфеля: 3"]
|
||
|
||
nothing = build_plan(
|
||
_mixed(fixture_trades), _context(), "UW", {4: EXCLUDE, 5: EXCLUDE, 6: EXCLUDE}
|
||
)
|
||
nothing.rows[2] = replace(nothing.rows[2], status=RowStatus.OTHER)
|
||
assert nothing.blockers == ["нет строк для загрузки в UW"]
|
||
|
||
duplicate = build_plan(
|
||
fixture_trades,
|
||
_context(same_file=[_batch("SALES", {1, 2, 3, 4, 5, 6})]),
|
||
"SALES",
|
||
{},
|
||
)
|
||
assert duplicate.blockers == ["этот файл уже загружен в SALES"]
|
||
|
||
|
||
# --- итоги ------------------------------------------------------------------
|
||
|
||
|
||
def test_totals_by_operation_and_currency(fixture_trades: list[QuikTrade]) -> None:
|
||
trades = list(fixture_trades)
|
||
trades[0] = replace(trades[0], currency="CNY")
|
||
|
||
rows = {(r.operation, r.currency): r for r in totals(trades)}
|
||
|
||
assert set(rows) == {("BUY", "CNY"), ("BUY", "RUB"), ("SELL", "RUB")}
|
||
assert rows[("BUY", "RUB")].trades == 3
|
||
assert rows[("BUY", "RUB")].quantity == Decimal("1043216")
|
||
assert rows[("BUY", "RUB")].amount == Decimal("1040816000")
|
||
assert rows[("SELL", "RUB")].settlement_amount == Decimal("1078380000")
|
||
assert rows[("BUY", "CNY")].quantity == Decimal("54784")
|
||
|
||
|
||
def test_commission_totals_keep_five_fields(fixture_trades: list[QuikTrade]) -> None:
|
||
[rub] = commission_totals(fixture_trades)
|
||
|
||
assert rub.currency == "RUB"
|
||
assert rub.commission_ts == Decimal("276217.0")
|
||
assert rub.commission_clearing == Decimal("174754.22")
|
||
assert rub.commission_exchange == Decimal("101462.78")
|
||
assert rub.commission_ts == rub.commission_clearing + rub.commission_exchange
|
||
assert rub.commission_tc == rub.commission_broker == Decimal("0")
|
||
|
||
|
||
# --- БД: повтор файла, пересечения, бумаги ----------------------------------------
|
||
|
||
|
||
def _add_batch(session_local, portfolio_code: str, sha: str, loaded: list[int]):
|
||
async def _add() -> None:
|
||
async with session_local() as session:
|
||
portfolio_id = (
|
||
await session.execute(
|
||
select(Portfolio.id).where(Portfolio.code == portfolio_code)
|
||
)
|
||
).scalar_one()
|
||
await ImportBatchRepository(session).add(
|
||
source="QUIK",
|
||
portfolio_id=portfolio_id,
|
||
file_name="сделки_300326.xlsx",
|
||
content_sha256=sha,
|
||
date_from=date(2026, 3, 30),
|
||
date_to=date(2026, 3, 30),
|
||
rows_total=6,
|
||
loaded_rows=loaded,
|
||
rows_excluded=0,
|
||
user=USER,
|
||
)
|
||
await session.commit()
|
||
|
||
run_db(_add())
|
||
|
||
|
||
def _add_security(session_local, name: str, isin: str) -> None:
|
||
async def _add() -> None:
|
||
async with session_local() as session:
|
||
rub = (
|
||
await session.execute(select(Currency.id).where(Currency.code == "RUB"))
|
||
).scalar_one()
|
||
session.add(
|
||
Security(
|
||
isin=isin,
|
||
secid=isin,
|
||
name=name,
|
||
currency_id=rub,
|
||
created_by=USER,
|
||
updated_by=USER,
|
||
)
|
||
)
|
||
await session.commit()
|
||
|
||
run_db(_add())
|
||
|
||
|
||
def test_context_from_db_detects_duplicate_overlap_and_securities(
|
||
session_local, fixtures_dir: Path
|
||
) -> None:
|
||
parsed = parse_quik_xlsx(
|
||
(fixtures_dir / "quik_sample_300326_anon.xlsx").read_bytes()
|
||
)
|
||
sha = parsed.content_hash
|
||
_add_security(session_local, SHORT, "RU000A10EVH2")
|
||
_add_batch(session_local, "SALES", sha, [6, 1, 2, 3, 4, 5])
|
||
_add_batch(session_local, "SALES", "0" * 64, [1]) # другой файл, тот же день
|
||
|
||
context = run_db(quik_import.load_context(parsed))
|
||
|
||
assert [p.code for p in context.portfolios] == ["UW", "SALES"]
|
||
assert [b.portfolio_code for b in context.same_file] == ["SALES"]
|
||
assert context.same_file[0].loaded_rows == frozenset(range(1, 7))
|
||
assert context.same_file[0].created_by == USER
|
||
assert len(context.overlapping) == 1
|
||
assert context.securities == {SHORT: [SecurityMatch(1, "RU000A10EVH2")]}
|
||
|
||
marked = replace(
|
||
parsed, trades=[replace(t, instrument=f"{SHORT}_1") for t in parsed.trades]
|
||
)
|
||
marked_context = run_db(quik_import.load_context(marked))
|
||
assert marked_context.securities == {SHORT: [SecurityMatch(1, "RU000A10EVH2")]}
|
||
|
||
in_sales = build_plan(parsed.trades, context, "SALES", {})
|
||
assert in_sales.duplicate is not None
|
||
assert in_sales.summary.loaded == {"UW": 0, "SALES": 6}
|
||
|
||
in_uw = build_plan(parsed.trades, context, "UW", {})
|
||
assert in_uw.duplicate is None
|
||
assert in_uw.overlapping == []
|
||
|
||
|
||
def test_same_file_same_portfolio_rejected_by_unique_index(
|
||
session_local, fixtures_dir: Path
|
||
) -> None:
|
||
_add_batch(session_local, "SALES", "a" * 64, [1, 2])
|
||
_add_batch(session_local, "UW", "a" * 64, [3]) # тот же файл в другой портфель
|
||
|
||
with pytest.raises(IntegrityError):
|
||
_add_batch(session_local, "SALES", "a" * 64, [4])
|
||
|
||
|
||
def test_context_without_hash_has_only_portfolios(
|
||
session_local, fixtures_dir: Path
|
||
) -> None:
|
||
parsed = parse_quik_xlsx(
|
||
(fixtures_dir / "quik_sample_300326_anon.xlsx").read_bytes()
|
||
)
|
||
parsed.errors.append(RowIssue(2, "Цена", "не число")) # хэша нет
|
||
|
||
context = run_db(quik_import.load_context(parsed))
|
||
|
||
assert [p.code for p in context.portfolios] == ["UW", "SALES"]
|
||
assert context.same_file == [] and context.securities == {}
|