perf(api): свод 2 — миграция 0053: нормализаторы поиска как в JS и компактная проекция
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
b134cfd118
commit
a62add9779
15
api/alembic/versions/0053_svod2_server_paging.py
Normal file
15
api/alembic/versions/0053_svod2_server_paging.py
Normal file
@ -0,0 +1,15 @@
|
||||
from sql_migration import run_sql_file_migration
|
||||
|
||||
|
||||
revision = "0053"
|
||||
down_revision = "0049"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
run_sql_file_migration("0053_svod2_server_paging.sql")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
237
api/alembic/versions/sql/0053_svod2_server_paging.sql
Normal file
237
api/alembic/versions/sql/0053_svod2_server_paging.sql
Normal file
@ -0,0 +1,237 @@
|
||||
-- AURORA-1843, миграция 0053, часть 1: нормализаторы поиска как в JS и компактная проекция строки свода 2.
|
||||
|
||||
-- Регистр без зависимости от локали: явная таблица translate (ASCII, кириллица, Ґ, латиница-1 без U+00D7).
|
||||
CREATE OR REPLACE FUNCTION v3.svod2_casefold(x text)
|
||||
RETURNS text
|
||||
LANGUAGE sql IMMUTABLE
|
||||
AS $function$
|
||||
SELECT translate(
|
||||
x,
|
||||
E'\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004A\u004B\u004C\u004D\u004E\u004F\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005A\u0410\u0411\u0412\u0413\u0414\u0415\u0416\u0417\u0418\u0419\u041A\u041B\u041C\u041D\u041E\u041F\u0420\u0421\u0422\u0423\u0424\u0425\u0426\u0427\u0428\u0429\u042A\u042B\u042C\u042D\u042E\u042F\u0400\u0401\u0402\u0403\u0404\u0405\u0406\u0407\u0408\u0409\u040A\u040B\u040C\u040D\u040E\u040F\u0490\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF\u00D0\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE',
|
||||
E'\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006A\u006B\u006C\u006D\u006E\u006F\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007A\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043A\u043B\u043C\u043D\u043E\u043F\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044A\u044B\u044C\u044D\u044E\u044F\u0450\u0451\u0452\u0453\u0454\u0455\u0456\u0457\u0458\u0459\u045A\u045B\u045C\u045D\u045E\u045F\u0491\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE'
|
||||
)
|
||||
$function$;
|
||||
|
||||
-- Обрезка как String.prototype.trim: ECMAScript WhiteSpace и LineTerminator.
|
||||
CREATE OR REPLACE FUNCTION v3.svod2_js_trim(x text)
|
||||
RETURNS text
|
||||
LANGUAGE sql IMMUTABLE
|
||||
AS $function$
|
||||
SELECT btrim(x, E'\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF')
|
||||
$function$;
|
||||
|
||||
-- Запись числа как JS Number::toString. Цифры берутся из float8::text при extra_float_digits = 1
|
||||
-- (кратчайшая точная запись), SET в функции изолирует результат от настроек сессии.
|
||||
CREATE OR REPLACE FUNCTION v3.svod2_js_number_text(x double precision)
|
||||
RETURNS text
|
||||
LANGUAGE plpgsql IMMUTABLE
|
||||
SET extra_float_digits = 1
|
||||
AS $function$
|
||||
DECLARE
|
||||
t text;
|
||||
neg boolean := false;
|
||||
mant text;
|
||||
ex integer := 0;
|
||||
int_part text;
|
||||
digits text;
|
||||
lead integer;
|
||||
n integer;
|
||||
k integer;
|
||||
res text;
|
||||
BEGIN
|
||||
IF x IS NULL THEN
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
IF x = 0 THEN
|
||||
RETURN '0';
|
||||
END IF;
|
||||
t := x::text;
|
||||
IF t IN ('NaN', 'Infinity', '-Infinity') THEN
|
||||
RETURN t;
|
||||
END IF;
|
||||
IF left(t, 1) = '-' THEN
|
||||
neg := true;
|
||||
t := substr(t, 2);
|
||||
END IF;
|
||||
IF position('e' IN t) > 0 THEN
|
||||
mant := split_part(t, 'e', 1);
|
||||
ex := split_part(t, 'e', 2)::integer;
|
||||
ELSE
|
||||
mant := t;
|
||||
END IF;
|
||||
IF position('.' IN mant) > 0 THEN
|
||||
int_part := split_part(mant, '.', 1);
|
||||
ELSE
|
||||
int_part := mant;
|
||||
END IF;
|
||||
digits := replace(mant, '.', '');
|
||||
lead := length(digits) - length(ltrim(digits, '0'));
|
||||
digits := ltrim(digits, '0');
|
||||
digits := rtrim(digits, '0');
|
||||
n := length(int_part) + ex - lead;
|
||||
k := length(digits);
|
||||
IF n >= k AND n <= 21 THEN
|
||||
res := digits || repeat('0', n - k);
|
||||
ELSIF n > 0 AND n <= 21 THEN
|
||||
res := substr(digits, 1, n) || '.' || substr(digits, n + 1);
|
||||
ELSIF n > -6 AND n <= 0 THEN
|
||||
res := '0.' || repeat('0', -n) || digits;
|
||||
ELSE
|
||||
res := substr(digits, 1, 1);
|
||||
IF k > 1 THEN
|
||||
res := res || '.' || substr(digits, 2);
|
||||
END IF;
|
||||
res := res || 'e' || CASE WHEN n - 1 >= 0 THEN '+' ELSE '-' END || abs(n - 1)::text;
|
||||
END IF;
|
||||
IF neg THEN
|
||||
res := '-' || res;
|
||||
END IF;
|
||||
RETURN res;
|
||||
END
|
||||
$function$;
|
||||
|
||||
-- Компактная проекция data строки свода 2: jsonb_strip_nulls по перечисленным путям.
|
||||
CREATE OR REPLACE FUNCTION v3.svod2_compact(d jsonb)
|
||||
RETURNS jsonb
|
||||
LANGUAGE sql IMMUTABLE
|
||||
AS $function$
|
||||
SELECT jsonb_strip_nulls(jsonb_build_object(
|
||||
'line_id', d #> '{line_id}',
|
||||
'header', jsonb_build_object(
|
||||
'source_form', d #> '{header,source_form}',
|
||||
'source_id', d #> '{header,source_id}',
|
||||
'smeta_type', d #> '{header,smeta_type}',
|
||||
'smeta_direction', d #> '{header,smeta_direction}',
|
||||
'org_name', d #> '{header,org_name}',
|
||||
'name', d #> '{header,name}',
|
||||
'item_id', d #> '{header,item_id}',
|
||||
'num_group', d #> '{header,num_group}',
|
||||
'justification', d #> '{header,justification}'
|
||||
),
|
||||
'plan', jsonb_build_object(
|
||||
'plan_q1', d #> '{plan,plan_q1}',
|
||||
'plan_q2', d #> '{plan,plan_q2}',
|
||||
'plan_q3', d #> '{plan,plan_q3}',
|
||||
'plan_q4', d #> '{plan,plan_q4}'
|
||||
),
|
||||
'allocation', jsonb_build_object(
|
||||
'internal_order', d #> '{allocation,internal_order}',
|
||||
'property_object', d #> '{allocation,property_object}'
|
||||
),
|
||||
'sequestration', jsonb_build_object(
|
||||
'DFIP', jsonb_build_object(
|
||||
'adj_q1', d #> '{sequestration,DFIP,adj_q1}',
|
||||
'adj_q2', d #> '{sequestration,DFIP,adj_q2}',
|
||||
'adj_q3', d #> '{sequestration,DFIP,adj_q3}',
|
||||
'adj_q4', d #> '{sequestration,DFIP,adj_q4}'
|
||||
)
|
||||
),
|
||||
'reserve', jsonb_build_object(
|
||||
'amount_q1', d #> '{reserve,amount_q1}',
|
||||
'amount_q2', d #> '{reserve,amount_q2}',
|
||||
'amount_q3', d #> '{reserve,amount_q3}',
|
||||
'amount_q4', d #> '{reserve,amount_q4}'
|
||||
),
|
||||
'approved', jsonb_build_object(
|
||||
'approved_q1', d #> '{approved,approved_q1}',
|
||||
'approved_q2', d #> '{approved,approved_q2}',
|
||||
'approved_q3', d #> '{approved,approved_q3}',
|
||||
'approved_q4', d #> '{approved,approved_q4}'
|
||||
),
|
||||
'contract_detail', jsonb_build_object(
|
||||
'counterparty', d #> '{contract_detail,counterparty}',
|
||||
'reference', d #> '{contract_detail,reference}',
|
||||
'subject', d #> '{contract_detail,subject}',
|
||||
'currency', d #> '{contract_detail,currency}',
|
||||
'ceiling_amount', d #> '{contract_detail,ceiling_amount}',
|
||||
'expenses_q1', d #> '{contract_detail,expenses_q1}',
|
||||
'expenses_q2', d #> '{contract_detail,expenses_q2}',
|
||||
'expenses_q3', d #> '{contract_detail,expenses_q3}',
|
||||
'expenses_q4', d #> '{contract_detail,expenses_q4}',
|
||||
'vat_rate', d #> '{contract_detail,vat_rate}',
|
||||
'deadline', d #> '{contract_detail,deadline}'
|
||||
),
|
||||
'q1', jsonb_build_object(
|
||||
'adj_current', d #> '{q1,adj_current}',
|
||||
'adj_ssp', d #> '{q1,adj_ssp}',
|
||||
'adj_rf', d #> '{q1,adj_rf}',
|
||||
'adj_reserve', d #> '{q1,adj_reserve}',
|
||||
'payment_date', d #> '{q1,payment_date}',
|
||||
'payment_amount', d #> '{q1,payment_amount}',
|
||||
'payment_amount_ho', d #> '{q1,payment_amount_ho}',
|
||||
'payment_amount_rf', d #> '{q1,payment_amount_rf}',
|
||||
'payment_act', d #> '{q1,payment_act}',
|
||||
'booking_amount', d #> '{q1,booking_amount}',
|
||||
'actual_m1', d #> '{q1,actual_m1}',
|
||||
'actual_m2', d #> '{q1,actual_m2}',
|
||||
'actual_m3', d #> '{q1,actual_m3}',
|
||||
'transfer_to_q2', d #> '{q1,transfer_to_q2}',
|
||||
'transfer_to_q3', d #> '{q1,transfer_to_q3}',
|
||||
'transfer_to_q4', d #> '{q1,transfer_to_q4}',
|
||||
'transfer_to_economy', d #> '{q1,transfer_to_economy}'
|
||||
),
|
||||
'q2', jsonb_build_object(
|
||||
'adj_current', d #> '{q2,adj_current}',
|
||||
'adj_ssp', d #> '{q2,adj_ssp}',
|
||||
'adj_rf', d #> '{q2,adj_rf}',
|
||||
'adj_reserve', d #> '{q2,adj_reserve}',
|
||||
'payment_date', d #> '{q2,payment_date}',
|
||||
'payment_amount', d #> '{q2,payment_amount}',
|
||||
'payment_amount_ho', d #> '{q2,payment_amount_ho}',
|
||||
'payment_amount_rf', d #> '{q2,payment_amount_rf}',
|
||||
'payment_act', d #> '{q2,payment_act}',
|
||||
'booking_amount', d #> '{q2,booking_amount}',
|
||||
'actual_m1', d #> '{q2,actual_m1}',
|
||||
'actual_m2', d #> '{q2,actual_m2}',
|
||||
'actual_m3', d #> '{q2,actual_m3}',
|
||||
'transfer_to_q3', d #> '{q2,transfer_to_q3}',
|
||||
'transfer_to_q4', d #> '{q2,transfer_to_q4}',
|
||||
'transfer_to_economy', d #> '{q2,transfer_to_economy}',
|
||||
'plan_revision_eco_change', d #> '{q2,plan_revision_eco_change}',
|
||||
'plan_revision_item_adj', d #> '{q2,plan_revision_item_adj}',
|
||||
'plan_revision_increase', d #> '{q2,plan_revision_increase}',
|
||||
'plan_revision_sequester', d #> '{q2,plan_revision_sequester}'
|
||||
),
|
||||
'q3', jsonb_build_object(
|
||||
'adj_current', d #> '{q3,adj_current}',
|
||||
'adj_ssp', d #> '{q3,adj_ssp}',
|
||||
'adj_rf', d #> '{q3,adj_rf}',
|
||||
'adj_reserve', d #> '{q3,adj_reserve}',
|
||||
'payment_date', d #> '{q3,payment_date}',
|
||||
'payment_amount', d #> '{q3,payment_amount}',
|
||||
'payment_amount_ho', d #> '{q3,payment_amount_ho}',
|
||||
'payment_amount_rf', d #> '{q3,payment_amount_rf}',
|
||||
'payment_act', d #> '{q3,payment_act}',
|
||||
'booking_amount', d #> '{q3,booking_amount}',
|
||||
'actual_m1', d #> '{q3,actual_m1}',
|
||||
'actual_m2', d #> '{q3,actual_m2}',
|
||||
'actual_m3', d #> '{q3,actual_m3}',
|
||||
'transfer_to_q4', d #> '{q3,transfer_to_q4}',
|
||||
'transfer_to_economy', d #> '{q3,transfer_to_economy}',
|
||||
'plan_revision_eco_change', d #> '{q3,plan_revision_eco_change}',
|
||||
'plan_revision_item_adj', d #> '{q3,plan_revision_item_adj}',
|
||||
'plan_revision_increase', d #> '{q3,plan_revision_increase}',
|
||||
'plan_revision_sequester', d #> '{q3,plan_revision_sequester}'
|
||||
),
|
||||
'q4', jsonb_build_object(
|
||||
'adj_current', d #> '{q4,adj_current}',
|
||||
'adj_ssp', d #> '{q4,adj_ssp}',
|
||||
'adj_rf', d #> '{q4,adj_rf}',
|
||||
'adj_reserve', d #> '{q4,adj_reserve}',
|
||||
'payment_date', d #> '{q4,payment_date}',
|
||||
'payment_amount', d #> '{q4,payment_amount}',
|
||||
'payment_amount_ho', d #> '{q4,payment_amount_ho}',
|
||||
'payment_amount_rf', d #> '{q4,payment_amount_rf}',
|
||||
'payment_act', d #> '{q4,payment_act}',
|
||||
'booking_amount', d #> '{q4,booking_amount}',
|
||||
'actual_m1', d #> '{q4,actual_m1}',
|
||||
'actual_m2', d #> '{q4,actual_m2}',
|
||||
'actual_m3', d #> '{q4,actual_m3}',
|
||||
'actual_spod', d #> '{q4,actual_spod}',
|
||||
'plan_revision_eco_change', d #> '{q4,plan_revision_eco_change}',
|
||||
'plan_revision_item_adj', d #> '{q4,plan_revision_item_adj}',
|
||||
'plan_revision_increase', d #> '{q4,plan_revision_increase}',
|
||||
'plan_revision_sequester', d #> '{q4,plan_revision_sequester}'
|
||||
)
|
||||
))
|
||||
$function$;
|
||||
53
api/tests/integration/test_svod2_sql.py
Normal file
53
api/tests/integration/test_svod2_sql.py
Normal file
@ -0,0 +1,53 @@
|
||||
"""SQL-нормализаторы свода 2 (миграция 0053): регистр, обрезка, запись числа как в JS, компактная проекция."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration.conftest import legacy_full
|
||||
from tests.svod2_oracle import compact_projection
|
||||
|
||||
NUM = json.loads(
|
||||
(Path(__file__).resolve().parent.parent / "fixtures" / "js_number_text_vectors.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_js_number_text_matches_node_vectors(svod2_sql):
|
||||
for v in NUM:
|
||||
got = svod2_sql(
|
||||
"SELECT v3.svod2_js_number_text((CAST(:x AS jsonb) #>> '{}')::float8) AS t",
|
||||
{"x": v["input"]},
|
||||
)[0][0]
|
||||
assert got == v["js"], v
|
||||
|
||||
|
||||
def test_js_number_text_ignores_session_extra_float_digits(svod2_sql):
|
||||
svod2_sql("SET LOCAL extra_float_digits = 0", None)
|
||||
got = svod2_sql("SELECT v3.svod2_js_number_text(0.1::float8 + 0.2::float8) AS t", None)[0][0]
|
||||
assert got == "0.30000000000000004"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src",
|
||||
["АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ", "ЄІЇЎЂЏҐ", "ABCxyz", "ÀÉÖÞ", "×", "аРЕНДА 1"],
|
||||
)
|
||||
def test_casefold_matches_js_lower(svod2_sql, src):
|
||||
assert svod2_sql("SELECT v3.svod2_casefold(:s) AS t", {"s": src})[0][0] == src.lower()
|
||||
|
||||
|
||||
def test_casefold_independent_of_collation(svod2_sql):
|
||||
got = svod2_sql("""SELECT v3.svod2_casefold('ЁЛКА' COLLATE "C") AS t""", None)[0][0]
|
||||
assert got == "ёлка"
|
||||
|
||||
|
||||
def test_js_trim_ecmascript_whitespace(svod2_sql):
|
||||
assert svod2_sql("SELECT v3.svod2_js_trim(:s) AS t", {"s": " x \t"})[0][0] == "x"
|
||||
assert svod2_sql("SELECT v3.svod2_js_trim(:s) AS t", {"s": " x"})[0][0] == " x"
|
||||
|
||||
|
||||
def test_compact_matches_python_projection(svod2_sql, svod2_data, client, admin_tokens, auth_headers):
|
||||
full = legacy_full(client, "RF", auth_headers(admin_tokens))[0]["data"]
|
||||
got = svod2_sql("SELECT v3.svod2_compact(CAST(:d AS jsonb)) AS c", {"d": json.dumps(full)})[0][0]
|
||||
assert got == compact_projection(full)
|
||||
Loading…
x
Reference in New Issue
Block a user