diff --git a/api/alembic/versions/0053_svod2_server_paging.py b/api/alembic/versions/0053_svod2_server_paging.py new file mode 100644 index 0000000..4346091 --- /dev/null +++ b/api/alembic/versions/0053_svod2_server_paging.py @@ -0,0 +1,15 @@ +from sql_migration import run_sql_file_migration + + +revision = "0053" +down_revision = "0049" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + run_sql_file_migration("0053_svod2_server_paging.sql") + + +def downgrade() -> None: + pass diff --git a/api/alembic/versions/sql/0053_svod2_server_paging.sql b/api/alembic/versions/sql/0053_svod2_server_paging.sql new file mode 100644 index 0000000..40c3cac --- /dev/null +++ b/api/alembic/versions/sql/0053_svod2_server_paging.sql @@ -0,0 +1,237 @@ +-- AURORA-1843, миграция 0053, часть 1: нормализаторы поиска как в JS и компактная проекция строки свода 2. + +-- Регистр без зависимости от локали: явная таблица translate (ASCII, кириллица, Ґ, латиница-1 без U+00D7). +CREATE OR REPLACE FUNCTION v3.svod2_casefold(x text) +RETURNS text +LANGUAGE sql IMMUTABLE +AS $function$ + SELECT translate( + x, + E'\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004A\u004B\u004C\u004D\u004E\u004F\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005A\u0410\u0411\u0412\u0413\u0414\u0415\u0416\u0417\u0418\u0419\u041A\u041B\u041C\u041D\u041E\u041F\u0420\u0421\u0422\u0423\u0424\u0425\u0426\u0427\u0428\u0429\u042A\u042B\u042C\u042D\u042E\u042F\u0400\u0401\u0402\u0403\u0404\u0405\u0406\u0407\u0408\u0409\u040A\u040B\u040C\u040D\u040E\u040F\u0490\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF\u00D0\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE', + E'\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006A\u006B\u006C\u006D\u006E\u006F\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007A\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043A\u043B\u043C\u043D\u043E\u043F\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044A\u044B\u044C\u044D\u044E\u044F\u0450\u0451\u0452\u0453\u0454\u0455\u0456\u0457\u0458\u0459\u045A\u045B\u045C\u045D\u045E\u045F\u0491\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE' + ) +$function$; + +-- Обрезка как String.prototype.trim: ECMAScript WhiteSpace и LineTerminator. +CREATE OR REPLACE FUNCTION v3.svod2_js_trim(x text) +RETURNS text +LANGUAGE sql IMMUTABLE +AS $function$ + SELECT btrim(x, E'\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF') +$function$; + +-- Запись числа как JS Number::toString. Цифры берутся из float8::text при extra_float_digits = 1 +-- (кратчайшая точная запись), SET в функции изолирует результат от настроек сессии. +CREATE OR REPLACE FUNCTION v3.svod2_js_number_text(x double precision) +RETURNS text +LANGUAGE plpgsql IMMUTABLE +SET extra_float_digits = 1 +AS $function$ +DECLARE + t text; + neg boolean := false; + mant text; + ex integer := 0; + int_part text; + digits text; + lead integer; + n integer; + k integer; + res text; +BEGIN + IF x IS NULL THEN + RETURN NULL; + END IF; + IF x = 0 THEN + RETURN '0'; + END IF; + t := x::text; + IF t IN ('NaN', 'Infinity', '-Infinity') THEN + RETURN t; + END IF; + IF left(t, 1) = '-' THEN + neg := true; + t := substr(t, 2); + END IF; + IF position('e' IN t) > 0 THEN + mant := split_part(t, 'e', 1); + ex := split_part(t, 'e', 2)::integer; + ELSE + mant := t; + END IF; + IF position('.' IN mant) > 0 THEN + int_part := split_part(mant, '.', 1); + ELSE + int_part := mant; + END IF; + digits := replace(mant, '.', ''); + lead := length(digits) - length(ltrim(digits, '0')); + digits := ltrim(digits, '0'); + digits := rtrim(digits, '0'); + n := length(int_part) + ex - lead; + k := length(digits); + IF n >= k AND n <= 21 THEN + res := digits || repeat('0', n - k); + ELSIF n > 0 AND n <= 21 THEN + res := substr(digits, 1, n) || '.' || substr(digits, n + 1); + ELSIF n > -6 AND n <= 0 THEN + res := '0.' || repeat('0', -n) || digits; + ELSE + res := substr(digits, 1, 1); + IF k > 1 THEN + res := res || '.' || substr(digits, 2); + END IF; + res := res || 'e' || CASE WHEN n - 1 >= 0 THEN '+' ELSE '-' END || abs(n - 1)::text; + END IF; + IF neg THEN + res := '-' || res; + END IF; + RETURN res; +END +$function$; + +-- Компактная проекция data строки свода 2: jsonb_strip_nulls по перечисленным путям. +CREATE OR REPLACE FUNCTION v3.svod2_compact(d jsonb) +RETURNS jsonb +LANGUAGE sql IMMUTABLE +AS $function$ + SELECT jsonb_strip_nulls(jsonb_build_object( + 'line_id', d #> '{line_id}', + 'header', jsonb_build_object( + 'source_form', d #> '{header,source_form}', + 'source_id', d #> '{header,source_id}', + 'smeta_type', d #> '{header,smeta_type}', + 'smeta_direction', d #> '{header,smeta_direction}', + 'org_name', d #> '{header,org_name}', + 'name', d #> '{header,name}', + 'item_id', d #> '{header,item_id}', + 'num_group', d #> '{header,num_group}', + 'justification', d #> '{header,justification}' + ), + 'plan', jsonb_build_object( + 'plan_q1', d #> '{plan,plan_q1}', + 'plan_q2', d #> '{plan,plan_q2}', + 'plan_q3', d #> '{plan,plan_q3}', + 'plan_q4', d #> '{plan,plan_q4}' + ), + 'allocation', jsonb_build_object( + 'internal_order', d #> '{allocation,internal_order}', + 'property_object', d #> '{allocation,property_object}' + ), + 'sequestration', jsonb_build_object( + 'DFIP', jsonb_build_object( + 'adj_q1', d #> '{sequestration,DFIP,adj_q1}', + 'adj_q2', d #> '{sequestration,DFIP,adj_q2}', + 'adj_q3', d #> '{sequestration,DFIP,adj_q3}', + 'adj_q4', d #> '{sequestration,DFIP,adj_q4}' + ) + ), + 'reserve', jsonb_build_object( + 'amount_q1', d #> '{reserve,amount_q1}', + 'amount_q2', d #> '{reserve,amount_q2}', + 'amount_q3', d #> '{reserve,amount_q3}', + 'amount_q4', d #> '{reserve,amount_q4}' + ), + 'approved', jsonb_build_object( + 'approved_q1', d #> '{approved,approved_q1}', + 'approved_q2', d #> '{approved,approved_q2}', + 'approved_q3', d #> '{approved,approved_q3}', + 'approved_q4', d #> '{approved,approved_q4}' + ), + 'contract_detail', jsonb_build_object( + 'counterparty', d #> '{contract_detail,counterparty}', + 'reference', d #> '{contract_detail,reference}', + 'subject', d #> '{contract_detail,subject}', + 'currency', d #> '{contract_detail,currency}', + 'ceiling_amount', d #> '{contract_detail,ceiling_amount}', + 'expenses_q1', d #> '{contract_detail,expenses_q1}', + 'expenses_q2', d #> '{contract_detail,expenses_q2}', + 'expenses_q3', d #> '{contract_detail,expenses_q3}', + 'expenses_q4', d #> '{contract_detail,expenses_q4}', + 'vat_rate', d #> '{contract_detail,vat_rate}', + 'deadline', d #> '{contract_detail,deadline}' + ), + 'q1', jsonb_build_object( + 'adj_current', d #> '{q1,adj_current}', + 'adj_ssp', d #> '{q1,adj_ssp}', + 'adj_rf', d #> '{q1,adj_rf}', + 'adj_reserve', d #> '{q1,adj_reserve}', + 'payment_date', d #> '{q1,payment_date}', + 'payment_amount', d #> '{q1,payment_amount}', + 'payment_amount_ho', d #> '{q1,payment_amount_ho}', + 'payment_amount_rf', d #> '{q1,payment_amount_rf}', + 'payment_act', d #> '{q1,payment_act}', + 'booking_amount', d #> '{q1,booking_amount}', + 'actual_m1', d #> '{q1,actual_m1}', + 'actual_m2', d #> '{q1,actual_m2}', + 'actual_m3', d #> '{q1,actual_m3}', + 'transfer_to_q2', d #> '{q1,transfer_to_q2}', + 'transfer_to_q3', d #> '{q1,transfer_to_q3}', + 'transfer_to_q4', d #> '{q1,transfer_to_q4}', + 'transfer_to_economy', d #> '{q1,transfer_to_economy}' + ), + 'q2', jsonb_build_object( + 'adj_current', d #> '{q2,adj_current}', + 'adj_ssp', d #> '{q2,adj_ssp}', + 'adj_rf', d #> '{q2,adj_rf}', + 'adj_reserve', d #> '{q2,adj_reserve}', + 'payment_date', d #> '{q2,payment_date}', + 'payment_amount', d #> '{q2,payment_amount}', + 'payment_amount_ho', d #> '{q2,payment_amount_ho}', + 'payment_amount_rf', d #> '{q2,payment_amount_rf}', + 'payment_act', d #> '{q2,payment_act}', + 'booking_amount', d #> '{q2,booking_amount}', + 'actual_m1', d #> '{q2,actual_m1}', + 'actual_m2', d #> '{q2,actual_m2}', + 'actual_m3', d #> '{q2,actual_m3}', + 'transfer_to_q3', d #> '{q2,transfer_to_q3}', + 'transfer_to_q4', d #> '{q2,transfer_to_q4}', + 'transfer_to_economy', d #> '{q2,transfer_to_economy}', + 'plan_revision_eco_change', d #> '{q2,plan_revision_eco_change}', + 'plan_revision_item_adj', d #> '{q2,plan_revision_item_adj}', + 'plan_revision_increase', d #> '{q2,plan_revision_increase}', + 'plan_revision_sequester', d #> '{q2,plan_revision_sequester}' + ), + 'q3', jsonb_build_object( + 'adj_current', d #> '{q3,adj_current}', + 'adj_ssp', d #> '{q3,adj_ssp}', + 'adj_rf', d #> '{q3,adj_rf}', + 'adj_reserve', d #> '{q3,adj_reserve}', + 'payment_date', d #> '{q3,payment_date}', + 'payment_amount', d #> '{q3,payment_amount}', + 'payment_amount_ho', d #> '{q3,payment_amount_ho}', + 'payment_amount_rf', d #> '{q3,payment_amount_rf}', + 'payment_act', d #> '{q3,payment_act}', + 'booking_amount', d #> '{q3,booking_amount}', + 'actual_m1', d #> '{q3,actual_m1}', + 'actual_m2', d #> '{q3,actual_m2}', + 'actual_m3', d #> '{q3,actual_m3}', + 'transfer_to_q4', d #> '{q3,transfer_to_q4}', + 'transfer_to_economy', d #> '{q3,transfer_to_economy}', + 'plan_revision_eco_change', d #> '{q3,plan_revision_eco_change}', + 'plan_revision_item_adj', d #> '{q3,plan_revision_item_adj}', + 'plan_revision_increase', d #> '{q3,plan_revision_increase}', + 'plan_revision_sequester', d #> '{q3,plan_revision_sequester}' + ), + 'q4', jsonb_build_object( + 'adj_current', d #> '{q4,adj_current}', + 'adj_ssp', d #> '{q4,adj_ssp}', + 'adj_rf', d #> '{q4,adj_rf}', + 'adj_reserve', d #> '{q4,adj_reserve}', + 'payment_date', d #> '{q4,payment_date}', + 'payment_amount', d #> '{q4,payment_amount}', + 'payment_amount_ho', d #> '{q4,payment_amount_ho}', + 'payment_amount_rf', d #> '{q4,payment_amount_rf}', + 'payment_act', d #> '{q4,payment_act}', + 'booking_amount', d #> '{q4,booking_amount}', + 'actual_m1', d #> '{q4,actual_m1}', + 'actual_m2', d #> '{q4,actual_m2}', + 'actual_m3', d #> '{q4,actual_m3}', + 'actual_spod', d #> '{q4,actual_spod}', + 'plan_revision_eco_change', d #> '{q4,plan_revision_eco_change}', + 'plan_revision_item_adj', d #> '{q4,plan_revision_item_adj}', + 'plan_revision_increase', d #> '{q4,plan_revision_increase}', + 'plan_revision_sequester', d #> '{q4,plan_revision_sequester}' + ) + )) +$function$; diff --git a/api/tests/integration/test_svod2_sql.py b/api/tests/integration/test_svod2_sql.py new file mode 100644 index 0000000..4944dc4 --- /dev/null +++ b/api/tests/integration/test_svod2_sql.py @@ -0,0 +1,53 @@ +"""SQL-нормализаторы свода 2 (миграция 0053): регистр, обрезка, запись числа как в JS, компактная проекция.""" +import json +from pathlib import Path + +import pytest + +from tests.integration.conftest import legacy_full +from tests.svod2_oracle import compact_projection + +NUM = json.loads( + (Path(__file__).resolve().parent.parent / "fixtures" / "js_number_text_vectors.json").read_text( + encoding="utf-8" + ) +) + + +def test_js_number_text_matches_node_vectors(svod2_sql): + for v in NUM: + got = svod2_sql( + "SELECT v3.svod2_js_number_text((CAST(:x AS jsonb) #>> '{}')::float8) AS t", + {"x": v["input"]}, + )[0][0] + assert got == v["js"], v + + +def test_js_number_text_ignores_session_extra_float_digits(svod2_sql): + svod2_sql("SET LOCAL extra_float_digits = 0", None) + got = svod2_sql("SELECT v3.svod2_js_number_text(0.1::float8 + 0.2::float8) AS t", None)[0][0] + assert got == "0.30000000000000004" + + +@pytest.mark.parametrize( + "src", + ["АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ", "ЄІЇЎЂЏҐ", "ABCxyz", "ÀÉÖÞ", "×", "аРЕНДА 1"], +) +def test_casefold_matches_js_lower(svod2_sql, src): + assert svod2_sql("SELECT v3.svod2_casefold(:s) AS t", {"s": src})[0][0] == src.lower() + + +def test_casefold_independent_of_collation(svod2_sql): + got = svod2_sql("""SELECT v3.svod2_casefold('ЁЛКА' COLLATE "C") AS t""", None)[0][0] + assert got == "ёлка" + + +def test_js_trim_ecmascript_whitespace(svod2_sql): + assert svod2_sql("SELECT v3.svod2_js_trim(:s) AS t", {"s": "   x \t"})[0][0] == "x" + assert svod2_sql("SELECT v3.svod2_js_trim(:s) AS t", {"s": "​ x"})[0][0] == "​ x" + + +def test_compact_matches_python_projection(svod2_sql, svod2_data, client, admin_tokens, auth_headers): + full = legacy_full(client, "RF", auth_headers(admin_tokens))[0]["data"] + got = svod2_sql("SELECT v3.svod2_compact(CAST(:d AS jsonb)) AS c", {"d": json.dumps(full)})[0][0] + assert got == compact_projection(full)