RSHB/DKB_ARMKA
9
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'test-init' (#1) from test-init into main

Browse Source 
Reviewed-on: #1
This commit is contained in:
PotapovaA 2026-09-16 16:19:44 +03:00
commit 1667e11add
9 changed files with 2940 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
api/alembic/env.py
View File

@ -1,12 +1,27 @@
import asyncio
from logging.config import fileConfig
from pathlib import Path
from pydantic import AliasChoices, Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import AsyncEngine
from alembic import context
from src.core.config import settings
from src.models import *
class MigrationSettings(BaseSettings):
database_url: str = Field(validation_alias=AliasChoices(
"OPENBAO__DATABASE__DATABASE_URL",
"OPENBAO__SETTINGS__DATABASE_URL",
))
model_config = SettingsConfigDict(
env_file=Path(__file__).resolve().parents[1] / ".env",
extra="ignore",
)
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
@ -16,7 +31,7 @@ config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
config.set_main_option("sqlalchemy.url", settings.OPENBAO.DATABASE.DATABASE_URL)
config.set_main_option("sqlalchemy.url", MigrationSettings().database_url.replace("%", "%%"))
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
@ -45,6 +60,7 @@ def do_run_migrations(connection) -> None:
)
with context.begin_transaction():
context.execute('CREATE SCHEMA IF NOT EXISTS "dkb"')
context.run_migrations()
def run_migrations_offline() -> None:
@ -71,6 +87,7 @@ def run_migrations_offline() -> None:
)
with context.begin_transaction():
context.execute('CREATE SCHEMA IF NOT EXISTS "dkb"')
context.run_migrations()
@ -96,7 +113,9 @@ async def run_migrations_online() -> None:
await connectable.dispose()
if context.is_offline_mode():
if config.attributes.get("connection") is not None:
do_run_migrations(config.attributes["connection"])
elif context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_migrations_online())

4
api/run_server.py
View File

@ -1,5 +1,6 @@
import uvicorn
from src import app, get_settings
from src.app import app
from src.core.config import get_settings
if __name__ == "__main__":
settings = get_settings()
@ -9,4 +10,3 @@ if __name__ == "__main__":
port=settings.server.PORT,
lifespan="on",
)

8
api/src/__init__.py
View File

@ -1,7 +1 @@
from .app import app
from .core.config import get_settings
__all__ = [
'app',
'get_settings',
]
"""API package. Import application entry points explicitly from src.app."""

141
api/src/app.py
View File

@ -24,8 +24,8 @@ from src.core.clients import S3Storage, PollWorker
from src.core.async_cache import AsyncInMemoryCache
# from src.core.logger import setup_logger
from nats.aio.client import Client as NATS
import nkeys
import base64
import nkeys
from loguru import logger
import os
@ -42,20 +42,6 @@ VERSION = "1.0.0"
UP_TIME = datetime.now(timezone.utc)
kp = nkeys.from_seed(settings.OPENBAO.NATS.NATS_SEED.encode('utf-8'))
def jwt_cb():
return settings.OPENBAO.NATS.NATS_JWT.encode('utf-8')
def sig_cb(nonce):
if (isinstance(nonce, str)):
nonce = nonce.encode('utf-8')
signed = kp.sign(nonce)
return base64.b64encode(signed)
async def sub_cb(msg):
return None
@ -94,80 +80,75 @@ async def lifespan(app):
app.state.nats_subs = []
app.state.storage = None
app.state.nats = None
app.state.poller = None
app.state.pdf_poller = None
s3_client = None
smb_registered = False
try:
smbclient.register_session(settings.OPENBAO.SMB.SMB_HOST, settings.OPENBAO.SMB.SMB_LOGIN, settings.OPENBAO.SMB.SMB_PASSWORD)
except:
logger.exception("Ошибка при регистрации сессии SMB")
s3_client = Minio(settings.OPENBAO.MINIO.MINIO_URL, access_key=settings.OPENBAO.MINIO.MINIO_ADMIN_LOGIN, secret_key=settings.OPENBAO.MINIO.MINIO_ADMIN_PASSWORD, cert_check=False)
try:
bucket_exists = await s3_client.bucket_exists(settings.OPENBAO.MINIO.MINIO_BUCKET)
logger.info("Бакет проверен. bucket_exists={bucket_exists}", bucket_exists=bucket_exists)
except Exception as exc:
logger.exception(
"Не удалось проверить наличие бакетa. error_category=s3_error, error_type={error_type}, error={error}",
error_type=type(exc).__name__,
error=repr(exc),
)
return
storage = S3Storage(client=s3_client, bucket_name=settings.OPENBAO.MINIO.MINIO_BUCKET)
app.state.storage = storage
cache = AsyncInMemoryCache()
nc = NATS()
if not settings.DEBUG:
try:
await nc.connect(settings.OPENBAO.NATS.NATS_URL, user_jwt_cb=jwt_cb, signature_cb=sig_cb,
max_reconnect_attempts=2)
logger.info("Подключение к NATS произошло")
except Exception as exc:
logger.exception(
"Не удалось проверить наличие бакетa. error_category=nats_error, error_type={error_type}, error={error}",
error_type=type(exc).__name__,
error=repr(exc),
minio = settings.OPENBAO.MINIO
if minio is not None:
s3_client = Minio(
minio.MINIO_URL,
access_key=minio.MINIO_ADMIN_LOGIN,
secret_key=minio.MINIO_ADMIN_PASSWORD,
secure=minio.MINIO_SECURE,
)
return
poller = PollWorker(nc, cache, storage)
await poller.start()
pdf_poller = PDFPollWorker(cache, storage)
await pdf_poller.start()
app.state.nats = nc
app.state.poller = poller
app.state.pdf_poller = pdf_poller
if not settings.DEBUG:
handler_cb = build_file_handler(nc, storage)
if not await s3_client.bucket_exists(minio.MINIO_BUCKET):
raise RuntimeError(f"Бакет MinIO {minio.MINIO_BUCKET!r} не создан")
app.state.storage = S3Storage(s3_client, minio.MINIO_BUCKET)
logger.info("MinIO подключён, бакет: {}", minio.MINIO_BUCKET)
smb = settings.OPENBAO.SMB
if smb is not None:
smbclient.register_session(smb.SMB_HOST, smb.SMB_LOGIN, smb.SMB_PASSWORD,
port=smb.SMB_PORT)
smb_registered = True
logger.info("SMB подключён")
nats_settings = settings.OPENBAO.NATS
if nats_settings is not None:
nc = NATS()
app.state.nats = nc
if not settings.DEBUG:
keypair = nkeys.from_seed(nats_settings.NATS_SEED.encode("utf-8"))
def user_jwt_cb():
return nats_settings.NATS_JWT.encode("utf-8")
def signature_cb(nonce):
return base64.b64encode(keypair.sign(nonce.encode("utf-8")))
await nc.connect(nats_settings.NATS_URL, user_jwt_cb=user_jwt_cb,
signature_cb=signature_cb, max_reconnect_attempts=2)
logger.info("NATS подключён")
missing = [name for name in ("MINIO", "NATS", "SMB")
if getattr(settings.OPENBAO, name) is None]
if missing:
logger.info("Распознавание файлов отключено: не настроены {}", ", ".join(missing))
else:
cache = AsyncInMemoryCache()
app.state.poller = PollWorker(nc, cache, app.state.storage)
await app.state.poller.start()
app.state.pdf_poller = PDFPollWorker(cache, app.state.storage)
await app.state.pdf_poller.start()
try:
# sub_xls = await nc.subscribe('files.tabular_parsed', cb=handler_cb)
# sub_img = await nc.subscribe('files.img_parsed', cb=handler_cb)
#
# app.state.nats_subs = [sub_xls, sub_img]
logger.info("Подписка на топики завершена")
except Exception as exc:
logger.warning(
"Не удалось подписаться на тему. error_category=nats_error, error_type={error_type}, error={error}",
error_type=type(exc).__name__,
error=repr(exc),
)
try:
logger.info("Приложение запущено")
yield
except Exception as exc:
logger.exception("ОШИБКА ПРИ РАБОТЕ С СЕКРЕТАМИ error={error}", error=repr(exc))
finally:
logger.info("Приложение закрывается")
smbclient.reset_connection_cache()
for worker in (app.state.poller, app.state.pdf_poller):
if worker is not None:
await worker.stop()
if not settings.DEBUG:
for sub in app.state.nats_subs:
logger.info("Отписываемся от топика")
await sub.unsubscribe()
if app.state.nats:
if app.state.nats is not None:
await app.state.nats.close()
if app.state.poller:
await app.state.poller.stop()
if app.state.storage:
await app.state.storage.client.close_session()
if app.state.pdf_poller:
await app.state.pdf_poller.stop()
logger.info("CLOSING APP AT ALL")
if s3_client is not None:
await s3_client.close_session()
if smb_registered:
smbclient.reset_connection_cache()
logger.info("Приложение закрывается")
app = FastAPI(
title="Fast API",
@ -247,4 +228,4 @@ class SPAStaticFiles(StaticFiles):
app.mount("/", SPAStaticFiles(directory="web", html=True), name="web")
app.mount("/", StaticFiles(directory="web", html=True), name="web")
app.mount("/", StaticFiles(directory="web", html=True), name="web")

7
api/src/core/PDFWorker/PDFTempPoller.py
View File

@ -65,7 +65,7 @@ class PDFPollWorker:
logger.exception("Не получилось достать данные из кэша")
jobs = []
all_files = smbclient.listdir(self.smb_path)
all_files = smbclient.listdir(self.smb_path, port=settings.OPENBAO.SMB.SMB_PORT)
for job in jobs:
search_fname = f"{job.file_id}"
@ -85,8 +85,9 @@ class PDFPollWorker:
logger.info("Начинаю обработку файла по пути smb_dir={smb_dir}", smb_dir=new_dir)
files = []
for fname in smbclient.listdir(new_dir):
with smbclient.open_file(f'{new_dir}/{fname}', 'r') as f:
for fname in smbclient.listdir(new_dir, port=settings.OPENBAO.SMB.SMB_PORT):
with smbclient.open_file(f'{new_dir}/{fname}', 'r',
port=settings.OPENBAO.SMB.SMB_PORT) as f:
files.append(json.loads(f.read()))
s3_json_dict = {
'file_id': job.file_id,

5
api/src/core/clients.py
View File

@ -108,7 +108,8 @@ class PollWorker:
if row['subject'] == 'file.img_uploaded':
last_part = row['row_payload']['s3_key'].split('/')[-1]
with smbclient.open_file(f'{settings.OPENBAO.SMB.SMB_PATH}/Файлы/{last_part}', 'wb') as f:
with smbclient.open_file(f'{settings.OPENBAO.SMB.SMB_PATH}/Файлы/{last_part}', 'wb',
port=settings.OPENBAO.SMB.SMB_PORT) as f:
data = await self.storage.get_object(row['row_payload']['s3_key'])
content = await data.read()
f.write(content)
@ -177,4 +178,4 @@ class PollWorker:
file_stmt = update(StoredFile).where(StoredFile.current_recognition_job_id == row_id).values(status=None)
await session.execute(stmt)
await session.execute(file_stmt)
await session.commit()
await session.commit()

16
api/src/core/config.py
View File

@ -1,4 +1,5 @@
from pydantic_settings import BaseSettings, SettingsConfigDict
from pydantic import Field
class DBSettings(BaseSettings):
@ -9,23 +10,25 @@ class MinioSettings(BaseSettings):
MINIO_ADMIN_LOGIN: str
MINIO_ADMIN_PASSWORD: str
MINIO_BUCKET: str
MINIO_SECURE: bool = True
class NATSSettings(BaseSettings):
NATS_URL: str
NATS_SEED: str
NATS_JWT: str
NATS_JWT: str = Field(min_length=1)
NATS_SEED: str = Field(min_length=1)
class SMBSettings(BaseSettings):
SMB_HOST: str
SMB_PORT: int = 445
SMB_PATH: str
SMB_LOGIN: str
SMB_PASSWORD: str
class OpenBAOSettings(BaseSettings):
DATABASE: DBSettings
MINIO: MinioSettings
NATS: NATSSettings
SMB: SMBSettings
MINIO: MinioSettings | None = None
NATS: NATSSettings | None = None
SMB: SMBSettings | None = None
class ServerSettings(BaseSettings):
HOST: str = "0.0.0.0"
@ -39,6 +42,7 @@ class Settings(BaseSettings):
VERSION: str = '1.0.0'
class Config:
case_sensitive = True
extra = "ignore"
env_file = ".env"
env_nested_delimiter = "__"
@ -47,4 +51,4 @@ class Settings(BaseSettings):
def get_settings() -> Settings:
return Settings()
settings = get_settings()
settings = get_settings()

7
api/src/services/file_service.py
View File

@ -1,7 +1,7 @@
import json
import uuid
from fastapi import UploadFile
from fastapi import HTTPException, UploadFile
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.inspection import inspect
from loguru import logger
@ -21,6 +21,8 @@ class FileService:
self.session = session
async def save_file(self, file: UploadFile, legal_entity_id: uuid.UUID, quarter: str, year: int):
if self.storage is None:
raise HTTPException(status_code=503, detail="Обработка файлов не настроена")
try:
db_payload = {
'original_name': file.filename,
@ -70,6 +72,8 @@ class FileService:
return [self._get_transformed_file_job_info(new_file) for new_file in result]
async def get_file_bytes(self, file_id):
if self.storage is None:
raise HTTPException(status_code=503, detail="Файловое хранилище не настроено")
result = await self.file_repo.get_by_id(file_id)
return (await self.storage.get_object(result.s3_key))
@ -82,4 +86,3 @@ class FileService:
'original_name': stored_file_db_obj.original_name,
'document_type': stored_file_db_obj.document_type
}

2832
web/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff