allowlgroup/parser
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

добавил новые поля и работу с донесениями
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Игорь Бандурист
2026-05-01 19:44:51 +10:00
parent 843bdcfefd
commit f4cabfacbe
3 changed files with 67 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
parser_bd.py
View File

@@ -13,6 +13,10 @@ class ParsedData(BaseModel):
article_date: str article_date: str
status: Optional[bool] = False status: Optional[bool] = False
viewed: Optional[bool] = False viewed: Optional[bool] = False
tematik: Optional[bool] = False
svodka: Optional[bool] = False
donesenie: Optional[bool] = False
bilutene: Optional[bool] = False
other: str other: str
category: str category: str
translation_text: str translation_text: str
@@ -34,8 +38,8 @@ def save_parsed_data_to_db(data: ParsedData):
conn = get_connection() conn = get_connection()
with conn.cursor() as cur: with conn.cursor() as cur:
cur.execute(""" cur.execute("""
INSERT INTO url (url, parsed_at, title, original_text, article_date, status, viewed, other, category, translation_text, short_text) INSERT INTO url (url, parsed_at, title, original_text, article_date, status, viewed, tematik, svodka, donesenie, bilutene, other, category, translation_text, short_text)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (url) DO UPDATE SET ON CONFLICT (url) DO UPDATE SET
parsed_at = EXCLUDED.parsed_at, parsed_at = EXCLUDED.parsed_at,
title = EXCLUDED.title, title = EXCLUDED.title,
@@ -43,11 +47,15 @@ def save_parsed_data_to_db(data: ParsedData):
article_date = EXCLUDED.article_date, article_date = EXCLUDED.article_date,
status = EXCLUDED.status, status = EXCLUDED.status,
viewed = EXCLUDED.viewed, viewed = EXCLUDED.viewed,
tematik = EXCLUDED.tematik,
svodka = EXCLUDED.svodka,
donesenie = EXCLUDED.donesenie,
bilutene = EXCLUDED.bilutene,
other = EXCLUDED.other, other = EXCLUDED.other,
category = EXCLUDED.category, category = EXCLUDED.category,
translation_text = EXCLUDED.translation_text, translation_text = EXCLUDED.translation_text,
short_text = EXCLUDED.short_text; short_text = EXCLUDED.short_text;
""", (data.url, data.parsed_at, data.title, data.original_text, data.article_date, data.status, data.viewed, data.other, data.category, data.translation_text, data.short_text)) """, (data.url, data.parsed_at, data.title, data.original_text, data.article_date, data.status, data.viewed, data.tematik, data.svodka, data.donesenie, data.bilutene, data.other, data.category, data.translation_text, data.short_text))
conn.commit() conn.commit()
return {"status": "success", "message": "Данные успешно сохранены"} return {"status": "success", "message": "Данные успешно сохранены"}
except Exception as e: except Exception as e:

6
services/document_builder.py
View File

@@ -6,7 +6,7 @@ import os
from docx import Document from docx import Document
from config import DOCUMENTS_DIR from config import DOCUMENTS_DIR
from utils import logger from utils import logger
import parser_bd as pbd import work_parser as wp
# Обязательные поля для модели ParsedData # Обязательные поля для модели ParsedData
@@ -62,8 +62,8 @@ def update_bd_and_create_document(
data['other'] = other data['other'] = other
# Сохранение в БД через pbd # Сохранение в БД через pbd
parsed_data = pbd.ParsedData(**data) parsed_data = wp.ParsedData(**data)
pbd.save_parsed_data_to_db(parsed_data) wp.save_parsed_data_to_db(parsed_data)
print("Данные успешно сохранены в БД") print("Данные успешно сохранены в БД")
# Создание DOCX документа # Создание DOCX документа

53
work_parser.py
View File

@@ -10,6 +10,59 @@ DB_CONFIG = {
"connect_timeout": 10, "connect_timeout": 10,
"options": "-c statement_timeout=30000" "options": "-c statement_timeout=30000"
} }
# Модель для данных, которые приходят в POST
class ParsedData(BaseModel):
url: str
parsed_at: str
title: str
original_text: str
article_date: str
status: Optional[bool] = False
viewed: Optional[bool] = False
tematik: Optional[bool] = False
svodka: Optional[bool] = False
donesenie: Optional[bool] = False
bilutene: Optional[bool] = False
other: str
category: str
translation_text: str
short_text: str
# Функции для работы с БД (без эндпоинтов)
def save_parsed_data_to_db(data: ParsedData):
conn = None
try:
conn = get_connection()
with conn.cursor() as cur:
cur.execute("""
INSERT INTO url (url, parsed_at, title, original_text, article_date, status, viewed, tematik, svodka, donesenie, bilutene, other, category, translation_text, short_text)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (url) DO UPDATE SET
parsed_at = EXCLUDED.parsed_at,
title = EXCLUDED.title,
original_text = EXCLUDED.original_text,
article_date = EXCLUDED.article_date,
status = EXCLUDED.status,
viewed = EXCLUDED.viewed,
tematik = EXCLUDED.tematik,
svodka = EXCLUDED.svodka,
donesenie = EXCLUDED.donesenie,
bilutene = EXCLUDED.bilutene,
other = EXCLUDED.other,
category = EXCLUDED.category,
translation_text = EXCLUDED.translation_text,
short_text = EXCLUDED.short_text;
""", (data.url, data.parsed_at, data.title, data.original_text, data.article_date, data.status, data.viewed, data.tematik, data.svodka, data.donesenie, data.bilutene, data.other, data.category, data.translation_text, data.short_text))
conn.commit()
return {"status": "success", "message": "Данные успешно сохранены"}
except Exception as e:
if conn:
conn.rollback()
raise e
finally:
if conn:
conn.close()
# Глобальное подключение к БД # Глобальное подключение к БД
conn = None conn = None