allowlgroup/parser
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ytljxtns
All checks were successful
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone Build is passing
Details

Browse Source
This commit is contained in:
admin
2026-04-11 14:10:05 +10:00
parent bf8bc173a1
commit f39ade5245
1 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
main.py
View File

@@ -336,7 +336,7 @@ def start_pars_one_istochnik(data_init=""):
print(f"Страница {page_number} [{i}/{len(hrefs)}] parsing {link}") print(f"Страница {page_number} [{i}/{len(hrefs)}] parsing {link}")
text = extract_text_from_url_one(link) text = extract_text_from_url_one(link)
if len(text) >= 100: if len(text) >= 100:
response_text = gpt_response_message(text, url_ist = "http://epaper.hljnews.cn/hljrb/pc/layout") response_text = gpt_response_message(text, url_ist = "source1")
print(response_text) print(response_text)
if response_text: if response_text:
update_bd_and_create_document(response_text=response_text, article_date=f"{current_year}/{current_month}/{current_day}", url=link, parsed_at=str(dt.now()), original_text=text, other=url) update_bd_and_create_document(response_text=response_text, article_date=f"{current_year}/{current_month}/{current_day}", url=link, parsed_at=str(dt.now()), original_text=text, other=url)
@@ -364,7 +364,7 @@ def start_pars_two_istochnik():
try: try:
text, time_text = extract_text_from_url(hrefs) text, time_text = extract_text_from_url(hrefs)
if len(text) >= 100: if len(text) >= 100:
response_text = gpt_response_message(text, url_ist = "https://def.ltn.com.tw/breakingnewslist") response_text = gpt_response_message(text, url_ist = "source2")
print(response_text) print(response_text)
if response_text: if response_text:
update_bd_and_create_document(response_text=response_text, article_date=time_text, url=hrefs, parsed_at=str(dt.now()), original_text=text, other=url) update_bd_and_create_document(response_text=response_text, article_date=time_text, url=hrefs, parsed_at=str(dt.now()), original_text=text, other=url)
@@ -483,14 +483,13 @@ def get_categories_promt():
return wp.get_all_categories_promt() return wp.get_all_categories_promt()
class Source(BaseModel): class Source(BaseModel):
url: str
name: str name: str
promt: str promt: str
# POST метод для установки настроек # POST метод для установки настроек
@app.post("/settings", summary="Метод сохранения настроек парсера") @app.post("/settings", summary="Метод сохранения настроек парсера")
def set_settings(settings: Source): def set_settings(settings: Source):
return wp.update_promt(settings.url, settings.name, settings.promt) return wp.update_promt(settings.name, settings.promt)
@app.delete("/delete_task/{task_id}", summary="Метод удаления задачи") @app.delete("/delete_task/{task_id}", summary="Метод удаления задачи")
def delete_task(task_id: int): def delete_task(task_id: int):