diff --git a/main.py b/main.py index 468e333..c85b27d 100644 --- a/main.py +++ b/main.py @@ -215,31 +215,50 @@ def gpt_response_message(content: str, name_promt: str): url = GPT_SERVER_URL params = {'text': contentGPT} + # Получаем список прокси для GPT (если настроен) + gpt_proxies_list = download_proxies(GPT_PROXIES_URL) + gpt_proxies_list = get_shuffled_proxies(gpt_proxies_list) + max_retries = 5 retries = 0 while retries < max_retries: - try: - response = requests.get(url, params=params, timeout=60) - return response.text - except requests.exceptions.ConnectTimeout as e: - print(f"Ошибка подключения (timeout): {e}") - logger.warning(f"gpt_response_message timeout:") #{e}") - retries += 1 - if retries < max_retries: - time.sleep(2 ** (retries - 1)) - except requests.exceptions.ConnectionError as e: - print(f"Ошибка соединения: {e}") - logger.warning(f"gpt_response_message connection error: ") #{e}") - retries += 1 - if retries < max_retries: - time.sleep(2 ** (retries - 1)) - except Exception as ex: - print(f"Ошибка при запросе к GPT: {ex}") - logger.error(f"gpt_response_message: ") #{ex}") - retries += 1 - if retries < max_retries: - time.sleep(2 ** (retries - 1)) + print(f"Попытка подключения к GPT через прокси ({len(gpt_proxies_list)} прокси доступны)") + for proxy in gpt_proxies_list: + try: + proxies = { + 'http': f'http://{proxy}', + 'https': f'http://{proxy}', + } + print(f"Прокси: {proxy}") + response = requests.get(url, params=params, timeout=60, proxies=proxies) + if response.status_code == 200: + print(f"✓ GPT-сервис ответил через прокси: {response.status_code}") + return response.text + except Exception as ex: + print(f"Ошибка через прокси {proxy}: {ex}") + continue + # try: + # response = requests.get(url, params=params, timeout=60) + # return response.text + # except requests.exceptions.ConnectTimeout as e: + # print(f"Ошибка подключения (timeout): {e}") + # logger.warning(f"gpt_response_message timeout:") #{e}") + # retries += 1 + # if retries < max_retries: + # time.sleep(2 ** (retries - 1)) + # except requests.exceptions.ConnectionError as e: + # print(f"Ошибка соединения: {e}") + # logger.warning(f"gpt_response_message connection error: ") #{e}") + # retries += 1 + # if retries < max_retries: + # time.sleep(2 ** (retries - 1)) + # except Exception as ex: + # print(f"Ошибка при запросе к GPT: {ex}") + # logger.error(f"gpt_response_message: ") #{ex}") + # retries += 1 + # if retries < max_retries: + # time.sleep(2 ** (retries - 1)) logger.info(f"Привышен лимит запросов {max_retries}") return "" @@ -402,6 +421,25 @@ def start_pars_all_istochnik(url:str, promt:str): # print(f"Начало парсинга: {url} с промтом: {promt}") task_id = wp.insert_task(status='queued', source_url=url) + # Прокси + # proxies_list = download_proxies(PROXIES_URL) + # proxies_list = get_shuffled_proxies(proxies_list) + + # # Загружаем главную страницу через прокси + # main_response = None + # for proxy in proxies_list: + # main_response = fetch_with_proxy(url, proxy=proxy, timeout=30, verify=True) + # if main_response: + # break + # else: + # try: + # main_response = requests.get(url, timeout=30).text + # except requests.RequestException as e: + # print(f"Ошибка при загрузке страницы {url}: {e}") + # wp.update_task(task_id, status='failed', finished_at=datetime.utcnow()) + # return set() + # soup = BeautifulSoup(main_response, 'html.parser') + try: response = requests.get(url) response.raise_for_status()