From fd133a7de2391b2c8f9eb73e742b41f1281c77ea Mon Sep 17 00:00:00 2001 From: xtaodada Date: Thu, 25 May 2023 20:23:38 +0800 Subject: [PATCH] fix api too low --- .gitignore | 1 + defs/feed.py | 25 +++++++++++------ defs/glover.py.example | 2 +- defs/update.py | 63 +++++++++++++++++++++++++++--------------- init.py | 5 +++- 5 files changed, 63 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 3b072b5..f7298a3 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,4 @@ cython_debug/ data/ bot* defs/glover.py +log.txt diff --git a/defs/feed.py b/defs/feed.py index 38c2116..35f498a 100644 --- a/defs/feed.py +++ b/defs/feed.py @@ -14,13 +14,17 @@ class UsernameNotFound(Exception): pass -async def get(username: str): - url = f"{rss_hub_host}/twitter/user/{username}" +class HostNeedChange(Exception): + pass + + +async def get(username: str, host: str) -> Optional[FeedParserDict]: + url = f"{host}/twitter/user/{username}" response = await request.get(url) if response.status_code == 200: return parse(response.text) elif response.status_code == 404: - raise UsernameNotFound + raise HostNeedChange else: return None @@ -57,8 +61,13 @@ async def parse_user(username: str, data: FeedParserDict) -> User: async def get_user(username: str) -> Optional[User]: - data = await get(username) - if data: - return await parse_user(username, data) - else: - return None + for host in rss_hub_host: + try: + data = await get(username, host) + if data: + return await parse_user(username, data) + except HostNeedChange: + if host == rss_hub_host[-1]: + raise UsernameNotFound + continue + return None diff --git a/defs/glover.py.example b/defs/glover.py.example index a03d657..841a1e4 100644 --- a/defs/glover.py.example +++ b/defs/glover.py.example @@ -1,6 +1,6 @@ api_id = 1 api_hash = "a" -rss_hub_host = "https://rsshub.app" +rss_hub_host = ["https://rsshub.app"] cid = 11 tid = None owner = 11 diff --git a/defs/update.py b/defs/update.py index b73a2a9..46638cd 100644 --- a/defs/update.py +++ b/defs/update.py @@ -1,6 +1,7 @@ import asyncio +import time import traceback -from typing import List +from typing import List, Dict from pyrogram.enums import ParseMode from pyrogram.errors import FloodWait @@ -93,12 +94,6 @@ async def send_username_changed(user: str): await bot.send_message(owner, text) -@flood_wait() -async def send_api_error(): - text = "获取数据过多，可能 API 失效" - await bot.send_message(owner, text) - - async def send_check(user_data: User): need_send_tweets = [ tweet for tweet in user_data.tweets @@ -114,29 +109,51 @@ async def send_check(user_data: User): TweetDB.add(user_data.username, tweet.id) +async def async_get_user(user_data: Dict, username: str) -> None: + try: + data = await get_user(username) + if data: + user_data[username] = data + else: + user_data[username] = None + logs.warning(f"获取 {username} 的数据失败，未知原因") + except UsernameNotFound: + logs.warning(f"获取 {username} 的数据失败，可能用户名已改变") + user_data[username] = UsernameNotFound + except Exception: + logs.error(f"获取 {username} 的数据失败") + traceback.print_exc() + user_data[username] = None + + async def check_update(): logs.info("开始检查更新") users = UserDB.get_all() + users_data = {user: None for user in users} + tasks = [async_get_user(users_data, user) for user in users] + tasks_count = len(tasks) // 20 + 1 if len(tasks) % 20 else len(tasks) // 20 + start_time = time.time() + for idx in range(0, len(tasks), 20): + tasks_group = tasks[idx:idx + 20] + logs.info(f"开始获取第 {idx // 20 + 1} / {tasks_count} 组用户的数据") + await asyncio.gather(*tasks_group) + logs.info(f"获取数据用时 {time.time() - start_time:.2f} 秒") failed_users = [] - nums = len(users) - for idx, user in enumerate(users): - try: - user_data = await get_user(user) - if user_data: - logs.info(f"获取 {user_data.name} 的数据成功，共 {len(user_data.tweets)} 条推文") - await send_check(user_data) - else: - logs.warning(f"获取 {user} 的数据失败，未知原因") - failed_users.append(user) - except UsernameNotFound: - logs.warning(f"获取 {user} 的数据失败，可能用户名已改变") - failed_users.append(user) - except Exception: - traceback.print_exc() + nums = len(users_data) + keys = list(users_data.keys()) + values = list(users_data.values()) + for idx in range(nums): + username = keys[idx] + user_data = values[idx] + if isinstance(user_data, User): + logs.info(f"获取 {user_data.name} 的数据成功，共 {len(user_data.tweets)} 条推文") + await send_check(user_data) + elif isinstance(user_data, UsernameNotFound): + logs.warning(f"获取 {username} 的数据失败，可能用户名已改变") + failed_users.append(username) logs.info(f"处理完成，剩余 {nums - idx - 1} 个用户") if len(failed_users) > 5: logs.warning("失效数据过多，可能 API 失效") - await send_api_error() else: for user in failed_users: await send_username_changed(user) diff --git a/init.py b/init.py index 603b0d8..3332304 100644 --- a/init.py +++ b/init.py @@ -3,7 +3,7 @@ import httpx from defs.glover import api_id, api_hash from scheduler import scheduler -from logging import getLogger, INFO, ERROR, StreamHandler, basicConfig +from logging import getLogger, INFO, ERROR, StreamHandler, basicConfig, FileHandler, Formatter from coloredlogs import ColoredFormatter # Enable logging @@ -11,9 +11,12 @@ logs = getLogger("T2G") logging_format = "%(levelname)s [%(asctime)s] [%(name)s] %(message)s" logging_handler = StreamHandler() logging_handler.setFormatter(ColoredFormatter(logging_format)) +file_handler = FileHandler("log.txt", mode="w", encoding="utf-8") +file_handler.setFormatter(Formatter(logging_format)) root_logger = getLogger() root_logger.setLevel(ERROR) root_logger.addHandler(logging_handler) +root_logger.addHandler(file_handler) basicConfig(level=INFO) logs.setLevel(INFO)