fix api too low

.gitignore

@@ -161,3 +161,4 @@ cython_debug/
 data/
 bot*
 defs/glover.py
+log.txt

defs/feed.py

@@ -14,13 +14,17 @@ class UsernameNotFound(Exception):
     pass
 
 
-async def get(username: str):
+class HostNeedChange(Exception):
-    url = f"{rss_hub_host}/twitter/user/{username}"
+    pass
+
+
+async def get(username: str, host: str) -> Optional[FeedParserDict]:
+    url = f"{host}/twitter/user/{username}"
     response = await request.get(url)
     if response.status_code == 200:
         return parse(response.text)
     elif response.status_code == 404:
-        raise UsernameNotFound
+        raise HostNeedChange
     else:
         return None
 
@@ -57,8 +61,13 @@ async def parse_user(username: str, data: FeedParserDict) -> User:
 
 
 async def get_user(username: str) -> Optional[User]:
-    data = await get(username)
+    for host in rss_hub_host:
+        try:
+            data = await get(username, host)
             if data:
                 return await parse_user(username, data)
-    else:
+        except HostNeedChange:
+            if host == rss_hub_host[-1]:
+                raise UsernameNotFound
+            continue
     return None

defs/glover.py.example

@@ -1,6 +1,6 @@
 api_id = 1
 api_hash = "a"
-rss_hub_host = "https://rsshub.app"
+rss_hub_host = ["https://rsshub.app"]
 cid = 11
 tid = None
 owner = 11

defs/update.py

@@ -1,6 +1,7 @@
 import asyncio
+import time
 import traceback
-from typing import List
+from typing import List, Dict
 
 from pyrogram.enums import ParseMode
 from pyrogram.errors import FloodWait
@@ -93,12 +94,6 @@ async def send_username_changed(user: str):
     await bot.send_message(owner, text)
 
 
-@flood_wait()
-async def send_api_error():
-    text = "获取数据过多，可能 API 失效"
-    await bot.send_message(owner, text)
-
-
 async def send_check(user_data: User):
     need_send_tweets = [
         tweet for tweet in user_data.tweets
@@ -114,29 +109,51 @@ async def send_check(user_data: User):
         TweetDB.add(user_data.username, tweet.id)
 
 
+async def async_get_user(user_data: Dict, username: str) -> None:
+    try:
+        data = await get_user(username)
+        if data:
+            user_data[username] = data
+        else:
+            user_data[username] = None
+            logs.warning(f"获取 {username} 的数据失败，未知原因")
+    except UsernameNotFound:
+        logs.warning(f"获取 {username} 的数据失败，可能用户名已改变")
+        user_data[username] = UsernameNotFound
+    except Exception:
+        logs.error(f"获取 {username} 的数据失败")
+        traceback.print_exc()
+        user_data[username] = None
+
+
 async def check_update():
     logs.info("开始检查更新")
     users = UserDB.get_all()
+    users_data = {user: None for user in users}
+    tasks = [async_get_user(users_data, user) for user in users]
+    tasks_count = len(tasks) // 20 + 1 if len(tasks) % 20 else len(tasks) // 20
+    start_time = time.time()
+    for idx in range(0, len(tasks), 20):
+        tasks_group = tasks[idx:idx + 20]
+        logs.info(f"开始获取第 {idx // 20 + 1} / {tasks_count} 组用户的数据")
+        await asyncio.gather(*tasks_group)
+    logs.info(f"获取数据用时 {time.time() - start_time:.2f} 秒")
     failed_users = []
-    nums = len(users)
+    nums = len(users_data)
-    for idx, user in enumerate(users):
+    keys = list(users_data.keys())
-        try:
+    values = list(users_data.values())
-            user_data = await get_user(user)
+    for idx in range(nums):
-            if user_data:
+        username = keys[idx]
+        user_data = values[idx]
+        if isinstance(user_data, User):
             logs.info(f"获取 {user_data.name} 的数据成功，共 {len(user_data.tweets)} 条推文")
             await send_check(user_data)
-            else:
+        elif isinstance(user_data, UsernameNotFound):
-                logs.warning(f"获取 {user} 的数据失败，未知原因")
+            logs.warning(f"获取 {username} 的数据失败，可能用户名已改变")
-                failed_users.append(user)
+            failed_users.append(username)
-        except UsernameNotFound:
-            logs.warning(f"获取 {user} 的数据失败，可能用户名已改变")
-            failed_users.append(user)
-        except Exception:
-            traceback.print_exc()
         logs.info(f"处理完成，剩余 {nums - idx - 1} 个用户")
     if len(failed_users) > 5:
         logs.warning("失效数据过多，可能 API 失效")
-        await send_api_error()
     else:
         for user in failed_users:
             await send_username_changed(user)

init.py

@@ -3,7 +3,7 @@ import httpx
 
 from defs.glover import api_id, api_hash
 from scheduler import scheduler
-from logging import getLogger, INFO, ERROR, StreamHandler, basicConfig
+from logging import getLogger, INFO, ERROR, StreamHandler, basicConfig, FileHandler, Formatter
 from coloredlogs import ColoredFormatter
 
 # Enable logging
@@ -11,9 +11,12 @@ logs = getLogger("T2G")
 logging_format = "%(levelname)s [%(asctime)s] [%(name)s] %(message)s"
 logging_handler = StreamHandler()
 logging_handler.setFormatter(ColoredFormatter(logging_format))
+file_handler = FileHandler("log.txt", mode="w", encoding="utf-8")
+file_handler.setFormatter(Formatter(logging_format))
 root_logger = getLogger()
 root_logger.setLevel(ERROR)
 root_logger.addHandler(logging_handler)
+root_logger.addHandler(file_handler)
 basicConfig(level=INFO)
 logs.setLevel(INFO)