✨ 推送支持过滤重复图片

defs/post.py

@@ -2,6 +2,7 @@ import asyncio
 import contextlib
 import os
 import re
+import time
 from asyncio import sleep
 from io import BytesIO
 from typing import List, Dict
@@ -11,9 +12,11 @@ from pyrogram.types import Message
 
 from defs.glover import lofter_channel
 from defs.lofter import lofter_link
+from models.lofter import LofterPost as LofterPostModel
+from models.models.lofter import Lofter as LofterModel
 from init import request, bot
 
-pattern = re.compile(r'<[^>]+>',re.S)
+pattern = re.compile(r'<[^>]+>', re.S)
 
 
 class LofterPost:
@@ -36,10 +39,13 @@ class LofterPost:
         return res.json()
 
     class Item:
-        def __init__(self, url, origin_url, title, username, name, tags, comment, static):
+        def __init__(self, url, origin_url, title, user_id, username, name, tags, comment, post_id, first, static):
             self.url = url.split('?')[0]
             self.origin_url = origin_url
+            self.user_id = str(user_id)
             self.username = username
+            self.post_id = post_id
+            self.first = first
             self.static = static
             title = pattern.sub('\n', title).strip()[:500]
             self.text = f"<b>Lofter Status Info</b>\n\n" \
@@ -48,6 +54,18 @@ class LofterPost:
                         f"{tags}\n" \
                         f"{comment}"
 
+        async def check_exists(self):
+            return await LofterPostModel.get_by_post_and_user_id(self.user_id, self.post_id)
+
+        async def add_to_db(self):
+            post = LofterModel(
+                user_id=self.user_id,
+                username=self.username,
+                post_id=self.post_id,
+                timestamp=int(time.time())
+            )
+            await LofterPostModel.add_post(post)
+
         async def init(self):
             file = await request.get(self.url, timeout=30)
             file = BytesIO(file.content)
@@ -71,8 +89,9 @@ class LofterPost:
         datas = []
         for i in data:
             if post_data := i.get("postData"):
-                username, name, comment = "", "", ""
+                user_id, username, name, comment = 0, "", "", ""
                 if blog_info := post_data.get("blogInfo"):
+                    user_id = blog_info.get("blogId", 0)
                     username = blog_info.get("blogName", "")
                     name = blog_info.get("blogNickName", "")
                 if post_count_view := post_data.get("postCountView"):
@@ -87,6 +106,7 @@ class LofterPost:
                     tags = "".join(f"#{i} " for i in post_view.get("tagList", []))
                     if photo_post_view := post_view.get("photoPostView"):
                         if photo_links := photo_post_view.get("photoLinks"):
+                            first = True
                             for photo in photo_links:
                                 if url := photo.get("orign"):
                                     width = photo.get("ow", 0)
@@ -96,12 +116,16 @@ class LofterPost:
                                         url,
                                         origin_url,
                                         title,
+                                        user_id,
                                         username,
                                         name,
                                         tags,
                                         comment,
+                                        permalink,
+                                        first,
                                         static
                                     ))
+                                    first = False
         return datas
 
     async def get_items(self) -> List[Item]:
@@ -117,7 +141,8 @@ class LofterPost:
 
     async def upload(self, message: Message):
         msg = await message.reply_text("正在上传中...")
-        success, error = 0, 0
+        success, error, skip = 0, 0, 0
+        temp_skip = False
         while True:
             try:
                 items = await self.get_items()
@@ -125,17 +150,32 @@ class LofterPost:
                     break
                 for item in items:
                     try:
+                        if item.first:
+                            if await item.check_exists():
+                                temp_skip = True
+                                skip += 1
+                                await sleep(0.5)
+                                continue
+                            else:
+                                temp_skip = False
+                        elif temp_skip:
+                            skip += 1
+                            continue
                         file = await item.init()
                         await item.upload(file)
+                        if item.first:
+                            await item.add_to_db()
                         success += 1
-                        await sleep(1)
-                    except Exception:
+                        await sleep(0.5)
+                    except Exception as e:
+                        print(f"Error uploading file: {e}")
                         error += 1
                     if (success + error) % 10 == 0:
                         with contextlib.suppress(Exception):
-                            await msg.edit(f"已成功上传{success}条，失败{error}条，第 {success + error} 条")
+                            await msg.edit(f"已成功上传{success}条，失败{error}条，跳过 {skip} 条，第 {success + error + skip} 条")
                 if self.offset == -1:
                     break
-            except Exception:
+            except Exception as e:
+                print(f"Error uploading file: {e}")
                 continue
-        await msg.edit(f"上传完成，成功{success}条，失败{error}条")
+        await msg.edit(f"上传完成，成功{success}条，失败{error}条，跳过 {skip} 条，总共 {success + error + skip} 条")

init.py

@@ -2,6 +2,7 @@ import logging
 
 import httpx
 
+from models.sqlite import Sqlite
 from defs.glover import api_id, api_hash, ipv6
 from scheduler import scheduler
 from pyrogram import Client
@@ -34,6 +35,7 @@ class UserMe:
 
 
 user_me = UserMe()
+sqlite = Sqlite()
 bot = Client("bot", api_id=api_id, api_hash=api_hash, ipv6=ipv6, plugins=dict(root="modules"))
 headers = {
     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.72 Safari/537.36"

main.py

@@ -1,6 +1,12 @@
-from init import logs, user_me, bot
+from pyrogram import idle
+
+from init import logs, user_me, bot, sqlite
 
 
 if __name__ == "__main__":
     logs.info(f"@{user_me.username} 运行成功！")
-    bot.run()
+    bot.start()
+    bot.loop.create_task(sqlite.create_db_and_tables())
+    idle()
+    bot.stop()
+    sqlite.stop()

models/lofter.py

@@ -0,0 +1,31 @@
+from typing import cast, Optional
+
+from sqlalchemy import select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from init import sqlite
+from models.models.lofter import Lofter
+
+
+class LofterPost:
+    @staticmethod
+    async def get_by_post_and_user_id(user_id: str, post_id: str) -> Optional[Lofter]:
+        async with sqlite.Session() as session:
+            session = cast(AsyncSession, session)
+            if user_id != "0":
+                check = Lofter.post_id == post_id and Lofter.user_id == user_id
+            else:
+                check = Lofter.post_id == post_id
+            statement = select(Lofter).where(check)
+            results = await session.exec(statement)
+            if post := results.first():
+                return post[0]
+            else:
+                return None
+
+    @staticmethod
+    async def add_post(post: Lofter):
+        async with sqlite.Session() as session:
+            session = cast(AsyncSession, session)
+            session.add(post)
+            await session.commit()

models/models/lofter.py

@@ -0,0 +1,10 @@
+from sqlmodel import SQLModel, Field
+
+
+class Lofter(SQLModel, table=True):
+    __table_args__ = dict(mysql_charset='utf8mb4', mysql_collate="utf8mb4_general_ci")
+
+    user_id: str = Field(primary_key=True)
+    username: str = Field()
+    post_id: str = Field(primary_key=True)
+    timestamp: int = Field(default=0)

models/sqlite.py

@@ -0,0 +1,26 @@
+from sqlmodel import SQLModel
+
+from models.models.lofter import Lofter
+
+__all__ = ["Lofter"]
+
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.orm import sessionmaker
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+
+class Sqlite:
+    def __init__(self):
+        self.engine = create_async_engine("sqlite+aiosqlite:///data/data.db")
+        self.Session = sessionmaker(bind=self.engine, class_=AsyncSession)
+
+    async def create_db_and_tables(self):
+        async with self.engine.begin() as session:
+            await session.run_sync(SQLModel.metadata.create_all)
+
+    async def get_session(self):
+        async with self.Session() as session:
+            yield session
+
+    def stop(self):
+        self.Session.close_all()

modules/lofter.py

@@ -28,6 +28,8 @@ async def lofter_share(_: Client, message: Message):
                     await img[0].reply_to(message, static=static)
                 else:
                     await message.reply_media_group(media=await input_media(img[:9], static), quote=True)
+            elif "front/blog" in url:
+                pass
             else:
                 text, avatar, username, status_link = await get_loft_user(url)
                 if avatar:

requirements.txt

@@ -15,3 +15,6 @@ pytz
 python-twitter
 beautifulsoup4
 lxml
+sqlalchemy
+sqlmodel
+aiosqlite