Xtao-Labs/Formatted-RSS-to-Telegram
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Optimize

Browse Source
This commit is contained in:
Apocalypsor 2021-05-27 18:31:26 +08:00
parent 3e19eccce7
commit df267eda99
3 changed files with 78 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
FR2T/fr2t.py
View File

@ -27,25 +27,15 @@ class FR2T:
with open(self.config_path, "r", encoding="UTF-8") as c: with open(self.config_path, "r", encoding="UTF-8") as c:
rss_config = yaml.safe_load(c) rss_config = yaml.safe_load(c)
self.database_url = ( self.database_url = os.getenv("DATABASE") or rss_config["database_url"]
os.environ["DATABASE"] self.expire_time = (
if os.getenv("DATABASE") os.getenv("EXPIRE_TIME") or rss_config.get("expire_time") or "30d"
else rss_config["database_url"] )
self.user_agent = (
os.getenv("USER-AGENT")
or rss_config.get("user-agent")
or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
) )
if os.getenv("EXPIRE_TIME"):
self.expire_time = os.environ["EXPIRE_TIME"]
elif rss_config["expire_time"]:
self.expire_time = rss_config["expire_time"]
else:
self.expire_time = "30d"
if os.getenv("USER-AGENT"):
self.user_agent = os.environ["USER-AGENT"]
elif rss_config["user-agent"]:
self.user_agent = rss_config["user-agent"]
else:
self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
self.telegram = rss_config["telegram"] self.telegram = rss_config["telegram"]
@ -57,14 +47,15 @@ class FR2T:
self.telegram.update(telegram_update) self.telegram.update(telegram_update)
if not self.telegram["disable_notification"]: self.telegram["disable_notification"] = (
self.telegram["disable_notification"] = "false" self.telegram.get("disable_notification") or "false"
)
if not self.telegram["disable_web_page_preview"]: self.telegram["disable_web_page_preview"] = (
self.telegram["disable_web_page_preview"] = "false" self.telegram.get("disable_web_page_preview") or "false"
)
if not self.telegram["parse_mode"]: self.telegram["parse_mode"] = (
self.telegram["parse_mode"] = "MarkdownV2" self.telegram.get("parse_mode") or "MarkdownV2"
)
def run(self): def run(self):
def save_sslcontext(obj): def save_sslcontext(obj):
@ -123,13 +114,14 @@ def runProcess(rss, telegram, database_url, user_agent):
client = MongoClient(database_url) client = MongoClient(database_url)
db = client["RSS"] db = client["RSS"]
if isinstance(rss["url"], str): if rss.get("name") and rss.get("url"):
handleRSS(rss, rss["url"], telegram, db, user_agent) if isinstance(rss["url"], str):
elif isinstance(rss["url"], list): handleRSS(rss, rss["url"], telegram, db, user_agent)
for url in set(rss["url"]): elif isinstance(rss["url"], list):
handleRSS(rss, url, telegram, db, user_agent) for url in set(rss["url"]):
else: handleRSS(rss, url, telegram, db, user_agent)
print("{}: Error URL!".format(rss["name"])) else:
print("{}: Error URL!".format(rss["name"]))
def handleRSS(rss, url, telegram, db, user_agent): def handleRSS(rss, url, telegram, db, user_agent):
@ -147,9 +139,7 @@ def handleRSS(rss, url, telegram, db, user_agent):
{"$set": {"expired": expired_url["expired"] + 1}}, {"$set": {"expired": expired_url["expired"] + 1}},
) )
else: else:
new_expired_url = {"url": url, "expired": 1} db["Expire"].insert_one({"url": url, "expired": 1})
db["Expire"].insert_one(new_expired_url)
else: else:
db["Expire"].update_one( db["Expire"].update_one(
{"url": url}, {"url": url},
@ -158,45 +148,43 @@ def handleRSS(rss, url, telegram, db, user_agent):
for content in rss_content: for content in rss_content:
result = {} result = {}
for rule in rss["rules"]: if rss.get("rules"):
obj = objParser(content, rule["obj"]) for rule in rss["rules"]:
if not rule.get("type"): obj = objParser(content, rule["obj"])
rule["type"] = "regex"
if rule["type"] == "regex": if not rule.get("type") or rule["type"] == "regex":
matcher = re.compile(rule["matcher"]) matcher = re.compile(rule["matcher"])
matched = matcher.search(obj) matched = matcher.search(obj)
if len(matched.groups()) == 1: if len(matched.groups()) == 1:
matched = matched.groups()[0] matched = matched.groups()[0]
else: else:
tmp = list(matched.groups()) tmp_matched = list(matched.groups())
tmp.insert(0, matched.group()) tmp_matched.insert(0, matched.group())
matched = tmp matched = tmp_matched
result[rule["dest"]] = matched result[rule["dest"]] = matched
elif rule["type"] == "func": elif rule["type"] == "func":
loc = locals() loc = locals()
tmp_func = rule["matcher"] + "\ntmp_return = func(obj)\n" tmp_func = rule["matcher"] + "\ntmp_return = func(obj)\n"
exec(tmp_func) exec(tmp_func)
result[rule["dest"]] = loc["tmp_return"] result[rule["dest"]] = loc["tmp_return"]
template = Template(rss["text"]) template = Template(rss["text"])
args = dict(**result, **content) args = dict(**result, **content, rss_name=rss["name"], rss_url=rss["url"])
escapeAll(telegram["parse_mode"], args) escapeAll(telegram["parse_mode"], args)
text = template.render(args) text = template.render(args)
id1_hash = hashlib.md5(url.encode()).hexdigest() id1_hash = hashlib.md5(url.encode()).hexdigest()
id2 = content["id"] if "id" in content else content["guid"] id2 = content.get("id") or content.get("guid") or content.get("link")
id2_hash = hashlib.md5(id2.encode()).hexdigest() id2_hash = hashlib.md5(id2.encode()).hexdigest()
id = id1_hash + id2_hash id = id1_hash + id2_hash
tmp_tg = copy.deepcopy(telegram) tmp_tg = copy.deepcopy(telegram)
if rss.get("telegram"): if rss.get("telegram"):
tmp_tg.update(rss["telegram"]) tmp_tg.update(rss["telegram"])
@ -214,12 +202,11 @@ def handleText(name, id, text, tg, db):
id_posted = db[name].find_one({"id": id}) id_posted = db[name].find_one({"id": id})
if id_posted: if id_posted:
if editToTelegram(tg, id_posted["message"], text): if editToTelegram(tg, id_posted["message"], text):
rdb = db[name].update_one( db[name].update_one(
{"_id": id_posted["_id"]}, {"_id": id_posted["_id"]},
{"$set": {"text": text_hash, "edit_time": time.time()}}, {"$set": {"text": text_hash, "edit_time": time.time()}},
) )
print(rdb.matched_count)
print( print(
"Edited 1 message: ID {} TEXT {} in {}".format( "Edited 1 message: ID {} TEXT {} in {}".format(
id_posted["message"], text_hash, name id_posted["message"], text_hash, name
@ -228,15 +215,15 @@ def handleText(name, id, text, tg, db):
else: else:
message_id = sendToTelegram(tg, text) message_id = sendToTelegram(tg, text)
if message_id: if message_id:
post = { db[name].insert_one(
"id": id, {
"message": message_id, "id": id,
"text": text_hash, "message": message_id,
"create_time": time.time(), "text": text_hash,
"edit_time": time.time(), "create_time": time.time(),
} "edit_time": time.time(),
}
db[name].insert_one(post) )
print(f"Sent 1 message: {text_hash} in {name}") print(f"Sent 1 message: {text_hash} in {name}")

4
FR2T/parser.py
View File

@ -2,9 +2,7 @@ import feedparser
def rssParser(url: str, user_agent: str): def rssParser(url: str, user_agent: str):
d = feedparser.parse(url, agent=user_agent)["entries"] return feedparser.parse(url, agent=user_agent)["entries"]
return d
def objParser(obj: dict, url: str): def objParser(obj: dict, url: str):

21
data/rss.yaml
View File

@ -1,3 +1,12 @@
template:
WebSubTemplate: &WebSubTemplate
text: |
🌐 更新文章：*{{ title }}*
💡 直达链接：{{ link }}
\#Apocalypsor的个人订阅 \#{{ rss_name }}
rss: rss:
- name: 每日精品限免 / 促销应用 - name: 每日精品限免 / 促销应用
url: https://rss.dov.moe/appstore/xianmian url: https://rss.dov.moe/appstore/xianmian
@ -100,3 +109,15 @@ rss:
💡 直达链接：{{ link }} 💡 直达链接：{{ link }}
\#Apocalypsor的动漫订阅 \#{{ tag[2] }} \#Apocalypsor的动漫订阅 \#{{ tag[2] }}
- name: 少数派
url: https://sspai.com/feed
<<: *WebSubTemplate
- name: 小众软件
url: https://www.appinn.com/feed/
<<: *WebSubTemplate
- name: 趣乐子
url: https://www.7lez.com/?feed=rss2
<<: *WebSubTemplate