From c5e61ee01ee64659081c4ffc538a17440b6ded4f Mon Sep 17 00:00:00 2001 From: Apocalypsor <37431235+Apocalypsor@users.noreply.github.com> Date: Wed, 26 May 2021 17:32:39 +0800 Subject: [PATCH] Update --- .github/dependabot.yml | 8 + .github/pull.yml | 5 + .github/workflows/Clean-Workflows.yaml | 17 ++ .github/workflows/Purge-Database.yaml | 30 +++ .github/workflows/Subscribe.yaml | 36 ++++ .gitignore | 130 +++++++++++++ FR2T/__init__.py | 0 FR2T/fr2t.py | 242 +++++++++++++++++++++++++ FR2T/parser.py | 16 ++ FR2T/utils.py | 66 +++++++ Pipfile | 17 ++ Pipfile.lock | 235 ++++++++++++++++++++++++ clean.py | 8 + data/config.yaml | 10 + data/rss.yaml | 102 +++++++++++ main.py | 8 + test/example.json | 1 + test/rss.py | 8 + 18 files changed, 939 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/pull.yml create mode 100644 .github/workflows/Clean-Workflows.yaml create mode 100644 .github/workflows/Purge-Database.yaml create mode 100644 .github/workflows/Subscribe.yaml create mode 100644 .gitignore create mode 100644 FR2T/__init__.py create mode 100644 FR2T/fr2t.py create mode 100644 FR2T/parser.py create mode 100644 FR2T/utils.py create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 clean.py create mode 100644 data/config.yaml create mode 100644 data/rss.yaml create mode 100644 main.py create mode 100644 test/example.json create mode 100644 test/rss.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e324bea --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + timezone: Asia/Shanghai + open-pull-requests-limit: 10 diff --git a/.github/pull.yml b/.github/pull.yml new file mode 100644 index 0000000..9ffe59c --- /dev/null +++ b/.github/pull.yml @@ -0,0 +1,5 @@ +version: "1" +rules: + - base: master + upstream: Apocalypsor:master + mergeMethod: merge \ No newline at end of file diff --git a/.github/workflows/Clean-Workflows.yaml b/.github/workflows/Clean-Workflows.yaml new file mode 100644 index 0000000..2ca281c --- /dev/null +++ b/.github/workflows/Clean-Workflows.yaml @@ -0,0 +1,17 @@ +name: Clean Workflow + +on: + workflow_dispatch: + + schedule: + - cron: '26 19 */7 * *' + +jobs: + del_runs: + runs-on: ubuntu-latest + steps: + - name: Delete workflow runs + uses: GitRML/delete-workflow-runs@main + with: + token: ${{ secrets.REPO_TOKEN }} + retain_days: 1 diff --git a/.github/workflows/Purge-Database.yaml b/.github/workflows/Purge-Database.yaml new file mode 100644 index 0000000..19c5023 --- /dev/null +++ b/.github/workflows/Purge-Database.yaml @@ -0,0 +1,30 @@ +name: Purge-Database + +on: + workflow_dispatch: + + schedule: + - cron: "44 2 */30 * *" + +jobs: + Purge: + name: Purge Database + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + pip install pipenv + pipenv lock --requirements > requirements.txt + pip install -r requirements.txt + - name: Purge + env: + DATABASE: ${{ secrets.DATABASE }} + EXPIRE_TIME: ${{ secrets.EXPIRE_TIME }} + run: | + python ./clean.py || echo 'Check your configuration.' diff --git a/.github/workflows/Subscribe.yaml b/.github/workflows/Subscribe.yaml new file mode 100644 index 0000000..8b6f998 --- /dev/null +++ b/.github/workflows/Subscribe.yaml @@ -0,0 +1,36 @@ +name: Subscribe + +on: + workflow_dispatch: + + schedule: + - cron: "*/6 * * * *" + +jobs: + FR2T: + name: Formatted RSS to Telegram + runs-on: ubuntu-latest + timeout-minutes: 6 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + pip install pipenv + pipenv lock --requirements > requirements.txt + pip install -r requirements.txt + - name: Subscribe + env: + DATABASE: ${{ secrets.DATABASE }} + TG_TOKEN: ${{ secrets.TG_TOKEN }} + TG_CHAT_ID: ${{ secrets.TG_CHAT_ID }} + TG_DISABLE_NOTIFICATION: ${{ secrets.TG_DISABLE_NOTIFICATION }} + TG_DISABLE_WEB_PAGE_PREVIEW: ${{ secrets.TG_DISABLE_WEB_PAGE_PREVIEW }} + TG_PARSE_MODE: ${{ secrets.TG_PARSE_MODE }} + EXPIRE_TIME: ${{ secrets.EXPIRE_TIME }} + run: | + python ./main.py || echo 'Check your configuration.' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3b99e4d --- /dev/null +++ b/.gitignore @@ -0,0 +1,130 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +/.idea/ \ No newline at end of file diff --git a/FR2T/__init__.py b/FR2T/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FR2T/fr2t.py b/FR2T/fr2t.py new file mode 100644 index 0000000..b470cea --- /dev/null +++ b/FR2T/fr2t.py @@ -0,0 +1,242 @@ +from .parser import rssParser, objParser +from .utils import postData, escapeAll, escapeText +from jinja2 import Template +from pymongo import MongoClient +import datetime +import hashlib +import time +import re +import yaml +from multiprocessing import Pool +import ssl +import copyreg +import copy +import os + + +class FR2T: + def __init__(self, config_path, rss_path): + self.config_path = config_path + self.rss_path = rss_path + self.loadConfig() + + + def loadConfig(self): + with open(self.rss_path, "r") as c: + self.config = yaml.safe_load(c) + + with open(self.config_path, "r") as c: + rss_config = yaml.safe_load(c) + + self.database_url = os.environ["DATABASE"] if os.getenv("DATABASE") else rss_config["database_url"] + self.expire_time = os.environ["EXPIRE_TIME"] if os.getenv("EXPIRE_TIME") else rss_config["expire_time"] + + self.telegram = rss_config["telegram"] + telegram_update = {} + for up in self.telegram: + up_v = os.getenv("TG_" + up.upper()) + if up_v: + telegram_update[up] = up_v + + self.telegram.update(telegram_update) + if not self.telegram["disable_notification"]: + self.telegram["disable_notification"] = "false" + + if not self.telegram["disable_web_page_preview"]: + self.telegram["disable_web_page_preview"] = "false" + + if not self.telegram["parse_mode"]: + self.telegram["parse_mode"] = "MarkdownV2" + + def run(self): + def save_sslcontext(obj): + return obj.__class__, (obj.protocol,) + + copyreg.pickle(ssl.SSLContext, save_sslcontext) + + args = [(r, self.telegram, self.database_url) for r in self.config["rss"]] + + with Pool(8) as p: + p.map(mixInput, args) + + print("Finished!") + + def purge(self): + now_time = datetime.datetime.now() + days = hours = 0 + if self.expire_time.endswith("y"): + days = int(self.expire_time.strip("y")) * 365 + + if self.expire_time.endswith("m"): + days = int(self.expire_time.strip("m")) * 30 + + if self.expire_time.endswith("d"): + days = int(self.expire_time.strip("d")) + + if self.expire_time.endswith("h"): + hours = (self.expire_time.strip("h")) + + expired_time = now_time - datetime.timedelta(days=days, hours=hours) + expired_timestamp = datetime.datetime.timestamp(expired_time) + + deleted_num = 0 + client = MongoClient(self.database_url) + db = client["RSS"] + col_list = db.list_collection_names() + for col_name in col_list: + print(f"开始清理: {col_name}") + col = db[col_name] + purge_rule = {"create_time": { "$lt": expired_timestamp }} + + deleted_result = col.delete_many(purge_rule) + deleted_num += deleted_result.deleted_count + + print(f"已删除 {deleted_num} 个记录！") + + +def mixInput(mix_args): + runProcess(mix_args[0], mix_args[1], mix_args[2]) + + +def runProcess(rss, telegram, database_url): + client = MongoClient(database_url) + db = client["RSS"] + + if isinstance(rss["url"], str): + handleRSS(rss, rss["url"], telegram, db) + elif isinstance(rss["url"], list): + for url in set(rss["url"]): + handleRSS(rss, url, telegram, db) + else: + print("{}: Error URL!".format(rss["name"])) + + +def handleRSS(rss, url, telegram, db): + rss_content = rssParser(url) + if not rss_content: + msg = escapeText(telegram["parse_mode"], url) + print(f"订阅 {url} 已失效") + sendToTelegram(telegram, f"订阅 {msg} 已失效\n\n\#提醒") + else: + for content in rss_content: + result = {} + + for rule in rss["rules"]: + obj = objParser(content, rule["obj"]) + if not rule.get("type"): + rule["type"] = "regex" + + if rule["type"] == "regex": + matcher = re.compile(rule["matcher"]) + matched = matcher.search(obj) + + if len(matched.groups()) == 1: + matched = matched.groups()[0] + else: + tmp = list(matched.groups()) + tmp.insert(0, matched.group()) + matched = tmp + + result[rule["dest"]] = matched + elif rule["type"] == "func": + loc = locals() + tmp_func = rule["matcher"] + "\ntmp_return = func(obj)\n" + exec(tmp_func) + result[rule["dest"]] = loc["tmp_return"] + + template = Template(rss["text"]) + + args = dict(**result, **content) + escapeAll(telegram["parse_mode"], args) + + text = template.render(args) + + id1_hash = hashlib.md5(url.encode()).hexdigest() + + id2 = content["id"] if "id" in content else content["guid"] + id2_hash = hashlib.md5(id2.encode()).hexdigest() + + id = id1_hash + id2_hash + + tmp_tg = copy.deepcopy(telegram) + + if rss.get("telegram"): + tmp_tg.update(rss["telegram"]) + + handleText(rss["name"], id, text, tmp_tg, db) + + +def handleText(name, id, text, tg, db): + text_hash = hashlib.md5(text.encode()).hexdigest() + + text_posted = db[name].find_one({"text": text_hash}) + + if not text_posted: + time.sleep(1) + + id_posted = db[name].find_one({"id": id}) + if id_posted: + if editToTelegram(tg, id_posted["message"], text): + db[name].update_one( + {"_id": str(id_posted["_id"])}, + {"$set": {"text": text_hash, "edit_time": time.time()}}, + ) + + print("Edited 1 message: in", name) + else: + message_id = sendToTelegram(tg, text) + if message_id: + post = { + "id": id, + "message": message_id, + "text": text_hash, + "create_time": time.time(), + "edit_time": time.time(), + } + + db[name].insert_one(post) + + print("Sent 1 message in:", name) + + +def sendToTelegram(tg, text): + url = "https://api.telegram.org/bot" + tg["token"] + "/sendMessage" + payload = { + "chat_id": tg["chat_id"], + "text": text, + "parse_mode": tg["parse_mode"], + "disable_web_page_preview": tg["disable_web_page_preview"], + "disable_notification": tg["disable_notification"], + } + + r = postData(url, data=payload) + + if r.json()["ok"]: + return r.json()["result"]["message_id"] + elif r.json()["error_code"] == 429: + print("\nToo frequently! Sleep 30s.\n") + time.sleep(30) + sendToTelegram(tg, text) + else: + print("\nError: failed to sending message:") + print(text) + print(r.json()["description"] + "\n") + return False + + +def editToTelegram(tg, message_id, text): + url = "https://api.telegram.org/bot" + tg["token"] + "/editMessageText" + payload = { + "chat_id": tg["chat_id"], + "message_id": message_id, + "text": text, + "parse_mode": tg["parse_mode"], + "disable_web_page_preview": tg["disable_web_page_preview"], + } + + r = postData(url, data=payload) + + if r.json()["ok"]: + return True + else: + return False diff --git a/FR2T/parser.py b/FR2T/parser.py new file mode 100644 index 0000000..c1be47b --- /dev/null +++ b/FR2T/parser.py @@ -0,0 +1,16 @@ +import feedparser + + +def rssParser(url: str): + d = feedparser.parse(url)["entries"] + + return d + + +def objParser(obj: dict, url: str): + paths = url.split(".") + + for p in paths: + obj = obj[p] + + return obj diff --git a/FR2T/utils.py b/FR2T/utils.py new file mode 100644 index 0000000..1cce2d2 --- /dev/null +++ b/FR2T/utils.py @@ -0,0 +1,66 @@ +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +import requests + + +def postData(url, data, headers=None, retry=5, timeout=10): + retry_strategy = Retry(total=retry, backoff_factor=0.1) + + adapter = HTTPAdapter(max_retries=retry_strategy) + http = requests.Session() + http.mount("https://", adapter) + http.mount("http://", adapter) + + response = http.post(url, data=data, headers=headers, timeout=timeout) + + return response + + +def escapeText(mode, text): + escaped_chara = () + + if mode.lower() == "markdownv2": + escaped_chara = ( + "_", + "*", + "[", + "]", + "(", + ")", + "~", + "`", + ">", + "#", + "+", + "-", + "=", + "|", + "{", + "}", + ".", + "!", + ) + + elif mode.lower() == "markdown": + escaped_chara = ("_", "*", "`", "[") + + for e in escaped_chara: + text = text.replace(e, "\\" + e) + + return text + + +def escapeAll(mode, obj): + if isinstance(obj, str): + escaped = escapeText(mode, obj) + return escaped + + elif isinstance(obj, list): + for o in range(len(obj)): + obj[o] = escapeAll(mode, obj[o]) + return obj + + elif isinstance(obj, dict): + for k, v in obj.items(): + obj[k] = escapeAll(mode, v) + return obj diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..17760ee --- /dev/null +++ b/Pipfile @@ -0,0 +1,17 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +feedparser = "*" +pyyaml = "*" +jinja2 = "*" +requests = "*" +pymongo = "*" +dnspython = "*" + +[dev-packages] + +[requires] +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..a5102e6 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,235 @@ +{ + "_meta": { + "hash": { + "sha256": "2ca9ec84a3ecb99a485a86251e5473608e2e777fc09d76537c4b2a73e2ddf998" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "dnspython": { + "hashes": [ + "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216", + "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "feedparser": { + "hashes": [ + "sha256:1b00a105425f492f3954fd346e5b524ca9cef3a4bbf95b8809470e9857aa1074", + "sha256:f596c4b34fb3e2dc7e6ac3a8191603841e8d5d267210064e94d4238737452ddd" + ], + "index": "pypi", + "version": "==6.0.2" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "jinja2": { + "hashes": [ + "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4", + "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4" + ], + "index": "pypi", + "version": "==3.0.1" + }, + "markupsafe": { + "hashes": [ + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "pymongo": { + "hashes": [ + "sha256:03be7ad107d252bb7325d4af6309fdd2c025d08854d35f0e7abc8bf048f4245e", + "sha256:071552b065e809d24c5653fcc14968cfd6fde4e279408640d5ac58e3353a3c5f", + "sha256:08b8723248730599c9803ae4c97b8f3f76c55219104303c88cb962a31e3bb5ee", + "sha256:08bda7b2c522ff9f1e554570da16298271ebb0c56ab9699446aacba249008988", + "sha256:0aaf4d44f1f819360f9432df538d54bbf850f18152f34e20337c01b828479171", + "sha256:0cabfc297f4cf921f15bc789a8fbfd7115eb9f813d3f47a74b609894bc66ab0d", + "sha256:13acf6164ead81c9fc2afa0e1ea6d6134352973ce2bb35496834fee057063c04", + "sha256:15b083d1b789b230e5ac284442d9ecb113c93f3785a6824f748befaab803b812", + "sha256:161fcd3281c42f644aa8dec7753cca2af03ce654e17d76da4f0dab34a12480ca", + "sha256:1a994a42f49dab5b6287e499be7d3d2751776486229980d8857ad53b8333d469", + "sha256:20d75ea11527331a2980ab04762a9d960bcfea9475c54bbeab777af880de61cd", + "sha256:225c61e08fe517aede7912937939e09adf086c8e6f7e40d4c85ad678c2c2aea3", + "sha256:3135dd574ef1286189f3f04a36c8b7a256376914f8cbbce66b94f13125ded858", + "sha256:3491c7de09e44eded16824cb58cf9b5cc1dc6f066a0bb7aa69929d02aa53b828", + "sha256:3551912f5c34d8dd7c32c6bb00ae04192af47f7b9f653608f107d19c1a21a194", + "sha256:38a7b5140a48fc91681cdb5cb95b7cd64640b43d19259fdd707fa9d5a715f2b2", + "sha256:3a3498a8326111221560e930f198b495ea6926937e249f475052ffc6893a6680", + "sha256:3bfc7689a1bacb9bcd2f2d5185d99507aa29f667a58dd8adaa43b5a348139e46", + "sha256:421d13523d11c57f57f257152bc4a6bb463aadf7a3918e9c96fefdd6be8dbfb8", + "sha256:424799c71ff435094e5fb823c40eebb4500f0e048133311e9c026467e8ccebac", + "sha256:474e21d0e07cd09679e357d1dac76e570dab86665e79a9d3354b10a279ac6fb3", + "sha256:4c7e8c8e1e1918dcf6a652ac4b9d87164587c26fd2ce5dd81e73a5ab3b3d492f", + "sha256:506a6dab4c7ffdcacdf0b8e70bd20eb2e77fa994519547c9d88d676400fcad58", + "sha256:510cd3bfabb63a07405b7b79fae63127e34c118b7531a2cbbafc7a24fd878594", + "sha256:517ba47ca04a55b1f50ee8df9fd97f6c37df5537d118fb2718952b8623860466", + "sha256:539d4cb1b16b57026999c53e5aab857fe706e70ae5310cc8c232479923f932e6", + "sha256:5c36428cc4f7fae56354db7f46677fd21222fc3cb1e8829549b851172033e043", + "sha256:5db59223ed1e634d842a053325f85f908359c6dac9c8ddce8ef145061fae7df8", + "sha256:5e606846c049ed40940524057bfdf1105af6066688c0e6a1a3ce2038589bae70", + "sha256:6060794aac9f7b0644b299f46a9c6cbc0bc470bd01572f4134df140afd41ded6", + "sha256:62c29bc36a6d9be68fe7b5aaf1e120b4aa66a958d1e146601fcd583eb12cae7b", + "sha256:73326b211e7410c8bd6a74500b1e3f392f39cf10862e243d00937e924f112c01", + "sha256:78f07961f4f214ea8e80be63cffd5cc158eb06cd922ffbf6c7155b11728f28f9", + "sha256:7c97554ea521f898753d9773891d0347ebfaddcc1dee2ad94850b163171bf1f1", + "sha256:8898f6699f740ca93a0879ed07d8e6db02d68af889d0ebb3d13ab017e6b1af1e", + "sha256:8a41fdc751dc4707a4fafb111c442411816a7c225ebb5cadb57599534b5d5372", + "sha256:8e0004b0393d72d76de94b4792a006cb960c1c65c7659930fbf9a81ce4341982", + "sha256:977b1d4f868986b4ba5d03c317fde4d3b66e687d74473130cd598e3103db34fa", + "sha256:9a4f6e0b01df820ba9ed0b4e618ca83a1c089e48d4f268d0e00dcd49893d4549", + "sha256:9b9298964389c180a063a9e8bac8a80ed42de11d04166b20249bfa0a489e0e0f", + "sha256:a08c8b322b671857c81f4c30cd3c8df2895fd3c0e9358714f39e0ef8fb327702", + "sha256:ad31f184dcd3271de26ab1f9c51574afb99e1b0e484ab1da3641256b723e4994", + "sha256:aff3656af2add93f290731a6b8930b23b35c0c09569150130a58192b3ec6fc61", + "sha256:b2f41261b648cf5dee425f37ff14f4ad151c2f24b827052b402637158fd056ef", + "sha256:b413117210fa6d92664c3d860571e8e8727c3e8f2ff197276c5d0cb365abd3ad", + "sha256:b7efc7e7049ef366777cfd35437c18a4166bb50a5606a1c840ee3b9624b54fc9", + "sha256:b8f94acd52e530a38f25e4d5bf7ddfdd4bea9193e718f58419def0d4406b58d3", + "sha256:d0a70151d7de8a3194cdc906bcc1a42e14594787c64b0c1c9c975e5a2af3e251", + "sha256:d360e5d5dd3d55bf5d1776964625018d85b937d1032bae1926dd52253decd0db", + "sha256:d4e62417e89b717a7bcd8576ac3108cd063225942cc91c5b37ff5465fdccd386", + "sha256:d65bac5f6724d9ea6f0b5a0f0e4952fbbf209adcf6b5583b54c54bd2fcd74dc0", + "sha256:e02beaab433fd1104b2804f909e694cfbdb6578020740a9051597adc1cd4e19f", + "sha256:e4b631688dfbdd61b5610e20b64b99d25771c6d52d9da73349342d2a0f11c46a", + "sha256:e4e9db78b71db2b1684ee4ecc3e32c4600f18cdf76e6b9ae03e338e52ee4b168", + "sha256:eb4d176394c37a76e8b0afe54b12d58614a67a60a7f8c0dd3a5afbb013c01092", + "sha256:f08665d3cc5abc2f770f472a9b5f720a9b3ab0b8b3bb97c7c1487515e5653d39", + "sha256:f3d851af3852f16ad4adc7ee054fd9c90a7a5063de94d815b7f6a88477b9f4c6", + "sha256:f4ba58157e8ae33ee86fadf9062c506e535afd904f07f9be32731f4410a23b7f", + "sha256:f664ed7613b8b18f0ce5696b146776266a038c19c5cd6efffa08ecc189b01b73", + "sha256:f947b359cc4769af8b49be7e37af01f05fcf15b401da2528021148e4a54426d1", + "sha256:fe4189846448df013cd9df11bba38ddf78043f8c290a9f06430732a7a8601cce", + "sha256:fea5cb1c63efe1399f0812532c7cf65458d38fd011be350bc5021dfcac39fba8", + "sha256:fedf0dee7a412ca6d1d6d92c158fe9cbaa8ea0cae90d268f9ccc0744de7a97d0", + "sha256:fffff7bfb6799a763d3742c59c6ee7ffadda21abed557637bc44ed1080876484" + ], + "index": "pypi", + "version": "==3.11.4" + }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "index": "pypi", + "version": "==5.4.1" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "index": "pypi", + "version": "==2.25.1" + }, + "sgmllib3k": { + "hashes": [ + "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9" + ], + "version": "==1.0.0" + }, + "urllib3": { + "hashes": [ + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.4" + } + }, + "develop": {} +} diff --git a/clean.py b/clean.py new file mode 100644 index 0000000..49a977f --- /dev/null +++ b/clean.py @@ -0,0 +1,8 @@ +from FR2T import fr2t + +if __name__ == "__main__": + config_path = "data/config.yaml" + rss_path = "data/rss.yaml" + + fr = fr2t.FR2T(config_path, rss_path) + fr.purge() diff --git a/data/config.yaml b/data/config.yaml new file mode 100644 index 0000000..2195794 --- /dev/null +++ b/data/config.yaml @@ -0,0 +1,10 @@ +database_url: + +expire_time: + +telegram: + token: + chat_id: + disable_notification: + disable_web_page_preview: + parse_mode: \ No newline at end of file diff --git a/data/rss.yaml b/data/rss.yaml new file mode 100644 index 0000000..6f66604 --- /dev/null +++ b/data/rss.yaml @@ -0,0 +1,102 @@ +rss: + - name: 每日精品限免 / 促销应用 + url: https://rss.dov.moe/appstore/xianmian + telegram: + disable_web_page_preview: true + rules: + - obj: summary + type: regex + matcher: 原价：(.+) -> 现价：(.+) + dest: price + - obj: summary + type: regex + matcher: 平台：(.+) + dest: platform + - obj: summary + type: regex + matcher: \n *(.+)$ + dest: description + text: | + 🧬 App优惠：*{{ title }}* + + 🚀 价格：{{ price[1] }} —\> {{ price[2] }} + + 🚀 平台：{{ platform }} + + 🚀 描述：{% if description == "
" %}无{% else %}{{ description }}{% endif %} + + ⛱ 下载链接：{{ link }} + + \#App优惠 + + - name: Youtube 订阅 + url: + - https://www.youtube.com/feeds/videos.xml?channel_id=UCMUnInmOkrWN4gof9KlhNmQ + - https://www.youtube.com/feeds/videos.xml?channel_id=UC2tQpW0dPiyWPebwBSksJ_g + - https://www.youtube.com/feeds/videos.xml?channel_id=UCIF_gt4BfsWyM_2GOcKXyEQ + - https://www.youtube.com/feeds/videos.xml?channel_id=UCpPswAyGzdRwWmiW5oTNnvA + - https://www.youtube.com/feeds/videos.xml?channel_id=UChNxH3wxiElBSAdAyMfNhJQ + - https://www.youtube.com/feeds/videos.xml?channel_id=UCnmeiFVqv64lAmVaw2PUm7w + - https://www.youtube.com/feeds/videos.xml?channel_id=UCtFcF_I558yCsGkP_5SdJHA + - https://www.youtube.com/feeds/videos.xml?channel_id=UCsvn_Po0SmunchJYOWpOxMg + - https://www.youtube.com/feeds/videos.xml?channel_id=UC0vBXGSyV14uvJ4hECDOl0Q + - https://www.youtube.com/feeds/videos.xml?channel_id=UCJQJ4GjTiq5lmn8czf8oo0Q + - https://www.youtube.com/feeds/videos.xml?channel_id=UCsLWG2t7n9LFsvH0wR2rtpw + - https://www.youtube.com/feeds/videos.xml?channel_id=UCplkk3J5wrEl0TNrthHjq4Q + rules: + - obj: authors + type: func + matcher: | + def func(placeholder): + tmp_list = [] + for p in placeholder: + tmp_list.append(p["name"]) + + return " / ".join(tmp_list) + dest: author_list + text: | + 🎥 订阅更新：*{{ title }}* + + 🏆 Youtuber：*{{ author_list }}* + + 🎯 直达链接：{{ link }} + + \#Apocalypsor的Youtube订阅 + + - name: Bilibili 视频订阅 + url: https://rss.dov.moe/bilibili/followings/video/2975898 + rules: + - obj: authors + type: func + matcher: | + def func(placeholder): + tmp_list = [] + for p in placeholder: + tmp_list.append(p["name"]) + + return " / ".join(tmp_list) + dest: author_list + text: | + 📺 订阅更新：*{{ title }}* + + 🆙 UP主：*{{ author_list }}* + + ✨ 直达链接：{{ link }} + + \#Apocalypsor的B站订阅 + + - name: Bilibili 追番订阅 + url: https://rss.dov.moe/bilibili/user/bangumi/2975898 + telegram: + disable_web_page_preview: true + rules: + - obj: title + type: regex + matcher: ^(\[.+?\])?(.+?)((（僅)|$) + dest: tag + text: | + 🌟 更新剧集：*{{ title }}* + + 💡 直达链接：{{ link }} + + \#Apocalypsor的动漫订阅 \#{{ tag[2] }} diff --git a/main.py b/main.py new file mode 100644 index 0000000..a159e1d --- /dev/null +++ b/main.py @@ -0,0 +1,8 @@ +from FR2T import fr2t + +if __name__ == "__main__": + config_path = "data/config.yaml" + rss_path = "data/rss.yaml" + + fr = fr2t.FR2T(config_path, rss_path) + fr.run() diff --git a/test/example.json b/test/example.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/test/example.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/rss.py b/test/rss.py new file mode 100644 index 0000000..57dc382 --- /dev/null +++ b/test/rss.py @@ -0,0 +1,8 @@ +import feedparser +import json + +url = "https://rsshub.app/bilibili/user/bangumi/2975898" +d = feedparser.parse(url)["entries"] + +with open("example.json", "w") as t: + json.dump(d, t, indent=4, ensure_ascii=False)