diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml index 3ed4d51..3331398 100644 --- a/.github/workflows/upgrade.yml +++ b/.github/workflows/upgrade.yml @@ -18,6 +18,7 @@ jobs: username: root key: ${{ secrets.SSH_KEY }} script: | + cd /home/ytdlbot make bot make upgrade-all-worker curl "https://api.telegram.org/bot$BOT_TOKEN/sendMessage?chat_id=260260121&text=ytdl%20upgrade%20complete!" diff --git a/Makefile b/Makefile index 7b1ca0c..847d686 100644 --- a/Makefile +++ b/Makefile @@ -22,5 +22,5 @@ upgrade-all-worker: bash upgrade_worker.sh tag: - git tag -a v$(date "+%Y-%m-%d") -m v$(date "+%Y-%m-%d") + git tag -a v$(shell date "+%Y-%m-%d") -m v$(shell date "+%Y-%m-%d") git push --tags \ No newline at end of file diff --git a/ytdlbot/config.py b/ytdlbot/config.py index c5304a8..ef0d643 100644 --- a/ytdlbot/config.py +++ b/ytdlbot/config.py @@ -22,6 +22,9 @@ QUOTA = os.getenv("QUOTA", 5 * 1024 * 1024 * 1024) # 5G if os.uname().sysname == "Darwin": QUOTA = 10 * 1024 * 1024 # 10M +TG_MAX_SIZE = 2 * 1024 * 1024 * 1024 * 0.99 +# TG_MAX_SIZE = 10 * 1024 * 1024 + EX = os.getenv("EX", 24 * 3600) MULTIPLY = os.getenv("MULTIPLY", 5) # VIP1 is 5*5-25G, VIP2 is 50G USD2CNY = os.getenv("USD2CNY", 6) # $5 --> ¥30 diff --git a/ytdlbot/downloader.py b/ytdlbot/downloader.py index c2ab768..4c050de 100644 --- a/ytdlbot/downloader.py +++ b/ytdlbot/downloader.py @@ -10,6 +10,7 @@ __author__ = "Benny " import logging import os import pathlib +import random import re import subprocess import time @@ -26,13 +27,12 @@ else: import yt_dlp as ytdl from yt_dlp import DownloadError -from config import ENABLE_VIP +from config import ENABLE_VIP, TG_MAX_SIZE from db import Redis from limit import VIP from utils import adjust_formats, apply_log_formatter, get_user_settings r = fakeredis.FakeStrictRedis() -EXPIRE = 5 apply_log_formatter() @@ -48,7 +48,7 @@ def edit_text(bot_msg, text): key = f"{bot_msg.chat.id}-{bot_msg.message_id}" # if the key exists, we shouldn't send edit message if not r.exists(key): - r.set(key, "ok", ex=EXPIRE) + r.set(key, "ok", ex=random.randint(1, 5)) bot_msg.edit_text(text) @@ -183,6 +183,8 @@ def ytdl_download(url, tempdir, bm) -> dict: if settings[2] == "video": # only convert if send type is video convert_to_mp4(response, bm) + # disable it for now + # split_large_video(response) return response @@ -198,3 +200,24 @@ def convert_flac(flac_name, tmp): def add_instagram_cookies(url: "str", opt: "dict"): if url.startswith("https://www.instagram.com"): opt["cookiefile"] = os.path.join(os.path.dirname(__file__), "instagram.com_cookies.txt") + + +def run_splitter(video_path: "str"): + subprocess.check_output(f"sh split-video.sh {video_path} {TG_MAX_SIZE} ".split()) + os.remove(video_path) + + +def split_large_video(response: "dict"): + original_video = None + split = False + for original_video in response.get("filepath", []): + size = os.stat(original_video).st_size + if size > TG_MAX_SIZE: + split = True + logging.warning("file is too large %s, splitting...", size) + run_splitter(original_video) + + if split and original_video: + response["filepath"] = [i.as_posix() for i in pathlib.Path(original_video).parent.glob("*")] + + diff --git a/ytdlbot/split-video.sh b/ytdlbot/split-video.sh new file mode 100755 index 0000000..5265ea1 --- /dev/null +++ b/ytdlbot/split-video.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Short script to split videos by filesize using ffmpeg by LukeLR + +if [ $# -ne 2 ]; then + echo 'Illegal number of parameters. Needs 2 parameters:' + echo 'Usage:' + echo './split-video.sh FILE SIZELIMIT "FFMPEG_ARGS' + echo + echo 'Parameters:' + echo ' - FILE: Name of the video file to split' + echo ' - SIZELIMIT: Maximum file size of each part (in bytes)' + echo ' - FFMPEG_ARGS: Additional arguments to pass to each ffmpeg-call' + echo ' (video format and quality options etc.)' + exit 1 +fi + +FILE="$1" +SIZELIMIT="$2" +FFMPEG_ARGS="$3" + +# Duration of the source video +DURATION=$(ffprobe -i "$FILE" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1) + +# Duration that has been encoded so far +CUR_DURATION=0 + +# Filename of the source video (without extension) +BASENAME="${FILE%.*}" + +# Extension for the video parts +#EXTENSION="${FILE##*.}" +EXTENSION="mp4" + +# Number of the current video part +i=1 + +# Filename of the next video part +NEXTFILENAME="$BASENAME-$i.$EXTENSION" + +echo "Duration of source video: $DURATION" + +# Until the duration of all partial videos has reached the duration of the source video +while [[ $CUR_DURATION -lt $DURATION ]]; do + # Encode next part + echo ffmpeg -i "$FILE" -ss "$CUR_DURATION" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME" + ffmpeg -ss "$CUR_DURATION" -i "$FILE" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME" + + # Duration of the new part + NEW_DURATION=$(ffprobe -i "$NEXTFILENAME" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1) + + # Total duration encoded so far + CUR_DURATION=$((CUR_DURATION + NEW_DURATION)) + + i=$((i + 1)) + + echo "Duration of $NEXTFILENAME: $NEW_DURATION" + echo "Part No. $i starts at $CUR_DURATION" + + NEXTFILENAME="$BASENAME-$i.$EXTENSION" +done \ No newline at end of file diff --git a/ytdlbot/ytdl_bot.py b/ytdlbot/ytdl_bot.py index 25b3975..145bd30 100644 --- a/ytdlbot/ytdl_bot.py +++ b/ytdlbot/ytdl_bot.py @@ -27,7 +27,8 @@ from db import MySQL, Redis from downloader import convert_flac from limit import verify_payment from tasks import download_entrance -from utils import customize_logger, get_user_settings, set_user_settings, get_revision_tag +from utils import (customize_logger, get_revision_tag, get_user_settings, + set_user_settings) customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"]) app = create_app()