add split large video(disabled now), random edit text

fix Makefile

.github/workflows/upgrade.yml

@@ -18,6 +18,7 @@ jobs:
           username: root
           key: ${{ secrets.SSH_KEY }}
           script: |
+            cd /home/ytdlbot
             make bot
             make upgrade-all-worker
             curl "https://api.telegram.org/bot$BOT_TOKEN/sendMessage?chat_id=260260121&text=ytdl%20upgrade%20complete!"

Makefile

@@ -22,5 +22,5 @@ upgrade-all-worker:
 	bash upgrade_worker.sh
 
 tag:
-	git tag -a v$(date "+%Y-%m-%d") -m v$(date "+%Y-%m-%d")
+	git tag -a v$(shell date "+%Y-%m-%d") -m v$(shell date "+%Y-%m-%d")
 	git push --tags

ytdlbot/config.py

@@ -22,6 +22,9 @@ QUOTA = os.getenv("QUOTA", 5 * 1024 * 1024 * 1024)  # 5G
 if os.uname().sysname == "Darwin":
     QUOTA = 10 * 1024 * 1024  # 10M
 
+TG_MAX_SIZE = 2 * 1024 * 1024 * 1024 * 0.99
+# TG_MAX_SIZE = 10 * 1024 * 1024
+
 EX = os.getenv("EX", 24 * 3600)
 MULTIPLY = os.getenv("MULTIPLY", 5)  # VIP1 is 5*5-25G, VIP2 is 50G
 USD2CNY = os.getenv("USD2CNY", 6)  # $5 --> ¥30

ytdlbot/downloader.py

@@ -10,6 +10,7 @@ __author__ = "Benny <benny.think@gmail.com>"
 import logging
 import os
 import pathlib
+import random
 import re
 import subprocess
 import time
@@ -26,13 +27,12 @@ else:
     import yt_dlp as ytdl
     from yt_dlp import DownloadError
 
-from config import ENABLE_VIP
+from config import ENABLE_VIP, TG_MAX_SIZE
 from db import Redis
 from limit import VIP
 from utils import adjust_formats, apply_log_formatter, get_user_settings
 
 r = fakeredis.FakeStrictRedis()
-EXPIRE = 5
 apply_log_formatter()
 
 
@@ -48,7 +48,7 @@ def edit_text(bot_msg, text):
     key = f"{bot_msg.chat.id}-{bot_msg.message_id}"
     # if the key exists, we shouldn't send edit message
     if not r.exists(key):
-        r.set(key, "ok", ex=EXPIRE)
+        r.set(key, "ok", ex=random.randint(1, 5))
         bot_msg.edit_text(text)
 
 
@@ -183,6 +183,8 @@ def ytdl_download(url, tempdir, bm) -> dict:
     if settings[2] == "video":
         # only convert if send type is video
         convert_to_mp4(response, bm)
+    # disable it for now
+    # split_large_video(response)
     return response
 
 
@@ -198,3 +200,24 @@ def convert_flac(flac_name, tmp):
 def add_instagram_cookies(url: "str", opt: "dict"):
     if url.startswith("https://www.instagram.com"):
         opt["cookiefile"] = os.path.join(os.path.dirname(__file__), "instagram.com_cookies.txt")
+
+
+def run_splitter(video_path: "str"):
+    subprocess.check_output(f"sh split-video.sh {video_path} {TG_MAX_SIZE} ".split())
+    os.remove(video_path)
+
+
+def split_large_video(response: "dict"):
+    original_video = None
+    split = False
+    for original_video in response.get("filepath", []):
+        size = os.stat(original_video).st_size
+        if size > TG_MAX_SIZE:
+            split = True
+            logging.warning("file is too large %s, splitting...", size)
+            run_splitter(original_video)
+
+    if split and original_video:
+        response["filepath"] = [i.as_posix() for i in pathlib.Path(original_video).parent.glob("*")]
+
+

ytdlbot/split-video.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Short script to split videos by filesize using ffmpeg by LukeLR
+
+if [ $# -ne 2 ]; then
+    echo 'Illegal number of parameters. Needs 2 parameters:'
+    echo 'Usage:'
+    echo './split-video.sh FILE SIZELIMIT "FFMPEG_ARGS'
+    echo 
+    echo 'Parameters:'
+    echo '    - FILE:        Name of the video file to split'
+    echo '    - SIZELIMIT:   Maximum file size of each part (in bytes)'
+    echo '    - FFMPEG_ARGS: Additional arguments to pass to each ffmpeg-call'
+    echo '                   (video format and quality options etc.)'
+    exit 1
+fi
+
+FILE="$1"
+SIZELIMIT="$2"
+FFMPEG_ARGS="$3"
+
+# Duration of the source video
+DURATION=$(ffprobe -i "$FILE" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1)
+
+# Duration that has been encoded so far
+CUR_DURATION=0
+
+# Filename of the source video (without extension)
+BASENAME="${FILE%.*}"
+
+# Extension for the video parts
+#EXTENSION="${FILE##*.}"
+EXTENSION="mp4"
+
+# Number of the current video part
+i=1
+
+# Filename of the next video part
+NEXTFILENAME="$BASENAME-$i.$EXTENSION"
+
+echo "Duration of source video: $DURATION"
+
+# Until the duration of all partial videos has reached the duration of the source video
+while [[ $CUR_DURATION -lt $DURATION ]]; do
+    # Encode next part
+    echo ffmpeg -i "$FILE" -ss "$CUR_DURATION" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME"
+    ffmpeg -ss "$CUR_DURATION" -i "$FILE" -fs "$SIZELIMIT" $FFMPEG_ARGS "$NEXTFILENAME"
+
+    # Duration of the new part
+    NEW_DURATION=$(ffprobe -i "$NEXTFILENAME" -show_entries format=duration -v quiet -of default=noprint_wrappers=1:nokey=1|cut -d. -f1)
+
+    # Total duration encoded so far
+    CUR_DURATION=$((CUR_DURATION + NEW_DURATION))
+
+    i=$((i + 1))
+
+    echo "Duration of $NEXTFILENAME: $NEW_DURATION"
+    echo "Part No. $i starts at $CUR_DURATION"
+
+    NEXTFILENAME="$BASENAME-$i.$EXTENSION"
+done

ytdlbot/ytdl_bot.py

@@ -27,7 +27,8 @@ from db import MySQL, Redis
 from downloader import convert_flac
 from limit import verify_payment
 from tasks import download_entrance
-from utils import customize_logger, get_user_settings, set_user_settings, get_revision_tag
+from utils import (customize_logger, get_revision_tag, get_user_settings,
+                   set_user_settings)
 
 customize_logger(["pyrogram.client", "pyrogram.session.session", "pyrogram.connection.connection"])
 app = create_app()