From 1700cde152deb38cbf46a041fe958f9dfc2f1468 Mon Sep 17 00:00:00 2001 From: aNNiMON Date: Fri, 20 Jan 2023 23:21:32 +0200 Subject: [PATCH] Support for 20+ MiB files download by calling external Telegram Client file downloader --- .gitignore | 3 +- README.md | 1 + ffmpegbot.yaml.template | 5 ++ pytgfile.py | 64 +++++++++++++++++++ .../java/com/annimon/ffmpegbot/BotConfig.java | 4 +- .../com/annimon/ffmpegbot/MainBotHandler.java | 14 +++- .../ffmpegbot/TelegramRuntimeException.java | 7 ++ .../commands/admin/ClearCommand.java | 2 +- .../commands/ffmpeg/FFmpegCommandBuilder.java | 2 +- .../ffmpeg/MediaProcessingBundle.java | 21 +++--- .../commands/ytdlp/YtDlpCommandBuilder.java | 2 +- .../commands/ytdlp/YtDlpCommandBundle.java | 3 +- .../file/FallbackFileDownloader.java | 24 +++++++ .../ffmpegbot/file/FileDownloadException.java | 11 ++++ .../ffmpegbot/file/FileDownloader.java | 10 +++ .../ffmpegbot/{session => file}/FilePath.java | 12 +++- .../file/TelegramClientFileDownloader.java | 53 +++++++++++++++ .../file/TelegramFileDownloader.java | 24 +++++++ .../annimon/ffmpegbot/session/Resolver.java | 8 --- 19 files changed, 245 insertions(+), 25 deletions(-) create mode 100644 pytgfile.py create mode 100644 src/main/java/com/annimon/ffmpegbot/TelegramRuntimeException.java create mode 100644 src/main/java/com/annimon/ffmpegbot/file/FallbackFileDownloader.java create mode 100644 src/main/java/com/annimon/ffmpegbot/file/FileDownloadException.java create mode 100644 src/main/java/com/annimon/ffmpegbot/file/FileDownloader.java rename src/main/java/com/annimon/ffmpegbot/{session => file}/FilePath.java (64%) create mode 100644 src/main/java/com/annimon/ffmpegbot/file/TelegramClientFileDownloader.java create mode 100644 src/main/java/com/annimon/ffmpegbot/file/TelegramFileDownloader.java diff --git a/.gitignore b/.gitignore index 87ca788..55182af 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ build/ out gen -*.iml \ No newline at end of file +*.iml +*.session \ No newline at end of file diff --git a/README.md b/README.md index 2de3e9e..b166c4a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Telegram Bot for re-encoding media - Telegram bot username and token, [@BotFather](https://t.me/BotFather) - JRE 17+ or JDK 17+ (for build) - `ffmpeg` must be installed and available in `PATH`. + - `python3` version 3.8+ must be installed and available in `PATH`. - `yt-dlp` for `/dl` command. ## Installation diff --git a/ffmpegbot.yaml.template b/ffmpegbot.yaml.template index 0067290..52b37dc 100644 --- a/ffmpegbot.yaml.template +++ b/ffmpegbot.yaml.template @@ -1,6 +1,11 @@ # Telegram bot token and bot username botToken: 1234567890:AAAABBBBCCCCDDDDEEEEFF-GGGGHHHHIIII botUsername: yourbotname +# Telegram API app_id / app_hash (see https://core.telegram.org/api/obtaining_api_id) +appId: 12345 +appHash: abc123def456 +# Path to Telegram API file downloader script +downloaderScript: pytgfile.py # Superusers can execute /run command superUsers: [12345] # Allowed user ids diff --git a/pytgfile.py b/pytgfile.py new file mode 100644 index 0000000..be307e8 --- /dev/null +++ b/pytgfile.py @@ -0,0 +1,64 @@ +import argparse +import asyncio +import pathlib +import pyrogram +import pyrogram.file_id + +async def run_bot(args, func): + async with pyrogram.Client(args.bot_username, api_id=args.api_id, + api_hash=args.api_hash, bot_token=args.bot_token) as bot_client: + await func(bot_client) + +async def get_file(args): + args.outputpath.resolve() + if args.outputpath.is_dir(): + raise RuntimeError('Should be file') + + async def download_media_func(client: pyrogram.Client): + file_id_obj = pyrogram.file_id.FileId.decode(args.file_id) + await client.handle_download((file_id_obj, args.outputpath.parent, args.outputpath.name, False, 0, None, tuple())) + + await run_bot(args, download_media_func) + +async def put_file(args): + args.input.resolve() + if not args.input.exists(): + raise RuntimeError('Input not exists') + + async def upload_file_func(client: pyrogram.Client): + if args.type == 'audio': + await client.send_audio(args.chat_id, args.input) + elif args.type == 'video': + await client.send_video(args.chat_id, args.input) + elif args.type == 'gif': + await client.send_animation(args.chat_id, args.input) + + await run_bot(args, upload_file_func) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog = 'pytgfile', description = 'Get or put Telegram files by file id') + parser.add_argument('--api_id', type=str, required=True) + parser.add_argument('--api_hash', type=str, required=True) + parser.add_argument('--bot_username', type=str, required=True) + parser.add_argument('--bot_token', type=str, required=True) + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_get = subparsers.add_parser('get', help='get help') + parser_get.set_defaults(func=get_file) + parser_get.add_argument('--file_id', type=str, help='bar help', required=True) + parser_get.add_argument('-o', '--outputpath', type=pathlib.Path, help='bar help', required=True) + + parser_put = subparsers.add_parser('put', help='put help') + parser_put.set_defaults(func=put_file) + parser_put.add_argument('-c', '--chat_id', type=str, help='chat_id help', required=True) + parser_put.add_argument('-i', '--input', type=pathlib.Path, help='input help', required=True) + parser_put.add_argument('-t', '--type', type=str, help='type help', choices=['audio', 'video', 'gif'], required=True) + args = parser.parse_args() + + try: + asyncio.run(args.func(args)) + except: + exit(1) + + exit(0) \ No newline at end of file diff --git a/src/main/java/com/annimon/ffmpegbot/BotConfig.java b/src/main/java/com/annimon/ffmpegbot/BotConfig.java index 59d8d4b..d13e859 100644 --- a/src/main/java/com/annimon/ffmpegbot/BotConfig.java +++ b/src/main/java/com/annimon/ffmpegbot/BotConfig.java @@ -2,6 +2,8 @@ package com.annimon.ffmpegbot; import java.util.Set; -public record BotConfig(String botToken, String botUsername, +public record BotConfig(String appId, String appHash, + String botToken, String botUsername, + String downloaderScript, Set superUsers, Set allowedUsers) { } diff --git a/src/main/java/com/annimon/ffmpegbot/MainBotHandler.java b/src/main/java/com/annimon/ffmpegbot/MainBotHandler.java index 21b526d..7bbdc36 100644 --- a/src/main/java/com/annimon/ffmpegbot/MainBotHandler.java +++ b/src/main/java/com/annimon/ffmpegbot/MainBotHandler.java @@ -5,6 +5,9 @@ import com.annimon.ffmpegbot.commands.admin.AdminCommandBundle; import com.annimon.ffmpegbot.commands.ffmpeg.InputParametersBundle; import com.annimon.ffmpegbot.commands.ffmpeg.MediaProcessingBundle; import com.annimon.ffmpegbot.commands.ytdlp.YtDlpCommandBundle; +import com.annimon.ffmpegbot.file.FallbackFileDownloader; +import com.annimon.ffmpegbot.file.TelegramClientFileDownloader; +import com.annimon.ffmpegbot.file.TelegramFileDownloader; import com.annimon.ffmpegbot.session.Sessions; import com.annimon.tgbotsmodule.BotHandler; import com.annimon.tgbotsmodule.commands.CommandRegistry; @@ -28,7 +31,14 @@ public class MainBotHandler extends BotHandler { permissions = new Permissions(botConfig.superUsers(), botConfig.allowedUsers()); commands = new CommandRegistry<>(this, this::checkAccess); final var sessions = new Sessions(); - mediaProcessingBundle = new MediaProcessingBundle(sessions); + final var fallbackFileDownloader = new FallbackFileDownloader( + new TelegramFileDownloader(), + new TelegramClientFileDownloader( + botConfig.downloaderScript(), + botConfig.appId(), botConfig.appHash(), + botConfig.botToken(), botConfig.botUsername()) + ); + mediaProcessingBundle = new MediaProcessingBundle(sessions, fallbackFileDownloader); commands.registerBundle(mediaProcessingBundle); commands.registerBundle(new InputParametersBundle(sessions)); commands.registerBundle(new YtDlpCommandBundle()); @@ -64,6 +74,6 @@ public class MainBotHandler extends BotHandler { @Override public void handleTelegramApiException(TelegramApiException ex) { - throw new RuntimeException(ex); + throw new TelegramRuntimeException(ex); } } diff --git a/src/main/java/com/annimon/ffmpegbot/TelegramRuntimeException.java b/src/main/java/com/annimon/ffmpegbot/TelegramRuntimeException.java new file mode 100644 index 0000000..c566bd5 --- /dev/null +++ b/src/main/java/com/annimon/ffmpegbot/TelegramRuntimeException.java @@ -0,0 +1,7 @@ +package com.annimon.ffmpegbot; + +public class TelegramRuntimeException extends RuntimeException { + public TelegramRuntimeException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/annimon/ffmpegbot/commands/admin/ClearCommand.java b/src/main/java/com/annimon/ffmpegbot/commands/admin/ClearCommand.java index 09f402a..a51c92d 100644 --- a/src/main/java/com/annimon/ffmpegbot/commands/admin/ClearCommand.java +++ b/src/main/java/com/annimon/ffmpegbot/commands/admin/ClearCommand.java @@ -2,7 +2,7 @@ package com.annimon.ffmpegbot.commands.admin; import com.annimon.ffmpegbot.Permissions; import com.annimon.ffmpegbot.TextUtils; -import com.annimon.ffmpegbot.session.FilePath; +import com.annimon.ffmpegbot.file.FilePath; import com.annimon.tgbotsmodule.commands.TextCommand; import com.annimon.tgbotsmodule.commands.authority.For; import com.annimon.tgbotsmodule.commands.context.MessageContext; diff --git a/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/FFmpegCommandBuilder.java b/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/FFmpegCommandBuilder.java index 66f1b48..a0a72ae 100644 --- a/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/FFmpegCommandBuilder.java +++ b/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/FFmpegCommandBuilder.java @@ -1,7 +1,7 @@ package com.annimon.ffmpegbot.commands.ffmpeg; import com.annimon.ffmpegbot.parameters.*; -import com.annimon.ffmpegbot.session.FilePath; +import com.annimon.ffmpegbot.file.FilePath; import com.annimon.ffmpegbot.session.FileType; import com.annimon.ffmpegbot.session.FileTypes; import com.annimon.ffmpegbot.session.MediaSession; diff --git a/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/MediaProcessingBundle.java b/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/MediaProcessingBundle.java index 53fca9c..bdae8e1 100644 --- a/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/MediaProcessingBundle.java +++ b/src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/MediaProcessingBundle.java @@ -1,7 +1,9 @@ package com.annimon.ffmpegbot.commands.ffmpeg; +import com.annimon.ffmpegbot.file.FileDownloadException; +import com.annimon.ffmpegbot.file.FileDownloader; import com.annimon.ffmpegbot.parameters.Parameter; -import com.annimon.ffmpegbot.session.FilePath; +import com.annimon.ffmpegbot.file.FilePath; import com.annimon.ffmpegbot.session.MediaSession; import com.annimon.ffmpegbot.session.Resolver; import com.annimon.ffmpegbot.session.Sessions; @@ -14,7 +16,6 @@ import com.annimon.tgbotsmodule.commands.context.CallbackQueryContext; import com.annimon.tgbotsmodule.services.CommonAbsSender; import org.jetbrains.annotations.NotNull; import org.telegram.telegrambots.meta.api.objects.Message; -import org.telegram.telegrambots.meta.exceptions.TelegramApiException; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; @@ -28,9 +29,11 @@ import static com.annimon.ffmpegbot.commands.ffmpeg.MediaProcessingKeyboard.crea public class MediaProcessingBundle implements CommandBundle { private final Sessions sessions; + private final FileDownloader fileDownloader; - public MediaProcessingBundle(Sessions sessions) { + public MediaProcessingBundle(Sessions sessions, FileDownloader fileDownloader) { this.sessions = sessions; + this.fileDownloader = fileDownloader; } @Override @@ -127,11 +130,13 @@ public class MediaProcessingBundle implements CommandBundle { private void download(final CallbackQueryContext ctx, final MediaSession session) { try { - final var tgFile = Methods.getFile(session.getFileId()).call(ctx.sender); - final var localFilename = FilePath.generateFilename(tgFile.getFileId(), tgFile.getFilePath()); - session.setInputFile(ctx.sender.downloadFile(tgFile, FilePath.inputFile(localFilename))); - session.setOutputFile(FilePath.outputFile(localFilename)); - } catch (TelegramApiException e) { + final String defaultFilename = FilePath.defaultFilename(session.getFileType()); + final var file = fileDownloader.downloadFile(ctx.sender, session.getFileId(), defaultFilename); + final var filename = FilePath.generateFilename(session.getFileId(), file.getName()); + session.setInputFile(FilePath.inputFile(filename)); + file.renameTo(session.getInputFile()); + session.setOutputFile(FilePath.outputFile(filename)); + } catch (FileDownloadException e) { session.setStatus("Unable to download due to " + e.getMessage()); editMessage(ctx, session); } diff --git a/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBuilder.java b/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBuilder.java index 665dff0..66db61e 100644 --- a/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBuilder.java +++ b/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBuilder.java @@ -1,6 +1,6 @@ package com.annimon.ffmpegbot.commands.ytdlp; -import com.annimon.ffmpegbot.session.FilePath; +import com.annimon.ffmpegbot.file.FilePath; import com.annimon.ffmpegbot.session.YtDlpSession; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBundle.java b/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBundle.java index f85259b..af35e3c 100644 --- a/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBundle.java +++ b/src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBundle.java @@ -1,6 +1,7 @@ package com.annimon.ffmpegbot.commands.ytdlp; import com.annimon.ffmpegbot.Permissions; +import com.annimon.ffmpegbot.file.FilePath; import com.annimon.ffmpegbot.session.*; import com.annimon.tgbotsmodule.api.methods.Methods; import com.annimon.tgbotsmodule.commands.CommandBundle; @@ -32,7 +33,7 @@ public class YtDlpCommandBundle implements CommandBundle { .orElse("720"); final var fileType = downloadOption.equals("audio") ? FileType.AUDIO : FileType.VIDEO; final var ytDlpSession = new YtDlpSession(url, downloadOption, fileType); - final var filename = FilePath.generateFilename(url, Resolver.resolveDefaultFilename(fileType)); + final var filename = FilePath.generateFilename(url, FilePath.defaultFilename(fileType)); ytDlpSession.setOutputFilename(filename); Methods.sendChatAction(ctx.chatId(), Resolver.resolveAction(fileType)).callAsync(ctx.sender); diff --git a/src/main/java/com/annimon/ffmpegbot/file/FallbackFileDownloader.java b/src/main/java/com/annimon/ffmpegbot/file/FallbackFileDownloader.java new file mode 100644 index 0000000..47d7e7b --- /dev/null +++ b/src/main/java/com/annimon/ffmpegbot/file/FallbackFileDownloader.java @@ -0,0 +1,24 @@ +package com.annimon.ffmpegbot.file; + +import com.annimon.tgbotsmodule.services.CommonAbsSender; + +import java.io.File; + +public class FallbackFileDownloader implements FileDownloader { + private final FileDownloader defaultDownloader; + private final FileDownloader fallbackDownloader; + + public FallbackFileDownloader(FileDownloader defaultDownloader, FileDownloader fallbackDownloader) { + this.defaultDownloader = defaultDownloader; + this.fallbackDownloader = fallbackDownloader; + } + + @Override + public File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename) { + try { + return defaultDownloader.downloadFile(sender, fileId, defaultFilename); + } catch (FileDownloadException e) { + return fallbackDownloader.downloadFile(sender, fileId, defaultFilename); + } + } +} diff --git a/src/main/java/com/annimon/ffmpegbot/file/FileDownloadException.java b/src/main/java/com/annimon/ffmpegbot/file/FileDownloadException.java new file mode 100644 index 0000000..c8b8b40 --- /dev/null +++ b/src/main/java/com/annimon/ffmpegbot/file/FileDownloadException.java @@ -0,0 +1,11 @@ +package com.annimon.ffmpegbot.file; + +public class FileDownloadException extends RuntimeException { + public FileDownloadException(String message) { + super(message); + } + + public FileDownloadException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/com/annimon/ffmpegbot/file/FileDownloader.java b/src/main/java/com/annimon/ffmpegbot/file/FileDownloader.java new file mode 100644 index 0000000..1c13851 --- /dev/null +++ b/src/main/java/com/annimon/ffmpegbot/file/FileDownloader.java @@ -0,0 +1,10 @@ +package com.annimon.ffmpegbot.file; + +import com.annimon.tgbotsmodule.services.CommonAbsSender; + +import java.io.File; + +public interface FileDownloader { + + File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename); +} diff --git a/src/main/java/com/annimon/ffmpegbot/session/FilePath.java b/src/main/java/com/annimon/ffmpegbot/file/FilePath.java similarity index 64% rename from src/main/java/com/annimon/ffmpegbot/session/FilePath.java rename to src/main/java/com/annimon/ffmpegbot/file/FilePath.java index d669ae6..aaa2c5d 100644 --- a/src/main/java/com/annimon/ffmpegbot/session/FilePath.java +++ b/src/main/java/com/annimon/ffmpegbot/file/FilePath.java @@ -1,6 +1,8 @@ -package com.annimon.ffmpegbot.session; +package com.annimon.ffmpegbot.file; +import com.annimon.ffmpegbot.session.FileType; import org.apache.commons.io.FilenameUtils; +import org.jetbrains.annotations.NotNull; import java.io.File; @@ -26,4 +28,12 @@ public class FilePath { final var ext = FilenameUtils.getExtension(filename); return "%d_%d.%s".formatted(System.currentTimeMillis(), Math.abs(fileId.hashCode()), ext); } + + public static String defaultFilename(@NotNull FileType fileType) { + return "file." + switch (fileType) { + case ANIMATION, VIDEO, VIDEO_NOTE -> "mp4"; + case AUDIO -> "mp3"; + case VOICE -> "ogg"; + }; + } } diff --git a/src/main/java/com/annimon/ffmpegbot/file/TelegramClientFileDownloader.java b/src/main/java/com/annimon/ffmpegbot/file/TelegramClientFileDownloader.java new file mode 100644 index 0000000..10d04c9 --- /dev/null +++ b/src/main/java/com/annimon/ffmpegbot/file/TelegramClientFileDownloader.java @@ -0,0 +1,53 @@ +package com.annimon.ffmpegbot.file; + +import com.annimon.tgbotsmodule.services.CommonAbsSender; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class TelegramClientFileDownloader implements FileDownloader { + private final String scriptPath; + private final String appId; + private final String appHash; + private final String botToken; + private final String botUsername; + + public TelegramClientFileDownloader(String scriptPath, String appId, String appHash, String botToken, String botUsername) { + this.scriptPath = scriptPath; + this.appId = appId; + this.appHash = appHash; + this.botToken = botToken; + this.botUsername = botUsername; + } + + @Override + public File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename) { + try { + final var tempFile = File.createTempFile("tmp", defaultFilename); + final ProcessBuilder pb = new ProcessBuilder(buildCommand(fileId, tempFile)); + final Process process = pb.start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new FileDownloadException("Downloader process finished with the exit code " + exitCode); + } + return tempFile; + } catch (InterruptedException | IOException e) { + throw new FileDownloadException("Downloader process failed", e); + } + } + + private List buildCommand(String fileId, File destFile) { + final var commands = new ArrayList(); + commands.addAll(List.of("python3", scriptPath)); + commands.addAll(List.of("--api_id", appId)); + commands.addAll(List.of("--api_hash", appHash)); + commands.addAll(List.of("--bot_token", botToken)); + commands.addAll(List.of("--bot_username", botUsername)); + commands.add("get"); + commands.addAll(List.of("--file_id", fileId)); + commands.addAll(List.of("-o", destFile.getPath())); + return commands; + } +} diff --git a/src/main/java/com/annimon/ffmpegbot/file/TelegramFileDownloader.java b/src/main/java/com/annimon/ffmpegbot/file/TelegramFileDownloader.java new file mode 100644 index 0000000..05c047e --- /dev/null +++ b/src/main/java/com/annimon/ffmpegbot/file/TelegramFileDownloader.java @@ -0,0 +1,24 @@ +package com.annimon.ffmpegbot.file; + +import com.annimon.ffmpegbot.TelegramRuntimeException; +import com.annimon.tgbotsmodule.api.methods.Methods; +import com.annimon.tgbotsmodule.services.CommonAbsSender; +import org.apache.commons.io.FilenameUtils; +import org.telegram.telegrambots.meta.exceptions.TelegramApiException; + +import java.io.File; +import java.io.IOException; + +public class TelegramFileDownloader implements FileDownloader { + + @Override + public File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename) { + try { + final var tgFile = Methods.getFile(fileId).call(sender); + final var extension = FilenameUtils.getExtension(tgFile.getFilePath()); + return sender.downloadFile(tgFile, File.createTempFile("tmp", "file." + extension)); + } catch (IOException | TelegramApiException | TelegramRuntimeException e) { + throw new FileDownloadException(e.getMessage(), e); + } + } +} diff --git a/src/main/java/com/annimon/ffmpegbot/session/Resolver.java b/src/main/java/com/annimon/ffmpegbot/session/Resolver.java index 5ca3281..833dbb7 100644 --- a/src/main/java/com/annimon/ffmpegbot/session/Resolver.java +++ b/src/main/java/com/annimon/ffmpegbot/session/Resolver.java @@ -41,14 +41,6 @@ public class Resolver { }; } - public static String resolveDefaultFilename(@NotNull FileType fileType) { - return "file." + switch (fileType) { - case ANIMATION, VIDEO, VIDEO_NOTE -> "mp4"; - case AUDIO -> "mp3"; - case VOICE -> "ogg"; - }; - } - public static MediaMessageMethod, ?> resolveMethod(@NotNull FileType fileType) { return switch (fileType) { case ANIMATION -> Methods.sendAnimation();