Support for 20+ MiB files download by calling external Telegram Client file downloader

.gitignore

@@ -3,4 +3,5 @@
 build/
 out
 gen
-*.iml
+*.iml
+*.session

README.md

@@ -15,6 +15,7 @@ Telegram Bot for re-encoding media
  - Telegram bot username and token, [@BotFather](https://t.me/BotFather)
  - JRE 17+ or JDK 17+ (for build)
  - `ffmpeg` must be installed and available in `PATH`.
+ - `python3` version 3.8+ must be installed and available in `PATH`.
  - `yt-dlp` for `/dl` command.
 
 ## Installation

ffmpegbot.yaml.template

@@ -1,6 +1,11 @@
 # Telegram bot token and bot username
 botToken: 1234567890:AAAABBBBCCCCDDDDEEEEFF-GGGGHHHHIIII
 botUsername: yourbotname
+# Telegram API app_id / app_hash (see https://core.telegram.org/api/obtaining_api_id)
+appId: 12345
+appHash: abc123def456
+# Path to Telegram API file downloader script
+downloaderScript: pytgfile.py
 # Superusers can execute /run command
 superUsers: [12345]
 # Allowed user ids

pytgfile.py

@@ -0,0 +1,64 @@
+import argparse
+import asyncio
+import pathlib
+import pyrogram
+import pyrogram.file_id
+
+async def run_bot(args, func):
+    async with pyrogram.Client(args.bot_username, api_id=args.api_id,
+                 api_hash=args.api_hash, bot_token=args.bot_token) as bot_client:
+        await func(bot_client)
+
+async def get_file(args):
+    args.outputpath.resolve()
+    if args.outputpath.is_dir():
+        raise RuntimeError('Should be file')
+
+    async def download_media_func(client: pyrogram.Client):
+        file_id_obj = pyrogram.file_id.FileId.decode(args.file_id)
+        await client.handle_download((file_id_obj, args.outputpath.parent, args.outputpath.name, False, 0, None, tuple()))
+
+    await run_bot(args, download_media_func)
+
+async def put_file(args):
+    args.input.resolve()
+    if not args.input.exists():
+        raise RuntimeError('Input not exists')
+
+    async def upload_file_func(client: pyrogram.Client):
+        if args.type == 'audio':
+            await client.send_audio(args.chat_id, args.input)
+        elif args.type == 'video':
+            await client.send_video(args.chat_id, args.input)
+        elif args.type == 'gif':
+            await client.send_animation(args.chat_id, args.input)
+
+    await run_bot(args, upload_file_func)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(prog = 'pytgfile', description = 'Get or put Telegram files by file id')
+    parser.add_argument('--api_id', type=str, required=True)
+    parser.add_argument('--api_hash', type=str, required=True)
+    parser.add_argument('--bot_username', type=str, required=True)
+    parser.add_argument('--bot_token', type=str, required=True)
+    subparsers = parser.add_subparsers(dest='command', required=True)
+
+    parser_get = subparsers.add_parser('get', help='get help')
+    parser_get.set_defaults(func=get_file)
+    parser_get.add_argument('--file_id', type=str, help='bar help', required=True)
+    parser_get.add_argument('-o', '--outputpath', type=pathlib.Path, help='bar help', required=True)
+ 
+    parser_put = subparsers.add_parser('put', help='put help')
+    parser_put.set_defaults(func=put_file)
+    parser_put.add_argument('-c', '--chat_id', type=str, help='chat_id help', required=True)
+    parser_put.add_argument('-i', '--input', type=pathlib.Path, help='input help', required=True)
+    parser_put.add_argument('-t', '--type', type=str, help='type help', choices=['audio', 'video', 'gif'], required=True)
+    args = parser.parse_args()
+
+    try:
+        asyncio.run(args.func(args))
+    except:
+        exit(1)
+
+    exit(0)

src/main/java/com/annimon/ffmpegbot/BotConfig.java

@@ -2,6 +2,8 @@ package com.annimon.ffmpegbot;
 
 import java.util.Set;
 
-public record BotConfig(String botToken, String botUsername,
+public record BotConfig(String appId, String appHash,
+                        String botToken, String botUsername,
+                        String downloaderScript,
                         Set<Long> superUsers, Set<Long> allowedUsers) {
 }

src/main/java/com/annimon/ffmpegbot/MainBotHandler.java

@@ -5,6 +5,9 @@ import com.annimon.ffmpegbot.commands.admin.AdminCommandBundle;
 import com.annimon.ffmpegbot.commands.ffmpeg.InputParametersBundle;
 import com.annimon.ffmpegbot.commands.ffmpeg.MediaProcessingBundle;
 import com.annimon.ffmpegbot.commands.ytdlp.YtDlpCommandBundle;
+import com.annimon.ffmpegbot.file.FallbackFileDownloader;
+import com.annimon.ffmpegbot.file.TelegramClientFileDownloader;
+import com.annimon.ffmpegbot.file.TelegramFileDownloader;
 import com.annimon.ffmpegbot.session.Sessions;
 import com.annimon.tgbotsmodule.BotHandler;
 import com.annimon.tgbotsmodule.commands.CommandRegistry;
@@ -28,7 +31,14 @@ public class MainBotHandler extends BotHandler {
         permissions = new Permissions(botConfig.superUsers(), botConfig.allowedUsers());
         commands = new CommandRegistry<>(this, this::checkAccess);
         final var sessions = new Sessions();
-        mediaProcessingBundle = new MediaProcessingBundle(sessions);
+        final var fallbackFileDownloader = new FallbackFileDownloader(
+                new TelegramFileDownloader(),
+                new TelegramClientFileDownloader(
+                        botConfig.downloaderScript(),
+                        botConfig.appId(), botConfig.appHash(),
+                        botConfig.botToken(), botConfig.botUsername())
+        );
+        mediaProcessingBundle = new MediaProcessingBundle(sessions, fallbackFileDownloader);
         commands.registerBundle(mediaProcessingBundle);
         commands.registerBundle(new InputParametersBundle(sessions));
         commands.registerBundle(new YtDlpCommandBundle());
@@ -64,6 +74,6 @@ public class MainBotHandler extends BotHandler {
 
     @Override
     public void handleTelegramApiException(TelegramApiException ex) {
-        throw new RuntimeException(ex);
+        throw new TelegramRuntimeException(ex);
     }
 }

src/main/java/com/annimon/ffmpegbot/TelegramRuntimeException.java

@@ -0,0 +1,7 @@
+package com.annimon.ffmpegbot;
+
+public class TelegramRuntimeException extends RuntimeException {
+    public TelegramRuntimeException(Throwable cause) {
+        super(cause);
+    }
+}

src/main/java/com/annimon/ffmpegbot/commands/admin/ClearCommand.java

@@ -2,7 +2,7 @@ package com.annimon.ffmpegbot.commands.admin;
 
 import com.annimon.ffmpegbot.Permissions;
 import com.annimon.ffmpegbot.TextUtils;
-import com.annimon.ffmpegbot.session.FilePath;
+import com.annimon.ffmpegbot.file.FilePath;
 import com.annimon.tgbotsmodule.commands.TextCommand;
 import com.annimon.tgbotsmodule.commands.authority.For;
 import com.annimon.tgbotsmodule.commands.context.MessageContext;

src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/FFmpegCommandBuilder.java

@@ -1,7 +1,7 @@
 package com.annimon.ffmpegbot.commands.ffmpeg;
 
 import com.annimon.ffmpegbot.parameters.*;
-import com.annimon.ffmpegbot.session.FilePath;
+import com.annimon.ffmpegbot.file.FilePath;
 import com.annimon.ffmpegbot.session.FileType;
 import com.annimon.ffmpegbot.session.FileTypes;
 import com.annimon.ffmpegbot.session.MediaSession;

src/main/java/com/annimon/ffmpegbot/commands/ffmpeg/MediaProcessingBundle.java

@@ -1,7 +1,9 @@
 package com.annimon.ffmpegbot.commands.ffmpeg;
 
+import com.annimon.ffmpegbot.file.FileDownloadException;
+import com.annimon.ffmpegbot.file.FileDownloader;
 import com.annimon.ffmpegbot.parameters.Parameter;
-import com.annimon.ffmpegbot.session.FilePath;
+import com.annimon.ffmpegbot.file.FilePath;
 import com.annimon.ffmpegbot.session.MediaSession;
 import com.annimon.ffmpegbot.session.Resolver;
 import com.annimon.ffmpegbot.session.Sessions;
@@ -14,7 +16,6 @@ import com.annimon.tgbotsmodule.commands.context.CallbackQueryContext;
 import com.annimon.tgbotsmodule.services.CommonAbsSender;
 import org.jetbrains.annotations.NotNull;
 import org.telegram.telegrambots.meta.api.objects.Message;
-import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.function.BiConsumer;
@@ -28,9 +29,11 @@ import static com.annimon.ffmpegbot.commands.ffmpeg.MediaProcessingKeyboard.crea
 public class MediaProcessingBundle implements CommandBundle<For> {
 
     private final Sessions sessions;
+    private final FileDownloader fileDownloader;
 
-    public MediaProcessingBundle(Sessions sessions) {
+    public MediaProcessingBundle(Sessions sessions, FileDownloader fileDownloader) {
         this.sessions = sessions;
+        this.fileDownloader = fileDownloader;
     }
 
     @Override
@@ -127,11 +130,13 @@ public class MediaProcessingBundle implements CommandBundle<For> {
 
     private void download(final CallbackQueryContext ctx, final MediaSession session) {
         try {
-            final var tgFile = Methods.getFile(session.getFileId()).call(ctx.sender);
-            final var localFilename = FilePath.generateFilename(tgFile.getFileId(), tgFile.getFilePath());
-            session.setInputFile(ctx.sender.downloadFile(tgFile, FilePath.inputFile(localFilename)));
-            session.setOutputFile(FilePath.outputFile(localFilename));
-        } catch (TelegramApiException e) {
+            final String defaultFilename = FilePath.defaultFilename(session.getFileType());
+            final var file = fileDownloader.downloadFile(ctx.sender, session.getFileId(), defaultFilename);
+            final var filename = FilePath.generateFilename(session.getFileId(), file.getName());
+            session.setInputFile(FilePath.inputFile(filename));
+            file.renameTo(session.getInputFile());
+            session.setOutputFile(FilePath.outputFile(filename));
+        } catch (FileDownloadException e) {
             session.setStatus("Unable to download due to " + e.getMessage());
             editMessage(ctx, session);
         }

src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBuilder.java

@@ -1,6 +1,6 @@
 package com.annimon.ffmpegbot.commands.ytdlp;
 
-import com.annimon.ffmpegbot.session.FilePath;
+import com.annimon.ffmpegbot.file.FilePath;
 import com.annimon.ffmpegbot.session.YtDlpSession;
 import org.jetbrains.annotations.NotNull;
 

src/main/java/com/annimon/ffmpegbot/commands/ytdlp/YtDlpCommandBundle.java

@@ -1,6 +1,7 @@
 package com.annimon.ffmpegbot.commands.ytdlp;
 
 import com.annimon.ffmpegbot.Permissions;
+import com.annimon.ffmpegbot.file.FilePath;
 import com.annimon.ffmpegbot.session.*;
 import com.annimon.tgbotsmodule.api.methods.Methods;
 import com.annimon.tgbotsmodule.commands.CommandBundle;
@@ -32,7 +33,7 @@ public class YtDlpCommandBundle implements CommandBundle<For> {
                 .orElse("720");
         final var fileType = downloadOption.equals("audio") ? FileType.AUDIO : FileType.VIDEO;
         final var ytDlpSession = new YtDlpSession(url, downloadOption, fileType);
-        final var filename = FilePath.generateFilename(url, Resolver.resolveDefaultFilename(fileType));
+        final var filename = FilePath.generateFilename(url, FilePath.defaultFilename(fileType));
         ytDlpSession.setOutputFilename(filename);
 
         Methods.sendChatAction(ctx.chatId(), Resolver.resolveAction(fileType)).callAsync(ctx.sender);

src/main/java/com/annimon/ffmpegbot/file/FallbackFileDownloader.java

@@ -0,0 +1,24 @@
+package com.annimon.ffmpegbot.file;
+
+import com.annimon.tgbotsmodule.services.CommonAbsSender;
+
+import java.io.File;
+
+public class FallbackFileDownloader implements FileDownloader {
+    private final FileDownloader defaultDownloader;
+    private final FileDownloader fallbackDownloader;
+
+    public FallbackFileDownloader(FileDownloader defaultDownloader, FileDownloader fallbackDownloader) {
+        this.defaultDownloader = defaultDownloader;
+        this.fallbackDownloader = fallbackDownloader;
+    }
+
+    @Override
+    public File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename) {
+        try {
+            return defaultDownloader.downloadFile(sender, fileId, defaultFilename);
+        } catch (FileDownloadException e) {
+            return fallbackDownloader.downloadFile(sender, fileId, defaultFilename);
+        }
+    }
+}

src/main/java/com/annimon/ffmpegbot/file/FileDownloadException.java

@@ -0,0 +1,11 @@
+package com.annimon.ffmpegbot.file;
+
+public class FileDownloadException extends RuntimeException {
+    public FileDownloadException(String message) {
+        super(message);
+    }
+
+    public FileDownloadException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/com/annimon/ffmpegbot/file/FileDownloader.java

@@ -0,0 +1,10 @@
+package com.annimon.ffmpegbot.file;
+
+import com.annimon.tgbotsmodule.services.CommonAbsSender;
+
+import java.io.File;
+
+public interface FileDownloader {
+
+    File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename);
+}

src/main/java/com/annimon/ffmpegbot/file/FilePath.java

@@ -1,6 +1,8 @@
-package com.annimon.ffmpegbot.session;
+package com.annimon.ffmpegbot.file;
 
+import com.annimon.ffmpegbot.session.FileType;
 import org.apache.commons.io.FilenameUtils;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
 
@@ -26,4 +28,12 @@ public class FilePath {
         final var ext = FilenameUtils.getExtension(filename);
         return "%d_%d.%s".formatted(System.currentTimeMillis(), Math.abs(fileId.hashCode()), ext);
     }
+
+    public static String defaultFilename(@NotNull FileType fileType) {
+        return "file." + switch (fileType) {
+            case ANIMATION, VIDEO, VIDEO_NOTE -> "mp4";
+            case AUDIO -> "mp3";
+            case VOICE -> "ogg";
+        };
+    }
 }

src/main/java/com/annimon/ffmpegbot/file/TelegramClientFileDownloader.java

@@ -0,0 +1,53 @@
+package com.annimon.ffmpegbot.file;
+
+import com.annimon.tgbotsmodule.services.CommonAbsSender;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TelegramClientFileDownloader implements FileDownloader {
+    private final String scriptPath;
+    private final String appId;
+    private final String appHash;
+    private final String botToken;
+    private final String botUsername;
+
+    public TelegramClientFileDownloader(String scriptPath, String appId, String appHash, String botToken, String botUsername) {
+        this.scriptPath = scriptPath;
+        this.appId = appId;
+        this.appHash = appHash;
+        this.botToken = botToken;
+        this.botUsername = botUsername;
+    }
+
+    @Override
+    public File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename) {
+        try {
+            final var tempFile = File.createTempFile("tmp", defaultFilename);
+            final ProcessBuilder pb = new ProcessBuilder(buildCommand(fileId, tempFile));
+            final Process process = pb.start();
+            int exitCode = process.waitFor();
+            if (exitCode != 0) {
+                throw new FileDownloadException("Downloader process finished with the exit code " + exitCode);
+            }
+            return tempFile;
+        } catch (InterruptedException | IOException e) {
+            throw new FileDownloadException("Downloader process failed", e);
+        }
+    }
+
+    private List<String> buildCommand(String fileId, File destFile) {
+        final var commands = new ArrayList<String>();
+        commands.addAll(List.of("python3", scriptPath));
+        commands.addAll(List.of("--api_id", appId));
+        commands.addAll(List.of("--api_hash", appHash));
+        commands.addAll(List.of("--bot_token", botToken));
+        commands.addAll(List.of("--bot_username", botUsername));
+        commands.add("get");
+        commands.addAll(List.of("--file_id", fileId));
+        commands.addAll(List.of("-o", destFile.getPath()));
+        return commands;
+    }
+}

src/main/java/com/annimon/ffmpegbot/file/TelegramFileDownloader.java

@@ -0,0 +1,24 @@
+package com.annimon.ffmpegbot.file;
+
+import com.annimon.ffmpegbot.TelegramRuntimeException;
+import com.annimon.tgbotsmodule.api.methods.Methods;
+import com.annimon.tgbotsmodule.services.CommonAbsSender;
+import org.apache.commons.io.FilenameUtils;
+import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
+
+import java.io.File;
+import java.io.IOException;
+
+public class TelegramFileDownloader implements FileDownloader {
+
+    @Override
+    public File downloadFile(CommonAbsSender sender, String fileId, String defaultFilename) {
+        try {
+            final var tgFile = Methods.getFile(fileId).call(sender);
+            final var extension = FilenameUtils.getExtension(tgFile.getFilePath());
+            return sender.downloadFile(tgFile, File.createTempFile("tmp", "file." + extension));
+        } catch (IOException | TelegramApiException | TelegramRuntimeException e) {
+            throw new FileDownloadException(e.getMessage(), e);
+        }
+    }
+}

src/main/java/com/annimon/ffmpegbot/session/Resolver.java

@@ -41,14 +41,6 @@ public class Resolver {
         };
     }
 
-    public static String resolveDefaultFilename(@NotNull FileType fileType) {
-        return "file." + switch (fileType) {
-            case ANIMATION, VIDEO, VIDEO_NOTE -> "mp4";
-            case AUDIO -> "mp3";
-            case VOICE -> "ogg";
-        };
-    }
-
     public static MediaMessageMethod<? extends MediaMessageMethod<?, ?>, ?> resolveMethod(@NotNull FileType fileType) {
         return switch (fileType) {
             case ANIMATION -> Methods.sendAnimation();