feat: support minicpm-v video normal speed

Author: MrAMS

tools/mtmd/mtmd-cli.cpp

@@ -162,21 +162,10 @@ struct mtmd_cli_context {
         return true;
     }
 
-    // Load multiple frames from a directory as a "video" (sequence of images)
+    // Load multiple frames from a video file or a directory as a "video" (sequence of images)
     // Returns number of frames appended
-    size_t load_video_dir(const std::string & dir, int max_frames = 32, int stride = 1, bool recursive = false) {
-        mtmd_video::LoadVideoOptions opts;
-        opts.max_frames = max_frames;
-        opts.stride     = stride;
-        opts.recursive  = recursive;
-        return mtmd_video::append_frames_from_dir(ctx_vision.get(), dir, bitmaps, opts);
-    }
-
-    size_t load_video_path(const std::string & path, int max_frames = 32, int stride = 1) {
-        mtmd_video::LoadVideoOptions opts;
-        opts.max_frames = max_frames;
-        opts.stride     = stride;
-        return mtmd_video::append_frames_from_path(ctx_vision.get(), path, bitmaps, opts);
+    size_t load_video(const std::string & path) {
+        return mtmd_video::append_frames_from_path(ctx_vision.get(), path, bitmaps);
     }
 };
 
@@ -322,7 +311,7 @@ int main(int argc, char ** argv) {
         for (const auto & vpath : params.video) {
             // for video understanding: disable UHD slicing (overview only)
             mtmd_set_minicpmv_max_slice_nums(ctx.ctx_vision.get(), 0);
-            size_t n = ctx.load_video_path(vpath, /*max_frames*/3, /*stride*/1);
+            size_t n = ctx.load_video(vpath);
             if (n == 0) {
                 LOG_ERR("Unable to load video frames from %s\n", vpath.c_str());
                 return 1;
@@ -399,23 +388,19 @@ int main(int argc, char ** argv) {
                 }
                 std::string media_path = line.substr(7);
                 if (is_video) {
-                    // parse optional args: "/video <dir> [max_frames] [stride]"
+                    // parse optional args: "/video <file/dir path>"
                     // simple split by spaces
                     std::vector<std::string> parts = string_split(media_path, " ");
-                    std::string dir = parts.size() > 0 ? parts[0] : media_path;
-                    int max_frames = 32;
-                    int stride = 1;
-                    if (parts.size() > 1) max_frames = std::max(1, atoi(parts[1].c_str()));
-                    if (parts.size() > 2) stride     = std::max(1, atoi(parts[2].c_str()));
-                    size_t n = ctx.load_video_path(dir, max_frames, stride);
+                    std::string path = parts.size() > 0 ? parts[0] : media_path;
+                    size_t n = ctx.load_video(path);
                     if (n > 0) {
-                        LOG("%s video loaded with %zu frames\n", dir.c_str(), n);
+                        LOG("%s video loaded with %zu frames\n", path.c_str(), n);
                         // add one marker per frame to match mtmd_tokenize expectations
                         for (size_t i = 0; i < n; ++i) {
                             content += mtmd_default_marker();
                         }
                     } else {
-                        LOG_ERR("ERR: failed to load video frames from %s\n", dir.c_str());
+                        LOG_ERR("ERR: failed to load video frames from %s\n", path.c_str());
                     }
                 } else {
                     if (ctx.load_media(media_path)) {

tools/mtmd/mtmd-video.cpp

@@ -9,6 +9,7 @@
 #include <cerrno>
 #include <cstring>
 #include <memory>
+#include <cmath>
 
 #if defined(_WIN32)
 #define WIN32_LEAN_AND_MEAN
@@ -148,6 +149,44 @@ struct DecodedFrameRGBA {
     std::vector<unsigned char> rgba; // size = width * height * 4
 };
 
+bool get_video_info_ffmpeg(const std::string &file, VideoInfo &info) {
+    AVFormatContext *fmt = nullptr;
+    if (avformat_open_input(&fmt, file.c_str(), nullptr, nullptr) < 0) {
+        return false;
+    }
+
+    std::unique_ptr<AVFormatContext, void(*)(AVFormatContext*)> fmt_guard(fmt, 
+        [](AVFormatContext *f){ if (f) {avformat_close_input(&f);} });
+
+    if (avformat_find_stream_info(fmt, nullptr) < 0) {
+        return false;
+    }
+
+    // find video stream
+    int vstream = av_find_best_stream(fmt, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
+    if (vstream < 0) {
+        return false;
+    }
+
+    AVStream *st = fmt->streams[vstream];
+
+    // get fps
+    if (st->avg_frame_rate.num > 0 && st->avg_frame_rate.den > 0){
+        info.fps = av_q2d(st->avg_frame_rate);
+    }else if (st->r_frame_rate.num > 0 && st->r_frame_rate.den > 0){
+        info.fps = av_q2d(st->r_frame_rate);
+    }
+    // get total frames
+    if (st->nb_frames > 0){
+        info.total_frames = st->nb_frames;
+    }else if (fmt->duration > 0 && info.fps > 0.0){
+        // estimate total frames if nb_frames is not available
+        info.total_frames = std::llround((fmt->duration / (double)AV_TIME_BASE) * info.fps);
+    }
+
+    return true;
+}
+
 static bool decode_video_ffmpeg_to_rgba(const std::string & file,
                                         std::vector<DecodedFrameRGBA> & frames,
                                         int max_frames,
@@ -239,12 +278,40 @@ bool load_frames_from_file(mtmd_context * /*ctx*/,
                            const LoadVideoOptions & /*opts*/) {
     return false;
 }
+bool get_video_info_ffmpeg(const std::string &file, VideoInfo &info) {
+    return false;
+}
 #endif
 
 size_t append_frames_from_path(mtmd_context * ctx,
                                const std::string & path,
-                               mtmd::bitmaps & dst,
-                               const LoadVideoOptions & opts) {
+                               mtmd::bitmaps & dst) {
+    mtmd_video::LoadVideoOptions opts;
+    opts.max_frames = 32;
+    opts.stride     = 1;
+    opts.recursive  = false;
+
+    auto info = mtmd_video::VideoInfo{};
+    if(is_dir(path)) {
+        info.fps = 1;
+        std::vector<std::string> files;
+        list_files(path, files, opts.recursive);
+        info.total_frames = files.size();
+    } else {
+        mtmd_video::get_video_info_ffmpeg(path, info);
+    }
+
+    // minicpm normal speed
+    const int32_t minicpmv_max_video_frames = 64;
+    opts.max_frames = minicpmv_max_video_frames;
+    if(info.total_frames > minicpmv_max_video_frames) {
+        // uniform sample
+        opts.stride = (int)std::ceil((double)info.total_frames / minicpmv_max_video_frames);
+    } else {
+        // 1 frame per second
+        opts.stride = (info.fps > 1.0) ? (int)std::ceil(info.fps) : 1;
+    }
+
     if (is_dir(path)) {
         return append_frames_from_dir(ctx, path, dst, opts);
     } else {

tools/mtmd/mtmd-video.h

@@ -18,6 +18,11 @@ struct LoadVideoOptions {
     bool recursive = false;// scan subdirectories
 };
 
+struct VideoInfo {
+    double fps = 0.0;           // frames per second
+    int64_t total_frames = 0;   // total number of frames
+};
+
 // Load frames from a directory of images (jpg/png/bmp/webp etc.)
 // Returns true on success (>=1 frame loaded), false otherwise.
 bool load_frames_from_dir(mtmd_context * ctx,
@@ -32,6 +37,10 @@ size_t append_frames_from_dir(mtmd_context * ctx,
                               mtmd::bitmaps & dst,
                               const LoadVideoOptions & opts = {});
 
+// get video info (fps, total frames) via FFmpeg (mp4/mov/mkv/avi/webm...).
+// Returns true on success, false otherwise.
+bool get_video_info_ffmpeg(const std::string &file, VideoInfo &info);
+
 // Load frames from a video file via FFmpeg (mp4/mov/mkv/avi/webm...).
 // Returns true on success (>=1 frame loaded), false otherwise.
 bool load_frames_from_file(mtmd_context * ctx,
@@ -43,8 +52,7 @@ bool load_frames_from_file(mtmd_context * ctx,
 // Returns number of frames appended (0 on failure or unsupported input).
 size_t append_frames_from_path(mtmd_context * ctx,
                                const std::string & path,
-                               mtmd::bitmaps & dst,
-                               const LoadVideoOptions & opts = {});
+                               mtmd::bitmaps & dst);
 
 } // namespace mtmd_video