[None][fix] MiniMax-M3 VL: forward video metadata so videos get timestamps

tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py

@@ -2050,6 +2050,32 @@ def _video_frames(video: Any) -> List[Any]:
             return list(video.frames)
         return list(video) if isinstance(video, (list, tuple)) else [video]
 
+    @staticmethod
+    def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]:
+        """Timing of one raw video item, in the form the HF video processor takes.
+
+        ``VideoData.metadata`` carries the source ``fps`` and the indices of
+        the sampled frames. With both, the processor writes a
+        ``]<]{t} seconds[>[`` timestamp before each temporal group. A video
+        without usable timing (no metadata, ``fps`` missing or not positive,
+        no frame indices) gets ``fps=None`` and no timestamps. Only the keys
+        transformers' ``VideoMetadata`` accepts are passed, and
+        ``total_num_frames`` is the number of frames delivered, because the
+        media loader has already sampled them.
+        """
+        metadata = getattr(video, "metadata", None)
+        metadata = metadata if isinstance(metadata, Mapping) else {}
+        fps = metadata.get("fps")
+        frames_indices = metadata.get("frames_indices")
+        if fps is None or fps <= 0 or frames_indices is None or len(frames_indices) == 0:
+            fps, frames_indices = None, range(num_frames)
+        return {
+            "total_num_frames": num_frames,
+            "fps": None if fps is None else float(fps),
+            "duration": metadata.get("duration"),
+            "frames_indices": [int(i) for i in frames_indices],
+        }
+
     def _video_bracket_count(self, video: Any) -> int:
         """Temporal patches of a video item: one framed bracket each.
 
@@ -2273,6 +2299,15 @@ def call_with_text_prompt(
             frame_iter = iter(flat_frames)
             videos = [[next(frame_iter) for _ in frames] for frames in videos]
             processor_kwargs["videos_kwargs"] = {"do_resize": True, **video_kwargs}
+            # Frame timing goes in only when some video has it; the others get
+            # fps=None, so the processor gives timestamps to exactly the videos
+            # with timing. The token-ID fast path does not add timestamps.
+            video_metadata = [
+                self._video_metadata(video, len(frames))
+                for video, frames in zip(video_inputs, videos)
+            ]
+            if any(m["fps"] is not None for m in video_metadata):
+                processor_kwargs["videos_kwargs"]["video_metadata"] = video_metadata
         processed = self.processor(
             text=[templated_text],
             images=images if images else None,

tests/unittest/_torch/models/test_minimax_m3_vl.py

@@ -2909,3 +2909,86 @@ def test_mixed_format_media_fall_back_to_uint8(monkeypatch):
         assert isinstance(frame, np.ndarray) and frame.dtype == np.uint8
         np.testing.assert_array_equal(frame, array)
     assert recorder.call_kwargs[1] == {"videos_kwargs": {"do_resize": True}}
+
+
+@pytest.mark.cpu_only
+def test_video_timing_is_forwarded_to_the_processor(monkeypatch):
+    """``VideoData.metadata`` reaches the HF processor as ``video_metadata``.
+
+    With the source fps and the sampled frame indices, the processor writes a
+    timestamp before each temporal group. Only the keys transformers'
+    ``VideoMetadata`` accepts are passed, and ``total_num_frames`` counts the
+    delivered frames. A video without timing in the same request gets
+    ``fps=None``, so only the timed video gets timestamps.
+    """
+    recorder = _MediaRecordingProcessor()
+    proc, _ = _tokenizer_only_processor(monkeypatch, recorder)
+    frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 4
+    timed = SimpleNamespace(
+        frames=frames,
+        metadata={
+            "total_num_frames": 40,
+            "fps": 10,
+            "duration": 4.0,
+            "frames_indices": [0, 13, 26, 39],
+            "unrelated": "dropped",
+        },
+    )
+    expected = {
+        "total_num_frames": 4,
+        "fps": 10.0,
+        "duration": 4.0,
+        "frames_indices": [0, 13, 26, 39],
+    }
+
+    proc.call_with_text_prompt(
+        {"prompt": "watch", "multi_modal_data": {"video": [timed]}}, SimpleNamespace()
+    )
+    assert recorder.call_kwargs[0] == {
+        "videos_kwargs": {"do_resize": True, "video_metadata": [expected]}
+    }
+
+    proc.call_with_text_prompt(
+        {
+            "prompt": "watch watch",
+            "multi_modal_data": {"video": [timed, SimpleNamespace(frames=frames[:2])]},
+        },
+        SimpleNamespace(),
+    )
+    assert recorder.call_kwargs[1]["videos_kwargs"]["video_metadata"] == [
+        expected,
+        {"total_num_frames": 2, "fps": None, "duration": None, "frames_indices": [0, 1]},
+    ]
+
+    video_utils = pytest.importorskip("transformers.video_utils")
+    assert video_utils.VideoMetadata(**expected).timestamps == pytest.approx([0.0, 1.3, 2.6, 3.9])
+
+
+@pytest.mark.cpu_only
+@pytest.mark.parametrize(
+    "metadata",
+    [
+        "frame_list",
+        None,
+        {"fps": 0, "frames_indices": [0, 1]},
+        {"fps": 10.0},
+        {"fps": 10.0, "frames_indices": []},
+    ],
+    ids=["frame_list", "no_metadata", "zero_fps", "no_frame_indices", "empty_frame_indices"],
+)
+def test_video_without_timing_passes_no_metadata(monkeypatch, metadata):
+    """Without usable timing the processor gets no ``video_metadata`` and adds no timestamps."""
+    recorder = _MediaRecordingProcessor()
+    proc, _ = _tokenizer_only_processor(monkeypatch, recorder)
+    frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 2
+    if metadata == "frame_list":
+        video = frames
+    elif metadata is None:
+        video = SimpleNamespace(frames=frames)
+    else:
+        video = SimpleNamespace(frames=frames, metadata=metadata)
+
+    proc.call_with_text_prompt(
+        {"prompt": "watch", "multi_modal_data": {"video": [video]}}, SimpleNamespace()
+    )
+    assert recorder.call_kwargs[0] == {"videos_kwargs": {"do_resize": True}}