diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index e51200393e06..ddbe96851d32 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -2050,6 +2050,32 @@ def _video_frames(video: Any) -> List[Any]: return list(video.frames) return list(video) if isinstance(video, (list, tuple)) else [video] + @staticmethod + def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: + """Timing of one raw video item, in the form the HF video processor takes. + + ``VideoData.metadata`` carries the source ``fps`` and the indices of + the sampled frames. With both, the processor writes a + ``]<]{t} seconds[>[`` timestamp before each temporal group. A video + without usable timing (no metadata, ``fps`` missing or not positive, + no frame indices) gets ``fps=None`` and no timestamps. Only the keys + transformers' ``VideoMetadata`` accepts are passed, and + ``total_num_frames`` is the number of frames delivered, because the + media loader has already sampled them. + """ + metadata = getattr(video, "metadata", None) + metadata = metadata if isinstance(metadata, Mapping) else {} + fps = metadata.get("fps") + frames_indices = metadata.get("frames_indices") + if fps is None or fps <= 0 or frames_indices is None or len(frames_indices) == 0: + fps, frames_indices = None, range(num_frames) + return { + "total_num_frames": num_frames, + "fps": None if fps is None else float(fps), + "duration": metadata.get("duration"), + "frames_indices": [int(i) for i in frames_indices], + } + def _video_bracket_count(self, video: Any) -> int: """Temporal patches of a video item: one framed bracket each. @@ -2273,6 +2299,15 @@ def call_with_text_prompt( frame_iter = iter(flat_frames) videos = [[next(frame_iter) for _ in frames] for frames in videos] processor_kwargs["videos_kwargs"] = {"do_resize": True, **video_kwargs} + # Frame timing goes in only when some video has it; the others get + # fps=None, so the processor gives timestamps to exactly the videos + # with timing. The token-ID fast path does not add timestamps. + video_metadata = [ + self._video_metadata(video, len(frames)) + for video, frames in zip(video_inputs, videos) + ] + if any(m["fps"] is not None for m in video_metadata): + processor_kwargs["videos_kwargs"]["video_metadata"] = video_metadata processed = self.processor( text=[templated_text], images=images if images else None, diff --git a/tests/unittest/_torch/models/test_minimax_m3_vl.py b/tests/unittest/_torch/models/test_minimax_m3_vl.py index 1586a9d7483a..77a7eb55e878 100644 --- a/tests/unittest/_torch/models/test_minimax_m3_vl.py +++ b/tests/unittest/_torch/models/test_minimax_m3_vl.py @@ -2909,3 +2909,86 @@ def test_mixed_format_media_fall_back_to_uint8(monkeypatch): assert isinstance(frame, np.ndarray) and frame.dtype == np.uint8 np.testing.assert_array_equal(frame, array) assert recorder.call_kwargs[1] == {"videos_kwargs": {"do_resize": True}} + + +@pytest.mark.cpu_only +def test_video_timing_is_forwarded_to_the_processor(monkeypatch): + """``VideoData.metadata`` reaches the HF processor as ``video_metadata``. + + With the source fps and the sampled frame indices, the processor writes a + timestamp before each temporal group. Only the keys transformers' + ``VideoMetadata`` accepts are passed, and ``total_num_frames`` counts the + delivered frames. A video without timing in the same request gets + ``fps=None``, so only the timed video gets timestamps. + """ + recorder = _MediaRecordingProcessor() + proc, _ = _tokenizer_only_processor(monkeypatch, recorder) + frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 4 + timed = SimpleNamespace( + frames=frames, + metadata={ + "total_num_frames": 40, + "fps": 10, + "duration": 4.0, + "frames_indices": [0, 13, 26, 39], + "unrelated": "dropped", + }, + ) + expected = { + "total_num_frames": 4, + "fps": 10.0, + "duration": 4.0, + "frames_indices": [0, 13, 26, 39], + } + + proc.call_with_text_prompt( + {"prompt": "watch", "multi_modal_data": {"video": [timed]}}, SimpleNamespace() + ) + assert recorder.call_kwargs[0] == { + "videos_kwargs": {"do_resize": True, "video_metadata": [expected]} + } + + proc.call_with_text_prompt( + { + "prompt": "watch watch", + "multi_modal_data": {"video": [timed, SimpleNamespace(frames=frames[:2])]}, + }, + SimpleNamespace(), + ) + assert recorder.call_kwargs[1]["videos_kwargs"]["video_metadata"] == [ + expected, + {"total_num_frames": 2, "fps": None, "duration": None, "frames_indices": [0, 1]}, + ] + + video_utils = pytest.importorskip("transformers.video_utils") + assert video_utils.VideoMetadata(**expected).timestamps == pytest.approx([0.0, 1.3, 2.6, 3.9]) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "metadata", + [ + "frame_list", + None, + {"fps": 0, "frames_indices": [0, 1]}, + {"fps": 10.0}, + {"fps": 10.0, "frames_indices": []}, + ], + ids=["frame_list", "no_metadata", "zero_fps", "no_frame_indices", "empty_frame_indices"], +) +def test_video_without_timing_passes_no_metadata(monkeypatch, metadata): + """Without usable timing the processor gets no ``video_metadata`` and adds no timestamps.""" + recorder = _MediaRecordingProcessor() + proc, _ = _tokenizer_only_processor(monkeypatch, recorder) + frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 2 + if metadata == "frame_list": + video = frames + elif metadata is None: + video = SimpleNamespace(frames=frames) + else: + video = SimpleNamespace(frames=frames, metadata=metadata) + + proc.call_with_text_prompt( + {"prompt": "watch", "multi_modal_data": {"video": [video]}}, SimpleNamespace() + ) + assert recorder.call_kwargs[0] == {"videos_kwargs": {"do_resize": True}}