From 1c65d0f0fc2f388d5ac23ef5f8c2619f7ff2fdc5 Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:33:20 -0700 Subject: [PATCH 1/5] [None][fix] MiniMax-M3 VL: forward video metadata so videos get timestamps call_with_text_prompt kept only VideoData.frames and dropped VideoData.metadata, so the HF MiniMax processor never knew a video's fps or sampled frame indices and wrote no "]<]{t} seconds[>[" timestamp before its temporal groups. The model could not place events in time. Pass each video's timing as videos_kwargs["video_metadata"] (only the keys transformers' VideoMetadata accepts; total_num_frames = frames delivered), and only when some video in the request has timing. A video without usable timing gets fps=None and no timestamps, as before. The float-media rescale is handled by #20626 and is unchanged here. The token-ID fast path still expands videos without timestamps. Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../_torch/models/modeling_minimaxm3_vl.py | 35 ++++++++ .../_torch/models/test_minimax_m3_vl.py | 83 +++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index e51200393e06..ddbe96851d32 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -2050,6 +2050,32 @@ def _video_frames(video: Any) -> List[Any]: return list(video.frames) return list(video) if isinstance(video, (list, tuple)) else [video] + @staticmethod + def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: + """Timing of one raw video item, in the form the HF video processor takes. + + ``VideoData.metadata`` carries the source ``fps`` and the indices of + the sampled frames. With both, the processor writes a + ``]<]{t} seconds[>[`` timestamp before each temporal group. A video + without usable timing (no metadata, ``fps`` missing or not positive, + no frame indices) gets ``fps=None`` and no timestamps. Only the keys + transformers' ``VideoMetadata`` accepts are passed, and + ``total_num_frames`` is the number of frames delivered, because the + media loader has already sampled them. + """ + metadata = getattr(video, "metadata", None) + metadata = metadata if isinstance(metadata, Mapping) else {} + fps = metadata.get("fps") + frames_indices = metadata.get("frames_indices") + if fps is None or fps <= 0 or frames_indices is None or len(frames_indices) == 0: + fps, frames_indices = None, range(num_frames) + return { + "total_num_frames": num_frames, + "fps": None if fps is None else float(fps), + "duration": metadata.get("duration"), + "frames_indices": [int(i) for i in frames_indices], + } + def _video_bracket_count(self, video: Any) -> int: """Temporal patches of a video item: one framed bracket each. @@ -2273,6 +2299,15 @@ def call_with_text_prompt( frame_iter = iter(flat_frames) videos = [[next(frame_iter) for _ in frames] for frames in videos] processor_kwargs["videos_kwargs"] = {"do_resize": True, **video_kwargs} + # Frame timing goes in only when some video has it; the others get + # fps=None, so the processor gives timestamps to exactly the videos + # with timing. The token-ID fast path does not add timestamps. + video_metadata = [ + self._video_metadata(video, len(frames)) + for video, frames in zip(video_inputs, videos) + ] + if any(m["fps"] is not None for m in video_metadata): + processor_kwargs["videos_kwargs"]["video_metadata"] = video_metadata processed = self.processor( text=[templated_text], images=images if images else None, diff --git a/tests/unittest/_torch/models/test_minimax_m3_vl.py b/tests/unittest/_torch/models/test_minimax_m3_vl.py index 1586a9d7483a..77a7eb55e878 100644 --- a/tests/unittest/_torch/models/test_minimax_m3_vl.py +++ b/tests/unittest/_torch/models/test_minimax_m3_vl.py @@ -2909,3 +2909,86 @@ def test_mixed_format_media_fall_back_to_uint8(monkeypatch): assert isinstance(frame, np.ndarray) and frame.dtype == np.uint8 np.testing.assert_array_equal(frame, array) assert recorder.call_kwargs[1] == {"videos_kwargs": {"do_resize": True}} + + +@pytest.mark.cpu_only +def test_video_timing_is_forwarded_to_the_processor(monkeypatch): + """``VideoData.metadata`` reaches the HF processor as ``video_metadata``. + + With the source fps and the sampled frame indices, the processor writes a + timestamp before each temporal group. Only the keys transformers' + ``VideoMetadata`` accepts are passed, and ``total_num_frames`` counts the + delivered frames. A video without timing in the same request gets + ``fps=None``, so only the timed video gets timestamps. + """ + recorder = _MediaRecordingProcessor() + proc, _ = _tokenizer_only_processor(monkeypatch, recorder) + frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 4 + timed = SimpleNamespace( + frames=frames, + metadata={ + "total_num_frames": 40, + "fps": 10, + "duration": 4.0, + "frames_indices": [0, 13, 26, 39], + "unrelated": "dropped", + }, + ) + expected = { + "total_num_frames": 4, + "fps": 10.0, + "duration": 4.0, + "frames_indices": [0, 13, 26, 39], + } + + proc.call_with_text_prompt( + {"prompt": "watch", "multi_modal_data": {"video": [timed]}}, SimpleNamespace() + ) + assert recorder.call_kwargs[0] == { + "videos_kwargs": {"do_resize": True, "video_metadata": [expected]} + } + + proc.call_with_text_prompt( + { + "prompt": "watch watch", + "multi_modal_data": {"video": [timed, SimpleNamespace(frames=frames[:2])]}, + }, + SimpleNamespace(), + ) + assert recorder.call_kwargs[1]["videos_kwargs"]["video_metadata"] == [ + expected, + {"total_num_frames": 2, "fps": None, "duration": None, "frames_indices": [0, 1]}, + ] + + video_utils = pytest.importorskip("transformers.video_utils") + assert video_utils.VideoMetadata(**expected).timestamps == pytest.approx([0.0, 1.3, 2.6, 3.9]) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "metadata", + [ + "frame_list", + None, + {"fps": 0, "frames_indices": [0, 1]}, + {"fps": 10.0}, + {"fps": 10.0, "frames_indices": []}, + ], + ids=["frame_list", "no_metadata", "zero_fps", "no_frame_indices", "empty_frame_indices"], +) +def test_video_without_timing_passes_no_metadata(monkeypatch, metadata): + """Without usable timing the processor gets no ``video_metadata`` and adds no timestamps.""" + recorder = _MediaRecordingProcessor() + proc, _ = _tokenizer_only_processor(monkeypatch, recorder) + frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 2 + if metadata == "frame_list": + video = frames + elif metadata is None: + video = SimpleNamespace(frames=frames) + else: + video = SimpleNamespace(frames=frames, metadata=metadata) + + proc.call_with_text_prompt( + {"prompt": "watch", "multi_modal_data": {"video": [video]}}, SimpleNamespace() + ) + assert recorder.call_kwargs[0] == {"videos_kwargs": {"do_resize": True}} From a9af7b3fd3811efa71b259be367a5bfc2bff2822 Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:28:36 -0700 Subject: [PATCH 2/5] [None][fix] Forward MiniMax-M3 video timing only when it is finite and per frame _video_metadata let a NaN or infinite fps through, because only fps <= 0 was rejected: the processor would then write "nan seconds" or "0.0 seconds" for every temporal group. It also accepted frame indices of any non-zero length, and the processor clamps its lookups into that list, so indices that do not match the delivered frames gave silently wrong timestamps. Treat a video as timed only when fps is finite and positive and there is exactly one frame index per delivered frame; anything else takes the existing no-timing fallback (fps=None, no timestamps). Add NaN, inf and both index-count mismatches to test_video_without_timing_passes_no_metadata. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> --- .../_torch/models/modeling_minimaxm3_vl.py | 20 +++++++++++----- .../_torch/models/test_minimax_m3_vl.py | 23 +++++++++++++++++-- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index ddbe96851d32..c052946335e5 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -13,6 +13,7 @@ from __future__ import annotations +import math from dataclasses import dataclass, field from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -2057,17 +2058,24 @@ def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: ``VideoData.metadata`` carries the source ``fps`` and the indices of the sampled frames. With both, the processor writes a ``]<]{t} seconds[>[`` timestamp before each temporal group. A video - without usable timing (no metadata, ``fps`` missing or not positive, - no frame indices) gets ``fps=None`` and no timestamps. Only the keys - transformers' ``VideoMetadata`` accepts are passed, and - ``total_num_frames`` is the number of frames delivered, because the - media loader has already sampled them. + without usable timing (no metadata, ``fps`` missing, not finite or + not positive, or not one frame index per delivered frame) gets + ``fps=None`` and no timestamps. Only the keys transformers' + ``VideoMetadata`` accepts are passed, and ``total_num_frames`` is the + number of frames delivered, because the media loader has already + sampled them. """ metadata = getattr(video, "metadata", None) metadata = metadata if isinstance(metadata, Mapping) else {} fps = metadata.get("fps") frames_indices = metadata.get("frames_indices") - if fps is None or fps <= 0 or frames_indices is None or len(frames_indices) == 0: + if ( + fps is None + or not math.isfinite(fps) + or fps <= 0 + or frames_indices is None + or len(frames_indices) != num_frames + ): fps, frames_indices = None, range(num_frames) return { "total_num_frames": num_frames, diff --git a/tests/unittest/_torch/models/test_minimax_m3_vl.py b/tests/unittest/_torch/models/test_minimax_m3_vl.py index 77a7eb55e878..7157e63bee30 100644 --- a/tests/unittest/_torch/models/test_minimax_m3_vl.py +++ b/tests/unittest/_torch/models/test_minimax_m3_vl.py @@ -2971,13 +2971,32 @@ def test_video_timing_is_forwarded_to_the_processor(monkeypatch): "frame_list", None, {"fps": 0, "frames_indices": [0, 1]}, + {"fps": float("nan"), "frames_indices": [0, 1]}, + {"fps": float("inf"), "frames_indices": [0, 1]}, {"fps": 10.0}, {"fps": 10.0, "frames_indices": []}, + {"fps": 10.0, "frames_indices": [0]}, + {"fps": 10.0, "frames_indices": [0, 5, 9]}, + ], + ids=[ + "frame_list", + "no_metadata", + "zero_fps", + "nan_fps", + "inf_fps", + "no_frame_indices", + "empty_frame_indices", + "fewer_frame_indices_than_frames", + "more_frame_indices_than_frames", ], - ids=["frame_list", "no_metadata", "zero_fps", "no_frame_indices", "empty_frame_indices"], ) def test_video_without_timing_passes_no_metadata(monkeypatch, metadata): - """Without usable timing the processor gets no ``video_metadata`` and adds no timestamps.""" + """Without usable timing the processor gets no ``video_metadata`` and adds no timestamps. + + Usable timing is a finite, positive ``fps`` and one frame index per + delivered frame: a timestamp from a NaN or infinite rate, or from indices + that do not line up with the frames, would be wrong. + """ recorder = _MediaRecordingProcessor() proc, _ = _tokenizer_only_processor(monkeypatch, recorder) frames = [np.zeros((28, 42, 3), dtype=np.uint8)] * 2 From deea791c0a249ecd9dba9f0220c4b16fee723511 Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:33:42 -0700 Subject: [PATCH 3/5] [None][fix] Write MiniMax-M3 video timestamps on the token-ID fast path too With video timing forwarded to the HF processor, the text-prompt path writes "]<]{t:.1f} seconds[>[" before each temporal bracket of a timed video, but expand_prompt_token_ids_for_mm (the prompt_token_ids path that Dynamo and other pre-tokenizing front ends take) still put the VISION_START + VIDEO_TOKEN*n + VISION_END brackets next to each other. The same request got different ids by route, and the token-ID route lost the video's timing. Add _video_timestamp_ids: for a video that _video_metadata finds timed, the processor's timestamp for each bracket (frames_indices[min(b * temporal_patch_size, n - 1)] / fps), encoded with the model tokenizer without special tokens. expand_prompt_token_ids_for_mm puts those ids before each bracket. A video without timing, and every image, expands exactly as before and never reaches the tokenizer. The timestamps are plain text tokens outside the multimodal mask, so the per-video token count (get_num_tokens_per_video) is unchanged, and the KV-reuse hashing splits a timed video into one run per bracket, as it already does for the text path's ids. Each timestamp is encoded on its own, which matches the processor everywhere except a first timestamp whose preceding prompt text ends in something the tokenizer merges with "]<" (a space or a period, say); ids tokenized around the marker cannot carry that merge. trtllm-serve renders a newline there. Tests: a CPU test for a timed and an untimed video in one request (ids, tokenizer calls, mm runs), and a timed parametrization of the real-checkpoint parity test, whose prompt now has a newline before the marker. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> --- .../_torch/models/modeling_minimaxm3_vl.py | 49 ++++++++-- .../_torch/models/test_minimax_m3_vl.py | 89 ++++++++++++++++++- 2 files changed, 127 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index c052946335e5..edc37c1b2bd4 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -2015,10 +2015,10 @@ def get_num_tokens_per_video( The HF processor expands one ``]<]video[>[`` marker into one bracket per temporal patch, ``VISION_START + VIDEO_TOKEN * (grid_h*grid_w / merge^2) + VISION_END`` repeated ``grid_t`` times, so the count is - ``grid_t * (grid_h*grid_w / merge^2 + 2)``. The brackets are - adjacent: the processor prefixes a timestamp only when the video - carries fps metadata, and the frame lists handed over here carry - none. + ``grid_t * (grid_h*grid_w / merge^2 + 2)``. A video with timing + also gets a timestamp before each bracket (see + :meth:`_video_timestamp_ids`); those are plain text tokens, outside + the multimodal mask, so they are not counted here. ``video_grid_thw`` is the processor's grid for this video, computed by the dummy-text pass of :func:`find_mm_token_lengths` @@ -2084,6 +2084,36 @@ def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: "frames_indices": [int(i) for i in frames_indices], } + def _video_timestamp_ids(self, video: Any, brackets: int) -> List[List[int]]: + """Token ids of the timestamp the processor writes before each bracket of a video. + + For a video with timing (see :meth:`_video_metadata`), bracket ``b`` + gets ``]<]{t:.1f} seconds[>[``, where + ``t = frames_indices[min(b * temporal_patch_size, n - 1)] / fps``, as + the checkpoint's processor writes it; a video without timing gets none. + + Each timestamp is encoded on its own. That gives the processor's ids + after a VISION_END, where the tokenizer splits the processor's text at + the special token. Before the first bracket the processor's text + continues the prompt text before the marker: when that text ends in, + for example, a space or a period, the tokenizer merges it with the + timestamp, which prompt ids tokenized around the marker cannot + reproduce, and the first timestamp's ids differ. A newline before the + marker, as in trtllm-serve's rendered prompts, gives the same ids. + """ + metadata = self._video_metadata(video, len(self._video_frames(video))) + if metadata["fps"] is None: + return [[] for _ in range(brackets)] + temporal = int(self.processor.video_processor.temporal_patch_size) + indices = metadata["frames_indices"] + times = [ + indices[min(b * temporal, len(indices) - 1)] / metadata["fps"] for b in range(brackets) + ] + return [ + self._tokenizer.encode(f"]<]{t:.1f} seconds[>[", add_special_tokens=False) + for t in times + ] + def _video_bracket_count(self, video: Any) -> int: """Temporal patches of a video item: one framed bracket each. @@ -2116,7 +2146,9 @@ def expand_prompt_token_ids_for_mm( (which already accounts for the +2 framing per bracket). An image is one bracket; a video is one bracket per temporal patch, as the processor lays it out (see :meth:`get_num_tokens_per_video`), so the - placeholder runs line up with the vision rows at merge time. + placeholder runs line up with the vision rows at merge time. A video + with timing gets the processor's timestamp before each bracket (see + :meth:`_video_timestamp_ids`); one without keeps adjacent brackets. """ mm_data = mm_data or {} has_image = bool(mm_data.get("image")) @@ -2142,6 +2174,7 @@ def expand_prompt_token_ids_for_mm( f"num_mm_tokens entries ({expected})." ) brackets = self._video_bracket_count(items[consumed]) if has_video else 1 + stamps = self._video_timestamp_ids(items[consumed], brackets) if has_video else [[]] total = int(num_mm_tokens_per_placeholder[consumed]) inner, remainder = divmod(total - 2 * brackets, brackets) if inner < 1 or remainder: @@ -2149,7 +2182,8 @@ def expand_prompt_token_ids_for_mm( f"MiniMaxM3VL fast path: num_mm_tokens[{consumed}]={total} must be " f"{brackets} x (VISION_START + >=1 placeholder + VISION_END)." ) - for _ in range(brackets): + for stamp in stamps: + expanded.extend(stamp) expanded.append(self._vision_start_token_id) expanded.extend([placeholder_id] * inner) expanded.append(self._vision_end_token_id) @@ -2309,7 +2343,8 @@ def call_with_text_prompt( processor_kwargs["videos_kwargs"] = {"do_resize": True, **video_kwargs} # Frame timing goes in only when some video has it; the others get # fps=None, so the processor gives timestamps to exactly the videos - # with timing. The token-ID fast path does not add timestamps. + # with timing. ``expand_prompt_token_ids_for_mm`` writes the same + # timestamps on the token-ID fast path. video_metadata = [ self._video_metadata(video, len(frames)) for video, frames in zip(video_inputs, videos) diff --git a/tests/unittest/_torch/models/test_minimax_m3_vl.py b/tests/unittest/_torch/models/test_minimax_m3_vl.py index 7157e63bee30..8aa0951cd145 100644 --- a/tests/unittest/_torch/models/test_minimax_m3_vl.py +++ b/tests/unittest/_torch/models/test_minimax_m3_vl.py @@ -1809,8 +1809,11 @@ def _real_checkpoint_input_processor(): return cls(model_path=checkpoint, config=config, tokenizer=tokenizer), tokenizer +@pytest.mark.parametrize("timed", [False, True], ids=["untimed", "timed"]) @pytest.mark.parametrize("num_frames,side", [(3, 230), (4, 224), (7, 300)]) -def test_real_checkpoint_video_fast_path_matches_processor_input_ids(num_frames: int, side: int): +def test_real_checkpoint_video_fast_path_matches_processor_input_ids( + num_frames: int, side: int, timed: bool +): """The token-ID fast path expands a video exactly as the checkpoint's processor does. The same frames go through the text-prompt path (the HF processor writes @@ -1820,21 +1823,40 @@ def test_real_checkpoint_video_fast_path_matches_processor_input_ids(num_frames: of the processor's ``video_grid_thw``. Odd and even frame counts cover the last-frame padding, and the frame sides are not multiples of the patch size, so the processor's resize is on the path. + + A ``VideoData`` with timing gets a ``]<]{t:.1f} seconds[>[`` timestamp + before each bracket on both paths; a frame list gets none. The marker + follows a newline, as in trtllm-serve's rendered prompts (see + ``_video_timestamp_ids`` for text glued to the marker). """ + from tensorrt_llm.inputs.multimodal_data import VideoData + proc, tokenizer = _real_checkpoint_input_processor() rng = np.random.default_rng(num_frames) frames = [rng.integers(0, 256, size=(side, side, 3), dtype=np.uint8) for _ in range(num_frames)] - prompt = "Watch: ]<]video[>[ then summarize." + video = frames + if timed: + # Every 7th frame of a 3 fps source: 0.0, 4.7, 9.3, 14.0, ... seconds. + video = VideoData( + frames=frames, + metadata={ + "total_num_frames": 7 * num_frames, + "fps": 3.0, + "duration": 7 * num_frames / 3.0, + "frames_indices": list(range(0, 7 * num_frames, 7)), + }, + ) + prompt = "Watch:\n]<]video[>[ then summarize." sampling = SimpleNamespace(add_special_tokens=False, truncate_prompt_tokens=None) slow_ids, slow_extra = proc.call_with_text_prompt( - {"prompt": prompt, "multi_modal_data": {"video": [frames]}, "chat_template_applied": True}, + {"prompt": prompt, "multi_modal_data": {"video": [video]}, "chat_template_applied": True}, sampling, ) fast_ids, fast_extra = proc.call_with_token_ids( { "prompt_token_ids": tokenizer.encode(prompt, add_special_tokens=False), - "multi_modal_data": {"video": [frames]}, + "multi_modal_data": {"video": [video]}, }, sampling, ) @@ -1850,6 +1872,7 @@ def test_real_checkpoint_video_fast_path_matches_processor_input_ids(num_frames: assert int(grid[0, 0]) == brackets vs_id = tokenizer.convert_tokens_to_ids("]<]start of image[>[") assert fast_ids.count(vs_id) == brackets + assert tokenizer.decode(fast_ids).count(" seconds[>[") == (brackets if timed else 0) assert torch.equal( grid, torch.as_tensor(fast_extra["multimodal_data"]["video"]["video_grid_thw"]).reshape(-1, 3), @@ -2327,6 +2350,64 @@ def test_fast_path_expand_video_one_bracket_per_temporal_patch(): ) +class _CharTokenizer: + """Tokenizer stand-in: one id per character; ``encoded`` records every text.""" + + def __init__(self): + self.encoded = [] + + def encode(self, text, add_special_tokens=True): + assert not add_special_tokens + self.encoded.append(text) + return [ord(c) for c in text] + + +@pytest.mark.cpu_only +def test_fast_path_expand_timed_video_writes_timestamps(): + """A video with timing gets the processor's timestamp before each bracket. + + Three frames sampled at indices 0, 13 and 26 of a 10 fps source pad to + two temporal patches, each stamped with the time of its first frame: 0.0 + and 2.6 seconds. A video without timing in the same request keeps + adjacent brackets and never reaches the tokenizer. The timestamps are + text outside the multimodal mask, so each video still counts 8 tokens and + the timed one splits into one run per bracket. + """ + from tensorrt_llm.inputs.multimodal import _compute_mm_masks, _find_mm_token_runs_from_mask + + proc = _make_minimax_m3_vl_fast_path_processor() + proc._tokenizer = _CharTokenizer() + frames = [object()] * 3 + timed = SimpleNamespace(frames=frames, metadata={"fps": 10.0, "frames_indices": [0, 13, 26]}) + expanded, _ = proc.expand_prompt_token_ids_for_mm( + [100, _VID_TOK, 200, _VID_TOK, 300], + num_mm_tokens_per_placeholder=[8, 8], + mm_data={"video": [timed, frames]}, + ) + + bracket = [_VS_TOK, _VID_TOK, _VID_TOK, _VE_TOK] + stamps = ["]<]0.0 seconds[>[", "]<]2.6 seconds[>["] + assert proc._tokenizer.encoded == stamps + stamp_ids = [[ord(c) for c in stamp] for stamp in stamps] + assert expanded == [ + 100, + *stamp_ids[0], + *bracket, + *stamp_ids[1], + *bracket, + 200, + *bracket, + *bracket, + 300, + ] + mm_mask, _, _ = _compute_mm_masks( + torch.tensor(expanded), None, proc.get_mm_token_ids(), proc.get_mm_special_token_ids() + ) + item_run_cu_offsets, _, run_lengths = _find_mm_token_runs_from_mask(mm_mask, [8, 8]) + assert item_run_cu_offsets == [0, 2, 3] + assert run_lengths == [4, 4, 8] + + @pytest.mark.cpu_only def test_video_token_count_is_one_bracket_per_temporal_patch(): """``grid_t * (grid_h*grid_w/merge^2 + 2)`` for the processor's grid. From c9a01cd775c7cfb0c366d4b9e6b57de4ac957b8f Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:14:35 -0700 Subject: [PATCH 4/5] [None][fix] Treat non-numeric MiniMax-M3 video timing as no timing VideoData validates its metadata container but not the values in it, so a caller can pass fps="10" or a correctly sized frames_indices list that holds None. _video_metadata then raised in math.isfinite or int() instead of taking the no-timing fallback. Require fps to be a real number (not a bool) and every frame index to be an integer before using them, and add the string-fps and None-index cases to test_video_without_timing_passes_no_metadata. The video loader's own metadata (a float fps and a list of int indices) is unaffected. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> --- tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py | 11 ++++++++--- tests/unittest/_torch/models/test_minimax_m3_vl.py | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index edc37c1b2bd4..25e6c54083f2 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -14,6 +14,7 @@ from __future__ import annotations import math +import numbers from dataclasses import dataclass, field from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -2058,8 +2059,8 @@ def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: ``VideoData.metadata`` carries the source ``fps`` and the indices of the sampled frames. With both, the processor writes a ``]<]{t} seconds[>[`` timestamp before each temporal group. A video - without usable timing (no metadata, ``fps`` missing, not finite or - not positive, or not one frame index per delivered frame) gets + without usable timing (no metadata, ``fps`` missing or not a finite + positive number, or not one integer frame index per delivered frame) gets ``fps=None`` and no timestamps. Only the keys transformers' ``VideoMetadata`` accepts are passed, and ``total_num_frames`` is the number of frames delivered, because the media loader has already @@ -2070,11 +2071,15 @@ def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: fps = metadata.get("fps") frames_indices = metadata.get("frames_indices") if ( - fps is None + not isinstance(fps, numbers.Real) + or isinstance(fps, bool) or not math.isfinite(fps) or fps <= 0 or frames_indices is None or len(frames_indices) != num_frames + or not all( + isinstance(i, numbers.Integral) and not isinstance(i, bool) for i in frames_indices + ) ): fps, frames_indices = None, range(num_frames) return { diff --git a/tests/unittest/_torch/models/test_minimax_m3_vl.py b/tests/unittest/_torch/models/test_minimax_m3_vl.py index 8aa0951cd145..f382204a58dd 100644 --- a/tests/unittest/_torch/models/test_minimax_m3_vl.py +++ b/tests/unittest/_torch/models/test_minimax_m3_vl.py @@ -3054,10 +3054,12 @@ def test_video_timing_is_forwarded_to_the_processor(monkeypatch): {"fps": 0, "frames_indices": [0, 1]}, {"fps": float("nan"), "frames_indices": [0, 1]}, {"fps": float("inf"), "frames_indices": [0, 1]}, + {"fps": "10", "frames_indices": [0, 1]}, {"fps": 10.0}, {"fps": 10.0, "frames_indices": []}, {"fps": 10.0, "frames_indices": [0]}, {"fps": 10.0, "frames_indices": [0, 5, 9]}, + {"fps": 10.0, "frames_indices": [0, None]}, ], ids=[ "frame_list", @@ -3065,10 +3067,12 @@ def test_video_timing_is_forwarded_to_the_processor(monkeypatch): "zero_fps", "nan_fps", "inf_fps", + "string_fps", "no_frame_indices", "empty_frame_indices", "fewer_frame_indices_than_frames", "more_frame_indices_than_frames", + "none_frame_index", ], ) def test_video_without_timing_passes_no_metadata(monkeypatch, metadata): From 9dfda613063e0095b28f38496074bbc6bfe35882 Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:26:28 -0700 Subject: [PATCH 5/5] [None][fix] Treat negative MiniMax-M3 video frame indices as no timing _video_metadata accepted frames_indices such as [-10, 1], and both the text path and the token-ID path then wrote a negative timestamp before the first bracket. Require every frame index to be a non-negative integer before using the timing, and add the negative-index case to test_video_without_timing_passes_no_metadata. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> --- tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py | 13 +++++++------ tests/unittest/_torch/models/test_minimax_m3_vl.py | 2 ++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index 25e6c54083f2..6fe45d15ae3e 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -2060,11 +2060,11 @@ def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: the sampled frames. With both, the processor writes a ``]<]{t} seconds[>[`` timestamp before each temporal group. A video without usable timing (no metadata, ``fps`` missing or not a finite - positive number, or not one integer frame index per delivered frame) gets - ``fps=None`` and no timestamps. Only the keys transformers' - ``VideoMetadata`` accepts are passed, and ``total_num_frames`` is the - number of frames delivered, because the media loader has already - sampled them. + positive number, or not one non-negative integer frame index per + delivered frame) gets ``fps=None`` and no timestamps. Only the keys + transformers' ``VideoMetadata`` accepts are passed, and + ``total_num_frames`` is the number of frames delivered, because the + media loader has already sampled them. """ metadata = getattr(video, "metadata", None) metadata = metadata if isinstance(metadata, Mapping) else {} @@ -2078,7 +2078,8 @@ def _video_metadata(video: Any, num_frames: int) -> Dict[str, Any]: or frames_indices is None or len(frames_indices) != num_frames or not all( - isinstance(i, numbers.Integral) and not isinstance(i, bool) for i in frames_indices + isinstance(i, numbers.Integral) and not isinstance(i, bool) and i >= 0 + for i in frames_indices ) ): fps, frames_indices = None, range(num_frames) diff --git a/tests/unittest/_torch/models/test_minimax_m3_vl.py b/tests/unittest/_torch/models/test_minimax_m3_vl.py index f382204a58dd..f9c01cdf215f 100644 --- a/tests/unittest/_torch/models/test_minimax_m3_vl.py +++ b/tests/unittest/_torch/models/test_minimax_m3_vl.py @@ -3060,6 +3060,7 @@ def test_video_timing_is_forwarded_to_the_processor(monkeypatch): {"fps": 10.0, "frames_indices": [0]}, {"fps": 10.0, "frames_indices": [0, 5, 9]}, {"fps": 10.0, "frames_indices": [0, None]}, + {"fps": 10.0, "frames_indices": [-10, 1]}, ], ids=[ "frame_list", @@ -3073,6 +3074,7 @@ def test_video_timing_is_forwarded_to_the_processor(monkeypatch): "fewer_frame_indices_than_frames", "more_frame_indices_than_frames", "none_frame_index", + "negative_frame_index", ], ) def test_video_without_timing_passes_no_metadata(monkeypatch, metadata):