From 4c9830f7d8421b38a245b70a3d88515a94a30735 Mon Sep 17 00:00:00 2001 From: rwagwani Date: Thu, 29 Oct 2026 13:36:20 +0000 Subject: [PATCH] fix(inference): warn when a transfer setting cannot take effect Several transfer settings are accepted without any message in setups where the code never reads them, so users change them and get an unchanged video. Make the code say what the docs already say. The shipped cookbook specs and the Sim-Transfer skill requests build exactly as before, with no new warnings; new L0 tests that run in CI build them all through build_sample. Warnings only (the generated video does not change): - a single-control `weight` (weights are normalised by their sum, and the causal Sim-Transfer path hard-codes 1.0); - on single-view distilled causal Transfer, a table of fields the fixed-step sampler never reads (negative prompt, guidance, sigma_max, shift, normalize_cfg, num_conditional_frames and both guidance windows), a chunk size unlike num_frames, and model_mode other than video2video; and `--sampler` other than unipc on any fixed-step checkpoint; - a reversed guidance_interval / control_guidance_interval (lo >= hi) where that guidance pass would otherwise run; - on single-view Transfer, an explicitly requested fps or num_frames that differs from what the source video gives, checked in generate_transfer_sample (and the streaming path) against the resolved length and frame rate; and prompt_upsampling=True; - both negative_prompt and negative_prompt_file set explicitly. Error: on the bidirectional single-view path, negative or all-zero control weights raise ValueError when the sample is built, instead of a bare assert inside build_transfer_batch. A built OmniSampleArgs now records which fields the request set (a private attribute, not serialized), so generation can tell a request from a default. Pydantic includes private attributes in ==, so test_sample_args compares model_dump() instead. Signed-off-by: rwagwani Co-Authored-By: Claude Opus 5.5 --- cosmos_framework/inference/args.py | 138 +++++++- cosmos_framework/inference/args_test.py | 6 +- cosmos_framework/inference/inference.py | 7 + cosmos_framework/inference/inference_test.py | 36 ++ .../inference/sim_transfer_args_test.py | 314 ++++++++++++++++++ cosmos_framework/inference/transfer.py | 28 ++ .../inference/transfer_shipped_specs_test.py | 165 +++++++++ .../inference/transfer_streaming.py | 3 + 8 files changed, 693 insertions(+), 4 deletions(-) create mode 100644 cosmos_framework/inference/transfer_shipped_specs_test.py diff --git a/cosmos_framework/inference/args.py b/cosmos_framework/inference/args.py index 39002c55c..c5ca071b9 100644 --- a/cosmos_framework/inference/args.py +++ b/cosmos_framework/inference/args.py @@ -140,6 +140,14 @@ def _build_sampling( if self.sigma_max is None: self.sigma_max = 0.0 return + interval = self.guidance_interval + guided = getattr(model_config, "fixed_step_sampler_config", None) is None and self.guidance not in (None, 1.0) + if guided and interval is not None and len(interval) == 2 and interval[0] >= interval[1]: + # The sampler tests lo < t < hi, so a reversed window silently never applies text guidance. + log.warning( + f"'guidance_interval'={list(interval)} is reversed (lo >= hi): text guidance runs only on steps " + "with lo < t < hi, so it never runs. Use [lo, hi] with lo < hi." + ) fixed_sampler = getattr(model_config, "fixed_step_sampler_config", None) if fixed_sampler is not None: schedule = fixed_sampler.t_list @@ -1496,6 +1504,20 @@ def download(self, output_dir: Path): "control_guidance": 1.0, } + # Fields the distilled causal Transfer path accepts but never reads: field -> (neutral value, reason). + # A warning fires only when the sample or CLI sets a field to something other than its neutral value. + _DISTILLED_TRANSFER_IGNORED_FIELDS: ClassVar[dict[str, tuple[Any, str]]] = { + "negative_prompt": ("", "the distilled schedule has no negative-caption branch"), + "negative_prompt_file": (None, "the distilled schedule has no negative-caption branch"), + "guidance": (1.0, "the distilled sampler always runs at guidance 1.0"), + "sigma_max": (80.0, "the distilled sampler uses its fixed timestep list"), + "shift": (5.0, "the distilled sampler's fixed timestep list is already shifted"), + "normalize_cfg": (False, "guidance is fixed at 1.0, so there is no CFG output to normalize"), + "num_conditional_frames": (0, "generation starts from noise with no RGB history prefix"), + "control_guidance_interval": (None, "control guidance is fixed at 1.0, so there is no control-guidance window"), + "guidance_interval": (None, "guidance is fixed at 1.0, so there is no guidance window"), + } + def _build_transfer_data( self, model_config: "OmniMoTModelConfig", @@ -1513,8 +1535,20 @@ def _build_transfer_data( if not hints: return + user_set = user_fields if user_fields is not None else frozenset() if self.negative_prompt is None and self.negative_prompt_file is not None: self.negative_prompt = _load_transfer_negative_prompt_file(self.negative_prompt_file) + elif ( + self.negative_prompt_file is not None + and "negative_prompt" in user_set + and "negative_prompt_file" in user_set + ): + log.warning( + f"[transfer] both 'negative_prompt' and 'negative_prompt_file' are set; " + f"'negative_prompt_file' ({self.negative_prompt_file}) is ignored." + ) + + self._warn_ignored_transfer_fields(model_config, hints, user_set) # A multiview caption run takes its chunk length from the span each caption describes, so # the generic 93-frame transfer chunk must not stand in for it: a 300-frame caption would @@ -1628,6 +1662,97 @@ def _build_transfer_data( if user_fields is None or field not in user_fields: setattr(self, field, value) + # After the hint defaults, so control_guidance is final. Only the bidirectional single-view + # path runs a control-guidance pass, and only when control_guidance is not 1.0. + interval = self.control_guidance_interval + if ( + interval is not None + and len(interval) == 2 + and interval[0] >= interval[1] + and not self.autoregressive + and self.multiview is None + and getattr(model_config, "fixed_step_sampler_config", None) is None + and getattr(self, "control_guidance", None) not in (None, 1.0) + ): + # control_active() tests lo < t < hi, so a reversed window silently never applies control guidance. + log.warning( + f"[transfer] 'control_guidance_interval'={list(interval)} is reversed (lo >= hi): control guidance " + "runs only on steps with lo < t < hi, so it never runs. Use [lo, hi] with lo < hi." + ) + + def _warn_ignored_transfer_fields( + self, + model_config: "OmniMoTModelConfig", + hints: dict, + user_set: frozenset[str], + ) -> None: + """Reject invalid control weights and warn about requested settings that cannot take effect.""" + self = cast("SampleDataOverrides", self) + weights = {key: (1.0 if config.weight is None else config.weight) for key, config in hints.items()} + # Only the bidirectional path passes these weights to build_transfer_batch, which asserts the same + # rule after the model has loaded; the causal path hard-codes a single weight of 1.0. + # Joint camera+LiDAR transfer reads only the wsm weight, unnormalized, so it is left alone here. + reads_weights = not self.autoregressive and TransferHintKey.LIDAR not in hints + if reads_weights and (any(w < 0 for w in weights.values()) or sum(weights.values()) <= 0): + raise ValueError(f"Control weights must be non-negative with a positive sum, got {weights}") + if len(hints) == 1 and self.multiview is None: + ((hint_key, weight),) = weights.items() + if weight != 1.0: + # build_transfer_batch divides each weight by the sum, and the network only takes the + # weighted path for more than one control, so a lone control's weight is never read. + log.warning( + f"[transfer] '{hint_key}.weight'={weight} has no effect with a single control; " + "weights only set the relative mix between multiple controls." + ) + # Single-view Transfer returns through _generate_transfer_batch before the native upsampler, + # but native_prompt_upsampling still skips the duration/resolution text added to plain prompts. + if ( + self.prompt_upsampling is True + and not getattr(self, "prompt_upsampling_applied", None) + and self.multiview is None + ): + log.warning( + "[transfer] 'prompt_upsampling' is ignored by Transfer: the prompt is not upsampled, and a " + "plain-text prompt also loses the duration/resolution text normally added to it. " + "Leave 'prompt_upsampling' off and pass the detailed prompt directly." + ) + + # Mirrors the sampler choice in generate_transfer_sample: a fixed-step (distilled) + # sampler runs at guidance 1.0, so these settings are never read. + if ( + self.autoregressive + and self.multiview is None + and getattr(model_config, "fixed_step_sampler_config", None) is not None + ): + for field, (neutral, reason) in self._DISTILLED_TRANSFER_IGNORED_FIELDS.items(): + value = getattr(self, field) + if field in user_set and value is not None and value != neutral: + log.warning(f"[transfer] '{field}' is ignored by distilled causal Transfer: {reason}.") + # The recipe keeps the chunk equal to num_frames (601 for the long rollout), so a fixed + # neutral value does not fit; flag only a chunk that would imply chunked generation. + chunk = self.num_video_frames_per_chunk + if "num_video_frames_per_chunk" in user_set and chunk is not None and chunk != self.num_frames: + log.warning( + f"[transfer] 'num_video_frames_per_chunk'={chunk} is ignored by distilled causal Transfer: " + "it generates the whole clip one latent frame at a time." + ) + # Any control routes to the transfer path whatever the mode says, and this path has no + # first-frame conditioning, so only video2video describes what runs. Action modes fail on + # their own capability check, and multiview transfer uses text2video/image2video on purpose. + mode = self.model_mode + if ( + "model_mode" in user_set + and self.multiview is None + and mode is not None + and mode != ModelMode.VIDEO2VIDEO + and not mode.is_action + and not mode.is_reasoner + ): + log.warning( + f"[transfer] 'model_mode'='{mode.value}' is ignored by distilled causal Transfer: " + "a control video always runs video-to-video transfer here; use 'video2video'." + ) + @staticmethod def _is_sf_dmd_transfer_model(model_config: "OmniMoTModelConfig") -> bool: """Whether the loaded model is the causal SF-DMD Transfer student whose contract applies.""" @@ -1905,7 +2030,14 @@ class OmniSampleArgs( SamplingArgs, SampleDataArgs, PromptUpsamplingArgs, -): ... +): + # Fields the sample or CLI set explicitly (not filled from defaults), so generation can tell a + # request apart from a default when it reports a setting it had to override. Not serialized. + _requested_fields: frozenset[str] = pydantic.PrivateAttr(default_factory=frozenset) + + @property + def requested_fields(self) -> frozenset[str]: + return self._requested_fields class OmniSampleOverrides( @@ -2160,7 +2292,9 @@ def build_sample(self, *, model_config: Any) -> OmniSampleArgs: # than ``applied is False`` (which would miss case (a)). prompt_upsampling_applied = getattr(self, "prompt_upsampling_applied", None) native_prompt_upsampling = self.prompt_upsampling is True and not prompt_upsampling_applied - return self._build(OmniSampleArgs, native_prompt_upsampling=native_prompt_upsampling) + sample_args = self._build(OmniSampleArgs, native_prompt_upsampling=native_prompt_upsampling) + sample_args._requested_fields = user_fields + return sample_args _MODEL_MEMORY_FACTOR: int = int(1e9) * 2 * 2 # 1B params/tower * 2 bytes/param (bfloat16) * 2 towers diff --git a/cosmos_framework/inference/args_test.py b/cosmos_framework/inference/args_test.py index 58d93c7b5..10fa68404 100644 --- a/cosmos_framework/inference/args_test.py +++ b/cosmos_framework/inference/args_test.py @@ -317,9 +317,11 @@ def test_sample_args(tmp_path: Path): args = overrides.build_sample(model_config=model_dict.config) # Check idempotent - assert overrides.build_sample(model_config=model_dict.config) == args + # Compare the public fields: a built sample also records which fields were requested. + assert overrides.build_sample(model_config=model_dict.config).model_dump() == args.model_dump() overrides_dump = {k: v for k, v in args.model_dump().items() if k in OmniSampleOverrides.model_fields} - assert OmniSampleOverrides.model_validate(overrides_dump).build_sample(model_config=model_dict.config) == args + rebuilt = OmniSampleOverrides.model_validate(overrides_dump).build_sample(model_config=model_dict.config) + assert rebuilt.model_dump() == args.model_dump() text2image_args = OmniSampleOverrides( name="text2image", diff --git a/cosmos_framework/inference/inference.py b/cosmos_framework/inference/inference.py index afa2e0284..a21afece9 100644 --- a/cosmos_framework/inference/inference.py +++ b/cosmos_framework/inference/inference.py @@ -222,6 +222,12 @@ def _configure_vae_compile(model: OmniMoTModel, setup_args: OmniSetupArgs) -> No tokenizer.compile_encode([resolution], output_dir=str(setup_args.output_dir), aspect_ratio=aspect_ratio) +def _warn_ignored_sampler_override(model_config: Any, sampler: str) -> None: + """Warn when ``--sampler`` asks for a scheduler that a fixed-step (distilled) checkpoint never uses.""" + if sampler != "unipc" and getattr(model_config, "fixed_step_sampler_config", None) is not None: + log.warning(f"--sampler {sampler} is ignored: this checkpoint samples with its fixed-step distilled schedule.") + + def _unbatch_transfer_debug_tensor( tensor: torch.Tensor, *, name: str ) -> torch.Tensor: # tensor: [B,C,T,H,W], returns: [C,T,H,W] @@ -1754,6 +1760,7 @@ def _create(cls, setup_args: SetupArgs, **kwargs: Any) -> Self: model.set_up_scheduler_and_sampler() log.debug(f"Sampler overridden to: {sampler_override}") + _warn_ignored_sampler_override(model.config, sampler_override) _configure_vae_compile(model, setup_args) if hasattr(model, "maybe_install_ar_post_saturation_mode"): model.maybe_install_ar_post_saturation_mode() diff --git a/cosmos_framework/inference/inference_test.py b/cosmos_framework/inference/inference_test.py index dbf7498f8..ab40aaf5d 100644 --- a/cosmos_framework/inference/inference_test.py +++ b/cosmos_framework/inference/inference_test.py @@ -1175,3 +1175,39 @@ def test_parallelism_config_sizes_the_vae_sharing_group_to_one_replica( assert config.context_parallel_shard_degree == cp_size assert config.cfg_parallel_shard_degree == cfgp_size assert config.vae_load_balance_group_size == expected_group + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize( + ("fixed_step", "sampler", "warns"), + [(True, "edm", True), (True, "unipc", False), (False, "edm", False), (False, "unipc", False)], +) +def test_loading_warns_when_a_fixed_step_checkpoint_ignores_the_sampler( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fixed_step: bool, sampler: str, warns: bool +) -> None: + from cosmos_framework.inference import inference + from cosmos_framework.inference.args import DEFAULT_CHECKPOINT_NAME, OmniSetupOverrides + from cosmos_framework.inference.common.args import ConfigFileType + + setup_args = OmniSetupOverrides( + checkpoint_path=DEFAULT_CHECKPOINT_NAME, output_dir=tmp_path / "outputs", guardrails=False, sampler=sampler + ).build_setup(world_size=1, local_world_size=1, device_memory_bytes=80 * 1024**3) + setup_args.config_file_type = ConfigFileType.MODULE + model = SimpleNamespace( + config=SimpleNamespace( + rectified_flow_inference_config=SimpleNamespace(scheduler_type=sampler), + fixed_step_sampler_config=SimpleNamespace(t_list=[1.0]) if fixed_step else None, + ), + tokenizer_vision_gen=None, + ) + warnings: list[str] = [] + monkeypatch.setattr(inference, "_download_on_rank0", lambda _download: tmp_path) + monkeypatch.setattr( + inference.Cosmos3OmniModel, "from_pretrained_dcp", Mock(return_value=SimpleNamespace(model=model)) + ) + monkeypatch.setattr(inference.log, "warning", lambda message, *_args, **_kwargs: warnings.append(message)) + + inference.OmniInference._create(setup_args, guardrails=None, _timer=None) + + assert any(f"--sampler {sampler} is ignored" in w for w in warnings) is warns diff --git a/cosmos_framework/inference/sim_transfer_args_test.py b/cosmos_framework/inference/sim_transfer_args_test.py index df6e09ab6..96e33ff9e 100644 --- a/cosmos_framework/inference/sim_transfer_args_test.py +++ b/cosmos_framework/inference/sim_transfer_args_test.py @@ -377,6 +377,199 @@ def test_public_causal_cache_overrides_preserve_exported_config( assert overrides.build_setup(world_size=1, local_world_size=1, device_memory_bytes=_GB200_MEMORY_BYTES) == setup +def _distilled_causal_model_config() -> SimpleNamespace: + return SimpleNamespace( + fixed_step_sampler_config=SimpleNamespace(t_list=[1.0, 0.9375, 0.8333333333333334, 0.625]), + video_temporal_causal=True, + teacher_forcing_frames_per_chunk=1, + ) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("weight,warns", [(None, False), (1.0, False), (0.5, True), (5.0, True)]) +def test_single_control_weight_warns_when_it_cannot_take_effect( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, weight: float | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control), "weight": weight}, autoregressive=True) + sample._build_transfer_data(_distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth"})) + + assert any("'depth.weight'" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_multi_control_weights_do_not_warn(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + warnings = _record_contract_warnings(monkeypatch) + rgb = tmp_path / "rgb.mp4" + rgb.touch() + sample = OmniSampleOverrides(vision_path=str(rgb), edge={"weight": 0.9}, blur={"weight": 0.1}, autoregressive=False) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset({"edge", "blur"})) + + assert not any(".weight'" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("weights", [{"depth": 0.0}, {"depth": -1.0}, {"edge": 0.0, "blur": 0.0}]) +def test_invalid_control_weights_are_rejected(tmp_path: Path, weights: dict[str, float]) -> None: + control = tmp_path / "control.mp4" + control.touch() + sample = OmniSampleOverrides( + **{key: {"control_path": str(control), "weight": w} for key, w in weights.items()}, autoregressive=False + ) + with pytest.raises(ValueError, match="non-negative with a positive sum"): + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset(weights)) + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_negative_prompt_file_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + negative = tmp_path / "negative.json" + negative.write_text(json.dumps({"caption": "blurry"})) + # The skill's defaults file supplies negative_prompt=""; only the sample sets the file. + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=True, + negative_prompt="", + negative_prompt_file=str(negative), + ) + sample._build_transfer_data( + _distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth", "negative_prompt_file"}) + ) + + assert sample.negative_prompt == "" + assert any("'negative_prompt_file' is ignored by distilled causal Transfer" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_explicit_negative_prompt_wins_over_file_with_a_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + negative = tmp_path / "negative.json" + negative.write_text(json.dumps({"caption": "blurry"})) + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=False, + negative_prompt="grainy", + negative_prompt_file=str(negative), + ) + sample._build_transfer_data( + SimpleNamespace(), + sample.sample_meta, + user_fields=frozenset({"depth", "negative_prompt", "negative_prompt_file"}), + ) + + assert sample.negative_prompt == "grainy" + assert any("both 'negative_prompt' and 'negative_prompt_file' are set" in w for w in warnings) + + +@pytest.mark.parametrize( + "field,value,warns", + [ + ("guidance", 1.0, False), + ("guidance", 3.0, True), + ("sigma_max", 80.0, False), + ("sigma_max", 200.0, True), + ("shift", 5.0, False), + ("shift", 10.0, True), + ("normalize_cfg", False, False), + ("normalize_cfg", True, True), + ("num_conditional_frames", 0, False), + ("num_conditional_frames", 2, True), + ], +) +@pytest.mark.L0 +@pytest.mark.CPU +def test_sampler_fields_are_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, value: float | bool, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control)}, autoregressive=True, **{field: value}) + sample._build_transfer_data( + _distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth", field}) + ) + + assert any(f"'{field}' is ignored by distilled causal Transfer" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("upsampling,warns", [(True, True), (False, False), (None, False)]) +def test_prompt_upsampling_is_flagged_as_ignored_by_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, upsampling: bool | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=False, prompt_upsampling=upsampling + ) + model_config = SimpleNamespace(tokenizer=SimpleNamespace(temporal_compression_factor=4)) + sample._build_transfer_data(model_config, sample.sample_meta, user_fields=frozenset({"depth"})) + + assert any("'prompt_upsampling' is ignored by Transfer" in w for w in warnings) is warns + + +@pytest.mark.parametrize( + "mode,mode_set,warns", + [ + ("video2video", True, False), + ("text2video", True, True), + ("image2video", True, True), + ("text2video", False, False), + ], +) +@pytest.mark.L0 +@pytest.mark.CPU +def test_non_video2video_mode_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: str, mode_set: bool, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control)}, autoregressive=True, model_mode=mode) + user_fields = frozenset({"depth", "model_mode"} if mode_set else {"depth"}) + sample._build_transfer_data(_distilled_causal_model_config(), sample.sample_meta, user_fields=user_fields) + + assert any(f"'model_mode'='{mode}' is ignored" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("chunk,warns", [(121, False), (61, True)]) +def test_chunk_size_unlike_num_frames_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, chunk: int, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=True, num_frames=121, num_video_frames_per_chunk=chunk + ) + sample._build_transfer_data( + _distilled_causal_model_config(), + sample.sample_meta, + user_fields=frozenset({"depth", "num_frames", "num_video_frames_per_chunk"}), + ) + + assert any("'num_video_frames_per_chunk'=61 is ignored" in w for w in warnings) is warns + + @pytest.mark.L0 @pytest.mark.CPU @pytest.mark.parametrize("resolution", ["256", "704", "720", "768"]) @@ -419,3 +612,124 @@ def test_bidirectional_480_model_keeps_the_teacher_tier_message(tmp_path: Path) ) with pytest.raises(ValueError, match="teacher's trained 480 and 720 tiers only, got resolution='256'"): sample._build_transfer_data(model_config, sample.sample_meta, user_fields=frozenset()) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval,warns", [(None, False), ([0.0, 1000.0], True)]) +def test_control_guidance_interval_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interval: list[float] | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=True, control_guidance_interval=interval + ) + sample._build_transfer_data( + _distilled_causal_model_config(), + sample.sample_meta, + user_fields=frozenset({"depth", "control_guidance_interval"}), + ) + + assert any("'control_guidance_interval' is ignored" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval,warns", [(None, False), ([0.0, 1000.0], True)]) +def test_guidance_interval_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interval: list[float] | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control)}, autoregressive=True, guidance_interval=interval) + sample._build_transfer_data( + _distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth", "guidance_interval"}) + ) + + assert any("'guidance_interval' is ignored" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_unused_negative_prompt_file_is_not_opened(tmp_path: Path) -> None: + # Specs such as the cookbook's set a relative negative_prompt_file next to a negative_prompt from the + # defaults; the file is only read when it supplies the caption, so a path that does not resolve is fine. + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=False, + negative_prompt="grainy", + negative_prompt_file="../assets/negative_prompt.json", + ) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset({"depth"})) + + assert sample.negative_prompt == "grainy" + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_zero_weight_on_the_causal_path_warns_instead_of_failing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The causal path hard-codes one weight of 1.0, so the value is ignored rather than invalid there. + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control), "weight": 0.0}, autoregressive=True) + sample._build_transfer_data(_distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth"})) + + assert any("'depth.weight'=0.0 has no effect" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval", [[800.0, 200.0], [500.0, 500.0]]) +def test_reversed_control_guidance_interval_warns( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interval: list[float] +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=False, control_guidance_interval=interval + ) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset()) + + assert any("'control_guidance_interval'" in w and "is reversed" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval", [[800.0, 200.0], [500.0, 500.0]]) +def test_reversed_guidance_interval_warns(monkeypatch: pytest.MonkeyPatch, interval: list[float]) -> None: + warnings = _record_contract_warnings(monkeypatch) + sample = OmniSampleOverrides(guidance_interval=interval, num_steps=35, guidance=3.0) + sample._build_sampling(SimpleNamespace(fixed_step_sampler_config=None), sample.sample_meta) + + assert any("'guidance_interval'" in w and "is reversed" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize( + ("control_guidance", "autoregressive", "warns"), [(1.5, False, True), (1.0, False, False), (1.5, True, False)] +) +def test_reversed_control_guidance_interval_warns_only_where_control_guidance_runs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, control_guidance: float, autoregressive: bool, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=autoregressive, + control_guidance=control_guidance, + control_guidance_interval=[800.0, 200.0], + ) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset({"control_guidance"})) + + assert any("is reversed" in w for w in warnings) is warns diff --git a/cosmos_framework/inference/transfer.py b/cosmos_framework/inference/transfer.py index 5d5ca48a8..63c615469 100644 --- a/cosmos_framework/inference/transfer.py +++ b/cosmos_framework/inference/transfer.py @@ -762,6 +762,33 @@ def _load_direct_transfer_inputs( return DirectTransferInputs(input_frames, per_hint_frames, reference_media.fps, geometry) +def _warn_unused_length_and_fps( + sample_args: OmniSampleArgs, total_frames: int, input_fps: float, temporal_compression_factor: int +) -> None: + """Warn when a requested num_frames or fps differs from what this sample actually uses. + + The length comes from the decoded control (capped by max_frames) and the frame rate from the + source video; sample fps is only the fallback for image sources, which report 0 fps. + """ + # Stand-ins and samples rebuilt from JSON carry no request record; stay silent for them. + requested = getattr(sample_args, "requested_fields", None) + if not isinstance(requested, frozenset): + return + if "fps" in requested and input_fps > 0 and abs(sample_args.fps - input_fps) >= 0.5: + log.warning( + f"[transfer] 'fps'={sample_args.fps} has no effect with a video input; the output keeps the input " + f"video's {input_fps:g} fps. 'fps' is used only for image inputs." + ) + # num_frames was rounded up to the latent grid when the sample was built; compare on that grid. + length = math.ceil((total_frames - 1) / temporal_compression_factor) * temporal_compression_factor + 1 + if "num_frames" in requested and total_frames > 1 and sample_args.num_frames != length: + log.warning( + f"[transfer] 'num_frames'={sample_args.num_frames} does not set the Transfer output length; the " + f"length comes from the control video ({total_frames} frames after the 'max_frames' cap). " + "Set 'max_frames' to change it." + ) + + def generate_transfer_sample( sample_args: OmniSampleArgs, model: OmniMoTModel, @@ -847,6 +874,7 @@ def generate_transfer_sample( total_frames = first_frames.shape[1] temporal_compression_factor = model.config.tokenizer.temporal_compression_factor + _warn_unused_length_and_fps(sample_args, total_frames, input_fps, temporal_compression_factor) chunk_frames = 1 if total_frames == 1 else num_video_frames_per_chunk chunk_frames = math.ceil((chunk_frames - 1) / temporal_compression_factor) * temporal_compression_factor + 1 num_chunks, stride = _get_num_chunks(total_frames, chunk_frames, num_conditional_frames) diff --git a/cosmos_framework/inference/transfer_shipped_specs_test.py b/cosmos_framework/inference/transfer_shipped_specs_test.py new file mode 100644 index 000000000..f9e6d1cde --- /dev/null +++ b/cosmos_framework/inference/transfer_shipped_specs_test.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 + +"""Shipped Transfer specs and requests must build without errors or ignored-setting warnings.""" + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from cosmos_framework.inference import args as args_module +from cosmos_framework.inference import transfer +from cosmos_framework.inference.args import OmniSampleOverrides + +pytestmark = [pytest.mark.L0, pytest.mark.CPU] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_COOKBOOK_SPECS = _REPO_ROOT / ".agents/skills/cookbooks/cosmos3/generator/transfer/specs" +_SIM_TRANSFER_SKILL = _REPO_ROOT / ".agents/skills/cosmos3-nano-sim-transfer" + + +def _nano_model_config() -> SimpleNamespace: + return SimpleNamespace( + action_gen=False, + sound_gen=False, + resolution="720", + video_temporal_causal=False, + fixed_step_sampler_config=None, + tokenizer=SimpleNamespace(temporal_compression_factor=4), + vlm_config=SimpleNamespace(model_name="Qwen/Qwen3-VL-8B-Instruct"), + rectified_flow_inference_config=SimpleNamespace(scheduler_type="unipc"), + ) + + +def _sim_transfer_model_config() -> SimpleNamespace: + return SimpleNamespace( + action_gen=False, + sound_gen=False, + resolution="480", + video_temporal_causal=True, + causal_training_strategy="none", + simulation_mode=None, + fixed_step_sampler_config=SimpleNamespace(t_list=[1.0, 0.9375, 0.8333333333333334, 0.625]), + teacher_forcing_frames_per_chunk=1, + teacher_forcing_replay_policy=SimpleNamespace( + control_visibility="causal", controls_read_strict_past_clean_rgb=True, clean_pass_causality="frame" + ), + tokenizer=SimpleNamespace(temporal_compression_factor=4), + vlm_config=SimpleNamespace(model_name="Qwen/Qwen3-VL-8B-Instruct"), + rectified_flow_inference_config=SimpleNamespace(scheduler_type="unipc"), + ) + + +def _record_warnings(monkeypatch: pytest.MonkeyPatch) -> list[str]: + warnings: list[str] = [] + monkeypatch.setattr(args_module.log, "warning", lambda message, *_a, **_k: warnings.append(message)) + return warnings + + +@pytest.mark.parametrize("spec", sorted(_COOKBOOK_SPECS.glob("*.json")), ids=lambda path: path.stem) +def test_cookbook_transfer_spec_builds_without_ignored_setting_warnings( + spec: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (overrides,) = OmniSampleOverrides.from_files([spec]) + if str(overrides.vision_path).startswith("https://"): + # Keep the test offline; only the request fields matter for building the sample. + stand_in = tmp_path / "vision.mp4" + stand_in.touch() + overrides.vision_path = str(stand_in) + warnings = _record_warnings(monkeypatch) + # The cookbook runs from the repository root, so the spec's relative negative_prompt_file does not + # resolve; it must not be needed, because the defaults already supply the negative caption. + monkeypatch.chdir(tmp_path) + overrides.output_dir = tmp_path / "out" + overrides.download(overrides.output_dir / "inputs") + + sample = overrides.build_sample(model_config=_nano_model_config()) + + assert not [w for w in warnings if w.startswith("[transfer]")] + assert {"fps", "num_frames"} <= sample.requested_fields + + +def _sim_transfer_requests(tmp_path: Path) -> list[Path]: + """Write the skill's official requests with prepare_hf_sample.py, on empty stand-in media.""" + module_spec = importlib.util.spec_from_file_location( + "prepare_hf_sample", _SIM_TRANSFER_SKILL / "scripts/prepare_hf_sample.py" + ) + assert module_spec is not None and module_spec.loader is not None + prepare_hf_sample = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(prepare_hf_sample) + requests: list[Path] = [] + for example in ("example1", "example2"): + sample_dir = tmp_path / "sample" / example + sample_dir.mkdir(parents=True) + for suffix in (".mp4", "_depth.mp4", "_segmentation.mp4"): + (sample_dir / f"{example}{suffix}").touch() + prompt = sample_dir / f"{example}_upsampled_prompt.json" + prompt.write_text(json.dumps({"prompt": "a robot"}) if example == "example1" else "a street") + prepare_hf_sample.prepare(example, sample_dir, tmp_path / example) + requests += sorted((tmp_path / example / "requests").glob("*.json")) + return requests + + +def test_sim_transfer_skill_requests_build_without_ignored_setting_warnings( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + requests = _sim_transfer_requests(tmp_path) + assert len(requests) == 6 + # The skill's command: its defaults file plus these CLI flags. + cli = OmniSampleOverrides( + defaults_file=_SIM_TRANSFER_SKILL / "inference/assets/sample_args.json", + autoregressive=True, + seed=2025, + num_steps=4, + guidance=1.0, + control_guidance=1.0, + prompt_upsampling=False, + ) + warnings = _record_warnings(monkeypatch) + for request in requests: + for overrides in OmniSampleOverrides.from_files([request], overrides=cli): + overrides.output_dir = tmp_path / "out" / request.stem + overrides.download(overrides.output_dir / "inputs") + overrides.build_sample(model_config=_sim_transfer_model_config()) + + assert not [w for w in warnings if w.startswith("[transfer]")] + + +@pytest.mark.parametrize( + ("requested", "total_frames", "input_fps", "expected"), + [ + ({"fps": 30, "num_frames": 121}, 121, 30.0, []), + ({"fps": 30, "num_frames": 121}, 121, 29.97, []), + ({"fps": 10}, 121, 30.0, ["'fps'=10 has no effect with a video input"]), + ({"fps": 10}, 1, 0.0, []), + ({"num_frames": 61}, 121, 30.0, ["'num_frames'=61 does not set the Transfer output length"]), + ({"num_frames": 601}, 600, 30.0, []), + ({}, 93, 16.0, []), + ], +) +def test_requested_fps_and_num_frames_warn_only_when_the_source_disagrees( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + requested: dict, + total_frames: int, + input_fps: float, + expected: list[str], +) -> None: + control = tmp_path / "depth.mp4" + control.touch() + overrides = OmniSampleOverrides( + name="sample", depth={"control_path": str(control)}, output_dir=tmp_path, **requested + ) + overrides.download(tmp_path / "inputs") + sample = overrides.build_sample(model_config=_nano_model_config()) + warnings: list[str] = [] + monkeypatch.setattr(transfer.log, "warning", lambda message, *_a, **_k: warnings.append(message)) + + transfer._warn_unused_length_and_fps(sample, total_frames, input_fps, temporal_compression_factor=4) + + assert len(warnings) == len(expected) + for message, text in zip(warnings, expected, strict=True): + assert text in message diff --git a/cosmos_framework/inference/transfer_streaming.py b/cosmos_framework/inference/transfer_streaming.py index 104afa9a1..d1de5326e 100644 --- a/cosmos_framework/inference/transfer_streaming.py +++ b/cosmos_framework/inference/transfer_streaming.py @@ -19,6 +19,7 @@ _latent_frames_for_pixel_frames, _require_causal_transfer_pyav, _resolve_transfer_seed, + _warn_unused_length_and_fps, load_transfer_control_frames, ) from cosmos_framework.inference.vision import ( @@ -199,6 +200,8 @@ def _prepare_streaming_transfer_input( height = media.height width = media.width temporal_compression_factor = int(model.config.tokenizer.temporal_compression_factor) + # Same check as the non-streaming path; an image source reports its sample fps, so it never warns. + _warn_unused_length_and_fps(sample_args, total_frames, media.output_fps, temporal_compression_factor) chunk_size = int(getattr(model.config, "teacher_forcing_frames_per_chunk", 1) or 1) if input_frames is not None: input_frames = pad_temporal_frames(input_frames, total_frames) # [3,T,H,W]