diff --git a/cosmos_framework/inference/args.py b/cosmos_framework/inference/args.py index 39002c55..c5ca071b 100644 --- a/cosmos_framework/inference/args.py +++ b/cosmos_framework/inference/args.py @@ -140,6 +140,14 @@ def _build_sampling( if self.sigma_max is None: self.sigma_max = 0.0 return + interval = self.guidance_interval + guided = getattr(model_config, "fixed_step_sampler_config", None) is None and self.guidance not in (None, 1.0) + if guided and interval is not None and len(interval) == 2 and interval[0] >= interval[1]: + # The sampler tests lo < t < hi, so a reversed window silently never applies text guidance. + log.warning( + f"'guidance_interval'={list(interval)} is reversed (lo >= hi): text guidance runs only on steps " + "with lo < t < hi, so it never runs. Use [lo, hi] with lo < hi." + ) fixed_sampler = getattr(model_config, "fixed_step_sampler_config", None) if fixed_sampler is not None: schedule = fixed_sampler.t_list @@ -1496,6 +1504,20 @@ def download(self, output_dir: Path): "control_guidance": 1.0, } + # Fields the distilled causal Transfer path accepts but never reads: field -> (neutral value, reason). + # A warning fires only when the sample or CLI sets a field to something other than its neutral value. + _DISTILLED_TRANSFER_IGNORED_FIELDS: ClassVar[dict[str, tuple[Any, str]]] = { + "negative_prompt": ("", "the distilled schedule has no negative-caption branch"), + "negative_prompt_file": (None, "the distilled schedule has no negative-caption branch"), + "guidance": (1.0, "the distilled sampler always runs at guidance 1.0"), + "sigma_max": (80.0, "the distilled sampler uses its fixed timestep list"), + "shift": (5.0, "the distilled sampler's fixed timestep list is already shifted"), + "normalize_cfg": (False, "guidance is fixed at 1.0, so there is no CFG output to normalize"), + "num_conditional_frames": (0, "generation starts from noise with no RGB history prefix"), + "control_guidance_interval": (None, "control guidance is fixed at 1.0, so there is no control-guidance window"), + "guidance_interval": (None, "guidance is fixed at 1.0, so there is no guidance window"), + } + def _build_transfer_data( self, model_config: "OmniMoTModelConfig", @@ -1513,8 +1535,20 @@ def _build_transfer_data( if not hints: return + user_set = user_fields if user_fields is not None else frozenset() if self.negative_prompt is None and self.negative_prompt_file is not None: self.negative_prompt = _load_transfer_negative_prompt_file(self.negative_prompt_file) + elif ( + self.negative_prompt_file is not None + and "negative_prompt" in user_set + and "negative_prompt_file" in user_set + ): + log.warning( + f"[transfer] both 'negative_prompt' and 'negative_prompt_file' are set; " + f"'negative_prompt_file' ({self.negative_prompt_file}) is ignored." + ) + + self._warn_ignored_transfer_fields(model_config, hints, user_set) # A multiview caption run takes its chunk length from the span each caption describes, so # the generic 93-frame transfer chunk must not stand in for it: a 300-frame caption would @@ -1628,6 +1662,97 @@ def _build_transfer_data( if user_fields is None or field not in user_fields: setattr(self, field, value) + # After the hint defaults, so control_guidance is final. Only the bidirectional single-view + # path runs a control-guidance pass, and only when control_guidance is not 1.0. + interval = self.control_guidance_interval + if ( + interval is not None + and len(interval) == 2 + and interval[0] >= interval[1] + and not self.autoregressive + and self.multiview is None + and getattr(model_config, "fixed_step_sampler_config", None) is None + and getattr(self, "control_guidance", None) not in (None, 1.0) + ): + # control_active() tests lo < t < hi, so a reversed window silently never applies control guidance. + log.warning( + f"[transfer] 'control_guidance_interval'={list(interval)} is reversed (lo >= hi): control guidance " + "runs only on steps with lo < t < hi, so it never runs. Use [lo, hi] with lo < hi." + ) + + def _warn_ignored_transfer_fields( + self, + model_config: "OmniMoTModelConfig", + hints: dict, + user_set: frozenset[str], + ) -> None: + """Reject invalid control weights and warn about requested settings that cannot take effect.""" + self = cast("SampleDataOverrides", self) + weights = {key: (1.0 if config.weight is None else config.weight) for key, config in hints.items()} + # Only the bidirectional path passes these weights to build_transfer_batch, which asserts the same + # rule after the model has loaded; the causal path hard-codes a single weight of 1.0. + # Joint camera+LiDAR transfer reads only the wsm weight, unnormalized, so it is left alone here. + reads_weights = not self.autoregressive and TransferHintKey.LIDAR not in hints + if reads_weights and (any(w < 0 for w in weights.values()) or sum(weights.values()) <= 0): + raise ValueError(f"Control weights must be non-negative with a positive sum, got {weights}") + if len(hints) == 1 and self.multiview is None: + ((hint_key, weight),) = weights.items() + if weight != 1.0: + # build_transfer_batch divides each weight by the sum, and the network only takes the + # weighted path for more than one control, so a lone control's weight is never read. + log.warning( + f"[transfer] '{hint_key}.weight'={weight} has no effect with a single control; " + "weights only set the relative mix between multiple controls." + ) + # Single-view Transfer returns through _generate_transfer_batch before the native upsampler, + # but native_prompt_upsampling still skips the duration/resolution text added to plain prompts. + if ( + self.prompt_upsampling is True + and not getattr(self, "prompt_upsampling_applied", None) + and self.multiview is None + ): + log.warning( + "[transfer] 'prompt_upsampling' is ignored by Transfer: the prompt is not upsampled, and a " + "plain-text prompt also loses the duration/resolution text normally added to it. " + "Leave 'prompt_upsampling' off and pass the detailed prompt directly." + ) + + # Mirrors the sampler choice in generate_transfer_sample: a fixed-step (distilled) + # sampler runs at guidance 1.0, so these settings are never read. + if ( + self.autoregressive + and self.multiview is None + and getattr(model_config, "fixed_step_sampler_config", None) is not None + ): + for field, (neutral, reason) in self._DISTILLED_TRANSFER_IGNORED_FIELDS.items(): + value = getattr(self, field) + if field in user_set and value is not None and value != neutral: + log.warning(f"[transfer] '{field}' is ignored by distilled causal Transfer: {reason}.") + # The recipe keeps the chunk equal to num_frames (601 for the long rollout), so a fixed + # neutral value does not fit; flag only a chunk that would imply chunked generation. + chunk = self.num_video_frames_per_chunk + if "num_video_frames_per_chunk" in user_set and chunk is not None and chunk != self.num_frames: + log.warning( + f"[transfer] 'num_video_frames_per_chunk'={chunk} is ignored by distilled causal Transfer: " + "it generates the whole clip one latent frame at a time." + ) + # Any control routes to the transfer path whatever the mode says, and this path has no + # first-frame conditioning, so only video2video describes what runs. Action modes fail on + # their own capability check, and multiview transfer uses text2video/image2video on purpose. + mode = self.model_mode + if ( + "model_mode" in user_set + and self.multiview is None + and mode is not None + and mode != ModelMode.VIDEO2VIDEO + and not mode.is_action + and not mode.is_reasoner + ): + log.warning( + f"[transfer] 'model_mode'='{mode.value}' is ignored by distilled causal Transfer: " + "a control video always runs video-to-video transfer here; use 'video2video'." + ) + @staticmethod def _is_sf_dmd_transfer_model(model_config: "OmniMoTModelConfig") -> bool: """Whether the loaded model is the causal SF-DMD Transfer student whose contract applies.""" @@ -1905,7 +2030,14 @@ class OmniSampleArgs( SamplingArgs, SampleDataArgs, PromptUpsamplingArgs, -): ... +): + # Fields the sample or CLI set explicitly (not filled from defaults), so generation can tell a + # request apart from a default when it reports a setting it had to override. Not serialized. + _requested_fields: frozenset[str] = pydantic.PrivateAttr(default_factory=frozenset) + + @property + def requested_fields(self) -> frozenset[str]: + return self._requested_fields class OmniSampleOverrides( @@ -2160,7 +2292,9 @@ def build_sample(self, *, model_config: Any) -> OmniSampleArgs: # than ``applied is False`` (which would miss case (a)). prompt_upsampling_applied = getattr(self, "prompt_upsampling_applied", None) native_prompt_upsampling = self.prompt_upsampling is True and not prompt_upsampling_applied - return self._build(OmniSampleArgs, native_prompt_upsampling=native_prompt_upsampling) + sample_args = self._build(OmniSampleArgs, native_prompt_upsampling=native_prompt_upsampling) + sample_args._requested_fields = user_fields + return sample_args _MODEL_MEMORY_FACTOR: int = int(1e9) * 2 * 2 # 1B params/tower * 2 bytes/param (bfloat16) * 2 towers diff --git a/cosmos_framework/inference/args_test.py b/cosmos_framework/inference/args_test.py index 58d93c7b..10fa6840 100644 --- a/cosmos_framework/inference/args_test.py +++ b/cosmos_framework/inference/args_test.py @@ -317,9 +317,11 @@ def test_sample_args(tmp_path: Path): args = overrides.build_sample(model_config=model_dict.config) # Check idempotent - assert overrides.build_sample(model_config=model_dict.config) == args + # Compare the public fields: a built sample also records which fields were requested. + assert overrides.build_sample(model_config=model_dict.config).model_dump() == args.model_dump() overrides_dump = {k: v for k, v in args.model_dump().items() if k in OmniSampleOverrides.model_fields} - assert OmniSampleOverrides.model_validate(overrides_dump).build_sample(model_config=model_dict.config) == args + rebuilt = OmniSampleOverrides.model_validate(overrides_dump).build_sample(model_config=model_dict.config) + assert rebuilt.model_dump() == args.model_dump() text2image_args = OmniSampleOverrides( name="text2image", diff --git a/cosmos_framework/inference/inference.py b/cosmos_framework/inference/inference.py index afa2e028..a21afece 100644 --- a/cosmos_framework/inference/inference.py +++ b/cosmos_framework/inference/inference.py @@ -222,6 +222,12 @@ def _configure_vae_compile(model: OmniMoTModel, setup_args: OmniSetupArgs) -> No tokenizer.compile_encode([resolution], output_dir=str(setup_args.output_dir), aspect_ratio=aspect_ratio) +def _warn_ignored_sampler_override(model_config: Any, sampler: str) -> None: + """Warn when ``--sampler`` asks for a scheduler that a fixed-step (distilled) checkpoint never uses.""" + if sampler != "unipc" and getattr(model_config, "fixed_step_sampler_config", None) is not None: + log.warning(f"--sampler {sampler} is ignored: this checkpoint samples with its fixed-step distilled schedule.") + + def _unbatch_transfer_debug_tensor( tensor: torch.Tensor, *, name: str ) -> torch.Tensor: # tensor: [B,C,T,H,W], returns: [C,T,H,W] @@ -1754,6 +1760,7 @@ def _create(cls, setup_args: SetupArgs, **kwargs: Any) -> Self: model.set_up_scheduler_and_sampler() log.debug(f"Sampler overridden to: {sampler_override}") + _warn_ignored_sampler_override(model.config, sampler_override) _configure_vae_compile(model, setup_args) if hasattr(model, "maybe_install_ar_post_saturation_mode"): model.maybe_install_ar_post_saturation_mode() diff --git a/cosmos_framework/inference/inference_test.py b/cosmos_framework/inference/inference_test.py index dbf7498f..ab40aaf5 100644 --- a/cosmos_framework/inference/inference_test.py +++ b/cosmos_framework/inference/inference_test.py @@ -1175,3 +1175,39 @@ def test_parallelism_config_sizes_the_vae_sharing_group_to_one_replica( assert config.context_parallel_shard_degree == cp_size assert config.cfg_parallel_shard_degree == cfgp_size assert config.vae_load_balance_group_size == expected_group + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize( + ("fixed_step", "sampler", "warns"), + [(True, "edm", True), (True, "unipc", False), (False, "edm", False), (False, "unipc", False)], +) +def test_loading_warns_when_a_fixed_step_checkpoint_ignores_the_sampler( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fixed_step: bool, sampler: str, warns: bool +) -> None: + from cosmos_framework.inference import inference + from cosmos_framework.inference.args import DEFAULT_CHECKPOINT_NAME, OmniSetupOverrides + from cosmos_framework.inference.common.args import ConfigFileType + + setup_args = OmniSetupOverrides( + checkpoint_path=DEFAULT_CHECKPOINT_NAME, output_dir=tmp_path / "outputs", guardrails=False, sampler=sampler + ).build_setup(world_size=1, local_world_size=1, device_memory_bytes=80 * 1024**3) + setup_args.config_file_type = ConfigFileType.MODULE + model = SimpleNamespace( + config=SimpleNamespace( + rectified_flow_inference_config=SimpleNamespace(scheduler_type=sampler), + fixed_step_sampler_config=SimpleNamespace(t_list=[1.0]) if fixed_step else None, + ), + tokenizer_vision_gen=None, + ) + warnings: list[str] = [] + monkeypatch.setattr(inference, "_download_on_rank0", lambda _download: tmp_path) + monkeypatch.setattr( + inference.Cosmos3OmniModel, "from_pretrained_dcp", Mock(return_value=SimpleNamespace(model=model)) + ) + monkeypatch.setattr(inference.log, "warning", lambda message, *_args, **_kwargs: warnings.append(message)) + + inference.OmniInference._create(setup_args, guardrails=None, _timer=None) + + assert any(f"--sampler {sampler} is ignored" in w for w in warnings) is warns diff --git a/cosmos_framework/inference/sim_transfer_args_test.py b/cosmos_framework/inference/sim_transfer_args_test.py index df6e09ab..96e33ff9 100644 --- a/cosmos_framework/inference/sim_transfer_args_test.py +++ b/cosmos_framework/inference/sim_transfer_args_test.py @@ -377,6 +377,199 @@ def test_public_causal_cache_overrides_preserve_exported_config( assert overrides.build_setup(world_size=1, local_world_size=1, device_memory_bytes=_GB200_MEMORY_BYTES) == setup +def _distilled_causal_model_config() -> SimpleNamespace: + return SimpleNamespace( + fixed_step_sampler_config=SimpleNamespace(t_list=[1.0, 0.9375, 0.8333333333333334, 0.625]), + video_temporal_causal=True, + teacher_forcing_frames_per_chunk=1, + ) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("weight,warns", [(None, False), (1.0, False), (0.5, True), (5.0, True)]) +def test_single_control_weight_warns_when_it_cannot_take_effect( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, weight: float | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control), "weight": weight}, autoregressive=True) + sample._build_transfer_data(_distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth"})) + + assert any("'depth.weight'" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_multi_control_weights_do_not_warn(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + warnings = _record_contract_warnings(monkeypatch) + rgb = tmp_path / "rgb.mp4" + rgb.touch() + sample = OmniSampleOverrides(vision_path=str(rgb), edge={"weight": 0.9}, blur={"weight": 0.1}, autoregressive=False) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset({"edge", "blur"})) + + assert not any(".weight'" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("weights", [{"depth": 0.0}, {"depth": -1.0}, {"edge": 0.0, "blur": 0.0}]) +def test_invalid_control_weights_are_rejected(tmp_path: Path, weights: dict[str, float]) -> None: + control = tmp_path / "control.mp4" + control.touch() + sample = OmniSampleOverrides( + **{key: {"control_path": str(control), "weight": w} for key, w in weights.items()}, autoregressive=False + ) + with pytest.raises(ValueError, match="non-negative with a positive sum"): + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset(weights)) + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_negative_prompt_file_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + negative = tmp_path / "negative.json" + negative.write_text(json.dumps({"caption": "blurry"})) + # The skill's defaults file supplies negative_prompt=""; only the sample sets the file. + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=True, + negative_prompt="", + negative_prompt_file=str(negative), + ) + sample._build_transfer_data( + _distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth", "negative_prompt_file"}) + ) + + assert sample.negative_prompt == "" + assert any("'negative_prompt_file' is ignored by distilled causal Transfer" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_explicit_negative_prompt_wins_over_file_with_a_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + negative = tmp_path / "negative.json" + negative.write_text(json.dumps({"caption": "blurry"})) + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=False, + negative_prompt="grainy", + negative_prompt_file=str(negative), + ) + sample._build_transfer_data( + SimpleNamespace(), + sample.sample_meta, + user_fields=frozenset({"depth", "negative_prompt", "negative_prompt_file"}), + ) + + assert sample.negative_prompt == "grainy" + assert any("both 'negative_prompt' and 'negative_prompt_file' are set" in w for w in warnings) + + +@pytest.mark.parametrize( + "field,value,warns", + [ + ("guidance", 1.0, False), + ("guidance", 3.0, True), + ("sigma_max", 80.0, False), + ("sigma_max", 200.0, True), + ("shift", 5.0, False), + ("shift", 10.0, True), + ("normalize_cfg", False, False), + ("normalize_cfg", True, True), + ("num_conditional_frames", 0, False), + ("num_conditional_frames", 2, True), + ], +) +@pytest.mark.L0 +@pytest.mark.CPU +def test_sampler_fields_are_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, value: float | bool, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control)}, autoregressive=True, **{field: value}) + sample._build_transfer_data( + _distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth", field}) + ) + + assert any(f"'{field}' is ignored by distilled causal Transfer" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("upsampling,warns", [(True, True), (False, False), (None, False)]) +def test_prompt_upsampling_is_flagged_as_ignored_by_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, upsampling: bool | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=False, prompt_upsampling=upsampling + ) + model_config = SimpleNamespace(tokenizer=SimpleNamespace(temporal_compression_factor=4)) + sample._build_transfer_data(model_config, sample.sample_meta, user_fields=frozenset({"depth"})) + + assert any("'prompt_upsampling' is ignored by Transfer" in w for w in warnings) is warns + + +@pytest.mark.parametrize( + "mode,mode_set,warns", + [ + ("video2video", True, False), + ("text2video", True, True), + ("image2video", True, True), + ("text2video", False, False), + ], +) +@pytest.mark.L0 +@pytest.mark.CPU +def test_non_video2video_mode_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: str, mode_set: bool, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control)}, autoregressive=True, model_mode=mode) + user_fields = frozenset({"depth", "model_mode"} if mode_set else {"depth"}) + sample._build_transfer_data(_distilled_causal_model_config(), sample.sample_meta, user_fields=user_fields) + + assert any(f"'model_mode'='{mode}' is ignored" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("chunk,warns", [(121, False), (61, True)]) +def test_chunk_size_unlike_num_frames_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, chunk: int, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=True, num_frames=121, num_video_frames_per_chunk=chunk + ) + sample._build_transfer_data( + _distilled_causal_model_config(), + sample.sample_meta, + user_fields=frozenset({"depth", "num_frames", "num_video_frames_per_chunk"}), + ) + + assert any("'num_video_frames_per_chunk'=61 is ignored" in w for w in warnings) is warns + + @pytest.mark.L0 @pytest.mark.CPU @pytest.mark.parametrize("resolution", ["256", "704", "720", "768"]) @@ -419,3 +612,124 @@ def test_bidirectional_480_model_keeps_the_teacher_tier_message(tmp_path: Path) ) with pytest.raises(ValueError, match="teacher's trained 480 and 720 tiers only, got resolution='256'"): sample._build_transfer_data(model_config, sample.sample_meta, user_fields=frozenset()) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval,warns", [(None, False), ([0.0, 1000.0], True)]) +def test_control_guidance_interval_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interval: list[float] | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=True, control_guidance_interval=interval + ) + sample._build_transfer_data( + _distilled_causal_model_config(), + sample.sample_meta, + user_fields=frozenset({"depth", "control_guidance_interval"}), + ) + + assert any("'control_guidance_interval' is ignored" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval,warns", [(None, False), ([0.0, 1000.0], True)]) +def test_guidance_interval_is_flagged_as_ignored_by_distilled_transfer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interval: list[float] | None, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control)}, autoregressive=True, guidance_interval=interval) + sample._build_transfer_data( + _distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth", "guidance_interval"}) + ) + + assert any("'guidance_interval' is ignored" in w for w in warnings) is warns + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_unused_negative_prompt_file_is_not_opened(tmp_path: Path) -> None: + # Specs such as the cookbook's set a relative negative_prompt_file next to a negative_prompt from the + # defaults; the file is only read when it supplies the caption, so a path that does not resolve is fine. + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=False, + negative_prompt="grainy", + negative_prompt_file="../assets/negative_prompt.json", + ) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset({"depth"})) + + assert sample.negative_prompt == "grainy" + + +@pytest.mark.L0 +@pytest.mark.CPU +def test_zero_weight_on_the_causal_path_warns_instead_of_failing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The causal path hard-codes one weight of 1.0, so the value is ignored rather than invalid there. + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides(depth={"control_path": str(control), "weight": 0.0}, autoregressive=True) + sample._build_transfer_data(_distilled_causal_model_config(), sample.sample_meta, user_fields=frozenset({"depth"})) + + assert any("'depth.weight'=0.0 has no effect" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval", [[800.0, 200.0], [500.0, 500.0]]) +def test_reversed_control_guidance_interval_warns( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interval: list[float] +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, autoregressive=False, control_guidance_interval=interval + ) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset()) + + assert any("'control_guidance_interval'" in w and "is reversed" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize("interval", [[800.0, 200.0], [500.0, 500.0]]) +def test_reversed_guidance_interval_warns(monkeypatch: pytest.MonkeyPatch, interval: list[float]) -> None: + warnings = _record_contract_warnings(monkeypatch) + sample = OmniSampleOverrides(guidance_interval=interval, num_steps=35, guidance=3.0) + sample._build_sampling(SimpleNamespace(fixed_step_sampler_config=None), sample.sample_meta) + + assert any("'guidance_interval'" in w and "is reversed" in w for w in warnings) + + +@pytest.mark.L0 +@pytest.mark.CPU +@pytest.mark.parametrize( + ("control_guidance", "autoregressive", "warns"), [(1.5, False, True), (1.0, False, False), (1.5, True, False)] +) +def test_reversed_control_guidance_interval_warns_only_where_control_guidance_runs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, control_guidance: float, autoregressive: bool, warns: bool +) -> None: + warnings = _record_contract_warnings(monkeypatch) + control = tmp_path / "depth.mp4" + control.touch() + sample = OmniSampleOverrides( + depth={"control_path": str(control)}, + autoregressive=autoregressive, + control_guidance=control_guidance, + control_guidance_interval=[800.0, 200.0], + ) + sample._build_transfer_data(SimpleNamespace(), sample.sample_meta, user_fields=frozenset({"control_guidance"})) + + assert any("is reversed" in w for w in warnings) is warns diff --git a/cosmos_framework/inference/transfer.py b/cosmos_framework/inference/transfer.py index 5d5ca48a..63c61546 100644 --- a/cosmos_framework/inference/transfer.py +++ b/cosmos_framework/inference/transfer.py @@ -762,6 +762,33 @@ def _load_direct_transfer_inputs( return DirectTransferInputs(input_frames, per_hint_frames, reference_media.fps, geometry) +def _warn_unused_length_and_fps( + sample_args: OmniSampleArgs, total_frames: int, input_fps: float, temporal_compression_factor: int +) -> None: + """Warn when a requested num_frames or fps differs from what this sample actually uses. + + The length comes from the decoded control (capped by max_frames) and the frame rate from the + source video; sample fps is only the fallback for image sources, which report 0 fps. + """ + # Stand-ins and samples rebuilt from JSON carry no request record; stay silent for them. + requested = getattr(sample_args, "requested_fields", None) + if not isinstance(requested, frozenset): + return + if "fps" in requested and input_fps > 0 and abs(sample_args.fps - input_fps) >= 0.5: + log.warning( + f"[transfer] 'fps'={sample_args.fps} has no effect with a video input; the output keeps the input " + f"video's {input_fps:g} fps. 'fps' is used only for image inputs." + ) + # num_frames was rounded up to the latent grid when the sample was built; compare on that grid. + length = math.ceil((total_frames - 1) / temporal_compression_factor) * temporal_compression_factor + 1 + if "num_frames" in requested and total_frames > 1 and sample_args.num_frames != length: + log.warning( + f"[transfer] 'num_frames'={sample_args.num_frames} does not set the Transfer output length; the " + f"length comes from the control video ({total_frames} frames after the 'max_frames' cap). " + "Set 'max_frames' to change it." + ) + + def generate_transfer_sample( sample_args: OmniSampleArgs, model: OmniMoTModel, @@ -847,6 +874,7 @@ def generate_transfer_sample( total_frames = first_frames.shape[1] temporal_compression_factor = model.config.tokenizer.temporal_compression_factor + _warn_unused_length_and_fps(sample_args, total_frames, input_fps, temporal_compression_factor) chunk_frames = 1 if total_frames == 1 else num_video_frames_per_chunk chunk_frames = math.ceil((chunk_frames - 1) / temporal_compression_factor) * temporal_compression_factor + 1 num_chunks, stride = _get_num_chunks(total_frames, chunk_frames, num_conditional_frames) diff --git a/cosmos_framework/inference/transfer_shipped_specs_test.py b/cosmos_framework/inference/transfer_shipped_specs_test.py new file mode 100644 index 00000000..f9e6d1cd --- /dev/null +++ b/cosmos_framework/inference/transfer_shipped_specs_test.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 + +"""Shipped Transfer specs and requests must build without errors or ignored-setting warnings.""" + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from cosmos_framework.inference import args as args_module +from cosmos_framework.inference import transfer +from cosmos_framework.inference.args import OmniSampleOverrides + +pytestmark = [pytest.mark.L0, pytest.mark.CPU] + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_COOKBOOK_SPECS = _REPO_ROOT / ".agents/skills/cookbooks/cosmos3/generator/transfer/specs" +_SIM_TRANSFER_SKILL = _REPO_ROOT / ".agents/skills/cosmos3-nano-sim-transfer" + + +def _nano_model_config() -> SimpleNamespace: + return SimpleNamespace( + action_gen=False, + sound_gen=False, + resolution="720", + video_temporal_causal=False, + fixed_step_sampler_config=None, + tokenizer=SimpleNamespace(temporal_compression_factor=4), + vlm_config=SimpleNamespace(model_name="Qwen/Qwen3-VL-8B-Instruct"), + rectified_flow_inference_config=SimpleNamespace(scheduler_type="unipc"), + ) + + +def _sim_transfer_model_config() -> SimpleNamespace: + return SimpleNamespace( + action_gen=False, + sound_gen=False, + resolution="480", + video_temporal_causal=True, + causal_training_strategy="none", + simulation_mode=None, + fixed_step_sampler_config=SimpleNamespace(t_list=[1.0, 0.9375, 0.8333333333333334, 0.625]), + teacher_forcing_frames_per_chunk=1, + teacher_forcing_replay_policy=SimpleNamespace( + control_visibility="causal", controls_read_strict_past_clean_rgb=True, clean_pass_causality="frame" + ), + tokenizer=SimpleNamespace(temporal_compression_factor=4), + vlm_config=SimpleNamespace(model_name="Qwen/Qwen3-VL-8B-Instruct"), + rectified_flow_inference_config=SimpleNamespace(scheduler_type="unipc"), + ) + + +def _record_warnings(monkeypatch: pytest.MonkeyPatch) -> list[str]: + warnings: list[str] = [] + monkeypatch.setattr(args_module.log, "warning", lambda message, *_a, **_k: warnings.append(message)) + return warnings + + +@pytest.mark.parametrize("spec", sorted(_COOKBOOK_SPECS.glob("*.json")), ids=lambda path: path.stem) +def test_cookbook_transfer_spec_builds_without_ignored_setting_warnings( + spec: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (overrides,) = OmniSampleOverrides.from_files([spec]) + if str(overrides.vision_path).startswith("https://"): + # Keep the test offline; only the request fields matter for building the sample. + stand_in = tmp_path / "vision.mp4" + stand_in.touch() + overrides.vision_path = str(stand_in) + warnings = _record_warnings(monkeypatch) + # The cookbook runs from the repository root, so the spec's relative negative_prompt_file does not + # resolve; it must not be needed, because the defaults already supply the negative caption. + monkeypatch.chdir(tmp_path) + overrides.output_dir = tmp_path / "out" + overrides.download(overrides.output_dir / "inputs") + + sample = overrides.build_sample(model_config=_nano_model_config()) + + assert not [w for w in warnings if w.startswith("[transfer]")] + assert {"fps", "num_frames"} <= sample.requested_fields + + +def _sim_transfer_requests(tmp_path: Path) -> list[Path]: + """Write the skill's official requests with prepare_hf_sample.py, on empty stand-in media.""" + module_spec = importlib.util.spec_from_file_location( + "prepare_hf_sample", _SIM_TRANSFER_SKILL / "scripts/prepare_hf_sample.py" + ) + assert module_spec is not None and module_spec.loader is not None + prepare_hf_sample = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(prepare_hf_sample) + requests: list[Path] = [] + for example in ("example1", "example2"): + sample_dir = tmp_path / "sample" / example + sample_dir.mkdir(parents=True) + for suffix in (".mp4", "_depth.mp4", "_segmentation.mp4"): + (sample_dir / f"{example}{suffix}").touch() + prompt = sample_dir / f"{example}_upsampled_prompt.json" + prompt.write_text(json.dumps({"prompt": "a robot"}) if example == "example1" else "a street") + prepare_hf_sample.prepare(example, sample_dir, tmp_path / example) + requests += sorted((tmp_path / example / "requests").glob("*.json")) + return requests + + +def test_sim_transfer_skill_requests_build_without_ignored_setting_warnings( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + requests = _sim_transfer_requests(tmp_path) + assert len(requests) == 6 + # The skill's command: its defaults file plus these CLI flags. + cli = OmniSampleOverrides( + defaults_file=_SIM_TRANSFER_SKILL / "inference/assets/sample_args.json", + autoregressive=True, + seed=2025, + num_steps=4, + guidance=1.0, + control_guidance=1.0, + prompt_upsampling=False, + ) + warnings = _record_warnings(monkeypatch) + for request in requests: + for overrides in OmniSampleOverrides.from_files([request], overrides=cli): + overrides.output_dir = tmp_path / "out" / request.stem + overrides.download(overrides.output_dir / "inputs") + overrides.build_sample(model_config=_sim_transfer_model_config()) + + assert not [w for w in warnings if w.startswith("[transfer]")] + + +@pytest.mark.parametrize( + ("requested", "total_frames", "input_fps", "expected"), + [ + ({"fps": 30, "num_frames": 121}, 121, 30.0, []), + ({"fps": 30, "num_frames": 121}, 121, 29.97, []), + ({"fps": 10}, 121, 30.0, ["'fps'=10 has no effect with a video input"]), + ({"fps": 10}, 1, 0.0, []), + ({"num_frames": 61}, 121, 30.0, ["'num_frames'=61 does not set the Transfer output length"]), + ({"num_frames": 601}, 600, 30.0, []), + ({}, 93, 16.0, []), + ], +) +def test_requested_fps_and_num_frames_warn_only_when_the_source_disagrees( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + requested: dict, + total_frames: int, + input_fps: float, + expected: list[str], +) -> None: + control = tmp_path / "depth.mp4" + control.touch() + overrides = OmniSampleOverrides( + name="sample", depth={"control_path": str(control)}, output_dir=tmp_path, **requested + ) + overrides.download(tmp_path / "inputs") + sample = overrides.build_sample(model_config=_nano_model_config()) + warnings: list[str] = [] + monkeypatch.setattr(transfer.log, "warning", lambda message, *_a, **_k: warnings.append(message)) + + transfer._warn_unused_length_and_fps(sample, total_frames, input_fps, temporal_compression_factor=4) + + assert len(warnings) == len(expected) + for message, text in zip(warnings, expected, strict=True): + assert text in message diff --git a/cosmos_framework/inference/transfer_streaming.py b/cosmos_framework/inference/transfer_streaming.py index 104afa9a..d1de5326 100644 --- a/cosmos_framework/inference/transfer_streaming.py +++ b/cosmos_framework/inference/transfer_streaming.py @@ -19,6 +19,7 @@ _latent_frames_for_pixel_frames, _require_causal_transfer_pyav, _resolve_transfer_seed, + _warn_unused_length_and_fps, load_transfer_control_frames, ) from cosmos_framework.inference.vision import ( @@ -199,6 +200,8 @@ def _prepare_streaming_transfer_input( height = media.height width = media.width temporal_compression_factor = int(model.config.tokenizer.temporal_compression_factor) + # Same check as the non-streaming path; an image source reports its sample fps, so it never warns. + _warn_unused_length_and_fps(sample_args, total_frames, media.output_fps, temporal_compression_factor) chunk_size = int(getattr(model.config, "teacher_forcing_frames_per_chunk", 1) or 1) if input_frames is not None: input_frames = pad_temporal_frames(input_frames, total_frames) # [3,T,H,W]