diff --git a/tensorrt_llm/_torch/kv_cache_compression/triattention/triattention.py b/tensorrt_llm/_torch/kv_cache_compression/triattention/triattention.py index de0e1ef4f04a..04ad73b627df 100644 --- a/tensorrt_llm/_torch/kv_cache_compression/triattention/triattention.py +++ b/tensorrt_llm/_torch/kv_cache_compression/triattention/triattention.py @@ -277,7 +277,7 @@ def _resolve_attention_layers(self) -> Tuple[List[int], List[int], Optional[int] return (dense_layers, swa_layers, window_size) def _load_calibration(self) -> None: - raw = torch.load(self.calibration_path, map_location="cpu", weights_only=False) + raw = torch.load(self.calibration_path, map_location="cpu", weights_only=True) if isinstance(raw, dict) and _REQUIRED_CALIBRATION_KEYS <= set(raw): e_q = raw["E_q"] e_q_norm = raw["E_q_norm"] diff --git a/tensorrt_llm/_torch/pyexecutor/profiling.py b/tensorrt_llm/_torch/pyexecutor/profiling.py index e7bcbd5157fd..8cbb20a9d664 100644 --- a/tensorrt_llm/_torch/pyexecutor/profiling.py +++ b/tensorrt_llm/_torch/pyexecutor/profiling.py @@ -43,6 +43,7 @@ import functools import os +import tempfile import time import uuid import weakref @@ -287,15 +288,7 @@ def _prepare_and_broadcast_start( activities = ["CPU", "GPU"] if output_dir is None: - # ``/tmp`` is the documented developer default for the - # profile trace directory (matches the server docs); the - # user-facing override is the - # ``TLLM_TORCH_PROFILER_DIR`` env var or the - # ``output_dir`` request body field. - output_dir = os.environ.get( - "TLLM_TORCH_PROFILER_DIR", - "/tmp", # nosec B108 - ) + output_dir = os.environ.get("TLLM_TORCH_PROFILER_DIR") # Fail fast if the directory cannot be created. Returning # success here would schedule a profile window that later @@ -306,11 +299,16 @@ def _prepare_and_broadcast_start( # plumbing as the "already in progress" guard above, so the # HTTP handler can map it to a 5xx with a meaningful body. try: - os.makedirs(output_dir, exist_ok=True) + if output_dir is None: + with tempfile.TemporaryDirectory(prefix="trtllm-profile-"): + pass + else: + os.makedirs(output_dir, exist_ok=True) except OSError as e: from tensorrt_llm.executor.utils import RequestError - raise RequestError(f"Failed to create profile output_dir {output_dir}: {e}") from e + target = output_dir if output_dir is not None else "the system temporary directory" + raise RequestError(f"Failed to create profile output directory at {target}: {e}") from e # Include a unique profile id (monotonic timestamp + short # uuid fragment) so successive start/stop cycles under the @@ -453,9 +451,11 @@ def apply_start_config(self, config: Dict) -> None: """ executor = self._executor activities = config.get("activities") or ["CPU", "GPU"] - # ``/tmp`` matches the documented developer default; see - # ``start_profile`` for the rationale. - output_dir = config.get("output_dir") or "/tmp" # nosec B108 + output_dir = config.get("output_dir") + if not output_dir: + # Maintain compatibility for callers constructing + # configs without output_dir. + output_dir = tempfile.mkdtemp(prefix="trtllm-profile-") profile_id = config.get("profile_id") or f"{int(time.time())}-{uuid.uuid4().hex[:8]}" # Prefer the legacy absolute ``start_iter`` form (older unit # tests and env-var driven paths set this directly). For the diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 44768cbdb1bf..4ae5999c510c 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -2070,7 +2070,7 @@ def start_profile(self, Args: output_dir: Directory where chrome traces are written. If ``None``, falls back to the ``TLLM_TORCH_PROFILER_DIR`` - environment variable and finally to ``/tmp``. + environment variable and finally to a temporary directory. num_steps: Number of iterations to profile. If ``None``, profiling runs until ``stop_profile`` is called. start_step: Additional iterations to skip before profiling diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index d21e1b6303cc..9aedeeb874fa 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -410,9 +410,8 @@ def _publish_bound_address(report_addr: Optional[str], host: str, host = resolved report_addr = os.path.abspath(report_addr) parent = os.path.dirname(report_addr) - if parent: - os.makedirs(parent, exist_ok=True) - fd, tmp_path = tempfile.mkstemp(dir=parent or None, + os.makedirs(parent, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=parent, prefix=os.path.basename(report_addr) + ".", suffix=".tmp") try: @@ -2253,9 +2252,18 @@ def _serve_coordinator_and_fleet(disagg_cfg, config_file, # that dominated per-request latency). Opt-out via the UDS env = "0". use_uds = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_UDS, "1") == "1" - coord_uds = ( - f"/tmp/trtllm_disagg_coord_{public_port}.sock" # nosec B108 - if use_uds else None) + coord_uds_dir = (tempfile.TemporaryDirectory( + prefix="trtllm-disagg-coord-") if use_uds else None) + coord_uds = (os.path.join(coord_uds_dir.name, "coordinator.sock") + if coord_uds_dir else None) + # TMPDIR must be at most 61 bytes to keep this socket path within Linux's + # 107-byte limit. + if (coord_uds_dir is not None and coord_uds is not None + and len(os.fsencode(coord_uds)) > 107): + coord_uds_dir.cleanup() + raise ValueError( + "Coordinator Unix socket path exceeds 107 bytes; TMPDIR must be " + "at most 61 bytes") # Fleet points at the UDS when enabled; the TCP port stays up for health. coord_url = f"unix:{coord_uds}" if coord_uds else \ f"http://{public_host}:{coord_port}" @@ -2343,6 +2351,8 @@ async def _monitor_fleet(): for process in fleet: if process.poll() is None: process.terminate() + if coord_uds_dir is not None: + coord_uds_dir.cleanup() def _init_fleet_worker_process(): diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index fdda58cbbf31..64698cf5d44d 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -1671,7 +1671,7 @@ def start_profile(self, output_dir (str, optional): Directory where chrome traces are written. If ``None``, falls back to the ``TLLM_TORCH_PROFILER_DIR`` environment variable and finally - to ``/tmp``. Defaults to None. + to a temporary directory. Defaults to None. num_steps (int, optional): Number of engine iterations to capture. When set, the engine stops the window automatically and ``stop_profile`` does not need to be called. When diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 0d85fd554229..31cc276a1adb 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -2375,7 +2375,7 @@ class StartProfileRequest(OpenAIBaseModel): default=None, description="Directory where chrome traces are written. Defaults " "to the TLLM_TORCH_PROFILER_DIR environment variable, " - "then /tmp.") + "then a temporary directory.") num_steps: Optional[PositiveInt] = Field( default=None, description="Number of iterations to profile. Must be >= 1 if "