From 9e72a12298a10f1ca9f2ae3dbc641972f299da92 Mon Sep 17 00:00:00 2001 From: Dennis Date: Fri, 11 Sep 2026 14:51:47 -0700 Subject: [PATCH 01/29] added new items to gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c1b977c..0c31877 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,7 @@ __pycache__/ _build/ /demo/ -/cli_test_repos/ \ No newline at end of file +/cli_test_repos/ +AGENTS.md +TODO.md +workplan.md From 20ec72ae6603b71c505523e5b41b35f7f43428d0 Mon Sep 17 00:00:00 2001 From: Dennis Date: Mon, 14 Sep 2026 21:33:09 -0700 Subject: [PATCH 02/29] upd gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0c31877..a2c6200 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ _build/ AGENTS.md TODO.md workplan.md +data-origins-plan.md From 5002e8fd8340096c44ab67964bcffd6ce084b09f Mon Sep 17 00:00:00 2001 From: Dennis Date: Tue, 15 Sep 2026 20:07:05 -0700 Subject: [PATCH 03/29] Exclude personal ignore rules from the feature branch --- .gitignore | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a2c6200..c1b977c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,4 @@ __pycache__/ _build/ /demo/ -/cli_test_repos/ -AGENTS.md -TODO.md -workplan.md -data-origins-plan.md +/cli_test_repos/ \ No newline at end of file From 7b919c365f6567ef8ef6cf531f3d23ce72f87343 Mon Sep 17 00:00:00 2001 From: Dennis Date: Tue, 15 Sep 2026 20:07:05 -0700 Subject: [PATCH 04/29] Add private SSH and SFTP data sources --- .github/workflows/ci.yml | 21 + README.rst | 21 +- doc/api.rst | 16 +- doc/usecase.rst | 160 ++++++++ mod/hallmark/cli.py | 27 +- mod/hallmark/dothm.py | 2 + mod/hallmark/downloader.py | 267 ++++++------- mod/hallmark/repo.py | 5 +- mod/hallmark/repo_builder.py | 336 ++++++++-------- mod/hallmark/repo_config.py | 30 +- mod/hallmark/transport/__init__.py | 110 +++++ mod/hallmark/transport/auth.py | 129 ++++++ mod/hallmark/transport/base.py | 196 +++++++++ mod/hallmark/transport/http.py | 85 ++++ mod/hallmark/transport/ssh.py | 411 +++++++++++++++++++ pyproject.toml | 5 + test/mock_server.py | 10 +- test/test_downloader.py | 14 +- test/test_repo_builder.py | 20 +- test/test_ssh_integration.py | 462 +++++++++++++++++++++ test/test_transport.py | 623 +++++++++++++++++++++++++++++ 21 files changed, 2607 insertions(+), 343 deletions(-) create mode 100644 mod/hallmark/transport/__init__.py create mode 100644 mod/hallmark/transport/auth.py create mode 100644 mod/hallmark/transport/base.py create mode 100644 mod/hallmark/transport/http.py create mode 100644 mod/hallmark/transport/ssh.py create mode 100644 test/test_ssh_integration.py create mode 100644 test/test_transport.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16556b2..54e9f87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,3 +35,24 @@ jobs: - name: Test with pytest run: | pytest + + ssh-transport: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + - name: Install Linux SSH server + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y openssh-server + - name: Install Hallmark and tests + run: python -m pip install -e . pytest + - name: Exercise disposable OpenSSH server and client + env: + HALLMARK_RUN_SSH_TESTS: "1" + run: python -m pytest test/test_ssh_integration.py test/test_transport.py -q diff --git a/README.rst b/README.rst index 9df3b17..f769d42 100644 --- a/README.rst +++ b/README.rst @@ -69,4 +69,23 @@ Or install from source for development purposes:: .. _hallmark: https://github.com/l6a/hallmark .. _yukon: https://github.com/l6a/yukon .. _banyan: https://github.com/l6a/banyan -.. _eht: https://eventhorizontelescope.org \ No newline at end of file +.. _eht: https://eventhorizontelescope.org + +Private data remotes +-------------------- + +Hallmark can fetch cataloged files from HTTP(S), SSH and SFTP data remotes. +Data remotes locate file bytes separately from the Git remotes used for catalog +history. SSH uses system OpenSSH 9.6+, a trusted host key and noninteractive +key/agent authentication:: + + hallmark set-config --remote-name campus --remote-url ssh://campus/srv/export/ + hallmark download --remote campus --all --dry-run + hallmark download --remote campus --all + +Optional local auth profiles can be selected with ``--remote-auth PROFILE``. +Private catalogs can be built with ``build --dataset-url URL --dataset-auth +PROFILE --allow-remote-commands``; SSH listing requires a POSIX shell and Python 3 +on the server. SFTP-only accounts can download known files. See +`Private lab data over SSH `_ for setup, +profiles, bounded hashing, platform scope and disposable integration tests. diff --git a/doc/api.rst b/doc/api.rst index 6c7d878..0ef7bbe 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -146,4 +146,18 @@ Preview a complete repository download:: hallmark download --all --dry-run -Large selections require confirmation unless ``--yes`` is supplied. \ No newline at end of file +Large selections require confirmation unless ``--yes`` is supplied. + +Dataset builders and data remotes +--------------------------------- + +.. autofunction:: hallmark.repo_builder.build_repo + +.. autofunction:: hallmark.repo_builder.list_remote_files + +``Repo.set_config(remote_auth="PROFILE")`` records a local SSH profile reference; +``remote_auth=""`` removes it. ``download_remote_data`` keeps its existing result +mapping with ``succeeded``, ``failed``, ``total_bytes`` and ``errors``. Configuration +and SSH connection preflight failures raise ``DownloadError`` before file workers +start. HTTP, SSH and SFTP follow the same destination/checksum contract. See the +private-lab workflow in :doc:`usecase` for profile schema and capability limits. diff --git a/doc/usecase.rst b/doc/usecase.rst index 2996054..722c900 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -120,3 +120,163 @@ This is especially useful for data (re-)organization. .. |hallmark| replace:: ``hallmark`` .. _hallmark: https://github.com/l6a/hallmark + + +Private lab data over SSH +------------------------- + +A **data remote** locates dataset bytes. A **Git remote** synchronizes the +catalog's history in ``.hm``. They are separate configurations, even when both +are named ``origin``. Selecting ``download --remote campus`` selects only that +data remote; Hallmark does not automatically fall back to another server. + +Use OpenSSH ``ssh`` and ``sftp`` 9.6 or newer on Linux. Linux and macOS have +integration jobs; macOS support should be checked against that job's result. +Windows SSH transport is not supported. Requests remains required for HTTP, +including its existing environment and ``.netrc`` authentication behavior. + +Before downloading, configure your SSH alias, trust the server's host key, and +load a usable key into your agent (or configure an identity file). Workers use +``BatchMode=yes`` and default to ``StrictHostKeyChecking=yes``. Password prompts, +interactive 2FA, and an initial interactive hardware-key challenge cannot be +answered by Hallmark. Complete those prerequisites separately. Hallmark owns a +new multiplexed connection for each invocation and closes it afterward; existing +user-owned masters are neither reused nor stopped. Multiplexing is required. +ProxyJump and other trusted SSH policy remain in your local SSH configuration. + +Configure an absolute dataset root:: + + hallmark set-config --remote-name campus --remote-url ssh://campus/srv/export/ + hallmark download --remote campus --all --dry-run + hallmark download --remote campus --all + +``sftp://`` is an alias for ``ssh://``. Explicit users and ports are supported, +for example ``ssh://researcher@campus:2222/srv/export/``. IPv6 addresses need +brackets: ``ssh://[2001:db8::1]/srv/export/``. SCP shorthand and implicit +home-relative roots are unsupported. URL paths must encode reserved characters, +e.g. ``has%20space%23tag``. Catalog/manifest paths are literal: ``literal%20.h5`` +names a file containing a percent sign. Spaces, quotes, glob characters, and +leading dashes are supported; control characters and backslashes are rejected. + +Downloads stream into temporary files, verify recorded checksums, then replace +the requested destinations atomically. A failed transfer or checksum mismatch +preserves an existing destination and removes its temporary file. Other files +that already succeeded remain available. Cancellation terminates owned SSH +process groups, waits for workers and closes the master. HTTP workers check +cancellation between chunks, but blocked Requests calls can take their normal +``(10, 30)`` connect/read timeouts to return. A slowly trickling response can +keep a blocked read alive longer; no fixed five-second HTTP +cancellation guarantee is made. Existing destination symlinks and observed +symlink swaps are rejected. This is not protection against every race with a +concurrently hostile local writer. + +Local profiles +~~~~~~~~~~~~~~ + +Profiles live in ``$HALLMARK_AUTH_FILE``, otherwise +``$XDG_CONFIG_HOME/hallmark/auth.yml``, otherwise +``~/.config/hallmark/auth.yml``. Repository configuration stores only the profile +name. Profiles currently apply to SSH/SFTP, with this strict schema: + +.. code-block:: yaml + + version: 1 + defaults: + connect_timeout: 10 + transfer_timeout: 3600 + shutdown_timeout: 2 + max_sessions: 4 + profiles: + campus: + hosts: [campus] + user: researcher + port: 22 + identity_file: ~/.ssh/id_ed25519 + host_key_policy: strict + +Supported profile fields are ``hosts`` and the settings shown above; profiles +may also override the four time/concurrency defaults. Global defaults cannot +set ``hosts``, ``user``, ``port`` or ``identity_file``. Passwords, tokens, arbitrary +SSH options, executables and SSH config paths are not supported fields. The +profile name must match ``[A-Za-z0-9_-]{1,64}``. A missing profile or a URL host +outside its ``hosts`` list fails before connections start. URL user/port override +profile defaults, then SSH configuration and OpenSSH defaults apply. Host bindings +refer to the URL alias or literal host, before SSH config resolves ``HostName``. + +``transfer_timeout`` is a total wall-clock budget per file in seconds, not an +idle/read timeout; increase it locally for large datasets. ``max_sessions`` caps +concurrent transfers per operation to respect the server's session limit. +Only a local profile/default may select ``host_key_policy: accept-new`` for +first-contact trust; changed host keys still fail. + +Associate or remove a profile reference:: + + hallmark set-config --remote-name campus --remote-auth campus + hallmark set-config --remote-name campus --remote-auth '' + +The Python equivalents are ``repo.set_config(remote_auth="campus")`` and +``repo.set_config(remote_auth="")``. ``None`` leaves the reference unchanged. +No resolved identity settings are written into the repository. + +Build a private catalog +~~~~~~~~~~~~~~~~~~~~~~~ + +The crawl source is independent of remotes recorded in the resulting catalog. +``--dataset-url`` is the exact dataset root; the dataset name is only a label:: + + hallmark build catalogs lab \ + --dataset-url ssh://campus/srv/export/ \ + --dataset-auth campus --allow-remote-commands \ + --fmt 'runs/run_{i:d}.h5=data.tsv' + +SSH builds require a POSIX login shell and Python 3 on the server. The explicit +``--allow-remote-commands`` flag permits fixed read-only recursive listing and, +when requested, hashing commands. Repository configuration cannot enable this +permission. SFTP-only accounts support downloads of cataloged files but cannot +build catalogs. Listings omit symlinks and special files, including symlinked +directories. Remote roots are namespaces; enforce access boundaries on the server. + +Existing checksum manifests are preferred. Supported manifests use unescaped GNU +sum lines: a hexadecimal digest, a space, a space or ``*`` marker, then the literal +filename. Strong algorithm names such as ``sha256`` are preserved. GNU escaped +filename records are rejected. Missing digests remain absent/``unknown``; auth and +ambiguous SSH failures are errors, never treated as an absent optional manifest. + +Add ``--remote-hash`` to compute server SHA-256 for files without a manifest hash. +Hashing is limited to 10 MiB per file, 100 MiB total and a 60-second operation +budget. Larger files or exhausted budgets retain unknown digests. Obvious file +size/mtime changes during hashing fail. Recursive listing has a 100,000-file and +16 MiB output limit with a 60-second command timeout. Text reads have a 16 MiB +per-file and 64 MiB aggregate limit; new reads stop after a five-minute crawl +budget. No dataset file is downloaded merely to compute an SSH checksum. + +When no output remotes are supplied, the synthesized ``origin`` records the +source URL and its profile reference. ``--remote NAME=URL`` records a different +output location without changing the crawl source or inheriting its auth profile. +A name-only output remote defaults to the crawl URL but must be configured with +its own auth reference if needed. + +Without ``--dataset-url``, legacy CyVerse dataset-name lookup is preserved. +An explicit HTTP root requires ``--index-format cyverse-html`` and must expose +that HTML index dialect. Arbitrary HTTP directory listing and WebDAV PROPFIND +are not implemented. Static HTTP checksums retain the bounded small-file MD5 +fallback. Local add/commit/checkout workflows still require one format backed +by ``data.tsv``; builder and download workflows may use multiple catalogs. + +Use immutable exports or server snapshots when reproducibility matters: crawling +and hashing a live directory cannot produce a guaranteed snapshot. Subsequent +downloads always verify whatever digest the catalog records. + +Transport validation +~~~~~~~~~~~~~~~~~~~~ + +Run ordinary tests with ``pytest -q``. The SFTP parser tests use a local +``sftp-server`` process without networking. The complete OpenSSH tests use only +loopback sockets, ephemeral keys, isolated known-hosts/config files and temporary +data:: + + HALLMARK_RUN_SSH_TESTS=1 pytest test/test_ssh_integration.py -q + +The opt-in fails if required tools are unavailable. It never connects to a live +lab server or changes normal SSH trust files. CI installs OpenSSH server and +runs this suite separately from the Python-version matrix. diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 1413e09..6cfbd90 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -72,6 +72,7 @@ def _translate_cli_errors(*error_types, prefix=None): # exception types translated to a clean CLI error by the hallmark build command _BUILD_DATASET_ERRORS = ( + DownloadError, RuntimeError, ValueError, FileNotFoundError, @@ -306,13 +307,15 @@ def add(repo, encoding, inputs): @click.option("--fmt") @click.option("--remote-name") @click.option("--remote-url") +@click.option("--remote-auth", help="Local SSH profile; empty string removes it.") @click.option("--encoding", "encodings", multiple=True) @click.pass_obj -def set_config(repo, fmt, remote_name, remote_url, encodings): +def set_config(repo, fmt, remote_name, remote_url, remote_auth, encodings): """Update the current branch config.yml.""" # if no config changes are requested, raise a ClickException to inform the user if ( - fmt is None and remote_name is None and remote_url is None and not encodings): + fmt is None and remote_name is None and remote_url is None + and remote_auth is None and not encodings): raise ClickException("No config changes requested.") encoding_updates = {} for item in encodings: @@ -329,7 +332,8 @@ def set_config(repo, fmt, remote_name, remote_url, encodings): fmt=fmt, remote_name=remote_name, remote_url=remote_url, - encoding_updates=encoding_updates or None) + encoding_updates=encoding_updates or None, + **({"remote_auth": remote_auth} if remote_auth is not None else {})) click.echo("Updated hallmark config.") @@ -604,7 +608,16 @@ def clone(url, path, no_fetch_data, max_workers, yes): "--overwrite", is_flag=True, help="Replace the destination repository if it already exists.") -def build(directory, dataset_name, remotes, config_file, fmts, overwrite): +@click.option("--dataset-url", + help="Exact crawl root, independent of recorded remotes.") +@click.option("--dataset-auth", help="Local SSH profile for the crawl source.") +@click.option("--index-format", type=click.Choice(["cyverse-html"])) +@click.option("--allow-remote-commands", is_flag=True, + help="Allow fixed read-only SSH listing commands (server python3).") +@click.option("--remote-hash", is_flag=True, + help="Allow bounded server-side SHA-256 hashing for SSH builds.") +def build(directory, dataset_name, remotes, config_file, fmts, overwrite, + dataset_url, dataset_auth, index_format, allow_remote_commands, remote_hash): """ Build a hallmark repository at DIRECTORY for the remote dataset DATASET_NAME. @@ -673,6 +686,10 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite): # if the checks pass, append the fmt and db to the fmt_entries list fmt_entries.append({"fmt": fmt, "db": db}) + source_options = {key: value for key, value in { + "dataset_url": dataset_url, "dataset_auth": dataset_auth, + "index_format": index_format, "allow_remote_commands": allow_remote_commands, + "remote_hash": remote_hash}.items() if value is not None and value is not False} # build the hallmark repository with the specified parameters try: with _translate_cli_errors(*_BUILD_DATASET_ERRORS): @@ -682,7 +699,7 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite): fmt_entries=fmt_entries, config_file=config_file, remotes=parsed_remotes or None, - overwrite=overwrite) + overwrite=overwrite, **source_options) except requests.exceptions.RequestException as exc: raise ClickException( f"Failed to reach dataset {dataset_name!r}: {exc}") from exc diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index fbbedf5..fcc8c91 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -153,6 +153,8 @@ def config_template() -> str: remote: # name: origin # url: https://example.com/path/to/data/ + # SSH alternative: ssh://campus/srv/export/ (absolute root, trusted host/key) + # auth: campus # Optional local profile name; never put credentials here. """ @classmethod diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index a7992d0..e6014a3 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -6,24 +6,23 @@ from pathlib import Path from threading import local from typing import Mapping, Optional, Sequence, Union -from urllib.parse import quote, urljoin +from urllib.parse import unquote, urlsplit, urlunsplit from tqdm import tqdm import requests import pandas as pd -from .error import HallmarkError +from .transport import OperationContext, RemoteSpec +from .transport.base import DownloadError, literal_path from .helper_functions import ( CHECKSUM_ALGORITHMS_BY_STRENGTH, - REMOTE_REQUEST_TIMEOUT, SUPPORTED_CHECKSUM_ALGORITHMS, as_list_of_dicts, atomic_output_path, file_checksum, normalize_nonempty_string, resolve_contained_path, - valid_checksum, - validate_relative_path) + valid_checksum) from .repo_config import ( normalize_remotes, normalize_tsv_name, @@ -41,10 +40,6 @@ # The size of chunks to read from a file when downloading or computing checksums. DOWNLOAD_CHUNK_SIZE = 8192 -class DownloadError(HallmarkError): - """Raised when remote data download fails.""" - - def _repository_config(repo) -> dict: """ Used by _select_remote_config and select_download_files. @@ -90,13 +85,9 @@ def _config_section_entries(config: dict, section_name: str) -> list[dict]: return [entry for entry in entries if isinstance(entry, dict)] -def _initialize_download_worker() -> None: - """ - Used by download_remote_data. - Create one reusable HTTP session for each downloader thread. - This function is called once per thread in the ThreadPoolExecutor. - """ - _DOWNLOAD_WORKER_STATE.session = requests.Session() +def _initialize_download_worker(context) -> None: + """Bind this invocation's context to each worker; no cross-operation state.""" + _DOWNLOAD_WORKER_STATE.context = context def _require_positive_integer(value, *, label: str) -> int: @@ -360,33 +351,12 @@ def _safe_remote_path(value: Union[str, Path]) -> Path: Raises: DownloadError: If the path is unsafe. """ - # Convert the input value to a string and strip whitespace. - raw_path = str(value).strip() - # Validate the path using the validate_relative_path function, which checks for - # unsafe characters, absolute paths, and other potential issues. - try: - return validate_relative_path(raw_path, label="remote path") - # Handle exceptions raised by validate_relative_path and raise a DownloadError - except ValueError as exc: - raise DownloadError(str(exc)) from exc + return literal_path(value) def _remote_file_url(remote_url: str, relative_path: Path) -> str: - """ - Used by download_remote_data. - Construct the full URL for a remote file based on the remote URL and relative path. - - Args: - remote_url: The base URL of the remote repository. - relative_path: The relative path of the file to download. - - Returns: - The full URL for the remote file as a string. - """ - # quote the relative path to ensure it is safe for use in a URL, preserving slashes - encoded_path = quote(relative_path.as_posix(), safe="/") - # join the remote URL and the encoded path to create the full URL for the file - return urljoin(remote_url.rstrip("/") + "/", encoded_path) + """Append a literal catalog path to a supported remote root.""" + return RemoteSpec.parse(remote_url).file_url(relative_path.as_posix()) def _download_tsv_name(value) -> str: @@ -560,66 +530,67 @@ def _verify_validated_checksum( raise DownloadError(f"Checksum mismatch for {path.name} " f"({algorithm})") -def _download_file( - url: str, - destination: Path, - expected_checksum: Optional[ChecksumSpec] = None, - chunk_size: int = DOWNLOAD_CHUNK_SIZE, - ) -> int: - """ - Used by download_remote_data. - Download one file and return the number of bytes written. - - Args: - url: The URL of the file to download. - destination: The path where the downloaded file will be saved. - expected_checksum: The expected checksum of the file. Can be a tuple of - (algorithm, checksum) or a legacy SHA-1 checksum. - chunk_size: The size of chunks to read the file in. - - Returns: - The number of bytes written to the destination file. - - Raises: - DownloadError: If the download fails or the checksum verification fails. - """ - # Ensure that the chunk size is a positive integer for reading the file in chunks. - chunk_size = _require_positive_integer(chunk_size, label="chunk_size") - # Validate and normalize the expected checksum specification +def _fetch_file(context, relative_path, destination, expected_checksum, chunk_size): validated_checksum = _validate_checksum_spec(expected_checksum) - - # try to download the file from the specified URL and write it to the temporary file try: - # Ensure the destination directory exists, creating it if necessary. + context.check_cancelled() + destination = resolve_contained_path( + context.output_root, relative_path, label="download destination") destination.parent.mkdir(parents=True, exist_ok=True) - # use atomic_output_path to create a temporary file for the download - with atomic_output_path(destination, suffix=".part", ) as temp_path: - # get the request client from the thread-local state - request_client = getattr(_DOWNLOAD_WORKER_STATE, "session", requests) - # use the request client to perform a GET request to the URL - with request_client.get( - url, - stream=True, - timeout=REMOTE_REQUEST_TIMEOUT) as response: - response.raise_for_status() - # handle the response content in chunks - with temp_path.open("wb") as handle: - for chunk in response.iter_content(chunk_size=chunk_size): - if chunk: - # write the chunk to the temporary file - handle.write(chunk) - # verify the checksum of the downloaded file against the expected value + with atomic_output_path(destination, suffix=".part") as temp_path: + context.transport.fetch(relative_path.as_posix(), temp_path, + chunk_size=chunk_size) _verify_validated_checksum(temp_path, validated_checksum, chunk_size) - # size is the number of bytes written to the temporary file size = temp_path.stat().st_size - + # Recheck after transfer as well as before creation. This protects + # against observed symlink swaps, not a concurrently hostile writer. + resolve_contained_path(context.output_root, relative_path, + label="download destination") + context.check_cancelled() return size + except (OSError, ValueError) as exc: + raise DownloadError(f"Failed to write {destination.name}: {exc}") from None + - # raise a DownloadError if the download request fails or issue writing the file - except requests.RequestException as exc: - raise DownloadError(f"Failed to download {url}: {exc}") from exc - except OSError as exc: - raise DownloadError(f"Failed to write {destination}: {exc}") from exc +def _download_file( + url: str, + destination: Path, + expected_checksum: Optional[ChecksumSpec] = None, + chunk_size: int = DOWNLOAD_CHUNK_SIZE, + ) -> int: + """Compatibility entry point; pool workers use their invocation's context.""" + chunk_size = _require_positive_integer(chunk_size, label="chunk_size") + _validate_checksum_spec(expected_checksum) + context = getattr(_DOWNLOAD_WORKER_STATE, "context", None) + if context is not None: + relative = destination.relative_to(context.output_root) + return _fetch_file( + context, relative, destination, expected_checksum, chunk_size) + RemoteSpec.parse(url) + parsed = urlsplit(url) + base = urlunsplit(parsed._replace(path=parsed.path.rsplit("/", 1)[0] + "/")) + remote = RemoteSpec.parse(base) + relative = literal_path(unquote(parsed.path.rsplit("/", 1)[-1])) + destination = Path(destination).absolute() + # Direct callers may intentionally choose a different local filename. + with OperationContext(remote, destination.parent) as context: + if remote.scheme in {"http", "https"}: + context._local.session = requests + context.transport.direct_url = url + try: + destination.parent.mkdir(parents=True, exist_ok=True) + resolve_contained_path(destination.parent, destination.name) + with atomic_output_path(destination, suffix=".part") as temporary: + context.transport.fetch(relative.as_posix(), temporary, + chunk_size=chunk_size) + _verify_validated_checksum( + temporary, _validate_checksum_spec(expected_checksum), chunk_size) + size = temporary.stat().st_size + resolve_contained_path(destination.parent, destination.name) + context.check_cancelled() + return size + except OSError as exc: + raise DownloadError(f"Failed to write {destination}: {exc}") from None def select_download_files( @@ -814,13 +785,14 @@ def download_remote_data( remote_url = normalize_nonempty_string( remote_config.get("url"), label="Remote URL", - exception_type=DownloadError).rstrip("/") + exception_type=DownloadError) # raise a DownloadError if the remote URL is not configured or invalid except DownloadError as exc: raise DownloadError("Remote URL not configured in config.yml") from exc # if the remote URL is not configured, raise a DownloadError to indicate the issue if not remote_url: raise DownloadError("Remote URL not configured in config.yml") + remote_spec = RemoteSpec.parse(remote_url, remote_config.get("auth")) # If there are still no files selected for download if not selected_files: # return the results without attempting any downloads @@ -875,55 +847,68 @@ def download_remote_data( # try to download the files using a thread pool executor for concurrent downloads try: - with ThreadPoolExecutor(max_workers=max_workers, - initializer=_initialize_download_worker) as executor: - # create an iterator over the files to download - file_iterator = iter(files_to_download) - # initialize a set to keep track of pending download futures - pending = set() - # determine the window size for concurrent downloads, allow for buffering - window_size = max_workers * 2 - - def submit_next() -> bool: - """ - Submit the next file for download if available. - Returns: - bool: True if a file was submitted, - False if no more files are available. - """ - # try to get the next file from the iterator - try: - url, destination, checksum = next(file_iterator) - # Handle the exception when there are no more files to download - except StopIteration: - return False - # add the download task to the pending set using the executor - pending.add(executor.submit(_download_file, url, destination, checksum)) - # if the download was successfully submitted, return True - return True - - # for each file in the initial window size, submit it for download - for _ in range(min(window_size, len(files_to_download))): - submit_next() - while pending: - # wait for at least one of the pending download tasks to complete - completed, _ = wait(pending, return_when=FIRST_COMPLETED) - # for each completed download, process the result and update statistics - for future in completed: - pending.remove(future) - # try to get the result of the completed download task + with OperationContext(remote_spec, output_root) as context: + context.transport.prepare() + executor = ThreadPoolExecutor( + max_workers=min(max_workers, context.settings.max_sessions) + if remote_spec.scheme in {"ssh", "sftp"} else max_workers, + initializer=_initialize_download_worker, initargs=(context,)) + try: + # create an iterator over the files to download + file_iterator = iter(files_to_download) + # initialize a set to keep track of pending download futures + pending = set() + # Bound queued work as well as running workers. + window_size = max_workers * 2 + + def submit_next() -> bool: + """ + Submit the next file for download if available. + Returns: + bool: True if a file was submitted, + False if no more files are available. + """ + # try to get the next file from the iterator try: - results["total_bytes"] += future.result() - results["succeeded"] += 1 - # Handle exceptions raised during the download - except DownloadError as exc: - results["failed"] += 1 - results["errors"].append(str(exc)) - # always update the progress bar, regardless of success or failure - finally: - progress.update(1) - # submit the next file for download if available + url, destination, checksum = next(file_iterator) + # Handle the exception when there are no more files to download + except StopIteration: + return False + # add the download task to the pending set using the executor + pending.add(executor.submit( + _download_file, url, destination, checksum)) + # if the download was successfully submitted, return True + return True + + # for each file in the initial window size, submit it for download + for _ in range(min(window_size, len(files_to_download))): submit_next() + while pending: + # wait for at least one of the pending download tasks to complete + completed, _ = wait(pending, return_when=FIRST_COMPLETED) + # Aggregate per-file failures and keep successful downloads. + for future in completed: + pending.remove(future) + # try to get the result of the completed download task + try: + results["total_bytes"] += future.result() + results["succeeded"] += 1 + # Handle exceptions raised during the download + except DownloadError as exc: + results["failed"] += 1 + results["errors"].append(str(exc)) + # Every completed transfer advances the progress bar. + finally: + progress.update(1) + # submit the next file for download if available + submit_next() + except BaseException: + context.cancel() + for future in pending: + future.cancel() + raise + finally: + executor.shutdown(wait=True, cancel_futures=True) # close after all downloads are complete, regardless of success or failure finally: progress.close() diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index d17947b..74a99c7 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -320,6 +320,7 @@ def set_config( remote_name: Optional[str] = None, remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, + remote_auth: Optional[str] = None, ) -> dict: """ Update repository configuration values. @@ -328,6 +329,7 @@ def set_config( fmt (str, optional): Data format specification. remote_name (str, optional): Name of the remote repository. remote_url (str, optional): URL of the remote repository. + remote_auth (str, optional): Local profile name; empty string removes it. encoding_updates (dict[str, str], optional): Updates to encoding rules. Returns: @@ -338,7 +340,8 @@ def set_config( fmt=fmt, remote_name=remote_name, remote_url=remote_url, - encoding_updates=encoding_updates) + encoding_updates=encoding_updates, + **({"remote_auth": remote_auth} if remote_auth is not None else {})) self.dothm.dump(self.state) return self.state.config diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index c2c0998..1fc9e12 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -1,22 +1,21 @@ from __future__ import annotations -import hashlib import re import shutil import string import time -from concurrent.futures import ThreadPoolExecutor from functools import lru_cache from pathlib import Path from string import Formatter -from threading import local -from urllib.parse import quote, unquote, urljoin, urlsplit +from urllib.parse import unquote, urlsplit import pandas as pd import parse -import requests from .repo import Repo +from .transport import OperationContext, RemoteSpec +from .transport.base import ( + CapabilityError, DownloadError, RemoteObjectMissing, reject_controls) from .fmt_detection import ( detect_fmt, KNOWN_PROCESSING_STAGES, @@ -25,9 +24,9 @@ from .error import DothmError from .helper_functions import ( CHECKSUM_ALGORITHMS, + CHECKSUM_ALGORITHMS_BY_STRENGTH, CHECKSUM_ALGORITHM_PATTERN, SUPPORTED_CHECKSUM_ALGORITHMS, - REMOTE_REQUEST_TIMEOUT, valid_checksum, atomic_output_path, load_yaml_file, @@ -49,7 +48,7 @@ r'' r'.+)\."rf"(?P{CHECKSUM_ALGORITHM_PATTERN})sums$") @@ -57,8 +56,6 @@ _CHECKSUM_NAME_KEYWORDS = ("sum", "checksum", "hash", "manifest", *CHECKSUM_ALGORITHMS) # maximum number of worker threads for static checksum computation _STATIC_CHECKSUM_MAX_WORKERS = 8 -# thread-local storage for network worker state, including requests sessions -_NETWORK_WORKER_STATE = local() # maximum number of worker threads for remote crawling and checksum computation _REMOTE_CRAWL_MAX_WORKERS = 8 # minimum ratio of matching lines in a checksum file to consider it valid @@ -77,137 +74,12 @@ "algorithm": CHECKSUM_ALGORITHMS} -def _network_worker_session(): - """ - Used by _checksum_small_remote_url and _fetch_remote_text. - Get or create a requests session for the current thread. - - Returns: - A requests.Session object for the current thread. - """ - # get the requests session from thread-local storage - session = getattr(_NETWORK_WORKER_STATE, "session", None) - if session is None: - # create a new requests session for this thread if it doesn't exist - session = requests.Session() - # network worker state is thread-local, so each thread will have its own session - _NETWORK_WORKER_STATE.session = session - return session - - def _remote_url(base_url: str, relative_path: str) -> str: - """ - Used by list_remote_files and build_repo. - Construct a full remote URL by joining a base URL and a relative path. - - Args: - base_url: The base URL of the remote repository. - relative_path: The relative path to the file or directory. - - Returns: - The full remote URL as a string. - """ - # strip trailing slashes from the base URL and ensure it ends with a single slash - directory_url = f"{str(base_url).rstrip('/')}/" - # use urljoin to combine the base URL and the quoted relative path - return urljoin(directory_url, quote(str(relative_path), safe="/")) - - -def _fetch_remote_text(url: str) -> str: - """ - Used by _fetch_optional_remote_text and list_remote_files. - Fetch the text content of a remote URL using a thread-local requests session. - Args: - url: The URL to fetch. - Returns: - The text content of the response. - """ - # Use a requests session to fetch the HTML index page for the given URL. - response = _network_worker_session().get(url, timeout=REMOTE_REQUEST_TIMEOUT) - # Raise an exception if the request failed - response.raise_for_status() - - # Return the text content of the response, which is the HTML index page. - return response.text - - -def _fetch_optional_remote_text(url: str) -> str | None: - """ - Used by list_remote_files. - Fetch the text content of a remote URL, returning None if the request fails. - - Args: - url: The URL to fetch. - - Returns: - The text content of the response, or None if the request fails. - """ - try: - return _fetch_remote_text(url) - # if the request fails (e.g., network error, timeout, 404), return None - except requests.RequestException: - return None - - -def _checksum_small_remote_file(session, file_url: str) -> tuple[str, str]: - """ - Used by _checksum_small_remote_url. - Compute the MD5 checksum of a small remote file. - - Args: - session: The requests session to use for HTTP requests. - file_url: The URL of the remote file. - - Returns: - A tuple containing the checksum algorithm and the checksum value. - If the file is too large or an error occurs, returns ("unknown", "unknown"). - """ - unknown = (_UNKNOWN_CHECKSUM_ALGORITHM, _UNKNOWN_CHECKSUM_ALGORITHM) - # try to get the file size using a HEAD request to avoid downloading large files - try: - head_response = session.head(file_url, timeout=REMOTE_REQUEST_TIMEOUT) - head_response.raise_for_status() - # if the HEAD request fails, return unknown checksum values - except requests.RequestException: - return unknown - - # get the Content-Length header to determine the file size - content_length = head_response.headers.get("Content-Length") - # try to convert the Content-Length to an integer, return unknown if it fails - try: - file_size = int(content_length) - except (TypeError, ValueError): - return unknown - # if the file size is negative or exceeds the maximum allowed size - if (file_size < 0 or file_size > _MAX_DOWNLOAD_SIZE_FOR_CHECKSUM): - return unknown - - # try to download the file and compute its MD5 checksum - try: - file_response = session.get(file_url, timeout=REMOTE_REQUEST_TIMEOUT) - file_response.raise_for_status() - # if the GET request fails, return unknown checksum values - except requests.RequestException: - return unknown - - # if all requests succeed, compute and return the MD5 checksum of the file content - return ("md5", hashlib.md5(file_response.content).hexdigest()) - - -def _checksum_small_remote_url(file_url: str) -> tuple[str, str]: - """ - Used by build_repo. - Compute the checksum of a small remote file using a thread-local requests session. - - Args: - file_url: The URL of the remote file. - - Returns: - _checksum_small_remote_file(session, file_url), where session is a thread-local - requests session. Tuple contains the checksum algorithm and the checksum value. - If the file is too large or an error occurs, returns ("unknown", "unknown"). - """ - return _checksum_small_remote_file(_network_worker_session(), file_url) + """Render a transport URL from a literal path, preserving directory slashes.""" + url = RemoteSpec.parse(base_url).file_url(relative_path) + if relative_path.endswith("/") and not url.endswith("/"): + url += "/" + return url # cache for performance, since the same fmt may be used for many files in a dataset @@ -631,6 +503,18 @@ def _match_file_against_fmts(rel_path: str, fmt_entries: list[dict]): valid_fmts, key=lambda m: (m[2], -m[3])) return best_index, best_parse +def _record_checksum(checksums, path, algorithm, checksum): + previous_algorithm, previous = checksums.get(path, (None, None)) + if previous is not None and previous_algorithm == algorithm: + if previous.lower() != checksum.lower(): + raise DownloadError(f"Conflicting {algorithm} manifests for {path!r}") + return + strength = {name: i for i, name in enumerate(CHECKSUM_ALGORITHMS_BY_STRENGTH)} + if previous is None or strength.get(algorithm, 99) < strength.get( + previous_algorithm, 100): + checksums[path] = (algorithm, checksum) + + def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: """ Used by list_remote_files. @@ -645,6 +529,8 @@ def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: A list of (checksum, filename) tuples for each line in the manifest text that has a valid checksum for the given algorithm, allowing unknown algorithms. """ + if any(line.startswith("\\") for line in text.splitlines()): + raise ValueError("GNU escaped manifest filenames are unsupported") # return a list of (checksum, filename) tuples for each line in the manifest text # if the checksum is valid for the given algorithm, allowing unknown algorithms return [ @@ -670,7 +556,8 @@ def _resolve_manifest_path(filename: str, rel_dir: str) -> str: or if the resolved path is not a valid relative path. """ # strip whitespace from the filename to avoid issues with leading/trailing spaces - filename = str(filename).strip() + filename = str(filename) + reject_controls(filename, "Manifest path") # remove leading "./" from the filename to normalize the path while filename.startswith("./"): @@ -727,7 +614,8 @@ def _normalize_index_href(href: str, *, is_directory: bool) -> str: ValueError: If the href is not a valid relative path. """ # use urlsplit to check for scheme, netloc, query, and fragment - parsed = urlsplit(str(href).strip()) + reject_controls(str(href), "Index href") + parsed = urlsplit(str(href)) # if any components are present, raise a ValueError as the href must be relative if ( @@ -740,6 +628,7 @@ def _normalize_index_href(href: str, *, is_directory: bool) -> str: # use unquote to decode any percent-encoded characters in the path decoded_path = unquote(parsed.path) + reject_controls(decoded_path, "Index path") # if the path is a directory, remove any trailing slashes; otherwise, keep it as-is candidate = (decoded_path.rstrip("/") if is_directory else decoded_path) # validate the candidate path to ensure it is a valid relative path @@ -812,26 +701,39 @@ def _normalize_fmt_entries(fmt_entries: list[dict]) -> list[dict]: return normalized -def list_remote_files(base_url: str) \ - -> dict[str, tuple[str | None, str | None]]: - """ - Recursively list every file under a CyVerse WebDAV directory, - collecting checksums along the way. +def list_remote_files(base_url: str, *, _context=None): + """List a CyVerse HTML index (legacy default), or an authorized SSH source.""" + if _context is None: + with OperationContext(RemoteSpec.parse(base_url)) as context: + return _list_source(context) + return _list_source(_context) - Sibling checksum manifests enrich matching files with checksums, but - directories are still traversed because a manifest may be incomplete. - Args: - base_url: A CyVerse WebDAV directory URL. +def _list_source(context): + if context.remote.scheme in {"ssh", "sftp"}: + return _list_ssh_files(context) + return CyverseHtmlIndex(context).list_files() - Returns: - A dictionary mapping relative paths to a tuple of (algorithm, checksum), - where algorithm is the checksum algorithm used (e.g., "md5", "sha256"), - and checksum is the corresponding checksum value. If no checksum is found, - both values will be None. - """ - # normalize the base URL to ensure it is a valid WebDAV directory URL - base_url = _remote_url(base_url, "") + +class CyverseHtmlIndex: + """Adapter for CyVerse's HTML directory-index dialect, not WebDAV PROPFIND.""" + + def __init__(self, context): + self.context = context + + def list_files(self): + return _list_cyverse_files(self.context) + + +def _list_cyverse_files(context): + def fetch_text(path): + return context.read_text(path) + + def optional_text(path): + try: + return fetch_text(path) + except RemoteObjectMissing: + return None # stores all files with their checksums, keyed by relative path, for all algorithms file_checksums: dict[str, tuple[str | None, str | None]] = {} @@ -841,7 +743,7 @@ def list_remote_files(base_url: str) \ # raw index text for directories that have been fetched but not yet processed prefetched_indexes: dict[str, str] = {} # Use a requests session and ThreadPoolExecutor to fetch remote indexes concurrently - with ThreadPoolExecutor(max_workers=_REMOTE_CRAWL_MAX_WORKERS) as crawl_executor: + with context.executor(_REMOTE_CRAWL_MAX_WORKERS) as crawl_executor: # while there are directories to open or prefetched indexes to process while directories_to_open or prefetched_indexes: # if there are no prefetched indexes, fetch a batch of directories to open @@ -856,15 +758,17 @@ def list_remote_files(base_url: str) \ continue # mark directory as visited and add it to the batch for fetching visited_directories.add(candidate) + if len(visited_directories) > context.listing_limit: + raise DownloadError("Remote listing exceeds its entry limit") batch.append(candidate) # if the batch is empty, continue to the next iteration of the loop if not batch: continue # construct the full URLs for the batch of directories to fetch - directory_urls = [_remote_url(base_url, rel_dir) for rel_dir in batch] + directory_urls = batch # fetch the index texts for the batch of directories concurrently - index_texts = crawl_executor.map(_fetch_remote_text, directory_urls) + index_texts = crawl_executor.map(fetch_text, directory_urls) # update the prefetched indexes with the fetched index texts prefetched_indexes.update(zip(batch, index_texts)) @@ -874,6 +778,8 @@ def list_remote_files(base_url: str) \ index_text = prefetched_indexes.pop(rel_dir) # parse the index text to extract entries using regex raw_entries = _INDEX_ROW_RE.findall(index_text) + if (len(file_checksums) + len(raw_entries) > context.listing_limit): + raise DownloadError("Remote listing exceeds its entry limit") entries = [] for entry_type, href in raw_entries: # try to normalize the href to ensure it is a valid relative path @@ -883,7 +789,10 @@ def list_remote_files(base_url: str) \ is_directory=(entry_type == "collection")) # if the href is invalid, skip this entry and continue to the next one except ValueError: - continue + # Directory indexes often include a parent navigation link. + if href in {"../", "./", "/"}: + continue + raise # add the normalized entry to the list of entries for this directory entries.append((entry_type, normalized_href)) @@ -895,9 +804,9 @@ def list_remote_files(base_url: str) \ for keyword in _CHECKSUM_NAME_KEYWORDS)))] # call _remote_url to construct the full URLs for the manifest files manifest_urls = [ - _remote_url(base_url, rel_dir + href) for href in manifest_hrefs] + rel_dir + href for href in manifest_hrefs] # crawl the manifest URLs concurrently to fetch their text contents - manifest_texts = crawl_executor.map(_fetch_optional_remote_text, + manifest_texts = crawl_executor.map(optional_text, manifest_urls) # create a dictionary mapping manifest hrefs to their fetched text contents manifest_text_by_href = { @@ -918,9 +827,9 @@ def record_manifest_matches( try: full_path = _resolve_manifest_path(filename, rel_dir) except ValueError: - continue + raise - file_checksums[full_path] = (algorithm, checksum) + _record_checksum(file_checksums, full_path, algorithm, checksum) files_covered_by_manifest.add(full_path) # Iterate over each entry in the current directory @@ -974,6 +883,35 @@ def record_manifest_matches( return file_checksums +def _list_ssh_files(context): + paths = context.transport.list_entries() + result = {path: (None, None) for path in paths} + for path in paths: + name = Path(path).name + sibling = _SUMS_FILENAME_RE.match(name) + if not sibling and not any(k in name.lower() for k in _CHECKSUM_NAME_KEYWORDS): + continue + text = context.read_text(path) + name_match = _ALGORITHM_IN_NAME_RE.search(name) + algorithm = (sibling.group("algorithm") if sibling else + name_match.group(1).lower() if name_match else "unknown") + matches = _manifest_matches(text, algorithm) + lines = [line for line in text.splitlines() if line.strip()] + if not sibling and (not lines or len(matches) < len(lines) * 0.5): + continue + directory = str(Path(path).parent) + directory = "" if directory == "." else directory + "/" + for checksum, filename in matches: + target = _resolve_manifest_path(filename, directory) + if target in result: + _record_checksum(result, target, algorithm, checksum) + if context.remote_hash: + for path, (_, digest) in result.items(): + if digest is None: + result[path] = context.transport.checksum_small(path) + return result + + def build_repo( repo_path: Path, dataset_name: str, @@ -981,6 +919,53 @@ def build_repo( config_file: Path | str | None = None, remotes: list[dict | str] | dict | str | None = None, overwrite: bool = False, + *, + dataset_url: str | None = None, + dataset_auth: str | None = None, + index_format: str | None = None, + allow_remote_commands: bool = False, + remote_hash: bool = False, +) -> "Repo": + """Build catalogs from an explicit dataset root or the legacy CyVerse root. + + ``dataset_auth`` selects a host-bound local SSH profile independently of + output ``remotes``. Non-default HTTP roots require ``index_format`` set to + ``cyverse-html``. SSH crawling requires ``allow_remote_commands=True``, a + POSIX shell and server Python 3. Server SHA-256 hashing is opt-in through + ``remote_hash`` and bounded to 10 MiB/file, 100 MiB and 60 seconds in total. + """ + dataset_name = validate_path_component(dataset_name, label="dataset name") + base_url = (dataset_url if dataset_url is not None else + _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/")) + source = RemoteSpec.parse(base_url, dataset_auth) + if source.scheme in {"http", "https"}: + if index_format not in {None, "cyverse-html"}: + raise CapabilityError("Supported HTTP index format: cyverse-html") + if dataset_url is not None and index_format is None: + raise CapabilityError( + "Explicit HTTP roots require --index-format cyverse-html") + if remote_hash: + raise CapabilityError("--remote-hash is supported only for SSH sources") + elif not allow_remote_commands: + raise CapabilityError( + "SSH builds require --allow-remote-commands; " + "SFTP-only accounts can download") + elif index_format is not None: + raise CapabilityError("--index-format applies only to HTTP sources") + with OperationContext(source, allow_remote_commands=allow_remote_commands, + remote_hash=remote_hash) as context: + return _build_repo(repo_path, dataset_name, fmt_entries, config_file, + remotes, overwrite, context) + + +def _build_repo( + repo_path: Path, + dataset_name: str, + fmt_entries: list[dict] | None = None, + config_file: Path | str | None = None, + remotes: list[dict | str] | dict | str | None = None, + overwrite: bool = False, + source=None, ) -> "Repo": """ Build a hallmark repository from the given dataset and format entries. @@ -1001,7 +986,7 @@ def build_repo( dataset_name = validate_path_component(dataset_name, label="dataset name") # CyVerse's curated Data Commons datasets live at a predictable URL, # built directly from dataset_name. - base_url = _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/") + base_url = source.remote.url # Determine if remotes were provided by the user remotes_provided = remotes is not None @@ -1068,7 +1053,7 @@ def _ensure_remote_files_listed(): # use time.perf_counter() to measure the time taken to list remote files # perf_counter() is preferred for measuring elapsed time with high resolution list_start = time.perf_counter() - file_checksums = list_remote_files(base_url) + file_checksums = list_remote_files(base_url, _context=source) print(f"Found {len(file_checksums)} files in " f"{time.perf_counter() - list_start:.1f}s") # sort the remote files by their relative paths for consistent ordering @@ -1173,6 +1158,8 @@ def _ensure_remote_files_listed(): # normalize the fmt entries to ensure they are valid and consistent fmt_entries = _normalize_fmt_entries(fmt_entries) + _ensure_remote_files_listed() + # if the repo path already exists and has a config.yml, we will reuse it if reused_from_existing_repo: repo = existing_repo @@ -1263,13 +1250,10 @@ def _ensure_remote_files_listed(): if file_checksums.get(rel_path, (None, None))[1] is None] computed_checksums = {} if paths_needing_checksums: - # construct the full URLs for the unmatched paths that need checksums - urls = [_remote_url(base_url, rel_path) for rel_path in paths_needing_checksums] - # use ThreadPoolExecutor to compute checksums for small remote files in parallel - with ThreadPoolExecutor(max_workers=min(_STATIC_CHECKSUM_MAX_WORKERS, len(urls)) - ) as executor: - # map the _checksum_small_remote_url function over the list of URLs - results = executor.map(_checksum_small_remote_url, urls) + paths = paths_needing_checksums + with source.executor(min(_STATIC_CHECKSUM_MAX_WORKERS, len(paths))) as executor: + # The source owns all static checksum I/O, including SSH capability checks. + results = executor.map(source.transport.checksum_small, paths) # zip the paths needing checksums with their computed results into a dict computed_checksums = dict(zip(paths_needing_checksums, results)) @@ -1347,6 +1331,8 @@ def _ensure_remote_files_listed(): if not remotes: # use a default remote named "origin" pointing to the base_url if none provided remotes = [{"name": "origin"}] + if source.remote.auth is not None: + remotes[0]["auth"] = source.remote.auth # create the final remotes list by adding the base_url to each remote entry final_remotes = [{"url": base_url, **remote} for remote in remotes] repo.state.config["remote"] = final_remotes diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index bde03eb..b5142c0 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -7,6 +7,7 @@ from __future__ import annotations +from copy import deepcopy from pathlib import Path from string import Formatter from typing import Dict, Optional @@ -15,10 +16,14 @@ as_list_of_dicts, coerce_fmt_value, normalize_nonempty_string, validate_path_component, validate_relative_path) +from .transport.base import RemoteSpec, profile_name, reject_controls + + def _update_remote_config( config: dict, remote_name: Optional[str], remote_url: Optional[str], + remote_auth: Optional[str] = None, ) -> None: """ Used by set_config. @@ -81,6 +86,10 @@ def _update_remote_config( selected["name"] = remote_name if remote_url is not None: selected["url"] = remote_url + if remote_auth == "": + selected.pop("auth", None) + elif remote_auth is not None: + selected["auth"] = profile_name(remote_auth) # normalize the remotes configuration to ensure it is a list of dictionaries normalized = normalize_remotes(remotes) # if preserve_list is True, store the normalized list; @@ -156,6 +165,14 @@ def normalize_remotes(remotes) -> list[dict]: else: raise ValueError(f"remote {index} must be a string or dictionary") + if set(entry) - {"name", "url", "auth"}: + raise ValueError("Remote entries support only name, url and auth fields") + if "auth" in entry: + profile_name(entry["auth"]) + if isinstance(entry.get("url"), str): + reject_controls(entry["url"], "Remote URL") + # Names without URLs are allowed while configuring/building a catalog. + RemoteSpec.parse(entry["url"], entry.get("auth")) # for each required key ("name" and "url"), validate that it exists for key in ("name", "url"): # if the key is not present in the entry, skip to the next key @@ -321,6 +338,7 @@ def set_config( remote_name: Optional[str] = None, remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, + remote_auth: Optional[str] = None, ) -> dict: """ Update the repository configuration. @@ -333,13 +351,14 @@ def set_config( fmt (str, optional): Filename format. remote_name (str, optional): Remote repository name. remote_url (str, optional): Remote repository URL. + remote_auth (str, optional): Local SSH profile; empty string removes it. encoding_updates (dict, optional): Encoding values to merge into the existing configuration. Returns: dict: The updated configuration. """ - config = repo.state.config + config = deepcopy(repo.state.config) # raise a ValueError if the provided config is not a dictionary if not isinstance(config, dict): raise ValueError("repository config must be a mapping") @@ -374,7 +393,11 @@ def set_config( remote_name = normalize_nonempty_string(remote_name, label="remote_name") # if a new remote URL is provided, validate that it is a non-empty string if remote_url is not None: + if isinstance(remote_url, str): + reject_controls(remote_url, "Remote URL") remote_url = normalize_nonempty_string(remote_url, label="remote_url") + if remote_auth not in (None, ""): + profile_name(remote_auth) # if a new format string or encoding updates are provided if fmt is not None or encoding_updates is not None: @@ -411,9 +434,10 @@ def set_config( config["data"][0] = updated_spec # if a new remote name or URL is provided, update the remote configuration - if remote_name is not None or remote_url is not None: - _update_remote_config(config, remote_name, remote_url) + if remote_name is not None or remote_url is not None or remote_auth is not None: + _update_remote_config(config, remote_name, remote_url, remote_auth) + repo.state.config = config return config diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py new file mode 100644 index 0000000..fa25a2b --- /dev/null +++ b/mod/hallmark/transport/__init__.py @@ -0,0 +1,110 @@ +"""Explicit internal dispatch and per-invocation resource ownership.""" + +from __future__ import annotations + +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path +from threading import Event, Lock, local + +import requests + +from .auth import resolve_settings +from .base import DownloadError, RemoteSpec, TransferCancelled + + +class OperationContext: + """Own sessions, cancellation and one transport for a download or build.""" + + def __init__( + self, + remote, + output_root=None, + *, + allow_remote_commands=False, + remote_hash=False, + ): + self.remote = remote + self.settings = resolve_settings(remote) + self.output_root = Path(output_root) if output_root is not None else None + self.cancelled = Event() + self.allow_remote_commands = allow_remote_commands + self.remote_hash = remote_hash + self.text_limit = 16 * 1024 * 1024 + self.listing_limit = 100_000 + self.listing_timeout = 60 + self.hash_file_limit = 10 * 1024 * 1024 + self.hash_total_limit = 100 * 1024 * 1024 + self.hash_timeout = 60 + self._local = local() + self._lock = Lock() + self._sessions = [] + self._text_bytes = 0 + self._listing_started = None + if remote.scheme in {"http", "https"}: + from .http import HttpTransport + + self.transport = HttpTransport(self) + else: + from .ssh import SshTransport + + self.transport = SshTransport(self) + + @contextmanager + def executor(self, max_workers): + """Cancel active transport work before waiting for interrupted workers.""" + executor = ThreadPoolExecutor(max_workers=max_workers) + try: + yield executor + except BaseException: + self.cancel() + raise + finally: + executor.shutdown(wait=True, cancel_futures=True) + + def session(self): + if not hasattr(self._local, "session"): + session = requests.Session() + with self._lock: + self._sessions.append(session) + self._local.session = session + return self._local.session + + def read_text(self, path): + with self._lock: + if self._listing_started is None: + self._listing_started = time.monotonic() + if time.monotonic() - self._listing_started > 300: + raise DownloadError("Remote crawl exceeded its five minute budget") + text = self.transport.read_text(path, self.text_limit) + with self._lock: + self._text_bytes += len(text.encode("utf-8")) + if self._text_bytes > 64 * 1024 * 1024: + raise DownloadError("Remote crawl exceeded its 64 MiB text budget") + return text + + def check_cancelled(self): + if self.cancelled.is_set(): + raise TransferCancelled("Download cancelled") + + def cancel(self): + self.cancelled.set() + cancel = getattr(self.transport, "cancel", None) + if cancel is not None: + cancel() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + if exc_type is not None: + self.cancel() + try: + self.transport.close() + finally: + for session in self._sessions: + session.close() + + +__all__ = ["OperationContext", "RemoteSpec"] diff --git a/mod/hallmark/transport/auth.py b/mod/hallmark/transport/auth.py new file mode 100644 index 0000000..c01fd1c --- /dev/null +++ b/mod/hallmark/transport/auth.py @@ -0,0 +1,129 @@ +"""Strict local SSH settings; repository configuration stores references only.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, fields, replace +from pathlib import Path + +from ..helper_functions import load_yaml_file +from .base import ( + RemoteConfigurationError, + profile_name, + reject_controls, + ssh_host, + ssh_user, +) + + +@dataclass(frozen=True) +class SSHSettings: + user: str | None = None + port: int | None = None + identity_file: str | None = None + host_key_policy: str = "strict" + connect_timeout: int = 10 + transfer_timeout: int = 3600 + shutdown_timeout: int = 2 + max_sessions: int = 4 + + +def _settings(values): + allowed = {item.name for item in fields(SSHSettings)} + if not isinstance(values, dict) or set(values) - allowed: + raise RemoteConfigurationError("Unsupported local SSH settings fields") + values = dict(values) + if "user" in values: + ssh_user(values["user"]) + for key in ( + "port", + "connect_timeout", + "transfer_timeout", + "shutdown_timeout", + "max_sessions", + ): + if key not in values: + continue + value = values[key] + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RemoteConfigurationError(f"SSH {key} must be a positive integer") + if key == "port" and value > 65535: + raise RemoteConfigurationError("SSH port must be at most 65535") + if values.get("host_key_policy", "strict") not in {"strict", "accept-new"}: + raise RemoteConfigurationError("host_key_policy must be strict or accept-new") + if "identity_file" in values: + value = values["identity_file"] + if not isinstance(value, str) or not value: + raise RemoteConfigurationError("identity_file must be a local path") + reject_controls(value, "identity_file") + # OpenSSH expands % tokens and environment substitutions inside paths. + if "%" in value or "$" in value: + raise RemoteConfigurationError("identity_file cannot contain expansions") + values["identity_file"] = str(Path(value).expanduser().absolute()) + return values + + +def resolve_settings(remote): + """Resolve settings once, before any worker or connection starts.""" + settings = SSHSettings() + if remote.scheme not in {"ssh", "sftp"}: + return settings + configured = os.environ.get("HALLMARK_AUTH_FILE") + path = ( + Path(configured).expanduser() + if configured + else Path(os.environ.get("XDG_CONFIG_HOME", "~/.config")).expanduser() + / "hallmark" + / "auth.yml" + ) + if not path.exists() and not configured and remote.auth is None: + return replace(settings, user=remote.user, port=remote.port) + try: + document = load_yaml_file(path) + except Exception: + raise RemoteConfigurationError( + "Unable to read local Hallmark auth file" + ) from None + if ( + not isinstance(document, dict) + or set(document) - {"version", "profiles", "defaults"} + or type(document.get("version")) is not int + or document["version"] != 1 + ): + raise RemoteConfigurationError("Auth file must use version: 1") + defaults = _settings(document.get("defaults", {})) + # Endpoint/identity defaults must always be host-bound through a profile. + if set(defaults) & {"user", "port", "identity_file"}: + raise RemoteConfigurationError("Global SSH defaults cannot select an identity") + settings = replace(settings, **defaults) + profiles = document.get("profiles", {}) + if not isinstance(profiles, dict): + raise RemoteConfigurationError("Auth profiles must be a mapping") + validated = {} + for name, profile in profiles.items(): + profile_name(name) + if not isinstance(profile, dict): + raise RemoteConfigurationError("Each auth profile must be a mapping") + hosts = profile.get("hosts") + if ( + not isinstance(hosts, list) + or not hosts + or any(not isinstance(host, str) for host in hosts) + ): + raise RemoteConfigurationError("Each auth profile requires hosts") + hosts = [ssh_host(host).lower() for host in hosts] + values = _settings({k: v for k, v in profile.items() if k != "hosts"}) + validated[name] = (hosts, values) + if remote.auth is not None: + if remote.auth not in validated: + raise RemoteConfigurationError(f"Unresolved auth profile {remote.auth!r}") + hosts, values = validated[remote.auth] + if remote.host.lower() not in hosts: + raise RemoteConfigurationError( + f"Auth profile {remote.auth!r} is not bound to {remote.host}" + ) + settings = replace(settings, **values) + # Profile user/port are defaults; hosts are endpoint constraints. + return replace( + settings, user=remote.user or settings.user, port=remote.port or settings.port + ) diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py new file mode 100644 index 0000000..71bbbe4 --- /dev/null +++ b/mod/hallmark/transport/base.py @@ -0,0 +1,196 @@ +"""Internal transport contracts and validated data-remote locators.""" + +from __future__ import annotations + +import ipaddress +import re +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import quote, unquote, urljoin, urlsplit, urlunsplit + +from ..error import HallmarkError +from ..helper_functions import validate_relative_path + + +class DownloadError(HallmarkError): + """A remote transfer or its verification failed.""" + + +class RemoteConfigurationError(DownloadError, ValueError): + """A remote cannot be used with the selected local policy.""" + + +class RemoteObjectMissing(DownloadError): + """The transport positively identified an absent object.""" + + +class CapabilityError(DownloadError): + """The selected source does not support this operation.""" + + +class TransferCancelled(DownloadError): + """The owning operation cancelled a transfer.""" + + +def reject_controls(value: str, label: str) -> None: + try: + value.encode("utf-8") + except UnicodeError: + raise RemoteConfigurationError(f"{label} must be valid UTF-8") from None + if any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in value): + raise RemoteConfigurationError(f"{label} contains control characters") + + +def literal_path(value) -> Path: + """Validate a literal catalog path without decoding or trimming it.""" + raw = str(value) + reject_controls(raw, "Remote path") + if not raw.strip(): + raise RemoteConfigurationError("Remote path cannot be empty") + try: + return validate_relative_path(raw, label="remote path") + except ValueError as exc: + raise RemoteConfigurationError(str(exc)) from None + + +def profile_name(value: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", value): + raise RemoteConfigurationError("Auth profile must match [A-Za-z0-9_-]{1,64}") + return value + + +def ssh_host(value: str) -> str: + # Aliases are not necessarily DNS names. Limit expansion tokens to characters + # that cannot become shell syntax in a user's ProxyCommand/Match configuration. + if ":" in value: + if "%" in value: + raise RemoteConfigurationError("Scoped IPv6 SSH hosts are unsupported") + try: + ipaddress.IPv6Address(value) + except ValueError: + raise RemoteConfigurationError("Invalid IPv6 SSH host") from None + elif not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", value): + raise RemoteConfigurationError("Invalid SSH host or config alias") + return value + + +def ssh_user(value: str) -> str: + if not isinstance(value, str) or not re.fullmatch( + r"[A-Za-z0-9_][A-Za-z0-9_.-]*", value + ): + raise RemoteConfigurationError("Invalid SSH user") + return value + + +@dataclass(frozen=True) +class RemoteSpec: + """A parsed URL root; catalog paths appended to it are always literal.""" + + url: str = field(repr=False) + scheme: str + host: str + root: str = field(repr=False) + user: str | None = field(default=None, repr=False) + port: int | None = None + auth: str | None = None + + @classmethod + def parse(cls, url: str, auth: str | None = None) -> RemoteSpec: + if not isinstance(url, str) or not url.strip(): + raise RemoteConfigurationError("Remote URL must be a non-empty string") + reject_controls(url, "Remote URL") + url = url.strip() + if auth is not None: + profile_name(auth) + try: + parsed = urlsplit(url) + host, port = parsed.hostname, parsed.port + except ValueError: + raise RemoteConfigurationError("Invalid remote URL host or port") from None + scheme = parsed.scheme.lower() + if scheme not in {"http", "https", "ssh", "sftp"}: + raise RemoteConfigurationError( + "Use an http(s):// or ssh://HOST/absolute/root/ URL; " + "scp shorthand and this scheme are unsupported" + ) + if not host or port == 0 or parsed.netloc.endswith(":"): + raise RemoteConfigurationError("Invalid remote URL host or port") + if scheme in {"ssh", "sftp"}: + ssh_host(host) + if parsed.password is not None: + raise RemoteConfigurationError("SSH URLs cannot contain passwords") + if "?" in url or "#" in url: + raise RemoteConfigurationError( + "SSH URLs cannot contain query or fragment fields; " + "percent-encode filename characters" + ) + if re.search(r"%(?![0-9A-Fa-f]{2})", parsed.path): + raise RemoteConfigurationError("Invalid URL percent escape") + try: + root = unquote(parsed.path, errors="strict") + user = ( + ssh_user(unquote(parsed.username, errors="strict")) + if parsed.username is not None + else None + ) + except UnicodeError: + raise RemoteConfigurationError("SSH URL must encode UTF-8") from None + reject_controls(root, "SSH root") + if ( + not root.startswith("/") + or root.startswith("//") + or ".." in root.split("/") + or "\\" in root + ): + raise RemoteConfigurationError( + "SSH root must be absolute, without traversal or backslashes" + ) + else: + root, user = parsed.path, None + if auth is not None: + raise RemoteConfigurationError( + "Auth profiles currently support SSH/SFTP only; " + "HTTP retains Requests .netrc authentication" + ) + return cls(url, scheme, host, root, user, port, auth) + + def pathname(self, relative_path: str) -> str: + return self.root.rstrip("/") + "/" + literal_path(relative_path).as_posix() + + def file_url(self, relative_path: str) -> str: + path = literal_path(relative_path).as_posix() if relative_path else "" + if relative_path.endswith("/") and path: + path += "/" + if self.scheme in {"http", "https"}: + return urljoin(self.url.rstrip("/") + "/", quote(path, safe="/")) + parsed = urlsplit(self.url) + root = self.root.rstrip("/") + "/" + path + return urlunsplit((self.scheme, parsed.netloc, quote(root, safe="/"), "", "")) + + @property + def display(self) -> str: + """Safe endpoint context without URL credentials, query, or local paths.""" + host = self.host if re.fullmatch(r"[A-Za-z0-9_.:-]+", self.host) else "remote" + return f"{self.scheme}://{host}" + + +class Transport: + """Internal behavior shared by transports; no configuration-driven imports.""" + + def __init__(self, context): + self.context = context + + def prepare(self): + pass + + def fetch(self, relative_path, destination, *, chunk_size=8192): + raise NotImplementedError + + def list_entries(self): + raise CapabilityError("Recursive listing is unsupported by this transport") + + def checksum_small(self, relative_path): + return ("unknown", "unknown") + + def close(self): + pass diff --git a/mod/hallmark/transport/http.py b/mod/hallmark/transport/http.py new file mode 100644 index 0000000..0336b4e --- /dev/null +++ b/mod/hallmark/transport/http.py @@ -0,0 +1,85 @@ +"""Requests transport, including its default .netrc/environment behavior.""" + +from __future__ import annotations + +import hashlib + +import requests + +from ..helper_functions import REMOTE_REQUEST_TIMEOUT +from .base import DownloadError, RemoteObjectMissing, Transport + + +class HttpTransport(Transport): + def _get(self, path): + return self.context.session().get( + getattr(self, "direct_url", None) or self.context.remote.file_url(path), + stream=True, + timeout=REMOTE_REQUEST_TIMEOUT, + ) + + def _error(self, exc): + # Requests exceptions can embed credentials in their URL or response body. + status = getattr(getattr(exc, "response", None), "status_code", None) + cls = RemoteObjectMissing if status == 404 else DownloadError + detail = f"HTTP {status}" if isinstance(status, int) else type(exc).__name__ + return cls(f"Failed to download from {self.context.remote.display}: {detail}") + + def fetch(self, relative_path, destination, *, chunk_size=8192): + try: + with self._get(relative_path) as response: + response.raise_for_status() + with destination.open("wb") as handle: + for chunk in response.iter_content(chunk_size=chunk_size): + self.context.check_cancelled() + if chunk: + handle.write(chunk) + except requests.RequestException as exc: + raise self._error(exc) from None + + def read_text(self, relative_path, limit): + content = bytearray() + try: + with self._get(relative_path) as response: + response.raise_for_status() + for chunk in response.iter_content(chunk_size=8192): + self.context.check_cancelled() + content.extend(chunk) + if len(content) > limit: + raise DownloadError("Remote text exceeds the configured limit") + encoding = response.encoding or "utf-8" + return content.decode(encoding, errors="replace") + except requests.RequestException as exc: + raise self._error(exc) from None + + def checksum_small(self, relative_path): + unknown = ("unknown", "unknown") + try: + with self.context.session().head( + self.context.remote.file_url(relative_path), + timeout=REMOTE_REQUEST_TIMEOUT, + ) as response: + response.raise_for_status() + try: + size = int(response.headers.get("Content-Length")) + except (TypeError, ValueError): + return unknown + if size < 0 or size > self.context.hash_file_limit: + return unknown + digest, received = hashlib.md5(), 0 + with self._get(relative_path) as response: + response.raise_for_status() + for chunk in response.iter_content(chunk_size=8192): + self.context.check_cancelled() + received += len(chunk) + if received > self.context.hash_file_limit: + return unknown + digest.update(chunk) + if received != size: + raise DownloadError("Remote file changed while computing its checksum") + return "md5", digest.hexdigest() + except requests.RequestException as exc: + status = getattr(getattr(exc, "response", None), "status_code", None) + if status in {401, 403}: + raise self._error(exc) from None + return unknown diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py new file mode 100644 index 0000000..d5dfa82 --- /dev/null +++ b/mod/hallmark/transport/ssh.py @@ -0,0 +1,411 @@ +"""Supervised OpenSSH connection sharing and exact-file SFTP transfers.""" + +from __future__ import annotations + +import os +import re +import selectors +import shlex +import shutil +import signal +import subprocess +import tempfile +import time +from pathlib import Path +from threading import BoundedSemaphore, Lock + +from .base import ( + CapabilityError, + DownloadError, + RemoteConfigurationError, + Transport, + literal_path, + reject_controls, +) + + +def batch_argument(path): + """Encode one absolute SFTP argument (command quoting and optional globbing).""" + text = str(path) + reject_controls(text, "SFTP path") + if not text.startswith("/"): + raise RemoteConfigurationError("SFTP operands must be absolute paths") + # Use unquoted escapes: OpenSSH adds glob escapes itself inside quotes, + # so adding backslashes there would instead request literal backslashes. + special = " \\\"'#?[*" # ] has no special meaning outside a bracket pattern. + return "".join("\\" + char if char in special else char for char in text) + + +# Fixed read-only programs. The server needs a POSIX login shell and python3. +# Paths are operands, never interpolated into program text. Symlinks are omitted. +_LIST_SCRIPT = """import os, stat, sys, time +root, max_entries, max_bytes, seconds = sys.argv[1:] +limit, budget = int(max_entries), int(max_bytes) +deadline = time.monotonic() + int(seconds) +count = size = 0 +if not os.path.isdir(root): + raise RuntimeError("Not a directory") +def fail(error): + raise error +for directory, dirs, files in os.walk(root, followlinks=False, onerror=fail): + dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(directory, d))] + for name in files: + path = os.path.join(directory, name) + info = os.lstat(path) + if not stat.S_ISREG(info.st_mode): + continue + relative = os.path.relpath(path, root) + record = (os.fsencode(relative) + b"\\0" + str(info.st_size).encode() + + b"\\0" + str(info.st_mtime_ns).encode() + b"\\0") + count += 1 + size += len(record) + if count > limit or size > budget or time.monotonic() > deadline: + raise RuntimeError("Listing limit exceeded") + sys.stdout.buffer.write(record) +""" + +_HASH_SCRIPT = """import hashlib, os, stat, sys +path, size, mtime = sys.argv[1:] +with open(path, "rb") as handle: + before = os.fstat(handle.fileno()) + if (not stat.S_ISREG(before.st_mode) or before.st_size != int(size) + or before.st_mtime_ns != int(mtime)): + raise RuntimeError("File changed") + digest = hashlib.sha256() + remaining = int(size) + while remaining: + chunk = handle.read(min(65536, remaining)) + if not chunk: + raise RuntimeError("File shortened") + remaining -= len(chunk) + digest.update(chunk) + after = os.fstat(handle.fileno()) + if (handle.read(1) or after.st_size != before.st_size + or after.st_mtime_ns != before.st_mtime_ns): + raise RuntimeError("File changed") +print(digest.hexdigest()) +""" + + +class SshTransport(Transport): + def __init__(self, context, *, ssh=None, sftp=None): + super().__init__(context) + # Injection is internal, for offline process tests; never from repo YAML. + self.ssh = ssh or ["ssh"] + self.sftp = sftp or ["sftp"] + self._injected = ssh is not None or sftp is not None + self._prepare_lock = Lock() + self._process_lock = Lock() + self._processes = set() + self._master = None + self._socket_dir = None + self._socket = None + self._slots = BoundedSemaphore(context.settings.max_sessions) + self._entries = {} + self._hash_lock = Lock() + self._hash_bytes = 0 + self._hash_started = None + + def _options(self, *, master=False): + settings = self.context.settings + policy = "yes" if settings.host_key_policy == "strict" else "accept-new" + options = [ + "BatchMode=yes", + f"StrictHostKeyChecking={policy}", + "ForwardAgent=no", + "ForwardX11=no", + "PermitLocalCommand=no", + "ClearAllForwardings=yes", + "RequestTTY=no", + "RemoteCommand=none", + "ForkAfterAuthentication=no", + "ControlPersist=no", + "ServerAliveInterval=15", + "ServerAliveCountMax=2", + "ConnectionAttempts=1", + f"ConnectTimeout={settings.connect_timeout}", + f"ControlPath={self._socket}", + "ControlMaster=yes" if master else "ControlMaster=no", + ] + if not master: + # If the owned master dies, never authenticate a replacement per file. + options.append("ProxyCommand=false") + if settings.user is not None: + options.append(f"User={settings.user}") + if settings.port is not None: + options.append(f"Port={settings.port}") + args = [part for option in options for part in ("-o", option)] + if settings.identity_file is not None: + args.extend(["-i", settings.identity_file, "-o", "IdentitiesOnly=yes"]) + return args + + def _spawn(self, argv, **kwargs): + with self._process_lock: + self.context.check_cancelled() + try: + process = subprocess.Popen( + argv, shell=False, start_new_session=True, **kwargs + ) + except OSError: + raise DownloadError("Unable to start the OpenSSH client") from None + self._processes.add(process) + return process + + def _stop(self, process): + # Every owned client starts a new process group, including proxy children. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=self.context.settings.shutdown_timeout) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + with self._process_lock: + self._processes.discard(process) + + def _run( + self, + argv, + *, + timeout, + data=b"", + limit=65536, + destination=None, + file_limit=None, + capture_stderr=False, + ): + output = bytearray() + stderr_size = 0 + diagnostics = bytearray() + with tempfile.TemporaryFile() as batch: + batch.write(data) + batch.seek(0) + process = self._spawn( + argv, stdin=batch, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + deadline = time.monotonic() + timeout + try: + with selectors.DefaultSelector() as selector: + for pipe in (process.stdout, process.stderr): + os.set_blocking(pipe.fileno(), False) + selector.register(pipe, selectors.EVENT_READ) + while selector.get_map() or process.poll() is None: + self.context.check_cancelled() + if time.monotonic() >= deadline: + raise DownloadError("SSH operation exceeded its time limit") + if ( + file_limit is not None + and destination.exists() + and destination.stat().st_size > file_limit + ): + raise DownloadError("Remote text exceeds its size limit") + for key, _ in selector.select(timeout=0.05): + chunk = os.read(key.fileobj.fileno(), 65536) + if not chunk: + selector.unregister(key.fileobj) + elif key.fileobj is process.stdout: + output.extend(chunk) + else: + stderr_size += len(chunk) + if capture_stderr: + diagnostics.extend(chunk) + if len(output) > limit or stderr_size > 65536: + raise DownloadError( + "SSH output exceeded its size limit" + ) + self.context.check_cancelled() + if process.wait() != 0: + # stderr is untrusted and may contain echoed credentials/paths. + # SFTP exits do not reliably distinguish absence from auth errors. + raise DownloadError( + f"SSH operation failed for {self.context.remote.display}; " + "check access, file existence, and server capabilities" + ) + return bytes(diagnostics if capture_stderr else output) + finally: + self._stop(process) + process.stdout.close() + process.stderr.close() + + def prepare(self): + with self._prepare_lock: + if self._master is not None: + if self._master.poll() is not None: + raise DownloadError("The owned SSH master disconnected") + return + if os.name != "posix": + raise CapabilityError("SSH transport requires a Linux/macOS client") + if not self._injected: + if not shutil.which("ssh") or not shutil.which("sftp"): + raise CapabilityError("Install OpenSSH ssh and sftp clients (9.6+)") + version = self._run(self.ssh + ["-V"], timeout=5, capture_stderr=True) + match = re.search(rb"OpenSSH_(\d+)\.(\d+)", version) + if not match or tuple(map(int, match.groups())) < (9, 6): + raise CapabilityError("SSH transport requires OpenSSH 9.6 or newer") + self._socket_dir = tempfile.TemporaryDirectory(prefix="hm-", dir="/tmp") + os.chmod(self._socket_dir.name, 0o700) + self._socket = str(Path(self._socket_dir.name) / "s") + if len(os.fsencode(self._socket)) > 100: + raise CapabilityError("SSH control socket path is too long") + try: + self._master = self._spawn( + self.ssh + + self._options(master=True) + + ["-N", "--", self.context.remote.host], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.monotonic() + self.context.settings.connect_timeout + while not Path(self._socket).exists(): + self.context.check_cancelled() + if self._master.poll() is not None: + raise DownloadError( + f"Cannot establish SSH connection to " + f"{self.context.remote.display}; check trusted host key, " + "loaded key/agent, network and OpenSSH settings" + ) + if time.monotonic() > deadline: + raise DownloadError("SSH connection exceeded its time limit") + self.context.cancelled.wait(0.05) + self._run( + self.ssh + + self._options() + + ["-O", "check", "--", self.context.remote.host], + timeout=self.context.settings.connect_timeout, + ) + except BaseException: + self.close() + raise + + def _fetch(self, relative_path, destination, file_limit=None): + remote = batch_argument(self.context.remote.pathname(relative_path)) + local = batch_argument(destination) + batch = f"get {remote} {local}\n".encode("utf-8") + if len(batch) > 8000: + raise RemoteConfigurationError("SFTP command exceeds the client path limit") + self.prepare() + host = self.context.remote.host + if ":" in host: + host = f"[{host}]" + with self._slots: + self._run( + self.sftp + self._options() + ["-b", "-", "--", host], + timeout=self.context.settings.transfer_timeout, + data=batch, + destination=destination, + file_limit=file_limit, + ) + + def fetch(self, relative_path, destination, *, chunk_size=8192): + self._fetch(relative_path, destination) + + def read_text(self, relative_path, limit): + with tempfile.TemporaryDirectory(prefix="hm-text-") as directory: + destination = Path(directory) / "content" + self._fetch(relative_path, destination, file_limit=limit) + if destination.stat().st_size > limit: + raise DownloadError("Remote text exceeds its size limit") + try: + return destination.read_text(encoding="utf-8") + except UnicodeError: + raise DownloadError("Remote manifest must contain UTF-8 text") from None + + def _command(self, script, operands, *, timeout, limit): + if not self.context.allow_remote_commands: + raise CapabilityError( + "SSH builds require --allow-remote-commands, a POSIX shell and " + "server python3; SFTP-only accounts support downloads only" + ) + self.prepare() + command = " ".join( + shlex.quote(value) + for value in ["python3", "-c", script, *map(str, operands)] + ) + with self._slots: + return self._run( + self.ssh + self._options() + ["--", self.context.remote.host, command], + timeout=timeout, + limit=limit, + ) + + def list_entries(self): + context = self.context + data = self._command( + _LIST_SCRIPT, + [ + context.remote.root, + context.listing_limit, + context.text_limit, + context.listing_timeout, + ], + timeout=context.listing_timeout, + limit=context.text_limit, + ) + try: + fields = data.decode("utf-8").split("\0") + if fields.pop() != "" or len(fields) % 3: + raise ValueError + for i in range(0, len(fields), 3): + path = literal_path(fields[i]).as_posix() + size, mtime = int(fields[i + 1]), int(fields[i + 2]) + if size < 0 or path in self._entries: + raise ValueError + self._entries[path] = (size, mtime) + if len(self._entries) > context.listing_limit: + raise ValueError + except (UnicodeError, ValueError): + raise DownloadError("Invalid or oversized SSH listing") from None + return list(self._entries) + + def checksum_small(self, relative_path): + if not self.context.remote_hash: + return "unknown", "unknown" + size, mtime = self._entries.get(relative_path, (-1, 0)) + with self._hash_lock: + if ( + size < 0 + or size > self.context.hash_file_limit + or self._hash_bytes + size > self.context.hash_total_limit + ): + return "unknown", "unknown" + if self._hash_started is None: + self._hash_started = time.monotonic() + remaining = self.context.hash_timeout - ( + time.monotonic() - self._hash_started + ) + if remaining <= 0: + return "unknown", "unknown" + self._hash_bytes += size + result = ( + self._command( + _HASH_SCRIPT, + [self.context.remote.pathname(relative_path), size, mtime], + timeout=remaining, + limit=128, + ) + .decode("ascii", errors="replace") + .strip() + ) + if not re.fullmatch(r"[0-9a-f]{64}", result): + raise DownloadError("Invalid remote SHA-256 result") + return "sha256", result + + def cancel(self): + with self._process_lock: + processes = list(self._processes) + for process in processes: + self._stop(process) + + def close(self): + self.cancel() + self._master = None + if self._socket_dir is not None: + self._socket_dir.cleanup() + self._socket_dir = None diff --git a/pyproject.toml b/pyproject.toml index 20a137d..50c77bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,3 +55,8 @@ exclude = ["doc", "demo"] [tool.pytest.ini_options] testpaths = ["test"] +addopts = "--strict-markers" +markers = [ + "ssh_client: exercise the actual local OpenSSH SFTP command parser", + "ssh_integration: disposable loopback sshd (HALLMARK_RUN_SSH_TESTS=1 required)", +] diff --git a/test/mock_server.py b/test/mock_server.py index 7de10d6..5e22afd 100644 --- a/test/mock_server.py +++ b/test/mock_server.py @@ -33,7 +33,10 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): return False - def get(self, url, timeout=None): + def close(self): + pass + + def get(self, url, timeout=None, stream=False): return self.fake_get(url, timeout=timeout) def head(self, url, timeout=None): @@ -94,11 +97,15 @@ def fake_get(self, url, timeout=None): resp.raise_for_status = lambda: None if url in self._html_by_url: resp.text = self._html_by_url[url] + resp.content = resp.text.encode() elif url in self._text_by_url: resp.text = self._text_by_url[url] resp.content = self._content_by_url[url] else: raise AssertionError(f"MockServer: no registered response for GET {url}") + resp.encoding = "utf-8" + resp.iter_content = lambda chunk_size: iter([resp.content]) + resp.__enter__.return_value = resp return resp def fake_head(self, url, timeout=None): @@ -118,4 +125,5 @@ def fake_head(self, url, timeout=None): resp.headers = {"Content-Length": str(len(self._content_by_url[url]))} else: raise AssertionError(f"MockServer: no registered response for HEAD {url}") + resp.__enter__.return_value = resp return resp diff --git a/test/test_downloader.py b/test/test_downloader.py index a6f9fb5..a4af524 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -307,14 +307,9 @@ def test_safe_remote_path_rejects_unsafe_and_reserved_paths(unsafe_path): _safe_remote_path(unsafe_path) -def test_safe_remote_path_normalizes_valid_relative_path(): - """ - Test that _safe_remote_path correctly normalizes a valid relative path by stripping - leading/trailing whitespace and converting it to a Path object. - """ - assert _safe_remote_path(" nested/file.txt ") == Path("nested/file.txt"), \ - f"Expected normalized path 'nested/file.txt', \ - got {_safe_remote_path(' nested/file.txt ')}" +def test_safe_remote_path_preserves_literal_whitespace(): + """Catalog filenames are literal; significant spaces must survive download.""" + assert _safe_remote_path(" nested/file.txt ") == Path(" nested/file.txt ") @pytest.mark.parametrize( @@ -806,6 +801,8 @@ class FakeSession: def __init__(self): self.urls = [] sessions.append(self) + def close(self): + self.closed = True def get(self, url, **kwargs): self.urls.append(url) return _Response([b"contents"]) @@ -822,6 +819,7 @@ def get(self, url, **kwargs): assert result["succeeded"] == 2, \ f"Expected 2 successful downloads, but got {result['succeeded']}" + assert sessions[0].closed assert len(sessions) == 1, f"Expected a single session to be reused for both \ downloads, but got {len(sessions)}" assert sessions[0].urls == [ diff --git a/test/test_repo_builder.py b/test/test_repo_builder.py index f72852f..ae392ac 100755 --- a/test/test_repo_builder.py +++ b/test/test_repo_builder.py @@ -43,7 +43,7 @@ def _served(server: MockServer): Returns: A context manager that patches requests.get and requests.head. """ - return patch("hallmark.repo_builder.requests.Session", return_value=server) + return patch("hallmark.transport.requests.Session", return_value=server) # helper function to monkeypatch list_remote_files for testing build_repo @@ -58,7 +58,7 @@ def _inventory(monkeypatch, files): A list of base_url values passed to the fake list_remote_files function. """ calls = [] - def fake_list(base_url): + def fake_list(base_url, **kwargs): """create a fake list_remote_files list""" calls.append(base_url) return dict(files) @@ -333,7 +333,7 @@ def test_list_remote_files_fetches_each_directory_once(): server.add_directory("nested/", [("data-object", "file.dat")]) requested_urls = [] original_get = server.get - def recording_get(url, timeout=None): + def recording_get(url, timeout=None, **kwargs): """A wrapper around the original server.get that records requested URLs.""" requested_urls.append(url) return original_get(url, timeout=timeout) @@ -356,11 +356,13 @@ def test_list_remote_files_unavailable_manifest_does_not_hide_files(): server.add_directory( "", [("data-object", "data.tar"), ("data-object", "checksums.txt")]) original_get = server.get - def unavailable_manifest_get(url, timeout=None): + def unavailable_manifest_get(url, timeout=None, **kwargs): """A wrapper around the original server.get that simulates an unavailable manifest.""" if url.endswith("/checksums.txt"): - raise requests.HTTPError("manifest unavailable") + response = requests.Response() + response.status_code = 404 + raise requests.HTTPError("manifest unavailable", response=response) return original_get(url, timeout=timeout) server.get = unavailable_manifest_get with _served(server): @@ -820,6 +822,7 @@ def huge_head(url, timeout=None): resp = MagicMock() resp.raise_for_status = lambda: None resp.headers = {"Content-Length": str(5 * 1024 * 1024 * 1024)} + resp.__enter__.return_value = resp return resp server.fake_head = huge_head def refuse_get_body(url, timeout=None): @@ -1473,7 +1476,7 @@ def test_build_repo_replaces_destination_when_overwrite_is_explicit( sentinel.write_text("old data\n", encoding="utf-8") monkeypatch.setattr( "hallmark.repo_builder.list_remote_files", - lambda _base_url: {}) + lambda _base_url, **kwargs: {}) repo = build_repo( repo_path=destination, dataset_name="EHTC_TEST", @@ -1602,6 +1605,9 @@ def test_build_repo_continues_when_static_checksum_request_fails(monkeypatch, tm """ _inventory(monkeypatch, {"README.md": (None, None)}) class FailingChecksumSession: + def close(self): + pass + """ A mock requests.Session that simulates a failure to retrieve the checksum for a static file, by raising an HTTPError when the head() method is called. @@ -1617,7 +1623,7 @@ def __exit__(self, exc_type, exc_value, traceback): def head(self, *args, **kwargs): """Simulate a failure to retrieve the checksum for a static file.""" raise requests.HTTPError("HEAD unavailable") - monkeypatch.setattr("hallmark.repo_builder.requests.Session", + monkeypatch.setattr("hallmark.transport.requests.Session", FailingChecksumSession) repo = build_repo(tmp_path / "repo.hm", "EHTC_TEST", fmt_entries=[]) diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py new file mode 100644 index 0000000..c977972 --- /dev/null +++ b/test/test_ssh_integration.py @@ -0,0 +1,462 @@ +"""Disposable, loopback-only OpenSSH tests. No external hosts or persistent keys.""" + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import getpass +import hashlib +import os +from pathlib import Path +import shutil +import socket +import subprocess +import time +from urllib.parse import quote + +from click.testing import CliRunner +import pytest +import yaml + +from hallmark import Repo +from hallmark.cli import hallmark +from hallmark.downloader import download_remote_data, select_download_files +from hallmark.repo_builder import build_repo +from hallmark.transport import OperationContext, RemoteSpec +from hallmark.transport.base import DownloadError, TransferCancelled +from hallmark.transport.ssh import SshTransport + +pytestmark = pytest.mark.ssh_integration + + +@pytest.fixture +def ssh_server(tmp_path, monkeypatch, request): + if os.environ.get("HALLMARK_RUN_SSH_TESTS") != "1": + pytest.skip("Set HALLMARK_RUN_SSH_TESTS=1 for disposable loopback SSH tests") + sshd = shutil.which("sshd") or "/usr/sbin/sshd" + if not Path(sshd).exists(): + pytest.fail("Opted-in SSH tests require openssh-server") + for name in ("host", "client", "second", "untrusted"): + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / name)], + check=True, + ) + authorized = tmp_path / "authorized" + authorized.write_text( + (tmp_path / "client.pub").read_text() + (tmp_path / "second.pub").read_text() + ) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + config = tmp_path / "sshd_config" + config.write_text(f"""ListenAddress 127.0.0.1 +Port {port} +HostKey {tmp_path}/host +PidFile {tmp_path}/sshd.pid +AuthorizedKeysFile {authorized} +StrictModes no +UsePAM no +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin prohibit-password +MaxSessions 4 +Subsystem sftp internal-sftp +""") + mode = getattr(request, "param", None) + if mode == "sftp-only": + config.write_text(config.read_text() + "ForceCommand internal-sftp\n") + elif mode == "no-sftp": + config.write_text( + config.read_text().replace("Subsystem sftp internal-sftp\n", "") + ) + elif mode == "max-one": + config.write_text(config.read_text().replace("MaxSessions 4", "MaxSessions 1")) + log = (tmp_path / "sshd.log").open("wb") + daemon = subprocess.Popen( + [sshd, "-D", "-e", "-f", str(config)], stderr=log, start_new_session=True + ) + known = tmp_path / "known_hosts" + known.write_text(f"[127.0.0.1]:{port} " + (tmp_path / "host.pub").read_text()) + client_config = tmp_path / "ssh_config" + client_config.write_text(f"""Host hm-test + HostName 127.0.0.1 +Host * + User {getpass.getuser()} + Port {port} + IdentityFile {tmp_path}/client + IdentityAgent none + IdentitiesOnly yes + UserKnownHostsFile {known} + GlobalKnownHostsFile /dev/null +""") + original = SshTransport._options + monkeypatch.setattr( + SshTransport, + "_options", + lambda self, **kw: ["-F", str(client_config)] + original(self, **kw), + ) + monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + root = tmp_path / "export" + root.mkdir() + try: + deadline = time.monotonic() + 5 + while True: + if daemon.poll() is not None: + log.flush() + pytest.fail((tmp_path / "sshd.log").read_text()) + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.1): + break + except OSError: + if time.monotonic() >= deadline: + pytest.fail("Local SSH test server did not start") + time.sleep(0.05) + yield { + "root": root, + "port": port, + "known": known, + "base": tmp_path, + "config": client_config, + "daemon": daemon, + "url": "ssh://hm-test" + quote(str(root), safe="/") + "/", + } + finally: + daemon.terminate() + daemon.wait(timeout=5) + log.close() + + +@pytest.mark.parametrize("scheme", ["ssh", "sftp"]) +def test_parallel_exact_files_and_master_cleanup(ssh_server, tmp_path, scheme): + root = ssh_server["root"] + names = [ + "a b#?%+ü.h5", + "quotes'\"[x]*?.h5", + "-leading.h5", + " tail ", + "literal%20.h5", + "nested/data.bin", + ] + for name in names: + path = root / name + path.parent.mkdir(exist_ok=True) + path.write_text(name) + url = ssh_server["url"].replace("ssh:", scheme + ":") + with OperationContext(RemoteSpec.parse(url)) as context: + transport = context.transport + transport.prepare() + master = transport._master + control = transport._socket + assert Path(control).stat().st_mode & 0o077 == 0 + output = tmp_path / "output" + output.mkdir() + with ThreadPoolExecutor(4) as pool: + tasks = [ + pool.submit(transport.fetch, name, output / str(i)) + for i, name in enumerate(names) + ] + for task in tasks: + task.result() + assert transport._master is master + assert len(transport._processes) == 1 + for i, name in enumerate(names): + assert (output / str(i)).read_text() == name + assert master.poll() is not None + assert not Path(control).exists() + + +def test_explicit_endpoint_and_missing_file(ssh_server, tmp_path): + server = ssh_server + url = f"sftp://{getpass.getuser()}@127.0.0.1:{server['port']}{server['root']}/" + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=url) + output = repo.worktree / "missing" + output.write_bytes(b"keep") + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("missing"), None)] + ) + assert result["failed"] == 1 + assert output.read_bytes() == b"keep" + assert not list(Path(repo.worktree).glob("*.part")) + + +@pytest.mark.parametrize("trust", ["unknown", "changed", "no-key"]) +def test_host_trust_and_auth_fail_preflight(ssh_server, tmp_path, trust): + server = ssh_server + if trust == "unknown": + server["known"].write_text("") + elif trust == "changed": + server["known"].write_text( + f"[127.0.0.1]:{server['port']} " + + (server["base"] / "untrusted.pub").read_text() + ) + else: + # Both keys are ephemeral; the replacement is not authorized on the server. + config = server["config"].read_text().replace("/client\n", "/untrusted\n") + server["config"].write_text(config) + with OperationContext(RemoteSpec.parse(server["url"])) as context: + with pytest.raises(DownloadError, match="Cannot establish"): + context.transport.prepare() + assert not context.transport._processes + assert not (tmp_path / "download").exists() + + +def test_simultaneous_profiles_have_separate_masters(ssh_server, monkeypatch): + server = ssh_server + (server["root"] / "item").write_bytes(b"data") + auth = server["base"] / "auth.yml" + auth.write_text( + yaml.safe_dump( + { + "version": 1, + "profiles": { + name: { + "hosts": ["hm-test"], + "identity_file": str(server["base"] / key), + } + for name, key in [("one", "client"), ("two", "second")] + }, + } + ) + ) + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(auth)) + with OperationContext(RemoteSpec.parse(server["url"], "one")) as one: + with OperationContext(RemoteSpec.parse(server["url"], "two")) as two: + with ThreadPoolExecutor(2) as pool: + list(pool.map(lambda context: context.transport.prepare(), [one, two])) + assert one.transport._socket != two.transport._socket + assert one.transport._master.pid != two.transport._master.pid + first = one.transport._master + second = two.transport._master + assert second.poll() is not None + assert first.poll() is None + one.transport.fetch("item", server["base"] / "copy") + assert first.poll() is not None + + +def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): + root = ssh_server["root"] + (root / "nested").mkdir() + (root / "nested/item_1.dat").write_bytes(b"science") + (root / "README.md").write_bytes(b"notes") + (root / "bad_2.dat").write_bytes(b"changed") + strong = hashlib.sha256(b"science").hexdigest() + (root / "export.sha256sums").write_text( + strong + " nested/item_1.dat\n" + "a" * 64 + " bad_2.dat\n" + ) + repo = build_repo( + tmp_path / "catalog", + "lab", + [ + {"fmt": "nested/item_{i}.dat", "db": "data.tsv"}, + {"fmt": "bad_{i}.dat", "db": "bad.tsv"}, + ], + dataset_url=ssh_server["url"], + allow_remote_commands=True, + remote_hash=True, + ) + assert repo.state.config["remote"] == [{"name": "origin", "url": ssh_server["url"]}] + static = next( + entry for entry in repo.state.config["data"] if entry.get("file") == "README.md" + ) + assert static["sha256"] == hashlib.sha256(b"notes").hexdigest() + destination = tmp_path / "downloads" + files = select_download_files(repo, all_files=True) + result = download_remote_data(repo, destination, selected_files=files) + assert result["succeeded"] == 3 + assert result["failed"] == 1 + assert (destination / "nested/item_1.dat").read_bytes() == b"science" + assert not (destination / "bad_2.dat").exists() + assert not list(destination.rglob("*.part")) + + # Single-catalog public Python clone and CLI clone both route SSH downloads. + single = build_repo( + tmp_path / "single", + "lab", + [{"fmt": "nested/item_{i}.dat", "db": "data.tsv"}], + dataset_url=ssh_server["url"], + allow_remote_commands=True, + ) + # Remove the intentionally incorrect static entry before cloning. + single.state.config["data"] = [ + entry + for entry in single.state.config["data"] + if entry.get("file") != "bad_2.dat" + ] + single.dothm.dump(single.state) + single.dothm.index.add(["config.yml"]) + single.dothm.index.commit("Remove deliberately invalid test entry") + clone = Repo.clone(str(single.dothm.path), tmp_path / "python-clone") + assert (clone.worktree / "nested/item_1.dat").read_bytes() == b"science" + result = CliRunner().invoke( + hallmark, + ["clone", str(single.dothm.path), str(tmp_path / "cli-clone"), "--yes"], + ) + assert result.exit_code == 0, result.output + assert (tmp_path / "cli-clone/nested/item_1.dat").read_bytes() == b"science" + + +def test_real_cancel_transfer_and_preserve_other_master(ssh_server, tmp_path): + server = ssh_server + (server["root"] / "large").write_bytes(b"x" * 128 * 1024) + with OperationContext(RemoteSpec.parse(server["url"])) as survivor: + survivor.transport.prepare() + with OperationContext(RemoteSpec.parse(server["url"])) as context: + context.settings = replace(context.settings, shutdown_timeout=1) + context.transport.prepare() + # Throttle only this test's SFTP process to make interruption deterministic. + context.transport.sftp = ["sftp", "-l", "8"] + target = tmp_path / "partial" + with ThreadPoolExecutor(1) as pool: + task = pool.submit(context.transport.fetch, "large", target) + deadline = time.monotonic() + 5 + while ( + len(context.transport._processes) < 2 + and time.monotonic() < deadline + ): + time.sleep(0.01) + context.cancel() + with pytest.raises(TransferCancelled): + task.result(timeout=5) + assert not context.transport._processes + assert survivor.transport._master.poll() is None + + +@pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) +def test_sftp_only_fetches_but_cannot_build(ssh_server, tmp_path): + (ssh_server["root"] / "item").write_bytes(b"data") + with OperationContext( + RemoteSpec.parse(ssh_server["url"]), allow_remote_commands=True + ) as context: + context.transport.fetch("item", tmp_path / "copy") + assert (tmp_path / "copy").read_bytes() == b"data" + with pytest.raises(DownloadError): + context.transport.list_entries() + + +@pytest.mark.parametrize("ssh_server", ["no-sftp"], indirect=True) +def test_disabled_subsystem_fails(ssh_server, tmp_path): + (ssh_server["root"] / "item").write_bytes(b"data") + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + with pytest.raises(DownloadError): + context.transport.fetch("item", tmp_path / "copy") + assert not (tmp_path / "copy").exists() + + +@pytest.mark.parametrize("ssh_server", ["max-one"], indirect=True) +def test_local_session_limit(ssh_server, tmp_path, monkeypatch): + for i in range(4): + (ssh_server["root"] / str(i)).write_bytes(b"data") + auth = tmp_path / "auth.yml" + auth.write_text("version: 1\ndefaults:\n max_sessions: 1\n") + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(auth)) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=ssh_server["url"]) + result = download_remote_data( + repo, + repo.worktree, + max_workers=4, + selected_files=[(Path(str(i)), None) for i in range(4)], + ) + assert result["succeeded"] == 4 + assert result["failed"] == 0 + + +def test_profile_source_recording_and_explicit_output( + ssh_server, tmp_path, monkeypatch +): + auth = tmp_path / "auth.yml" + auth.write_text("version: 1\nprofiles:\n lab:\n hosts: [hm-test]\n") + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(auth)) + repo = build_repo( + tmp_path / "catalog", + "label", + [], + dataset_url=ssh_server["url"], + dataset_auth="lab", + allow_remote_commands=True, + ) + assert repo.state.config["remote"][0]["auth"] == "lab" + explicit = build_repo( + tmp_path / "explicit", + "label", + [], + remotes=[{"name": "mirror", "url": "https://example.test/data"}], + dataset_url=ssh_server["url"], + dataset_auth="lab", + allow_remote_commands=True, + ) + assert explicit.state.config["remote"] == [ + {"name": "mirror", "url": "https://example.test/data"} + ] + + +def test_ssh_listing_omits_symlinks_and_rejects_controls(ssh_server, tmp_path): + root = ssh_server["root"] + (root / "item").write_bytes(b"data") + (root / "link").symlink_to(root / "item") + with OperationContext( + RemoteSpec.parse(ssh_server["url"]), allow_remote_commands=True + ) as context: + assert context.transport.list_entries() == ["item"] + (root / "bad\nname").write_bytes(b"data") + with OperationContext( + RemoteSpec.parse(ssh_server["url"]), allow_remote_commands=True + ) as context: + with pytest.raises(DownloadError, match="listing"): + context.transport.list_entries() + + +def test_permission_denied_preserves_existing_file(ssh_server, tmp_path): + source = ssh_server["root"] / "private" + source.write_bytes(b"unreadable") + source.chmod(0) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=ssh_server["url"]) + destination = repo.worktree / "private" + destination.write_bytes(b"original") + try: + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("private"), None)] + ) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + finally: + source.chmod(0o600) + + +def test_disconnected_master_cleans_partial_transfer(ssh_server, tmp_path, monkeypatch): + (ssh_server["root"] / "large").write_bytes(b"x" * 128 * 1024) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=ssh_server["url"]) + destination = repo.worktree / "large" + destination.write_bytes(b"original") + transports = [] + prepare = SshTransport.prepare + + def slow_prepare(self): + prepare(self) + self.sftp = ["sftp", "-l", "8"] + if self not in transports: + transports.append(self) + + monkeypatch.setattr(SshTransport, "prepare", slow_prepare) + with ThreadPoolExecutor(1) as pool: + task = pool.submit( + download_remote_data, + repo, + repo.worktree, + selected_files=[(Path("large"), None)], + ) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if transports and len(transports[0]._processes) > 1: + break + time.sleep(0.01) + assert transports and len(transports[0]._processes) > 1 + transports[0]._master.terminate() + result = task.result(timeout=5) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + assert not transports[0]._processes diff --git a/test/test_transport.py b/test/test_transport.py new file mode 100644 index 0000000..c12e611 --- /dev/null +++ b/test/test_transport.py @@ -0,0 +1,623 @@ +"""Transport boundaries, local policy, HTTP compatibility and process ownership.""" + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import hashlib +from pathlib import Path +import subprocess +import sys +import time +import traceback +from urllib.parse import quote + +import pytest +import requests +import yaml + +from hallmark import Repo +from hallmark.downloader import _download_file, download_remote_data +from hallmark.repo_builder import _manifest_matches, build_repo +from hallmark.repo_config import normalize_remotes +from hallmark.transport import OperationContext, RemoteSpec +from hallmark.transport.auth import resolve_settings +from hallmark.transport.base import ( + CapabilityError, + DownloadError, + RemoteConfigurationError, + TransferCancelled, + literal_path, +) +from hallmark.transport.ssh import SshTransport, batch_argument + + +@pytest.fixture(autouse=True) +def isolated_auth(monkeypatch, tmp_path): + monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config")) + + +@pytest.mark.parametrize("scheme", ["ssh", "sftp"]) +@pytest.mark.parametrize( + "name", + [ + "has space#tag.h5", + "a?b%+ü.h5", + "quotes'\"[]*.h5", + "-weird.fits", + "literal%20.h5", + " leading ", + ], +) +def test_literal_url_round_trip(scheme, name): + remote = RemoteSpec.parse(f"{scheme}://user@campus:2222/srv/a%20b/") + url = remote.file_url(name) + assert RemoteSpec.parse(url).root == "/srv/a b/" + name + assert remote.pathname(name) == "/srv/a b/" + name + assert quote(name, safe="/") in url + + +@pytest.mark.parametrize( + "url", + [ + "campus:/data", + "file:///etc/passwd", + "ssh:///data", + "ssh://-o/data", + "ssh://u:secret@campus/data", + "ssh://campus/data?secret", + "ssh://campus/data#x", + "ssh://campus/../data", + "ssh://campus/%2e%2e/data", + "ssh://campus", + "ssh://campus:0/data", + "ssh://campus:65536/data", + "ssh://campus:/data", + "ssh://campus/%0adata", + "ssh://campus/d\nata", + "\tssh://campus/data", + "ssh://u%24%28id%29@campus/data", + "ssh://campus$(id)/data", + "ssh://[::1/data", + "ssh://campus/%xx", + "ssh://campus/%ff", + "ssh://campus//data", + "ssh://campus/data\\file", + ], +) +def test_bad_remote_fails_without_echoing_secrets(url): + with pytest.raises(RemoteConfigurationError) as error: + RemoteSpec.parse(url) + assert "secret" not in str(error.value) + + +@pytest.mark.parametrize("host", ["campus_alias", "a.b-c", "[::1]", "[2001:db8::1]"]) +def test_hosts_and_aliases(host): + assert RemoteSpec.parse(f"ssh://{host}/").host == host.strip("[]") + + +@pytest.mark.parametrize("path", ["bad\nget /secret /tmp/leak", "a\tb", "a\rb", "a\0b"]) +def test_controls_rejected_at_all_path_boundaries(path, tmp_path): + with pytest.raises(RemoteConfigurationError): + literal_path(path) + with pytest.raises(RemoteConfigurationError): + batch_argument("/" + path) + with pytest.raises(RemoteConfigurationError): + _download_file("ssh://campus/" + path, tmp_path / "out") + + +def write_auth(monkeypatch, tmp_path, profiles, defaults=None): + path = tmp_path / "auth.yml" + path.write_text( + yaml.safe_dump({"version": 1, "profiles": profiles, "defaults": defaults or {}}) + ) + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(path)) + return path + + +def test_profile_precedence_binding_and_isolation(monkeypatch, tmp_path): + write_auth( + monkeypatch, + tmp_path, + { + "one": { + "hosts": ["campus"], + "user": "alice", + "port": 2200, + "identity_file": "~/key-one", + }, + "two": { + "hosts": ["campus"], + "user": "bob", + "port": 2201, + "identity_file": "~/key-two", + }, + }, + {"transfer_timeout": 77}, + ) + first = resolve_settings(RemoteSpec.parse("ssh://carol@campus:2222/data", "one")) + second = resolve_settings(RemoteSpec.parse("sftp://campus/data", "two")) + assert (first.user, first.port, first.transfer_timeout) == ("carol", 2222, 77) + assert (second.user, second.port) == ("bob", 2201) + assert first.identity_file != second.identity_file + with pytest.raises(RemoteConfigurationError, match="not bound"): + resolve_settings(RemoteSpec.parse("ssh://other/data", "one")) + with pytest.raises(RemoteConfigurationError, match="Unresolved"): + resolve_settings(RemoteSpec.parse("ssh://campus/data", "missing")) + + +@pytest.mark.parametrize( + "profile", + [ + {"hosts": ["campus"], "password": "secret"}, + {"hosts": ["campus"], "ssh_options": {"ProxyCommand": "anything"}}, + {"hosts": ["campus"], "port": True}, + {"hosts": ["campus"], "host_key_policy": "no"}, + {"hosts": ["campus"], "identity_file": "%d/key"}, + {"hosts": []}, + {"hosts": ["$(id)"]}, + ], +) +def test_invalid_profiles_fail_preflight(monkeypatch, tmp_path, profile): + write_auth(monkeypatch, tmp_path, {"campus": profile}) + with pytest.raises(RemoteConfigurationError): + OperationContext(RemoteSpec.parse("ssh://campus/data", "campus")) + + +def test_profile_xdg_and_removal(monkeypatch, tmp_path): + folder = tmp_path / "config" / "hallmark" + folder.mkdir(parents=True) + (folder / "auth.yml").write_text( + "version: 1\nprofiles:\n lab:\n hosts: [campus]\n user: alice\n" + ) + assert ( + resolve_settings(RemoteSpec.parse("ssh://campus/data", "lab")).user == "alice" + ) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://campus/data", remote_auth="lab") + assert repo.state.config["remote"]["auth"] == "lab" + repo.set_config(remote_auth="") + assert "auth" not in Repo(repo.worktree).state.config["remote"] + before = repo.state.config.copy() + with pytest.raises(ValueError): + repo.set_config(remote_auth="invalid.name", fmt="changed-{x}") + assert repo.state.config == before + + +@pytest.mark.parametrize( + "entry", + [ + {"url": "ssh://campus/data", "token": "secret"}, + {"url": "ssh://campus/data", "auth": 3}, + ], +) +def test_remote_schema(entry): + with pytest.raises(ValueError): + normalize_remotes(entry) + + +def test_netrc_is_preserved(monkeypatch, tmp_path): + netrc = tmp_path / "netrc" + netrc.write_text("machine example.test login scientist password test-value\n") + monkeypatch.setenv("NETRC", str(netrc)) + with OperationContext(RemoteSpec.parse("https://example.test/data")) as context: + prepared = context.session().prepare_request( + requests.Request("GET", "https://example.test/data/file") + ) + assert prepared.headers["Authorization"].startswith("Basic ") + assert context.session() is context.session() + + +def test_http_error_redacts_exception_and_cause(monkeypatch, tmp_path): + def fail(*args, **kwargs): + raise requests.ConnectionError("https://user:secret@example.test/?token=secret") + + monkeypatch.setattr(requests, "get", fail) + url = "https://user:secret@example.test/data?token=secret" + with pytest.raises(DownloadError) as error: + _download_file(url, tmp_path / "out") + assert "secret" not in "".join(traceback.format_exception(error.value)) + assert "secret" not in repr(error.value) + + +def test_symlink_swap_after_planning(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="https://example.test/") + folder = repo.worktree / "sub" + folder.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + + def swap(self): + folder.rmdir() + folder.symlink_to(outside, target_is_directory=True) + + monkeypatch.setattr("hallmark.transport.http.HttpTransport.prepare", swap) + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("sub/data"), None)] + ) + assert result["failed"] == 1 + assert list(outside.iterdir()) == [] + + +def test_builder_requires_explicit_capabilities(tmp_path): + with pytest.raises(CapabilityError, match="index-format"): + build_repo( + tmp_path / "repo", "lab", [], dataset_url="https://example.test/data" + ) + with pytest.raises(CapabilityError, match="allow-remote-commands"): + build_repo(tmp_path / "repo", "lab", [], dataset_url="ssh://campus/data") + assert not (tmp_path / "repo").exists() + + +def test_manifest_preserves_whitespace_and_percent(): + text = "a" * 64 + " leading%20 name \n" + assert _manifest_matches(text, "sha256") == [("a" * 64, " leading%20 name ")] + with pytest.raises(ValueError, match="escaped"): + _manifest_matches("\\" + "a" * 64 + " line\\nbreak\n", "sha256") + + +@pytest.mark.ssh_client +@pytest.mark.parametrize( + "name", + ["a b#?%+ü.h5", "quotes'\"[x]*?.h5", "-leading.h5", " tail ", "literal%20.h5"], +) +def test_real_sftp_parser_without_network(tmp_path, name): + import shutil + + server = next( + ( + p + for p in ["/usr/lib/openssh/sftp-server", "/usr/libexec/sftp-server"] + if Path(p).exists() + ), + None, + ) + if not server or not shutil.which("sftp"): + pytest.skip("OpenSSH client and local SFTP server binary required") + source = tmp_path / name + source.write_text(name) + output = tmp_path / ("output-" + name) + # A glob collision must never produce two matches or replace another file. + (tmp_path / "a b#X%+ü.h5").write_text("decoy") + batch = f"get {batch_argument(source)} {batch_argument(output)}\n" + result = subprocess.run( + ["sftp", "-D", server, "-b", "-"], + input=batch.encode(), + capture_output=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + assert output.read_bytes() == source.read_bytes() + + +@pytest.fixture +def fake_process(tmp_path): + script = tmp_path / "fake.py" + script.write_text("""import os, signal, subprocess, sys, time +mode = sys.argv[1] +if mode == "fail": + os.write(2, b"secret-invalid-utf8-\\xff") + sys.exit(1) +elif mode == "flood": + os.write(2, b"x" * 100000) +elif mode == "partial": + open(sys.argv[2], "wb").write(b"partial") + sys.exit(1) +elif mode == "sleep": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + open(sys.argv[2], "w").write(str(os.getpid()) + " " + str(child.pid)) + time.sleep(60) +else: + raise RuntimeError("Unexpected fake request") +""") + return [sys.executable, str(script)] + + +@pytest.mark.parametrize( + "mode, message", [("fail", "SSH operation failed"), ("flood", "size limit")] +) +def test_bounded_process_errors(fake_process, mode, message): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + transport = context.transport + with pytest.raises(DownloadError, match=message) as error: + transport._run(fake_process + [mode], timeout=5) + assert "secret" not in str(error.value) + assert not transport._processes + + +def test_process_start_failure(): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + with pytest.raises(DownloadError, match="Unable to start"): + context.transport._run(["/nonexistent/hallmark-test"], timeout=1) + + +def test_cancel_active_process_group(fake_process, tmp_path): + pids = tmp_path / "pids" + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + context.settings = replace(context.settings, shutdown_timeout=1) + with ThreadPoolExecutor(1) as pool: + future = pool.submit( + context.transport._run, fake_process + ["sleep", str(pids)], timeout=30 + ) + deadline = time.monotonic() + 5 + while not pids.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert pids.exists() + started = time.monotonic() + context.cancel() + with pytest.raises(TransferCancelled): + future.result(timeout=5) + assert time.monotonic() - started < 4 + assert not context.transport._processes + for pid in map(int, pids.read_text().split()): + proc = Path(f"/proc/{pid}/stat") + assert not proc.exists() or proc.read_text().split()[2] == "Z" + + +def test_partial_failure_preserves_destination(monkeypatch, fake_process, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + destination = repo.worktree / "data.bin" + destination.write_bytes(b"original") + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + + def fetch(self, path, destination, **kwargs): + self._run(fake_process + ["partial", str(destination)], timeout=5) + + monkeypatch.setattr(SshTransport, "fetch", fetch) + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("data.bin"), None)] + ) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + + +def test_checksum_failure_is_atomic(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + destination = repo.worktree / "data.bin" + destination.write_bytes(b"original") + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + monkeypatch.setattr( + SshTransport, "fetch", lambda self, path, dest, **kw: dest.write_bytes(b"wrong") + ) + result = download_remote_data( + repo, + repo.worktree, + selected_files=[ + (Path("data.bin"), ("sha256", hashlib.sha256(b"right").hexdigest())) + ], + ) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + + +def test_cancel_before_publish(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + destination = repo.worktree / "item" + destination.write_bytes(b"original") + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + + def fetch(self, path, dest, **kwargs): + dest.write_bytes(b"complete") + self.context.cancel() + + monkeypatch.setattr(SshTransport, "fetch", fetch) + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("item"), None)] + ) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + + +def test_keyboard_interrupt_stops_owned_workers(monkeypatch, tmp_path, fake_process): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + transports = [] + ready = tmp_path / "pids" + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + + def fetch(self, path, destination, **kwargs): + transports.append(self) + destination.write_bytes(b"partial") + self._run(fake_process + ["sleep", str(ready)], timeout=30) + + def interrupt(*args, **kwargs): + deadline = time.monotonic() + 5 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert ready.exists() + raise KeyboardInterrupt + + monkeypatch.setattr(SshTransport, "fetch", fetch) + monkeypatch.setattr("hallmark.downloader.wait", interrupt) + with pytest.raises(KeyboardInterrupt): + download_remote_data( + repo, repo.worktree, max_workers=1, selected_files=[(Path("item"), None)] + ) + assert not (repo.worktree / "item").exists() + assert not list(Path(repo.worktree).glob("*.part")) + assert all(not transport._processes for transport in transports) + + +def test_cli_dry_run_does_not_resolve_auth_or_start_clients(monkeypatch, tmp_path): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://campus/data", remote_auth="missing") + monkeypatch.chdir(repo.worktree) + + def fail(*args, **kwargs): + raise AssertionError("Dry-run must not create an operation context") + + monkeypatch.setattr("hallmark.downloader.OperationContext", fail) + result = CliRunner().invoke(hallmark, ["download", "item.dat", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "item.dat" in result.output + + +def test_cli_profile_roundtrip(monkeypatch, tmp_path): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + monkeypatch.chdir(repo.worktree) + runner = CliRunner() + result = runner.invoke( + hallmark, + ["set-config", "--remote-url", "ssh://campus/data", "--remote-auth", "lab"], + ) + assert result.exit_code == 0, result.output + assert Repo(repo.worktree).state.config["remote"]["auth"] == "lab" + result = runner.invoke(hallmark, ["set-config", "--remote-auth", ""]) + assert result.exit_code == 0, result.output + assert "auth" not in Repo(repo.worktree).state.config["remote"] + + +def test_ssh_hash_budgets_skip_without_commands(monkeypatch): + with OperationContext( + RemoteSpec.parse("ssh://unused/data"), + allow_remote_commands=True, + remote_hash=True, + ) as context: + transport = context.transport + transport._entries = {"big": (context.hash_file_limit + 1, 0), "small": (1, 0)} + transport._hash_bytes = context.hash_total_limit + + def fail(*args, **kwargs): + raise AssertionError("Budgeted-out files must not run a command") + + monkeypatch.setattr(transport, "_command", fail) + assert transport.checksum_small("big") == ("unknown", "unknown") + assert transport.checksum_small("small") == ("unknown", "unknown") + + +def test_http_manifest_auth_failure_is_not_optional(monkeypatch): + from hallmark.repo_builder import list_remote_files + from mock_server import MockServer + + server = MockServer("https://example.test/data/") + server.add_directory("", [("data-object", "export.sha256sums")]) + get = server.get + + def unauthorized(url, **kwargs): + if url.endswith("sha256sums"): + response = requests.Response() + response.status_code = 401 + raise requests.HTTPError("credentials-must-not-leak", response=response) + return get(url, **kwargs) + + server.get = unauthorized + monkeypatch.setattr(requests, "Session", lambda: server) + with pytest.raises(DownloadError, match="HTTP 401"): + list_remote_files(server.base_url) + + +@pytest.mark.parametrize( + "url", + [ + "ssh://[fe80::1%25eth0]/data", + "ssh://[fe80::1%25$(id)]/data", + "ssh://campus/bad\udcff", + ], +) +def test_reject_scoped_ipv6_and_invalid_unicode(url): + with pytest.raises(RemoteConfigurationError): + RemoteSpec.parse(url) + + +def test_minimum_openssh_is_checked_before_connection(monkeypatch): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + monkeypatch.setattr("hallmark.transport.ssh.shutil.which", lambda name: name) + monkeypatch.setattr( + context.transport, "_run", lambda *a, **kw: b"OpenSSH_9.5p1" + ) + with pytest.raises(CapabilityError, match="9.6"): + context.transport.prepare() + assert context.transport._master is None + assert context.transport._socket_dir is None + + +def test_short_socket_ignores_long_tempdir(monkeypatch, tmp_path): + # Long macOS TMPDIR/worktree paths can exceed AF_UNIX limits. + monkeypatch.setenv("TMPDIR", str(tmp_path / ("long" * 35))) + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + monkeypatch.setattr( + context.transport, "_run", lambda *a, **kw: b"OpenSSH_9.6p1" + ) + monkeypatch.setattr("hallmark.transport.ssh.shutil.which", lambda name: name) + + def fail_start(*args, **kwargs): + assert len(context.transport._socket.encode()) <= 100 + assert context.transport._socket.startswith("/tmp/hm-") + raise DownloadError("deliberate startup failure") + + monkeypatch.setattr(context.transport, "_spawn", fail_start) + with pytest.raises(DownloadError, match="deliberate"): + context.transport.prepare() + assert context.transport._socket_dir is None + + +def test_manifest_strength_and_conflict(): + from hallmark.repo_builder import _record_checksum + + checksums = {} + _record_checksum(checksums, "file", "sha256", "a" * 64) + _record_checksum(checksums, "file", "md5", "b" * 32) + assert checksums["file"] == ("sha256", "a" * 64) + with pytest.raises(DownloadError, match="Conflicting"): + _record_checksum(checksums, "file", "sha256", "c" * 64) + + +def test_explicit_http_source_is_exact_and_keeps_output_remotes(monkeypatch, tmp_path): + from mock_server import MockServer + + server = MockServer("https://example.test/already-the-dataset/") + server.add_directory("", [("data-object", "README.md")]) + server.add_file("README.md", b"notes") + monkeypatch.setattr(requests, "Session", lambda: server) + repo = build_repo( + tmp_path / "catalog", + "label", + [], + dataset_url=server.base_url, + index_format="cyverse-html", + remotes=[{"name": "mirror", "url": "https://elsewhere.test/data"}], + ) + assert repo.state.config["remote"][0]["url"] == "https://elsewhere.test/data" + assert repo.state.config["data"][0]["md5"] == hashlib.md5(b"notes").hexdigest() + + +def test_build_cli_passes_source_controls(monkeypatch, tmp_path): + from click.testing import CliRunner + from hallmark.cli import hallmark + + calls = [] + monkeypatch.setattr("hallmark.cli.build_repo", lambda **kw: calls.append(kw)) + result = CliRunner().invoke( + hallmark, + [ + "build", + str(tmp_path), + "lab", + "--fmt", + "run_{i}.h5=data.tsv", + "--dataset-url", + "ssh://campus/data", + "--dataset-auth", + "lab", + "--allow-remote-commands", + "--remote-hash", + ], + ) + assert result.exit_code == 0, result.output + assert calls[0]["dataset_url"] == "ssh://campus/data" + assert calls[0]["dataset_auth"] == "lab" + assert calls[0]["allow_remote_commands"] is True + assert calls[0]["remote_hash"] is True From d856bb7d791e69f7f4cc751daf6cbc133848149b Mon Sep 17 00:00:00 2001 From: Dennis Date: Tue, 15 Sep 2026 20:07:05 -0700 Subject: [PATCH 05/29] Fix Sphinx API documentation warnings --- doc/api.rst | 16 ---------------- doc/conf.py | 2 +- mod/hallmark/objects.py | 5 ++++- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 0ef7bbe..fa12b75 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -23,18 +23,6 @@ Core Repository :members: :show-inheritance: -.. automodule:: hallmark.state - :members: - :show-inheritance: - -.. automodule:: hallmark.downloader - :members: - :show-inheritance: - -.. automodule:: hallmark.paraframe - :members: - :show-inheritance: - Repository Worktrees -------------------- @@ -64,10 +52,6 @@ Data Handling :members: :show-inheritance: -.. automodule:: hallmark.eht_datatree - :members: - :show-inheritance: - Downloading ----------- diff --git a/doc/conf.py b/doc/conf.py index 2c15f9f..be01646 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -39,4 +39,4 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = 'alabaster' -html_static_path = ['_static'] +html_static_path = [] diff --git a/mod/hallmark/objects.py b/mod/hallmark/objects.py index fc54798..a54af2d 100644 --- a/mod/hallmark/objects.py +++ b/mod/hallmark/objects.py @@ -195,16 +195,19 @@ def missing(self, checksums: Iterable[str]) -> list[str]: def store(self, src: Path, sha1: str, *, actual_sha1: Optional[str] = None) -> Path: """ Store a file in the object store using its SHA-1 checksum. + Args: src (Path): Path to the source file to store. sha1 (str): SHA-1 checksum of the source file. actual_sha1 (Optional[str]): Optional actual SHA-1 of the source file. If provided, it will be used to verify the integrity of the source file. + Returns: Path: Path to the stored file. + Raises: ValueError: If the actual SHA-1 checksum of the source file does not match - the expected SHA-1. + the expected SHA-1. """ src = Path(src) # split the SHA-1 checksum into its storage path From 28f1a1bef0546c4e8114a793d95abc3b07aca638 Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 16 Sep 2026 16:03:15 -0700 Subject: [PATCH 06/29] Run CI for dev branch changes --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54e9f87..9b31994 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,9 +5,9 @@ name: CI on: push: - branches: [ "main" ] + branches: [ "main", "dev" ] pull_request: - branches: [ "main" ] + branches: [ "main", "dev" ] jobs: build: From e33baa123343ad1cff4638e3e6caa63572891b37 Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 16 Sep 2026 18:24:52 -0700 Subject: [PATCH 07/29] Fix transport tests on Python 3.9 and macOS --- test/test_ssh_integration.py | 73 ++++++++++++++++++++++++------------ test/test_transport.py | 5 ++- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py index c977972..2c4673b 100644 --- a/test/test_ssh_integration.py +++ b/test/test_ssh_integration.py @@ -295,30 +295,53 @@ def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): assert (tmp_path / "cli-clone/nested/item_1.dat").read_bytes() == b"science" +def _wait_for_partial_file(directory, pattern, total_size, task): + """Wait for actual SFTP bytes before interrupting a transfer.""" + directory = Path(directory) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if task.done(): + task.result() # Surface a transfer error instead of a polling timeout. + pytest.fail("Transfer completed before it could be interrupted") + for path in directory.glob(pattern): + try: + size = path.stat().st_size + except FileNotFoundError: + continue + if 0 < size < total_size: + return + time.sleep(0.01) + pytest.fail("SFTP did not write a partial file before the deadline") + + def test_real_cancel_transfer_and_preserve_other_master(ssh_server, tmp_path): server = ssh_server - (server["root"] / "large").write_bytes(b"x" * 128 * 1024) + payload = b"x" * 128 * 1024 + (server["root"] / "large").write_bytes(payload) with OperationContext(RemoteSpec.parse(server["url"])) as survivor: survivor.transport.prepare() with OperationContext(RemoteSpec.parse(server["url"])) as context: - context.settings = replace(context.settings, shutdown_timeout=1) + context.settings = replace( + context.settings, shutdown_timeout=1, transfer_timeout=15 + ) context.transport.prepare() - # Throttle only this test's SFTP process to make interruption deterministic. - context.transport.sftp = ["sftp", "-l", "8"] + # Bound requests as well as bandwidth: a small file can otherwise + # finish inside the client's initial buffering window on macOS. + context.transport.sftp = ["sftp", "-B", "1024", "-R", "1", "-l", "8"] target = tmp_path / "partial" with ThreadPoolExecutor(1) as pool: task = pool.submit(context.transport.fetch, "large", target) - deadline = time.monotonic() + 5 - while ( - len(context.transport._processes) < 2 - and time.monotonic() < deadline - ): - time.sleep(0.01) - context.cancel() - with pytest.raises(TransferCancelled): - task.result(timeout=5) + try: + _wait_for_partial_file(tmp_path, "partial", len(payload), task) + context.cancel() + with pytest.raises(TransferCancelled): + task.result(timeout=5) + finally: + context.cancel() assert not context.transport._processes assert survivor.transport._master.poll() is None + survivor.transport.fetch("large", tmp_path / "survivor-copy") + assert (tmp_path / "survivor-copy").read_bytes() == payload @pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) @@ -426,7 +449,8 @@ def test_permission_denied_preserves_existing_file(ssh_server, tmp_path): def test_disconnected_master_cleans_partial_transfer(ssh_server, tmp_path, monkeypatch): - (ssh_server["root"] / "large").write_bytes(b"x" * 128 * 1024) + payload = b"x" * 128 * 1024 + (ssh_server["root"] / "large").write_bytes(payload) repo = Repo.init(tmp_path / "repo") repo.set_config(remote_url=ssh_server["url"]) destination = repo.worktree / "large" @@ -436,7 +460,10 @@ def test_disconnected_master_cleans_partial_transfer(ssh_server, tmp_path, monke def slow_prepare(self): prepare(self) - self.sftp = ["sftp", "-l", "8"] + self.context.settings = replace( + self.context.settings, shutdown_timeout=1, transfer_timeout=15 + ) + self.sftp = ["sftp", "-B", "1024", "-R", "1", "-l", "8"] if self not in transports: transports.append(self) @@ -448,14 +475,14 @@ def slow_prepare(self): repo.worktree, selected_files=[(Path("large"), None)], ) - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - if transports and len(transports[0]._processes) > 1: - break - time.sleep(0.01) - assert transports and len(transports[0]._processes) > 1 - transports[0]._master.terminate() - result = task.result(timeout=5) + try: + _wait_for_partial_file(repo.worktree, "*.part", len(payload), task) + assert len(transports) == 1 + transports[0]._master.terminate() + result = task.result(timeout=5) + finally: + for transport in transports: + transport.context.cancel() assert result["failed"] == 1 assert destination.read_bytes() == b"original" assert not list(Path(repo.worktree).glob("*.part")) diff --git a/test/test_transport.py b/test/test_transport.py index c12e611..16bed36 100644 --- a/test/test_transport.py +++ b/test/test_transport.py @@ -215,7 +215,10 @@ def fail(*args, **kwargs): url = "https://user:secret@example.test/data?token=secret" with pytest.raises(DownloadError) as error: _download_file(url, tmp_path / "out") - assert "secret" not in "".join(traceback.format_exception(error.value)) + formatted = traceback.format_exception( + type(error.value), error.value, error.value.__traceback__ + ) + assert "secret" not in "".join(formatted) assert "secret" not in repr(error.value) From 6e705b9c74e689285416102825322be02b977451 Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 16 Sep 2026 18:29:50 -0700 Subject: [PATCH 08/29] Handle exited SSH process groups on macOS --- mod/hallmark/transport/ssh.py | 21 ++++++++++------- test/test_transport.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index d5dfa82..811d42b 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -153,18 +153,23 @@ def _spawn(self, argv, **kwargs): def _stop(self, process): # Every owned client starts a new process group, including proxy children. - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass + def signal_group(signum): + try: + os.killpg(process.pid, signum) + except ProcessLookupError: + pass + except PermissionError: + # macOS can report EPERM for a group containing only zombies. + # Reap our child, but retain errors for a still-running process. + if process.poll() is None: + raise + + signal_group(signal.SIGTERM) try: process.wait(timeout=self.context.settings.shutdown_timeout) except subprocess.TimeoutExpired: pass - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass + signal_group(signal.SIGKILL) process.wait() with self._process_lock: self._processes.discard(process) diff --git a/test/test_transport.py b/test/test_transport.py index 16bed36..d06d147 100644 --- a/test/test_transport.py +++ b/test/test_transport.py @@ -2,8 +2,10 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import replace +import errno import hashlib from pathlib import Path +import signal import subprocess import sys import time @@ -335,6 +337,48 @@ def test_process_start_failure(): context.transport._run(["/nonexistent/hallmark-test"], timeout=1) +def test_cleanup_exited_process_group_permission_error(monkeypatch): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + transport = context.transport + process = transport._spawn([sys.executable, "-c", "pass"]) + process.wait(timeout=5) + signals = [] + + def zombie_group(pid, signum): + assert pid == process.pid + signals.append(signum) + raise PermissionError(errno.EPERM, "Operation not permitted") + + with monkeypatch.context() as patch: + patch.setattr("hallmark.transport.ssh.os.killpg", zombie_group) + transport._stop(process) + # Exited leaders may still have proxy children: attempt both signals. + assert signals == [signal.SIGTERM, signal.SIGKILL] + assert process.returncode == 0 + assert not transport._processes + + +def test_cleanup_live_process_permission_error_is_not_suppressed(monkeypatch): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + transport = context.transport + process = transport._spawn( + [sys.executable, "-c", "import time; time.sleep(60)"] + ) + + def denied(pid, signum): + raise PermissionError(errno.EPERM, "Operation not permitted") + + with monkeypatch.context() as patch: + patch.setattr("hallmark.transport.ssh.os.killpg", denied) + with pytest.raises(PermissionError): + transport._stop(process) + assert process.poll() is None + assert process in transport._processes + # Restore real signals before the context closes and reaps this child. + assert process.poll() is not None + assert not transport._processes + + def test_cancel_active_process_group(fake_process, tmp_path): pids = tmp_path / "pids" with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: From 960b8d8f2dc6b1092596926cc644607acaa1592e Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 16 Sep 2026 18:33:54 -0700 Subject: [PATCH 09/29] Wait for concurrent SSH process cleanup --- mod/hallmark/transport/ssh.py | 11 +++++++---- test/test_transport.py | 9 ++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index 811d42b..f1265dd 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -158,11 +158,14 @@ def signal_group(signum): os.killpg(process.pid, signum) except ProcessLookupError: pass - except PermissionError: + except PermissionError as exc: # macOS can report EPERM for a group containing only zombies. - # Reap our child, but retain errors for a still-running process. - if process.poll() is None: - raise + # poll() may return None while another thread holds Popen's + # wait lock, so allow a bounded wait for concurrent reaping. + try: + process.wait(timeout=self.context.settings.shutdown_timeout) + except subprocess.TimeoutExpired: + raise exc from None signal_group(signal.SIGTERM) try: diff --git a/test/test_transport.py b/test/test_transport.py index d06d147..a582960 100644 --- a/test/test_transport.py +++ b/test/test_transport.py @@ -337,7 +337,10 @@ def test_process_start_failure(): context.transport._run(["/nonexistent/hallmark-test"], timeout=1) -def test_cleanup_exited_process_group_permission_error(monkeypatch): +@pytest.mark.parametrize("reaping_in_progress", [False, True]) +def test_cleanup_exited_process_group_permission_error( + monkeypatch, reaping_in_progress +): with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: transport = context.transport process = transport._spawn([sys.executable, "-c", "pass"]) @@ -351,6 +354,10 @@ def zombie_group(pid, signum): with monkeypatch.context() as patch: patch.setattr("hallmark.transport.ssh.os.killpg", zombie_group) + if reaping_in_progress: + # Popen.poll() cannot acquire its wait lock while another + # thread reaps the child and can temporarily report None. + patch.setattr(process, "poll", lambda: None) transport._stop(process) # Exited leaders may still have proxy children: attempt both signals. assert signals == [signal.SIGTERM, signal.SIGKILL] From b1e93035d7320aafbf6d8cbda46f7f998e226e1a Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 16 Sep 2026 18:38:14 -0700 Subject: [PATCH 10/29] Match private data documentation to existing style --- README.rst | 27 +++-- doc/api.rst | 27 ++++- doc/usecase.rst | 284 +++++++++++++++++++++++++++++------------------- 3 files changed, 212 insertions(+), 126 deletions(-) diff --git a/README.rst b/README.rst index f769d42..2bedc09 100644 --- a/README.rst +++ b/README.rst @@ -74,18 +74,25 @@ Or install from source for development purposes:: Private data remotes -------------------- -Hallmark can fetch cataloged files from HTTP(S), SSH and SFTP data remotes. -Data remotes locate file bytes separately from the Git remotes used for catalog -history. SSH uses system OpenSSH 9.6+, a trusted host key and noninteractive -key/agent authentication:: +|hallmark|_ can download data products from HTTP(S), SSH, and SFTP servers. +A data remote specifies where the files are stored, while a Git remote +is used to share the history of the data index. +For a private server, |hallmark|_ uses OpenSSH 9.6 or newer with a trusted +host key and key-based authentication that does not require a prompt. + +Configure a data remote, preview the files, and download them:: hallmark set-config --remote-name campus --remote-url ssh://campus/srv/export/ hallmark download --remote campus --all --dry-run hallmark download --remote campus --all -Optional local auth profiles can be selected with ``--remote-auth PROFILE``. -Private catalogs can be built with ``build --dataset-url URL --dataset-auth -PROFILE --allow-remote-commands``; SSH listing requires a POSIX shell and Python 3 -on the server. SFTP-only accounts can download known files. See -`Private lab data over SSH `_ for setup, -profiles, bounded hashing, platform scope and disposable integration tests. +Optional authentication profiles keep local SSH settings outside the +repository and can be selected with ``--remote-auth PROFILE``. +To build a data index from a private server, use ``hallmark build`` with +``--dataset-url``, ``--allow-remote-commands``, and, if needed, +``--dataset-auth PROFILE``. +Building an index requires a POSIX shell and Python 3 on the server; +SFTP-only accounts can download files from an existing index. + +Examples and configuration details are described in +`Private lab data over SSH `_. diff --git a/doc/api.rst b/doc/api.rst index fa12b75..842c07f 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -139,9 +139,24 @@ Dataset builders and data remotes .. autofunction:: hallmark.repo_builder.list_remote_files -``Repo.set_config(remote_auth="PROFILE")`` records a local SSH profile reference; -``remote_auth=""`` removes it. ``download_remote_data`` keeps its existing result -mapping with ``succeeded``, ``failed``, ``total_bytes`` and ``errors``. Configuration -and SSH connection preflight failures raise ``DownloadError`` before file workers -start. HTTP, SSH and SFTP follow the same destination/checksum contract. See the -private-lab workflow in :doc:`usecase` for profile schema and capability limits. +Associate a data remote with a local SSH authentication profile:: + + repo.set_config(remote_name="campus", remote_auth="campus") + +Remove the profile reference:: + + repo.set_config(remote_name="campus", remote_auth="") + +The repository stores the profile name. The SSH settings remain in the +local authentication file. Passing ``remote_auth=None`` leaves the +reference unchanged. + +``download_remote_data`` returns a dictionary containing ``succeeded``, +``failed``, ``total_bytes``, and ``errors``. Check ``failed`` and ``errors`` +for individual transfer failures. Configuration errors and failed SSH +connection checks raise ``DownloadError`` before downloads begin. +HTTP, SSH, and SFTP downloads use the same rules for destination paths +and checksum verification. + +See :ref:`private-lab-data-over-ssh` for a CLI workflow, authentication +settings, and supported server configurations. diff --git a/doc/usecase.rst b/doc/usecase.rst index 722c900..ccb4108 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -122,61 +122,92 @@ This is especially useful for data (re-)organization. .. _hallmark: https://github.com/l6a/hallmark -Private lab data over SSH -------------------------- - -A **data remote** locates dataset bytes. A **Git remote** synchronizes the -catalog's history in ``.hm``. They are separate configurations, even when both -are named ``origin``. Selecting ``download --remote campus`` selects only that -data remote; Hallmark does not automatically fall back to another server. - -Use OpenSSH ``ssh`` and ``sftp`` 9.6 or newer on Linux. Linux and macOS have -integration jobs; macOS support should be checked against that job's result. -Windows SSH transport is not supported. Requests remains required for HTTP, -including its existing environment and ``.netrc`` authentication behavior. - -Before downloading, configure your SSH alias, trust the server's host key, and -load a usable key into your agent (or configure an identity file). Workers use -``BatchMode=yes`` and default to ``StrictHostKeyChecking=yes``. Password prompts, -interactive 2FA, and an initial interactive hardware-key challenge cannot be -answered by Hallmark. Complete those prerequisites separately. Hallmark owns a -new multiplexed connection for each invocation and closes it afterward; existing -user-owned masters are neither reused nor stopped. Multiplexing is required. -ProxyJump and other trusted SSH policy remain in your local SSH configuration. - -Configure an absolute dataset root:: +.. _private-lab-data-over-ssh: + +6. CLI: Private Lab Data over SSH +---------------------------------- + +Frank works with simulation data stored on a private lab server. +He has an existing |hallmark|_ repository that indexes the files and +wants to download them to his local worktree. +He first configures the SSH alias ``campus``, verifies the server's host +key, and loads his key into an SSH agent or configures an identity file. + +From the local repository, he sets the data location, previews the +selection, and downloads the files:: hallmark set-config --remote-name campus --remote-url ssh://campus/srv/export/ hallmark download --remote campus --all --dry-run hallmark download --remote campus --all -``sftp://`` is an alias for ``ssh://``. Explicit users and ports are supported, -for example ``ssh://researcher@campus:2222/srv/export/``. IPv6 addresses need -brackets: ``ssh://[2001:db8::1]/srv/export/``. SCP shorthand and implicit -home-relative roots are unsupported. URL paths must encode reserved characters, -e.g. ``has%20space%23tag``. Catalog/manifest paths are literal: ``literal%20.h5`` -names a file containing a percent sign. Spaces, quotes, glob characters, and -leading dashes are supported; control characters and backslashes are rejected. - -Downloads stream into temporary files, verify recorded checksums, then replace -the requested destinations atomically. A failed transfer or checksum mismatch -preserves an existing destination and removes its temporary file. Other files -that already succeeded remain available. Cancellation terminates owned SSH -process groups, waits for workers and closes the master. HTTP workers check -cancellation between chunks, but blocked Requests calls can take their normal -``(10, 30)`` connect/read timeouts to return. A slowly trickling response can -keep a blocked read alive longer; no fixed five-second HTTP -cancellation guarantee is made. Existing destination symlinks and observed -symlink swaps are rejected. This is not protection against every race with a -concurrently hostile local writer. +The data remote ``campus`` specifies where the dataset files are stored. +It is configured separately from the Git remote used to share the +history in ``.hm``, even when both remotes have the same name. +The ``--remote`` option selects one data server; |hallmark|_ does not +automatically try another server if the download fails. + +Downloads are written to temporary files and checked against any +checksums supplied with the selection before atomically replacing +their destinations. +The ``--tsv`` and ``--all`` options include catalog checksums; explicit +paths alone do not look them up. +If a transfer or checksum check fails, the existing destination is +preserved and the temporary file is removed. +Files that have already downloaded successfully remain available. + +SSH access and file paths +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The client requires OpenSSH ``ssh`` and ``sftp`` version 9.6 or newer +with connection multiplexing enabled. +Linux and macOS have integration-test jobs; check the macOS job's result +before relying on that platform. Windows SSH transport is not supported. +HTTP downloads continue to use Requests, including its environment +settings and ``.netrc`` authentication. + +|hallmark|_ uses ``BatchMode=yes`` and defaults to +``StrictHostKeyChecking=yes``. Authentication must succeed without +password prompts, interactive two-factor authentication, or an initial +hardware-key prompt. Complete any interactive setup before downloading. +Each invocation opens and closes its own shared SSH connection, leaving +existing SSH connections alone. Settings such as ``ProxyJump`` belong +in the local SSH configuration. + +The URL must specify an absolute dataset directory. +Both ``ssh://`` and ``sftp://`` use SFTP for file transfers. +Usernames and ports can be included, as in +``ssh://researcher@campus:2222/srv/export/``. +IPv6 addresses require brackets, as in +``ssh://[2001:db8::1]/srv/export/``. +SCP shorthand and paths relative to a home directory are not supported. + +Reserved characters in URL paths must be percent-encoded, e.g., +``has%20space%23tag``. Paths in catalogs and checksum manifests are +literal: ``literal%20.h5`` names a file containing a percent sign. +Spaces, quotes, glob characters, and leading dashes are supported; +control characters and backslashes are rejected. +Existing destination symlinks and detected changes to symlinks are +rejected, although these checks cannot prevent every race with another +process modifying the destination. + +Cancelling a download stops the SSH processes started by |hallmark|_, +waits for the download workers, and closes its shared connection. +HTTP workers check for cancellation between chunks. A pending Requests +call may wait for its 10-second connection timeout or 30-second read +timeout; a response that keeps sending small amounts of data can delay +cancellation further. Local profiles ~~~~~~~~~~~~~~ -Profiles live in ``$HALLMARK_AUTH_FILE``, otherwise -``$XDG_CONFIG_HOME/hallmark/auth.yml``, otherwise -``~/.config/hallmark/auth.yml``. Repository configuration stores only the profile -name. Profiles currently apply to SSH/SFTP, with this strict schema: +Frank can keep SSH settings in a local authentication profile and +record its name in the repository. +|hallmark|_ reads profiles from ``$HALLMARK_AUTH_FILE`` when set, +otherwise from ``$XDG_CONFIG_HOME/hallmark/auth.yml``, or from +``~/.config/hallmark/auth.yml`` if neither variable is set. +Profiles apply to SSH and SFTP connections. + +For example, his authentication file can contain: .. code-block:: yaml @@ -194,89 +225,122 @@ name. Profiles currently apply to SSH/SFTP, with this strict schema: identity_file: ~/.ssh/id_ed25519 host_key_policy: strict -Supported profile fields are ``hosts`` and the settings shown above; profiles -may also override the four time/concurrency defaults. Global defaults cannot -set ``hosts``, ``user``, ``port`` or ``identity_file``. Passwords, tokens, arbitrary -SSH options, executables and SSH config paths are not supported fields. The -profile name must match ``[A-Za-z0-9_-]{1,64}``. A missing profile or a URL host -outside its ``hosts`` list fails before connections start. URL user/port override -profile defaults, then SSH configuration and OpenSSH defaults apply. Host bindings -refer to the URL alias or literal host, before SSH config resolves ``HostName``. +He associates the profile with the data remote:: -``transfer_timeout`` is a total wall-clock budget per file in seconds, not an -idle/read timeout; increase it locally for large datasets. ``max_sessions`` caps -concurrent transfers per operation to respect the server's session limit. -Only a local profile/default may select ``host_key_policy: accept-new`` for -first-contact trust; changed host keys still fail. + hallmark set-config --remote-name campus --remote-auth campus -Associate or remove a profile reference:: +He can remove the reference later:: - hallmark set-config --remote-name campus --remote-auth campus hallmark set-config --remote-name campus --remote-auth '' -The Python equivalents are ``repo.set_config(remote_auth="campus")`` and -``repo.set_config(remote_auth="")``. ``None`` leaves the reference unchanged. -No resolved identity settings are written into the repository. +The Python equivalents use ``repo.set_config`` with +``remote_name="campus"`` and ``remote_auth="campus"`` or ``remote_auth=""``. +Passing ``None`` leaves the profile reference unchanged. +The repository stores only the profile name; identity settings remain local. + +Profile names must match ``[A-Za-z0-9_-]{1,64}``. +The ``hosts`` list contains the aliases or hostnames used in data URLs, +before SSH resolves ``HostName``. +A missing profile or a host outside this list causes an error before +connecting. A username or port in the URL overrides the profile setting; +remaining values come from SSH configuration and OpenSSH defaults. + +Profiles accept the fields shown above and may override the four +timeout and concurrency defaults. Global defaults cannot set ``hosts``, +``user``, ``port``, or ``identity_file``. +Passwords, tokens, arbitrary SSH options, executable paths, and SSH +configuration paths are not supported profile fields. + +``transfer_timeout`` limits the total transfer time for each file in +seconds, so large datasets may need a higher value. +``max_sessions`` limits concurrent transfers within an operation to +respect the server's session limit. +The local profile or defaults may use ``host_key_policy: accept-new`` +to accept a host key on first contact. Changed host keys still cause +an error. Repository configuration cannot enable this setting. Build a private catalog ~~~~~~~~~~~~~~~~~~~~~~~ -The crawl source is independent of remotes recorded in the resulting catalog. -``--dataset-url`` is the exact dataset root; the dataset name is only a label:: +Frank can also build a catalog directly from the files on the server. +He supplies the dataset directory and a filename format:: hallmark build catalogs lab \ --dataset-url ssh://campus/srv/export/ \ --dataset-auth campus --allow-remote-commands \ --fmt 'runs/run_{i:d}.h5=data.tsv' -SSH builds require a POSIX login shell and Python 3 on the server. The explicit -``--allow-remote-commands`` flag permits fixed read-only recursive listing and, -when requested, hashing commands. Repository configuration cannot enable this -permission. SFTP-only accounts support downloads of cataloged files but cannot -build catalogs. Listings omit symlinks and special files, including symlinked -directories. Remote roots are namespaces; enforce access boundaries on the server. - -Existing checksum manifests are preferred. Supported manifests use unescaped GNU -sum lines: a hexadecimal digest, a space, a space or ``*`` marker, then the literal -filename. Strong algorithm names such as ``sha256`` are preserved. GNU escaped -filename records are rejected. Missing digests remain absent/``unknown``; auth and -ambiguous SSH failures are errors, never treated as an absent optional manifest. - -Add ``--remote-hash`` to compute server SHA-256 for files without a manifest hash. -Hashing is limited to 10 MiB per file, 100 MiB total and a 60-second operation -budget. Larger files or exhausted budgets retain unknown digests. Obvious file -size/mtime changes during hashing fail. Recursive listing has a 100,000-file and -16 MiB output limit with a 60-second command timeout. Text reads have a 16 MiB -per-file and 64 MiB aggregate limit; new reads stop after a five-minute crawl -budget. No dataset file is downloaded merely to compute an SSH checksum. - -When no output remotes are supplied, the synthesized ``origin`` records the -source URL and its profile reference. ``--remote NAME=URL`` records a different -output location without changing the crawl source or inheriting its auth profile. -A name-only output remote defaults to the crawl URL but must be configured with -its own auth reference if needed. - -Without ``--dataset-url``, legacy CyVerse dataset-name lookup is preserved. -An explicit HTTP root requires ``--index-format cyverse-html`` and must expose -that HTML index dialect. Arbitrary HTTP directory listing and WebDAV PROPFIND -are not implemented. Static HTTP checksums retain the bounded small-file MD5 -fallback. Local add/commit/checkout workflows still require one format backed -by ``data.tsv``; builder and download workflows may use multiple catalogs. - -Use immutable exports or server snapshots when reproducibility matters: crawling -and hashing a live directory cannot produce a guaranteed snapshot. Subsequent -downloads always verify whatever digest the catalog records. +This creates ``catalogs/lab.hm`` with an index of the matching run files +in ``data.tsv``. The name ``lab`` labels the catalog; it is not appended +to ``--dataset-url``. + +Building over SSH requires a POSIX login shell and Python 3 on the server. +The ``--allow-remote-commands`` option permits fixed, read-only commands +to list files recursively and, when requested, compute checksums. +This permission must be supplied explicitly; repository configuration +cannot enable it. +SFTP-only accounts can download indexed files but cannot build catalogs. +Listings skip symlinks, including symlinked directories, and special files. +The dataset directory defines where listing starts; access permissions +must be enforced on the server. + +|hallmark|_ uses existing checksum manifests when available. +Supported manifests use unescaped GNU sum records: a hexadecimal +checksum, a space, a space or ``*`` marker, and the literal filename. +Algorithm names such as ``sha256`` are retained, and GNU records with +escaped filenames are rejected. +Missing checksums remain absent or ``unknown``. Authentication failures +and SSH errors with an unclear cause stop the build instead of being +treated as missing manifests. + +Frank can add ``--remote-hash`` to compute SHA-256 checksums on the server +for files without a manifest checksum. +This is limited to 10 MiB per file, 100 MiB in total, and 60 seconds for +the operation. Files beyond these limits retain unknown checksums. +Detected changes to file size or modification time during hashing cause +an error. Dataset files are not downloaded to compute SSH checksums. + +Recursive listing is limited to 100,000 files, 16 MiB of output, and a +60-second command timeout. Text reads are limited to 16 MiB per file +and 64 MiB in total; new reads stop after five minutes of crawling. + +By default, the generated ``origin`` data remote records the source URL +and its profile name. +The ``--remote NAME=URL`` option records a different download location +without changing the directory being indexed or copying its profile name. +A remote specified by name alone uses the source URL. +In either case, a profile for that remote must be configured separately +if needed. + +Without ``--dataset-url``, the builder looks up the dataset name on +CyVerse as before. +An explicit HTTP directory requires ``--index-format cyverse-html`` and +a matching HTML index. Other HTTP directory indexes and WebDAV PROPFIND +are not supported. For HTTP files without a recorded checksum, the +builder retains its existing size and time limits for computing MD5. +Building and downloading can use multiple catalogs; local ``add``, +``commit``, and ``checkout`` operations require one format backed by +``data.tsv``. + +For reproducible results, Frank uses an immutable export or a server +snapshot. Listing and hashing a directory while its files change cannot +guarantee a consistent snapshot. +Later downloads verify any usable checksums included in their selections; +missing or unknown checksums do not prevent a transfer. Transport validation ~~~~~~~~~~~~~~~~~~~~ -Run ordinary tests with ``pytest -q``. The SFTP parser tests use a local -``sftp-server`` process without networking. The complete OpenSSH tests use only -loopback sockets, ephemeral keys, isolated known-hosts/config files and temporary -data:: +The regular test suite runs with ``pytest -q``. +SFTP parser tests use a local ``sftp-server`` process without networking. +To run the complete OpenSSH integration tests:: HALLMARK_RUN_SSH_TESTS=1 pytest test/test_ssh_integration.py -q -The opt-in fails if required tools are unavailable. It never connects to a live -lab server or changes normal SSH trust files. CI installs OpenSSH server and -runs this suite separately from the Python-version matrix. +These tests use a loopback server, temporary keys and data, and separate +SSH configuration and known-hosts files. +They do not connect to a lab server or change the user's SSH trust files. +Enabling the tests requires the OpenSSH tools to be installed; missing +tools cause a test failure. +CI installs the server and runs these tests separately from the +Python-version matrix. From e10a884400b4a7eed271e4090da2ad43c7565d12 Mon Sep 17 00:00:00 2001 From: Dennis Date: Fri, 18 Sep 2026 14:33:59 -0700 Subject: [PATCH 11/29] Simplify remote catalogs and require download approval --- .gitignore | 5 +- README.rst | 29 +- demo/private_data_demo.md | 286 +++++++++++++++ doc/api.rst | 66 ++-- doc/index.rst | 5 + doc/private_data.rst | 361 +++++++++++++++++++ doc/usecase.rst | 125 ++++--- mod/hallmark/__init__.py | 3 +- mod/hallmark/catalog.py | 238 +++++++++++++ mod/hallmark/cli.py | 308 +++++----------- mod/hallmark/discovery.py | 324 +++++++++++++++++ mod/hallmark/download_plan.py | 113 ++++++ mod/hallmark/downloader.py | 249 ++++++++++--- mod/hallmark/repo.py | 125 +++---- mod/hallmark/repo_builder.py | 407 +++------------------- mod/hallmark/transport/__init__.py | 29 +- mod/hallmark/transport/base.py | 18 +- mod/hallmark/transport/http.py | 88 +++-- mod/hallmark/transport/sftp.py | 278 +++++++++++++++ mod/hallmark/transport/ssh.py | 216 +++++------- test/mock_server.py | 2 +- test/test_catalog.py | 212 +++++++++++ test/test_cli.py | 540 ++++++++++++----------------- test/test_discovery.py | 267 ++++++++++++++ test/test_download_plan.py | 235 +++++++++++++ test/test_downloader.py | 62 ++-- test/test_hallmark.py | 17 +- test/test_repo_builder.py | 334 +++--------------- test/test_sftp_metadata.py | 229 ++++++++++++ test/test_ssh_integration.py | 119 +++++-- test/test_transport.py | 117 +++++-- 31 files changed, 3705 insertions(+), 1702 deletions(-) create mode 100644 demo/private_data_demo.md create mode 100644 doc/private_data.rst create mode 100644 mod/hallmark/catalog.py create mode 100644 mod/hallmark/discovery.py create mode 100644 mod/hallmark/download_plan.py create mode 100644 mod/hallmark/transport/sftp.py create mode 100644 test/test_catalog.py create mode 100644 test/test_discovery.py create mode 100644 test/test_download_plan.py create mode 100644 test/test_sftp_metadata.py diff --git a/.gitignore b/.gitignore index c1b977c..9ad915c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,5 +17,6 @@ __pycache__/ # Ignore Sphinx build output by default _build/ -/demo/ -/cli_test_repos/ \ No newline at end of file +/demo/* +!/demo/private_data_demo.md +/cli_test_repos/ diff --git a/README.rst b/README.rst index 2bedc09..ac5ec89 100644 --- a/README.rst +++ b/README.rst @@ -80,19 +80,24 @@ is used to share the history of the data index. For a private server, |hallmark|_ uses OpenSSH 9.6 or newer with a trusted host key and key-based authentication that does not require a prompt. -Configure a data remote, preview the files, and download them:: +Clone a remote directory to prepare its catalog, then approve selected downloads:: - hallmark set-config --remote-name campus --remote-url ssh://campus/srv/export/ - hallmark download --remote campus --all --dry-run - hallmark download --remote campus --all + hallmark clone ssh://campus/srv/export/ campus --filter '**/*.h5' + cd campus + hallmark download --all --dry-run + hallmark download --all -Optional authentication profiles keep local SSH settings outside the -repository and can be selected with ``--remote-auth PROFILE``. -To build a data index from a private server, use ``hallmark build`` with -``--dataset-url``, ``--allow-remote-commands``, and, if needed, -``--dataset-auth PROFILE``. -Building an index requires a POSIX shell and Python 3 on the server; -SFTP-only accounts can download files from an existing index. +Cloning discovers files without downloading dataset contents. CyVerse and +ordinary browsable HTTPS directories, including DESI, use the same workflow. +SSH discovery also works with SFTP-only accounts; no server shell or Python +is required. Omit the filter to catalog everything beneath the supplied URL. + +Every dataset transfer requires confirmation. Python callers can inspect +``repo.plan_download()`` and then execute ``repo.download(plan, approved=True)``. +Optional authentication profiles remain local and can be selected with +``clone --auth PROFILE`` or ``set-config --remote-auth PROFILE``. +``init`` creates a local repository; the older remote ``build`` command is +deprecated in favor of ``clone``. Examples and configuration details are described in -`Private lab data over SSH `_. +`Private data over SSH and SFTP `_. diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md new file mode 100644 index 0000000..59c949d --- /dev/null +++ b/demo/private_data_demo.md @@ -0,0 +1,286 @@ +# Private data demo: copyable instructions + +These are illustrative client-side instructions, not a record of a live VM run. +The SSH alias retains the validation VM settings supplied for this demo. Use an +existing export that you are authorized to read, and replace the example dataset +root before running the commands. No remote fixture creation is required. + +For a small walkthrough, the examples assume `runs/run_001.dat`, +`runs/run_002.dat`, a `README.md`, and optionally a publisher-provided `SHA256SUMS`. +The client needs this Hallmark checkout, Git, and OpenSSH `ssh`/`sftp` 9.6+. +The server needs SFTP; a login shell, Python, Hallmark, and Git are unnecessary. +Run the Bash blocks in order in one local terminal. + +1. Install the checkout in your active Python environment. + +```bash +# Start in the Hallmark source checkout. +python -m pip install -e . +ssh -V +hallmark clone --help +``` + +2. Add or update this entry in `~/.ssh/config`, preserving your other entries. +Replace the host, user, or identity if your approved server differs. + +```text +Host lab-data + HostName 34.29.34.222 + User denniswu501 + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes + StrictHostKeyChecking yes +``` + +The verified host key must already be in `known_hosts`. Load an encrypted key +into your SSH agent first. Check SFTP access without an interactive login: + +```bash +sftp -o BatchMode=yes -o StrictHostKeyChecking=yes -b /dev/null lab-data +``` + +3. Set the exact export root and create an isolated local workspace. The example +path is illustrative; it is not asserted to exist on the validation VM. + +```bash +export HM_DEMO_URL='ssh://lab-data/srv/exports/lab/' +HM_DEMO_WORKSPACE="$(mktemp -d "${TMPDIR:-/tmp}/hallmark-private-client.XXXXXX")" +export HM_DEMO_WORKSPACE +cd "$HM_DEMO_WORKSPACE" +``` + +SSH configuration alone is enough. To demonstrate an optional local profile, +create this file outside the catalog and clone: + +```bash +export HALLMARK_AUTH_FILE="$HM_DEMO_WORKSPACE/auth.yml" +cat > "$HALLMARK_AUTH_FILE" <<'YAML' +version: 1 +profiles: + demo: + hosts: [lab-data] + host_key_policy: strict + connect_timeout: 10 + transfer_timeout: 60 + max_sessions: 2 +YAML +chmod 600 "$HALLMARK_AUTH_FILE" +``` + +The profile uses the identity from SSH configuration. `hosts` matches the URL +alias `lab-data`, before SSH resolves its `HostName`. Each collaborator defines +their own local profile; catalog configuration stores only its name. + +4. Discover the two indexed runs and prepare a local `.hm`. + +```bash +hallmark clone "$HM_DEMO_URL" ./client \ + --auth demo --fmt 'runs/run_{run:03d}.dat' +cat ./client/.hm/config.yml +cat ./client/.hm/data.tsv +``` + +Expected for the illustrative export: two run rows, with sizes and modification +times where available. Published manifest checksums are attached when present; +missing checksums stay unknown. The data remote is `origin` with `auth: demo`. +Only listing and catalog metadata are fetched. No run bodies are downloaded for +hashing, and no remote commands or server Python are required. + +Omit `--fmt` to recursively catalog every file below the URL, including supporting +files. Alternatively, `--filter '**/*.dat'` selects paths without defining +parameters. Filters and formats never authorize downloads. Discovery reports +completed directories and discovered files while its total remains unknown. + +5. Inspect the transfer plan before approving any payload. + +```bash +cd "$HM_DEMO_WORKSPACE/client" +hallmark info +hallmark download --tsv data.tsv --dry-run +hallmark download runs/run_001.dat --dry-run +``` + +A dry run contacts no server. It reports count, known bytes, unknown-size count, +source, and destination; duration is unknown without a credible rate estimate. +It does not validate credentials or remote-file existence. To download the +selected run, inspect the displayed plan and answer `y` at the prompt: + +```bash +hallmark download runs/run_001.dat --max-workers 2 +cat runs/run_001.dat +``` + +Every nonempty CLI transfer prompts, including one file. Refusal or end-of-input +transfers nothing. `--yes` is deprecated and no longer bypasses approval. +Explicit file paths retain their catalog checksums. `--tsv data.tsv` selects +all rows in that table; `--all` selects all cataloged files. Overlapping paths +are deduplicated, and `--all` cannot be combined with explicit paths or TSVs. + +6. Use the SFTP URL alias and change a profile reference. + +```bash +hallmark set-config --remote-name origin \ + --remote-url "sftp://${HM_DEMO_URL#ssh://}" --remote-auth demo +hallmark download --filter 'runs/run_00[12].dat' --output ../via-sftp --dry-run + +# Return to SSH configuration alone, without a named profile. +hallmark set-config --remote-name origin --remote-auth '' +hallmark download runs/run_001.dat --output ../via-ssh-config --dry-run + +# Restore the profile for the Python examples. +hallmark set-config --remote-name origin --remote-auth demo +``` + +Both schemes use SFTP for discovery and transfer. SFTP-only accounts can complete +the whole workflow. Omit `--dry-run` when ready to inspect and approve a transfer. +Relative output paths use the current directory. A bare `.hm` catalog requires +an explicit `--output`. + +7. Inspect and approve a Python download. Stay in the `client` directory. +The script is saved locally so its approval prompt can read terminal input. + +```bash +cat > ../download_subset.py <<'PYTHON' +from hallmark import Repo +from hallmark.downloader import DownloadError + +repo = Repo(".") +plan = repo.plan_download( + output_path="../python-subset", file_paths=["runs/run_001.dat"], + remote_name="origin", +) +print(plan.summary()) +for item in plan.items: + print(item.relative_path, item.checksum, item.size_bytes) +if input("Download these files? [y/N] ").strip().lower() != "y": + raise SystemExit("Download declined") +try: + result = repo.download(plan, approved=True, max_workers=2, progress=True) +except DownloadError as exc: + raise SystemExit(f"Download could not start: {exc}") from exc +if result["failed"]: + raise SystemExit("\n".join(result["errors"])) +print(result) +PYTHON +python ../download_subset.py +``` + +The plan is immutable and pins its source, profile reference, destination, paths, +and checksums. Later catalog/configuration changes do not redirect it. Without +`approved=True`, nonempty Python transfers fail before connecting. Check the +returned `failed` count; successful files remain when another transfer fails. + +Files are written to temporary paths and published atomically after transfer and +any checksum verification. Failed files preserve an existing destination and +remove their temporary file. Repeating a download transfers the selection again. +Cancellation closes Hallmark-owned connections and processes. Byte progress shows +an ETA when a total is known; unknown totals stay indeterminate. + +8. Request an approved download during cloning, using either interface. +The CLI prepares the catalog first and then displays its normal approval prompt: + +```bash +hallmark clone "$HM_DEMO_URL" ../clone-and-download \ + --auth demo --filter 'runs/run_001.dat' --download +``` + +Python receives the completed plan through a callback: + +```bash +cat > ../clone_with_approval.py <<'PYTHON' +import os +from hallmark import Repo + + +def approve(plan): + print(plan.summary()) + return input("Download these files? [y/N] ").strip().lower() == "y" + + +repo = Repo.clone( + os.environ["HM_DEMO_URL"], "../python-clone", auth="demo", + filter="runs/run_001.dat", download=True, approve=approve, progress=True, +) +print(repo.dothm.path) +PYTHON +python ../clone_with_approval.py +``` + +9. Explore CyVerse and DESI with the same metadata-only workflow. These examples +use public endpoints, so no named auth profile is supplied. A broad URL can +require many directory-listing requests; use a specific subtree when appropriate. + +```bash +cd "$HM_DEMO_WORKSPACE" +hallmark clone \ + 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/' \ + ./cyverse --filter '**/*.uvfits' +hallmark clone 'https://data.desi.lbl.gov/public/' ./desi --filter '**/*.fits' + +cd "$HM_DEMO_WORKSPACE/cyverse" +hallmark download --all --dry-run +cd "$HM_DEMO_WORKSPACE/desi" +hallmark download --all --dry-run +``` + +Choose actual catalog paths before downloading, for example +`hallmark download PATH-FROM-THE-CATALOG`. This displays a plan and prompts for +approval. A format such as `--fmt 'runs/run_{run:03d}.dat'` is a parameterized +path template; a filter such as `--filter '**/*.fits'` is a relative path glob. +Neither supplies approval. No `--index-format cyverse-html` option is needed. + +The equivalent Python preparation, from the workspace root, is: + +```bash +cd "$HM_DEMO_WORKSPACE" +python - <<'PYTHON' +from hallmark import Repo + +cyverse = Repo.clone( + "https://data.cyverse.org/dav-anon/iplant/commons/" + "cyverse_curated/EHTC_FirstM87Results_Apr2019/", + "cyverse-python", filter="**/*.uvfits", progress=True, +) +desi = Repo.clone( + "https://data.desi.lbl.gov/public/", "desi-python", + filter="**/*.fits", progress=True, +) +for repo in (cyverse, desi): + plan = repo.plan_download() + print(plan.summary()) +PYTHON +``` + +For a subsequent selected download, use the inspect/approve sequence in step 7 +with `Repo("cyverse-python")` or `Repo("desi-python")` and an actual catalog path. +HTTPS download authentication retains Requests' `.netrc` and environment behavior. +A server with no usable index must expose a published Hallmark catalog or a +browsable directory; arbitrary HTTPS URLs cannot reveal hidden files. + +10. Reuse a catalog and finish the session. + +```bash +cd "$HM_DEMO_WORKSPACE" +hallmark clone ./client/.hm ./catalog-copy +hallmark init ./local-experiment +unset HALLMARK_AUTH_FILE +``` + +A local or Git-hosted Hallmark repository retains its history. A published +HTTP/SFTP catalog directory is imported as a snapshot with new local history. +Use `--source-type git` for a Git endpoint without an obvious `.git` suffix, or +`--source-type catalog` to require a published snapshot. Git credentials are +separate from the data remote's `--auth` profile. Filtering an existing Git +catalog preserves fetched history and adds a local filtered-catalog commit. + +`clone` downloads no dataset files by default, so `--no-fetch-data` is now only +a compatibility alias. `init` creates a local repository; the old remote `build` +command is deprecated. Discovery uses no `--allow-remote-commands` or +`--remote-hash` flag, and never reads dataset bodies to create checksums. +For local `add`/`commit` experiments, use a separate compatible one-format +repository; downloading a remote catalog does not populate the local object store. + +See [the private data guide](../doc/private_data.rst) for profile settings, +error handling and operational details. The auth-file override above lasts only +until `unset HALLMARK_AUTH_FILE`; existing catalog profile references still require +a matching local profile for later downloads. diff --git a/doc/api.rst b/doc/api.rst index 842c07f..d8ab88b 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -88,49 +88,51 @@ and interacting with Hallmark repositories. :members: :show-inheritance: -Building a repository ---------------------- - -Build a repository and choose filename formats interactively:: - - hallmark build ./repositories EHTC_2018L1_Dec2024 - -Load formats and remotes from an existing configuration:: - - hallmark build ./repositories EHTC_2018L1_Dec2024 \ - --config-file ./existing/config.yml +Preparing a repository +---------------------- -Provide filename formats directly:: +Create a local repository:: - hallmark build ./repositories EHTC_EXAMPLE \ - --fmt "images/{source}_{date}.fits=data" \ - --fmt "README.{format}=readme" + hallmark init ./local-project -Provide a custom remote or remotes:: +Clone a remote dataset without downloading dataset files:: - hallmark build ./repositories EHTC_EXAMPLE \ - --remote "origin=https://data.example.org/EHTC_EXAMPLE/" \ - --remote "backup=https://backup.example.org/EHTC_EXAMPLE/" + hallmark clone https://data.desi.lbl.gov/public/ ./desi --filter '**/*.fits' + hallmark clone ssh://lab-data/srv/data/ ./lab --fmt 'run{run:d}.h5' -The generated repository is named ``DATASET_NAME.hm``. The ``--fmt`` option -may be repeated, but it cannot be combined with ``--config-file``. +An existing Hallmark Git repository is cloned with its history. A published +HTTP/SFTP catalog snapshot starts new local history. Use ``--source-type git`` +for a Git endpoint whose URL cannot be distinguished from a directory URL. +The older ``build`` command remains a deprecated compatibility interface. Downloading remote data ----------------------- -Download specific remote-relative paths:: +Inspect the offline transfer plan and then confirm a download:: - hallmark download README.md data/example.fits + hallmark download --all --dry-run + hallmark download --filter 'runs/**' -Download everything represented by a configured TSV:: +Explicit paths and ``--tsv data.tsv`` also select files. Every nonempty transfer +requires interactive confirmation, including transfers requested through +``clone --download``. The old ``--yes`` option no longer bypasses approval. +A filter or filename format never authorizes a transfer. - hallmark download --tsv data.tsv +Python uses the same plan and requires explicit approval:: -Preview a complete repository download:: + from hallmark import Repo - hallmark download --all --dry-run + repo = Repo.clone('ssh://lab-data/srv/data/', 'lab', progress=True) + plan = repo.plan_download(filter='runs/**') + print(plan.summary()) + result = repo.download(plan, approved=True, progress=True) -Large selections require confirmation unless ``--yes`` is supplied. +Plans preserve their selected remote, destination and checksums even when +repository configuration later changes. Size and time estimates remain unknown +when metadata or a credible throughput estimate is unavailable. + +.. automodule:: hallmark.download_plan + :members: Dataset builders and data remotes --------------------------------- @@ -151,12 +153,14 @@ The repository stores the profile name. The SSH settings remain in the local authentication file. Passing ``remote_auth=None`` leaves the reference unchanged. -``download_remote_data`` returns a dictionary containing ``succeeded``, +The legacy ``download_remote_data`` function also requires ``approved=True``. +Both download APIs return a dictionary containing ``succeeded``, ``failed``, ``total_bytes``, and ``errors``. Check ``failed`` and ``errors`` for individual transfer failures. Configuration errors and failed SSH connection checks raise ``DownloadError`` before downloads begin. HTTP, SSH, and SFTP downloads use the same rules for destination paths and checksum verification. -See :ref:`private-lab-data-over-ssh` for a CLI workflow, authentication -settings, and supported server configurations. +See :doc:`private_data` for CLI and Python examples, and +:ref:`private-transport-reference` for authentication settings and +supported server configurations. diff --git a/doc/index.rst b/doc/index.rst index d1b0f53..3eec9a9 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -4,6 +4,10 @@ contain the root `toctree` directive. .. include:: ../README.rst + :end-before: Private data remotes + +For private datasets, see :doc:`private_data` for SSH setup, CLI workflows, +Python examples and troubleshooting. .. toctree:: :maxdepth: 2 @@ -11,6 +15,7 @@ design usecase + private_data api diff --git a/doc/private_data.rst b/doc/private_data.rst new file mode 100644 index 0000000..624a250 --- /dev/null +++ b/doc/private_data.rst @@ -0,0 +1,361 @@ +.. _private-data: + +Private data over SSH and SFTP +============================== + +Hallmark can discover a remote dataset, prepare a local catalog, and download +selected files after approval. ``clone`` accepts both an existing Hallmark +catalog and an ordinary data directory. Its default result is a local ``.hm`` +with no dataset payloads. A **data remote** identifies the dataset location; +it is separate from the Git remote used to share catalog history. + +The SSH examples use the alias ``lab-data`` and the absolute dataset root +``/srv/exports/lab/``. Replace the hostname, username, identity path and root +with your own values. Run Hallmark locally. The server needs SFTP access, but +neither Hallmark, Git, Python nor a login shell. + +1. Configure and check SSH access +--------------------------------- + +Install this checkout with ``python -m pip install -e .``. The client requires +system OpenSSH ``ssh`` and ``sftp`` version 9.6 or newer, with connection +multiplexing available. Linux and macOS have integration-test jobs; Windows +SSH transport is unsupported. + +Add an entry to your local ``~/.ssh/config``: + +.. code-block:: text + + Host lab-data + HostName data.example.org + User researcher + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes + StrictHostKeyChecking yes + +Have your administrator provision the server's verified host key in +``known_hosts``, or verify its fingerprint through a trusted channel before +accepting it. Load an encrypted key into your SSH agent before running Hallmark. +Check that SFTP works without an interactive prompt: + +.. code-block:: bash + + ssh -V + sftp -o BatchMode=yes -b /dev/null lab-data + +Hallmark uses noninteractive authentication and strict host-key checking. It +cannot answer password, MFA or initial hardware-key prompts. Ordinary SSH +configuration, including ``ProxyJump``, remains available. + +2. Discover a remote catalog +---------------------------- + +Suppose the export contains ``runs/run_001.h5``, ``runs/run_002.h5`` and +``README.md``. From an empty local workspace, run: + +.. code-block:: bash + + hallmark clone 'ssh://lab-data/srv/exports/lab/' ./lab \ + --fmt 'runs/run_{run:03d}.h5' + +This creates ``./lab/.hm`` and catalogs only the matching run files. The URL +is the exact discovery root. A filename format selects paths and extracts +parameters; it does not approve a download. A glob filter can select paths +without defining parameters: + +.. code-block:: bash + + hallmark clone 'sftp://lab-data/srv/exports/lab/' ./lab-h5 \ + --filter '**/*.h5' + +Without ``--filter`` or ``--fmt``, discovery recursively covers every directory +beneath the URL and catalogs all discovered files. A filter still traverses +directories needed to find matching files. Discovery shows an indeterminate +progress bar with completed directories and discovered files; it does not +invent a percentage or completion time while the total is unknown. + +Both URL schemes use structured SFTP directory enumeration and file transfers. +SFTP-only accounts work for discovery as well as downloading. No +``--allow-remote-commands`` or ``--remote-hash`` option is needed. Discovery +reads listings and published checksum manifests, never dataset bodies to +manufacture a checksum. Missing digests stay unknown. For reproducible +catalogs, publish an immutable export with a SHA-256 manifest, for example +`` runs/run_001.h5`` records in ``SHA256SUMS``. + +The generated data remote is named ``origin`` and points to the source URL. +Use ``hallmark init PATH`` to initialize a local repository. The older remote +``build`` command and ``build_repo`` API are deprecated in favor of ``clone``; +server-side hashing is no longer part of discovery. + +3. Preview, approve and download +-------------------------------- + +Continue from the same workspace: + +.. code-block:: bash + + cd lab + hallmark info + hallmark download --tsv data.tsv --dry-run + hallmark download runs/run_001.h5 + +The dry run reads only the local catalog. It reports file count, known bytes, +the number of unknown file sizes, destination and source. Duration stays +unknown unless a credible transfer-rate estimate is available. The dry run +does not test credentials, host trust or remote-file existence. + +Every nonempty CLI download displays its plan and asks +``Download these files? [y/N]``. Answer ``y`` to authorize the displayed +transfer. Refusal, a blank answer or end-of-input transfers no dataset files. +The deprecated ``--yes`` option is accepted but does not bypass confirmation. +During transfer, byte progress and measured throughput provide an ETA when the +total is known; unknown totals remain indeterminate. + +Choose a scope from inside ``lab``: + +.. code-block:: bash + + hallmark download --filter 'runs/run_00[12].h5' --dry-run + hallmark download --all --output ./payload + +Explicit paths use their recorded catalog checksums when available, just like +``--tsv`` and ``--all``. A path absent from the catalog can be requested, but +its size and checksum are unknown. Files without a usable publisher checksum +can download; successful transfer alone does not establish catalog integrity. + +``--tsv`` can be repeated and combined with explicit paths; overlapping entries +are deduplicated. ``--all`` cannot be combined with either selection mode. +Relative output paths are relative to the current directory. Without +``--output``, downloads go to the repository worktree; bare ``.hm`` repositories +require an explicit output directory. Repeating a download transfers the +selection again and replaces destinations only after successful transfer and +any recorded checksum verification. + +To request downloads during cloning, add ``--download``. Catalog preparation +finishes first, then the same plan and approval prompt appear: + +.. code-block:: bash + + hallmark clone 'ssh://lab-data/srv/exports/lab/' ../lab-one \ + --filter 'runs/run_001.h5' --download + +An existing catalog can be cloned from a local path, a Git endpoint, or an +HTTP/SFTP directory containing its metadata: + +.. code-block:: bash + + hallmark clone ./.hm ../lab-copy + hallmark clone 'https://git.example.org/team/lab.git' ../lab-history + hallmark clone 'https://data.example.org/catalogs/lab.hm/' ../lab-snapshot + +Git endpoints preserve history. Published HTTP/SFTP snapshots start new local +history. For a Git URL without an obvious ``.git`` suffix, specify +``--source-type git``; ``--source-type catalog`` or ``directory`` explicitly +selects a published snapshot or discovery. Filtering a Git clone preserves +fetched history and commits the filtered catalog locally. Git authentication +and data-server authentication are independent. ``--no-fetch-data`` remains a +compatibility alias for the default metadata-only behavior. + +4. Use a local authentication profile +------------------------------------- + +Profiles are optional when SSH configuration already provides access. They let +each collaborator supply credentials locally while the catalog records only a +profile name. Add the following to ``~/.config/hallmark/auth.yml``, merging it +with any existing profiles: + +.. code-block:: yaml + + version: 1 + profiles: + lab: + hosts: [lab-data] + user: researcher + identity_file: ~/.ssh/id_ed25519 + host_key_policy: strict + connect_timeout: 10 + transfer_timeout: 3600 + shutdown_timeout: 2 + max_sessions: 4 + +Keep this file outside the repository and restrict it to your account, for +example with ``chmod 600 ~/.config/hallmark/auth.yml``. Hallmark reads +``$HALLMARK_AUTH_FILE`` when set, otherwise +``$XDG_CONFIG_HOME/hallmark/auth.yml``, otherwise the path above. Every client +using a catalog's profile reference must define that profile locally. + +Associate an existing data remote with a profile, or select one when cloning +an ordinary remote directory: + +.. code-block:: bash + + hallmark set-config --remote-name origin --remote-auth lab + hallmark download --remote origin --tsv data.tsv --dry-run + hallmark clone 'ssh://lab-data/srv/exports/lab/' ../lab-profile --auth lab + +The new catalog's ``origin`` records ``auth: lab``. Clear a reference with +``hallmark set-config --remote-name origin --remote-auth ''`` to use SSH +configuration alone. ``--auth`` applies to data access; Git cloning uses Git's +own authentication configuration. + +``hosts`` binds the profile to the URL's alias or literal hostname before +``HostName`` resolution. This example binds to ``lab-data``, not +``data.example.org``. Explicit URL user/port values override profile settings; +unset values fall through to SSH configuration. Passwords, tokens and arbitrary +SSH options are not valid profile fields. ``transfer_timeout`` is the total +per-file time budget in seconds, not an idle timeout. Effective SSH download +concurrency is the smaller of ``--max-workers`` and local ``max_sessions`` +(both default to 4). + +5. Use the Python API +--------------------- + +These examples use fresh local destinations. Like the CLI, ``Repo.clone`` +discovers metadata without downloading dataset files: + +.. code-block:: python + + from hallmark import Repo + + repo = Repo.clone( + "ssh://lab-data/srv/exports/lab/", "lab-python", + auth="lab", # Omit when SSH configuration already supplies access. + fmt="runs/run_{run:03d}.h5", + progress=True, + ) + plan = repo.plan_download(file_paths=["runs/run_001.h5"]) + print(plan.summary()) + print(plan.items) # Paths, checksums, optional size_bytes and mtime. + +Inspect the plan before making the explicit approval call: + +.. code-block:: python + + result = repo.download(plan, approved=True, progress=True) + if result["failed"]: + raise RuntimeError("\n".join(result["errors"])) + +Without ``approved=True``, a nonempty transfer raises ``DownloadError`` before +opening a connection. A plan freezes the selected files, source URL, profile +reference and destination; later catalog or remote-config changes do not +redirect it. ``plan_download(filter="**/*.h5")`` scopes the complete catalog; +``plan_download(output_path="subset", tsv_names=["data.tsv"])`` selects a TSV +and destination. Planning makes no network requests. Known size totals and +unknown-size counts are available as ``known_bytes`` and +``unknown_size_count``. ``estimated_seconds`` stays ``None`` unless all sizes +are known and ``estimated_bytes_per_second`` was supplied when planning. + +For downloads during cloning, supply a callback that reviews the completed +plan and returns the Boolean ``True`` to approve: + +.. code-block:: python + + def approve(plan): + print(plan.summary()) + return input("Download these files? [y/N] ").strip().lower() == "y" + + repo = Repo.clone( + "ssh://lab-data/srv/exports/lab/", "lab-with-download", + filter="runs/run_001.h5", download=True, approve=approve, progress=True, + ) + +``repo.download`` returns ``succeeded``, ``failed``, ``total_bytes`` and +``errors``. Inspect ``failed`` because successful files remain when another +file fails. Setup failures raise ``hallmark.downloader.DownloadError``. +``repo.set_config(remote_auth="")`` removes a profile reference; ``None`` +leaves it unchanged. + +6. Discover public HTTPS collections +------------------------------------ + +The same workflow supports CyVerse and ordinary browsable HTTPS collections +such as DESI. No CyVerse-specific index option is required: + +.. code-block:: bash + + hallmark clone \ + 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/' \ + ./cyverse --filter '**/*.uvfits' + hallmark clone 'https://data.desi.lbl.gov/public/' ./desi --filter '**/*.fits' + +Each URL is the full recursive discovery root. A broad root can require many +listing requests even with a file filter; use a specific subtree when that is +your intended scope. Public collections need no auth profile. Private HTTP +access retains Requests' normal ``.netrc`` and environment behavior; Hallmark's +named auth profiles apply to SSH/SFTP. + +From either clone, inspect and approve a selected subset using +``hallmark download --filter PATTERN --dry-run`` and then the same command +without ``--dry-run``. Python follows the same steps: + +.. code-block:: python + + cyverse = Repo.clone( + "https://data.cyverse.org/dav-anon/iplant/commons/" + "cyverse_curated/EHTC_FirstM87Results_Apr2019/", + "cyverse-python", filter="**/*.uvfits", progress=True, + ) + desi = Repo.clone( + "https://data.desi.lbl.gov/public/", "desi-python", + filter="**/*.fits", progress=True, + ) + plan = desi.plan_download(filter="dr1/**/selected-file.fits") + print(plan.summary()) + # Replace the illustrative pattern with an actual catalog path, inspect, + # and authorize only the intended selection: + # result = desi.download(plan, approved=True, progress=True) + +HTTP provides file transfer but does not define directory enumeration. Hallmark +recognizes supported HTML indexes and directory landing pages automatically. +If a server supplies no usable listing, provide a published Hallmark catalog +or a more specific browsable URL; Hallmark cannot +infer hidden file URLs. + +Operational behavior and troubleshooting +---------------------------------------- + +Files are downloaded to temporary paths and published atomically after transfer +and any recorded checksum verification. A failed transfer preserves an existing +destination and removes its temporary file. Successful files remain available +when another file fails; a multi-file download is not a single transaction. +The CLI exits nonzero on transfer failures. Cancellation stops operation-owned +work and closes Hallmark's SSH connections and processes. + +.. list-table:: Common failures + :header-rows: 1 + :widths: 30 70 + + * - Symptom + - Action + * - SSH connection preflight fails + - Check the trusted host key, loaded identity, username, server availability + and OpenSSH version. For profile-only settings, pass the same + identity/user/port to the manual SFTP check. + * - Profile unresolved or not bound to the host + - Check the auth-file path, profile name and URL host against ``hosts``. + Another collaborator's local profile is not shared with the catalog. + * - No supported remote listing + - Use a browsable collection root or published catalog. Confirm that an + SFTP account can enumerate the requested directories. + * - Checksum mismatch + - Check whether the export changed since catalog creation. Reconcile the + trusted catalog and source snapshot before retrying. + * - Transfer timeout or session limit + - Adjust the local per-file time budget or lower concurrency to match + the server's capacity. + +Data URLs require absolute roots, for example +``ssh://researcher@lab-data:2222/srv/exports/lab/``. SCP shorthand is unsupported +for data directories. Percent-encode reserved characters in URL roots; catalog +paths are literal relative names. Hallmark rejects traversal and observed +destination symlinks; server permissions remain the access-control boundary. + +Discovered catalogs can use path rows and published catalogs can contain +multiple TSVs. Existing local ``add``/``commit``/``checkout`` workflows still +require a compatible one-format ``data.tsv`` repository; downloading does not +seed the local object store. For local experiments, initialize a separate +repository, copy in the downloaded files, configure their filename format, +then add and commit them. + +See :ref:`private-transport-reference` for profile settings, HTTP compatibility +and disposable transport tests, and :doc:`api` for function signatures. diff --git a/doc/usecase.rst b/doc/usecase.rst index ccb4108..5226b83 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -122,11 +122,15 @@ This is especially useful for data (re-)organization. .. _hallmark: https://github.com/l6a/hallmark +.. _private-transport-reference: + .. _private-lab-data-over-ssh: 6. CLI: Private Lab Data over SSH ---------------------------------- +For a step-by-step CLI workflow and Python examples, see :doc:`private_data`. + Frank works with simulation data stored on a private lab server. He has an existing |hallmark|_ repository that indexes the files and wants to download them to his local worktree. @@ -259,74 +263,63 @@ The local profile or defaults may use ``host_key_policy: accept-new`` to accept a host key on first contact. Changed host keys still cause an error. Repository configuration cannot enable this setting. -Build a private catalog +Clone a private catalog ~~~~~~~~~~~~~~~~~~~~~~~ -Frank can also build a catalog directly from the files on the server. -He supplies the dataset directory and a filename format:: - - hallmark build catalogs lab \ - --dataset-url ssh://campus/srv/export/ \ - --dataset-auth campus --allow-remote-commands \ - --fmt 'runs/run_{i:d}.h5=data.tsv' - -This creates ``catalogs/lab.hm`` with an index of the matching run files -in ``data.tsv``. The name ``lab`` labels the catalog; it is not appended -to ``--dataset-url``. - -Building over SSH requires a POSIX login shell and Python 3 on the server. -The ``--allow-remote-commands`` option permits fixed, read-only commands -to list files recursively and, when requested, compute checksums. -This permission must be supplied explicitly; repository configuration -cannot enable it. -SFTP-only accounts can download indexed files but cannot build catalogs. -Listings skip symlinks, including symlinked directories, and special files. -The dataset directory defines where listing starts; access permissions -must be enforced on the server. - -|hallmark|_ uses existing checksum manifests when available. -Supported manifests use unescaped GNU sum records: a hexadecimal -checksum, a space, a space or ``*`` marker, and the literal filename. -Algorithm names such as ``sha256`` are retained, and GNU records with -escaped filenames are rejected. -Missing checksums remain absent or ``unknown``. Authentication failures -and SSH errors with an unclear cause stop the build instead of being -treated as missing manifests. - -Frank can add ``--remote-hash`` to compute SHA-256 checksums on the server -for files without a manifest checksum. -This is limited to 10 MiB per file, 100 MiB in total, and 60 seconds for -the operation. Files beyond these limits retain unknown checksums. -Detected changes to file size or modification time during hashing cause -an error. Dataset files are not downloaded to compute SSH checksums. - -Recursive listing is limited to 100,000 files, 16 MiB of output, and a -60-second command timeout. Text reads are limited to 16 MiB per file -and 64 MiB in total; new reads stop after five minutes of crawling. - -By default, the generated ``origin`` data remote records the source URL -and its profile name. -The ``--remote NAME=URL`` option records a different download location -without changing the directory being indexed or copying its profile name. -A remote specified by name alone uses the source URL. -In either case, a profile for that remote must be configured separately -if needed. - -Without ``--dataset-url``, the builder looks up the dataset name on -CyVerse as before. -An explicit HTTP directory requires ``--index-format cyverse-html`` and -a matching HTML index. Other HTTP directory indexes and WebDAV PROPFIND -are not supported. For HTTP files without a recorded checksum, the -builder retains its existing size and time limits for computing MD5. -Building and downloading can use multiple catalogs; local ``add``, -``commit``, and ``checkout`` operations require one format backed by -``data.tsv``. - -For reproducible results, Frank uses an immutable export or a server -snapshot. Listing and hashing a directory while its files change cannot -guarantee a consistent snapshot. -Later downloads verify any usable checksums included in their selections; -missing or unknown checksums do not prevent a transfer. +Frank can prepare a catalog directly from the files on the server:: + + hallmark clone ssh://campus/srv/export/ lab \ + --auth campus --fmt 'runs/run_{i:d}.h5' + cd lab + hallmark download --all --dry-run + hallmark download --all + +This creates ``lab/.hm`` with only the matching run files in ``data.tsv``. +Without a filter or format, the catalog contains all discovered files below +the supplied URL. No dataset contents are downloaded while cloning. +The final command displays the transfer plan and asks Frank for confirmation. + +Both SSH and SFTP sources use structured SFTP directory enumeration. The server +needs no login shell, Python or Hallmark, and SFTP-only accounts work for +both discovery and downloads. Symlinks and special files are skipped. +Authentication and permission failures stop discovery rather than producing +an apparently complete catalog. + +The generated data ``origin`` records the source URL and optional profile name. +Existing Git-hosted catalogs keep their recorded data remotes when cloned. +A filter on an existing Git catalog creates a local metadata commit while +preserving fetched history; it does not fetch the dataset's content objects. + +Published checksum manifests are retained. Missing checksums remain unknown; +Hallmark never downloads files or hashes them on the server merely to finish +a catalog. Downloads verify any available publisher checksum. No global +five-minute crawl deadline or 100,000-entry cap limits the requested tree; +individual requests remain bounded and discovery can be cancelled. + +CyVerse and common HTTPS directory indexes are detected automatically:: + + hallmark clone https://data.desi.lbl.gov/public/ desi --filter '**/*.fits' + +A server with no usable listing must provide a published catalog or file +manifest through a supported adapter. Discovery cannot enumerate arbitrary +hidden URLs. HTTP/SFTP catalog snapshots start local history; Git endpoints +supply catalog history. Use ``--source-type git`` for ambiguous Git URLs. + +Frank can also inspect and approve a transfer in Python:: + + from hallmark import Repo + + repo = Repo('lab') + plan = repo.plan_download(filter='runs/run_1.h5') + print(plan.summary()) + result = repo.download(plan, approved=True, progress=True) + +Plans are built from local metadata without contacting the server. Missing +sizes and duration estimates are reported as unknown. During transfer, +progress reports bytes and estimates remaining time when possible. +``build`` and ``build_repo`` remain deprecated compatibility interfaces. +Local ``add``, ``commit`` and ``checkout`` retain their existing format and +local-object requirements; preparing a remote catalog does not materialize it. Transport validation ~~~~~~~~~~~~~~~~~~~~ diff --git a/mod/hallmark/__init__.py b/mod/hallmark/__init__.py index efbd637..1bfdf5e 100644 --- a/mod/hallmark/__init__.py +++ b/mod/hallmark/__init__.py @@ -17,5 +17,6 @@ from .repo import Repo from .paraframe import ParaFrame +from .download_plan import DownloadItem, DownloadPlan -__all__ = ["Repo", "ParaFrame"] +__all__ = ["Repo", "ParaFrame", "DownloadItem", "DownloadPlan"] diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py new file mode 100644 index 0000000..1aff9a8 --- /dev/null +++ b/mod/hallmark/catalog.py @@ -0,0 +1,238 @@ +"""Prepare remote inventories and published catalogs without dataset transfers.""" + +from __future__ import annotations + +import re +from io import StringIO +from pathlib import Path +from shutil import rmtree +from urllib.parse import urlsplit + +import pandas as pd +import parse +import yaml + +from .discovery import discover, path_matches +from .dothm import Dothm +from .error import CloneError, DestinationExistsError +from .helper_functions import as_list_of_dicts +from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name, row_to_path +from .transport import OperationContext, RemoteSpec +from .transport.base import RemoteObjectMissing, literal_path +from .worktree import Worktree + + +def _catalog_names(config): + """Validate catalog filenames before reading any remote metadata.""" + entries = as_list_of_dicts(config.get("data", [])) + if entries is None or any(not isinstance(entry, dict) for entry in entries): + raise CloneError("Catalog data must be a mapping or list of mappings") + names = {"data.tsv"} + for entry in entries: + if entry.get("db"): + names.add(normalize_tsv_name(entry["db"])) + if entry.get("file"): + literal_path(entry["file"]) + normalize_remotes(config.get("remote")) + return sorted(names) + + +def _catalog_formats(config, name): + entries = as_list_of_dicts(config.get("data", [])) or [] + return [entry["fmt"] for entry in entries if entry.get("fmt") + and normalize_tsv_name(entry.get("db", "data.tsv")) == name] + + +def _row_path(row, formats): + value = row.get("path") + if value is not None and not pd.isna(value) and str(value): + return literal_path(str(value)).as_posix() + paths = [] + for template in formats: + try: + paths.append(row_to_path(row, template).as_posix()) + except (ValueError, KeyError): + continue + if len(set(paths)) != 1: + raise CloneError("Cannot resolve a catalog row to a unique file path") + return paths[0] + + +def _validate_snapshot(files, config): + """Reject login pages and unsafe inventories before publishing a snapshot.""" + for name in _catalog_names(config): + try: + frame = pd.read_csv(StringIO(files[name]), sep="\t", dtype=str, + keep_default_na=False) + except (pd.errors.EmptyDataError, pd.errors.ParserError) as exc: + raise CloneError(f"Invalid published catalog table: {name}") from exc + formats = _catalog_formats(config, name) + has_fields = any(fmt_fields(fmt) and set(fmt_fields(fmt)) <= set(frame.columns) + for fmt in formats) + if not ({"path", "sha1"} & set(frame.columns) or has_fields): + raise CloneError(f"Unrecognized published catalog columns: {name}") + for _, row in frame.iterrows(): + _row_path(row, formats) + + +def _snapshot(context): + """Probe fixed catalog metadata names, never a dataset file or object store.""" + try: + config_text = context.read_text("config.yml") + except RemoteObjectMissing: + return None + try: + config = yaml.safe_load(config_text) + except yaml.YAMLError: + return None + if not isinstance(config, dict) or "data" not in config: + return None + try: + meta_text = context.read_text("meta.yml") + data_text = context.read_text("data.tsv") + except RemoteObjectMissing: + return None + try: + meta = yaml.safe_load(meta_text) + except yaml.YAMLError as exc: + raise CloneError("Invalid published catalog metadata") from exc + if not isinstance(meta, dict): + raise CloneError("Published catalog meta.yml must be a mapping") + files = {"config.yml": config_text, "meta.yml": meta_text, + "data.tsv": data_text} + for name in _catalog_names(config): + if name not in files: + files[name] = context.read_text(name) + _validate_snapshot(files, config) + return files + + +def _metadata_commit(repo, files, message): + """Commit only catalog metadata, without capturing local dataset objects.""" + repo.dothm.index.add(list(files)) + if repo.dothm.index.diff("HEAD"): + repo.dothm.index.commit(message) + repo.state = repo.dothm.load() + + +def _filter_catalog(repo, *, filter=None, fmt=None): + if filter is None and fmt is None: + return + config = repo.state.config + names = _catalog_names(config) + for name in names: + frame = repo.dothm.load_tsv(name) + formats = _catalog_formats(config, name) + keep = [] + for _, row in frame.iterrows(): + path = _row_path(row, formats) + keep.append(path_matches(path, filter=filter, fmt=fmt)) + repo.dothm.dump_tsv(frame.loc[keep], name) + for section in ("data", "meta"): + values = as_list_of_dicts(config.get(section, [])) + if values is None: + continue + config[section] = [entry for entry in values + if not entry.get("file") or path_matches( + entry["file"], filter=filter, fmt=fmt)] + repo.dothm.dump_yml(config, "config") + _metadata_commit(repo, [*names, "config.yml"], "Filter local catalog") + + +def _write_inventory(repo, source, entries, fmt): + columns = ["path", "checksum_algorithm", "checksum", "size_bytes", "mtime"] + rows = [] + for entry in entries: + row = dict(zip(columns, (entry.path, entry.checksum_algorithm, + entry.checksum, entry.size, entry.mtime))) + if fmt: + matched = parse.parse(fmt, entry.path) + if matched is not None: + row.update({key: value for key, value in matched.named.items() + if key not in columns}) + rows.append(row) + frame = pd.DataFrame(rows) + if frame.empty: + frame = pd.DataFrame(columns=columns) + data_spec = {"db": "data.tsv"} + if fmt: + data_spec["fmt"] = fmt + remote = {"name": "origin", "url": source.url} + if source.auth: + remote["auth"] = source.auth + repo.dothm.dump_yml({"data": [data_spec], "remote": [remote]}, "config") + repo.dothm.dump_yml({"source": source.url}, "meta") + repo.dothm.dump_tsv(frame, "data", na_rep="") + _metadata_commit(repo, ["config.yml", "meta.yml", "data.tsv"], + "Discover remote catalog") + + +def _git_source(url, source_type): + if source_type == "git": + return True + if source_type != "auto": + return False + parsed = urlsplit(url) + return (not parsed.scheme or parsed.scheme in {"git", "file"} + or bool(re.match(r"^[^/]+@[^/]+:", url)) + or (parsed.scheme != "sftp" and parsed.path.rstrip("/").endswith(".git"))) + + +def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, + source_type="auto", progress=False): + """Create a complete local catalog or remove this operation's destination.""" + if source_type not in {"auto", "git", "directory", "catalog"}: + raise ValueError("source_type must be auto, git, directory, or catalog") + url = str(url) + path_matches("validation", filter=filter, fmt=fmt) + destination = Path(path).expanduser().absolute() + if destination.exists() or destination.is_symlink(): + raise DestinationExistsError( + f"fatal: destination path '{path}' already exists and is not empty.") + is_git = _git_source(url, source_type) + if is_git and auth is not None: + raise ValueError("Git cloning uses Git/SSH authentication, not auth profiles") + # Reserve ownership before cleanup becomes possible; never remove another repo. + destination.parent.mkdir(parents=True, exist_ok=True) + try: + destination.mkdir() + except FileExistsError as exc: + raise DestinationExistsError(f"Destination already exists: {path}") from exc + try: + if is_git: + dothm_path, worktree_path = cls.lwpaths(destination) + local = Path(url).expanduser() + git_url = str(local / ".hm") if (local / ".hm").is_dir() else url + Dothm.clone(git_url, dothm_path, display_path=path) + if worktree_path: + Worktree.init(worktree_path) + repo = cls(destination) + _filter_catalog(repo, filter=filter, fmt=fmt) + return repo + + source = RemoteSpec.parse(url, auth) + snapshot = None + with OperationContext(source) as context: + if source_type != "directory": + snapshot = _snapshot(context) + if snapshot is None and not source.root.rstrip("/").endswith(".hm"): + nested = RemoteSpec.parse(url.rstrip("/") + "/.hm/", auth) + with OperationContext(nested) as nested_context: + snapshot = _snapshot(nested_context) + if snapshot is None: + if source_type == "catalog": + raise CloneError("No published Hallmark catalog at this URL") + entries = discover(context, filter=filter, fmt=fmt, + progress=progress) + repo = cls.init(destination) + if snapshot is not None: + for name, contents in snapshot.items(): + (repo.dothm.path / name).write_text(contents, encoding="utf-8") + _metadata_commit(repo, snapshot, "Import published catalog snapshot") + _filter_catalog(repo, filter=filter, fmt=fmt) + else: + _write_inventory(repo, source, entries, fmt) + return repo + except BaseException: + rmtree(destination, ignore_errors=True) + raise diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 6cfbd90..719eaba 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -26,11 +26,7 @@ from . import Repo from .helper_functions import validate_path_component from .repo_builder import build_repo -from .downloader import ( - BULK_DOWNLOAD_WARNING_FILE_COUNT, - DownloadError, - download_remote_data, - select_download_files) +from .downloader import DownloadError from .error import CheckoutError, CloneError from .repo_config import normalize_tsv_name @@ -81,30 +77,6 @@ def _translate_cli_errors(*error_types, prefix=None): yaml.YAMLError) -def _confirm_bulk_download( - selected_files, - assume_yes: bool, - ) -> None: - """ - Used by _run_download. - Prompt the user for confirmation if the number of selected files exceeds - the warning threshold and the user has not opted to assume yes. - Args: - selected_files (list): List of files selected for download. - assume_yes (bool): Flag indicating whether to assume yes for prompts. - Raises: - ClickException: If the user does not confirm the download. - """ - file_count = len(selected_files) - if ( - file_count >= BULK_DOWNLOAD_WARNING_FILE_COUNT - and not assume_yes): - click.echo( - f"Selected {file_count} files for download.\n" - "The total size is not recorded and may be very large.") - click.confirm("Continue?", abort=True) - - def _report_download_results(results: dict) -> None: """ Used by _run_download. @@ -143,29 +115,15 @@ def _report_download_results(results: dict) -> None: raise ClickException(f"Failed to download {failed} file(s)") -def _run_download( - repo, - selected_files, - output_path, - *, - max_workers, - assume_yes, - remote_name=None, - ) -> None: - """ - Used by download and clone. - Confirm, execute, and report one selected download.""" - # confirm with user if the number of selected files exceeds warning threshold - _confirm_bulk_download(selected_files, assume_yes) - # download the selected files from the remote repository - results = download_remote_data( - repo, - output_path, - max_workers=max_workers, - show_progress=True, - selected_files=selected_files, - remote_name=remote_name) - # report the results of the download operation to the user +def _run_download(repo, plan, *, max_workers): + """Approve and execute exactly the plan displayed by clone or download.""" + click.echo(plan.summary()) + if not plan.file_count: + click.echo("No files selected for download.") + return + click.confirm("Download these files?", default=False, abort=True) + results = repo.download(plan, approved=True, max_workers=max_workers, + progress=True) _report_download_results(results) @@ -404,190 +362,91 @@ def checkout(repo, target_branch): click.echo(f'Switched to branch "{target_branch}".') -@hallmark.command( - short_help="Download files from the configured data remote.") +@hallmark.command(short_help="Plan and approve a dataset download.") @click.argument("files", nargs=-1) -@click.option( - "--tsv", - "tsv_names", - multiple=True, - help="Download every file represented by a configured TSV. " - "May be repeated.") -@click.option( - "--all", - "download_all", - is_flag=True, - help="Download every configured TSV, static file, and metadata file.") -@click.option( - "--remote", - "remote_name", - help="Name of the configured remote to use.") -@click.option( - "--output", - type=click.Path(file_okay=False), - help="Output directory. Defaults to the repository worktree.") -@click.option( - "--max-workers", - type=click.IntRange(min=1), - default=4, - show_default=True, - help="Number of concurrent downloads.") -@click.option( - "--dry-run", - is_flag=True, - help="Show the selected files without downloading them.") -@click.option( - "-y", - "--yes", - is_flag=True, - help="Skip the bulk-download confirmation.") +@click.option("--tsv", "tsv_names", multiple=True, + help="Select a catalog TSV. May be repeated.") +@click.option("--all", "download_all", is_flag=True, + help="Select all cataloged files.") +@click.option("--filter", "filters", multiple=True, + help="Relative path glob; ** matches recursively. May be repeated.") +@click.option("--fmt", help="Select a parameterized filename template.") +@click.option("--remote", "remote_name", help="Configured data remote name.") +@click.option("--output", type=click.Path(file_okay=False), + help="Output directory; defaults to the worktree.") +@click.option("--max-workers", type=click.IntRange(min=1), default=4, + show_default=True) +@click.option("--dry-run", is_flag=True, + help="Show the offline transfer plan without downloading.") +@click.option("-y", "--yes", is_flag=True, hidden=True) @click.pass_obj -def download(repo, files, tsv_names, download_all, remote_name, output, max_workers, - dry_run, yes): - """ - Download files from the configured data remote. Options allow for selecting - specific files, TSVs, or downloading all files. Supports concurrent downloads - and dry-run mode for previewing selected files. - - Options: - --tsv: Download every file represented by a configured TSV. May be repeated. - --all: Download every configured TSV, static file, and metadata file. - --remote: Name of the configured remote to use. - --output: Output directory. Defaults to the repository worktree. - --max-workers: Number of concurrent downloads. Default is 4. - --dry-run: Show the selected files without downloading them. - -y, --yes: Skip the bulk-download confirmation. - - Raises: - ClickException: If there are any issues with the provided arguments or - during the download process. - """ - # do not allow --all to be combined with file paths or --tsv +def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, + output, max_workers, dry_run, yes): + """Inspect sizes and approve every nonempty dataset transfer.""" if download_all and (files or tsv_names): raise ClickException("--all cannot be combined with file paths or --tsv") - # require at least one of file paths, --tsv, or --all to be provided - if not files and not tsv_names and not download_all: - raise ClickException("Provide one or more file paths, --tsv, or --all") - - # if an output directory is specified, use it; otherwise, use the repo worktree - if output: - output_path = Path(output).expanduser() - elif repo.worktree is not None: - output_path = Path(repo.worktree) - # output is required when downloading from a bare .hm repository - else: - raise ClickException( - "--output is required when downloading from a bare .hm repository") - - # use the _translate_cli_errors context manager to handle DownloadError exceptions - with _translate_cli_errors(DownloadError): - selected_files = select_download_files( - repo, - file_paths=files, - tsv_names=tsv_names, - all_files=download_all) - click.echo(f"Selected {len(selected_files)} file(s) "f"for {output_path}") - - # don't actually download the files if --dry-run is specified + if not files and not tsv_names and not download_all and not filters and not fmt: + raise ClickException("Provide file paths, --tsv, --all, --filter, or --fmt") + if repo.worktree is None and output is None: + raise ClickException("--output is required when downloading from a bare repo") + if yes: + click.echo("--yes is deprecated; downloads still require confirmation.", + err=True) + with _translate_cli_errors(DownloadError, ValueError): + plan = repo.plan_download( + output, file_paths=files, tsv_names=tsv_names, all_files=download_all, + filter=filters or None, fmt=fmt, remote_name=remote_name) if dry_run: - # limit the number of files to preview to avoid overwhelming the user - preview_limit = 20 - # for each selected file, print its relative path to the user - for rel_path, _sha1 in selected_files[:preview_limit]: - click.echo(f" {rel_path.as_posix()}") - - remaining = len(selected_files) - preview_limit - # if there are more files than the preview limit - if remaining > 0: - # indicate how many more files are selected - click.echo(f" ... {remaining} more file(s)") - # bail out of the function early since this is a dry run + click.echo(plan.summary()) + for item in plan.items[:20]: + click.echo(f" {item.relative_path.as_posix()}") + if plan.file_count > 20: + click.echo(f" ... {plan.file_count - 20} more file(s)") return - - if not selected_files: - click.echo("No files selected for download.") - # bail out of the function early since there are no files to download - return - - # attempt to download the selected files, handling any errors - _run_download( - repo, - selected_files, - output_path, - max_workers=max_workers, - assume_yes=yes, - remote_name=remote_name) + _run_download(repo, plan, max_workers=max_workers) -@hallmark.command(short_help="Clone a hallmark repository from a remote URL.") +@hallmark.command(short_help="Clone a catalog or discover a remote dataset.") @click.argument("url") @click.argument("path") -@click.option( - "--no-fetch-data", - is_flag=True, - help="Skip downloading remote data files after clone.") -@click.option( - "--max-workers", - type=click.IntRange(min=1), - default=4, - show_default=True, - help="Number of concurrent downloads.") -@click.option( - "-y", - "--yes", - is_flag=True, - help="Skip the bulk-download confirmation.") -def clone(url, path, no_fetch_data, max_workers, yes): - """ - Clone a hallmark repository from a remote URL to the specified path. - - Options: - --no-fetch-data: Skip downloading remote data files after clone. - --max-workers: Number of concurrent downloads. Default is 4. - -y, --yes: Skip the bulk-download confirmation. - - Raises: - ClickException: If there are any issues with the provided arguments or - during the clone process, such as a destination already existing, - a clone error, or a download error. - """ - # use context manager to handle DownloadError and GitError exceptions - with _translate_cli_errors(DownloadError, GitError): +@click.option("--auth", help="Optional local SSH authentication profile.") +@click.option("--filter", "filters", multiple=True, + help="Relative path glob; ** matches recursively. May be repeated.") +@click.option("--fmt", help="Parameterized filename template to catalog.") +@click.option("--source-type", default="auto", show_default=True, + type=click.Choice(["auto", "git", "directory", "catalog"]), + help="Override automatic source detection.") +@click.option("--download", "fetch_data", is_flag=True, + help="Plan and request approval for downloads after catalog creation.") +@click.option("--no-fetch-data", is_flag=True, hidden=True) +@click.option("--max-workers", type=click.IntRange(min=1), default=4, + show_default=True) +@click.option("-y", "--yes", is_flag=True, hidden=True) +def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, + max_workers, yes): + """Prepare local .hm metadata; dataset files are not downloaded by default.""" + if fetch_data and no_fetch_data: + raise ClickException("--download conflicts with --no-fetch-data") + if yes: + click.echo("--yes is deprecated; downloads still require confirmation.", + err=True) + with _translate_cli_errors(DownloadError, GitError, ValueError): try: - repo = Repo.clone(url, path, fetch_data=False) - # handle CloneError exceptions and provide a user-friendly error message + repo = Repo.clone(url, path, auth=auth, filter=filters or None, fmt=fmt, + source_type=source_type, progress=True, + max_workers=max_workers) except CloneError as exc: click.echo(str(exc), err=True) raise SystemExit(1) from exc click.echo(f'Successfully cloned to "{path}"') - # if the user has not opted to skip data fetching - if not no_fetch_data: - # get the worktree path of the cloned repository - _, worktree_path = Repo.lwpaths(path) - if worktree_path is None: - click.echo( - "Bare repository clone; skipping data download.") - # bail out of the function early since there is no worktree - return - - # select the files to download from the cloned repository - selected_files = select_download_files(repo, all_files=True) - # if there are no selected files, inform the user and exit - if not selected_files: - click.echo("No remote data files are configured.") - return - - # confirm with user if the num of selected files exceeds warning threshold - click.echo("Downloading remote data files...") - _run_download( - repo, - selected_files, - worktree_path, - max_workers=max_workers, - assume_yes=yes) - - -@hallmark.command(short_help="Build a hallmark repository from a remote dataset.") + if fetch_data: + if repo.worktree is None: + raise ClickException("Use a worktree destination for --download") + _run_download(repo, repo.plan_download(), max_workers=max_workers) + + + +@hallmark.command(short_help="Deprecated: use clone for remote datasets.") @click.argument("directory") @click.argument("dataset_name") @click.option( @@ -611,11 +470,11 @@ def clone(url, path, no_fetch_data, max_workers, yes): @click.option("--dataset-url", help="Exact crawl root, independent of recorded remotes.") @click.option("--dataset-auth", help="Local SSH profile for the crawl source.") -@click.option("--index-format", type=click.Choice(["cyverse-html"])) +@click.option("--index-format", type=click.Choice(["cyverse-html"]), hidden=True) @click.option("--allow-remote-commands", is_flag=True, - help="Allow fixed read-only SSH listing commands (server python3).") + hidden=True, help="Deprecated; discovery uses SFTP.") @click.option("--remote-hash", is_flag=True, - help="Allow bounded server-side SHA-256 hashing for SSH builds.") + hidden=True, help="Unsupported; discovery does not hash dataset files.") def build(directory, dataset_name, remotes, config_file, fmts, overwrite, dataset_url, dataset_auth, index_format, allow_remote_commands, remote_hash): """ @@ -644,6 +503,7 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, a network error, Git error, or invalid dataset name. """ + click.echo("build is deprecated; use hallmark clone URL PATH.", err=True) if config_file and fmts: raise ClickException("Use only one of --config-file or --fmt, not both.") # validate the dataset name to ensure it is a valid path component diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py new file mode 100644 index 0000000..8d15584 --- /dev/null +++ b/mod/hallmark/discovery.py @@ -0,0 +1,324 @@ +"""Discover remote file metadata without transferring dataset payloads.""" + +from __future__ import annotations + +import fnmatch +import re +from collections import deque +from dataclasses import replace +from functools import lru_cache +from html.parser import HTMLParser +from pathlib import PurePosixPath +from urllib.parse import quote, unquote, urljoin, urlsplit + +import parse +from tqdm import tqdm + +from .helper_functions import CHECKSUM_ALGORITHMS_BY_STRENGTH, valid_checksum +from .transport.base import ( + CapabilityError, DownloadError, RemoteEntry, RemoteObjectMissing, + literal_path, reject_controls, +) + + +def _glob_matches(path, pattern): + """Match path segments, including zero-directory matches for ``**/``.""" + parts, patterns = path.split("/"), pattern.split("/") + + @lru_cache(maxsize=None) + def match(path_index, pattern_index): + if pattern_index == len(patterns): + return path_index == len(parts) + if patterns[pattern_index] == "**": + return (match(path_index, pattern_index + 1) + or path_index < len(parts) and match(path_index + 1, pattern_index)) + return (path_index < len(parts) + and fnmatch.fnmatchcase(parts[path_index], patterns[pattern_index]) + and match(path_index + 1, pattern_index + 1)) + + return match(0, 0) + + +@lru_cache(maxsize=128) +def _format_parser(fmt): + return parse.compile(fmt, case_sensitive=True) + + +def path_matches(path: str, filter=None, fmt: str | None = None) -> bool: + """Match root-relative paths by glob(s), exact Hallmark format, or both.""" + path = str(path) + parser = _format_parser(fmt) if fmt is not None else None + if filter is not None: + if not isinstance(filter, (str, list, tuple)): + raise ValueError( + "Path filters must be a glob string or a list of glob strings") + patterns = [filter] if isinstance(filter, str) else list(filter) + if not all(isinstance(pattern, str) for pattern in patterns): + raise ValueError("Path filters must be glob strings") + if not any(_glob_matches(path, pattern) for pattern in patterns): + return False + return parser is None or parser.parse(path) is not None + + +class _IndexParser(HTMLParser): + """Read links and recognizable index markers without depending on layout.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.links = [] + self.text = [] + self.row_kind = None + self.listing = False + self.has_table = False + self.has_form = False + self.current = None + self.in_anchor = False + self.cell_role = None + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "table": + self.has_table = True + elif tag == "form": + self.has_form = True + elif tag == "tr": + self.current = None + classes = attrs.get("class", "").split() + self.row_kind = None + if "object" in classes: + self.listing = True + self.row_kind = "directory" if "collection" in classes else "file" + elif tag == "td": + self.cell_role = attrs.get("class", "") + elif tag == "a" and "href" in attrs: + self.current = {"href": attrs["href"], "kind": self.row_kind, + "tail": "", "size": None, "mtime": None} + for field in ("size", "mtime"): + value = attrs.get(f"data-{field}") + if value is not None and value.isdigit(): + self.current[field] = int(value) + self.links.append(self.current) + self.in_anchor = True + + def handle_endtag(self, tag): + if tag == "tr": + self.row_kind = None + self.current = None + elif tag == "a": + self.in_anchor = False + elif tag == "td": + if self.current is not None: + self.current["tail"] += " " + self.cell_role = None + + def handle_data(self, data): + self.text.append(data) + if self.current is not None and not self.in_anchor: + self.current["tail"] += data + if self.cell_role in {"size", "bytes"}: + raw = data.strip().replace(",", "") + if raw.isdigit(): + self.current["size"] = int(raw) + + +def _origin(url): + return (url.scheme, url.hostname, + url.port or (443 if url.scheme == "https" else 80), + url.username, url.password) + + +def _index_link(root_url, directory, href, kind): + """Resolve a link only when it stays beneath the supplied URL root.""" + reject_controls(href, "Index href") + root = urlsplit(root_url.rstrip("/") + "/") + current = urljoin(root.geturl(), quote(directory, safe="/")) + resolved = urlsplit(urljoin(current, href)) + if _origin(resolved) != _origin(root): + return None + # Sorting/navigation query links are not dataset objects. + if resolved.query or resolved.fragment: + return None + root_path = unquote(root.path, errors="strict") + path = unquote(resolved.path, errors="strict") + reject_controls(path, "Index path") + if "\\" in path or ".." in path.split("/"): + raise DownloadError("Remote index contains an unsafe path") + if not path.startswith(root_path): + return None + relative = path[len(root_path):] + if not relative: + return None + if any(part.lower() in {".hm", ".git"} for part in relative.split("/")): + return None + is_directory = kind == "directory" or resolved.path.endswith("/") + relative = literal_path(relative.rstrip("/")).as_posix() + return relative + ("/" if is_directory else ""), is_directory + + +def _parse_index(text, root_url, directory): + parser = _IndexParser() + parser.feed(text) + links = [] + for entry in parser.links: + link = _index_link(root_url, directory, entry["href"], entry["kind"]) + if link is not None: + path, is_directory = link + size = entry["size"] + if size is None: + tail = re.search( + r"(?:\d{4}-\d\d-\d\d|\d\d-[A-Za-z]{3}-\d{4})" + r"\s+\d\d:\d\d(?::\d\d)?\s+(\d+)(?:\s|$)", entry["tail"]) + if tail: + size = int(tail.group(1)) + links.append((path, is_directory, size, entry["mtime"])) + markers = " ".join(parser.text).lower() + recognized = (parser.listing or "index of" in markers + or "directory listing" in markers or "parent directory" in markers + or any(is_dir for _, is_dir, _, _ in links)) + # CyVerse can emit a completely empty table without row-level markers. + empty_table = (parser.has_table and not parser.links + and not parser.has_form and not markers.strip()) + if not recognized and not empty_table: + raise CapabilityError( + "This URL exposes no usable directory listing. Use a browsable " + "directory URL or a published Hallmark catalog.") + return links + + +def _response_directory(context, requested): + """Use a redirected listing's canonical location when resolving child links.""" + transport = getattr(context, "transport", None) + final_url = getattr(transport, "text_urls", {}).get(requested) + if final_url is None: + return requested + root = urlsplit(context.remote.url.rstrip("/") + "/") + final = urlsplit(final_url) + if (_origin(root) == _origin(final) + and unquote(root.path).rstrip("/") == unquote(final.path).rstrip("/")): + return "" + link = _index_link(context.remote.url, "", final_url, "directory") + if link is None: + raise DownloadError("Redirected directory is outside the dataset root") + return link[0] + + +def _manifest_algorithm(path): + """Recognize conventional manifests; never guess from arbitrary substrings.""" + name = PurePosixPath(path).name.lower() + if re.fullmatch(r"checksums?(?:\.txt)?", name): + return "unknown" + for algorithm in CHECKSUM_ALGORITHMS_BY_STRENGTH: + if (re.fullmatch(rf"{algorithm}sums?(?:\.txt)?", name) + or re.fullmatch(rf".+\.{algorithm}sums?", name)): + return algorithm + return None + + +def _manifest_checksums(context, entries): + """Read published checksum metadata and attach it to discovered paths.""" + strength = {name: index for index, name in + enumerate(CHECKSUM_ALGORITHMS_BY_STRENGTH)} + for path in sorted(entries): + algorithm = _manifest_algorithm(path) + if algorithm is None: + continue + try: + text = context.read_text(path) + except RemoteObjectMissing: + continue + parent = PurePosixPath(path).parent + for line in text.splitlines(): + if line.startswith("\\"): + raise DownloadError("GNU escaped manifest filenames are unsupported") + match = re.fullmatch(r"([0-9a-fA-F]{8,}) [ *](.+)", line) + if not match: + continue + digest, filename = match.groups() + if not valid_checksum(algorithm, digest, allow_unknown_algorithm=True): + continue + while filename.startswith("./"): + filename = filename[2:] + filename = literal_path(filename).as_posix() + candidates = [(parent / filename).as_posix(), filename] + if parent.name == PurePosixPath(filename).parts[0]: + candidates.append((parent.parent / filename).as_posix()) + target = next((candidate for candidate in candidates + if candidate in entries), None) + if target is None: + continue + previous = entries[target] + if (previous.checksum_algorithm == algorithm and previous.checksum + and previous.checksum.lower() != digest.lower()): + raise DownloadError(f"Conflicting {algorithm} manifests for {target!r}") + if (previous.checksum is None + or strength.get(algorithm, 99) + < strength.get(previous.checksum_algorithm, 100)): + entries[target] = replace( + previous, checksum_algorithm=algorithm, checksum=digest.lower()) + + +def discover(context, *, filter=None, fmt=None, progress=False) -> list[RemoteEntry]: + """Recursively collect file metadata; formats and globs never authorize transfer. + + ``progress=True`` displays an indeterminate progress bar. A callable instead + receives dictionaries with ``directories``, ``files``, ``matched``, and + ``current`` keys, suitable for notebook or application rendering. + """ + # Validate selectors before contacting the source, even for an empty index. + path_matches("", filter=filter, fmt=fmt) + entries = {} + counts = {"directories": 0, "files": 0, "matched": 0, "current": ""} + bar = tqdm(total=None, unit="dir", desc="Discovering", disable=not bool(progress)) \ + if not callable(progress) else None + + def report(directory=None): + if directory is not None: + counts["directories"] += 1 + counts["current"] = directory or "/" + if bar is not None: + bar.update() + if callable(progress): + progress(dict(counts)) + elif bar is not None: + bar.set_postfix(files=counts["files"], matched=counts["matched"]) + + def add(entry): + path = literal_path(entry.path).as_posix() + if path not in entries: + entries[path] = entry + counts["files"] += 1 + counts["matched"] += int(path_matches(path, filter=filter, fmt=fmt)) + report() + + try: + if context.remote.scheme in {"ssh", "sftp"}: + for entry in context.transport.iter_entries(on_directory=report): + context.check_cancelled() + add(entry) + else: + queue, visited = deque([""]), set() + while queue: + directory = queue.popleft() + if directory in visited: + continue + visited.add(directory) + context.check_cancelled() + text = context.read_text(directory) + canonical = _response_directory(context, directory) + if canonical != directory and canonical in visited: + continue + visited.add(canonical) + links = _parse_index(text, context.remote.url, canonical) + for path, is_directory, size, mtime in links: + if is_directory: + if path not in visited: + queue.append(path) + else: + add(RemoteEntry(path=path, size=size, mtime=mtime)) + report(canonical) + _manifest_checksums(context, entries) + return [entries[path] for path in sorted(entries) + if path_matches(path, filter=filter, fmt=fmt)] + finally: + if bar is not None: + bar.close() diff --git a/mod/hallmark/download_plan.py b/mod/hallmark/download_plan.py new file mode 100644 index 0000000..f42ae60 --- /dev/null +++ b/mod/hallmark/download_plan.py @@ -0,0 +1,113 @@ +"""Immutable descriptions of dataset transfers, separate from their approval.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from math import isfinite +from pathlib import Path +from typing import Optional, Union +from urllib.parse import urlsplit, urlunsplit + + +@dataclass(frozen=True) +class DownloadItem: + """A catalog file and the metadata available when a transfer was planned.""" + + relative_path: Path + checksum: Optional[Union[str, tuple[str, str]]] = None + size_bytes: Optional[int] = None + mtime: Optional[str] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "relative_path", Path(self.relative_path)) + if isinstance(self.checksum, list): + object.__setattr__(self, "checksum", tuple(self.checksum)) + if self.checksum is not None and not ( + isinstance(self.checksum, str) + or (isinstance(self.checksum, tuple) and len(self.checksum) == 2 + and all(isinstance(part, str) for part in self.checksum)) + ): + raise ValueError("checksum must be a string or algorithm/value pair") + if self.size_bytes is not None and ( + isinstance(self.size_bytes, bool) + or not isinstance(self.size_bytes, int) + or self.size_bytes < 0 + ): + raise ValueError("size_bytes must be a nonnegative integer or None") + if self.mtime is not None and not isinstance(self.mtime, str): + raise ValueError("mtime must be a string or None") + + +@dataclass(frozen=True) +class DownloadPlan: + """The exact files, source, and destination presented for approval. + + Sizes describe catalog metadata; they are estimates until the transfer ends. + A duration is available only when every size and a supplied rate are known. + """ + + items: tuple[DownloadItem, ...] + remote_url: Optional[str] = field(repr=False) + output_path: Path + remote_auth: Optional[str] = None + remote_name: Optional[str] = None + estimated_bytes_per_second: Optional[float] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "items", tuple(self.items)) + object.__setattr__( + self, "output_path", Path(self.output_path).expanduser().absolute()) + if any(not isinstance(item, DownloadItem) for item in self.items): + raise TypeError("items must contain DownloadItem values") + if any(value is not None and not isinstance(value, str) for value in + (self.remote_url, self.remote_auth, self.remote_name)): + raise TypeError("remote fields must be strings or None") + rate = self.estimated_bytes_per_second + if rate is not None and ( + isinstance(rate, bool) or not isinstance(rate, (int, float)) + or not isfinite(rate) or rate <= 0 + ): + raise ValueError("estimated_bytes_per_second must be a positive rate") + + @property + def file_count(self) -> int: + """Number of files authorized if this plan is approved.""" + return len(self.items) + + @property + def known_bytes(self) -> int: + """Sum of recorded sizes, excluding files whose sizes are unknown.""" + return sum(item.size_bytes or 0 for item in self.items) + + @property + def unknown_size_count(self) -> int: + """Number of files without a recorded size.""" + return sum(item.size_bytes is None for item in self.items) + + @property + def total_bytes(self) -> Optional[int]: + """Estimated transfer size, or None if any file size is unknown.""" + return None if self.unknown_size_count else self.known_bytes + + @property + def estimated_seconds(self) -> Optional[float]: + """Estimated duration from a supplied rate, never a network probe.""" + if self.total_bytes is None or self.estimated_bytes_per_second is None: + return None + return self.total_bytes / self.estimated_bytes_per_second + + def summary(self) -> str: + """Describe the planned transfer without hiding unavailable estimates.""" + size = f"{self.known_bytes:,} bytes" + if self.unknown_size_count: + size += f" known; {self.unknown_size_count} file(s) with unknown size" + duration = ("unknown" if self.estimated_seconds is None + else f"about {self.estimated_seconds:.1f} seconds") + source = "" + if self.remote_url: + parsed = urlsplit(self.remote_url) + source = urlunsplit(parsed._replace( + netloc=parsed.netloc.rsplit("@", 1)[-1], query="", fragment="")) + return (f"{self.file_count} file(s); {size}; estimated duration: {duration}" + f"\nSource: {source}" + f"\nDestination: {self.output_path}") diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index e6014a3..d20646d 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -3,8 +3,9 @@ from __future__ import annotations from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from decimal import Decimal, InvalidOperation from pathlib import Path -from threading import local +from threading import Lock from typing import Mapping, Optional, Sequence, Union from urllib.parse import unquote, urlsplit, urlunsplit from tqdm import tqdm @@ -14,6 +15,7 @@ from .transport import OperationContext, RemoteSpec from .transport.base import DownloadError, literal_path +from .download_plan import DownloadItem, DownloadPlan from .helper_functions import ( CHECKSUM_ALGORITHMS_BY_STRENGTH, SUPPORTED_CHECKSUM_ALGORITHMS, @@ -32,9 +34,6 @@ BULK_DOWNLOAD_WARNING_FILE_COUNT = 100 # Maximum number of rows to read from a file at once when computing checksums. TSV_READ_CHUNK_SIZE = 10_000 -# Thread-local storage for download worker state, -# allowing each thread to maintain its own state. -_DOWNLOAD_WORKER_STATE = local() # union type for checksum specifications, string or a tuple of (algorithm, checksum). ChecksumSpec = Union[str, tuple[str, str]] # The size of chunks to read from a file when downloading or computing checksums. @@ -85,11 +84,6 @@ def _config_section_entries(config: dict, section_name: str) -> list[dict]: return [entry for entry in entries if isinstance(entry, dict)] -def _initialize_download_worker(context) -> None: - """Bind this invocation's context to each worker; no cross-operation state.""" - _DOWNLOAD_WORKER_STATE.context = context - - def _require_positive_integer(value, *, label: str) -> int: """ Used by download_remote_data and _download_file. @@ -191,7 +185,8 @@ def _row_checksum(row: Union[pd.Series, Mapping[str, object]] if legacy_sha1 is not None: return legacy_sha1 # If not found, check the "checksum" and "checksum_algorithm" columns - return _checksum_spec(row.get("checksum"), row.get("checksum_algorithm")) + return (_checksum_spec(row.get("checksum"), row.get("checksum_algorithm")) + or _entry_checksum(row)) def _entry_checksum(entry: dict) -> Optional[ChecksumSpec]: @@ -558,14 +553,9 @@ def _download_file( expected_checksum: Optional[ChecksumSpec] = None, chunk_size: int = DOWNLOAD_CHUNK_SIZE, ) -> int: - """Compatibility entry point; pool workers use their invocation's context.""" + """Standalone compatibility entry point for downloading one known URL.""" chunk_size = _require_positive_integer(chunk_size, label="chunk_size") _validate_checksum_spec(expected_checksum) - context = getattr(_DOWNLOAD_WORKER_STATE, "context", None) - if context is not None: - relative = destination.relative_to(context.output_root) - return _fetch_file( - context, relative, destination, expected_checksum, chunk_size) RemoteSpec.parse(url) parsed = urlsplit(url) base = urlunsplit(parsed._replace(path=parsed.path.rsplit("/", 1)[0] + "/")) @@ -593,12 +583,12 @@ def _download_file( raise DownloadError(f"Failed to write {destination}: {exc}") from None -def select_download_files( +def _select_download_items( repo, file_paths: Sequence[str] = (), tsv_names: Sequence[str] = (), all_files: bool = False, - ) -> list[tuple[Path, Optional[ChecksumSpec]]]: + ) -> list[DownloadItem]: """ Select files to download from a hallmark repository based on the provided file paths, TSV names, and the repository's configuration. @@ -610,8 +600,7 @@ def select_download_files( all_files: If True, include all files from the repository's configuration. Returns: - A list of tuples containing the relative path and optional checksum - specification for each selected file. + Catalog items with their relative paths, checksums, and available metadata. """ # Get the repository configuration, defaulting to an empty dictionary if not found. config = _repository_config(repo) @@ -620,10 +609,13 @@ def select_download_files( data_config = _config_section_entries(config, "data") # dictionary to store selected files with relative paths and optional checksums. selected: dict[str, tuple[Path, Optional[ChecksumSpec]]] = {} + metadata = {} + explicit_paths = {_safe_remote_path(path).as_posix() for path in file_paths} def add_file( value: Union[str, Path], - expected_checksum: Optional[ChecksumSpec] = None + expected_checksum: Optional[ChecksumSpec] = None, + row=None, ) -> None: """Add file to the selected files dictionary, ensuring it is safe and valid.""" # Resolve the input value to a safe relative path @@ -631,8 +623,22 @@ def add_file( # Merge the selected file into the dictionary of selected files, ensuring no # conflicting checksums exist. _merge_selected_file(selected, relative_path, expected_checksum) - - def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: + if row is not None: + size = _catalog_size(row.get("size_bytes")) + modified = row.get("mtime") + modified = (None if modified is None or pd.isna(modified) + or str(modified).strip() == "" else str(modified)) + previous_size, previous_modified = metadata.get( + relative_path.as_posix(), (None, None)) + if size is not None and previous_size not in (None, size): + raise DownloadError( + f"Conflicting sizes for {relative_path.as_posix()!r}") + metadata[relative_path.as_posix()] = ( + size if size is not None else previous_size, + modified if modified is not None else previous_modified) + + def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], + *, explicit_only=False) -> None: """ Add every file represented by a manifest DataFrame.""" # if the DataFrame is None or empty, return early without adding any files if frame is None or frame.empty: @@ -644,8 +650,9 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: # create a dictionary mapping column names to their corresponding values row = dict(zip(columns, values)) # add the resolved remote path and its checksum to the selected files - add_file( - _resolve_remote_path(row, fmt_entries), _row_checksum(row)) + relative_path = _safe_remote_path(_resolve_remote_path(row, fmt_entries)) + if not explicit_only or relative_path.as_posix() in explicit_paths: + add_file(relative_path, _row_checksum(row), row) # Add explicitly requested file paths to the selected files. for file_path in file_paths: @@ -653,8 +660,8 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: # Organize data configuration entries by their TSV names for easier access. entries_by_tsv: dict[str, list[dict]] = {} for entry in data_config: - # Skip entries that do not have both "fmt" and "db" keys - if not entry.get("fmt") or not entry.get("db"): + # Path catalogs need no parameterized filename format. + if not entry.get("db"): continue # download_tsv_name will normalize and validate the TSV name tsv_name = _download_tsv_name(entry["db"]) @@ -679,7 +686,11 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: requested_tsvs.append(tsv_name) seen_tsvs.add(tsv_name) - for tsv_name in requested_tsvs: + lookup_tsvs = list(requested_tsvs) + if explicit_paths: + lookup_tsvs.extend(name for name in entries_by_tsv + if name not in requested_tsvs) + for tsv_name in lookup_tsvs: fmt_entries = entries_by_tsv.get(tsv_name) # if there are no format entries for the requested TSV if fmt_entries is None: @@ -703,7 +714,8 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: keep_default_na=False, chunksize=TSV_READ_CHUNK_SIZE) for frame in frames: - add_frame(frame, fmt_entries) + add_frame(frame, fmt_entries, + explicit_only=tsv_name not in requested_tsvs) # skip empty TSV files without raising an error except pd.errors.EmptyDataError: continue @@ -711,7 +723,7 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: except (OSError, UnicodeError, pd.errors.ParserError) as exc: raise DownloadError(f"Unable to read TSV {tsv_path}: {exc}") from exc - if all_files: + if all_files or explicit_paths: # for each section ("data" and "meta") in the repository configuration for section_name in ("data", "meta"): # iterate through each entry in the section @@ -719,8 +731,9 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: # get the file path from the entry, if it exists file_path = entry.get("file") # if a file path is specified in the entry, add it to the selected files - if file_path: - add_file(file_path, _entry_checksum(entry)) + if file_path and (all_files or _safe_remote_path(file_path).as_posix() + in explicit_paths): + add_file(file_path, _entry_checksum(entry), entry) # Determine whether to use legacy data formats based on the presence of file paths, # TSV names, and all_files flag. @@ -731,8 +744,117 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: legacy_formats = [entry for entry in data_config if entry.get("fmt")] # add files from legacy formats to the selected files add_frame(repo.state.data, legacy_formats) + elif explicit_paths and not entries_by_tsv: + add_frame(repo.state.data, data_config, explicit_only=True) + + return [DownloadItem(path, checksum, *metadata.get(path.as_posix(), (None, None))) + for path, checksum in selected.values()] + + +def _catalog_size(value) -> Optional[int]: + """Read a recorded size without treating missing or malformed sizes as zero.""" + if value is None or isinstance(value, bool) or pd.isna(value): + return None + try: + size = Decimal(str(value).strip()) + except InvalidOperation: + return None + if not size.is_finite() or size < 0 or size != size.to_integral_value(): + return None + return int(size) + + +def select_download_files( + repo, + file_paths: Sequence[str] = (), + tsv_names: Sequence[str] = (), + all_files: bool = False, +) -> list[tuple[Path, Optional[ChecksumSpec]]]: + """Select paths and checksums, including catalog checksums for explicit paths.""" + return [(item.relative_path, item.checksum) for item in _select_download_items( + repo, file_paths=file_paths, tsv_names=tsv_names, all_files=all_files)] - return list(selected.values()) + +def plan_download( + repo, + output_path: Optional[Union[Path, str]] = None, + *, + file_paths: Optional[Sequence[str]] = None, + tsv_names: Optional[Sequence[str]] = None, + all_files: bool = False, + filter: Optional[Union[str, Sequence[str]]] = None, + fmt: Optional[str] = None, + remote_name: Optional[str] = None, + estimated_bytes_per_second: Optional[float] = None, +) -> DownloadPlan: + """Plan a transfer using local catalog metadata, without contacting a server. + + With no explicit path or TSV selection, include the complete catalog before + applying the optional path filter and format. Unknown sizes stay unknown. + """ + if output_path is None: + output_path = getattr(repo, "worktree", None) + if output_path is None: + raise DownloadError("output_path is required for a bare repository") + output_root = Path(output_path).expanduser().resolve() + if output_root.exists() and not output_root.is_dir(): + raise DownloadError(f"Download output is not a directory: {output_root}") + if isinstance(file_paths, (str, Path)): + file_paths = (str(file_paths),) + if isinstance(tsv_names, str): + tsv_names = (tsv_names,) + file_paths, tsv_names = tuple(file_paths or ()), tuple(tsv_names or ()) + if all_files and (file_paths or tsv_names): + raise DownloadError("all_files cannot be combined with paths or TSVs") + items = _select_download_items( + repo, file_paths=file_paths, tsv_names=tsv_names, + all_files=all_files or (not file_paths and not tsv_names)) + if filter is not None or fmt is not None: + from .discovery import path_matches + items = [item for item in items if path_matches( + item.relative_path.as_posix(), filter=filter, fmt=fmt)] + remote = _select_remote_config(repo, remote_name) + if items and (remote is None or not remote.get("url")): + raise DownloadError("No remote URL is configured in config.yml") + remote = remote or {} + if remote.get("url"): + RemoteSpec.parse(remote["url"], remote.get("auth")) + for item in items: + try: + resolve_contained_path(output_root, item.relative_path, + label="download destination") + except ValueError as exc: + raise DownloadError(str(exc)) from exc + return DownloadPlan( + tuple(items), remote.get("url"), output_root, + remote_auth=remote.get("auth"), remote_name=remote.get("name"), + estimated_bytes_per_second=estimated_bytes_per_second) + + +def execute_download_plan( + repo, + plan: DownloadPlan, + *, + approved: bool = False, + max_workers: int = 4, + show_progress: bool = False, +) -> dict: + """Execute the approved plan's source and selection, ignoring later config edits.""" + if not isinstance(plan, DownloadPlan): + raise TypeError("plan must be a DownloadPlan") + if plan.items and approved is not True: + raise DownloadError("Dataset downloads require explicit approval") + max_workers = _require_positive_integer(max_workers, label="max_workers") + if not plan.items: + return {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + if plan.output_path.resolve() != plan.output_path: + raise DownloadError("Download destination changed since planning") + remote = RemoteSpec.parse(plan.remote_url, plan.remote_auth) + return _download_selected( + remote, plan.output_path, + [(item.relative_path, item.checksum) for item in plan.items], + max_workers=max_workers, show_progress=show_progress, + byte_progress=True, total_bytes=plan.total_bytes) def download_remote_data( @@ -742,6 +864,8 @@ def download_remote_data( show_progress: bool = False, selected_files: Optional[Sequence[tuple[Path, Optional[ChecksumSpec]]]] = None, remote_name: Optional[str] = None, + *, + approved: bool = False, ) -> dict: """ Download remote data files for a hallmark repository. @@ -754,6 +878,7 @@ def download_remote_data( selected_files: A sequence of tuples containing the relative path and optional checksum of files to download. remote_name: The name of the remote configuration to use. + approved: Explicit approval to transfer the selected dataset files. Returns: A dict with the download results (succeeded, failed, total_bytes, and errors) @@ -798,6 +923,26 @@ def download_remote_data( # return the results without attempting any downloads return results + if approved is not True: + raise DownloadError("Dataset downloads require explicit approval") + return _download_selected( + remote_spec, worktree_path, selected_files, + max_workers=max_workers, show_progress=show_progress) + + +def _download_selected( + remote_spec, + worktree_path, + selected_files, + *, + max_workers, + show_progress, + byte_progress=False, + total_bytes=None, +): + """Run one approved transfer with operation-owned resources and atomic output.""" + results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + # output_root is the resolved absolute path where files will be downloaded output_root = Path(worktree_path).expanduser().resolve() # if the output root exists and is not a directory, raise a DownloadError @@ -833,26 +978,30 @@ def download_remote_data( except ValueError as exc: raise DownloadError(str(exc)) from exc - # Append the download info (URL, destination, expected checksum) to the list - files_to_download.append(( - _remote_file_url(remote_url, relative_path), - destination, - expected_checksum)) + files_to_download.append((relative_path, destination, expected_checksum)) # track the download progress using tqdm, with the number of files to download progress = tqdm( - total=len(files_to_download), - unit="file", + total=total_bytes if byte_progress else len(files_to_download), + unit="B" if byte_progress else "file", + unit_scale=byte_progress, disable=not show_progress,) + progress_lock = Lock() + + def advance_bytes(count): + # Transport callbacks run concurrently, one for each active worker. + with progress_lock: + progress.update(count) # try to download the files using a thread pool executor for concurrent downloads try: with OperationContext(remote_spec, output_root) as context: + if byte_progress: + context.on_bytes = advance_bytes context.transport.prepare() executor = ThreadPoolExecutor( max_workers=min(max_workers, context.settings.max_sessions) - if remote_spec.scheme in {"ssh", "sftp"} else max_workers, - initializer=_initialize_download_worker, initargs=(context,)) + if remote_spec.scheme in {"ssh", "sftp"} else max_workers) try: # create an iterator over the files to download file_iterator = iter(files_to_download) @@ -870,13 +1019,14 @@ def submit_next() -> bool: """ # try to get the next file from the iterator try: - url, destination, checksum = next(file_iterator) + relative_path, destination, checksum = next(file_iterator) # Handle the exception when there are no more files to download except StopIteration: return False # add the download task to the pending set using the executor pending.add(executor.submit( - _download_file, url, destination, checksum)) + _fetch_file, context, relative_path, destination, checksum, + DOWNLOAD_CHUNK_SIZE)) # if the download was successfully submitted, return True return True @@ -899,7 +1049,14 @@ def submit_next() -> bool: results["errors"].append(str(exc)) # Every completed transfer advances the progress bar. finally: - progress.update(1) + if byte_progress: + with progress_lock: + finished = results["succeeded"] + results["failed"] + progress.set_postfix( + files=f"{finished}/{len(files_to_download)}", + failed=results["failed"]) + else: + progress.update(1) # submit the next file for download if available submit_next() except BaseException: @@ -913,4 +1070,4 @@ def submit_next() -> bool: finally: progress.close() - return results \ No newline at end of file + return results diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 74a99c7..ce28422 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -203,72 +203,75 @@ def clone( url: str, path: Union[Path, str], *, - fetch_data: bool = True, + auth: Optional[str] = None, + filter=None, + fmt: Optional[str] = None, + source_type: str = "auto", + progress: bool = False, + download: bool = False, + approve=None, max_workers: int = 4, - show_progress: bool = False, + fetch_data: Optional[bool] = None, + show_progress: Optional[bool] = None, ) -> "Repo": - ''' - Clone a remote hallmark repository. Raises DestinationExistsError - if the destination path already exists. Raises DownloadError if - data download is enabled and files fail to download. + """Clone catalog metadata, discovering ordinary remote directories as needed. - Args: - url(string): remote repository URL. - path(path|string): destination path for clone. - fetch_data (boolean): if true, downloads associated data files. - max_workers (integer): Number of parallel workers for downloading data. - show_progress (boolean): wether to display download progress. - Returns: - Repo: Cloned Repository instance - ''' - clone_path = Path(path) - if clone_path.exists(): - raise DestinationExistsError( - f"fatal: destination path '{clone_path}' already exists " - "and is not an empty directory." - ) + ``filter`` is a relative path glob; ``fmt`` is a filename template. + Dataset files are transferred only with ``download=True`` and an + ``approve(plan)`` callback returning True after inspecting the plan. + ``source_type`` can explicitly select git, directory, or catalog sources. + The legacy ``fetch_data`` and ``show_progress`` names remain aliases. + """ + from .catalog import clone_catalog + from .downloader import DownloadError, _require_positive_integer + + _require_positive_integer(max_workers, label="max_workers") + if fetch_data is not None: + if download and not fetch_data: + raise ValueError("download and fetch_data conflict") + download = fetch_data + if show_progress is not None: + progress = show_progress + if download and not callable(approve): + raise DownloadError( + "Downloading during clone requires an approve(plan) callback") + if download and cls.lwpaths(path)[1] is None: + raise DownloadError( + "Clone a worktree to download; bare catalogs need an output path") + repo = clone_catalog(cls, url, path, auth=auth, filter=filter, fmt=fmt, + source_type=source_type, progress=progress) + if download: + plan = repo.plan_download() + if plan.file_count and approve(plan) is True: + repo.download_result = repo.download( + plan, approved=True, max_workers=max_workers, progress=progress) + if repo.download_result["failed"]: + details = "\n".join(repo.download_result["errors"][:5]) + raise DownloadError( + f"Failed to download {repo.download_result['failed']} " + f"file(s):\n{details}") + return repo - dothm_path, worktree_path = cls.lwpaths(path) - # try to clone the repository, and if it fails, clean up the destination path - try: - Dothm.clone(url, dothm_path, display_path=path) - except Exception: - # remove the partially created directory to avoid leaving a broken state - rmtree(clone_path, ignore_errors=True) - # re-raise the exception to propagate the error to the caller - raise - - # Initialize worktree if non-bare - if worktree_path: - Worktree.init(worktree_path) + def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, + all_files=False, filter=None, fmt=None, remote_name=None, + estimated_bytes_per_second=None): + """Inspect a transfer using local catalog metadata without network access.""" + from .downloader import plan_download - repo = cls(path) - # If fetch_data is True and a worktree exists, download remote data files - if fetch_data and worktree_path: - from .downloader import (DownloadError, download_remote_data, - select_download_files) - # Select files to download from the remote repository - selected_files = select_download_files(repo, all_files=True) - result = download_remote_data( - repo, - worktree_path, - max_workers=max_workers, - show_progress=show_progress, - selected_files=selected_files,) - - repo.download_result = result - if result["failed"]: - errors = result.get("errors", []) - details = "\n".join(f" - {error}" for error in errors[:5]) - remaining = result["failed"] - len(errors[:5]) - if remaining > 0: - details += f"\n - ... {remaining} more error(s)" - raise DownloadError( - f"Failed to download {result['failed']} file(s):\n" - f"{details}" - ) + return plan_download( + self, output_path, file_paths=file_paths, tsv_names=tsv_names, + all_files=all_files, filter=filter, fmt=fmt, remote_name=remote_name, + estimated_bytes_per_second=estimated_bytes_per_second) - return repo + def download(self, plan, *, approved=False, max_workers=4, progress=False): + """Execute an inspected DownloadPlan only with explicit approval.""" + from .downloader import execute_download_plan + + result = execute_download_plan( + self, plan, approved=approved, max_workers=max_workers, + show_progress=progress) + self.download_result = result + return result @staticmethod def checksum(path: Path, chunk_size: int = FILE_IO_CHUNK_SIZE) -> str: @@ -947,4 +950,4 @@ def add_worktree(self, target_branch: str) -> bool: f'failed to populate worktree for branch ' f'"{target_branch}": {exc}') from exc - return True \ No newline at end of file + return True diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index 1fc9e12..78be871 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -4,6 +4,7 @@ import shutil import string import time +import warnings from functools import lru_cache from pathlib import Path from string import Formatter @@ -15,9 +16,8 @@ from .repo import Repo from .transport import OperationContext, RemoteSpec from .transport.base import ( - CapabilityError, DownloadError, RemoteObjectMissing, reject_controls) + CapabilityError, DownloadError, reject_controls) from .fmt_detection import ( - detect_fmt, KNOWN_PROCESSING_STAGES, KNOWN_STATIC_FILE_STEMS) from .dothm import dump_yaml @@ -25,13 +25,11 @@ from .helper_functions import ( CHECKSUM_ALGORITHMS, CHECKSUM_ALGORITHMS_BY_STRENGTH, - CHECKSUM_ALGORITHM_PATTERN, SUPPORTED_CHECKSUM_ALGORITHMS, valid_checksum, atomic_output_path, load_yaml_file, normalize_nonempty_string, - prompt_choice, validate_path_component, validate_relative_path) from .repo_config import ( @@ -43,29 +41,10 @@ # base URL for the CyVerse curated data repository _CYVERSE_CURATED_BASE = \ "https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/" -# regular expression to match rows in an HTML index page for a directory listing -_INDEX_ROW_RE = re.compile( - r'' - r'.+)\."rf"(?P{CHECKSUM_ALGORITHM_PATTERN})sums$") -# keywords to identify checksum files based on their names -_CHECKSUM_NAME_KEYWORDS = ("sum", "checksum", "hash", "manifest", *CHECKSUM_ALGORITHMS) -# maximum number of worker threads for static checksum computation -_STATIC_CHECKSUM_MAX_WORKERS = 8 -# maximum number of worker threads for remote crawling and checksum computation -_REMOTE_CRAWL_MAX_WORKERS = 8 -# minimum ratio of matching lines in a checksum file to consider it valid -_MANIFEST_LINE_MATCH_RATIO = 0.5 -# regular expression to match checksum algorithms in file names, case-insensitive -_ALGORITHM_IN_NAME_RE = re.compile(rf"({CHECKSUM_ALGORITHM_PATTERN})", re.IGNORECASE) # default checksum algorithm to use when the algorithm cannot be determined _UNKNOWN_CHECKSUM_ALGORITHM = "unknown" -# maximum file size (in bytes) for which we will compute a checksum -_MAX_DOWNLOAD_SIZE_FOR_CHECKSUM = 10 * 1024 * 1024 # 10 MB # maximum length of a literal segment in a format string that can be cleared _MAX_CLEARABLE_LITERAL_LENGTH = 2 # known field values for specific format fields, used for parsing and matching @@ -702,214 +681,17 @@ def _normalize_fmt_entries(fmt_entries: list[dict]) -> list[dict]: def list_remote_files(base_url: str, *, _context=None): - """List a CyVerse HTML index (legacy default), or an authorized SSH source.""" - if _context is None: - with OperationContext(RemoteSpec.parse(base_url)) as context: - return _list_source(context) - return _list_source(_context) - - -def _list_source(context): - if context.remote.scheme in {"ssh", "sftp"}: - return _list_ssh_files(context) - return CyverseHtmlIndex(context).list_files() - - -class CyverseHtmlIndex: - """Adapter for CyVerse's HTML directory-index dialect, not WebDAV PROPFIND.""" + """Return a checksum inventory from an automatically detected remote listing.""" + from .discovery import discover - def __init__(self, context): - self.context = context + def inventory(context): + return {entry.path: (entry.checksum_algorithm, entry.checksum) + for entry in discover(context)} - def list_files(self): - return _list_cyverse_files(self.context) - - -def _list_cyverse_files(context): - def fetch_text(path): - return context.read_text(path) - - def optional_text(path): - try: - return fetch_text(path) - except RemoteObjectMissing: - return None - - # stores all files with their checksums, keyed by relative path, for all algorithms - file_checksums: dict[str, tuple[str | None, str | None]] = {} - - directories_to_open = [""] - visited_directories: set[str] = set() - # raw index text for directories that have been fetched but not yet processed - prefetched_indexes: dict[str, str] = {} - # Use a requests session and ThreadPoolExecutor to fetch remote indexes concurrently - with context.executor(_REMOTE_CRAWL_MAX_WORKERS) as crawl_executor: - # while there are directories to open or prefetched indexes to process - while directories_to_open or prefetched_indexes: - # if there are no prefetched indexes, fetch a batch of directories to open - if not prefetched_indexes: - batch = [] - # while there are directories to open and we haven't reached max workers - while (directories_to_open and len(batch) < _REMOTE_CRAWL_MAX_WORKERS): - # pop a directory from the stack to process - candidate = directories_to_open.pop() - # skip if this directory has already been visited to avoid cycles - if candidate in visited_directories: - continue - # mark directory as visited and add it to the batch for fetching - visited_directories.add(candidate) - if len(visited_directories) > context.listing_limit: - raise DownloadError("Remote listing exceeds its entry limit") - batch.append(candidate) - # if the batch is empty, continue to the next iteration of the loop - if not batch: - continue - - # construct the full URLs for the batch of directories to fetch - directory_urls = batch - # fetch the index texts for the batch of directories concurrently - index_texts = crawl_executor.map(fetch_text, directory_urls) - # update the prefetched indexes with the fetched index texts - prefetched_indexes.update(zip(batch, index_texts)) - - # get the next directory to process from the prefetched indexes - rel_dir = next(iter(prefetched_indexes)) - # pop the index text for this directory from the prefetched indexes - index_text = prefetched_indexes.pop(rel_dir) - # parse the index text to extract entries using regex - raw_entries = _INDEX_ROW_RE.findall(index_text) - if (len(file_checksums) + len(raw_entries) > context.listing_limit): - raise DownloadError("Remote listing exceeds its entry limit") - entries = [] - for entry_type, href in raw_entries: - # try to normalize the href to ensure it is a valid relative path - try: - normalized_href = _normalize_index_href( - href, - is_directory=(entry_type == "collection")) - # if the href is invalid, skip this entry and continue to the next one - except ValueError: - # Directory indexes often include a parent navigation link. - if href in {"../", "./", "/"}: - continue - raise - # add the normalized entry to the list of entries for this directory - entries.append((entry_type, normalized_href)) - - # find manifest files that are not directories and match expected patterns - manifest_hrefs = [href for entry_type, href in entries - if (entry_type != "collection" - and (_SUMS_FILENAME_RE.match(href) - or any(keyword in href.lower() - for keyword in _CHECKSUM_NAME_KEYWORDS)))] - # call _remote_url to construct the full URLs for the manifest files - manifest_urls = [ - rel_dir + href for href in manifest_hrefs] - # crawl the manifest URLs concurrently to fetch their text contents - manifest_texts = crawl_executor.map(optional_text, - manifest_urls) - # create a dictionary mapping manifest hrefs to their fetched text contents - manifest_text_by_href = { - href: text for href, text in zip(manifest_hrefs, manifest_texts) - if text is not None} - # track only files explicitly covered by parsed manifest lines - files_covered_by_manifest: set[str] = set() - - def record_manifest_matches( - matches: list[tuple[str, str]], algorithm: str) -> None: - """ - Record matches from a manifest file into the file_checksums dictionary. - Args: - matches: List of (checksum, filename) extracted from the manifest. - algorithm: The checksum algorithm used in the manifest. - """ - for checksum, filename in matches: - try: - full_path = _resolve_manifest_path(filename, rel_dir) - except ValueError: - raise - - _record_checksum(file_checksums, full_path, algorithm, checksum) - files_covered_by_manifest.add(full_path) - - # Iterate over each entry in the current directory - for entry_type, href in entries: - # if the entry is a collection (directory), skip it for now - if entry_type == "collection": - continue - # get the manifest text for this href from the prefetched manifest texts - text = manifest_text_by_href.get(href) - # skip if manifest text is None, meaning the file could not be fetched - if text is None: - continue - # Match against expected pattern to extract covered directory and algo - sibling_match = _SUMS_FILENAME_RE.match(href) - if sibling_match: - algorithm = sibling_match.group("algorithm") - # match if the checksum is valid for the given algorithm - matches = _manifest_matches(text, algorithm) - # add each path and its checksum to the file_checksums dictionary - record_manifest_matches(matches, algorithm) - # go to the next entry since this one has been processed - continue - - # lines in manifest text that are not empty after stripping whitespace - lines = [line for line in text.splitlines() if line.strip()] - - # search for the checksum algorithm in the filename using regex - name_match = _ALGORITHM_IN_NAME_RE.search(href) - # if a match is found, use the matched algorithm; otherwise, use unknown - algorithm = (name_match.group(1).lower() if name_match - else _UNKNOWN_CHECKSUM_ALGORITHM) - # only matches that are valid for the given algorithm are considered - matches = _manifest_matches(text, algorithm) - # if the file is empty or the number of valid matches is too low, skip - if (not lines or len(matches) < _MANIFEST_LINE_MATCH_RATIO * len(lines) - ): - continue - # record the matches from the manifest into file_checksums dictionary - record_manifest_matches(matches, algorithm) - - # After processing all entries, add any uncovered directories to the stack - for entry_type, href in entries: - rel_path = rel_dir + href - if entry_type == "collection": - directories_to_open.append(rel_path) - else: - # add files not explicitly covered by any manifest line with None - if rel_path not in files_covered_by_manifest: - file_checksums.setdefault(rel_path, (None, None)) - # return the complete mapping of relative paths to their checksums organized by algo - return file_checksums - - -def _list_ssh_files(context): - paths = context.transport.list_entries() - result = {path: (None, None) for path in paths} - for path in paths: - name = Path(path).name - sibling = _SUMS_FILENAME_RE.match(name) - if not sibling and not any(k in name.lower() for k in _CHECKSUM_NAME_KEYWORDS): - continue - text = context.read_text(path) - name_match = _ALGORITHM_IN_NAME_RE.search(name) - algorithm = (sibling.group("algorithm") if sibling else - name_match.group(1).lower() if name_match else "unknown") - matches = _manifest_matches(text, algorithm) - lines = [line for line in text.splitlines() if line.strip()] - if not sibling and (not lines or len(matches) < len(lines) * 0.5): - continue - directory = str(Path(path).parent) - directory = "" if directory == "." else directory + "/" - for checksum, filename in matches: - target = _resolve_manifest_path(filename, directory) - if target in result: - _record_checksum(result, target, algorithm, checksum) - if context.remote_hash: - for path, (_, digest) in result.items(): - if digest is None: - result[path] = context.transport.checksum_small(path) - return result + if _context is None: + with OperationContext(RemoteSpec.parse(base_url)) as context: + return inventory(context) + return inventory(_context) def build_repo( @@ -926,34 +708,30 @@ def build_repo( allow_remote_commands: bool = False, remote_hash: bool = False, ) -> "Repo": - """Build catalogs from an explicit dataset root or the legacy CyVerse root. + """Compatibility wrapper for legacy catalog construction. - ``dataset_auth`` selects a host-bound local SSH profile independently of - output ``remotes``. Non-default HTTP roots require ``index_format`` set to - ``cyverse-html``. SSH crawling requires ``allow_remote_commands=True``, a - POSIX shell and server Python 3. Server SHA-256 hashing is opt-in through - ``remote_hash`` and bounded to 10 MiB/file, 100 MiB and 60 seconds in total. + Prefer ``Repo.clone(url, path)`` for remote catalogs and ``Repo.init(path)`` + for local repositories. Discovery reads listings and published manifests only. + Missing formats default to a generic path catalog without prompting. """ + warnings.warn( + "build_repo is deprecated; use Repo.clone(url, path) or Repo.init(path)", + DeprecationWarning, stacklevel=2) + if remote_hash: + raise CapabilityError( + "remote_hash is no longer supported: catalog discovery does not read " + "dataset payloads to compute checksums") + if index_format not in {None, "auto", "cyverse-html"}: + raise CapabilityError("Directory indexes are detected automatically") + if index_format is not None or allow_remote_commands: + warnings.warn( + "index_format and allow_remote_commands are obsolete and ignored", + DeprecationWarning, stacklevel=2) dataset_name = validate_path_component(dataset_name, label="dataset name") base_url = (dataset_url if dataset_url is not None else _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/")) source = RemoteSpec.parse(base_url, dataset_auth) - if source.scheme in {"http", "https"}: - if index_format not in {None, "cyverse-html"}: - raise CapabilityError("Supported HTTP index format: cyverse-html") - if dataset_url is not None and index_format is None: - raise CapabilityError( - "Explicit HTTP roots require --index-format cyverse-html") - if remote_hash: - raise CapabilityError("--remote-hash is supported only for SSH sources") - elif not allow_remote_commands: - raise CapabilityError( - "SSH builds require --allow-remote-commands; " - "SFTP-only accounts can download") - elif index_format is not None: - raise CapabilityError("--index-format applies only to HTTP sources") - with OperationContext(source, allow_remote_commands=allow_remote_commands, - remote_hash=remote_hash) as context: + with OperationContext(source) as context: return _build_repo(repo_path, dataset_name, fmt_entries, config_file, remotes, overwrite, context) @@ -974,7 +752,7 @@ def _build_repo( Args: repo_path: Path where the hallmark repo will be created. fmt_entries: dict entries with "fmt", "db", and optional "name" keys. - If None, the function will attempt to detect the format entries automatically. + If None, preserve existing formats or create a generic path catalog. config_file: Path to an existing config.yml file to load fmt entries from. remotes: Optional list of remote repos to add to the repo. Each remote can be a dict with "name" and "url" keys, or a str representing the name of the remote. @@ -984,8 +762,7 @@ def _build_repo( """ # validate the dataset name to ensure it is a valid path component dataset_name = validate_path_component(dataset_name, label="dataset name") - # CyVerse's curated Data Commons datasets live at a predictable URL, - # built directly from dataset_name. + # The wrapper has already resolved the exact URL, including the legacy default. base_url = source.remote.url # Determine if remotes were provided by the user @@ -1041,8 +818,7 @@ def _ensure_remote_files_listed(): """ Used by build_repo. Ensure that the remote files have been listed and checksums collected. - This function is called lazily to avoid unnecessary network requests if the - user provides their own fmt entries or chooses to reuse an existing config. + Defer network requests until destination and configuration validation pass. """ # create nonlocal references to the outer variables so they can be modified nonlocal file_checksums, remote_files @@ -1059,101 +835,16 @@ def _ensure_remote_files_listed(): # sort the remote files by their relative paths for consistent ordering remote_files = sorted(file_checksums) - # if the repo path already exists and has a config.yml, we will reuse it - reused_from_existing_repo = False - # records the interactive include_drives choice when fmts are auto-detected, so it - # can be persisted to meta.yml and isn't lost once the build finishes - detect_include_drives: bool | None = None - - if fmt_entries is None and existing_fmt_entries: - print(f"Found existing config.yml with {len(existing_fmt_entries)} fmt(s):") - for entry in existing_fmt_entries: - print(f" {entry['fmt']!r} -> {entry['db']}") - # prompt the user to either use the existing fmt entries or update them - keep_choice = prompt_choice("Use these fmts as-is, or update the list? " - "[use/update]: ", {"use", "update"}) - # if user chooses to use existing fmts, set them as the current fmt_entries - if keep_choice == "use": - fmt_entries = existing_fmt_entries - reused_from_existing_repo = True - - # if repo path exists and we are not reusing it, check if overwrite is allowed - if repo_path.exists() and not reused_from_existing_repo and not overwrite: - # raise an error to prevent accidental overwriting of an existing repo + # Keep existing catalog formats when refreshing through the legacy wrapper. + reused_from_existing_repo = fmt_entries is None and bool(existing_fmt_entries) + if reused_from_existing_repo: + fmt_entries = existing_fmt_entries + elif repo_path.exists() and not overwrite: raise FileExistsError( f'Destination "{repo_path}" already exists. ' "Use overwrite=True to replace it.") - if fmt_entries is None: - # prompt the user to choose how to supply fmt entries - choice = prompt_choice( - "Load fmts from an existing config file, detect them " - "automatically, or input them yourself? " - "[config/detect/input]: ", {"config", "detect", "input"}) - if choice == "config": - # prompt the user for the path to an existing config.yml or repo directory - config_path = Path( - input( - "Path to the existing config.yml or repository " - "directory to load fmts from: ").strip()).expanduser() - # if the provided path is a directory, assume it contains a config.yml - if config_path.is_dir(): - config_path = config_path / "config.yml" - # if the config file does not exist, raise an error - if not config_path.is_file(): - raise FileNotFoundError(f"Config file does not exist: {config_path}") - - # load the config.yml file and extract the fmt entries - loaded_config = load_yaml_file(config_path) - fmt_entries = fmt_entries_from_config(loaded_config) - if not fmt_entries: - raise ValueError( - f"No fmt entries found in config file {config_path}.") - # if remotes were not provided and the loaded config has a "remote" key - if not remotes_provided and loaded_config.get("remote"): - # normalize the remotes from the loaded config and use them - remotes = normalize_remotes(loaded_config["remote"]) - - elif choice == "detect": - # ask the user if they want to include drive files during fmt detection - include_drives = prompt_choice("Include drive/archive files during fmt " \ - "detection? [yes/no]: ", {"yes", "no"}, ) == "yes" - # remember the choice so it can be recorded in meta.yml below - detect_include_drives = include_drives - - # check if the remote files have already been listed; if not, list them now - _ensure_remote_files_listed() - detected_fmts = detect_fmt(remote_files, include_drives=include_drives) - if not detected_fmts: - raise ValueError( - "No fmts could be automatically detected from this dataset.") - fmt_entries = [] - # create a fmt entry for each detected format - if len(detected_fmts) == 1: - # if there is only one detected format, assume it is "data.tsv" - print(f"Detected fmt: {detected_fmts[0]!r}\n" - f" Only one fmt detected; using db name 'data.tsv'.") - fmt_entries.append({"fmt": detected_fmts[0], "db": "data.tsv"}) - else: - for fmt in detected_fmts: - # prompt the user to enter a database name for each detected format - db_name = normalize_tsv_name(input( - f"Detected fmt: {fmt!r}\n" - f" Enter a db name for this fmt (e.g. 'data.tsv'): ")) - fmt_entries.append({"fmt": fmt, "db": db_name}) - elif choice == "input": - fmt_entries = [] - print("Enter fmt entries one at a time. Leave the fmt blank to finish.") - # allow the user to input multiple fmt entries manually - while True: - fmt = input("fmt (blank to finish): ").strip() - # if the user inputs a blank fmt, exit the loop - if not fmt: - break - # prompt the user to enter a database name for the provided format - db_name = normalize_tsv_name(input(" db name for this fmt: ")) - # append the user-provided fmt and db name to the fmt_entries list - fmt_entries.append({"fmt": fmt, "db": db_name}) + fmt_entries = [{"fmt": "{path}", "db": "data.tsv"}] # normalize the fmt entries to ensure they are valid and consistent fmt_entries = _normalize_fmt_entries(fmt_entries) @@ -1243,20 +934,6 @@ def _ensure_remote_files_listed(): static_file_entries = [] meta_file_entries: list[dict] = [] - # identify which unmatched paths need checksums computed (those without a checksum) - paths_needing_checksums = [ - rel_path - for rel_path in unmatched_paths - if file_checksums.get(rel_path, (None, None))[1] is None] - computed_checksums = {} - if paths_needing_checksums: - paths = paths_needing_checksums - with source.executor(min(_STATIC_CHECKSUM_MAX_WORKERS, len(paths))) as executor: - # The source owns all static checksum I/O, including SSH capability checks. - results = executor.map(source.transport.checksum_small, paths) - # zip the paths needing checksums with their computed results into a dict - computed_checksums = dict(zip(paths_needing_checksums, results)) - for rel_path in unmatched_paths: path = Path(rel_path) stem_name = path.stem.split(".")[-1] @@ -1265,9 +942,6 @@ def _ensure_remote_files_listed(): is_meta_file = stem_name.lower() == "meta" # for each unmatched path, look up its checksum algo, checksum = file_checksums.get(rel_path, (None, None)) - # if not available, use the computed checksum from the parallel computation - if checksum is None: - algo, checksum = computed_checksums[rel_path] # create an entry for the static file with its relative path entry = {"file": rel_path} @@ -1292,9 +966,6 @@ def _ensure_remote_files_listed(): # hallmark's bookkeeping: create and commit the meta.yml file meta_dict: dict = {"dataset": dataset_name} - # record the include_drives choice so a detect-built repo's fmts stay consistent - if detect_include_drives is not None: - meta_dict["detect_include_drives"] = detect_include_drives repo.dothm.dump_yml(meta_dict, "meta") repo.dothm.index.add(["meta.yml"]) repo.dothm.index.commit(f"Initialize dataset: {dataset_name}") @@ -1377,4 +1048,4 @@ def _ensure_remote_files_listed(): # reload the repo state from the .dothm directory to reflect the latest changes repo.state = repo.dothm.load() - return repo \ No newline at end of file + return repo diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py index fa25a2b..9799bcf 100644 --- a/mod/hallmark/transport/__init__.py +++ b/mod/hallmark/transport/__init__.py @@ -2,7 +2,6 @@ from __future__ import annotations -import time from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from pathlib import Path @@ -11,7 +10,7 @@ import requests from .auth import resolve_settings -from .base import DownloadError, RemoteSpec, TransferCancelled +from .base import RemoteEntry, RemoteSpec, TransferCancelled class OperationContext: @@ -21,27 +20,17 @@ def __init__( self, remote, output_root=None, - *, - allow_remote_commands=False, - remote_hash=False, ): self.remote = remote self.settings = resolve_settings(remote) self.output_root = Path(output_root) if output_root is not None else None self.cancelled = Event() - self.allow_remote_commands = allow_remote_commands - self.remote_hash = remote_hash + self.on_bytes = None self.text_limit = 16 * 1024 * 1024 - self.listing_limit = 100_000 self.listing_timeout = 60 - self.hash_file_limit = 10 * 1024 * 1024 - self.hash_total_limit = 100 * 1024 * 1024 - self.hash_timeout = 60 self._local = local() self._lock = Lock() self._sessions = [] - self._text_bytes = 0 - self._listing_started = None if remote.scheme in {"http", "https"}: from .http import HttpTransport @@ -72,17 +61,7 @@ def session(self): return self._local.session def read_text(self, path): - with self._lock: - if self._listing_started is None: - self._listing_started = time.monotonic() - if time.monotonic() - self._listing_started > 300: - raise DownloadError("Remote crawl exceeded its five minute budget") - text = self.transport.read_text(path, self.text_limit) - with self._lock: - self._text_bytes += len(text.encode("utf-8")) - if self._text_bytes > 64 * 1024 * 1024: - raise DownloadError("Remote crawl exceeded its 64 MiB text budget") - return text + return self.transport.read_text(path, self.text_limit) def check_cancelled(self): if self.cancelled.is_set(): @@ -107,4 +86,4 @@ def __exit__(self, exc_type, exc, tb): session.close() -__all__ = ["OperationContext", "RemoteSpec"] +__all__ = ["OperationContext", "RemoteEntry", "RemoteSpec"] diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py index 71bbbe4..4a36c6c 100644 --- a/mod/hallmark/transport/base.py +++ b/mod/hallmark/transport/base.py @@ -82,6 +82,17 @@ def ssh_user(value: str) -> str: return value +@dataclass(frozen=True) +class RemoteEntry: + """A discovered file; absent attributes remain unknown without reading it.""" + + path: str + size: int | None = None + mtime: float | None = None + checksum_algorithm: str | None = None + checksum: str | None = None + + @dataclass(frozen=True) class RemoteSpec: """A parsed URL root; catalog paths appended to it are always literal.""" @@ -187,10 +198,13 @@ def fetch(self, relative_path, destination, *, chunk_size=8192): raise NotImplementedError def list_entries(self): + return [entry.path for entry in self.iter_entries()] + + def iter_entries(self, on_directory=None): raise CapabilityError("Recursive listing is unsupported by this transport") - def checksum_small(self, relative_path): - return ("unknown", "unknown") + def read_text(self, relative_path, limit): + raise CapabilityError("Reading metadata is unsupported by this transport") def close(self): pass diff --git a/mod/hallmark/transport/http.py b/mod/hallmark/transport/http.py index 0336b4e..45882f0 100644 --- a/mod/hallmark/transport/http.py +++ b/mod/hallmark/transport/http.py @@ -2,15 +2,19 @@ from __future__ import annotations -import hashlib +from urllib.parse import unquote, urljoin, urlsplit import requests from ..helper_functions import REMOTE_REQUEST_TIMEOUT -from .base import DownloadError, RemoteObjectMissing, Transport +from .base import DownloadError, RemoteObjectMissing, Transport, reject_controls class HttpTransport(Transport): + def __init__(self, context): + super().__init__(context) + self.text_urls = {} + def _get(self, path): return self.context.session().get( getattr(self, "direct_url", None) or self.context.remote.file_url(path), @@ -25,6 +29,45 @@ def _error(self, exc): detail = f"HTTP {status}" if isinstance(status, int) else type(exc).__name__ return cls(f"Failed to download from {self.context.remote.display}: {detail}") + def _metadata_get(self, relative_path): + """Follow bounded metadata redirects only within the declared source root.""" + url = self.context.remote.file_url(relative_path) + root = urlsplit(self.context.remote.url) + root_path = unquote(root.path, errors="strict").rstrip("/") or "/" + + def endpoint(parsed): + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return parsed.scheme, parsed.hostname, port + + for _ in range(11): + self.context.check_cancelled() + try: + reject_controls(url, "Metadata URL") + target = urlsplit(url) + path = unquote(target.path, errors="strict") + reject_controls(path, "Metadata path") + within_root = (root_path == "/" or path == root_path + or path.startswith(root_path + "/")) + if (endpoint(target) != endpoint(root) or not within_root + or ".." in path.split("/") or "\\" in path + or target.username != root.username + or target.password != root.password): + raise DownloadError("Metadata redirect leaves the source root") + except (ValueError, UnicodeError): + raise DownloadError("Invalid metadata redirect URL") from None + response = self.context.session().get( + url, stream=True, timeout=REMOTE_REQUEST_TIMEOUT, + allow_redirects=False, + ) + if response.status_code not in {301, 302, 303, 307, 308}: + return response, url + location = response.headers.get("Location") + response.close() + if not isinstance(location, str) or not location: + raise DownloadError("Metadata redirect has no location") + url = urljoin(url, location) + raise DownloadError("Metadata redirect limit exceeded") + def fetch(self, relative_path, destination, *, chunk_size=8192): try: with self._get(relative_path) as response: @@ -34,13 +77,16 @@ def fetch(self, relative_path, destination, *, chunk_size=8192): self.context.check_cancelled() if chunk: handle.write(chunk) + if self.context.on_bytes is not None: + self.context.on_bytes(len(chunk)) except requests.RequestException as exc: raise self._error(exc) from None def read_text(self, relative_path, limit): content = bytearray() try: - with self._get(relative_path) as response: + response, final_url = self._metadata_get(relative_path) + with response: response.raise_for_status() for chunk in response.iter_content(chunk_size=8192): self.context.check_cancelled() @@ -48,38 +94,8 @@ def read_text(self, relative_path, limit): if len(content) > limit: raise DownloadError("Remote text exceeds the configured limit") encoding = response.encoding or "utf-8" - return content.decode(encoding, errors="replace") + text = content.decode(encoding, errors="replace") + self.text_urls[relative_path] = final_url + return text except requests.RequestException as exc: raise self._error(exc) from None - - def checksum_small(self, relative_path): - unknown = ("unknown", "unknown") - try: - with self.context.session().head( - self.context.remote.file_url(relative_path), - timeout=REMOTE_REQUEST_TIMEOUT, - ) as response: - response.raise_for_status() - try: - size = int(response.headers.get("Content-Length")) - except (TypeError, ValueError): - return unknown - if size < 0 or size > self.context.hash_file_limit: - return unknown - digest, received = hashlib.md5(), 0 - with self._get(relative_path) as response: - response.raise_for_status() - for chunk in response.iter_content(chunk_size=8192): - self.context.check_cancelled() - received += len(chunk) - if received > self.context.hash_file_limit: - return unknown - digest.update(chunk) - if received != size: - raise DownloadError("Remote file changed while computing its checksum") - return "md5", digest.hexdigest() - except requests.RequestException as exc: - status = getattr(getattr(exc, "response", None), "status_code", None) - if status in {401, 403}: - raise self._error(exc) from None - return unknown diff --git a/mod/hallmark/transport/sftp.py b/mod/hallmark/transport/sftp.py new file mode 100644 index 0000000..67acc0e --- /dev/null +++ b/mod/hallmark/transport/sftp.py @@ -0,0 +1,278 @@ +"""Read-only SFTP v3 metadata over an operation-owned OpenSSH subsystem. + +OpenSSH owns authentication and encryption. This module only frames the small +metadata protocol, so names and attributes never pass through shell/ls parsing. +""" + +from __future__ import annotations + +import os +import selectors +import struct +import subprocess +import time + +from .base import CapabilityError, DownloadError, RemoteObjectMissing + + +_MAX_PACKET = 1024 * 1024 +_KNOWN_ATTRIBUTES = 0x8000000F + + +def _string(value): + if isinstance(value, str): + value = value.encode("utf-8") + return struct.pack(">I", len(value)) + value + + +class _Packet: + def __init__(self, data): + self.data = data + self.offset = 0 + + def take(self, size): + end = self.offset + size + if end > len(self.data): + raise DownloadError("Truncated SFTP metadata packet") + result = self.data[self.offset:end] + self.offset = end + return result + + def uint32(self): + return struct.unpack(">I", self.take(4))[0] + + def string(self): + return self.take(self.uint32()) + + def text(self): + try: + return self.string().decode("utf-8") + except UnicodeError: + raise DownloadError("SFTP filename must contain valid UTF-8") from None + + def finish(self): + if self.offset != len(self.data): + raise DownloadError("Unexpected trailing SFTP metadata") + + def attributes(self): + flags = self.uint32() + if flags & ~_KNOWN_ATTRIBUTES: + raise DownloadError("Unsupported SFTP attribute flags") + size = struct.unpack(">Q", self.take(8))[0] if flags & 1 else None + if flags & 2: + self.take(8) # uid and gid + mode = self.uint32() if flags & 4 else None + mtime = None + if flags & 8: + self.uint32() # atime + mtime = self.uint32() + if flags & 0x80000000: + count = self.uint32() + if count > (len(self.data) - self.offset) // 8: + raise DownloadError("Invalid SFTP attribute count") + for _ in range(count): + self.string() + self.string() + return {"size": size, "mode": mode, "mtime": mtime} + + +class SftpMetadata: + """One bounded, cancellable request at a time on a binary subsystem pipe.""" + + def __init__(self, transport): + self.transport = transport + self.context = transport.context + self.process = None + self.selector = None + self.buffer = bytearray() + self.stderr_size = 0 + self.request_id = 0 + self.deadline = None + self._slot = False + + def __enter__(self): + self.transport.prepare() + while not self.transport._slots.acquire(timeout=0.05): + self.context.check_cancelled() + self._slot = True + try: + self.process = self.transport._spawn( + self.transport.ssh + self.transport._options() + + ["-T", "-s", "--", self.context.remote.host, "sftp"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0, + ) + self.selector = selectors.DefaultSelector() + for pipe in (self.process.stdin, self.process.stdout, self.process.stderr): + os.set_blocking(pipe.fileno(), False) + self.selector.register(self.process.stdout, selectors.EVENT_READ) + self.selector.register(self.process.stderr, selectors.EVENT_READ) + self._new_deadline() + self._send(b"\x01" + struct.pack(">I", 3)) + kind, packet = self._receive() + if kind != 2 or packet.uint32() != 3: + raise CapabilityError("Server must support SFTP version 3") + while packet.offset < len(packet.data): + packet.string() + packet.string() # Ignore unneeded negotiated extensions. + return self + except BaseException: + self.__exit__(None, None, None) + raise + + def __exit__(self, exc_type, exc, tb): + try: + if self.process is not None: + self.transport._stop(self.process) + finally: + if self.selector is not None: + self.selector.close() + if self.process is not None: + for pipe in ( + self.process.stdin, self.process.stdout, self.process.stderr, + ): + pipe.close() + if self._slot: + self.transport._slots.release() + self._slot = False + + def _new_deadline(self): + self.deadline = time.monotonic() + self.context.listing_timeout + + def _check(self): + self.context.check_cancelled() + if time.monotonic() >= self.deadline: + raise DownloadError("SFTP metadata request exceeded its time limit") + + def _pump(self): + self._check() + writable = False + for key, events in self.selector.select(timeout=0.05): + if events & selectors.EVENT_WRITE: + writable = True + continue + try: + chunk = os.read(key.fileobj.fileno(), 65536) + except BlockingIOError: + continue + if not chunk: + self.selector.unregister(key.fileobj) + elif key.fileobj is self.process.stdout: + self.buffer.extend(chunk) + if len(self.buffer) > _MAX_PACKET + 4: + raise DownloadError("SFTP metadata packet exceeds its size limit") + else: + self.stderr_size += len(chunk) + if self.stderr_size > 65536: + raise DownloadError("SSH diagnostics exceeded their size limit") + return writable + + def _send(self, payload): + if len(payload) > _MAX_PACKET: + raise DownloadError("SFTP metadata request exceeds its size limit") + data = memoryview(struct.pack(">I", len(payload)) + payload) + self.selector.register(self.process.stdin, selectors.EVENT_WRITE) + try: + while data: + if self._pump(): + try: + written = os.write(self.process.stdin.fileno(), data) + except BlockingIOError: + continue + except BrokenPipeError: + self.context.check_cancelled() + raise DownloadError("SFTP subsystem disconnected") from None + data = data[written:] + finally: + self.selector.unregister(self.process.stdin) + + def _read(self, size): + while len(self.buffer) < size: + self._pump() + self.context.check_cancelled() + if (len(self.buffer) < size + and self.process.stdout.fileno() not in self.selector.get_map()): + raise DownloadError("SFTP subsystem returned truncated metadata") + self._check() + result = bytes(self.buffer[:size]) + del self.buffer[:size] + return result + + def _receive(self): + length = struct.unpack(">I", self._read(4))[0] + if not 1 <= length <= _MAX_PACKET: + raise DownloadError("Invalid or oversized SFTP metadata packet") + data = self._read(length) + return data[0], _Packet(data[1:]) + + def _request(self, kind, data, expected, *, eof=False): + self._new_deadline() + self.request_id = (self.request_id + 1) % (2 ** 32) + self._send(bytes([kind]) + struct.pack(">I", self.request_id) + data) + response, packet = self._receive() + if packet.uint32() != self.request_id: + raise DownloadError("Mismatched SFTP response identifier") + if response == 101: + status = packet.uint32() + packet.string() # Untrusted server diagnostics are never echoed. + packet.string() + packet.finish() + if status == 1 and eof: + return None + if status == 0 and expected == 101: + return packet + if status == 2: + raise RemoteObjectMissing("Remote metadata object does not exist") + if status == 8: + raise CapabilityError("Server does not support SFTP metadata access") + raise DownloadError(f"SFTP metadata operation failed (status {status})") + if response != expected: + raise DownloadError("Unexpected SFTP metadata response") + return packet + + def realpath(self, path): + packet = self._request(16, _string(path), 104) + if packet.uint32() != 1: + raise DownloadError("Invalid SFTP canonical path response") + result = packet.text() + packet.string() + packet.attributes() + packet.finish() + return result + + def lstat(self, path): + packet = self._request(7, _string(path), 105) + result = packet.attributes() + packet.finish() + return result + + def iterdir(self, path): + packet = self._request(11, _string(path), 102) + handle = packet.string() + packet.finish() + if not handle or len(handle) > 256: + raise DownloadError("Invalid SFTP directory handle") + completed = False + try: + while True: + packet = self._request(12, _string(handle), 104, eof=True) + if packet is None: + completed = True + return + count = packet.uint32() + if count == 0 or count > (len(packet.data) - packet.offset) // 12: + raise DownloadError("Invalid SFTP directory entry count") + entries = [] + for _ in range(count): + name = packet.text() + packet.string() # Human-readable longname is not parsed. + entries.append((name, packet.attributes())) + packet.finish() + yield from entries + finally: + if not self.context.cancelled.is_set(): + try: + self._request(4, _string(handle), 101) + except DownloadError: + if completed: + raise diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index f1265dd..3dcb801 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -5,7 +5,7 @@ import os import re import selectors -import shlex +import stat import shutil import signal import subprocess @@ -18,6 +18,7 @@ CapabilityError, DownloadError, RemoteConfigurationError, + RemoteEntry, Transport, literal_path, reject_controls, @@ -36,57 +37,6 @@ def batch_argument(path): return "".join("\\" + char if char in special else char for char in text) -# Fixed read-only programs. The server needs a POSIX login shell and python3. -# Paths are operands, never interpolated into program text. Symlinks are omitted. -_LIST_SCRIPT = """import os, stat, sys, time -root, max_entries, max_bytes, seconds = sys.argv[1:] -limit, budget = int(max_entries), int(max_bytes) -deadline = time.monotonic() + int(seconds) -count = size = 0 -if not os.path.isdir(root): - raise RuntimeError("Not a directory") -def fail(error): - raise error -for directory, dirs, files in os.walk(root, followlinks=False, onerror=fail): - dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(directory, d))] - for name in files: - path = os.path.join(directory, name) - info = os.lstat(path) - if not stat.S_ISREG(info.st_mode): - continue - relative = os.path.relpath(path, root) - record = (os.fsencode(relative) + b"\\0" + str(info.st_size).encode() - + b"\\0" + str(info.st_mtime_ns).encode() + b"\\0") - count += 1 - size += len(record) - if count > limit or size > budget or time.monotonic() > deadline: - raise RuntimeError("Listing limit exceeded") - sys.stdout.buffer.write(record) -""" - -_HASH_SCRIPT = """import hashlib, os, stat, sys -path, size, mtime = sys.argv[1:] -with open(path, "rb") as handle: - before = os.fstat(handle.fileno()) - if (not stat.S_ISREG(before.st_mode) or before.st_size != int(size) - or before.st_mtime_ns != int(mtime)): - raise RuntimeError("File changed") - digest = hashlib.sha256() - remaining = int(size) - while remaining: - chunk = handle.read(min(65536, remaining)) - if not chunk: - raise RuntimeError("File shortened") - remaining -= len(chunk) - digest.update(chunk) - after = os.fstat(handle.fileno()) - if (handle.read(1) or after.st_size != before.st_size - or after.st_mtime_ns != before.st_mtime_ns): - raise RuntimeError("File changed") -print(digest.hexdigest()) -""" - - class SshTransport(Transport): def __init__(self, context, *, ssh=None, sftp=None): super().__init__(context) @@ -101,10 +51,6 @@ def __init__(self, context, *, ssh=None, sftp=None): self._socket_dir = None self._socket = None self._slots = BoundedSemaphore(context.settings.max_sessions) - self._entries = {} - self._hash_lock = Lock() - self._hash_bytes = 0 - self._hash_started = None def _options(self, *, master=False): settings = self.context.settings @@ -187,10 +133,22 @@ def _run( destination=None, file_limit=None, capture_stderr=False, + report_progress=False, ): output = bytearray() stderr_size = 0 diagnostics = bytearray() + reported_bytes = 0 + + def report_bytes(): + nonlocal reported_bytes + if report_progress and destination is not None and destination.exists(): + current = destination.stat().st_size + if current > reported_bytes: + if self.context.on_bytes is not None: + self.context.on_bytes(current - reported_bytes) + reported_bytes = current + with tempfile.TemporaryFile() as batch: batch.write(data) batch.seek(0) @@ -205,6 +163,7 @@ def _run( selector.register(pipe, selectors.EVENT_READ) while selector.get_map() or process.poll() is None: self.context.check_cancelled() + report_bytes() if time.monotonic() >= deadline: raise DownloadError("SSH operation exceeded its time limit") if ( @@ -228,6 +187,7 @@ def _run( "SSH output exceeded its size limit" ) self.context.check_cancelled() + report_bytes() if process.wait() != 0: # stderr is untrusted and may contain echoed credentials/paths. # SFTP exits do not reliably distinguish absence from auth errors. @@ -309,12 +269,16 @@ def _fetch(self, relative_path, destination, file_limit=None): data=batch, destination=destination, file_limit=file_limit, + report_progress=file_limit is None, ) def fetch(self, relative_path, destination, *, chunk_size=8192): self._fetch(relative_path, destination) def read_text(self, relative_path, limit): + entry = self.stat(relative_path) + if entry.size is not None and entry.size > limit: + raise DownloadError("Remote text exceeds its size limit") with tempfile.TemporaryDirectory(prefix="hm-text-") as directory: destination = Path(directory) / "content" self._fetch(relative_path, destination, file_limit=limit) @@ -325,85 +289,71 @@ def read_text(self, relative_path, limit): except UnicodeError: raise DownloadError("Remote manifest must contain UTF-8 text") from None - def _command(self, script, operands, *, timeout, limit): - if not self.context.allow_remote_commands: - raise CapabilityError( - "SSH builds require --allow-remote-commands, a POSIX shell and " - "server python3; SFTP-only accounts support downloads only" - ) - self.prepare() - command = " ".join( - shlex.quote(value) - for value in ["python3", "-c", script, *map(str, operands)] - ) - with self._slots: - return self._run( - self.ssh + self._options() + ["--", self.context.remote.host, command], - timeout=timeout, - limit=limit, - ) + def _metadata(self): + from .sftp import SftpMetadata - def list_entries(self): - context = self.context - data = self._command( - _LIST_SCRIPT, - [ - context.remote.root, - context.listing_limit, - context.text_limit, - context.listing_timeout, - ], - timeout=context.listing_timeout, - limit=context.text_limit, - ) - try: - fields = data.decode("utf-8").split("\0") - if fields.pop() != "" or len(fields) % 3: - raise ValueError - for i in range(0, len(fields), 3): - path = literal_path(fields[i]).as_posix() - size, mtime = int(fields[i + 1]), int(fields[i + 2]) - if size < 0 or path in self._entries: - raise ValueError - self._entries[path] = (size, mtime) - if len(self._entries) > context.listing_limit: - raise ValueError - except (UnicodeError, ValueError): - raise DownloadError("Invalid or oversized SSH listing") from None - return list(self._entries) + return SftpMetadata(self) - def checksum_small(self, relative_path): - if not self.context.remote_hash: - return "unknown", "unknown" - size, mtime = self._entries.get(relative_path, (-1, 0)) - with self._hash_lock: - if ( - size < 0 - or size > self.context.hash_file_limit - or self._hash_bytes + size > self.context.hash_total_limit - ): - return "unknown", "unknown" - if self._hash_started is None: - self._hash_started = time.monotonic() - remaining = self.context.hash_timeout - ( - time.monotonic() - self._hash_started - ) - if remaining <= 0: - return "unknown", "unknown" - self._hash_bytes += size - result = ( - self._command( - _HASH_SCRIPT, - [self.context.remote.pathname(relative_path), size, mtime], - timeout=remaining, - limit=128, - ) - .decode("ascii", errors="replace") - .strip() - ) - if not re.fullmatch(r"[0-9a-f]{64}", result): - raise DownloadError("Invalid remote SHA-256 result") - return "sha256", result + def stat(self, relative_path): + """Read attributes without fetching file contents or following symlinks.""" + path = literal_path(relative_path).as_posix() + with self._metadata() as session: + attrs = session.lstat(self.context.remote.pathname(path)) + mode = attrs["mode"] + if mode is None or not stat.S_ISREG(mode): + raise DownloadError("Remote metadata must be a regular file") + return RemoteEntry(path, attrs["size"], attrs["mtime"]) + + def iter_entries(self, on_directory=None): + """Yield regular files recursively using only the SFTP subsystem.""" + with self._metadata() as session: + root = session.realpath(self.context.remote.root).rstrip("/") or "/" + reject_controls(root, "SFTP root") + if not root.startswith("/") or ".." in root.split("/"): + raise DownloadError("Invalid SFTP canonical root") + pending = [("", root)] + visited = set() + while pending: + self.context.check_cancelled() + relative, directory = pending.pop() + canonical = session.realpath(directory).rstrip("/") or "/" + if (root != "/" and canonical != root + and not canonical.startswith(root + "/")): + raise DownloadError("SFTP directory escapes the source root") + if canonical in visited: + continue + visited.add(canonical) + attrs = session.lstat(directory) + if attrs["mode"] is None: + raise CapabilityError("SFTP server omitted directory type") + if not stat.S_ISDIR(attrs["mode"]): + raise DownloadError("SFTP discovery root must be a directory") + if on_directory is not None: + on_directory(relative) + names = set() + for name, attrs in session.iterdir(directory): + self.context.check_cancelled() + if name in {".", ".."}: + continue + if name.lower() in {".hm", ".git"}: + continue + if not name or "/" in name or name in names: + raise DownloadError("Invalid or duplicate SFTP directory name") + names.add(name) + path = literal_path(relative + name).as_posix() + absolute = directory.rstrip("/") + "/" + name + mode = attrs["mode"] + if mode is None: + attrs = session.lstat(absolute) + mode = attrs["mode"] + if mode is None: + raise CapabilityError("SFTP server omitted file type") + if stat.S_ISDIR(mode): + pending.append((path + "/", absolute)) + elif stat.S_ISREG(mode): + entry = RemoteEntry(path, attrs["size"], attrs["mtime"]) + yield entry + # Symlinks and special files are deliberately not traversed. def cancel(self): with self._process_lock: diff --git a/test/mock_server.py b/test/mock_server.py index 5e22afd..4191bc9 100644 --- a/test/mock_server.py +++ b/test/mock_server.py @@ -36,7 +36,7 @@ def __exit__(self, exc_type, exc_value, traceback): def close(self): pass - def get(self, url, timeout=None, stream=False): + def get(self, url, timeout=None, stream=False, allow_redirects=True): return self.fake_get(url, timeout=timeout) def head(self, url, timeout=None): diff --git a/test/test_catalog.py b/test/test_catalog.py new file mode 100644 index 0000000..a89603b --- /dev/null +++ b/test/test_catalog.py @@ -0,0 +1,212 @@ +"""Catalog cloning must never need dataset bodies or local content objects.""" + +from pathlib import Path + +import pytest +import yaml + +from hallmark import Repo +from hallmark.error import CloneError, DestinationExistsError +from hallmark.transport import OperationContext +from hallmark.transport.base import DownloadError, RemoteObjectMissing + + +@pytest.fixture +def metadata_server(monkeypatch): + pages, reads = {}, [] + + def read_text(context, path): + url = context.remote.url.rstrip("/") + "/" + path + reads.append(url) + if url not in pages: + raise RemoteObjectMissing("No such metadata object") + return pages[url] + + monkeypatch.setattr(OperationContext, "read_text", read_text) + return pages, reads + + +def test_directory_clone_is_recursive_filtered_and_payload_free( + tmp_path, metadata_server): + pages, reads = metadata_server + root = "https://example.test/data/" + pages[root] = ('

Index of /data/

root.h5' + 'note.txtsub/') + pages[root + "sub/"] = '

Index of sub

data.h5' + events = [] + repo = Repo.clone(root, tmp_path / "clone", filter="**/*.h5", + progress=events.append) + assert repo.state.data["path"].tolist() == ["root.h5", "sub/data.h5"] + assert repo.state.config["remote"] == [{"name": "origin", "url": root}] + assert repo.state.config["data"] == [{"db": "data.tsv"}] + assert not (repo.worktree / "root.h5").exists() + assert not any(url.endswith((".h5", ".txt")) for url in reads) + plan = repo.plan_download() + assert plan.file_count == 2 + assert plan.unknown_size_count == 2 + assert events[-1]["directories"] == 2 + + +def test_url_only_clone_does_not_require_filename_inference(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/odd/" + pages[root] = '

Index of odd

odd name' + repo = Repo.clone(root, tmp_path / "clone") + assert repo.state.data["path"].tolist() == ["odd name"] + assert repo.plan_download().file_count == 1 + + +def test_empty_filtered_clone_is_valid(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root] = '

Index of data

notes.txt' + repo = Repo.clone(root, tmp_path / "clone", fmt="run{run:d}.h5") + assert repo.state.data.empty + assert repo.plan_download().file_count == 0 + + +def test_template_enriches_and_filters_inventory(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root] = ('

Index of data

run2.h5' + 'notes.txt') + repo = Repo.clone(root, tmp_path / "clone", fmt="run{run:d}.h5") + assert repo.state.data["path"].tolist() == ["run2.h5"] + assert str(repo.state.data.iloc[0]["run"]) == "2" + + +@pytest.mark.parametrize("nested", [False, True]) +def test_published_snapshot_preserves_data_remote(tmp_path, metadata_server, nested): + pages, reads = metadata_server + root = "https://example.test/published/" + prefix = root + (".hm/" if nested else "") + pages[prefix + "config.yml"] = yaml.safe_dump({ + "data": [{"db": "data.tsv"}], + "remote": [{"name": "origin", "url": "https://mirror.test/data/"}]}) + pages[prefix + "meta.yml"] = "dataset: example\n" + pages[prefix + "data.tsv"] = "path\tsize_bytes\na.h5\t12\nb.txt\t20\n" + repo = Repo.clone(root, tmp_path / "clone", filter="**/*.h5") + assert repo.state.data["path"].tolist() == ["a.h5"] + assert repo.plan_download().total_bytes == 12 + assert repo.plan_download().remote_url == "https://mirror.test/data/" + assert not any(url.endswith("a.h5") for url in reads) + assert repo.dothm.head.commit.message.strip() == "Filter local catalog" + + +def test_filtered_git_clone_preserves_history_without_objects(tmp_path, monkeypatch): + source = Repo.init(tmp_path / "source") + (source.worktree / "run1.h5").write_text("one") + (source.worktree / "run2.h5").write_text("two") + source.add("run{run:d}.h5") + source.set_config(remote_url="https://example.test/data/") + source.commit("Original scientific data") + head = source.dothm.head.commit.hexsha + assert list(source.objects.root.rglob("*")) + + def fail(*args, **kwargs): + raise AssertionError("Filtered metadata clone must not transfer data") + + monkeypatch.setattr("hallmark.downloader._fetch_file", fail) + repo = Repo.clone(str(source.dothm.path), tmp_path / "clone", filter="run1.h5") + assert repo.dothm.head.commit.parents[0].hexsha == head + assert len(repo.state.data) == 1 + assert repo.plan_download().items[0].relative_path == Path("run1.h5") + assert not repo.objects.root.exists() + assert not (repo.worktree / "run1.h5").exists() + assert repo.dothm.remotes.origin.url == str(source.dothm.path) + + +def test_clone_requires_callback_before_download_intent(tmp_path, metadata_server): + _, reads = metadata_server + with pytest.raises(DownloadError, match="approve"): + Repo.clone("https://example.test/data/", tmp_path / "clone", download=True) + assert reads == [] + assert not (tmp_path / "clone").exists() + + +def test_clone_refusal_keeps_catalog_without_payload( + tmp_path, metadata_server, monkeypatch): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root] = '

Index of data

a.h5' + plans = [] + + def decline(plan): + plans.append(plan) + return False + + def fail(*args, **kwargs): + raise AssertionError("Refused transfer must not start") + + monkeypatch.setattr(Repo, "download", fail) + repo = Repo.clone(root, tmp_path / "clone", download=True, approve=decline) + assert len(plans) == 1 + assert plans[0].file_count == 1 + assert repo.download_result is None + assert not (repo.worktree / "a.h5").exists() + + +def test_interrupted_discovery_removes_incomplete_destination( + tmp_path, metadata_server, monkeypatch): + def fail(*args, **kwargs): + raise KeyboardInterrupt() + + monkeypatch.setattr("hallmark.catalog.discover", fail) + with pytest.raises(KeyboardInterrupt): + Repo.clone("https://example.test/data/", tmp_path / "clone", + source_type="directory") + assert not (tmp_path / "clone").exists() + + +def test_existing_destination_is_never_changed(tmp_path, metadata_server): + destination = tmp_path / "clone" + destination.mkdir() + (destination / "keep").write_text("keep") + with pytest.raises(DestinationExistsError): + Repo.clone("https://example.test/data/", destination) + assert (destination / "keep").read_text() == "keep" + assert metadata_server[1] == [] + + +def test_missing_snapshot_does_not_publish_empty_success(tmp_path, metadata_server): + with pytest.raises(CloneError, match="No published"): + Repo.clone("https://example.test/data/", tmp_path / "clone", + source_type="catalog") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_rejects_external_catalog_paths(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root + "config.yml"] = "data:\n- db: ../escape.tsv\n" + pages[root + "meta.yml"] = "{}\n" + pages[root + "data.tsv"] = "path\n" + with pytest.raises(ValueError): + Repo.clone(root, tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_filter_normalizes_legacy_db_names(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root + "config.yml"] = ( + "data:\n- db: data\n fmt: 'item_{i}.dat'\n") + pages[root + "meta.yml"] = "{}\n" + pages[root + "data.tsv"] = "i\n1\n2\n" + repo = Repo.clone(root, tmp_path / "clone", filter="item_1.dat") + assert repo.state.data["i"].astype(str).tolist() == ["1"] + + +@pytest.mark.parametrize("table", [ + "Login required\n", "", "path\n../escape\n", + "path\n.hm/config.yml\n", "unrelated\nrecord\n", +]) +def test_snapshot_validates_tables_without_filter(tmp_path, metadata_server, table): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root + "config.yml"] = "data:\n- db: data.tsv\n" + pages[root + "meta.yml"] = "{}\n" + pages[root + "data.tsv"] = table + with pytest.raises((CloneError, ValueError)): + Repo.clone(root, tmp_path / "clone") + assert not (tmp_path / "clone").exists() diff --git a/test/test_cli.py b/test/test_cli.py index 86e5ad7..39b9957 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -18,15 +18,17 @@ import yaml import importlib import pytest +import pandas as pd from pathlib import Path from click.testing import CliRunner from git import Repo as GitRepo from git.exc import GitError from types import SimpleNamespace -from hallmark import ParaFrame +from hallmark import ParaFrame, Repo from hallmark.cli import hallmark -from hallmark.downloader import DownloadError, BULK_DOWNLOAD_WARNING_FILE_COUNT +from hallmark.downloader import DownloadError +from hallmark.download_plan import DownloadItem, DownloadPlan from hallmark.helper_functions import chdir cli_module = importlib.import_module("hallmark.cli") @@ -50,21 +52,34 @@ def _install_repo(monkeypatch, worktree=Path("worktree")): A SimpleNamespace object representing the fake repository, with a 'worktree' attribute set to the given worktree path. """ - repo = SimpleNamespace(worktree=worktree) + repo = SimpleNamespace( + worktree=worktree, + plan_download=lambda output=None, **kwargs: _download_plan( + 0, output or worktree or "downloads")) monkeypatch.setattr(cli_module, "Repo", lambda path: repo) return repo -def _selection(count): - """ - Return a list of (Path, None) tuples for testing download selection, - with the given count of files. - Args: - count: The number of files to generate for the selection. - Returns: - A list of tuples, each containing a Path object for a file and None. - """ - return [(Path(f"file-{index:03d}.dat"), None) for index in range(count)] +def _download_plan(count, output=Path("worktree")): + """A real immutable plan for testing CLI rendering and approval boundaries.""" + return DownloadPlan(tuple(DownloadItem(Path(f"file-{index:03d}.dat"), + size_bytes=8 if index == 0 else None) + for index in range(count)), + "https://example.test/data/", Path(output)) + + +def _local_cli_catalog(path): + """Create a metadata-only repository backed by a tiny fake HTTP dataset.""" + repo = Repo.init(path) + repo.state.config = { + "data": [{"db": "data.tsv"}], + "remote": {"name": "origin", "url": "https://example.test/data/"}} + repo.state.data = pd.DataFrame([ + {"path": "tiny.fits", "size_bytes": 4}, + {"path": "other.txt", "size_bytes": 5}]) + repo.dothm.dump(repo.state) + repo.dothm.index.commit("Catalog two remote files") + return repo def parse(result): @@ -557,7 +572,7 @@ def test_clone_existing_destination_fails_with_plain_git_stderr(): assert ( result.output.strip() == "fatal: destination path 'repo3' already exists and " - "is not an empty directory."), \ + "is not empty."), \ f"Expected git error message, got: {result.output.strip()}" @@ -618,74 +633,37 @@ def test_clone_copies_committed_hallmark_state(): "Expected data.tsv to exist in target after clone" -def test_clone_reports_download_error_cleanly(monkeypatch): - """ - Test that the hallmark CLI 'clone' command reports a download error cleanly - when the remote URL is not configured in config.yml. - This test initializes a hallmark repository, monkeypatches the download function - to raise a DownloadError, and verifies that the clone command fails with the - expected error message. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - runner = CliRunner() - with runner.isolated_filesystem(): - source = Path("source") - result = runner.invoke(hallmark, ["init", str(source)]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for init, got {result.exit_code}" +def test_clone_reports_download_error_cleanly(monkeypatch, tmp_path): + source = _local_cli_catalog(tmp_path / "source") - GitRepo(str(source / ".hm")).index.commit("commit initial hallmark state") - def boom(*args, **kwargs): - """Simulate a download error by raising a DownloadError.""" - raise DownloadError("Remote URL not configured in config.yml") - monkeypatch.setattr("hallmark.cli.download_remote_data", boom) - monkeypatch.setattr( - "hallmark.cli.select_download_files", - lambda *args, **kwargs: [(Path("data.bin"), None)]) - result = runner.invoke( - hallmark, - ["clone", str(source / ".hm"), "target"]) + def fail_download(self, plan, **kwargs): + raise DownloadError("Remote download failed") - assert result.exit_code != 0, f"Expected non-zero exit code for clone with \ - download error, got {result.exit_code}" - assert "Remote URL not configured in config.yml" in result.output, \ - f"Expected download error message in output, got: {result.output}" + monkeypatch.setattr(Repo, "download", fail_download) + result = CliRunner().invoke(hallmark, [ + "clone", str(source.dothm.path), str(tmp_path / "target"), "--download"], + input="y\n") + assert result.exit_code != 0 + assert "Error: Remote download failed" in result.output + assert "Download these files? [y/N]" in result.output -def test_clone_cli_skips_download_when_no_remote_files(monkeypatch): - """ - Test that the hallmark CLI 'clone' command skips the download step when there are - no remote files configured in the source repository. - This test monkeypatches the Repo class to simulate a source repository with no - remote files and verifies that the clone command completes successfully without - attempting to download any files. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - repo = SimpleNamespace() +def test_clone_cli_skips_download_when_no_remote_files(monkeypatch, tmp_path): + repo = SimpleNamespace(worktree=tmp_path, + plan_download=lambda: _download_plan(0, tmp_path)) + class FakeRepo: - """Fake Repo class to simulate a hallmark repository with no remote files.""" @staticmethod - def clone(url, path, fetch_data=False): - """Simulate cloning a repository by returning a SimpleNamespace.""" + def clone(url, path, **kwargs): return repo - @staticmethod - def lwpaths(path): - """Return the paths to the .hm directory and the worktree.""" - return Path(path) / ".hm", Path(path) monkeypatch.setattr(cli_module, "Repo", FakeRepo) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: []) - result = CliRunner().invoke(hallmark, ["clone", "source", "target"]) - - assert result.exit_code == 0, f"Expected exit code 0 for clone with no remote \ - files, got {result.exit_code}" - assert 'Successfully cloned to "target"' in result.output, \ - f"Expected success message in output, got: {result.output}" - assert "No remote data files are configured." in result.output, \ - f"Expected message about no remote files, got: {result.output}" + result = CliRunner().invoke(hallmark, [ + "clone", "source", "target", "--download"]) + assert result.exit_code == 0, result.output + assert 'Successfully cloned to "target"' in result.output + assert "No files selected for download." in result.output + assert "Download these files?" not in result.output @pytest.mark.parametrize("max_workers", [0, -1]) @@ -1174,7 +1152,7 @@ def test_build_cli_reports_missing_config_yml_in_directory(tmp_path): @pytest.mark.parametrize( "arguments, message", [ - (["download"], "Provide one or more file paths, --tsv, or --all"), + (["download"], "Provide file paths, --tsv, --all, --filter, or --fmt"), (["download", "file.dat", "--all"], "--all cannot be combined"), (["download", "--tsv", "data", "--all"], "--all cannot be combined")]) def test_download_cli_rejects_invalid_selection_combinations( @@ -1236,272 +1214,206 @@ def test_download_cli_rejects_nonpositive_worker_count(monkeypatch): def test_download_cli_dry_run_limits_preview(monkeypatch): - """ - Test that the hallmark CLI 'download' command in dry-run mode limits the preview - of selected files to the first 20 files. This test monkeypatches the repository - installation and the selection function to simulate a selection of 23 files, and - verifies that the download command in dry-run mode outputs only the first 20 files - and indicates that there are more files selected. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - Raises: - AssertionError: If the download function is called during the dry-run, which - should not happen. - """ - _install_repo(monkeypatch) - selected = _selection(23) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: selected) - def should_not_download(*args, **kwargs): - """Simulate a dry-run by raising an AssertionError if download is attempted.""" + repo = _install_repo(monkeypatch) + plan = _download_plan(23) + repo.plan_download = lambda *args, **kwargs: plan + + def reject_download(*args, **kwargs): raise AssertionError("dry-run must not download") - monkeypatch.setattr(cli_module, "download_remote_data", should_not_download) - result = CliRunner().invoke(hallmark, ["download", "--all", "--dry-run"]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for dry-run download, got {result.exit_code}" - assert "Selected 23 file(s)" in result.output, \ - f"Expected message about 23 selected files, got: {result.output}" - assert "file-000.dat" in result.output, \ - f"Expected first selected file in output, got: {result.output}" - assert "file-019.dat" in result.output, \ - f"Expected 20th selected file in output, got: {result.output}" - assert "file-020.dat" not in result.output, \ - f"Expected 21st selected file not in output, got: {result.output}" - assert "... 3 more file(s)" in result.output, \ - f"Expected message about 3 more files, got: {result.output}" + repo.download = reject_download + result = CliRunner().invoke(hallmark, ["download", "--all", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "23 file(s)" in result.output + assert "8 bytes known; 22 file(s) with unknown size" in result.output + assert "estimated duration: unknown" in result.output + assert "Source: https://example.test/data/" in result.output + assert f"Destination: {plan.output_path}" in result.output + assert "file-000.dat" in result.output + assert "file-019.dat" in result.output + assert "file-020.dat" not in result.output + assert "... 3 more file(s)" in result.output + assert "Download these files?" not in result.output def test_download_cli_reports_empty_selection(monkeypatch): - """ - Test that the hallmark CLI 'download' command reports an empty selection when no - files are selected for download. This test monkeypatches the repository installation - and the selection function to simulate an empty selection, and verifies that the - download command outputs the expected message indicating that no files were selected - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ _install_repo(monkeypatch) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: []) result = CliRunner().invoke(hallmark, ["download", "--all"]) - - assert result.exit_code == 0, f"Expected exit code 0 for download with empty \ - selection, got {result.exit_code}" - assert "No files selected for download." in result.output, \ - f"Expected message about no files selected, got: {result.output}" + assert result.exit_code == 0, result.output + assert "No files selected for download." in result.output + assert "Download these files?" not in result.output def test_download_cli_passes_selection_and_options_to_downloader(monkeypatch): - """ - Test that the hallmark CLI 'download' command passes the selected files and options - to the downloader function. This test monkeypatches the repository installation, - the selection function, and the downloader function to capture the arguments passed - to them, and verifies that the download command outputs the expected success message - and that the captured arguments match the expected values. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ repo = _install_repo(monkeypatch) - selected = [(Path("nested/file.dat"), "abc")] + plan = _download_plan(1) captured = {} - def fake_select(actual_repo, **kwargs): - """Fake selection function to capture arguments passed to it.""" - captured["select"] = (actual_repo, kwargs) - return selected - - def fake_download(actual_repo, output_path, **kwargs): - """Fake downloader function to capture arguments passed to it.""" - captured["download"] = (actual_repo, output_path, kwargs) - return { - "succeeded": 1, - "failed": 0, - "total_bytes": 1024 * 1024, - "errors": []} - monkeypatch.setattr(cli_module, "select_download_files", fake_select) - monkeypatch.setattr(cli_module, "download_remote_data", fake_download) - result = CliRunner().invoke( - hallmark, - [ - "download", - "nested/file.dat", - "--remote", - "mirror", - "--max-workers", - "2", - "--yes"]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for download, got {result.exit_code}" - assert "Successfully downloaded 1 files (1.0 MB)" in result.output, \ - f"Expected success message in output, got: {result.output}" - assert captured["select"] == ( - repo, - {"file_paths": ("nested/file.dat",), "tsv_names": (), "all_files": False}), \ - f"Expected captured selection arguments to match expected values, \ - got: {captured['select']}" - assert captured["download"] == ( - repo, - Path("worktree"), - { - "max_workers": 2, - "show_progress": True, - "selected_files": selected, - "remote_name": "mirror"}), f"Expected captured download arguments to match \ - expected values, got: {captured['download']}" + def fake_plan(output, **kwargs): + captured["plan"] = (output, kwargs) + return plan + + def fake_download(actual_plan, **kwargs): + captured["download"] = (actual_plan, kwargs) + return {"succeeded": 1, "failed": 0, "total_bytes": 1024 * 1024, + "errors": []} + + repo.plan_download = fake_plan + repo.download = fake_download + result = CliRunner().invoke(hallmark, [ + "download", "nested/file.dat", "--remote", "mirror", "--max-workers", "2", + "--filter", "**/*.dat", "--fmt", "nested/{name}.dat"], input="y\n") + assert result.exit_code == 0, result.output + assert "Successfully downloaded 1 files (1.0 MB)" in result.output + assert captured["plan"] == (None, { + "file_paths": ("nested/file.dat",), "tsv_names": (), "all_files": False, + "filter": ("**/*.dat",), "fmt": "nested/{name}.dat", "remote_name": "mirror"}) + assert captured["download"][0] is plan + assert captured["download"][1] == { + "max_workers": 2, "progress": True, "approved": True} def test_download_cli_allows_explicit_output_for_bare_repository(monkeypatch): - """ - Test that the hallmark CLI 'download' command allows an explicit output path to be - specified when the repository is bare (i.e., has no worktree). This test - monkeypatches the repository installation to simulate a bare repository and verifies - the download command completes successfully when an explicit output path is provided - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ _install_repo(monkeypatch, worktree=None) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: []) - - result = CliRunner().invoke( - hallmark, ["download", "--all", "--output", "downloads", "--dry-run"]) - - assert result.exit_code == 0, f"Expected exit code 0 for download with explicit \ - output, got {result.exit_code}" - assert "for downloads" in result.output, \ - f"Expected message about output path, got: {result.output}" + result = CliRunner().invoke(hallmark, [ + "download", "--all", "--output", "downloads", "--dry-run"]) + assert result.exit_code == 0, result.output + assert f"Destination: {Path('downloads').absolute()}" in result.output def test_download_cli_converts_selection_errors_to_click_errors(monkeypatch): - """ - Test that the CLI 'download' command converts selection errors to click errors. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - Raises: - AssertionError: If the download function is called during the selection error, - which should not happen. - """ - _install_repo(monkeypatch) - def fail_selection(*args, **kwargs): - """Simulate a selection error by raising a DownloadError.""" + repo = _install_repo(monkeypatch) + + def fail_plan(*args, **kwargs): raise DownloadError("bad selection") - monkeypatch.setattr(cli_module, "select_download_files", fail_selection) - result = CliRunner().invoke(hallmark, ["download", "--tsv", "missing"]) - assert result.exit_code != 0, f"Expected non-zero exit code for download with \ - selection error, got {result.exit_code}" - assert "Error: bad selection" in result.output, \ - f"Expected error message about selection error, got: {result.output}" + repo.plan_download = fail_plan + result = CliRunner().invoke(hallmark, ["download", "--tsv", "missing"]) + assert result.exit_code != 0 + assert "Error: bad selection" in result.output def test_download_cli_reports_only_first_ten_errors(monkeypatch): - """ - Test that the hallmark CLI 'download' command reports only the first ten errors - when there are more than ten failed downloads. This test monkeypatches the repo - installation, the selection function, and the downloader function to simulate a - scenario with 12 failed downloads, and verifies that the download command outputs - only the first ten errors and indicates that there are more errors. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - _install_repo(monkeypatch) - monkeypatch.setattr( - cli_module, - "select_download_files", - lambda *args, **kwargs: [(Path("file.dat"), None)]) - errors = [f"failure-{index}" for index in range(12)] - monkeypatch.setattr( - cli_module, - "download_remote_data", - lambda *args, **kwargs: { - "succeeded": 2, - "failed": 12, - "total_bytes": 0, - "errors": errors}) - result = CliRunner().invoke(hallmark, ["download", "file.dat", "--yes"]) + repo = _install_repo(monkeypatch) + repo.plan_download = lambda *args, **kwargs: _download_plan(14) + repo.download = lambda *args, **kwargs: { + "succeeded": 2, "failed": 12, "total_bytes": 0, + "errors": [f"failure-{index}" for index in range(12)]} + result = CliRunner().invoke(hallmark, ["download", "--all"], input="y\n") + assert result.exit_code != 0 + assert "2 succeeded, 12 failed" in result.output + assert "failure-0" in result.output + assert "failure-9" in result.output + assert "failure-10" not in result.output + assert "... 2 more error(s)" in result.output + assert "Failed to download 12 file(s)" in result.output + + +@pytest.mark.parametrize("count", [1, 100]) +@pytest.mark.parametrize("answer", ["n\n", "", "\n"]) +@pytest.mark.parametrize("legacy_yes", [False, True]) +def test_download_cli_requires_affirmative_approval( + monkeypatch, count, answer, legacy_yes): + repo = _install_repo(monkeypatch) + repo.plan_download = lambda *args, **kwargs: _download_plan(count) - assert result.exit_code != 0, f"Expected non-zero exit code for download with \ - multiple errors, got {result.exit_code}" - assert "2 succeeded, 12 failed" in result.output, \ - f"Expected summary of succeeded and failed downloads, got: {result.output}" - assert "failure-0" in result.output, \ - f"Expected first error in output, got: {result.output}" - assert "failure-9" in result.output, \ - f"Expected tenth error in output, got: {result.output}" - assert "failure-10" not in result.output, \ - f"Expected eleventh error not in output, got: {result.output}" - assert "... 2 more error(s)" in result.output, \ - f"Expected message about additional errors, got: {result.output}" - assert "Failed to download 12 file(s)" in result.output, \ - f"Expected summary of failed downloads, got: {result.output}" - - -def test_download_cli_prompts_for_bulk_selection(monkeypatch): - """ - Test that the hallmark CLI 'download' command prompts for confirmation when a bulk - selection of files is made. This test monkeypatches the repository installation, - the selection function, and the downloader function to simulate a bulk selection, - and verifies that the download command prompts for confirmation and aborts when - the user responds with 'n'. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - Raises: - AssertionError: If the download function is called during the aborted selection, - which should not happen. - """ - _install_repo(monkeypatch) - selected = _selection(BULK_DOWNLOAD_WARNING_FILE_COUNT) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: selected) + def reject_download(*args, **kwargs): + raise AssertionError("rejected or absent approval must not download") - def should_not_download(*args, **kwargs): - """Simulate an aborted selection by raising an AssertionError.""" - raise AssertionError("aborted selection must not download") - monkeypatch.setattr(cli_module, "download_remote_data", should_not_download) - result = CliRunner().invoke( - hallmark, ["download", "--all"], input="n\n") + repo.download = reject_download + args = ["download", "--all"] + (["--yes"] if legacy_yes else []) + result = CliRunner().invoke(hallmark, args, input=answer) + assert result.exit_code != 0 + assert f"{count} file(s)" in result.output + assert "Download these files? [y/N]" in result.output + assert "Aborted!" in result.output + if legacy_yes: + assert "--yes is deprecated" in result.output - assert result.exit_code != 0, f"Expected non-zero exit code for download with \ - aborted selection, got {result.exit_code}" - assert ( - f"Selected {BULK_DOWNLOAD_WARNING_FILE_COUNT} files for download" in - result.output), f"Expected message about bulk selection, got: {result.output}" - assert "Continue? [y/N]" in result.output, \ - f"Expected prompt for confirmation, got: {result.output}" - assert "Aborted!" in result.output, \ - f"Expected message about aborted download, got: {result.output}" - - -def test_download_cli_yes_skips_bulk_prompt(monkeypatch): - """ - Test that the hallmark CLI 'download' command skips the bulk selection prompt when - the --yes option is provided. This test monkeypatches the repository installation, - the selection function, and the downloader function to simulate a bulk selection, - and verifies that the download command completes successfully without prompting for - confirmation when the --yes option is used. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - _install_repo(monkeypatch) - selected = _selection(BULK_DOWNLOAD_WARNING_FILE_COUNT) - called = {"download": False} - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: selected) - def fake_download(*args, **kwargs): - """Fake downloader function to simulate a successful download.""" - called["download"] = True - return { - "succeeded": len(selected), - "failed": 0, - "total_bytes": 0, - "errors": []} - monkeypatch.setattr(cli_module, "download_remote_data", fake_download) - result = CliRunner().invoke(hallmark, ["download", "--all", "--yes"]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for download with --yes, got {result.exit_code}" - assert called["download"], \ - "Expected download function to be called with --yes, but it was not" - assert "Continue?" not in result.output, \ - f"Expected no prompt for confirmation with --yes, got: {result.output}" +@pytest.mark.parametrize("arguments", [[], ["--no-fetch-data"]]) +def test_clone_cli_defaults_to_metadata_only(monkeypatch, tmp_path, arguments): + source = _local_cli_catalog(tmp_path / "source") + + def reject_download(*args, **kwargs): + raise AssertionError("default clone must not plan or download payloads") + + monkeypatch.setattr(Repo, "plan_download", reject_download) + monkeypatch.setattr(Repo, "download", reject_download) + target = tmp_path / "target" + result = CliRunner().invoke(hallmark, [ + "clone", str(source.dothm.path), str(target), *arguments]) + assert result.exit_code == 0, result.output + assert (target / ".hm/data.tsv").is_file() + assert not (target / "tiny.fits").exists() + assert "Download these files?" not in result.output + + +def test_clone_cli_forwards_discovery_options(monkeypatch, tmp_path): + captured = {} + + class FakeRepo: + @staticmethod + def clone(url, path, **kwargs): + captured.update(url=url, path=path, **kwargs) + return SimpleNamespace(worktree=Path(path)) + + monkeypatch.setattr(cli_module, "Repo", FakeRepo) + result = CliRunner().invoke(hallmark, [ + "clone", "ssh://lab-data/export/", str(tmp_path / "target"), "--auth", "lab", + "--filter", "**/*.fits", "--filter", "README*", "--fmt", "{name}.fits", + "--source-type", "directory", "--max-workers", "2"]) + assert result.exit_code == 0, result.output + assert captured == { + "url": "ssh://lab-data/export/", "path": str(tmp_path / "target"), + "auth": "lab", "filter": ("**/*.fits", "README*"), "fmt": "{name}.fits", + "source_type": "directory", "progress": True, "max_workers": 2} + + +def test_clone_cli_rejects_conflicting_download_aliases(): + result = CliRunner().invoke(hallmark, [ + "clone", "source", "target", "--download", "--no-fetch-data"]) + assert result.exit_code != 0 + assert "--download conflicts with --no-fetch-data" in result.output + + +@pytest.mark.parametrize("during_clone", [False, True]) +@pytest.mark.parametrize("answer", ["y\n", "n\n", ""]) +def test_cli_downloads_only_approved_selected_payload( + monkeypatch, tmp_path, during_clone, answer): + from mock_server import MockServer + + source = _local_cli_catalog(tmp_path / "source") + server = MockServer("https://example.test/data/") + server.add_file("tiny.fits", b"fits") + server.add_file("other.txt", b"notes") + requests_made = [] + original_get = server.get + + def capture_get(url, **kwargs): + requests_made.append(url) + return original_get(url, **kwargs) + + server.get = capture_get + monkeypatch.setattr(requests, "Session", lambda: server) + if during_clone: + target = tmp_path / "target" + arguments = ["clone", str(source.dothm.path), str(target), + "--filter", "*.fits", "--download"] + else: + target = source.worktree + monkeypatch.chdir(target) + arguments = ["download", "--filter", "*.fits"] + result = CliRunner().invoke(hallmark, arguments, input=answer) + assert "1 file(s); 4 bytes" in result.output + assert "Download these files? [y/N]" in result.output + assert "Source: https://example.test/data/" in result.output + if answer == "y\n": + assert result.exit_code == 0, result.output + assert requests_made == ["https://example.test/data/tiny.fits"] + assert (target / "tiny.fits").read_bytes() == b"fits" + else: + assert result.exit_code != 0 + assert requests_made == [] + assert not (target / "tiny.fits").exists() + assert not (target / "other.txt").exists() diff --git a/test/test_discovery.py b/test/test_discovery.py new file mode 100644 index 0000000..964b170 --- /dev/null +++ b/test/test_discovery.py @@ -0,0 +1,267 @@ +"""Metadata-only discovery against local fixtures; no public network is used.""" + +from types import SimpleNamespace + +import pytest + +from hallmark.discovery import discover, path_matches +from hallmark.transport.base import ( + CapabilityError, DownloadError, RemoteEntry, RemoteSpec, TransferCancelled, +) + + +class Source: + def __init__(self, pages, url="https://data.test/public/"): + self.remote = RemoteSpec.parse(url) + self.pages = pages + self.reads = [] + + def read_text(self, path): + self.reads.append(path) + assert path in self.pages, f"Unexpected payload request: {path}" + return self.pages[path] + + def check_cancelled(self): + pass + + +def index(*links): + return 'Index of /' + ''.join( + f'{path}' for path in links) + + +@pytest.mark.parametrize("url", ["https://data.test/public/", "http://data.test/public"]) +def test_auto_index_walks_all_directories_without_payloads(url): + source = Source({ + "": index("root.fits", "nested/", "nested/", "../"), + "nested/": index("item.bin", "deeper/", "../"), + "nested/deeper/": index("file.txt"), + }, url) + entries = discover(source) + assert [entry.path for entry in entries] == [ + "nested/deeper/file.txt", "nested/item.bin", "root.fits"] + assert source.reads == ["", "nested/", "nested/deeper/"] + assert all(entry.checksum is None for entry in entries) + + +def test_cyverse_markup_is_detected_automatically(): + source = Source({"": '' + '
file' + '
'}) + assert [entry.path for entry in discover(source)] == ["file.fits"] + + +@pytest.mark.parametrize("page, size, mtime", [ + ('Index of /file' + ' 17-Sep-2026 02:31 12345\n', 12345, None), + ('Index of /
file' + '2026-09-17 02:3112345
', 12345, None), + ('
file' + '12,345
', 12345, None), + ('Index of /file', 12345, 1750000000), + ('Index of /file' + ' 17-Sep-2026 02:31 1.2M\n', None, None), +]) +def test_exact_sizes_are_preserved_and_rounded_sizes_remain_unknown(page, size, mtime): + assert discover(Source({"": page})) == [ + RemoteEntry(path="file.fits", size=size, mtime=mtime)] + + +def test_hallmark_and_git_directories_are_not_dataset_files(): + source = Source({"": index(".hm/", ".git/", ".HM/", "file.fits")}) + assert [entry.path for entry in discover(source)] == ["file.fits"] + assert source.reads == [""] + + +def test_desi_style_landing_page_links_to_scoped_indexes(): + source = Source({ + "": '

DESI public data

', + "dr1/": index("spectro/"), + "dr1/spectro/": index("redshift.fits"), + "edr/": index("old.fits"), + }) + assert [entry.path for entry in discover(source)] == [ + "dr1/spectro/redshift.fits", "edr/old.fits"] + + +@pytest.mark.parametrize("page", [ + 'Index of /empty/', '
', +]) +def test_recognized_empty_index(page): + assert discover(Source({"": page})) == [] + + +@pytest.mark.parametrize("page", [ + '

Welcome

Log in', + '
', + '
Access denied
', + 'not a directory index', +]) +def test_unsupported_page_is_not_an_empty_catalog(page): + with pytest.raises(CapabilityError, match="no usable directory listing"): + discover(Source({"": page})) + + +def test_link_resolution_preserves_literal_names_and_stays_under_root(): + source = Source({ + "": index( + "https://other.test/x/", "/elsewhere/", "../", "?C=N;O=D", + "#top", "/public/sub%20dir/", "/public/%2520/", "root%25.fits"), + "sub dir/": index("file%20name.fits", "../"), + "%20/": index("literal%2520.fits"), + }) + assert [entry.path for entry in discover(source)] == [ + "%20/literal%20.fits", "root%.fits", "sub dir/file name.fits"] + assert source.reads == ["", "sub dir/", "%20/"] + + +@pytest.mark.parametrize("href", ["%2e%2e/secret/", "sub%5cfile", "bad%00file"]) +def test_encoded_unsafe_paths_are_rejected(href): + with pytest.raises((DownloadError, ValueError)): + discover(Source({"": index(href)})) + + +@pytest.mark.parametrize("path, pattern, matched", [ + ("root.fits", "**/*.fits", True), + ("a/b/root.fits", "**/*.fits", True), + ("a/b/root.txt", "**/*.fits", False), + ("a/b/root.fits", "a/*.fits", False), + ("a/root.fits", "a/?.fits", False), + ("a/a.fits", "a/[ab].fits", True), + ("root.txt", ["**/*.fits", "**/*.txt"], True), +]) +def test_recursive_globs(path, pattern, matched): + assert path_matches(path, filter=pattern) is matched + + +def test_exact_format_and_filter_are_both_required(): + assert path_matches("sample_001.fits", fmt="sample_{id:03d}.fits") + assert not path_matches("sample_bad.fits", fmt="sample_{id:03d}.fits") + assert not path_matches("Sample_001.fits", fmt="sample_{id:03d}.fits") + assert not path_matches("sample_001.fits", filter="*.txt", fmt="sample_{id}.fits") + + +@pytest.mark.parametrize("invalid", [12, {"*.fits": True}, iter(["*.fits"])]) +def test_invalid_filter_types_fail_before_source_access(invalid): + source = Source({"": index("file.fits")}) + with pytest.raises(ValueError, match="glob string"): + discover(source, filter=invalid) + assert source.reads == [] + + +@pytest.mark.parametrize("final_url", [ + "https://data.test/public/release%20one/", + "https://DATA.TEST:443/public/release%20one/", +]) +def test_redirected_listing_uses_final_directory_and_deduplicates_crawl(final_url): + source = Source({ + "": index("alias/", "release%20one/"), + "alias/": index("file.fits", "nested/"), + "release one/nested/": index("other.fits"), + }) + source.transport = SimpleNamespace(text_urls={"alias/": final_url}) + entries = discover(source) + assert [entry.path for entry in entries] == [ + "release one/file.fits", "release one/nested/other.fits"] + assert source.reads == ["", "alias/", "release one/nested/"] + + +def test_filter_traverses_unmatched_directories_and_excludes_other_payloads(): + source = Source({"": index("root.fits", "nested/", "notes.txt"), + "nested/": index("other.fits", "other.bin")}) + assert [entry.path for entry in discover(source, filter="**/*.fits")] == [ + "nested/other.fits", "root.fits"] + assert source.reads == ["", "nested/"] + + +def test_only_conventional_checksum_metadata_is_read(): + digest = "a" * 64 + source = Source({ + "": index("data.fits", "sha256sums.txt", "custom_checksum_payload.bin"), + "sha256sums.txt": f"{digest} data.fits\n", + }) + entries = discover(source, filter="*.fits") + assert entries == [RemoteEntry(path="data.fits", checksum_algorithm="sha256", + checksum=digest)] + assert source.reads == ["", "sha256sums.txt"] + + +def test_checksum_manifest_cannot_add_unlisted_files(): + source = Source({"": index("sha256sums.txt"), + "sha256sums.txt": "a" * 64 + " phantom.fits\n"}) + assert [entry.path for entry in discover(source)] == ["sha256sums.txt"] + + +def test_nested_manifest_accepts_published_directory_prefixed_paths(): + source = Source({ + "": index("release/"), + "release/": index("project/"), + "release/project/": index("sha256sums", "file.fits"), + "release/project/sha256sums": "a" * 64 + " project/file.fits\n", + }) + assert discover(source, filter="**/*.fits")[0].checksum == "a" * 64 + + +def test_conflicting_manifests_fail(): + source = Source({ + "": index("file.fits", "sha256sums", "project.sha256sums"), + "sha256sums": "a" * 64 + " file.fits\n", + "project.sha256sums": "b" * 64 + " file.fits\n", + }) + with pytest.raises(DownloadError, match="Conflicting"): + discover(source) + + +def test_sftp_metadata_and_progress_callback(): + source = Source({}, url="sftp://lab/srv/data/") + + def entries(on_directory): + on_directory("") + yield RemoteEntry(path="root.fits", size=12, mtime=123) + on_directory("nested") + yield RemoteEntry(path="nested/file.bin", size=20) + + source.transport = SimpleNamespace(iter_entries=entries) + snapshots = [] + selected = discover(source, filter="**/*.fits", progress=snapshots.append) + assert selected == [RemoteEntry(path="root.fits", size=12, mtime=123)] + assert snapshots[-1] == {"directories": 2, "files": 2, + "matched": 1, "current": "nested"} + assert source.reads == [] + + +def test_cancellation_does_not_return_a_partial_inventory(): + source = Source({"": index("a/"), "a/": index("file")}) + + def cancel(): + if source.reads: + raise TransferCancelled("cancelled") + + source.check_cancelled = cancel + with pytest.raises(TransferCancelled): + discover(source) + + +def test_discovery_bar_has_unknown_total(monkeypatch): + bars = [] + + class Bar: + def __init__(self, **kwargs): + bars.append(kwargs) + + def update(self): + pass + + def set_postfix(self, **kwargs): + pass + + def close(self): + pass + + monkeypatch.setattr("hallmark.discovery.tqdm", Bar) + discover(Source({"": index("a.fits")}), progress=True) + assert bars[0]["total"] is None + assert not bars[0]["disable"] diff --git a/test/test_download_plan.py b/test/test_download_plan.py new file mode 100644 index 0000000..386651c --- /dev/null +++ b/test/test_download_plan.py @@ -0,0 +1,235 @@ +"""Transfer approval must authorize exactly the locally inspected catalog plan.""" + +from dataclasses import FrozenInstanceError, replace +from hashlib import sha256 +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from hallmark.download_plan import DownloadItem, DownloadPlan +from hallmark.downloader import ( + DownloadError, + download_remote_data, + execute_download_plan, + plan_download, + select_download_files, +) + + +@pytest.fixture +def catalog(tmp_path): + metadata = tmp_path / ".hm" + metadata.mkdir() + frame = pd.DataFrame([ + {"path": "nested/a.fits", "size_bytes": "6", "mtime": "2026-09-18", + "sha256": sha256(b"abcdef").hexdigest()}, + {"path": "b.txt", "size_bytes": "", "mtime": "", "sha256": ""}, + {"path": "empty.fits", "size_bytes": "0", "mtime": "", "sha256": ""}, + ]) + frame.to_csv(metadata / "data.tsv", sep="\t", index=False) + return SimpleNamespace( + dothm=SimpleNamespace(path=metadata), worktree=tmp_path, + state=SimpleNamespace(data=frame, config={ + "data": [{"db": "data.tsv"}], + "remote": {"name": "origin", "url": "https://source.test/data/"}, + })) + + +def test_plan_reads_path_catalog_offline_and_preserves_unknown_sizes( + catalog, monkeypatch): + def reject_network(*args, **kwargs): + raise AssertionError("planning must not contact a server") + monkeypatch.setattr("hallmark.downloader.OperationContext", reject_network) + monkeypatch.setattr("requests.sessions.Session.request", reject_network) + plan = plan_download(catalog) + assert plan.file_count == 3 + assert plan.known_bytes == 6 + assert plan.unknown_size_count == 1 + assert plan.total_bytes is None + assert plan.estimated_seconds is None + assert "1 file(s) with unknown size" in plan.summary() + assert "estimated duration: unknown" in plan.summary() + assert plan.items[0].mtime == "2026-09-18" + assert plan.items[2].size_bytes == 0 + + +def test_explicit_paths_retain_catalog_checksums_and_metadata(catalog): + plan = plan_download(catalog, file_paths=["nested/a.fits", "unlisted.bin"]) + assert plan.file_count == 2 + assert plan.items[0].checksum == ("sha256", sha256(b"abcdef").hexdigest()) + assert plan.items[0].size_bytes == 6 + assert plan.items[1].checksum is None + assert plan.items[1].size_bytes is None + assert select_download_files(catalog, file_paths=["nested/a.fits"]) == [ + (Path("nested/a.fits"), plan.items[0].checksum)] + + +def test_sizes_survive_nullable_numeric_tsv_serialization(catalog): + frame = catalog.state.data.copy() + frame["size_bytes"] = [6, None, 0] + frame.to_csv(catalog.dothm.path / "data.tsv", sep="\t", index=False) + plan = plan_download(catalog) + assert [item.size_bytes for item in plan.items] == [6, None, 0] + + +def test_plan_filters_without_authorizing_download(catalog): + plan = plan_download(catalog, filter="**/*.fits") + assert [item.relative_path.as_posix() for item in plan.items] == [ + "nested/a.fits", "empty.fits"] + assert plan.total_bytes == 6 + with pytest.raises(DownloadError, match="approval"): + execute_download_plan(catalog, plan) + + +def test_duration_requires_supplied_rate_and_complete_sizes(catalog): + known = plan_download(catalog, file_paths="nested/a.fits", + estimated_bytes_per_second=3) + assert known.estimated_seconds == 2 + unknown = plan_download(catalog, estimated_bytes_per_second=3) + assert unknown.estimated_seconds is None + with pytest.raises(ValueError, match="positive rate"): + replace(known, estimated_bytes_per_second=float("nan")) + + +def test_plan_is_immutable_and_copies_input_sequences(tmp_path): + checksum = ["sha256", "a" * 64] + item = DownloadItem(Path("a.bin"), checksum) + items = [item] + plan = DownloadPlan(items, "https://source.test/", tmp_path) + checksum[1] = "b" * 64 + items.clear() + assert plan.file_count == 1 + assert plan.items[0].checksum == ("sha256", "a" * 64) + with pytest.raises(FrozenInstanceError): + plan.remote_url = "https://different.test/" + with pytest.raises(FrozenInstanceError): + plan.items[0].size_bytes = 4 + + +def test_plan_display_redacts_url_credentials_without_changing_source(tmp_path): + url = "https://user:secret@example.test:8443/data/?token=secret#secret" + plan = DownloadPlan((DownloadItem(Path("a.bin")),), url, tmp_path) + assert plan.remote_url == url + assert "Source: https://example.test:8443/data/" in plan.summary() + assert "secret" not in plan.summary() + assert "user" not in plan.summary() + assert "secret" not in repr(plan) + + +@pytest.mark.parametrize("approval", [False, None, 1, "yes"]) +def test_unapproved_execution_never_opens_transport(catalog, monkeypatch, approval): + plan = plan_download(catalog) + def reject_open(*args, **kwargs): + raise AssertionError("unapproved transfer opened its transport") + monkeypatch.setattr("hallmark.downloader.OperationContext", reject_open) + with pytest.raises(DownloadError, match="approval"): + execute_download_plan(catalog, plan, approved=approval) + with pytest.raises(DownloadError, match="approval"): + download_remote_data(catalog, catalog.worktree, approved=approval) + + +@pytest.mark.parametrize("known_size", [True, False]) +@pytest.mark.parametrize("transfer_fails", [False, True]) +def test_approved_execution_uses_pinned_source_and_byte_progress( + catalog, monkeypatch, known_size, transfer_fails): + plan = plan_download(catalog, file_paths="nested/a.fits") + if not known_size: + plan = replace(plan, items=(replace(plan.items[0], size_bytes=None),)) + catalog.state.config["remote"]["url"] = "https://changed.test/" + (catalog.dothm.path / "data.tsv").write_text("path\nother.bin\n") + opened = [] + ticks = [] + progress_options = [] + completion_counts = [] + + class Progress: + def __init__(self, **kwargs): + progress_options.append(kwargs) + + def update(self, count): + ticks.append(count) + + def set_postfix(self, **kwargs): + completion_counts.append(kwargs) + + def close(self): + pass + + class Context: + def __init__(self, remote, output_root): + opened.append(remote) + self.output_root = output_root + self.transport = self + self.on_bytes = None + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def prepare(self): + pass + + def check_cancelled(self): + pass + + def fetch(self, relative_path, destination, **kwargs): + assert relative_path == "nested/a.fits" + with destination.open("wb") as handle: + for chunk in (b"abc", b"def"): + handle.write(chunk) + self.on_bytes(len(chunk)) + if transfer_fails: + raise DownloadError("interrupted transfer") + + monkeypatch.setattr("hallmark.downloader.OperationContext", Context) + monkeypatch.setattr("hallmark.downloader.tqdm", Progress) + result = execute_download_plan(catalog, plan, approved=True, show_progress=True) + assert opened[0].url == "https://source.test/data/" + assert ticks == [3, 3] + assert progress_options[0]["total"] == (6 if known_size else None) + assert progress_options[0]["unit"] == "B" + assert completion_counts[-1] == {"files": "1/1", "failed": int(transfer_fails)} + if transfer_fails: + assert result["failed"] == 1 + assert not (catalog.worktree / "nested/a.fits").exists() + else: + assert result["total_bytes"] == 6 + assert (catalog.worktree / "nested/a.fits").read_bytes() == b"abcdef" + + +def test_execution_rechecks_destination_after_approval(catalog, tmp_path): + output = tmp_path / "output" + output.mkdir() + plan = plan_download(catalog, output, file_paths="nested/a.fits") + outside = tmp_path / "outside" + outside.mkdir() + (output / "nested").symlink_to(outside, target_is_directory=True) + with pytest.raises(DownloadError, match="outside|symbolic link"): + execute_download_plan(catalog, plan, approved=True) + assert not (outside / "a.fits").exists() + + +def test_execution_rejects_changed_output_root(catalog, tmp_path): + output = tmp_path / "output" + output.mkdir() + plan = plan_download(catalog, output, file_paths="nested/a.fits") + output.rmdir() + output.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(DownloadError, match="destination changed"): + execute_download_plan(catalog, plan, approved=True) + + +def test_empty_plan_needs_no_approval(catalog): + plan = plan_download(catalog, filter="**/*.absent") + assert execute_download_plan(catalog, plan) == { + "succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + + +def test_bare_catalog_requires_explicit_output(catalog): + catalog.worktree = None + with pytest.raises(DownloadError, match="output_path"): + plan_download(catalog) diff --git a/test/test_downloader.py b/test/test_downloader.py index a4af524..38783a9 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -451,11 +451,12 @@ def test_download_remote_data_builds_urls_and_destinations_from_fmt( tmp_path: A pytest fixture providing a temporary directory. """ captured = {} - def fake_download_file(url, destination, sha1, chunk_size=8192): + def fake_download_file(context, relative_path, destination, sha1, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """A fake _download_file function that records its arguments""" captured.setdefault("files", []).append((url, destination, sha1)) return 123 - monkeypatch.setattr("hallmark.downloader._download_file", fake_download_file) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download_file) repo = SimpleNamespace( state=SimpleNamespace( config={ @@ -474,7 +475,7 @@ def fake_download_file(url, destination, sha1, chunk_size=8192): "pipeline": "hops", "step": "netcal", "type": "StokesI"}]))) - result = download_remote_data(repo, tmp_path) + result = download_remote_data(repo, tmp_path, approved=True) assert result == {"succeeded": 1, "failed": 0, "total_bytes": 123, "errors": []},\ f"Expected result {{'succeeded': 1, 'failed': 0, 'total_bytes': 123, \ @@ -494,12 +495,12 @@ def test_download_remote_data_returns_empty_result_without_remote(tmp_path): tmp_path: A pytest fixture providing a temporary directory.""" repo = _repo(tmp_path) - assert download_remote_data(repo, tmp_path) == { + assert download_remote_data(repo, tmp_path, approved=True) == { "succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []}, f"Expected empty result for repo without remote, but got \ - {download_remote_data(repo, tmp_path)}" + {download_remote_data(repo, tmp_path, approved=True)}" def test_download_remote_data_requires_remote_url(tmp_path): @@ -513,7 +514,7 @@ def test_download_remote_data_requires_remote_url(tmp_path): repo = _repo(tmp_path, {"remote": {"name": "origin"}}) with pytest.raises(DownloadError, match="Remote URL not configured"): - download_remote_data(repo, tmp_path, selected_files=[]) + download_remote_data(repo, tmp_path, selected_files=[], approved=True) def test_download_remote_data_returns_empty_result_for_empty_selection(tmp_path): @@ -526,7 +527,7 @@ def test_download_remote_data_returns_empty_result_for_empty_selection(tmp_path) repo = _repo( tmp_path, {"remote": {"name": "origin", "url": "https://example.test/data"}}) - result = download_remote_data(repo, tmp_path, selected_files=[]) + result = download_remote_data(repo, tmp_path, selected_files=[], approved=True) assert result["succeeded"] == result["failed"] == result["total_bytes"] == 0, \ f"Expected all counts to be 0, but got {result}" @@ -551,7 +552,8 @@ def test_download_remote_data_aggregates_successes_and_failures(monkeypatch, tmp {"name": "origin", "url": "https://origin.test/data"}, {"name": "mirror", "url": "https://mirror.test/base/"}]}) calls = [] - def fake_download(url, destination, sha1): + def fake_download(context, relative_path, destination, sha1, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """A fake _download_file function that records its arguments and simulates a failure for a specific file""" calls.append((url, destination, sha1)) @@ -570,7 +572,7 @@ def update(self, amount): def close(self): self.closed = True progress = Progress() - monkeypatch.setattr("hallmark.downloader._download_file", fake_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download) monkeypatch.setattr("hallmark.downloader.tqdm", lambda **kwargs: progress) result = download_remote_data( repo, @@ -580,7 +582,7 @@ def close(self): selected_files=[ (Path("nested/good.bin"), "good-sha"), (Path("bad.bin"), None)], - remote_name="mirror") + remote_name="mirror", approved=True) assert result == { "succeeded": 1, @@ -619,7 +621,7 @@ def test_download_remote_data_revalidates_selected_paths(tmp_path): download_remote_data( repo, tmp_path, - selected_files=[(Path("../escape.bin"), None)]) + selected_files=[(Path("../escape.bin"), None)], approved=True) def test_download_remote_data_rejects_symlink_parent_escape(tmp_path): """ @@ -652,7 +654,7 @@ def test_download_remote_data_rejects_symlink_parent_escape(tmp_path): download_remote_data( repo, output_root, - selected_files=[(Path("linked/file.bin"), None)]) + selected_files=[(Path("linked/file.bin"), None)], approved=True) assert not (outside_root / "file.bin").exists(), \ "Expected no file to be created outside the download root, but it exists" @@ -689,7 +691,7 @@ def test_download_remote_data_rejects_symlink_destination(tmp_path): download_remote_data( repo, output_root, - selected_files=[(Path("file.bin"), None)]) + selected_files=[(Path("file.bin"), None)], approved=True) assert outside_file.read_bytes() == b"do not overwrite", \ "Expected the outside file to remain unchanged, but it was modified" @@ -709,19 +711,20 @@ def test_download_remote_data_deduplicates_selected_paths(monkeypatch, tmp_path) "name": "origin", "url": "https://example.test/data"}}) calls = [] - def fake_download(url, destination, checksum): + def fake_download(context, relative_path, destination, checksum, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """A fake _download_file function that records its arguments and simulates a download.""" calls.append((url, destination, checksum)) return 4 - monkeypatch.setattr("hallmark.downloader._download_file", fake_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download) result = download_remote_data( repo, tmp_path, selected_files=[ (Path("file.bin"), None), (Path("file.bin"), "abc123"), - (Path("file.bin"), "abc123")]) + (Path("file.bin"), "abc123")], approved=True) assert result["succeeded"] == 1, \ f"Expected succeeded to be 1, but got {result['succeeded']}" @@ -748,7 +751,8 @@ def test_download_remote_data_rejects_conflicting_checksums(tmp_path): download_remote_data( repo, tmp_path, - selected_files=[(Path("file.bin"), "first"), (Path("file.bin"), "second")]) + selected_files=[(Path("file.bin"), "first"), (Path("file.bin"), "second")], + approved=True) @pytest.mark.parametrize("max_workers", [0, -1, 1.5, True, None]) @@ -766,7 +770,7 @@ def test_download_remote_data_rejects_invalid_worker_count(max_workers, tmp_path with pytest.raises( DownloadError, match="max_workers must be a positive integer"): - download_remote_data(repo, tmp_path, max_workers=max_workers) + download_remote_data(repo, tmp_path, max_workers=max_workers, approved=True) def test_download_remote_data_rejects_selected_files_without_remote(tmp_path): @@ -782,7 +786,8 @@ def test_download_remote_data_rejects_selected_files_without_remote(tmp_path): repo = _repo(tmp_path) with pytest.raises(DownloadError, match="No remote is configured"): - download_remote_data(repo, tmp_path, selected_files=[(Path("data.bin"), None)]) + download_remote_data(repo, tmp_path, + selected_files=[(Path("data.bin"), None)], approved=True) def test_download_remote_data_reuses_session_per_worker(monkeypatch, tmp_path): @@ -815,7 +820,8 @@ def get(self, url, **kwargs): repo, tmp_path, selected_files=[ - (Path("first.dat"), None), (Path("second.dat"), None)], max_workers=1) + (Path("first.dat"), None), (Path("second.dat"), None)], + max_workers=1, approved=True) assert result["succeeded"] == 2, \ f"Expected 2 successful downloads, but got {result['succeeded']}" @@ -844,7 +850,7 @@ def test_download_remote_data_rejects_malformed_selection(tmp_path): with pytest.raises(DownloadError, match="path, checksum"): download_remote_data( - repo, tmp_path, selected_files=[("file.dat", None, "extra")]) + repo, tmp_path, selected_files=[("file.dat", None, "extra")], approved=True) def test_download_remote_data_rejects_malformed_checksum_tuple(tmp_path): @@ -864,7 +870,8 @@ def test_download_remote_data_rejects_malformed_checksum_tuple(tmp_path): download_remote_data( repo, tmp_path, - selected_files=[(Path("file.dat"), ("sha1", "abc123", "extra"))]) + selected_files=[(Path("file.dat"), ("sha1", "abc123", "extra"))], + approved=True) def test_download_remote_data_rejects_output_path_that_is_file(tmp_path): @@ -882,7 +889,7 @@ def test_download_remote_data_rejects_output_path_that_is_file(tmp_path): download_remote_data( repo, output_path, - selected_files=[(Path("file.bin"), None)]) + selected_files=[(Path("file.bin"), None)], approved=True) def test_download_remote_data_deduplicates_equivalent_checksum_tuples( @@ -900,11 +907,12 @@ def test_download_remote_data_deduplicates_equivalent_checksum_tuples( "name": "origin", "url": "https://example.test/data"}}) calls = [] - def fake_download(url, destination, checksum): + def fake_download(context, relative_path, destination, checksum, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """Record download requests and return a fixed byte count.""" calls.append((url, destination, checksum)) return 11 - monkeypatch.setattr("hallmark.downloader._download_file", fake_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download) checksum_upper = ("SHA256", "A" * 64) checksum_lower = ("sha256", "a" * 64) result = download_remote_data( @@ -912,7 +920,7 @@ def fake_download(url, destination, checksum): tmp_path, selected_files=[ (Path("file.bin"), checksum_upper), - (Path("file.bin"), checksum_lower)]) + (Path("file.bin"), checksum_lower)], approved=True) assert result["succeeded"] == 1, \ f"Expected succeeded to be 1, but got {result['succeeded']}" @@ -1489,4 +1497,4 @@ def test_entry_checksum_prefers_strongest_named_checksum(): entry = {"md5": "a" * 32, "sha1": "b" * 40, "sha256": "c" * 64, "sha512": "d" * 128} assert _entry_checksum(entry) == ("sha512", "d" * 128), \ - f"Expected strongest checksum to be sha512, but got {_entry_checksum(entry)}" \ No newline at end of file + f"Expected strongest checksum to be sha512, but got {_entry_checksum(entry)}" diff --git a/test/test_hallmark.py b/test/test_hallmark.py index 524d2aa..1e615ff 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -1277,7 +1277,7 @@ def record_restore(sha1, destination): ### Repo.clone() tests ### -def test_repo_clone_downloads_remote_data_by_default(monkeypatch, tmp_path): +def test_repo_clone_downloads_after_plan_approval(monkeypatch, tmp_path): source = Repo.init(tmp_path / "source") _write_files(source.worktree, ["a0_i0.h5"]) source.add("a{a}_i{i}.h5") @@ -1287,16 +1287,17 @@ def test_repo_clone_downloads_remote_data_by_default(monkeypatch, tmp_path): captured = {} - def fake_download_file(url, destination, sha1, chunk_size=8192): - captured["url"] = url + def fake_download_file(context, relative_path, destination, sha1, chunk_size): + captured["url"] = context.remote.file_url(str(relative_path)) captured["sha1"] = sha1 destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text("downloaded\n", encoding="utf-8") return destination.stat().st_size - monkeypatch.setattr("hallmark.downloader._download_file", fake_download_file) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download_file) - clone = Repo.clone(str(source.dothm.path), tmp_path / "clone") + clone = Repo.clone(str(source.dothm.path), tmp_path / "clone", + download=True, approve=lambda plan: plan.file_count == 1) assert captured == { "url": "https://example.com/data/a0_i0.h5", @@ -1318,9 +1319,9 @@ def test_repo_clone_can_skip_remote_data_download(monkeypatch, tmp_path): def fail_download(*args, **kwargs): raise AssertionError("download should not be attempted") - monkeypatch.setattr("hallmark.downloader._download_file", fail_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fail_download) - clone = Repo.clone(str(source.dothm.path), tmp_path / "clone", fetch_data=False) + clone = Repo.clone(str(source.dothm.path), tmp_path / "clone") assert not (clone.worktree / "a0_i0.h5").exists() assert clone.download_result is None @@ -2846,4 +2847,4 @@ def test_checkout_remote_branch(tmp_path): # The worktree should now contain the version from the remote branch. assert (clone.worktree / "data.txt").read_text( encoding="utf-8" - ) == "experiment contents\n" \ No newline at end of file + ) == "experiment contents\n" diff --git a/test/test_repo_builder.py b/test/test_repo_builder.py index ae392ac..45b9b36 100755 --- a/test/test_repo_builder.py +++ b/test/test_repo_builder.py @@ -10,6 +10,7 @@ import requests from mock_server import MockServer +from hallmark.transport.base import CapabilityError from hallmark.repo_builder import ( KNOWN_FIELD_VALUES, _match_file_against_fmts, @@ -182,24 +183,15 @@ def test_list_remote_files_partial_sibling_manifest_does_not_hide_files(): expected (None, None) because the file was not listed in the manifest" -def test_list_remote_files_generic_manifest_detected_by_content(): - """ - A manifest file whose name is generic (like "md5sum.txt") should - still be detected as a manifest if its content looks like one. - """ +def test_list_remote_files_does_not_probe_custom_checksum_payloads(): server = MockServer(BASE_URL) server.add_directory("", [ ("data-object", "data.tar"), - ("data-object", "custom_checksum_manifest.log"),]) - server.add_file("custom_checksum_manifest.log", - "aa11bb22cc33dd44ee55ff66aa11bb22 data.tar\n") + ("data-object", "custom_checksum_manifest.log")]) + # No payload is registered: fetching either body would fail the test. with _served(server): - file_checksums = list_remote_files(server.base_url) - - assert file_checksums["data.tar"] == \ - ("unknown", "aa11bb22cc33dd44ee55ff66aa11bb22"), \ - f"unexpected checksum for data.tar: {file_checksums}, \ - expected unknown aa11bb22cc33dd44ee55ff66aa11bb22" + files = list_remote_files(server.base_url) + assert files["data.tar"] == (None, None) def test_list_remote_files_generic_manifest_algorithm_from_own_filename(): @@ -842,114 +834,37 @@ def refuse_get_body(url, timeout=None): e for e in repo.state.config["data"] if e.get("file") == "huge_unmatched.dat") - assert entry == {"file": "huge_unmatched.dat", "checksum": "unknown"}, \ + assert entry == {"file": "huge_unmatched.dat"}, \ f"unexpected entry for huge_unmatched.dat: {entry}, expected checksum 'unknown'" -def test_build_repo_small_unmatched_file_still_downloaded_and_hashed(tmp_path, - simple_dataset_server): - """ - Test that build_repo still downloads and hashes a small unmatched file, - using a simple mock dataset server to avoid hitting the real network. - Args: - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - """ - with _served(simple_dataset_server): - repo = build_repo( - repo_path=tmp_path / "repo4.hm", - dataset_name="EHTC_TEST", - fmt_entries=[],) - entry = next(e for e in repo.state.config["data"] if e.get("file") == "README.md") - - has_legacy_md5 = ("md5" in entry and entry["md5"] != "unknown") - has_normalized_md5 = ( - entry.get("checksum_algorithm") == "md5" - and entry.get("checksum") not in (None, "unknown")) - - assert has_legacy_md5 or has_normalized_md5, f"unexpected entry for README.md: \ - {entry}, expected either legacy md5 or normalized checksum schema" - - -def test_build_repo_interactive_detect_branch(monkeypatch, tmp_path, - simple_dataset_server): - """ - fmt_entries=None should enter interactive detect flow and accept strict yes/no. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - """ - answers = iter(["detect", "no", "data.tsv"]) - monkeypatch.setattr("builtins.input", lambda _: next(answers)) - called = {"include_drives": None} - def _fake_detect_fmt(rel_paths, include_drives=False): - called["include_drives"] = include_drives - return ["proj/data.{format}"] - monkeypatch.setattr("hallmark.repo_builder.detect_fmt", _fake_detect_fmt) - with _served(simple_dataset_server): - repo = build_repo( - repo_path=tmp_path / "interactive_detect.hm", - dataset_name="EHTC_TEST", - fmt_entries=None,) +def test_build_repo_small_unmatched_file_is_not_downloaded(tmp_path): + server = MockServer(BASE_URL) + server.add_directory("", [("data-object", "README.md")]) + # Even a tiny file stays remote until download approval. + with _served(server): + repo = build_repo(tmp_path / "repo.hm", "EHTC_TEST", fmt_entries=[]) + assert repo.state.config["data"] == [{"name": "readme", "file": "README.md"}] - assert called["include_drives"] is False, \ - f"expected include_drives=False, got {called['include_drives']}" - data_tsv = repo.dothm.path / "data.tsv" - assert data_tsv.exists(), f"expected {data_tsv} to be created" - df = pd.read_csv(data_tsv, sep="\t", dtype=str) - assert "proj/data.tar" in set(df["path"]), \ - f"expected proj/data.tar in data.tsv, got {df.to_dict(orient='records')}" +def test_build_repo_defaults_to_generic_path_catalog_without_input( + monkeypatch, tmp_path): + _inventory(monkeypatch, { + "arbitrary.bin": (None, None), "nested/a.fits": (None, None)}) + monkeypatch.setattr("builtins.input", lambda *_: pytest.fail("Unexpected prompt")) + repo = build_repo(tmp_path / "catalog", "label") + assert repo.state.config["data"] == [{"fmt": "{path}", "db": "data.tsv"}] + frame = pd.read_csv(repo.dothm.path / "data.tsv", sep="\t") + assert set(frame["path"]) == {"arbitrary.bin", "nested/a.fits"} -def test_build_repo_interactive_input_branch(monkeypatch, - tmp_path, simple_dataset_server): - """ - fmt_entries=None should allow manual interactive input flow. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - """ - answers = iter(["input", "proj/data.{format}", "data.tsv", ""]) - monkeypatch.setattr("builtins.input", lambda _: next(answers)) - with _served(simple_dataset_server): - repo = build_repo( - repo_path=tmp_path / "interactive_input.hm", - dataset_name="EHTC_TEST", - fmt_entries=None,) - data_tsv = repo.dothm.path / "data.tsv" - assert data_tsv.exists(), f"expected {data_tsv} to be created" - df = pd.read_csv(data_tsv, sep="\t", dtype=str) - assert "proj/data.tar" in set(df["path"]), \ - f"expected proj/data.tar in data.tsv, got {df.to_dict(orient='records')}" -def test_build_repo_interactive_detect_invalid_yes_no_raises( - monkeypatch, tmp_path, simple_dataset_server -): - """ - Detect flow should raise on any include_drives answer besides strict yes/no. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - Raises: - ValueError: If the include_drives answer is not "yes" or "no". - """ - answers = iter(["detect", "y"]) - monkeypatch.setattr("builtins.input", lambda _: next(answers)) - with _served(simple_dataset_server): - with pytest.raises(ValueError): - build_repo( - repo_path=tmp_path / "interactive_invalid.hm", - dataset_name="EHTC_TEST", - fmt_entries=None,) +def test_build_repo_rejects_payload_hashing_before_network(monkeypatch, tmp_path): + monkeypatch.setattr("hallmark.repo_builder.list_remote_files", + lambda *_: pytest.fail("Unexpected network access")) + with pytest.raises(CapabilityError, match="remote_hash"): + build_repo(tmp_path / "catalog", "label", remote_hash=True) def test_build_repo_multiple_remotes(tmp_path, simple_dataset_server): @@ -1159,7 +1074,7 @@ def test_build_repo_empty_dataset_writes_valid_empty_data_config(monkeypatch, tm expected 'data: []' at the start" -def test_build_repo_loads_formats_and_remotes_interactively_from_config( +def test_build_repo_loads_formats_and_remotes_from_explicit_config( monkeypatch, tmp_path): """ If the user selects a config file that has formats and remotes, build_repo should @@ -1181,7 +1096,8 @@ def test_build_repo_loads_formats_and_remotes_interactively_from_config( answers = iter(["config", str(source_config)]) monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - repo = build_repo(tmp_path / "from-config.hm", "EHTC_TEST") + repo = build_repo( + tmp_path / "from-config.hm", "EHTC_TEST", config_file=source_config) assert {entry.get("fmt") for entry in repo.state.config["data"]} == { "item_{number}.dat"}, f"unexpected fmt entries: {repo.state.config['data']}, \ @@ -1191,7 +1107,7 @@ def test_build_repo_loads_formats_and_remotes_interactively_from_config( {repo.state.config['remote']}, expected to load from config" -def test_build_repo_explicit_remotes_override_interactive_config(monkeypatch, tmp_path): +def test_build_repo_explicit_remotes_override_loaded_config(monkeypatch, tmp_path): """ If the user selects a config file that has remotes, but also provides explicit remotes to build_repo, the explicit remotes should override the config remotes. @@ -1212,14 +1128,15 @@ def test_build_repo_explicit_remotes_override_interactive_config(monkeypatch, tm repo = build_repo( tmp_path / "override.hm", "EHTC_TEST", - remotes=[{"name": "new", "url": "https://new.test"}]) + remotes=[{"name": "new", "url": "https://new.test"}], + config_file=source_config) assert repo.state.config["remote"] == [{"name": "new", "url": "https://new.test"}],\ f"unexpected remotes: {repo.state.config['remote']}, \ expected to override with new remote" -def test_build_repo_interactive_config_requires_a_format(monkeypatch, tmp_path): +def test_build_repo_explicit_config_requires_a_format(monkeypatch, tmp_path): """ If the user selects a config file that has no fmt entries, build_repo should raise a ValueError indicating that no formats were found. @@ -1235,7 +1152,7 @@ def test_build_repo_interactive_config_requires_a_format(monkeypatch, tmp_path): monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) with pytest.raises(ValueError, match="No fmt entries found"): - build_repo(tmp_path / "missing-fmt.hm", "EHTC_TEST") + build_repo(tmp_path / "missing-fmt.hm", "EHTC_TEST", config_file=source_config) def test_build_repo_reuses_existing_formats_and_remotes(monkeypatch, tmp_path): @@ -1269,88 +1186,18 @@ def test_build_repo_reuses_existing_formats_and_remotes(monkeypatch, tmp_path): {calls}, expected two calls to {BASE_URL}" -def test_build_repo_rejects_invalid_existing_repo_choice_before_network( - monkeypatch, tmp_path): - """ - If the user chooses an invalid option when prompted to reuse an existing repo, - build_repo should raise a ValueError before attempting any network operations. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - AssertionError: If build_repo attempts to perform network operations after an - invalid choice is made. - ValueError: If the user provides an unrecognized choice when prompted to reuse - an existing repo. - """ - repo_path = tmp_path / "existing.hm" - _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - build_repo( - repo_path, - "EHTC_TEST", - fmt_entries=[{"fmt": "item_{number}.dat", "db": "items.tsv"}]) - monkeypatch.setattr("builtins.input", lambda prompt: "invalid") - def network_must_not_run(base_url): - """check the invalid input is caught before attempting network operations""" - raise AssertionError("remote listing should remain lazy") - monkeypatch.setattr( - "hallmark.repo_builder.list_remote_files", network_must_not_run) +def test_build_repo_reuses_existing_catalog_without_input(monkeypatch, tmp_path): + _inventory(monkeypatch, {"item_1.dat": (None, None)}) + path = tmp_path / "catalog" + build_repo(path, "label", [{"fmt": "item_{number}.dat", "db": "data.tsv"}]) + monkeypatch.setattr("builtins.input", lambda *_: pytest.fail("Unexpected prompt")) + repo = build_repo(path, "label") + assert repo.state.config["data"] == [{"fmt": "item_{number}.dat", "db": "data.tsv"}] - with pytest.raises(ValueError, match="Unrecognized choice"): - build_repo(repo_path, "EHTC_TEST") -def test_build_repo_detects_single_format_and_uses_data_tsv(monkeypatch, tmp_path): - """ - If detect_fmt returns a single format, build_repo should use it and default to - "data.tsv" as the db name. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - """ - _inventory(monkeypatch, - {"item_1.dat": ("md5", "a" * 32), "item_2.dat": ("md5", "b" * 32)}) - answers = iter(["detect", "yes"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - detected = {} - def fake_detect(paths, include_drives=False): - """return a single format for testing purposes, and record the arguments""" - detected["args"] = (paths, include_drives) - return ["item_{number}.dat"] - monkeypatch.setattr("hallmark.repo_builder.detect_fmt", fake_detect) - repo = build_repo(tmp_path / "detected.hm", "EHTC_TEST") - - assert detected["args"] == (["item_1.dat", "item_2.dat"], True), \ - f"unexpected arguments to detect_fmt: {detected['args']}, expected \ - (['item_1.dat', 'item_2.dat'], True)" - assert (repo.dothm.path / "data.tsv").is_file(), f"expected data.tsv to be created,\ - but it does not exist at {repo.dothm.path / 'data.tsv'}" - assert [entry for entry in repo.state.config["data"] if "fmt" in entry] == [ - {"fmt": "item_{number}.dat", "db": "data.tsv"}], f"unexpected data config: \ - {repo.state.config['data']}, expected a single fmt entry with db 'data.tsv'" -def test_build_repo_records_detect_include_drives_in_meta(monkeypatch, tmp_path): - """ - When fmts are auto-detected, build_repo should record the include_drives choice - in meta.yml so it can be recovered later without re-running detection interactively. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - """ - _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - answers = iter(["detect", "yes"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - monkeypatch.setattr( - "hallmark.repo_builder.detect_fmt", - lambda paths, include_drives=False: ["item_{number}.dat"]) - repo = build_repo(tmp_path / "detected.hm", "EHTC_TEST") - - assert repo.state.meta.get("detect_include_drives") is True, f"expected meta.yml to\ - record detect_include_drives=True, got: {repo.state.meta}" - assert repo.dothm.load_yml("meta").get("detect_include_drives") is True, \ - "expected meta.yml on disk to record detect_include_drives=True" - def test_build_repo_omits_detect_include_drives_when_fmt_entries_given( monkeypatch, tmp_path): @@ -1370,63 +1217,10 @@ def test_build_repo_omits_detect_include_drives_when_fmt_entries_given( f"did not expect detect_include_drives in meta.yml, got: {repo.state.meta}" -def test_build_repo_detects_multiple_formats_and_normalizes_db_names( - monkeypatch, tmp_path): - """ - If detect_fmt returns multiple formats, build_repo should prompt for a db name - for each format and normalize the db names to be unique. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - """ - _inventory(monkeypatch, - {"a_1.dat": ("md5", "a" * 32), "b_2.dat": ("md5", "b" * 32)}) - answers = iter(["detect", "no", "a", "b.tsv"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - monkeypatch.setattr( - "hallmark.repo_builder.detect_fmt", - lambda paths, include_drives=False: ["a_{n}.dat", "b_{n}.dat"]) - repo = build_repo(tmp_path / "multi-detect.hm", "EHTC_TEST") - assert [entry for entry in repo.state.config["data"] if "fmt" in entry] == [ - {"fmt": "a_{n}.dat", "db": "a.tsv"}, - {"fmt": "b_{n}.dat", "db": "b.tsv"}], f"unexpected data config: \ - {repo.state.config['data']}, expected two fmt entries with normalized db names" - - -def test_build_repo_detect_rejects_no_detected_formats(monkeypatch, tmp_path): - """ - If detect_fmt returns no formats, build_repo should raise a ValueError. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - ValueError: If no formats could be automatically detected. - """ - _inventory(monkeypatch, {"item.dat": ("md5", "a" * 32)}) - answers = iter(["detect", "no"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - monkeypatch.setattr( - "hallmark.repo_builder.detect_fmt", - lambda paths, include_drives=False: [],) - with pytest.raises(ValueError, match="No fmts could be automatically detected"): - build_repo(tmp_path / "nothing-detected.hm", "EHTC_TEST") -def test_build_repo_rejects_unknown_format_input_mode(monkeypatch, tmp_path): - monkeypatch.setattr("builtins.input", lambda prompt: "unknown") - """ - Test that build_repo raises a ValueError when an unrecognized choice is provided - in interactive mode. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - ValueError: If the provided choice is not recognized. - """ - with pytest.raises(ValueError, match="Unrecognized choice"): - build_repo(tmp_path / "unknown.hm", "EHTC_TEST") def test_build_repo_preserves_existing_destination_by_default( @@ -1568,7 +1362,7 @@ def test_build_repo_accepts_config_repository_directory(monkeypatch, tmp_path): answers = iter(["config", str(source_repo)]) monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - repo = build_repo(tmp_path / "result.hm", "EHTC_TEST") + repo = build_repo(tmp_path / "result.hm", "EHTC_TEST", config_file=source_repo) actual = [ entry for entry in repo.state.config["data"] if "fmt" in entry] expected = [{"fmt": "item_{number}.dat", "db": "items.tsv"}] @@ -1593,44 +1387,12 @@ def test_build_repo_preserves_meta_file_checksum(monkeypatch, tmp_path): expected meta.yml with checksum {checksum}" -def test_build_repo_continues_when_static_checksum_request_fails(monkeypatch, tmp_path): - """ - Test that build_repo continues to create the repo even if a request to retrieve - the checksum for a static file fails, and that it records the checksum as "unknown". - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - HTTPError: If the request to retrieve the checksum for a static file fails. - """ +def test_build_repo_never_requests_static_checksum(monkeypatch, tmp_path): _inventory(monkeypatch, {"README.md": (None, None)}) - class FailingChecksumSession: - def close(self): - pass - - """ - A mock requests.Session that simulates a failure to retrieve the checksum for - a static file, by raising an HTTPError when the head() method is called. - """ - def __enter__(self): - """enter the context manager""" - return self - - def __exit__(self, exc_type, exc_value, traceback): - """exit the context manager""" - return False - - def head(self, *args, **kwargs): - """Simulate a failure to retrieve the checksum for a static file.""" - raise requests.HTTPError("HEAD unavailable") monkeypatch.setattr("hallmark.transport.requests.Session", - FailingChecksumSession) + lambda: pytest.fail("Unexpected checksum request")) repo = build_repo(tmp_path / "repo.hm", "EHTC_TEST", fmt_entries=[]) - - assert repo.state.config["data"] == [ - {"name": "readme", "file": "README.md", "checksum": "unknown"}], \ - f"unexpected data config: {repo.state.config['data']}, \ - expected README.md with checksum 'unknown'" + assert repo.state.config["data"] == [{"name": "readme", "file": "README.md"}] def test_build_repo_preserves_multiple_meta_files(monkeypatch, tmp_path): @@ -1951,4 +1713,4 @@ def test_build_repo_explicit_remotes_override_config_file_remotes( assert repo.state.config["remote"] == [{"name": "new", "url": "https://new.test"}],\ f"unexpected remotes: {repo.state.config['remote']}, \ - expected explicit remotes to override config_file remotes" \ No newline at end of file + expected explicit remotes to override config_file remotes" diff --git a/test/test_sftp_metadata.py b/test/test_sftp_metadata.py new file mode 100644 index 0000000..0740568 --- /dev/null +++ b/test/test_sftp_metadata.py @@ -0,0 +1,229 @@ +"""Structured SFTP discovery, using a local subsystem without any network.""" + +from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext +from pathlib import Path +import stat +import struct +import sys +import time + +import pytest + +from hallmark.transport import OperationContext, RemoteSpec +from hallmark.transport.base import ( + CapabilityError, DownloadError, RemoteConfigurationError, + RemoteObjectMissing, TransferCancelled, +) +from hallmark.transport.sftp import _Packet + + +@pytest.fixture +def local_sftp(tmp_path, monkeypatch): + server = next((path for path in ( + "/usr/lib/openssh/sftp-server", "/usr/libexec/sftp-server", + ) if Path(path).exists()), None) + if server is None: + pytest.skip("Local OpenSSH SFTP subsystem binary is unavailable") + root = tmp_path / "source" + root.mkdir() + with OperationContext(RemoteSpec.parse(f"sftp://unused{root}/")) as context: + transport = context.transport + monkeypatch.setattr(transport, "prepare", lambda: None) + spawn = transport._spawn + + def spawn_subsystem(argv, **kwargs): + assert argv[-5:] == ["-T", "-s", "--", "unused", "sftp"] + assert "BatchMode=yes" in argv + assert "StrictHostKeyChecking=yes" in argv + return spawn([server], **kwargs) + + monkeypatch.setattr(transport, "_spawn", spawn_subsystem) + yield context, root + + +def test_structured_paths_sizes_progress_and_missing_metadata(local_sftp): + context, root = local_sftp + names = ["plain", "a b#?%+ü'\"[]*.fits", "-leading", " leading ", "literal%20"] + for name in names: + (root / name).write_text(name) + (root / "nested").mkdir() + (root / "nested" / "child").write_bytes(b"child") + (root / "link").symlink_to(root / "nested", target_is_directory=True) + directories = [] + iterator = context.transport.iter_entries(on_directory=directories.append) + first = next(iterator) + assert directories == [""] + entries = [first, *iterator] + expected = {name: len(name.encode()) for name in names} + expected["nested/child"] = 5 + assert {entry.path: entry.size for entry in entries} == expected + assert directories == ["", "nested/"] + assert all(entry.mtime is not None for entry in entries) + assert len(context.transport._processes) == 0 + assert context.transport.stat("plain").size == 5 + with pytest.raises(RemoteObjectMissing): + context.transport.stat("missing") + assert not context.transport._processes + + +def test_multiple_readdir_packets_and_cancellation_cleanup(local_sftp): + context, root = local_sftp + for index in range(1500): + (root / f"file-{index:04d}").touch() + assert len(list(context.transport.iter_entries())) == 1500 + iterator = context.transport.iter_entries() + next(iterator) + context.cancel() + with pytest.raises(TransferCancelled): + next(iterator) + assert not context.transport._processes + + +def test_reject_controls_without_silently_omitting_file(local_sftp): + context, root = local_sftp + (root / "bad\nname").touch() + with pytest.raises(RemoteConfigurationError, match="control"): + list(context.transport.iter_entries()) + assert not context.transport._processes + + +def test_discovery_excludes_root_repository_metadata_and_objects(local_sftp): + context, root = local_sftp + for metadata in (".hm", ".git", ".HM", ".GIT"): + (root / metadata / "objects").mkdir(parents=True) + (root / metadata / "objects" / "payload").write_bytes(b"private") + (root / "nested" / ".hm" / "objects").mkdir(parents=True) + (root / "nested" / ".hm" / "objects" / "payload").write_bytes(b"private") + (root / "science.fits").write_bytes(b"science") + directories = [] + entries = list(context.transport.iter_entries(on_directory=directories.append)) + assert [entry.path for entry in entries] == ["science.fits"] + assert directories == ["", "nested/"] + + +def test_metadata_file_size_limit_before_transfer(local_sftp, monkeypatch): + context, root = local_sftp + (root / "metadata").write_bytes(b"oversized") + + def unexpected(*args, **kwargs): + raise AssertionError("Known oversized metadata must not be fetched") + + monkeypatch.setattr(context.transport, "_fetch", unexpected) + with pytest.raises(DownloadError, match="size limit"): + context.transport.read_text("metadata", 2) + + +def _fake_subsystem(monkeypatch, context, script): + transport = context.transport + monkeypatch.setattr(transport, "prepare", lambda: None) + spawn = transport._spawn + monkeypatch.setattr( + transport, "_spawn", + lambda argv, **kw: spawn([sys.executable, "-c", script], **kw), + ) + + +@pytest.mark.parametrize("payload, message", [ + (struct.pack(">I", 2 ** 30), "oversized"), + (struct.pack(">I", 5) + b"\x02\x00", "truncated"), + (struct.pack(">IBI", 5, 2, 4), "version 3"), +]) +def test_bad_handshake_is_bounded_and_cleans_process(monkeypatch, payload, message): + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + _fake_subsystem( + monkeypatch, context, + "import os; os.read(0, 9); os.write(1, " + repr(payload) + ")", + ) + with pytest.raises((DownloadError, CapabilityError), match=message): + with context.transport._metadata(): + pytest.fail("Invalid handshake was accepted") + assert not context.transport._processes + + +def test_metadata_timeout_is_per_request_and_cancellable(monkeypatch): + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + _fake_subsystem(monkeypatch, context, "import time; time.sleep(30)") + context.listing_timeout = 0.1 + with pytest.raises(DownloadError, match="time limit"): + with context.transport._metadata(): + pass + assert not context.transport._processes + context.listing_timeout = 30 + with ThreadPoolExecutor(1) as pool: + future = pool.submit(context.transport._metadata().__enter__) + deadline = time.monotonic() + 2 + while not context.transport._processes and time.monotonic() < deadline: + time.sleep(0.01) + context.cancel() + with pytest.raises(TransferCancelled): + future.result(timeout=2) + assert not context.transport._processes + + +def test_attributes_preserve_unknowns_and_reject_truncation(): + assert _Packet(struct.pack(">I", 0)).attributes() == { + "size": None, "mode": None, "mtime": None, + } + data = struct.pack(">IQIII", 13, 2 ** 40, stat.S_IFREG | 0o600, 123, 456) + assert _Packet(data).attributes() == { + "size": 2 ** 40, "mode": stat.S_IFREG | 0o600, "mtime": 456, + } + with pytest.raises(DownloadError, match="Truncated"): + _Packet(data[:-1]).attributes() + with pytest.raises(DownloadError, match="flags"): + _Packet(struct.pack(">I", 0x10)).attributes() + + +def test_missing_type_uses_lstat_and_cannot_escape_root(monkeypatch): + class Session: + escape = False + + def realpath(self, path): + return "/outside" if self.escape and path.endswith("/nested") else path + + def lstat(self, path): + mode = stat.S_IFREG if path.endswith("/file") else stat.S_IFDIR + return {"size": 4, "mode": mode, "mtime": 123} + + def iterdir(self, path): + if path == "/data": + yield "nested", {"size": None, "mode": None, "mtime": None} + else: + yield "file", {"size": None, "mode": None, "mtime": None} + + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + session = Session() + monkeypatch.setattr( + context.transport, "_metadata", lambda: nullcontext(session), + ) + entries = list(context.transport.iter_entries()) + assert [(entry.path, entry.size) for entry in entries] == [("nested/file", 4)] + session.escape = True + with pytest.raises(DownloadError, match="escapes"): + list(context.transport.iter_entries()) + + +@pytest.mark.parametrize("kind, response_id, message", [ + (105, 999, "identifier"), + (104, 1, "Unexpected"), +]) +def test_wrong_response_cannot_be_treated_as_file_attributes( + monkeypatch, kind, response_id, message, +): + response = struct.pack(">BII", kind, response_id, 0) + script = """import struct, sys +source, output = sys.stdin.buffer, sys.stdout.buffer +source.read(9) +output.write(struct.pack('>IBI', 5, 2, 3)); output.flush() +length = struct.unpack('>I', source.read(4))[0] +source.read(length) +payload = """ + repr(response) + """ +output.write(struct.pack('>I', len(payload)) + payload); output.flush() +""" + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + _fake_subsystem(monkeypatch, context, script) + with pytest.raises(DownloadError, match=message): + with context.transport._metadata() as session: + session.lstat("/data") + assert not context.transport._processes diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py index 2c4673b..80813c2 100644 --- a/test/test_ssh_integration.py +++ b/test/test_ssh_integration.py @@ -21,7 +21,9 @@ from hallmark.downloader import download_remote_data, select_download_files from hallmark.repo_builder import build_repo from hallmark.transport import OperationContext, RemoteSpec -from hallmark.transport.base import DownloadError, TransferCancelled +from hallmark.transport.base import ( + DownloadError, RemoteObjectMissing, TransferCancelled, +) from hallmark.transport.ssh import SshTransport pytestmark = pytest.mark.ssh_integration @@ -173,7 +175,7 @@ def test_explicit_endpoint_and_missing_file(ssh_server, tmp_path): output.write_bytes(b"keep") result = download_remote_data( repo, repo.worktree, selected_files=[(Path("missing"), None)] - ) + , approved=True) assert result["failed"] == 1 assert output.read_bytes() == b"keep" assert not list(Path(repo.worktree).glob("*.part")) @@ -251,17 +253,18 @@ def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): {"fmt": "bad_{i}.dat", "db": "bad.tsv"}, ], dataset_url=ssh_server["url"], - allow_remote_commands=True, - remote_hash=True, ) assert repo.state.config["remote"] == [{"name": "origin", "url": ssh_server["url"]}] static = next( entry for entry in repo.state.config["data"] if entry.get("file") == "README.md" ) - assert static["sha256"] == hashlib.sha256(b"notes").hexdigest() + assert not any(key in static for key in ("md5", "sha1", "sha256")) + assert static.get("checksum") in (None, "unknown") destination = tmp_path / "downloads" files = select_download_files(repo, all_files=True) - result = download_remote_data(repo, destination, selected_files=files) + result = download_remote_data( + repo, destination, selected_files=files, approved=True, + ) assert result["succeeded"] == 3 assert result["failed"] == 1 assert (destination / "nested/item_1.dat").read_bytes() == b"science" @@ -274,7 +277,6 @@ def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): "lab", [{"fmt": "nested/item_{i}.dat", "db": "data.tsv"}], dataset_url=ssh_server["url"], - allow_remote_commands=True, ) # Remove the intentionally incorrect static entry before cloning. single.state.config["data"] = [ @@ -285,11 +287,15 @@ def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): single.dothm.dump(single.state) single.dothm.index.add(["config.yml"]) single.dothm.index.commit("Remove deliberately invalid test entry") - clone = Repo.clone(str(single.dothm.path), tmp_path / "python-clone") + clone = Repo.clone( + str(single.dothm.path), tmp_path / "python-clone", + download=True, approve=lambda plan: True, + ) assert (clone.worktree / "nested/item_1.dat").read_bytes() == b"science" result = CliRunner().invoke( hallmark, - ["clone", str(single.dothm.path), str(tmp_path / "cli-clone"), "--yes"], + ["clone", str(single.dothm.path), str(tmp_path / "cli-clone"), "--download"], + input="y\n", ) assert result.exit_code == 0, result.output assert (tmp_path / "cli-clone/nested/item_1.dat").read_bytes() == b"science" @@ -329,10 +335,17 @@ def test_real_cancel_transfer_and_preserve_other_master(ssh_server, tmp_path): # finish inside the client's initial buffering window on macOS. context.transport.sftp = ["sftp", "-B", "1024", "-R", "1", "-l", "8"] target = tmp_path / "partial" + progress = [] + context.on_bytes = progress.append with ThreadPoolExecutor(1) as pool: task = pool.submit(context.transport.fetch, "large", target) try: _wait_for_partial_file(tmp_path, "partial", len(payload), task) + deadline = time.monotonic() + 2 + while not progress and time.monotonic() < deadline: + time.sleep(0.01) + assert 0 < sum(progress) < len(payload) + assert all(delta > 0 for delta in progress) context.cancel() with pytest.raises(TransferCancelled): task.result(timeout=5) @@ -345,15 +358,71 @@ def test_real_cancel_transfer_and_preserve_other_master(ssh_server, tmp_path): @pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) -def test_sftp_only_fetches_but_cannot_build(ssh_server, tmp_path): - (ssh_server["root"] / "item").write_bytes(b"data") - with OperationContext( - RemoteSpec.parse(ssh_server["url"]), allow_remote_commands=True - ) as context: +def test_sftp_only_discovers_metadata_and_fetches(ssh_server, tmp_path): + root = ssh_server["root"] + (root / "item").write_bytes(b"data") + (root / "nested").mkdir() + name = "nested/a b#?%+ü'\"[*].fits" + (root / name).write_bytes(b"science") + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + directories = [] + entries = list(context.transport.iter_entries(on_directory=directories.append)) + assert {entry.path: entry.size for entry in entries} == {"item": 4, name: 7} + assert directories == ["", "nested/"] + assert all(entry.mtime is not None for entry in entries) + assert context.read_text("item") == "data" + with pytest.raises(RemoteObjectMissing): + context.read_text("absent-config.yml") + transferred = [] + context.on_bytes = transferred.append context.transport.fetch("item", tmp_path / "copy") assert (tmp_path / "copy").read_bytes() == b"data" - with pytest.raises(DownloadError): - context.transport.list_entries() + assert sum(transferred) == 4 + + +@pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) +@pytest.mark.parametrize("scheme", ["ssh", "sftp"]) +def test_sftp_only_clone_plans_then_requires_payload_approval( + ssh_server, tmp_path, monkeypatch, scheme, +): + root = ssh_server["root"] + (root / "nested").mkdir() + (root / "nested" / "science.fits").write_bytes(b"science") + (root / "notes.txt").write_bytes(b"notes") + fetched = [] + fetch = SshTransport._fetch + + def record_fetch(self, path, destination, file_limit=None): + fetched.append(path) + return fetch(self, path, destination, file_limit) + + def reject_git_probe(*args, **kwargs): + raise AssertionError("SFTP directory detection must not invoke remote Git") + + monkeypatch.setattr(SshTransport, "_fetch", record_fetch) + monkeypatch.setattr("hallmark.catalog.Dothm.clone", reject_git_probe) + plans = [] + + def decline(plan): + plans.append(plan) + return False + + repo = Repo.clone( + ssh_server["url"].replace("ssh:", scheme + ":"), tmp_path / "clone", + filter="**/*.fits", download=True, approve=decline, + ) + assert fetched == [] + assert repo.state.data["path"].tolist() == ["nested/science.fits"] + assert len(plans) == 1 + assert plans[0].total_bytes == 7 + assert not (repo.worktree / "nested").exists() + with pytest.raises(DownloadError, match="approval"): + repo.download(plans[0]) + assert fetched == [] + result = repo.download(plans[0], approved=True) + assert result["succeeded"] == 1 + assert fetched == ["nested/science.fits"] + assert (repo.worktree / "nested/science.fits").read_bytes() == b"science" @pytest.mark.parametrize("ssh_server", ["no-sftp"], indirect=True) @@ -379,7 +448,7 @@ def test_local_session_limit(ssh_server, tmp_path, monkeypatch): repo.worktree, max_workers=4, selected_files=[(Path(str(i)), None) for i in range(4)], - ) + approved=True) assert result["succeeded"] == 4 assert result["failed"] == 0 @@ -396,7 +465,6 @@ def test_profile_source_recording_and_explicit_output( [], dataset_url=ssh_server["url"], dataset_auth="lab", - allow_remote_commands=True, ) assert repo.state.config["remote"][0]["auth"] == "lab" explicit = build_repo( @@ -406,7 +474,6 @@ def test_profile_source_recording_and_explicit_output( remotes=[{"name": "mirror", "url": "https://example.test/data"}], dataset_url=ssh_server["url"], dataset_auth="lab", - allow_remote_commands=True, ) assert explicit.state.config["remote"] == [ {"name": "mirror", "url": "https://example.test/data"} @@ -417,15 +484,11 @@ def test_ssh_listing_omits_symlinks_and_rejects_controls(ssh_server, tmp_path): root = ssh_server["root"] (root / "item").write_bytes(b"data") (root / "link").symlink_to(root / "item") - with OperationContext( - RemoteSpec.parse(ssh_server["url"]), allow_remote_commands=True - ) as context: + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: assert context.transport.list_entries() == ["item"] (root / "bad\nname").write_bytes(b"data") - with OperationContext( - RemoteSpec.parse(ssh_server["url"]), allow_remote_commands=True - ) as context: - with pytest.raises(DownloadError, match="listing"): + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + with pytest.raises(DownloadError, match="control"): context.transport.list_entries() @@ -440,7 +503,7 @@ def test_permission_denied_preserves_existing_file(ssh_server, tmp_path): try: result = download_remote_data( repo, repo.worktree, selected_files=[(Path("private"), None)] - ) + , approved=True) assert result["failed"] == 1 assert destination.read_bytes() == b"original" assert not list(Path(repo.worktree).glob("*.part")) @@ -474,7 +537,7 @@ def slow_prepare(self): repo, repo.worktree, selected_files=[(Path("large"), None)], - ) + approved=True) try: _wait_for_partial_file(repo.worktree, "*.part", len(payload), task) assert len(transports) == 1 diff --git a/test/test_transport.py b/test/test_transport.py index a582960..a6befbe 100644 --- a/test/test_transport.py +++ b/test/test_transport.py @@ -224,6 +224,76 @@ def fail(*args, **kwargs): assert "secret" not in repr(error.value) +@pytest.mark.parametrize("location", [ + "https://other.test/data/", "https://example.test/outside/", + "http://example.test/data/", "/data/%2e%2e/outside/", + "https://new:secret@example.test/data/", "https://example.test:444/data/", +]) +def test_metadata_redirect_never_requests_outside_root(monkeypatch, location): + calls = [] + closed = [] + with OperationContext(RemoteSpec.parse("https://example.test/data/")) as context: + response = requests.Response() + response.status_code = 302 + response.headers["Location"] = location + response.close = lambda: closed.append(True) + + def get(url, **kwargs): + calls.append(url) + assert kwargs["allow_redirects"] is False + return response + + monkeypatch.setattr(context.session(), "get", get) + with pytest.raises(DownloadError, match="source root"): + context.read_text("") + assert calls == ["https://example.test/data/"] + assert closed == [True] + + +def test_metadata_redirect_within_root_and_default_port_is_supported(monkeypatch): + calls = [] + with OperationContext(RemoteSpec.parse("https://example.test/data/")) as context: + first = requests.Response() + first.status_code = 301 + first.headers["Location"] = "https://example.test:443/data/listing/" + first.close = lambda: None + second = requests.Response() + second.status_code = 200 + second.encoding = "utf-8" + second._content = b"

Index of data

" + second._content_consumed = True + + def get(url, **kwargs): + calls.append(url) + assert kwargs["allow_redirects"] is False + return first if len(calls) == 1 else second + + monkeypatch.setattr(context.session(), "get", get) + assert context.read_text("") == "

Index of data

" + assert context.transport.text_urls[""] == ( + "https://example.test:443/data/listing/") + assert calls == ["https://example.test/data/", + "https://example.test:443/data/listing/"] + + +def test_metadata_redirect_loop_is_bounded(monkeypatch): + calls = [] + with OperationContext(RemoteSpec.parse("https://example.test/data/")) as context: + response = requests.Response() + response.status_code = 307 + response.headers["Location"] = "/data/" + response.close = lambda: None + + def get(url, **kwargs): + calls.append(url) + return response + + monkeypatch.setattr(context.session(), "get", get) + with pytest.raises(DownloadError, match="redirect limit"): + context.read_text("") + assert len(calls) == 11 + + def test_symlink_swap_after_planning(monkeypatch, tmp_path): repo = Repo.init(tmp_path / "repo") repo.set_config(remote_url="https://example.test/") @@ -239,18 +309,19 @@ def swap(self): monkeypatch.setattr("hallmark.transport.http.HttpTransport.prepare", swap) result = download_remote_data( repo, repo.worktree, selected_files=[(Path("sub/data"), None)] - ) + , approved=True) assert result["failed"] == 1 assert list(outside.iterdir()) == [] -def test_builder_requires_explicit_capabilities(tmp_path): - with pytest.raises(CapabilityError, match="index-format"): - build_repo( - tmp_path / "repo", "lab", [], dataset_url="https://example.test/data" - ) - with pytest.raises(CapabilityError, match="allow-remote-commands"): - build_repo(tmp_path / "repo", "lab", [], dataset_url="ssh://campus/data") +@pytest.mark.parametrize("scheme", ["https", "ssh"]) +def test_builder_discovers_without_backend_flags(monkeypatch, tmp_path, scheme): + calls = [] + monkeypatch.setattr( + "hallmark.repo_builder._build_repo", lambda *a, **kw: calls.append((a, kw)) + ) + build_repo(tmp_path / "repo", "lab", [], dataset_url=f"{scheme}://unused/data") + assert len(calls) == 1 assert not (tmp_path / "repo").exists() @@ -422,7 +493,7 @@ def fetch(self, path, destination, **kwargs): monkeypatch.setattr(SshTransport, "fetch", fetch) result = download_remote_data( repo, repo.worktree, selected_files=[(Path("data.bin"), None)] - ) + , approved=True) assert result["failed"] == 1 assert destination.read_bytes() == b"original" assert not list(Path(repo.worktree).glob("*.part")) @@ -443,7 +514,7 @@ def test_checksum_failure_is_atomic(monkeypatch, tmp_path): selected_files=[ (Path("data.bin"), ("sha256", hashlib.sha256(b"right").hexdigest())) ], - ) + approved=True) assert result["failed"] == 1 assert destination.read_bytes() == b"original" assert not list(Path(repo.worktree).glob("*.part")) @@ -463,7 +534,7 @@ def fetch(self, path, dest, **kwargs): monkeypatch.setattr(SshTransport, "fetch", fetch) result = download_remote_data( repo, repo.worktree, selected_files=[(Path("item"), None)] - ) + , approved=True) assert result["failed"] == 1 assert destination.read_bytes() == b"original" assert not list(Path(repo.worktree).glob("*.part")) @@ -493,7 +564,7 @@ def interrupt(*args, **kwargs): with pytest.raises(KeyboardInterrupt): download_remote_data( repo, repo.worktree, max_workers=1, selected_files=[(Path("item"), None)] - ) + , approved=True) assert not (repo.worktree / "item").exists() assert not list(Path(repo.worktree).glob("*.part")) assert all(not transport._processes for transport in transports) @@ -534,24 +605,6 @@ def test_cli_profile_roundtrip(monkeypatch, tmp_path): assert "auth" not in Repo(repo.worktree).state.config["remote"] -def test_ssh_hash_budgets_skip_without_commands(monkeypatch): - with OperationContext( - RemoteSpec.parse("ssh://unused/data"), - allow_remote_commands=True, - remote_hash=True, - ) as context: - transport = context.transport - transport._entries = {"big": (context.hash_file_limit + 1, 0), "small": (1, 0)} - transport._hash_bytes = context.hash_total_limit - - def fail(*args, **kwargs): - raise AssertionError("Budgeted-out files must not run a command") - - monkeypatch.setattr(transport, "_command", fail) - assert transport.checksum_small("big") == ("unknown", "unknown") - assert transport.checksum_small("small") == ("unknown", "unknown") - - def test_http_manifest_auth_failure_is_not_optional(monkeypatch): from hallmark.repo_builder import list_remote_files from mock_server import MockServer @@ -645,7 +698,9 @@ def test_explicit_http_source_is_exact_and_keeps_output_remotes(monkeypatch, tmp remotes=[{"name": "mirror", "url": "https://elsewhere.test/data"}], ) assert repo.state.config["remote"][0]["url"] == "https://elsewhere.test/data" - assert repo.state.config["data"][0]["md5"] == hashlib.md5(b"notes").hexdigest() + entry = repo.state.config["data"][0] + assert not any(key in entry for key in ("md5", "sha1", "sha256")) + assert entry.get("checksum") in (None, "unknown") def test_build_cli_passes_source_controls(monkeypatch, tmp_path): From 0b4b17f23d87b5d6773295998df13aadd1b85787 Mon Sep 17 00:00:00 2001 From: Dennis Date: Sun, 20 Sep 2026 19:37:16 -0700 Subject: [PATCH 12/29] Align private data prose with existing conventions --- README.rst | 4 +- demo/private_data_demo.md | 35 +++--- doc/api.rst | 6 +- doc/private_data.rst | 55 ++++++---- doc/usecase.rst | 31 +++--- mod/hallmark/catalog.py | 48 +++++++- mod/hallmark/cli.py | 99 ++++++++++------- mod/hallmark/discovery.py | 53 +++++++-- mod/hallmark/dothm.py | 4 +- mod/hallmark/download_plan.py | 42 +++++-- mod/hallmark/downloader.py | 169 +++++++++++++++++++++++------ mod/hallmark/repo.py | 102 +++++++++++++++-- mod/hallmark/repo_builder.py | 100 ++++++++++++----- mod/hallmark/repo_config.py | 11 +- mod/hallmark/transport/__init__.py | 28 ++++- mod/hallmark/transport/auth.py | 31 +++++- mod/hallmark/transport/base.py | 105 ++++++++++++++++-- mod/hallmark/transport/http.py | 7 +- mod/hallmark/transport/sftp.py | 46 +++++++- mod/hallmark/transport/ssh.py | 61 ++++++++++- test/mock_server.py | 2 + test/test_catalog.py | 3 +- test/test_cli.py | 4 +- test/test_discovery.py | 2 + test/test_download_plan.py | 3 +- test/test_downloader.py | 2 +- test/test_repo_builder.py | 28 ++--- test/test_sftp_metadata.py | 2 + test/test_ssh_integration.py | 3 +- test/test_transport.py | 3 + 30 files changed, 843 insertions(+), 246 deletions(-) diff --git a/README.rst b/README.rst index ac5ec89..949392d 100644 --- a/README.rst +++ b/README.rst @@ -80,7 +80,7 @@ is used to share the history of the data index. For a private server, |hallmark|_ uses OpenSSH 9.6 or newer with a trusted host key and key-based authentication that does not require a prompt. -Clone a remote directory to prepare its catalog, then approve selected downloads:: +Clone a remote directory to create its catalog, then select files to download:: hallmark clone ssh://campus/srv/export/ campus --filter '**/*.h5' cd campus @@ -92,7 +92,7 @@ ordinary browsable HTTPS directories, including DESI, use the same workflow. SSH discovery also works with SFTP-only accounts; no server shell or Python is required. Omit the filter to catalog everything beneath the supplied URL. -Every dataset transfer requires confirmation. Python callers can inspect +Every dataset transfer requires approval. Python callers can inspect ``repo.plan_download()`` and then execute ``repo.download(plan, approved=True)``. Optional authentication profiles remain local and can be selected with ``clone --auth PROFILE`` or ``set-config --remote-auth PROFILE``. diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md index 59c949d..e5d827e 100644 --- a/demo/private_data_demo.md +++ b/demo/private_data_demo.md @@ -1,14 +1,14 @@ # Private data demo: copyable instructions -These are illustrative client-side instructions, not a record of a live VM run. -The SSH alias retains the validation VM settings supplied for this demo. Use an -existing export that you are authorized to read, and replace the example dataset -root before running the commands. No remote fixture creation is required. +Run these instructions on your local machine using an existing data export. +The SSH alias uses the validation VM settings supplied for this demo. Replace +the example dataset root with your export path; the commands below have not +been run against that path on the VM. For a small walkthrough, the examples assume `runs/run_001.dat`, `runs/run_002.dat`, a `README.md`, and optionally a publisher-provided `SHA256SUMS`. The client needs this Hallmark checkout, Git, and OpenSSH `ssh`/`sftp` 9.6+. -The server needs SFTP; a login shell, Python, Hallmark, and Git are unnecessary. +The server needs SFTP, but does not require a login shell, Python, Hallmark, or Git. Run the Bash blocks in order in one local terminal. 1. Install the checkout in your active Python environment. @@ -21,7 +21,7 @@ hallmark clone --help ``` 2. Add or update this entry in `~/.ssh/config`, preserving your other entries. -Replace the host, user, or identity if your approved server differs. +Replace the host, user, or identity when using a different server. ```text Host lab-data @@ -40,7 +40,7 @@ sftp -o BatchMode=yes -o StrictHostKeyChecking=yes -b /dev/null lab-data ``` 3. Set the exact export root and create an isolated local workspace. The example -path is illustrative; it is not asserted to exist on the validation VM. +path is illustrative and may not exist on the validation VM. ```bash export HM_DEMO_URL='ssh://lab-data/srv/exports/lab/' @@ -83,15 +83,15 @@ cat ./client/.hm/data.tsv Expected for the illustrative export: two run rows, with sizes and modification times where available. Published manifest checksums are attached when present; missing checksums stay unknown. The data remote is `origin` with `auth: demo`. -Only listing and catalog metadata are fetched. No run bodies are downloaded for -hashing, and no remote commands or server Python are required. +Only listings and catalog metadata are fetched. Dataset files are not read to +compute checksums, and no remote commands or server Python are required. Omit `--fmt` to recursively catalog every file below the URL, including supporting files. Alternatively, `--filter '**/*.dat'` selects paths without defining parameters. Filters and formats never authorize downloads. Discovery reports completed directories and discovered files while its total remains unknown. -5. Inspect the transfer plan before approving any payload. +5. Inspect the transfer plan before approving a download. ```bash cd "$HM_DEMO_WORKSPACE/client" @@ -101,7 +101,8 @@ hallmark download runs/run_001.dat --dry-run ``` A dry run contacts no server. It reports count, known bytes, unknown-size count, -source, and destination; duration is unknown without a credible rate estimate. +source, and destination. The CLI reports duration as unknown; Python callers +can estimate it from a supplied transfer rate when all file sizes are known. It does not validate credentials or remote-file existence. To download the selected run, inspect the displayed plan and answer `y` at the prompt: @@ -165,16 +166,16 @@ PYTHON python ../download_subset.py ``` -The plan is immutable and pins its source, profile reference, destination, paths, -and checksums. Later catalog/configuration changes do not redirect it. Without +The plan fixes its source, profile reference, destination, paths, and checksums. +Later catalog or configuration changes do not redirect it. Without `approved=True`, nonempty Python transfers fail before connecting. Check the returned `failed` count; successful files remain when another transfer fails. Files are written to temporary paths and published atomically after transfer and any checksum verification. Failed files preserve an existing destination and remove their temporary file. Repeating a download transfers the selection again. -Cancellation closes Hallmark-owned connections and processes. Byte progress shows -an ETA when a total is known; unknown totals stay indeterminate. +Cancellation closes the connections and processes started for the download. +Byte progress shows an ETA when a total is known; unknown totals stay indeterminate. 8. Request an approved download during cloning, using either interface. The CLI prepares the catalog first and then displays its normal approval prompt: @@ -274,9 +275,9 @@ separate from the data remote's `--auth` profile. Filtering an existing Git catalog preserves fetched history and adds a local filtered-catalog commit. `clone` downloads no dataset files by default, so `--no-fetch-data` is now only -a compatibility alias. `init` creates a local repository; the old remote `build` +a compatibility alias. `init` creates a local repository; the older remote `build` command is deprecated. Discovery uses no `--allow-remote-commands` or -`--remote-hash` flag, and never reads dataset bodies to create checksums. +`--remote-hash` flag, and does not read dataset files to compute checksums. For local `add`/`commit` experiments, use a separate compatible one-format repository; downloading a remote catalog does not populate the local object store. diff --git a/doc/api.rst b/doc/api.rst index d8ab88b..a8c342d 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -108,7 +108,7 @@ The older ``build`` command remains a deprecated compatibility interface. Downloading remote data ----------------------- -Inspect the offline transfer plan and then confirm a download:: +Preview the selected files using the local catalog, then confirm a download:: hallmark download --all --dry-run hallmark download --filter 'runs/**' @@ -128,8 +128,8 @@ Python uses the same plan and requires explicit approval:: result = repo.download(plan, approved=True, progress=True) Plans preserve their selected remote, destination and checksums even when -repository configuration later changes. Size and time estimates remain unknown -when metadata or a credible throughput estimate is unavailable. +repository configuration later changes. Size estimates require recorded file +sizes; duration estimates also require a supplied transfer rate. .. automodule:: hallmark.download_plan :members: diff --git a/doc/private_data.rst b/doc/private_data.rst index 624a250..e8043eb 100644 --- a/doc/private_data.rst +++ b/doc/private_data.rst @@ -3,21 +3,21 @@ Private data over SSH and SFTP ============================== -Hallmark can discover a remote dataset, prepare a local catalog, and download +|hallmark|_ can discover a remote dataset, prepare a local catalog, and download selected files after approval. ``clone`` accepts both an existing Hallmark catalog and an ordinary data directory. Its default result is a local ``.hm`` -with no dataset payloads. A **data remote** identifies the dataset location; +with no dataset files. A **data remote** identifies the dataset location; it is separate from the Git remote used to share catalog history. The SSH examples use the alias ``lab-data`` and the absolute dataset root ``/srv/exports/lab/``. Replace the hostname, username, identity path and root with your own values. Run Hallmark locally. The server needs SFTP access, but -neither Hallmark, Git, Python nor a login shell. +does not require Hallmark, Git, Python or a login shell. 1. Configure and check SSH access --------------------------------- -Install this checkout with ``python -m pip install -e .``. The client requires +Install the checkout with ``python -m pip install -e .``. The client requires system OpenSSH ``ssh`` and ``sftp`` version 9.6 or newer, with connection multiplexing available. Linux and macOS have integration-test jobs; Windows SSH transport is unsupported. @@ -71,14 +71,14 @@ without defining parameters: Without ``--filter`` or ``--fmt``, discovery recursively covers every directory beneath the URL and catalogs all discovered files. A filter still traverses directories needed to find matching files. Discovery shows an indeterminate -progress bar with completed directories and discovered files; it does not -invent a percentage or completion time while the total is unknown. +progress bar with counts of completed directories and discovered files. +The percentage and completion time remain unknown until a total is available. Both URL schemes use structured SFTP directory enumeration and file transfers. SFTP-only accounts work for discovery as well as downloading. No ``--allow-remote-commands`` or ``--remote-hash`` option is needed. Discovery -reads listings and published checksum manifests, never dataset bodies to -manufacture a checksum. Missing digests stay unknown. For reproducible +reads listings and published checksum manifests without reading dataset files +to compute checksums. Missing checksums remain unknown. For reproducible catalogs, publish an immutable export with a SHA-256 manifest, for example `` runs/run_001.h5`` records in ``SHA256SUMS``. @@ -100,9 +100,10 @@ Continue from the same workspace: hallmark download runs/run_001.h5 The dry run reads only the local catalog. It reports file count, known bytes, -the number of unknown file sizes, destination and source. Duration stays -unknown unless a credible transfer-rate estimate is available. The dry run -does not test credentials, host trust or remote-file existence. +the number of unknown file sizes, destination and source. The CLI reports +duration as unknown. Python callers can supply a transfer rate to estimate +duration when all file sizes are known. The dry run does not test credentials, +host trust or remote-file existence. Every nonempty CLI download displays its plan and asks ``Download these files? [y/N]``. Answer ``y`` to authorize the displayed @@ -121,7 +122,7 @@ Choose a scope from inside ``lab``: Explicit paths use their recorded catalog checksums when available, just like ``--tsv`` and ``--all``. A path absent from the catalog can be requested, but its size and checksum are unknown. Files without a usable publisher checksum -can download; successful transfer alone does not establish catalog integrity. +can be downloaded, but their contents cannot be checked against the catalog. ``--tsv`` can be repeated and combined with explicit paths; overlapping entries are deduplicated. ``--all`` cannot be combined with either selection mode. @@ -193,7 +194,7 @@ an ordinary remote directory: hallmark download --remote origin --tsv data.tsv --dry-run hallmark clone 'ssh://lab-data/srv/exports/lab/' ../lab-profile --auth lab -The new catalog's ``origin`` records ``auth: lab``. Clear a reference with +The new catalog's ``origin`` remote records ``auth: lab``. Clear a reference with ``hallmark set-config --remote-name origin --remote-auth ''`` to use SSH configuration alone. ``--auth`` applies to data access; Git cloning uses Git's own authentication configuration. @@ -201,7 +202,7 @@ own authentication configuration. ``hosts`` binds the profile to the URL's alias or literal hostname before ``HostName`` resolution. This example binds to ``lab-data``, not ``data.example.org``. Explicit URL user/port values override profile settings; -unset values fall through to SSH configuration. Passwords, tokens and arbitrary +unset values use SSH configuration. Passwords, tokens and arbitrary SSH options are not valid profile fields. ``transfer_timeout`` is the total per-file time budget in seconds, not an idle timeout. Effective SSH download concurrency is the smaller of ``--max-workers`` and local ``max_sessions`` @@ -227,7 +228,7 @@ discovers metadata without downloading dataset files: print(plan.summary()) print(plan.items) # Paths, checksums, optional size_bytes and mtime. -Inspect the plan before making the explicit approval call: +Inspect the plan, then approve the download: .. code-block:: python @@ -237,7 +238,7 @@ Inspect the plan before making the explicit approval call: Without ``approved=True``, a nonempty transfer raises ``DownloadError`` before opening a connection. A plan freezes the selected files, source URL, profile -reference and destination; later catalog or remote-config changes do not +reference and destination; later catalog or remote configuration changes do not redirect it. ``plan_download(filter="**/*.h5")`` scopes the complete catalog; ``plan_download(output_path="subset", tsv_names=["data.tsv"])`` selects a TSV and destination. Planning makes no network requests. Known size totals and @@ -246,7 +247,7 @@ unknown-size counts are available as ``known_bytes`` and are known and ``estimated_bytes_per_second`` was supplied when planning. For downloads during cloning, supply a callback that reviews the completed -plan and returns the Boolean ``True`` to approve: +plan and returns ``True`` to approve: .. code-block:: python @@ -311,15 +312,17 @@ If a server supplies no usable listing, provide a published Hallmark catalog or a more specific browsable URL; Hallmark cannot infer hidden file URLs. -Operational behavior and troubleshooting ----------------------------------------- +.. _operational-behavior-and-troubleshooting: + +Download behavior and troubleshooting +------------------------------------- Files are downloaded to temporary paths and published atomically after transfer and any recorded checksum verification. A failed transfer preserves an existing destination and removes its temporary file. Successful files remain available when another file fails; a multi-file download is not a single transaction. -The CLI exits nonzero on transfer failures. Cancellation stops operation-owned -work and closes Hallmark's SSH connections and processes. +The CLI exits nonzero on transfer failures. Cancellation stops the active +operation and closes the SSH connections and processes it started. .. list-table:: Common failures :header-rows: 1 @@ -327,7 +330,7 @@ work and closes Hallmark's SSH connections and processes. * - Symptom - Action - * - SSH connection preflight fails + * - SSH connection check fails - Check the trusted host key, loaded identity, username, server availability and OpenSSH version. For profile-only settings, pass the same identity/user/port to the manual SFTP check. @@ -339,7 +342,7 @@ work and closes Hallmark's SSH connections and processes. SFTP account can enumerate the requested directories. * - Checksum mismatch - Check whether the export changed since catalog creation. Reconcile the - trusted catalog and source snapshot before retrying. + catalog and source against a trusted version before retrying. * - Transfer timeout or session limit - Adjust the local per-file time budget or lower concurrency to match the server's capacity. @@ -353,9 +356,13 @@ destination symlinks; server permissions remain the access-control boundary. Discovered catalogs can use path rows and published catalogs can contain multiple TSVs. Existing local ``add``/``commit``/``checkout`` workflows still require a compatible one-format ``data.tsv`` repository; downloading does not -seed the local object store. For local experiments, initialize a separate +populate the local object store. For local experiments, initialize a separate repository, copy in the downloaded files, configure their filename format, then add and commit them. See :ref:`private-transport-reference` for profile settings, HTTP compatibility and disposable transport tests, and :doc:`api` for function signatures. + +.. |hallmark| replace:: ``hallmark`` + +.. _hallmark: https://github.com/l6a/hallmark diff --git a/doc/usecase.rst b/doc/usecase.rst index 5226b83..6a44479 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -153,8 +153,8 @@ automatically try another server if the download fails. Downloads are written to temporary files and checked against any checksums supplied with the selection before atomically replacing their destinations. -The ``--tsv`` and ``--all`` options include catalog checksums; explicit -paths alone do not look them up. +The ``--tsv`` and ``--all`` options include catalog checksums. Explicit +paths also use their recorded checksums when available. If a transfer or checksum check fails, the existing destination is preserved and the temporary file is removed. Files that have already downloaded successfully remain available. @@ -164,8 +164,8 @@ SSH access and file paths The client requires OpenSSH ``ssh`` and ``sftp`` version 9.6 or newer with connection multiplexing enabled. -Linux and macOS have integration-test jobs; check the macOS job's result -before relying on that platform. Windows SSH transport is not supported. +Linux and macOS have integration-test jobs. Windows SSH transport is +not supported. HTTP downloads continue to use Requests, including its environment settings and ``.netrc`` authentication. @@ -285,24 +285,26 @@ both discovery and downloads. Symlinks and special files are skipped. Authentication and permission failures stop discovery rather than producing an apparently complete catalog. -The generated data ``origin`` records the source URL and optional profile name. +The generated data remote ``origin`` records the source URL and optional +profile name. Existing Git-hosted catalogs keep their recorded data remotes when cloned. A filter on an existing Git catalog creates a local metadata commit while preserving fetched history; it does not fetch the dataset's content objects. -Published checksum manifests are retained. Missing checksums remain unknown; -Hallmark never downloads files or hashes them on the server merely to finish -a catalog. Downloads verify any available publisher checksum. No global -five-minute crawl deadline or 100,000-entry cap limits the requested tree; -individual requests remain bounded and discovery can be cancelled. +Published manifest checksums are recorded in the catalog. Missing checksums +remain unknown; |hallmark|_ does not read dataset files locally or on the +server to compute them during discovery. Downloads verify any available +publisher checksum. Discovery covers the requested tree without an overall +time or entry limit. Individual requests have limits, and discovery can +be cancelled. CyVerse and common HTTPS directory indexes are detected automatically:: hallmark clone https://data.desi.lbl.gov/public/ desi --filter '**/*.fits' -A server with no usable listing must provide a published catalog or file -manifest through a supported adapter. Discovery cannot enumerate arbitrary -hidden URLs. HTTP/SFTP catalog snapshots start local history; Git endpoints +A server with no usable listing must provide a published Hallmark catalog. +Discovery cannot find files whose URLs are absent from both the listing and +catalog. HTTP/SFTP catalog snapshots start local history; Git endpoints supply catalog history. Use ``--source-type git`` for ambiguous Git URLs. Frank can also inspect and approve a transfer in Python:: @@ -319,7 +321,8 @@ sizes and duration estimates are reported as unknown. During transfer, progress reports bytes and estimates remaining time when possible. ``build`` and ``build_repo`` remain deprecated compatibility interfaces. Local ``add``, ``commit`` and ``checkout`` retain their existing format and -local-object requirements; preparing a remote catalog does not materialize it. +local-object requirements; preparing a remote catalog does not populate the +local object store. Transport validation ~~~~~~~~~~~~~~~~~~~~ diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py index 1aff9a8..f373c8c 100644 --- a/mod/hallmark/catalog.py +++ b/mod/hallmark/catalog.py @@ -1,4 +1,4 @@ -"""Prepare remote inventories and published catalogs without dataset transfers.""" +"""Create local catalogs from Git repositories and remote data directories.""" from __future__ import annotations @@ -38,12 +38,14 @@ def _catalog_names(config): def _catalog_formats(config, name): + """Return the filename formats associated with a catalog TSV.""" entries = as_list_of_dicts(config.get("data", [])) or [] return [entry["fmt"] for entry in entries if entry.get("fmt") and normalize_tsv_name(entry.get("db", "data.tsv")) == name] def _row_path(row, formats): + """Resolve a catalog row to one literal relative path.""" value = row.get("path") if value is not None and not pd.isna(value) and str(value): return literal_path(str(value)).as_posix() @@ -59,7 +61,7 @@ def _row_path(row, formats): def _validate_snapshot(files, config): - """Reject login pages and unsafe inventories before publishing a snapshot.""" + """Validate catalog tables and paths before writing the snapshot.""" for name in _catalog_names(config): try: frame = pd.read_csv(StringIO(files[name]), sep="\t", dtype=str, @@ -76,7 +78,12 @@ def _validate_snapshot(files, config): def _snapshot(context): - """Probe fixed catalog metadata names, never a dataset file or object store.""" + """ + Read and validate a published catalog without fetching dataset files. + + Return None when required catalog files are absent or ``config.yml`` + does not describe a catalog. Invalid tables and metadata raise CloneError. + """ try: config_text = context.read_text("config.yml") except RemoteObjectMissing: @@ -108,7 +115,7 @@ def _snapshot(context): def _metadata_commit(repo, files, message): - """Commit only catalog metadata, without capturing local dataset objects.""" + """Commit catalog metadata and reload the repository state.""" repo.dothm.index.add(list(files)) if repo.dothm.index.diff("HEAD"): repo.dothm.index.commit(message) @@ -116,6 +123,7 @@ def _metadata_commit(repo, files, message): def _filter_catalog(repo, *, filter=None, fmt=None): + """Filter catalog tables and file entries, then commit the selection.""" if filter is None and fmt is None: return config = repo.state.config @@ -140,6 +148,7 @@ def _filter_catalog(repo, *, filter=None, fmt=None): def _write_inventory(repo, source, entries, fmt): + """Write discovered files and their source to a new catalog.""" columns = ["path", "checksum_algorithm", "checksum", "size_bytes", "mtime"] rows = [] for entry in entries: @@ -168,6 +177,7 @@ def _write_inventory(repo, source, entries, fmt): def _git_source(url, source_type): + """Determine whether the selected source should be cloned with Git.""" if source_type == "git": return True if source_type != "auto": @@ -180,7 +190,33 @@ def _git_source(url, source_type): def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, source_type="auto", progress=False): - """Create a complete local catalog or remove this operation's destination.""" + """ + Create a local catalog from Git, a published snapshot, or a data directory. + + An incomplete destination created by this call is removed on failure. + Existing destinations are rejected before any source access. + + Args: + cls: Repository class used to initialize or open the result. + url (str): Catalog location or URL of the dataset directory. + path (Path | str): New repository destination. + auth (str, optional): Local SSH profile for data access. + filter (str | list[str], optional): Relative path glob or globs. + fmt (str, optional): Filename format used to select paths. + source_type (str): ``auto``, ``git``, ``directory``, or ``catalog``. + Defaults to automatic detection. + progress (bool | callable): Discovery progress display or callback. + Defaults to False. + + Returns: + Repo: Repository containing catalog metadata without dataset files. + + Raises: + DestinationExistsError: If the destination already exists. + CloneError: If a requested catalog is missing or invalid. + ValueError: If source options are invalid. + DownloadError: If remote metadata cannot be read or validated. + """ if source_type not in {"auto", "git", "directory", "catalog"}: raise ValueError("source_type must be auto, git, directory, or catalog") url = str(url) @@ -192,7 +228,7 @@ def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, is_git = _git_source(url, source_type) if is_git and auth is not None: raise ValueError("Git cloning uses Git/SSH authentication, not auth profiles") - # Reserve ownership before cleanup becomes possible; never remove another repo. + # Create the destination exclusively so cleanup only removes our own directory. destination.parent.mkdir(parents=True, exist_ok=True) try: destination.mkdir() diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 719eaba..8794d34 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -116,7 +116,19 @@ def _report_download_results(results: dict) -> None: def _run_download(repo, plan, *, max_workers): - """Approve and execute exactly the plan displayed by clone or download.""" + """ + Display a download plan, request approval, and report the results. + + Args: + repo: The hallmark repository object. + plan (DownloadPlan): Files, source, and destination to display. + max_workers (int): Maximum concurrent download workers. + + Raises: + Abort: If a nonempty download is declined. + ClickException: If any file transfers fail. + DownloadError: If the download cannot be started. + """ click.echo(plan.summary()) if not plan.file_count: click.echo("No files selected for download.") @@ -265,7 +277,8 @@ def add(repo, encoding, inputs): @click.option("--fmt") @click.option("--remote-name") @click.option("--remote-url") -@click.option("--remote-auth", help="Local SSH profile; empty string removes it.") +@click.option("--remote-auth", + help="Local SSH profile name. An empty string removes the reference.") @click.option("--encoding", "encodings", multiple=True) @click.pass_obj def set_config(repo, fmt, remote_name, remote_url, remote_auth, encodings): @@ -362,27 +375,34 @@ def checkout(repo, target_branch): click.echo(f'Switched to branch "{target_branch}".') -@hallmark.command(short_help="Plan and approve a dataset download.") +@hallmark.command(short_help="Download files from the configured data remote.") @click.argument("files", nargs=-1) @click.option("--tsv", "tsv_names", multiple=True, help="Select a catalog TSV. May be repeated.") @click.option("--all", "download_all", is_flag=True, help="Select all cataloged files.") @click.option("--filter", "filters", multiple=True, - help="Relative path glob; ** matches recursively. May be repeated.") -@click.option("--fmt", help="Select a parameterized filename template.") -@click.option("--remote", "remote_name", help="Configured data remote name.") + help="Select paths matching a glob. ** matches recursively. " + "May be repeated.") +@click.option("--fmt", help="Select paths matching a filename format.") +@click.option("--remote", "remote_name", + help="Name of the configured data remote to use.") @click.option("--output", type=click.Path(file_okay=False), - help="Output directory; defaults to the worktree.") + help="Output directory. Defaults to the repository worktree.") @click.option("--max-workers", type=click.IntRange(min=1), default=4, show_default=True) @click.option("--dry-run", is_flag=True, - help="Show the offline transfer plan without downloading.") + help="Show the download plan using only local catalog metadata.") @click.option("-y", "--yes", is_flag=True, hidden=True) @click.pass_obj def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, output, max_workers, dry_run, yes): - """Inspect sizes and approve every nonempty dataset transfer.""" + """ + Download selected files from a configured data remote. + + Preview the selection with --dry-run. Every nonempty download displays + its plan and asks for confirmation before transferring files. + """ if download_all and (files or tsv_names): raise ClickException("--all cannot be combined with file paths or --tsv") if not files and not tsv_names and not download_all and not filters and not fmt: @@ -411,20 +431,27 @@ def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, @click.argument("path") @click.option("--auth", help="Optional local SSH authentication profile.") @click.option("--filter", "filters", multiple=True, - help="Relative path glob; ** matches recursively. May be repeated.") -@click.option("--fmt", help="Parameterized filename template to catalog.") + help="Select paths matching a glob. ** matches recursively. " + "May be repeated.") +@click.option("--fmt", + help="Select paths and extract parameters using a filename format.") @click.option("--source-type", default="auto", show_default=True, type=click.Choice(["auto", "git", "directory", "catalog"]), help="Override automatic source detection.") @click.option("--download", "fetch_data", is_flag=True, - help="Plan and request approval for downloads after catalog creation.") + help="Request a download after catalog creation, with confirmation.") @click.option("--no-fetch-data", is_flag=True, hidden=True) @click.option("--max-workers", type=click.IntRange(min=1), default=4, show_default=True) @click.option("-y", "--yes", is_flag=True, hidden=True) def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, max_workers, yes): - """Prepare local .hm metadata; dataset files are not downloaded by default.""" + """ + Clone a catalog or index a remote data directory at PATH. + + Dataset files are not downloaded by default. Use --download to review + and approve a transfer after the catalog has been created. + """ if fetch_data and no_fetch_data: raise ClickException("--download conflicts with --no-fetch-data") if yes: @@ -457,19 +484,19 @@ def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, "--config-file", "config_file", type=click.Path(exists=True, dir_okay=True, file_okay=True), help="Path to config.yml or a repository directory containing config.yml. " - "build_repo loads fmts (and remotes unless --remote is provided).") + "Load formats and, unless --remote is given, data remotes.") @click.option( "--fmt", "fmts", multiple=True, - help="A fmt entry to use directly, as FMT=DB (e.g. " - "'a{a}_i{i}.h5=data.tsv'). May be repeated for multiple fmts; " - "skips the prompt entirely.") + help="Filename format and TSV name, as FMT=DB (e.g. " + "'a{a}_i{i}.h5=data.tsv'). May be repeated for multiple formats.") @click.option( "--overwrite", is_flag=True, help="Replace the destination repository if it already exists.") @click.option("--dataset-url", - help="Exact crawl root, independent of recorded remotes.") -@click.option("--dataset-auth", help="Local SSH profile for the crawl source.") + help="Exact dataset URL to search recursively. Recorded remotes " + "do not change this root.") +@click.option("--dataset-auth", help="Local SSH profile for the dataset source.") @click.option("--index-format", type=click.Choice(["cyverse-html"]), hidden=True) @click.option("--allow-remote-commands", is_flag=True, hidden=True, help="Deprecated; discovery uses SFTP.") @@ -478,30 +505,18 @@ def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, def build(directory, dataset_name, remotes, config_file, fmts, overwrite, dataset_url, dataset_auth, index_format, allow_remote_commands, remote_hash): """ - Build a hallmark repository at DIRECTORY for the remote dataset DATASET_NAME. - - The dataset is fetched from the remote index and stored in a new hallmark - repository at DIRECTORY. The remotes can be specified with - --remote NAME=URL or --remote NAME. - if no remotes are specified, the default remote from the dataset index will be used. - --config-file: Optional path to an existing config.yml to load fmts and remotes - --fmt: Optional fmt entries to use directly, specified as FMT=DB. May be repeated - for multiple fmts. - --overwrite: Optional flag to replace the destination repo if it already exists. - - Arguments: - - DIRECTORY: The file system path where the hallmark repository will be created. - DATASET_NAME: The name of the remote dataset to fetch. - --remote: Optional remote(s) to record, specified as NAME=URL or just NAME. - May be repeated for multiple remotes. - --config-file: Optional path to an existing config.yml to load fmts and remotes. - --fmt: Optional fmt entries to use directly, specified as FMT=DB. + Build a catalog at DIRECTORY/DATASET_NAME.hm. - Raises: - ClickException: If there is an error during the build process, such as - a network error, Git error, or invalid dataset name. + This command is deprecated. Use hallmark clone URL PATH for remote + datasets, or hallmark init PATH to create a local repository. + + --dataset-url selects the exact discovery root; otherwise DATASET_NAME + identifies a dataset beneath the CyVerse curated-data directory. + --remote records data locations without changing the discovery root. + Supply filename formats with --fmt or --config-file. If neither is + provided, existing formats are reused or a generic path catalog is + created. Dataset files are not downloaded during catalog creation. """ click.echo("build is deprecated; use hallmark clone URL PATH.", err=True) if config_file and fmts: @@ -564,4 +579,4 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, raise ClickException( f"Failed to reach dataset {dataset_name!r}: {exc}") from exc - click.echo(f'Successfully built hallmark repository at "{repo_path}".') \ No newline at end of file + click.echo(f'Successfully built hallmark repository at "{repo_path}".') diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py index 8d15584..001f143 100644 --- a/mod/hallmark/discovery.py +++ b/mod/hallmark/discovery.py @@ -1,4 +1,4 @@ -"""Discover remote file metadata without transferring dataset payloads.""" +"""Discover remote file metadata without downloading dataset contents.""" from __future__ import annotations @@ -41,11 +41,26 @@ def match(path_index, pattern_index): @lru_cache(maxsize=128) def _format_parser(fmt): + """Compile and cache a case-sensitive filename format.""" return parse.compile(fmt, case_sensitive=True) def path_matches(path: str, filter=None, fmt: str | None = None) -> bool: - """Match root-relative paths by glob(s), exact Hallmark format, or both.""" + """ + Match a relative path against globs and a filename format. + + Args: + path (str): Path relative to the dataset root. + filter (str | list[str], optional): Globs to match. At least one + must match; ``**`` spans zero or more directories. + fmt (str, optional): Filename format that the complete path must match. + + Returns: + bool: True when the path satisfies every supplied selector. + + Raises: + ValueError: If the filter or format is invalid. + """ path = str(path) parser = _format_parser(fmt) if fmt is not None else None if filter is not None: @@ -122,6 +137,7 @@ def handle_data(self, data): def _origin(url): + """Return connection fields used to compare directory-listing URLs.""" return (url.scheme, url.hostname, url.port or (443 if url.scheme == "https" else 80), url.username, url.password) @@ -135,7 +151,7 @@ def _index_link(root_url, directory, href, kind): resolved = urlsplit(urljoin(current, href)) if _origin(resolved) != _origin(root): return None - # Sorting/navigation query links are not dataset objects. + # Sorting and navigation links do not identify dataset files. if resolved.query or resolved.fragment: return None root_path = unquote(root.path, errors="strict") @@ -156,6 +172,7 @@ def _index_link(root_url, directory, href, kind): def _parse_index(text, root_url, directory): + """Extract file and directory metadata from a supported HTML index.""" parser = _IndexParser() parser.feed(text) links = [] @@ -203,7 +220,7 @@ def _response_directory(context, requested): def _manifest_algorithm(path): - """Recognize conventional manifests; never guess from arbitrary substrings.""" + """Identify checksum manifests by their conventional filenames.""" name = PurePosixPath(path).name.lower() if re.fullmatch(r"checksums?(?:\.txt)?", name): return "unknown" @@ -258,11 +275,29 @@ def _manifest_checksums(context, entries): def discover(context, *, filter=None, fmt=None, progress=False) -> list[RemoteEntry]: - """Recursively collect file metadata; formats and globs never authorize transfer. - - ``progress=True`` displays an indeterminate progress bar. A callable instead - receives dictionaries with ``directories``, ``files``, ``matched``, and - ``current`` keys, suitable for notebook or application rendering. + """ + Discover remote files and their published metadata recursively. + + Directory listings and checksum manifests are read before filtering. + Dataset files are not downloaded to compute missing checksums. + + Args: + context (OperationContext): Source connection and cancellation state. + filter (str | list[str], optional): Relative path glob or globs. + fmt (str, optional): Filename format that selected paths must match. + progress (bool | callable): True displays a progress bar with an + unknown total. A callback receives dictionaries containing + ``directories``, ``files``, ``matched``, and ``current``. + Defaults to False. + + Returns: + list[RemoteEntry]: Matching files sorted by relative path. + Unavailable sizes, modification times, and checksums remain None. + + Raises: + ValueError: If a filter or format is invalid. + CapabilityError: If the source provides no supported listing. + DownloadError: If discovery fails, paths are unsafe, or manifests conflict. """ # Validate selectors before contacting the source, even for an empty index. path_matches("", filter=filter, fmt=fmt) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index fcc8c91..2ea03c5 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -153,8 +153,8 @@ def config_template() -> str: remote: # name: origin # url: https://example.com/path/to/data/ - # SSH alternative: ssh://campus/srv/export/ (absolute root, trusted host/key) - # auth: campus # Optional local profile name; never put credentials here. + # SSH URL example: ssh://campus/srv/export/ (requires a trusted host key) + # auth: campus # Optional local profile name. Credentials stay local. """ @classmethod diff --git a/mod/hallmark/download_plan.py b/mod/hallmark/download_plan.py index f42ae60..df1ef0e 100644 --- a/mod/hallmark/download_plan.py +++ b/mod/hallmark/download_plan.py @@ -11,7 +11,15 @@ @dataclass(frozen=True) class DownloadItem: - """A catalog file and the metadata available when a transfer was planned.""" + """ + A file and its catalog metadata at the time a download is planned. + + Attributes: + relative_path (Path): File path relative to the data remote. + checksum (str | tuple, optional): Digest or (algorithm, digest) pair. + size_bytes (int, optional): Recorded file size; None if unknown. + mtime (str, optional): Recorded modification time; None if unknown. + """ relative_path: Path checksum: Optional[Union[str, tuple[str, str]]] = None @@ -19,6 +27,7 @@ class DownloadItem: mtime: Optional[str] = None def __post_init__(self) -> None: + """Normalize the path and validate optional file metadata.""" object.__setattr__(self, "relative_path", Path(self.relative_path)) if isinstance(self.checksum, list): object.__setattr__(self, "checksum", tuple(self.checksum)) @@ -40,10 +49,20 @@ def __post_init__(self) -> None: @dataclass(frozen=True) class DownloadPlan: - """The exact files, source, and destination presented for approval. - - Sizes describe catalog metadata; they are estimates until the transfer ends. - A duration is available only when every size and a supplied rate are known. + """ + An immutable selection of files, source, and destination for a download. + + Creating a plan does not approve a transfer. Sizes come from the catalog; + a duration estimate requires every file size and a supplied transfer rate. + + Attributes: + items (tuple[DownloadItem]): Selected files and their metadata. + remote_url (str, optional): Data source URL; may be None for an empty plan. + output_path (Path): Absolute destination directory. + remote_auth (str, optional): Local SSH profile name. + remote_name (str, optional): Name of the selected data remote. + estimated_bytes_per_second (float, optional): Positive rate supplied + by the caller for duration estimates. """ items: tuple[DownloadItem, ...] @@ -54,6 +73,7 @@ class DownloadPlan: estimated_bytes_per_second: Optional[float] = None def __post_init__(self) -> None: + """Copy the selection and validate the source fields and rate.""" object.__setattr__(self, "items", tuple(self.items)) object.__setattr__( self, "output_path", Path(self.output_path).expanduser().absolute()) @@ -71,7 +91,7 @@ def __post_init__(self) -> None: @property def file_count(self) -> int: - """Number of files authorized if this plan is approved.""" + """Number of files selected for download.""" return len(self.items) @property @@ -91,13 +111,19 @@ def total_bytes(self) -> Optional[int]: @property def estimated_seconds(self) -> Optional[float]: - """Estimated duration from a supplied rate, never a network probe.""" + """Estimated duration, or None when sizes or the transfer rate are unknown.""" if self.total_bytes is None or self.estimated_bytes_per_second is None: return None return self.total_bytes / self.estimated_bytes_per_second def summary(self) -> str: - """Describe the planned transfer without hiding unavailable estimates.""" + """ + Describe the planned transfer, including unavailable estimates. + + Returns: + str: File count, size and duration estimates, source, and destination. + URL credentials, query parameters, and fragments are omitted. + """ size = f"{self.known_bytes:,} bytes" if self.unknown_size_count: size += f" known; {self.unknown_size_count} file(s) with unknown size" diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index d20646d..3259187 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -41,7 +41,7 @@ def _repository_config(repo) -> dict: """ - Used by _select_remote_config and select_download_files. + Used by _select_remote_config and _select_download_items. Return repository configuration as a dictionary. Args: @@ -66,7 +66,7 @@ def _repository_config(repo) -> dict: def _config_section_entries(config: dict, section_name: str) -> list[dict]: """ - Used by select_download_files. + Used by _select_download_items. Return the dict entries of a config section (e.g. "data" or "meta"), silently dropping any entries that aren't mappings. @@ -191,7 +191,7 @@ def _row_checksum(row: Union[pd.Series, Mapping[str, object]] def _entry_checksum(entry: dict) -> Optional[ChecksumSpec]: """ - Used by select_download_files. + Used by _select_download_items. Extract either legacy or builder checksum metadata from a config entry. Args: @@ -251,7 +251,7 @@ def _checksum_identity(expected_checksum: Optional[ChecksumSpec] def _validate_checksum_spec(expected_checksum: Optional[ChecksumSpec] ) -> Optional[tuple[str, str]]: """ - Used by _download_file. + Used by _download_file and _fetch_file. Validate and normalize a checksum specification. Args: @@ -287,7 +287,7 @@ def _merge_selected_file( expected_checksum: Optional[ChecksumSpec] ) -> None: """ - Used by add_file and download_remote_data. + Used by add_file and _download_selected. Merge a selected file into the dictionary of selected files, ensuring no conflicting checksums exist. @@ -335,7 +335,7 @@ def _merge_selected_file( def _safe_remote_path(value: Union[str, Path]) -> Path: """ - Used by select_download_files and download_remote_data. + Used by _select_download_items and _download_selected. Validate and return a safe relative Path object for a remote file path. Args: value: The input path to validate. @@ -350,13 +350,22 @@ def _safe_remote_path(value: Union[str, Path]) -> Path: def _remote_file_url(remote_url: str, relative_path: Path) -> str: - """Append a literal catalog path to a supported remote root.""" + """ + Construct a file URL from a data remote and a literal catalog path. + + Args: + remote_url (str): URL of the dataset root. + relative_path (Path): File path relative to the dataset root. + + Returns: + str: URL with the relative path encoded for the transport. + """ return RemoteSpec.parse(remote_url).file_url(relative_path.as_posix()) def _download_tsv_name(value) -> str: """ - Used by select_download_files. + Used by _select_download_items. Normalize a TSV name and expose validation failures as DownloadError. Args: @@ -376,7 +385,7 @@ def _download_tsv_name(value) -> str: def _select_remote_config(repo, remote_name: Optional[str] = None) -> Optional[dict]: """ - Used by download_remote_data. + Used by download_remote_data and plan_download. Select a remote configuration from the repository. Args: @@ -502,7 +511,7 @@ def _verify_validated_checksum( chunk_size: int, ) -> None: """ - Used by _download_file. + Used by _download_file and _fetch_file. Verify a checksum that has already been normalized. Args: @@ -526,6 +535,23 @@ def _verify_validated_checksum( def _fetch_file(context, relative_path, destination, expected_checksum, chunk_size): + """ + Download and verify a file before replacing its destination. + + Args: + context (OperationContext): Transport and destination for this download. + relative_path (Path): Literal path relative to the dataset root. + destination (Path): Destination used for error reporting before + resolving the path against the context's output root. + expected_checksum: Optional digest or (algorithm, digest) pair. + chunk_size (int): Bytes per read when downloading or verifying. + + Returns: + int: Number of downloaded bytes. + + Raises: + DownloadError: If the transfer, checksum, or destination is invalid. + """ validated_checksum = _validate_checksum_spec(expected_checksum) try: context.check_cancelled() @@ -537,8 +563,8 @@ def _fetch_file(context, relative_path, destination, expected_checksum, chunk_si chunk_size=chunk_size) _verify_validated_checksum(temp_path, validated_checksum, chunk_size) size = temp_path.stat().st_size - # Recheck after transfer as well as before creation. This protects - # against observed symlink swaps, not a concurrently hostile writer. + # Recheck for symlink changes before replacing the destination. + # Another process can still change the path after this check. resolve_contained_path(context.output_root, relative_path, label="download destination") context.check_cancelled() @@ -553,7 +579,23 @@ def _download_file( expected_checksum: Optional[ChecksumSpec] = None, chunk_size: int = DOWNLOAD_CHUNK_SIZE, ) -> int: - """Standalone compatibility entry point for downloading one known URL.""" + """ + Download one URL and verify its checksum before replacing the destination. + + This compatibility helper creates its own transport context. + + Args: + url (str): Full URL of the remote file. + destination (Path): Local path, which may use a different filename. + expected_checksum: Optional digest or (algorithm, digest) pair. + chunk_size (int): Bytes per read. Defaults to DOWNLOAD_CHUNK_SIZE. + + Returns: + int: Number of downloaded bytes. + + Raises: + DownloadError: If the URL, transfer, checksum, or destination is invalid. + """ chunk_size = _require_positive_integer(chunk_size, label="chunk_size") _validate_checksum_spec(expected_checksum) RemoteSpec.parse(url) @@ -600,7 +642,7 @@ def _select_download_items( all_files: If True, include all files from the repository's configuration. Returns: - Catalog items with their relative paths, checksums, and available metadata. + list[DownloadItem]: Relative paths, checksums, and available file metadata. """ # Get the repository configuration, defaulting to an empty dictionary if not found. config = _repository_config(repo) @@ -617,7 +659,7 @@ def add_file( expected_checksum: Optional[ChecksumSpec] = None, row=None, ) -> None: - """Add file to the selected files dictionary, ensuring it is safe and valid.""" + """Add a file and its catalog metadata, checking for conflicting records.""" # Resolve the input value to a safe relative path relative_path = _safe_remote_path(value) # Merge the selected file into the dictionary of selected files, ensuring no @@ -639,7 +681,7 @@ def add_file( def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], *, explicit_only=False) -> None: - """ Add every file represented by a manifest DataFrame.""" + """Add matching catalog rows, retaining their checksums and file metadata.""" # if the DataFrame is None or empty, return early without adding any files if frame is None or frame.empty: return @@ -770,7 +812,22 @@ def select_download_files( tsv_names: Sequence[str] = (), all_files: bool = False, ) -> list[tuple[Path, Optional[ChecksumSpec]]]: - """Select paths and checksums, including catalog checksums for explicit paths.""" + """ + Select remote paths and their available catalog checksums. + + Args: + repo: The hallmark repository object. + file_paths (sequence[str]): Explicit remote-relative paths. + tsv_names (sequence[str]): Catalog TSVs to select. + all_files (bool): Include all configured files. Defaults to False. + + Returns: + list[tuple]: Pairs of relative paths and optional checksums. + Explicit paths use catalog checksums when available. + + Raises: + DownloadError: If catalog paths or checksum records are invalid. + """ return [(item.relative_path, item.checksum) for item in _select_download_items( repo, file_paths=file_paths, tsv_names=tsv_names, all_files=all_files)] @@ -787,10 +844,34 @@ def plan_download( remote_name: Optional[str] = None, estimated_bytes_per_second: Optional[float] = None, ) -> DownloadPlan: - """Plan a transfer using local catalog metadata, without contacting a server. + """ + Plan a download using local catalog metadata. + + No network requests are made. With no explicit paths or TSVs, select + the complete catalog before applying filters. Missing sizes remain unknown. + + Args: + repo: The hallmark repository object. + output_path (Path | str, optional): Destination directory. Defaults + to the worktree; required for a bare repository. + file_paths (sequence[str], optional): Remote-relative file paths. + tsv_names (sequence[str], optional): Catalog TSVs to select. + all_files (bool): Select all configured files. Cannot be combined + with explicit paths or TSVs. Defaults to False. + filter (str | list[str], optional): Relative path glob or globs. + fmt (str, optional): Filename format that selected paths must match. + remote_name (str, optional): Configured data remote to use. + estimated_bytes_per_second (float, optional): Positive rate used + to estimate duration when every file size is known. + + Returns: + DownloadPlan: Immutable files, source, destination, and metadata + for review before approval. - With no explicit path or TSV selection, include the complete catalog before - applying the optional path filter and format. Unknown sizes stay unknown. + Raises: + DownloadError: If the catalog, remote, selection, or destination + is invalid. + ValueError: If a filter, format, or supplied rate is invalid. """ if output_path is None: output_path = getattr(repo, "worktree", None) @@ -839,7 +920,26 @@ def execute_download_plan( max_workers: int = 4, show_progress: bool = False, ) -> dict: - """Execute the approved plan's source and selection, ignoring later config edits.""" + """ + Execute a download plan using its recorded source and destination. + + Args: + repo: Repository associated with the plan. Current configuration + is not used to change the planned transfer. + plan (DownloadPlan): Files and destination presented for approval. + approved (bool): Must be True for a nonempty plan. Defaults to False. + max_workers (int): Maximum concurrent download workers. Defaults to 4. + show_progress (bool): Show byte progress. Defaults to False. + + Returns: + dict: Results with ``succeeded``, ``failed``, ``total_bytes``, and + ``errors`` keys. Individual transfer failures are recorded here. + + Raises: + TypeError: If ``plan`` is not a DownloadPlan. + DownloadError: If approval is missing, setup fails, or the + destination is invalid. + """ if not isinstance(plan, DownloadPlan): raise TypeError("plan must be a DownloadPlan") if plan.items and approved is not True: @@ -872,19 +972,22 @@ def download_remote_data( Args: repo: The hallmark repository object. - worktree_path: The path to the working tree where files will be downloaded. - max_workers: The maximum number of concurrent download threads. - show_progress: Whether to display a progress bar. - selected_files: A sequence of tuples containing the relative path and optional - checksum of files to download. - remote_name: The name of the remote configuration to use. - approved: Explicit approval to transfer the selected dataset files. + worktree_path (Path): Directory where files will be downloaded. + max_workers (int): Maximum concurrent download workers. Defaults to 4. + show_progress (bool): Show progress by file count. Defaults to False. + selected_files (sequence[tuple], optional): Relative paths paired + with optional checksums. If omitted, use ``select_download_files``. + remote_name (str, optional): Configured data remote to use. + approved (bool): Must be True for a nonempty transfer. Defaults to False. Returns: - A dict with the download results (succeeded, failed, total_bytes, and errors) + dict: Results with ``succeeded``, ``failed``, ``total_bytes``, and + ``errors`` keys. Individual transfer failures are recorded here; + successful downloads remain available. Raises: - DownloadError: If the remote URL is not configured or if any download fails. + DownloadError: If approval is missing, configuration is invalid, + or connection setup fails before transfers begin. """ # Ensure that max_workers is a positive integer for concurrent downloads. max_workers = _require_positive_integer(max_workers, label="max_workers") @@ -940,7 +1043,7 @@ def _download_selected( byte_progress=False, total_bytes=None, ): - """Run one approved transfer with operation-owned resources and atomic output.""" + """Download selected files with shared resources and atomic file replacement.""" results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} # output_root is the resolved absolute path where files will be downloaded @@ -980,7 +1083,7 @@ def _download_selected( files_to_download.append((relative_path, destination, expected_checksum)) - # track the download progress using tqdm, with the number of files to download + # track byte totals for plans and file counts for the legacy API progress = tqdm( total=total_bytes if byte_progress else len(files_to_download), unit="B" if byte_progress else "file", @@ -1047,7 +1150,7 @@ def submit_next() -> bool: except DownloadError as exc: results["failed"] += 1 results["errors"].append(str(exc)) - # Every completed transfer advances the progress bar. + # report completed files even when a transfer fails finally: if byte_progress: with progress_lock: diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index ce28422..666b4b6 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -214,13 +214,45 @@ def clone( fetch_data: Optional[bool] = None, show_progress: Optional[bool] = None, ) -> "Repo": - """Clone catalog metadata, discovering ordinary remote directories as needed. + """ + Clone a Hallmark catalog or index a remote data directory. + + Git sources retain their history. Published catalog snapshots and + discovered directories start new local history. Dataset files are + downloaded only when requested and approved after catalog creation. + Declining approval leaves the catalog available without dataset files. + + Args: + url (str): Catalog location or URL of the dataset directory. + path (Path | str): New worktree or bare ``.hm`` repository path. + auth (str, optional): Local SSH profile for data access. Git + sources use Git's authentication configuration instead. + filter (str | list[str], optional): Relative path glob or globs. + A path must match at least one glob when provided. + fmt (str, optional): Filename format used to select paths and + extract parameters when discovering a directory. + source_type (str): ``auto``, ``git``, ``directory``, or ``catalog``. + Defaults to automatic source detection. + progress (bool): Show discovery and download progress. Defaults + to False. + download (bool): Request downloads after preparing the catalog. + Defaults to False and requires an approval callback when True. + approve (callable, optional): Called with the completed download + plan. Return the Boolean True to approve the transfer. + max_workers (int): Maximum download workers. Defaults to 4. + fetch_data (bool, optional): Compatibility alias for ``download``. + show_progress (bool, optional): Compatibility alias for ``progress``. - ``filter`` is a relative path glob; ``fmt`` is a filename template. - Dataset files are transferred only with ``download=True`` and an - ``approve(plan)`` callback returning True after inspecting the plan. - ``source_type`` can explicitly select git, directory, or catalog sources. - The legacy ``fetch_data`` and ``show_progress`` names remain aliases. + Returns: + Repo: The cloned repository. ``download_result`` contains results + when an approved, nonempty download was performed. + + Raises: + DestinationExistsError: If the destination already exists. + CloneError: If a requested catalog is missing or invalid. + ValueError: If source options are invalid or conflict. + DownloadError: If discovery or downloading fails, or a download + is requested without a callback or worktree destination. """ from .catalog import clone_catalog from .downloader import DownloadError, _require_positive_integer @@ -255,7 +287,35 @@ def clone( def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, all_files=False, filter=None, fmt=None, remote_name=None, estimated_bytes_per_second=None): - """Inspect a transfer using local catalog metadata without network access.""" + """ + Plan a download using the local catalog without contacting a server. + + With no explicit paths or TSVs, select the complete catalog before + applying any filter or filename format. Planning does not approve + the transfer. + + Args: + output_path (Path | str, optional): Destination directory. Defaults + to the worktree; required for a bare repository. + file_paths (sequence[str], optional): Remote-relative file paths. + tsv_names (sequence[str], optional): Catalog TSVs to select. + all_files (bool): Select all configured files. Cannot be combined + with explicit paths or TSVs. Defaults to False. + filter (str | list[str], optional): Relative path glob or globs. + fmt (str, optional): Filename format that selected paths must match. + remote_name (str, optional): Configured data remote to use. + estimated_bytes_per_second (float, optional): Positive transfer + rate for duration estimates. No rate is measured while planning. + + Returns: + DownloadPlan: Immutable selection with its source, destination, + checksums, and available file metadata. + + Raises: + DownloadError: If the catalog, remote, selection, or destination + is invalid. + ValueError: If a filter, format, or supplied rate is invalid. + """ from .downloader import plan_download return plan_download( @@ -264,7 +324,30 @@ def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, estimated_bytes_per_second=estimated_bytes_per_second) def download(self, plan, *, approved=False, max_workers=4, progress=False): - """Execute an inspected DownloadPlan only with explicit approval.""" + """ + Download the files in an approved plan. + + The plan fixes the source and destination even if repository settings + change. Successful files remain available when another transfer fails. + + Args: + plan (DownloadPlan): Plan returned by ``plan_download``. + approved (bool): Must be True for a nonempty transfer. Defaults + to False; an empty plan requires no approval. + max_workers (int): Maximum concurrent download workers. Defaults + to 4; the local SSH session limit may reduce concurrency. + progress (bool): Show byte progress. Defaults to False. + + Returns: + dict: Results with ``succeeded``, ``failed``, ``total_bytes``, and + ``errors`` keys. Also stored in ``download_result``. Individual + transfer failures are recorded in ``failed`` and ``errors``. + + Raises: + TypeError: If ``plan`` is not a DownloadPlan. + DownloadError: If approval is missing, setup fails, or the + destination is invalid. + """ from .downloader import execute_download_plan result = execute_download_plan( @@ -332,7 +415,8 @@ def set_config( fmt (str, optional): Data format specification. remote_name (str, optional): Name of the remote repository. remote_url (str, optional): URL of the remote repository. - remote_auth (str, optional): Local profile name; empty string removes it. + remote_auth (str, optional): Local SSH profile name. An empty string + removes the reference; None leaves it unchanged. encoding_updates (dict[str, str], optional): Updates to encoding rules. Returns: diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index 78be871..e04ada6 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -54,7 +54,7 @@ def _remote_url(base_url: str, relative_path: str) -> str: - """Render a transport URL from a literal path, preserving directory slashes.""" + """Construct a remote URL, preserving literal paths and directory slashes.""" url = RemoteSpec.parse(base_url).file_url(relative_path) if relative_path.endswith("/") and not url.endswith("/"): url += "/" @@ -378,7 +378,7 @@ def solve(segment_idx, path_idx): def _match_static_named_file_plain_only(rel_path: str, fmt_entries: list[dict] ) -> tuple[int, dict] | tuple[None, None]: """ - Used by build_repo. + Used by _build_repo. Match a static named file against a list of fmt entries using only the plain parse. Args: @@ -403,7 +403,7 @@ def _match_static_named_file_plain_only(rel_path: str, fmt_entries: list[dict] def _match_file_against_fmts(rel_path: str, fmt_entries: list[dict]): """ - Used by build_repo. + Used by _build_repo. Find the best fmt entry for one file, across all of them at once. Args: @@ -483,6 +483,7 @@ def _match_file_against_fmts(rel_path: str, fmt_entries: list[dict]): return best_index, best_parse def _record_checksum(checksums, path, algorithm, checksum): + """Keep the strongest checksum and reject conflicting digests.""" previous_algorithm, previous = checksums.get(path, (None, None)) if previous is not None and previous_algorithm == algorithm: if previous.lower() != checksum.lower(): @@ -496,7 +497,6 @@ def _record_checksum(checksums, path, algorithm, checksum): def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: """ - Used by list_remote_files. Parse a manifest text and return a list of (checksum, filename) tuples for each line in the manifest that has a valid checksum for the given algorithm. @@ -520,7 +520,6 @@ def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: def _resolve_manifest_path(filename: str, rel_dir: str) -> str: """ - Used by list_remote_files. Resolve a manifest path relative to a given directory, ensuring it is valid. Args: @@ -579,7 +578,6 @@ def _resolve_manifest_path(filename: str, rel_dir: str) -> str: def _normalize_index_href(href: str, *, is_directory: bool) -> str: """ - Used by list_remote_files. Normalize an index href to ensure it is a valid relative path. Args: @@ -622,7 +620,7 @@ def _normalize_index_href(href: str, *, is_directory: bool) -> str: def _normalize_fmt_entries(fmt_entries: list[dict]) -> list[dict]: """ - Used by build_repo. + Used by _build_repo. Normalize the fmt entries to ensure they are valid and consistent. Args: @@ -681,7 +679,22 @@ def _normalize_fmt_entries(fmt_entries: list[dict]) -> list[dict]: def list_remote_files(base_url: str, *, _context=None): - """Return a checksum inventory from an automatically detected remote listing.""" + """ + List remote files and their published checksums. + + Args: + base_url (str): Exact URL root to search recursively. + _context (OperationContext, optional): Existing internal transport + context. Its remote is used when supplied. + + Returns: + dict: Relative paths mapped to (algorithm, checksum) pairs. + Missing checksum values remain None. + + Raises: + DownloadError: If discovery fails or checksum manifests conflict. + CapabilityError: If the source provides no supported listing. + """ from .discovery import discover def inventory(context): @@ -708,11 +721,44 @@ def build_repo( allow_remote_commands: bool = False, remote_hash: bool = False, ) -> "Repo": - """Compatibility wrapper for legacy catalog construction. + """ + Build a remote catalog using the deprecated builder interface. Prefer ``Repo.clone(url, path)`` for remote catalogs and ``Repo.init(path)`` - for local repositories. Discovery reads listings and published manifests only. - Missing formats default to a generic path catalog without prompting. + for local repositories. Discovery reads listings and published checksum + manifests without downloading dataset files. + + Args: + repo_path (Path): Destination repository path. + dataset_name (str): Dataset name recorded in metadata and used for + the default CyVerse URL. + fmt_entries (list[dict], optional): Filename formats and TSV names. + If omitted, reuse existing formats or create a generic path catalog. + config_file (Path | str, optional): Configuration file or repository + directory supplying formats and, unless overridden, data remotes. + Cannot be combined with ``fmt_entries``. + remotes (list | dict | str, optional): Data remotes to record. + These do not change the discovery root. + overwrite (bool): Allow replacement of an existing destination. + Defaults to False; existing formats may be reused without it. + dataset_url (str, optional): Exact discovery root. Defaults to the + CyVerse curated-data directory for ``dataset_name``. + dataset_auth (str, optional): Local SSH profile for discovery. + index_format (str, optional): Obsolete listing option. None, + ``auto``, and ``cyverse-html`` are accepted; detection is automatic. + allow_remote_commands (bool): Obsolete option, ignored when supplied. + remote_hash (bool): Unsupported when True. Defaults to False; + discovery does not compute checksums from dataset contents. + + Returns: + Repo: Repository containing the prepared catalog. + + Raises: + ValueError: If the dataset name or catalog configuration is invalid. + FileExistsError: If the destination exists and cannot be reused. + FileNotFoundError: If the supplied configuration file is missing. + CapabilityError: If remote hashing or an unsupported index is requested. + DownloadError: If remote discovery fails. """ warnings.warn( "build_repo is deprecated; use Repo.clone(url, path) or Repo.init(path)", @@ -746,23 +792,27 @@ def _build_repo( source=None, ) -> "Repo": """ - Build a hallmark repository from the given dataset and format entries. - Creates separate TSV files for each fmt, and one config.yml file with the manifest. + Build a catalog using a resolved data source. Args: - repo_path: Path where the hallmark repo will be created. - fmt_entries: dict entries with "fmt", "db", and optional "name" keys. - If None, preserve existing formats or create a generic path catalog. - config_file: Path to an existing config.yml file to load fmt entries from. - remotes: Optional list of remote repos to add to the repo. Each remote can be a - dict with "name" and "url" keys, or a str representing the name of the remote. + repo_path (Path): Destination repository path. + dataset_name (str): Dataset name recorded in metadata. + fmt_entries (list[dict], optional): Entries with ``fmt``, ``db``, + and optional ``name`` keys. Reuse existing formats or a generic + path format when omitted. + config_file (Path | str, optional): Configuration file or directory + supplying formats and data remotes. + remotes (list | dict | str, optional): Data remotes to record. + overwrite (bool): Allow replacement of the destination. Defaults + to False, except when reusing its existing catalog formats. + source (OperationContext): Resolved source supplied by ``build_repo``. Returns: - The initialized Repo object, sitting on the main branch. + Repo: Repository containing the prepared catalog. """ # validate the dataset name to ensure it is a valid path component dataset_name = validate_path_component(dataset_name, label="dataset name") - # The wrapper has already resolved the exact URL, including the legacy default. + # build_repo resolves the source URL before calling this helper base_url = source.remote.url # Determine if remotes were provided by the user @@ -816,9 +866,9 @@ def _build_repo( remote_files = None def _ensure_remote_files_listed(): """ - Used by build_repo. - Ensure that the remote files have been listed and checksums collected. - Defer network requests until destination and configuration validation pass. + List remote files after validating the destination and configuration. + + Reuse the inventory on subsequent calls within this build. """ # create nonlocal references to the outer variables so they can be modified nonlocal file_checksums, remote_files @@ -835,7 +885,7 @@ def _ensure_remote_files_listed(): # sort the remote files by their relative paths for consistent ordering remote_files = sorted(file_checksums) - # Keep existing catalog formats when refreshing through the legacy wrapper. + # preserve existing formats when refreshing through the legacy builder reused_from_existing_repo = fmt_entries is None and bool(existing_fmt_entries) if reused_from_existing_repo: fmt_entries = existing_fmt_entries diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index b5142c0..799286d 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -27,12 +27,14 @@ def _update_remote_config( ) -> None: """ Used by set_config. - Update the repository configuration with a new remote name and/or URL. + Update the repository data remote and its local profile reference. Args: config (dict): The repository configuration dictionary. remote_name (str, optional): New remote repository name. remote_url (str, optional): New remote repository URL. + remote_auth (str, optional): Local SSH profile name. An empty string + removes the reference; None leaves it unchanged. Raises: ValueError: If the remote configuration is invalid or if the specified @@ -171,7 +173,7 @@ def normalize_remotes(remotes) -> list[dict]: profile_name(entry["auth"]) if isinstance(entry.get("url"), str): reject_controls(entry["url"], "Remote URL") - # Names without URLs are allowed while configuring/building a catalog. + # allow names without URLs while configuring a data remote RemoteSpec.parse(entry["url"], entry.get("auth")) # for each required key ("name" and "url"), validate that it exists for key in ("name", "url"): @@ -351,7 +353,8 @@ def set_config( fmt (str, optional): Filename format. remote_name (str, optional): Remote repository name. remote_url (str, optional): Remote repository URL. - remote_auth (str, optional): Local SSH profile; empty string removes it. + remote_auth (str, optional): Local SSH profile name. An empty string + removes the reference; None leaves it unchanged. encoding_updates (dict, optional): Encoding values to merge into the existing configuration. @@ -433,7 +436,7 @@ def set_config( # update the first entry in the "data" list of the configuration config["data"][0] = updated_spec - # if a new remote name or URL is provided, update the remote configuration + # update the remote when its name, URL, or profile reference changes if remote_name is not None or remote_url is not None or remote_auth is not None: _update_remote_config(config, remote_name, remote_url, remote_auth) diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py index 9799bcf..029331a 100644 --- a/mod/hallmark/transport/__init__.py +++ b/mod/hallmark/transport/__init__.py @@ -1,4 +1,4 @@ -"""Explicit internal dispatch and per-invocation resource ownership.""" +"""Manage transport selection and resources for each remote operation.""" from __future__ import annotations @@ -14,7 +14,16 @@ class OperationContext: - """Own sessions, cancellation and one transport for a download or build.""" + """ + Manage transport resources for one discovery or download operation. + + Leaving the context closes its transport and HTTP sessions. An exception + also cancels active work. + + Args: + remote (RemoteSpec): Data source and optional local profile name. + output_root (Path | str, optional): Root used to validate download paths. + """ def __init__( self, @@ -42,7 +51,16 @@ def __init__( @contextmanager def executor(self, max_workers): - """Cancel active transport work before waiting for interrupted workers.""" + """ + Create a worker pool whose tasks share this operation's resources. + + Args: + max_workers (int): Maximum concurrent workers. + + Yields: + ThreadPoolExecutor: Worker pool. Interrupted work is cancelled + before waiting for workers to finish. + """ executor = ThreadPoolExecutor(max_workers=max_workers) try: yield executor @@ -53,6 +71,7 @@ def executor(self, max_workers): executor.shutdown(wait=True, cancel_futures=True) def session(self): + """Return the reusable HTTP session for the current thread.""" if not hasattr(self._local, "session"): session = requests.Session() with self._lock: @@ -61,13 +80,16 @@ def session(self): return self._local.session def read_text(self, path): + """Read remote metadata within the configured size limit.""" return self.transport.read_text(path, self.text_limit) def check_cancelled(self): + """Raise TransferCancelled if this operation was cancelled.""" if self.cancelled.is_set(): raise TransferCancelled("Download cancelled") def cancel(self): + """Cancel this operation and stop its active transport work.""" self.cancelled.set() cancel = getattr(self.transport, "cancel", None) if cancel is not None: diff --git a/mod/hallmark/transport/auth.py b/mod/hallmark/transport/auth.py index c01fd1c..eac3dff 100644 --- a/mod/hallmark/transport/auth.py +++ b/mod/hallmark/transport/auth.py @@ -1,4 +1,4 @@ -"""Strict local SSH settings; repository configuration stores references only.""" +"""Read local SSH settings for profile names stored in repository configuration.""" from __future__ import annotations @@ -18,6 +18,13 @@ @dataclass(frozen=True) class SSHSettings: + """ + Local SSH settings shared by the connections in one operation. + + Timeouts are in seconds. ``transfer_timeout`` limits the total time + for each file, and ``max_sessions`` limits concurrent SFTP sessions. + Unset user, port, and identity values fall back to SSH configuration. + """ user: str | None = None port: int | None = None identity_file: str | None = None @@ -29,6 +36,7 @@ class SSHSettings: def _settings(values): + """Validate local SSH options and normalize identity-file paths.""" allowed = {item.name for item in fields(SSHSettings)} if not isinstance(values, dict) or set(values) - allowed: raise RemoteConfigurationError("Unsupported local SSH settings fields") @@ -64,7 +72,22 @@ def _settings(values): def resolve_settings(remote): - """Resolve settings once, before any worker or connection starts.""" + """ + Resolve local SSH settings before opening a connection. + + URL user and port values override a host-bound profile, which overrides + global timeout and concurrency defaults. Remaining values use SSH config. + + Args: + remote (RemoteSpec): Source URL and optional local profile name. + + Returns: + SSHSettings: Validated settings, or defaults for an HTTP source. + + Raises: + RemoteConfigurationError: If the auth file, settings, or host binding + is invalid, or a referenced profile is missing. + """ settings = SSHSettings() if remote.scheme not in {"ssh", "sftp"}: return settings @@ -92,7 +115,7 @@ def resolve_settings(remote): ): raise RemoteConfigurationError("Auth file must use version: 1") defaults = _settings(document.get("defaults", {})) - # Endpoint/identity defaults must always be host-bound through a profile. + # User, port, and identity settings require a profile bound to specific hosts. if set(defaults) & {"user", "port", "identity_file"}: raise RemoteConfigurationError("Global SSH defaults cannot select an identity") settings = replace(settings, **defaults) @@ -123,7 +146,7 @@ def resolve_settings(remote): f"Auth profile {remote.auth!r} is not bound to {remote.host}" ) settings = replace(settings, **values) - # Profile user/port are defaults; hosts are endpoint constraints. + # URL user and port override profile defaults after checking the host binding. return replace( settings, user=remote.user or settings.user, port=remote.port or settings.port ) diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py index 4a36c6c..179ff83 100644 --- a/mod/hallmark/transport/base.py +++ b/mod/hallmark/transport/base.py @@ -1,4 +1,4 @@ -"""Internal transport contracts and validated data-remote locators.""" +"""Shared transport interfaces and data-remote URL validation.""" from __future__ import annotations @@ -13,26 +13,27 @@ class DownloadError(HallmarkError): - """A remote transfer or its verification failed.""" + """Raised when download setup, transfer, or verification fails.""" class RemoteConfigurationError(DownloadError, ValueError): - """A remote cannot be used with the selected local policy.""" + """Raised when a data remote or its local settings are invalid.""" class RemoteObjectMissing(DownloadError): - """The transport positively identified an absent object.""" + """Raised when the server reports that a remote file is missing.""" class CapabilityError(DownloadError): - """The selected source does not support this operation.""" + """Raised when the source or client does not support an operation.""" class TransferCancelled(DownloadError): - """The owning operation cancelled a transfer.""" + """Raised when a transfer is cancelled.""" def reject_controls(value: str, label: str) -> None: + """Reject control characters and text that cannot be encoded as UTF-8.""" try: value.encode("utf-8") except UnicodeError: @@ -54,6 +55,7 @@ def literal_path(value) -> Path: def profile_name(value: str) -> str: + """Validate and return a local authentication profile name.""" if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", value): raise RemoteConfigurationError("Auth profile must match [A-Za-z0-9_-]{1,64}") return value @@ -62,6 +64,7 @@ def profile_name(value: str) -> str: def ssh_host(value: str) -> str: # Aliases are not necessarily DNS names. Limit expansion tokens to characters # that cannot become shell syntax in a user's ProxyCommand/Match configuration. + """Validate an SSH hostname, configuration alias, or IPv6 address.""" if ":" in value: if "%" in value: raise RemoteConfigurationError("Scoped IPv6 SSH hosts are unsupported") @@ -75,6 +78,7 @@ def ssh_host(value: str) -> str: def ssh_user(value: str) -> str: + """Validate and return an SSH username.""" if not isinstance(value, str) or not re.fullmatch( r"[A-Za-z0-9_][A-Za-z0-9_.-]*", value ): @@ -84,7 +88,18 @@ def ssh_user(value: str) -> str: @dataclass(frozen=True) class RemoteEntry: - """A discovered file; absent attributes remain unknown without reading it.""" + """ + A discovered file and its available metadata. + + Attributes: + path (str): Literal path relative to the dataset root. + size (int, optional): File size in bytes. + mtime (float, optional): Modification time reported by the server. + checksum_algorithm (str, optional): Published checksum algorithm. + checksum (str, optional): Published digest. + + Unavailable metadata remains None; file contents are not read to fill it. + """ path: str size: int | None = None @@ -95,7 +110,21 @@ class RemoteEntry: @dataclass(frozen=True) class RemoteSpec: - """A parsed URL root; catalog paths appended to it are always literal.""" + """ + A parsed data-remote URL and optional local authentication profile. + + Use ``parse`` to validate a URL. SSH roots are decoded once; catalog + paths appended to the root are treated as literal filenames. + + Attributes: + url (str): Original source URL after trimming surrounding whitespace. + scheme (str): HTTP, HTTPS, SSH, or SFTP scheme in lowercase. + host (str): Hostname or SSH configuration alias. + root (str): Dataset root, decoded for SSH and SFTP. + user (str, optional): Explicit SSH username. + port (int, optional): Explicit port number. + auth (str, optional): Local SSH authentication profile name. + """ url: str = field(repr=False) scheme: str @@ -107,6 +136,19 @@ class RemoteSpec: @classmethod def parse(cls, url: str, auth: str | None = None) -> RemoteSpec: + """ + Parse and validate a data-remote URL. + + Args: + url (str): HTTP(S) URL or SSH/SFTP URL with an absolute dataset root. + auth (str, optional): Local SSH profile name. Unsupported for HTTP(S). + + Returns: + RemoteSpec: Validated source description. + + Raises: + RemoteConfigurationError: If the URL or profile reference is invalid. + """ if not isinstance(url, str) or not url.strip(): raise RemoteConfigurationError("Remote URL must be a non-empty string") reject_controls(url, "Remote URL") @@ -166,9 +208,11 @@ def parse(cls, url: str, auth: str | None = None) -> RemoteSpec: return cls(url, scheme, host, root, user, port, auth) def pathname(self, relative_path: str) -> str: + """Append a literal relative path to the dataset root.""" return self.root.rstrip("/") + "/" + literal_path(relative_path).as_posix() def file_url(self, relative_path: str) -> str: + """Encode a literal catalog path as a URL beneath the dataset root.""" path = literal_path(relative_path).as_posix() if relative_path else "" if relative_path.endswith("/") and path: path += "/" @@ -180,31 +224,72 @@ def file_url(self, relative_path: str) -> str: @property def display(self) -> str: - """Safe endpoint context without URL credentials, query, or local paths.""" + """Source scheme and host, with credentials and paths omitted.""" host = self.host if re.fullmatch(r"[A-Za-z0-9_.:-]+", self.host) else "remote" return f"{self.scheme}://{host}" class Transport: - """Internal behavior shared by transports; no configuration-driven imports.""" + """ + Common interface for internal data transports. + + Args: + context (OperationContext): Resources and settings for this operation. + """ def __init__(self, context): self.context = context def prepare(self): + """Check connection requirements before starting transfers.""" pass def fetch(self, relative_path, destination, *, chunk_size=8192): + """ + Transfer a file to a caller-managed temporary destination. + + Args: + relative_path (str): Literal path relative to the dataset root. + destination (Path): Local file to write. + chunk_size (int): Read size for transports that support it. + + Checksum verification and atomic replacement are handled by the downloader. + """ raise NotImplementedError def list_entries(self): + """Return the relative paths from a recursive directory listing.""" return [entry.path for entry in self.iter_entries()] def iter_entries(self, on_directory=None): + """ + Yield file metadata from a recursive directory listing. + + Args: + on_directory (callable, optional): Callback receiving directory paths + relative to the dataset root. + + Raises: + CapabilityError: If the transport does not provide directory listings. + """ raise CapabilityError("Recursive listing is unsupported by this transport") def read_text(self, relative_path, limit): + """ + Read a remote metadata file within a byte limit. + + Args: + relative_path (str): Metadata path relative to the dataset root. + limit (int): Maximum number of bytes to read. + + Returns: + str: Decoded metadata text. + + Raises: + CapabilityError: If the transport does not support metadata reads. + """ raise CapabilityError("Reading metadata is unsupported by this transport") def close(self): + """Release connections and processes used by this transport.""" pass diff --git a/mod/hallmark/transport/http.py b/mod/hallmark/transport/http.py index 45882f0..14f3fed 100644 --- a/mod/hallmark/transport/http.py +++ b/mod/hallmark/transport/http.py @@ -1,4 +1,4 @@ -"""Requests transport, including its default .netrc/environment behavior.""" +"""HTTP transport using Requests with its normal .netrc and environment settings.""" from __future__ import annotations @@ -11,11 +11,13 @@ class HttpTransport(Transport): + """Transfer files and metadata using reusable Requests sessions.""" def __init__(self, context): super().__init__(context) self.text_urls = {} def _get(self, path): + """Open a streaming response with the configured request timeout.""" return self.context.session().get( getattr(self, "direct_url", None) or self.context.remote.file_url(path), stream=True, @@ -24,6 +26,7 @@ def _get(self, path): def _error(self, exc): # Requests exceptions can embed credentials in their URL or response body. + """Describe an HTTP failure without exposing response contents.""" status = getattr(getattr(exc, "response", None), "status_code", None) cls = RemoteObjectMissing if status == 404 else DownloadError detail = f"HTTP {status}" if isinstance(status, int) else type(exc).__name__ @@ -69,6 +72,7 @@ def endpoint(parsed): raise DownloadError("Metadata redirect limit exceeded") def fetch(self, relative_path, destination, *, chunk_size=8192): + """Stream a file to the destination, checking for cancellation.""" try: with self._get(relative_path) as response: response.raise_for_status() @@ -83,6 +87,7 @@ def fetch(self, relative_path, destination, *, chunk_size=8192): raise self._error(exc) from None def read_text(self, relative_path, limit): + """Read metadata within the source root and byte limit.""" content = bytearray() try: response, final_url = self._metadata_get(relative_path) diff --git a/mod/hallmark/transport/sftp.py b/mod/hallmark/transport/sftp.py index 67acc0e..fae1ae6 100644 --- a/mod/hallmark/transport/sftp.py +++ b/mod/hallmark/transport/sftp.py @@ -1,7 +1,7 @@ -"""Read-only SFTP v3 metadata over an operation-owned OpenSSH subsystem. +"""Read SFTP v3 metadata through an OpenSSH subsystem. -OpenSSH owns authentication and encryption. This module only frames the small -metadata protocol, so names and attributes never pass through shell/ls parsing. +OpenSSH handles authentication and encryption. Names and attributes are read +from binary protocol fields without parsing shell commands or ``ls`` output. """ from __future__ import annotations @@ -20,17 +20,20 @@ def _string(value): + """Encode a byte string or UTF-8 text as an SFTP string.""" if isinstance(value, str): value = value.encode("utf-8") return struct.pack(">I", len(value)) + value class _Packet: + """Read and validate fields in an SFTP response packet.""" def __init__(self, data): self.data = data self.offset = 0 def take(self, size): + """Read the requested bytes, rejecting truncated packets.""" end = self.offset + size if end > len(self.data): raise DownloadError("Truncated SFTP metadata packet") @@ -39,22 +42,27 @@ def take(self, size): return result def uint32(self): + """Read an unsigned 32-bit integer in network byte order.""" return struct.unpack(">I", self.take(4))[0] def string(self): + """Read a length-prefixed byte string.""" return self.take(self.uint32()) def text(self): + """Read a UTF-8 filename from a length-prefixed string.""" try: return self.string().decode("utf-8") except UnicodeError: raise DownloadError("SFTP filename must contain valid UTF-8") from None def finish(self): + """Reject unconsumed bytes after the expected fields.""" if self.offset != len(self.data): raise DownloadError("Unexpected trailing SFTP metadata") def attributes(self): + """Read size, mode, and modification time; retain unknowns as None.""" flags = self.uint32() if flags & ~_KNOWN_ATTRIBUTES: raise DownloadError("Unsupported SFTP attribute flags") @@ -77,7 +85,15 @@ def attributes(self): class SftpMetadata: - """One bounded, cancellable request at a time on a binary subsystem pipe.""" + """ + Read SFTP metadata through a dedicated OpenSSH subsystem process. + + Requests run sequentially with size limits, timeouts, and cancellation + checks. Use this object as a context manager to close the process and pipes. + + Args: + transport (SshTransport): Shared SSH connection and session limits. + """ def __init__(self, transport): self.transport = transport @@ -137,14 +153,17 @@ def __exit__(self, exc_type, exc, tb): self._slot = False def _new_deadline(self): + """Start the timeout for one metadata request.""" self.deadline = time.monotonic() + self.context.listing_timeout def _check(self): + """Check cancellation and the current request deadline.""" self.context.check_cancelled() if time.monotonic() >= self.deadline: raise DownloadError("SFTP metadata request exceeded its time limit") def _pump(self): + """Buffer available output and report whether stdin is writable.""" self._check() writable = False for key, events in self.selector.select(timeout=0.05): @@ -168,6 +187,7 @@ def _pump(self): return writable def _send(self, payload): + """Write one size-limited packet while checking cancellation.""" if len(payload) > _MAX_PACKET: raise DownloadError("SFTP metadata request exceeds its size limit") data = memoryview(struct.pack(">I", len(payload)) + payload) @@ -187,6 +207,7 @@ def _send(self, payload): self.selector.unregister(self.process.stdin) def _read(self, size): + """Read the requested bytes within the current deadline.""" while len(self.buffer) < size: self._pump() self.context.check_cancelled() @@ -199,6 +220,7 @@ def _read(self, size): return result def _receive(self): + """Read a response type and its validated packet body.""" length = struct.unpack(">I", self._read(4))[0] if not 1 <= length <= _MAX_PACKET: raise DownloadError("Invalid or oversized SFTP metadata packet") @@ -206,6 +228,7 @@ def _receive(self): return data[0], _Packet(data[1:]) def _request(self, kind, data, expected, *, eof=False): + """Send a request and validate its response identifier and status.""" self._new_deadline() self.request_id = (self.request_id + 1) % (2 ** 32) self._send(bytes([kind]) + struct.pack(">I", self.request_id) + data) @@ -231,6 +254,7 @@ def _request(self, kind, data, expected, *, eof=False): return packet def realpath(self, path): + """Return the canonical path reported by the SFTP server.""" packet = self._request(16, _string(path), 104) if packet.uint32() != 1: raise DownloadError("Invalid SFTP canonical path response") @@ -241,12 +265,26 @@ def realpath(self, path): return result def lstat(self, path): + """Read file attributes without following a final symlink.""" packet = self._request(7, _string(path), 105) result = packet.attributes() packet.finish() return result def iterdir(self, path): + """ + Yield names and attributes from one remote directory. + + Args: + path (str): Absolute directory path on the server. + + Yields: + tuple: Filename and attribute dictionary. Missing sizes, modes, + and modification times are None. + + Raises: + DownloadError: If the directory cannot be read or responses are invalid. + """ packet = self._request(11, _string(path), 102) handle = packet.string() packet.finish() diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index 3dcb801..9ef7d7b 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -1,4 +1,4 @@ -"""Supervised OpenSSH connection sharing and exact-file SFTP transfers.""" +"""Share an OpenSSH connection for SFTP transfers and metadata reads.""" from __future__ import annotations @@ -26,7 +26,18 @@ def batch_argument(path): - """Encode one absolute SFTP argument (command quoting and optional globbing).""" + """ + Escape an absolute path for a literal SFTP batch argument. + + Args: + path (Path | str): Absolute local or remote path. + + Returns: + str: Argument with command and glob characters escaped. + + Raises: + RemoteConfigurationError: If the path is relative or contains controls. + """ text = str(path) reject_controls(text, "SFTP path") if not text.startswith("/"): @@ -38,9 +49,17 @@ def batch_argument(path): class SshTransport(Transport): + """ + Share one OpenSSH connection for SFTP transfers and metadata requests. + + Args: + context (OperationContext): Local settings and cancellation state. + ssh (list[str], optional): Internal command override for offline tests. + sftp (list[str], optional): Internal command override for offline tests. + """ def __init__(self, context, *, ssh=None, sftp=None): super().__init__(context) - # Injection is internal, for offline process tests; never from repo YAML. + # Command overrides support offline tests and cannot come from repo YAML. self.ssh = ssh or ["ssh"] self.sftp = sftp or ["sftp"] self._injected = ssh is not None or sftp is not None @@ -53,6 +72,7 @@ def __init__(self, context, *, ssh=None, sftp=None): self._slots = BoundedSemaphore(context.settings.max_sessions) def _options(self, *, master=False): + """Build noninteractive SSH options for the shared connection.""" settings = self.context.settings policy = "yes" if settings.host_key_policy == "strict" else "accept-new" options = [ @@ -74,7 +94,7 @@ def _options(self, *, master=False): "ControlMaster=yes" if master else "ControlMaster=no", ] if not master: - # If the owned master dies, never authenticate a replacement per file. + # A failed shared connection must not reauthenticate for each file. options.append("ProxyCommand=false") if settings.user is not None: options.append(f"User={settings.user}") @@ -86,6 +106,7 @@ def _options(self, *, master=False): return args def _spawn(self, argv, **kwargs): + """Start a client in a separate process group and track it for cleanup.""" with self._process_lock: self.context.check_cancelled() try: @@ -99,6 +120,7 @@ def _spawn(self, argv, **kwargs): def _stop(self, process): # Every owned client starts a new process group, including proxy children. + """Stop a client process group and wait for the child to exit.""" def signal_group(signum): try: os.killpg(process.pid, signum) @@ -135,6 +157,12 @@ def _run( capture_stderr=False, report_progress=False, ): + """ + Run an OpenSSH command with output limits and a total time limit. + + Optionally bound a downloaded metadata file's size or report received + dataset bytes. Always stop and reap the client before returning. + """ output = bytearray() stderr_size = 0 diagnostics = bytearray() @@ -202,6 +230,7 @@ def report_bytes(): process.stderr.close() def prepare(self): + """Check client support and establish the shared SSH connection.""" with self._prepare_lock: if self._master is not None: if self._master.poll() is not None: @@ -253,6 +282,7 @@ def prepare(self): raise def _fetch(self, relative_path, destination, file_limit=None): + """Fetch one literal path within the session and transfer limits.""" remote = batch_argument(self.context.remote.pathname(relative_path)) local = batch_argument(destination) batch = f"get {remote} {local}\n".encode("utf-8") @@ -273,9 +303,11 @@ def _fetch(self, relative_path, destination, file_limit=None): ) def fetch(self, relative_path, destination, *, chunk_size=8192): + """Download one file over SFTP; OpenSSH controls the read size.""" self._fetch(relative_path, destination) def read_text(self, relative_path, limit): + """Read UTF-8 metadata with size checks before and after transfer.""" entry = self.stat(relative_path) if entry.size is not None and entry.size > limit: raise DownloadError("Remote text exceeds its size limit") @@ -290,12 +322,13 @@ def read_text(self, relative_path, limit): raise DownloadError("Remote manifest must contain UTF-8 text") from None def _metadata(self): + """Create an SFTP metadata session on the shared connection.""" from .sftp import SftpMetadata return SftpMetadata(self) def stat(self, relative_path): - """Read attributes without fetching file contents or following symlinks.""" + """Read regular-file attributes without fetching the file contents.""" path = literal_path(relative_path).as_posix() with self._metadata() as session: attrs = session.lstat(self.context.remote.pathname(path)) @@ -305,7 +338,21 @@ def stat(self, relative_path): return RemoteEntry(path, attrs["size"], attrs["mtime"]) def iter_entries(self, on_directory=None): - """Yield regular files recursively using only the SFTP subsystem.""" + """ + Yield regular files recursively using the SFTP subsystem. + + Args: + on_directory (callable, optional): Callback receiving each directory + path relative to the dataset root. + + Yields: + RemoteEntry: File path, size, and modification time when available. + Symlinks, special files, and repository metadata directories are skipped. + + Raises: + DownloadError: If discovery fails or a path escapes the dataset root. + CapabilityError: If the server omits required file types. + """ with self._metadata() as session: root = session.realpath(self.context.remote.root).rstrip("/") or "/" reject_controls(root, "SFTP root") @@ -356,12 +403,14 @@ def iter_entries(self, on_directory=None): # Symlinks and special files are deliberately not traversed. def cancel(self): + """Stop the client process groups started by this transport.""" with self._process_lock: processes = list(self._processes) for process in processes: self._stop(process) def close(self): + """Stop clients and remove the shared connection socket directory.""" self.cancel() self._master = None if self._socket_dir is not None: diff --git a/test/mock_server.py b/test/mock_server.py index 4191bc9..487442b 100644 --- a/test/mock_server.py +++ b/test/mock_server.py @@ -34,9 +34,11 @@ def __exit__(self, exc_type, exc_value, traceback): return False def close(self): + """Provide the Requests session cleanup interface for tests.""" pass def get(self, url, timeout=None, stream=False, allow_redirects=True): + """Return a registered response through the Requests session interface.""" return self.fake_get(url, timeout=timeout) def head(self, url, timeout=None): diff --git a/test/test_catalog.py b/test/test_catalog.py index a89603b..9f86b19 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -1,4 +1,4 @@ -"""Catalog cloning must never need dataset bodies or local content objects.""" +"""Test catalog cloning without downloading dataset files or stored objects.""" from pathlib import Path @@ -13,6 +13,7 @@ @pytest.fixture def metadata_server(monkeypatch): + """Serve registered metadata and record each requested URL.""" pages, reads = {}, [] def read_text(context, path): diff --git a/test/test_cli.py b/test/test_cli.py index 39b9957..e465eee 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -61,7 +61,7 @@ def _install_repo(monkeypatch, worktree=Path("worktree")): def _download_plan(count, output=Path("worktree")): - """A real immutable plan for testing CLI rendering and approval boundaries.""" + """Create a download plan for CLI display and approval tests.""" return DownloadPlan(tuple(DownloadItem(Path(f"file-{index:03d}.dat"), size_bytes=8 if index == 0 else None) for index in range(count)), @@ -69,7 +69,7 @@ def _download_plan(count, output=Path("worktree")): def _local_cli_catalog(path): - """Create a metadata-only repository backed by a tiny fake HTTP dataset.""" + """Create a local catalog for a small, simulated HTTP dataset.""" repo = Repo.init(path) repo.state.config = { "data": [{"db": "data.tsv"}], diff --git a/test/test_discovery.py b/test/test_discovery.py index 964b170..d00939d 100644 --- a/test/test_discovery.py +++ b/test/test_discovery.py @@ -11,6 +11,7 @@ class Source: + """Serve directory listings and record requests for discovery tests.""" def __init__(self, pages, url="https://data.test/public/"): self.remote = RemoteSpec.parse(url) self.pages = pages @@ -26,6 +27,7 @@ def check_cancelled(self): def index(*links): + """Render a small HTML directory index from the supplied links.""" return 'Index of /' + ''.join( f'{path}' for path in links) diff --git a/test/test_download_plan.py b/test/test_download_plan.py index 386651c..d4d13da 100644 --- a/test/test_download_plan.py +++ b/test/test_download_plan.py @@ -1,4 +1,4 @@ -"""Transfer approval must authorize exactly the locally inspected catalog plan.""" +"""Test download planning and approval using local catalog metadata.""" from dataclasses import FrozenInstanceError, replace from hashlib import sha256 @@ -20,6 +20,7 @@ @pytest.fixture def catalog(tmp_path): + """Create a local catalog with known and unknown file metadata.""" metadata = tmp_path / ".hm" metadata.mkdir() frame = pd.DataFrame([ diff --git a/test/test_downloader.py b/test/test_downloader.py index 38783a9..c60c12c 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -308,7 +308,7 @@ def test_safe_remote_path_rejects_unsafe_and_reserved_paths(unsafe_path): def test_safe_remote_path_preserves_literal_whitespace(): - """Catalog filenames are literal; significant spaces must survive download.""" + """Test that catalog paths retain significant leading and trailing spaces.""" assert _safe_remote_path(" nested/file.txt ") == Path(" nested/file.txt ") diff --git a/test/test_repo_builder.py b/test/test_repo_builder.py index 45b9b36..a2f7f05 100755 --- a/test/test_repo_builder.py +++ b/test/test_repo_builder.py @@ -60,7 +60,7 @@ def _inventory(monkeypatch, files): """ calls = [] def fake_list(base_url, **kwargs): - """create a fake list_remote_files list""" + """Return the configured inventory and record the requested URL.""" calls.append(base_url) return dict(files) monkeypatch.setattr("hallmark.repo_builder.list_remote_files", fake_list) @@ -184,11 +184,12 @@ def test_list_remote_files_partial_sibling_manifest_does_not_hide_files(): def test_list_remote_files_does_not_probe_custom_checksum_payloads(): + """Test that discovery reads only recognized checksum manifests.""" server = MockServer(BASE_URL) server.add_directory("", [ ("data-object", "data.tar"), ("data-object", "custom_checksum_manifest.log")]) - # No payload is registered: fetching either body would fail the test. + # No file contents are registered, so fetching either file would fail the test. with _served(server): files = list_remote_files(server.base_url) assert files["data.tar"] == (None, None) @@ -326,7 +327,7 @@ def test_list_remote_files_fetches_each_directory_once(): requested_urls = [] original_get = server.get def recording_get(url, timeout=None, **kwargs): - """A wrapper around the original server.get that records requested URLs.""" + """Record requested URLs before returning their registered responses.""" requested_urls.append(url) return original_get(url, timeout=timeout) server.get = recording_get @@ -349,8 +350,7 @@ def test_list_remote_files_unavailable_manifest_does_not_hide_files(): "", [("data-object", "data.tar"), ("data-object", "checksums.txt")]) original_get = server.get def unavailable_manifest_get(url, timeout=None, **kwargs): - """A wrapper around the original server.get that simulates - an unavailable manifest.""" + """Simulate a missing checksum manifest while serving other registered files.""" if url.endswith("/checksums.txt"): response = requests.Response() response.status_code = 404 @@ -839,9 +839,9 @@ def refuse_get_body(url, timeout=None): def test_build_repo_small_unmatched_file_is_not_downloaded(tmp_path): + """Test that even small unmatched files remain remote during catalog creation.""" server = MockServer(BASE_URL) server.add_directory("", [("data-object", "README.md")]) - # Even a tiny file stays remote until download approval. with _served(server): repo = build_repo(tmp_path / "repo.hm", "EHTC_TEST", fmt_entries=[]) assert repo.state.config["data"] == [{"name": "readme", "file": "README.md"}] @@ -849,6 +849,7 @@ def test_build_repo_small_unmatched_file_is_not_downloaded(tmp_path): def test_build_repo_defaults_to_generic_path_catalog_without_input( monkeypatch, tmp_path): + """Test that omitted formats produce a path catalog without prompting.""" _inventory(monkeypatch, { "arbitrary.bin": (None, None), "nested/a.fits": (None, None)}) monkeypatch.setattr("builtins.input", lambda *_: pytest.fail("Unexpected prompt")) @@ -861,6 +862,7 @@ def test_build_repo_defaults_to_generic_path_catalog_without_input( def test_build_repo_rejects_payload_hashing_before_network(monkeypatch, tmp_path): + """Test that remote hashing is rejected before opening a connection.""" monkeypatch.setattr("hallmark.repo_builder.list_remote_files", lambda *_: pytest.fail("Unexpected network access")) with pytest.raises(CapabilityError, match="remote_hash"): @@ -1077,8 +1079,7 @@ def test_build_repo_empty_dataset_writes_valid_empty_data_config(monkeypatch, tm def test_build_repo_loads_formats_and_remotes_from_explicit_config( monkeypatch, tmp_path): """ - If the user selects a config file that has formats and remotes, build_repo should - load those formats and remotes into the new repo's config + Test that build_repo loads formats and remotes from an explicit config file. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1109,8 +1110,7 @@ def test_build_repo_loads_formats_and_remotes_from_explicit_config( def test_build_repo_explicit_remotes_override_loaded_config(monkeypatch, tmp_path): """ - If the user selects a config file that has remotes, but also provides explicit - remotes to build_repo, the explicit remotes should override the config remotes. + Test that explicit remotes override those loaded from a config file. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1138,8 +1138,7 @@ def test_build_repo_explicit_remotes_override_loaded_config(monkeypatch, tmp_pat def test_build_repo_explicit_config_requires_a_format(monkeypatch, tmp_path): """ - If the user selects a config file that has no fmt entries, build_repo should raise - a ValueError indicating that no formats were found. + Test that a supplied config file must contain at least one filename format. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1157,8 +1156,7 @@ def test_build_repo_explicit_config_requires_a_format(monkeypatch, tmp_path): def test_build_repo_reuses_existing_formats_and_remotes(monkeypatch, tmp_path): """ - If a repo already exists, build_repo should prompt to reuse its formats and remotes. - If user agrees, the existing formats and remotes should be preserved in the new repo + Test that an existing repository retains its formats and data remotes. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1187,6 +1185,7 @@ def test_build_repo_reuses_existing_formats_and_remotes(monkeypatch, tmp_path): def test_build_repo_reuses_existing_catalog_without_input(monkeypatch, tmp_path): + """Test that an existing catalog is refreshed without interactive input.""" _inventory(monkeypatch, {"item_1.dat": (None, None)}) path = tmp_path / "catalog" build_repo(path, "label", [{"fmt": "item_{number}.dat", "db": "data.tsv"}]) @@ -1388,6 +1387,7 @@ def test_build_repo_preserves_meta_file_checksum(monkeypatch, tmp_path): def test_build_repo_never_requests_static_checksum(monkeypatch, tmp_path): + """Test that missing static-file checksums do not trigger dataset downloads.""" _inventory(monkeypatch, {"README.md": (None, None)}) monkeypatch.setattr("hallmark.transport.requests.Session", lambda: pytest.fail("Unexpected checksum request")) diff --git a/test/test_sftp_metadata.py b/test/test_sftp_metadata.py index 0740568..42a992c 100644 --- a/test/test_sftp_metadata.py +++ b/test/test_sftp_metadata.py @@ -20,6 +20,7 @@ @pytest.fixture def local_sftp(tmp_path, monkeypatch): + """Connect the transport to a local SFTP subsystem without networking.""" server = next((path for path in ( "/usr/lib/openssh/sftp-server", "/usr/libexec/sftp-server", ) if Path(path).exists()), None) @@ -115,6 +116,7 @@ def unexpected(*args, **kwargs): def _fake_subsystem(monkeypatch, context, script): + """Use a supplied script when starting an SFTP subsystem process.""" transport = context.transport monkeypatch.setattr(transport, "prepare", lambda: None) spawn = transport._spawn diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py index 80813c2..7115b5b 100644 --- a/test/test_ssh_integration.py +++ b/test/test_ssh_integration.py @@ -31,6 +31,7 @@ @pytest.fixture def ssh_server(tmp_path, monkeypatch, request): + """Start a loopback SSH server with temporary keys and configuration.""" if os.environ.get("HALLMARK_RUN_SSH_TESTS") != "1": pytest.skip("Set HALLMARK_RUN_SSH_TESTS=1 for disposable loopback SSH tests") sshd = shutil.which("sshd") or "/usr/sbin/sshd" @@ -271,7 +272,7 @@ def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): assert not (destination / "bad_2.dat").exists() assert not list(destination.rglob("*.part")) - # Single-catalog public Python clone and CLI clone both route SSH downloads. + # Python and CLI clones both use the recorded SSH data remote. single = build_repo( tmp_path / "single", "lab", diff --git a/test/test_transport.py b/test/test_transport.py index a6befbe..798bf5a 100644 --- a/test/test_transport.py +++ b/test/test_transport.py @@ -34,6 +34,7 @@ @pytest.fixture(autouse=True) def isolated_auth(monkeypatch, tmp_path): + """Isolate local authentication settings from the user environment.""" monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config")) @@ -108,6 +109,7 @@ def test_controls_rejected_at_all_path_boundaries(path, tmp_path): def write_auth(monkeypatch, tmp_path, profiles, defaults=None): + """Write a local authentication file for a transport test.""" path = tmp_path / "auth.yml" path.write_text( yaml.safe_dump({"version": 1, "profiles": profiles, "defaults": defaults or {}}) @@ -368,6 +370,7 @@ def test_real_sftp_parser_without_network(tmp_path, name): @pytest.fixture def fake_process(tmp_path): + """Create a local command for testing SSH process failures.""" script = tmp_path / "fake.py" script.write_text("""import os, signal, subprocess, sys, time mode = sys.argv[1] From dd0e5c4f12032d2c76b17ffb8b1c3a7bbc14e103 Mon Sep 17 00:00:00 2001 From: Dennis Date: Mon, 21 Sep 2026 11:46:25 -0700 Subject: [PATCH 13/29] updated gitignore --- .gitignore | 2 ++ README.rst | 7 ++++++- doc/index.rst | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9ad915c..74770d8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,6 @@ _build/ /demo/* !/demo/private_data_demo.md +!/demo/scientific_workflows_cli.md +!/demo/scientific_workflows_python.ipynb /cli_test_repos/ diff --git a/README.rst b/README.rst index 949392d..bf12886 100644 --- a/README.rst +++ b/README.rst @@ -46,7 +46,7 @@ Features: ------------- Examples of using ``ParaFrame`` with Python API or Command Line Interface (CLI) -can be found in the Jupyter Notebook tutorials in the ``demos`` folder. +can be found in the Jupyter Notebook tutorials in the ``demo`` folder. ``Installation`` ----------------- @@ -101,3 +101,8 @@ deprecated in favor of ``clone``. Examples and configuration details are described in `Private data over SSH and SFTP `_. + +For workflows with real EHT, DESI, Roman–Rubin simulation, and DES data, +see the `scientific CLI guide `_ and +`Python notebook `_. Both include +small-data alternatives and explicit download approval. diff --git a/doc/index.rst b/doc/index.rst index 3eec9a9..f0f7545 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -9,6 +9,11 @@ For private datasets, see :doc:`private_data` for SSH setup, CLI workflows, Python examples and troubleshooting. +Download the :download:`scientific CLI guide <../demo/scientific_workflows_cli.md>` +or :download:`Python notebook <../demo/scientific_workflows_python.ipynb>` +for four real-data workflows, including branches, Python worktrees, and +alternatives using less than 100 MB per workflow. + .. toctree:: :maxdepth: 2 :caption: Contents: From dbf427ac664f65e9b2b4f6d87c7ffdea94c186bc Mon Sep 17 00:00:00 2001 From: Dennis Date: Mon, 21 Sep 2026 20:25:44 -0700 Subject: [PATCH 14/29] Initialize remote datasets through extensible data backends --- .gitignore | 1 + README.rst | 31 ++- demo/multi_server_backend.py | 94 ++++++++ demo/private_data_demo.md | 86 +++++--- doc/api.rst | 35 ++- doc/backends.rst | 221 +++++++++++++++++++ doc/index.rst | 8 +- doc/private_data.rst | 173 ++++++++++----- doc/usecase.rst | 26 +-- mod/hallmark/__init__.py | 9 +- mod/hallmark/backends.py | 97 +++++++++ mod/hallmark/catalog.py | 194 ++++++++++------- mod/hallmark/cli.py | 100 +++++++-- mod/hallmark/discovery.py | 179 +--------------- mod/hallmark/download_plan.py | 17 +- mod/hallmark/downloader.py | 17 +- mod/hallmark/repo.py | 126 ++++++++--- mod/hallmark/repo_builder.py | 28 ++- mod/hallmark/repo_config.py | 46 +++- mod/hallmark/transport/__init__.py | 32 +-- mod/hallmark/transport/base.py | 98 ++++++++- mod/hallmark/transport/cyverse.py | 40 ++++ mod/hallmark/transport/http.py | 45 +++- mod/hallmark/transport/index.py | 149 +++++++++++++ mod/hallmark/transport/ssh.py | 9 +- test/test_backend_config.py | 100 +++++++++ test/test_backends.py | 334 +++++++++++++++++++++++++++++ test/test_catalog.py | 323 +++++++++++++++++++++++++--- test/test_cli.py | 110 +++++++++- test/test_discovery.py | 7 +- test/test_download_plan.py | 45 ++++ test/test_ssh_integration.py | 7 +- 32 files changed, 2281 insertions(+), 506 deletions(-) create mode 100644 demo/multi_server_backend.py create mode 100644 doc/backends.rst create mode 100644 mod/hallmark/backends.py create mode 100644 mod/hallmark/transport/cyverse.py create mode 100644 mod/hallmark/transport/index.py create mode 100644 test/test_backend_config.py create mode 100644 test/test_backends.py diff --git a/.gitignore b/.gitignore index 74770d8..ad0c224 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ _build/ /demo/* !/demo/private_data_demo.md +!/demo/multi_server_backend.py !/demo/scientific_workflows_cli.md !/demo/scientific_workflows_python.ipynb /cli_test_repos/ diff --git a/README.rst b/README.rst index bf12886..9eaf72a 100644 --- a/README.rst +++ b/README.rst @@ -80,29 +80,38 @@ is used to share the history of the data index. For a private server, |hallmark|_ uses OpenSSH 9.6 or newer with a trusted host key and key-based authentication that does not require a prompt. -Clone a remote directory to create its catalog, then select files to download:: +Initialize a catalog from a remote dataset, then select files to download:: - hallmark clone ssh://campus/srv/export/ campus --filter '**/*.h5' + hallmark init campus --from ssh://campus/srv/export/ --filter '**/*.h5' cd campus hallmark download --all --dry-run hallmark download --all -Cloning discovers files without downloading dataset contents. CyVerse and -ordinary browsable HTTPS directories, including DESI, use the same workflow. +Remote initialization discovers files without downloading dataset contents. +CyVerse and ordinary browsable HTTPS directories, including DESI, use the same +workflow. SSH discovery also works with SFTP-only accounts; no server shell or Python is required. Omit the filter to catalog everything beneath the supplied URL. Every dataset transfer requires approval. Python callers can inspect ``repo.plan_download()`` and then execute ``repo.download(plan, approved=True)``. Optional authentication profiles remain local and can be selected with -``clone --auth PROFILE`` or ``set-config --remote-auth PROFILE``. -``init`` creates a local repository; the older remote ``build`` command is -deprecated in favor of ``clone``. +``init --auth PROFILE`` or ``set-config --remote-auth PROFILE``. Add +``--with-download`` to ``init`` to review and approve a transfer after discovery. +The older remote ``build`` command is deprecated in favor of ``init --from``. + +Use ``hallmark clone CATALOG PATH`` for an existing Git-hosted ``.hm`` or a +published HTTP/SFTP catalog snapshot. Catalogs can live on GitHub or another +server while their data remotes point elsewhere. Git clones preserve the full +catalog and its history; clone filters select optional downloads only. + +The public ``DataBackend`` interface supports generic HTTP/SSH, CyVerse, and +registered plugins for collaboration-specific APIs and multiple data servers. +See `Data backends `_ for configuration and an extension example. Examples and configuration details are described in `Private data over SSH and SFTP `_. -For workflows with real EHT, DESI, Roman–Rubin simulation, and DES data, -see the `scientific CLI guide `_ and -`Python notebook `_. Both include -small-data alternatives and explicit download approval. +For a sequential CLI and Python walkthrough, see the +`private data demo `_. It includes catalog +initialization, local authentication profiles and explicit download approval. diff --git a/demo/multi_server_backend.py b/demo/multi_server_backend.py new file mode 100644 index 0000000..d5633a5 --- /dev/null +++ b/demo/multi_server_backend.py @@ -0,0 +1,94 @@ +"""Example backend joining independent HTTP roots into one logical dataset. + +Call ``register()`` before ``Repo.init(..., backend="multi-server", +backend_options={"routes": {"north": north_url, "south": south_url}})``. +An installed plugin instead exposes ``MultiServerBackend`` through the +``hallmark.backends`` entry-point group. No servers are contacted on import. +""" + +from collections.abc import Mapping +from contextlib import ExitStack +from dataclasses import replace + +from hallmark.backends import ( + DataBackend, OperationContext, RemoteConfigurationError, + RemoteObjectMissing, RemoteSpec, register_backend, +) +from hallmark.transport.base import literal_path + + +class MultiServerBackend(DataBackend): + """Map each first path segment to a separate HTTP(S) server root. + + Child contexts provide thread-local sessions. Their cancellation event is + shared with the parent; cleanup closes every child's sessions. Credentials + use Requests' usual local .netrc configuration for each server. + """ + + def __init__(self, context): + super().__init__(context) + routes = context.remote.backend_options.get("routes") + if not isinstance(routes, Mapping) or not routes: + raise RemoteConfigurationError("Multi-server options require routes") + specs = {} + for prefix, url in routes.items(): + if literal_path(prefix).as_posix() != prefix or "/" in prefix: + raise RemoteConfigurationError("Route names must be one path segment") + spec = RemoteSpec.parse(url, backend="http") + if spec.scheme not in {"http", "https"}: + raise RemoteConfigurationError("Example routes require HTTP(S) URLs") + specs[prefix] = spec + self._stack = ExitStack() + self._children = {} + try: + for prefix, spec in specs.items(): + child = self._stack.enter_context( + OperationContext(spec, context.output_root)) + child.cancelled = context.cancelled + self._children[prefix] = child + except BaseException: + self._stack.close() + raise + + def _route(self, relative_path): + path = literal_path(relative_path).as_posix() + prefix, separator, suffix = path.partition("/") + if not separator or prefix not in self._children: + raise RemoteObjectMissing("No route for this logical dataset path") + return self._children[prefix], suffix + + def prepare(self): + for child in self._children.values(): + child.transport.prepare() + + def iter_entries(self, on_directory=None): + for prefix, child in self._children.items(): + self.context.check_cancelled() + + def report(directory, prefix=prefix): + if on_directory is not None: + on_directory(prefix + "/" + directory) + + for entry in child.transport.iter_entries(on_directory=report): + yield replace(entry, path=prefix + "/" + entry.path) + + def read_text(self, relative_path, limit): + child, path = self._route(relative_path) + return child.transport.read_text(path, limit) + + def fetch(self, relative_path, destination, *, chunk_size=8192): + child, path = self._route(relative_path) + child.on_bytes = self.context.on_bytes + child.transport.fetch(path, destination, chunk_size=chunk_size) + + def cancel(self): + for child in self._children.values(): + child.cancel() + + def close(self): + self._stack.close() + + +def register(): + """Register this example in the current process before using it.""" + register_backend("multi-server", MultiServerBackend) diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md index e5d827e..c7b5e09 100644 --- a/demo/private_data_demo.md +++ b/demo/private_data_demo.md @@ -17,7 +17,7 @@ Run the Bash blocks in order in one local terminal. # Start in the Hallmark source checkout. python -m pip install -e . ssh -V -hallmark clone --help +hallmark init --help ``` 2. Add or update this entry in `~/.ssh/config`, preserving your other entries. @@ -50,7 +50,7 @@ cd "$HM_DEMO_WORKSPACE" ``` SSH configuration alone is enough. To demonstrate an optional local profile, -create this file outside the catalog and clone: +create this file outside the catalog: ```bash export HALLMARK_AUTH_FILE="$HM_DEMO_WORKSPACE/auth.yml" @@ -74,12 +74,15 @@ their own local profile; catalog configuration stores only its name. 4. Discover the two indexed runs and prepare a local `.hm`. ```bash -hallmark clone "$HM_DEMO_URL" ./client \ +hallmark init ./client --from "$HM_DEMO_URL" \ --auth demo --fmt 'runs/run_{run:03d}.dat' cat ./client/.hm/config.yml cat ./client/.hm/data.tsv ``` +The destination can contain existing files when `.hm` is absent; those files +are preserved. An existing `.hm` is rejected before contacting the data server. + Expected for the illustrative export: two run rows, with sizes and modification times where available. Published manifest checksums are attached when present; missing checksums stay unknown. The data remote is `origin` with `auth: demo`. @@ -166,7 +169,8 @@ PYTHON python ../download_subset.py ``` -The plan fixes its source, profile reference, destination, paths, and checksums. +The plan fixes its source, profile reference, backend settings, destination, +paths, and checksums. Later catalog or configuration changes do not redirect it. Without `approved=True`, nonempty Python transfers fail before connecting. Check the returned `failed` count; successful files remain when another transfer fails. @@ -177,18 +181,19 @@ remove their temporary file. Repeating a download transfers the selection again. Cancellation closes the connections and processes started for the download. Byte progress shows an ETA when a total is known; unknown totals stay indeterminate. -8. Request an approved download during cloning, using either interface. +8. Request an approved download during initialization, using either interface. The CLI prepares the catalog first and then displays its normal approval prompt: ```bash -hallmark clone "$HM_DEMO_URL" ../clone-and-download \ - --auth demo --filter 'runs/run_001.dat' --download +hallmark init ../init-and-download --from "$HM_DEMO_URL" \ + --auth demo --filter 'runs/run_001.dat' --with-download ``` -Python receives the completed plan through a callback: +Declining keeps the completed catalog available. Empty selections need no +approval. Python receives the completed plan through a callback: ```bash -cat > ../clone_with_approval.py <<'PYTHON' +cat > ../init_with_approval.py <<'PYTHON' import os from hallmark import Repo @@ -198,13 +203,13 @@ def approve(plan): return input("Download these files? [y/N] ").strip().lower() == "y" -repo = Repo.clone( - os.environ["HM_DEMO_URL"], "../python-clone", auth="demo", +repo = Repo.init( + "../python-init", from_url=os.environ["HM_DEMO_URL"], auth="demo", filter="runs/run_001.dat", download=True, approve=approve, progress=True, ) print(repo.dothm.path) PYTHON -python ../clone_with_approval.py +python ../init_with_approval.py ``` 9. Explore CyVerse and DESI with the same metadata-only workflow. These examples @@ -213,10 +218,10 @@ require many directory-listing requests; use a specific subtree when appropriate ```bash cd "$HM_DEMO_WORKSPACE" -hallmark clone \ +hallmark init ./cyverse --from \ 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/' \ - ./cyverse --filter '**/*.uvfits' -hallmark clone 'https://data.desi.lbl.gov/public/' ./desi --filter '**/*.fits' + --filter '**/*.uvfits' +hallmark init ./desi --from 'https://data.desi.lbl.gov/public/' --filter '**/*.fits' cd "$HM_DEMO_WORKSPACE/cyverse" hallmark download --all --dry-run @@ -237,13 +242,14 @@ cd "$HM_DEMO_WORKSPACE" python - <<'PYTHON' from hallmark import Repo -cyverse = Repo.clone( - "https://data.cyverse.org/dav-anon/iplant/commons/" - "cyverse_curated/EHTC_FirstM87Results_Apr2019/", - "cyverse-python", filter="**/*.uvfits", progress=True, +cyverse = Repo.init( + "cyverse-python", + from_url="https://data.cyverse.org/dav-anon/iplant/commons/" + "cyverse_curated/EHTC_FirstM87Results_Apr2019/", + filter="**/*.uvfits", progress=True, ) -desi = Repo.clone( - "https://data.desi.lbl.gov/public/", "desi-python", +desi = Repo.init( + "desi-python", from_url="https://data.desi.lbl.gov/public/", filter="**/*.fits", progress=True, ) for repo in (cyverse, desi): @@ -269,18 +275,42 @@ unset HALLMARK_AUTH_FILE A local or Git-hosted Hallmark repository retains its history. A published HTTP/SFTP catalog directory is imported as a snapshot with new local history. -Use `--source-type git` for a Git endpoint without an obvious `.git` suffix, or -`--source-type catalog` to require a published snapshot. Git credentials are -separate from the data remote's `--auth` profile. Filtering an existing Git -catalog preserves fetched history and adds a local filtered-catalog commit. +Use `--source-type git` to require a Git endpoint, or `--source-type catalog` +to require a published snapshot. Automatic HTTP(S) detection checks for +metadata at the root and in `.hm`, then tries Git if neither snapshot exists. +Git credentials are separate from data-remote profile references. Cloning +preserves the complete catalog and Git HEAD. To download selected files after +cloning, add `--with-download --filter PATTERN`; clone filters require download +intent and do not remove catalog rows. `--download` remains an alias. +Declining approval leaves the catalog available, and empty selections do not +prompt. Raw dataset URLs belong to `init --from`. `clone` downloads no dataset files by default, so `--no-fetch-data` is now only -a compatibility alias. `init` creates a local repository; the older remote `build` -command is deprecated. Discovery uses no `--allow-remote-commands` or -`--remote-hash` flag, and does not read dataset files to compute checksums. +a compatibility alias. Plain `init` creates a local repository; the older remote +`build` command is deprecated in favor of `init --from`. Discovery uses no +`--allow-remote-commands` or `--remote-hash` flag, and does not read dataset files +to compute checksums. For local `add`/`commit` experiments, use a separate compatible one-format repository; downloading a remote catalog does not populate the local object store. +The catalog can also live on GitHub or an HTTPS metadata server while its +data remote remains on the SSH server. For example, after the team publishes +`.hm` as a Git repository, collaborators can run: + +```bash +hallmark clone https://github.com/example/lab-catalog.git ./shared-catalog +``` + +This uses Git credentials and preserves the stored data URL and profile +reference. A published snapshot at `https://catalogs.example.org/lab/` +can be cloned in the same way, starting local history. Cloning metadata alone +does not require the payload profile. + +Use `init --backend NAME --backend-options FILE` to select an installed data +backend and load its nonsecret options from a YAML mapping. Python accepts +`backend=NAME, backend_options=options`. See [data backends](../doc/backends.rst) +for registration and a multiple-server example. + See [the private data guide](../doc/private_data.rst) for profile settings, error handling and operational details. The auth-file override above lasts only until `unset HALLMARK_AUTH_FILE`; existing catalog profile references still require diff --git a/doc/api.rst b/doc/api.rst index a8c342d..aceb8c1 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -95,15 +95,16 @@ Create a local repository:: hallmark init ./local-project -Clone a remote dataset without downloading dataset files:: +Initialize a catalog from a remote dataset without downloading dataset files:: - hallmark clone https://data.desi.lbl.gov/public/ ./desi --filter '**/*.fits' - hallmark clone ssh://lab-data/srv/data/ ./lab --fmt 'run{run:d}.h5' + hallmark init ./desi --from https://data.desi.lbl.gov/public/ --filter '**/*.fits' + hallmark init ./lab --from ssh://lab-data/srv/data/ --fmt 'run{run:d}.h5' -An existing Hallmark Git repository is cloned with its history. A published -HTTP/SFTP catalog snapshot starts new local history. Use ``--source-type git`` -for a Git endpoint whose URL cannot be distinguished from a directory URL. -The older ``build`` command remains a deprecated compatibility interface. +Use ``hallmark clone SOURCE PATH`` for an existing Hallmark Git repository +or published HTTP/SFTP snapshot. Git clones retain the full catalog and its +history; snapshots start new local history. Use ``--source-type git`` or +``--source-type catalog`` to override automatic detection. The older ``build`` +command remains available but is deprecated in favor of ``init --from``. Downloading remote data ----------------------- @@ -115,25 +116,39 @@ Preview the selected files using the local catalog, then confirm a download:: Explicit paths and ``--tsv data.tsv`` also select files. Every nonempty transfer requires interactive confirmation, including transfers requested through -``clone --download``. The old ``--yes`` option no longer bypasses approval. +``init --with-download`` or ``clone --with-download``. Clone accepts its older +``--download`` alias. Clone filters and formats require download intent and +leave the complete catalog unchanged. The old ``--yes`` option no longer +bypasses approval. A filter or filename format never authorizes a transfer. Python uses the same plan and requires explicit approval:: from hallmark import Repo - repo = Repo.clone('ssh://lab-data/srv/data/', 'lab', progress=True) + repo = Repo.init('lab', from_url='ssh://lab-data/srv/data/', progress=True) plan = repo.plan_download(filter='runs/**') print(plan.summary()) result = repo.download(plan, approved=True, progress=True) -Plans preserve their selected remote, destination and checksums even when +Plans preserve their selected remote, backend settings, destination and checksums even when repository configuration later changes. Size estimates require recorded file sizes; duration estimates also require a supplied transfer rate. .. automodule:: hallmark.download_plan :members: +Data backends +------------- + +See :doc:`backends` for registration, installed plugins and the transfer +contract. Backend classes are also exported from ``hallmark``. Existing +``hallmark.transport`` imports remain compatibility aliases. + +.. automodule:: hallmark.backends + :members: + :show-inheritance: + Dataset builders and data remotes --------------------------------- diff --git a/doc/backends.rst b/doc/backends.rst new file mode 100644 index 0000000..842d0a7 --- /dev/null +++ b/doc/backends.rst @@ -0,0 +1,221 @@ +Data backends +============= + +A data backend discovers files and transfers them from a data source. It does +not determine where the Hallmark catalog is hosted. The same catalog can be +shared through GitHub, an HTTP server or SFTP while its backend accesses a +separate data service or several servers. + +Built-in backends and configuration +----------------------------------- + +``hallmark.backends`` exposes the common ``DataBackend`` base and these +implementations: + +* ``HttpBackend`` transfers HTTP(S) files and discovers supported HTML indexes. +* ``SshBackend`` uses SFTP for both SSH and SFTP data URLs, including accounts + without a login shell. +* ``CyVerseBackend`` extends HTTP support with CyVerse directory and metadata + parsing. + +Without an explicit backend, Hallmark selects one from the data URL. Explicit +selection takes precedence. For example: + +.. code-block:: bash + + hallmark init ./observations --from 'https://data.example.org/export/' \ + --backend http --filter '**/*.fits' + +For an installed plugin, pass its registered name and an optional YAML mapping: + +.. code-block:: bash + + hallmark init ./survey --from 'https://survey.example.org/release/' \ + --backend survey --backend-options ./survey-options.yml + +``Repo.init(..., from_url=url, backend="survey", backend_options=options)`` +accepts the same settings in Python. The data remote stores ``backend`` and +``backend_options`` alongside its ``name``, ``url`` and optional ``auth``: + +.. code-block:: yaml + + remote: + - name: origin + url: https://survey.example.org/release/ + backend: survey + backend_options: + release: dr1 + +Options contain ordinary configuration that can be shared with the catalog. +Keep tokens, passwords and private keys outside them. The built-in SSH backend +uses local auth profiles; HTTP retains Requests' authentication and environment +behavior. A plugin documents how it obtains any additional credentials locally. +The example ``release`` option above is interpreted by the plugin, not Hallmark. + +Update an existing remote in Python with +``repo.set_config(remote_backend="survey", remote_backend_options=options)``. +Passing ``remote_backend=""`` returns to URL-based selection; passing +``remote_backend_options={}`` clears the options. ``None`` leaves the respective +setting unchanged. Configurations without backend fields remain valid. + +``repo.plan_download()`` reads the local catalog without opening a connection. +The plan captures its selected backend name and deeply immutable options, in +addition to its source URL, profile reference, destination and file metadata. +Changes to repository settings or the caller's original options mapping do not +redirect an approved plan. Install or register the selected plugin before +executing a transfer that uses it. + +Writing a backend +----------------- + +Implement ``DataBackend`` or extend an existing backend. All classes receive +an ``OperationContext`` through ``__init__(context)``. The context exposes the +validated ``RemoteSpec`` as ``context.remote`` and its frozen options as +``context.remote.backend_options``. ``context.session()`` supplies a Requests +session per worker thread; ``context.check_cancelled()`` raises when the +operation has been cancelled. + +.. list-table:: Backend contract + :header-rows: 1 + :widths: 35 65 + + * - Method + - Responsibility + * - ``prepare()`` + - Check access and prepare operation resources before discovery or transfer. + Repeated calls must be safe. + * - ``iter_entries(on_directory=None)`` + - Yield ``RemoteEntry`` records with literal logical paths relative to the + dataset root. Report completed directories through the optional callback. + Missing sizes, timestamps and published checksums remain ``None``. + * - ``read_text(relative_path, limit)`` + - Read metadata within the supplied byte limit and return decoded text. + Reject oversized responses and propagate access or connection errors. + * - ``fetch(relative_path, destination, *, chunk_size=8192)`` + - Write a payload to the caller-managed temporary path. Check cancellation + during transfer and call ``context.on_bytes(count)`` when set. + * - ``cancel()`` + - Stop resources started by this backend, including active child operations. + * - ``close()`` + - Release connections and processes when the operation ends. + +``list_entries()`` is a convenience wrapper over ``iter_entries()``. Raise +``CapabilityError`` when a backend cannot list or read metadata, and +``RemoteObjectMissing`` for an absent metadata object. These exceptions are +available from ``hallmark.backends``. Do not disguise authentication failures +as missing objects. + +Discovery reads listings and published metadata only. It must not download +payloads to infer checksums. The shared downloader owns approval, destination +path validation, checksum verification, atomic replacement and worker +concurrency. Backends map logical paths to server paths or API requests and +write only to the temporary destination they are given. + +An instance can serve concurrent ``fetch`` calls. Protect shared mutable +state, use thread-local sessions, and check cancellation in listing, metadata +and transfer loops. Document any server concurrency limit. ``cancel`` may run +while workers are active; make it and ``close`` safe to call during cleanup. +Keep resource acquisition in ``prepare`` or managed contexts so partial setup +can be cleaned up after failures. + +For example, an HTTP service can publish a small ``manifest.json`` array with +``path``, optional ``size`` and optional ``sha256`` fields. Save this adapter as +``my_survey/backend.py`` in an installable Python package: + +.. code-block:: python + + import json + + from hallmark.backends import HttpBackend, RemoteEntry + + + class SurveyBackend(HttpBackend): + """Read a published manifest; inherit bounded HTTP reads and transfers.""" + + def iter_entries(self, on_directory=None): + records = json.loads(self.read_text("manifest.json", self.context.text_limit)) + for record in records: + self.context.check_cancelled() + digest = record.get("sha256") + yield RemoteEntry( + path=record["path"], + size=record.get("size"), + checksum_algorithm="sha256" if digest else None, + checksum=digest, + ) + if on_directory is not None: + on_directory("") + +Register the class for the current Python process before initialization or +transfer: + +.. code-block:: python + + from hallmark import Repo, register_backend + from my_survey.backend import SurveyBackend + + register_backend("survey", SurveyBackend) + repo = Repo.init( + "survey", from_url="https://survey.example.org/release/", + backend="survey", + ) + +For CLI use and other collaborators, publish the class as an entry point in +the plugin package's ``pyproject.toml``: + +.. code-block:: toml + + [project.entry-points."hallmark.backends"] + survey = "my_survey.backend:SurveyBackend" + +Install that package in the same Python environment as Hallmark. Entry points +and ``register_backend`` use the same registry; duplicate names and classes +that do not subclass ``DataBackend`` are rejected. Register through one route +per process. Repository configuration stores registered names, never arbitrary +Python import paths. Old ``hallmark.transport`` imports remain supported as +compatibility aliases. + +Mapping one catalog across servers +---------------------------------- + +The :download:`multiple-server backend example <../demo/multi_server_backend.py>` +combines HTTP roots beneath logical prefixes. For example, the options file +can contain: + +.. code-block:: yaml + + routes: + north: https://north.example.org/export/ + south: https://south.example.org/export/ + +The catalog records ``north/run_001.fits`` and ``south/run_002.fits``. The +backend maps those paths to their configured server roots and retains each +server's available metadata. Download plans capture the complete routing map. +These prefixes organize catalog paths; they do not require moving the data or +placing ``.hm`` on either server. + +From the source checkout, register the demonstration backend explicitly: + +.. code-block:: python + + from hallmark import Repo + from demo.multi_server_backend import register + + register() + repo = Repo.init( + "combined", + from_url="https://north.example.org/export/", + backend="multi-server", + backend_options={"routes": { + "north": "https://north.example.org/export/", + "south": "https://south.example.org/export/", + }}, + ) + plan = repo.plan_download(filter="north/**/*.fits") + print(plan.summary()) + +The example is tested with two disposable local HTTP servers. The URLs above +are illustrative. Collaboration-specific adapters for DESI, LSST, Roman, JWST +or Euclid can implement their own manifests, directory conventions or API +routing through the same contract; those service-specific plugins are not +included with Hallmark. diff --git a/doc/index.rst b/doc/index.rst index f0f7545..e6e97d4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -9,10 +9,9 @@ For private datasets, see :doc:`private_data` for SSH setup, CLI workflows, Python examples and troubleshooting. -Download the :download:`scientific CLI guide <../demo/scientific_workflows_cli.md>` -or :download:`Python notebook <../demo/scientific_workflows_python.ipynb>` -for four real-data workflows, including branches, Python worktrees, and -alternatives using less than 100 MB per workflow. +Download the :download:`private data demo <../demo/private_data_demo.md>` +for a sequential CLI and Python walkthrough. See :doc:`backends` to add +a data service or combine several servers under one catalog. .. toctree:: :maxdepth: 2 @@ -21,6 +20,7 @@ alternatives using less than 100 MB per workflow. design usecase private_data + backends api diff --git a/doc/private_data.rst b/doc/private_data.rst index e8043eb..c5659cb 100644 --- a/doc/private_data.rst +++ b/doc/private_data.rst @@ -4,10 +4,11 @@ Private data over SSH and SFTP ============================== |hallmark|_ can discover a remote dataset, prepare a local catalog, and download -selected files after approval. ``clone`` accepts both an existing Hallmark -catalog and an ordinary data directory. Its default result is a local ``.hm`` -with no dataset files. A **data remote** identifies the dataset location; -it is separate from the Git remote used to share catalog history. +selected files after approval. ``init --from`` creates a new ``.hm`` from a +remote dataset; ``clone`` copies an existing Hallmark Git catalog or published +catalog snapshot. Both leave dataset files on their servers by default. +A **data remote** identifies the dataset location; it is independent of the +Git remote or snapshot URL used to share the catalog. The SSH examples use the alias ``lab-data`` and the absolute dataset root ``/srv/exports/lab/``. Replace the hostname, username, identity path and root @@ -47,15 +48,15 @@ Hallmark uses noninteractive authentication and strict host-key checking. It cannot answer password, MFA or initial hardware-key prompts. Ordinary SSH configuration, including ``ProxyJump``, remains available. -2. Discover a remote catalog ----------------------------- +2. Initialize a catalog from remote data +---------------------------------------- Suppose the export contains ``runs/run_001.h5``, ``runs/run_002.h5`` and ``README.md``. From an empty local workspace, run: .. code-block:: bash - hallmark clone 'ssh://lab-data/srv/exports/lab/' ./lab \ + hallmark init ./lab --from 'ssh://lab-data/srv/exports/lab/' \ --fmt 'runs/run_{run:03d}.h5' This creates ``./lab/.hm`` and catalogs only the matching run files. The URL @@ -65,7 +66,7 @@ without defining parameters: .. code-block:: bash - hallmark clone 'sftp://lab-data/srv/exports/lab/' ./lab-h5 \ + hallmark init ./lab-h5 --from 'sftp://lab-data/srv/exports/lab/' \ --filter '**/*.h5' Without ``--filter`` or ``--fmt``, discovery recursively covers every directory @@ -83,9 +84,18 @@ catalogs, publish an immutable export with a SHA-256 manifest, for example `` runs/run_001.h5`` records in ``SHA256SUMS``. The generated data remote is named ``origin`` and points to the source URL. -Use ``hallmark init PATH`` to initialize a local repository. The older remote -``build`` command and ``build_repo`` API are deprecated in favor of ``clone``; -server-side hashing is no longer part of discovery. +The destination may already contain files, provided it has no ``.hm``; +initialization preserves those files. An existing ``.hm`` is rejected before +network access. Use ``hallmark init PATH`` without ``--from`` for a local +repository. The older remote ``build`` command and ``build_repo`` API remain +available but are deprecated in favor of ``init --from``. + +Select a built-in backend with ``--backend http``, ``ssh``, or ``cyverse``, or +name an installed plugin. Without an explicit choice, Hallmark selects a +backend from the URL. ``--backend-options FILE`` reads a YAML mapping of +nonsecret backend settings. These settings are saved with the data remote; +credentials belong in local authentication configuration. See :doc:`backends` +for the plugin interface and a multiple-server example. 3. Preview, approve and download -------------------------------- @@ -132,13 +142,14 @@ require an explicit output directory. Repeating a download transfers the selection again and replaces destinations only after successful transfer and any recorded checksum verification. -To request downloads during cloning, add ``--download``. Catalog preparation -finishes first, then the same plan and approval prompt appear: +To request downloads during initialization, add ``--with-download``. Catalog +preparation finishes first, then the same plan and approval prompt appear. +Declining keeps the new catalog available; an empty selection needs no approval: .. code-block:: bash - hallmark clone 'ssh://lab-data/srv/exports/lab/' ../lab-one \ - --filter 'runs/run_001.h5' --download + hallmark init ../lab-one --from 'ssh://lab-data/srv/exports/lab/' \ + --filter 'runs/run_001.h5' --with-download An existing catalog can be cloned from a local path, a Git endpoint, or an HTTP/SFTP directory containing its metadata: @@ -149,13 +160,28 @@ HTTP/SFTP directory containing its metadata: hallmark clone 'https://git.example.org/team/lab.git' ../lab-history hallmark clone 'https://data.example.org/catalogs/lab.hm/' ../lab-snapshot -Git endpoints preserve history. Published HTTP/SFTP snapshots start new local -history. For a Git URL without an obvious ``.git`` suffix, specify -``--source-type git``; ``--source-type catalog`` or ``directory`` explicitly -selects a published snapshot or discovery. Filtering a Git clone preserves -fetched history and commits the filtered catalog locally. Git authentication -and data-server authentication are independent. ``--no-fetch-data`` remains a -compatibility alias for the default metadata-only behavior. +Git endpoints preserve the complete catalog and history. Published HTTP/SFTP +snapshots start new local history. A snapshot URL may name the metadata +directory itself or its parent containing ``.hm``. Use ``--source-type git`` +to require Git or ``--source-type catalog`` to require a snapshot. Automatic +selection treats local paths, SCP-style addresses, Git/file/SSH URLs and URLs +ending in ``.git`` as Git sources. SFTP URLs select snapshots. Other HTTP(S) +URLs are checked for a snapshot before Git is attempted. Authentication errors, +connection failures and malformed snapshots are reported rather than treated +as dataset directories. Use ``init --from`` for raw datasets. + +Clone filters and formats select files to download, leaving the full catalog +and Git HEAD unchanged. They require ``--with-download`` (the older +``--download`` spelling remains an alias): + +.. code-block:: bash + + hallmark clone ./.hm ../lab-selected \ + --filter 'runs/run_001.h5' --with-download + +``--no-fetch-data`` remains a compatibility alias for the default behavior +without payload downloads. Git authentication and data authentication are +independent; cloning a catalog does not require its payload credentials. 4. Use a local authentication profile ------------------------------------- @@ -185,19 +211,21 @@ example with ``chmod 600 ~/.config/hallmark/auth.yml``. Hallmark reads ``$XDG_CONFIG_HOME/hallmark/auth.yml``, otherwise the path above. Every client using a catalog's profile reference must define that profile locally. -Associate an existing data remote with a profile, or select one when cloning -an ordinary remote directory: +Associate an existing data remote with a profile, or select one when +initializing a catalog from a remote dataset: .. code-block:: bash hallmark set-config --remote-name origin --remote-auth lab hallmark download --remote origin --tsv data.tsv --dry-run - hallmark clone 'ssh://lab-data/srv/exports/lab/' ../lab-profile --auth lab + hallmark init ../lab-profile --from 'ssh://lab-data/srv/exports/lab/' --auth lab The new catalog's ``origin`` remote records ``auth: lab``. Clear a reference with ``hallmark set-config --remote-name origin --remote-auth ''`` to use SSH -configuration alone. ``--auth`` applies to data access; Git cloning uses Git's -own authentication configuration. +configuration alone. With ``init``, ``--auth`` selects data access. With +``clone``, it selects access to an SSH/SFTP catalog snapshot; cloned data +remotes retain their own profile references. Git cloning uses Git's own +authentication configuration. ``hosts`` binds the profile to the URL's alias or literal hostname before ``HostName`` resolution. This example binds to ``lab-data``, not @@ -211,15 +239,15 @@ concurrency is the smaller of ``--max-workers`` and local ``max_sessions`` 5. Use the Python API --------------------- -These examples use fresh local destinations. Like the CLI, ``Repo.clone`` -discovers metadata without downloading dataset files: +These examples use fresh local destinations. Like the CLI, ``Repo.init`` +with ``from_url`` discovers metadata without downloading dataset files: .. code-block:: python from hallmark import Repo - repo = Repo.clone( - "ssh://lab-data/srv/exports/lab/", "lab-python", + repo = Repo.init( + "lab-python", from_url="ssh://lab-data/srv/exports/lab/", auth="lab", # Omit when SSH configuration already supplies access. fmt="runs/run_{run:03d}.h5", progress=True, @@ -238,16 +266,16 @@ Inspect the plan, then approve the download: Without ``approved=True``, a nonempty transfer raises ``DownloadError`` before opening a connection. A plan freezes the selected files, source URL, profile -reference and destination; later catalog or remote configuration changes do not -redirect it. ``plan_download(filter="**/*.h5")`` scopes the complete catalog; +reference, backend, backend options and destination. Later catalog or remote +configuration changes do not redirect it. ``plan_download(filter="**/*.h5")`` scopes the complete catalog; ``plan_download(output_path="subset", tsv_names=["data.tsv"])`` selects a TSV and destination. Planning makes no network requests. Known size totals and unknown-size counts are available as ``known_bytes`` and ``unknown_size_count``. ``estimated_seconds`` stays ``None`` unless all sizes are known and ``estimated_bytes_per_second`` was supplied when planning. -For downloads during cloning, supply a callback that reviews the completed -plan and returns ``True`` to approve: +For downloads during initialization or cloning, supply a callback that reviews +the completed plan and returns ``True`` to approve: .. code-block:: python @@ -255,8 +283,8 @@ plan and returns ``True`` to approve: print(plan.summary()) return input("Download these files? [y/N] ").strip().lower() == "y" - repo = Repo.clone( - "ssh://lab-data/srv/exports/lab/", "lab-with-download", + repo = Repo.init( + "lab-with-download", from_url="ssh://lab-data/srv/exports/lab/", filter="runs/run_001.h5", download=True, approve=approve, progress=True, ) @@ -274,10 +302,10 @@ such as DESI. No CyVerse-specific index option is required: .. code-block:: bash - hallmark clone \ + hallmark init ./cyverse --from \ 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/' \ - ./cyverse --filter '**/*.uvfits' - hallmark clone 'https://data.desi.lbl.gov/public/' ./desi --filter '**/*.fits' + --filter '**/*.uvfits' + hallmark init ./desi --from 'https://data.desi.lbl.gov/public/' --filter '**/*.fits' Each URL is the full recursive discovery root. A broad root can require many listing requests even with a file filter; use a specific subtree when that is @@ -285,19 +313,20 @@ your intended scope. Public collections need no auth profile. Private HTTP access retains Requests' normal ``.netrc`` and environment behavior; Hallmark's named auth profiles apply to SSH/SFTP. -From either clone, inspect and approve a selected subset using +From either initialized catalog, inspect and approve a selected subset using ``hallmark download --filter PATTERN --dry-run`` and then the same command without ``--dry-run``. Python follows the same steps: .. code-block:: python - cyverse = Repo.clone( - "https://data.cyverse.org/dav-anon/iplant/commons/" - "cyverse_curated/EHTC_FirstM87Results_Apr2019/", - "cyverse-python", filter="**/*.uvfits", progress=True, + cyverse = Repo.init( + "cyverse-python", + from_url="https://data.cyverse.org/dav-anon/iplant/commons/" + "cyverse_curated/EHTC_FirstM87Results_Apr2019/", + filter="**/*.uvfits", progress=True, ) - desi = Repo.clone( - "https://data.desi.lbl.gov/public/", "desi-python", + desi = Repo.init( + "desi-python", from_url="https://data.desi.lbl.gov/public/", filter="**/*.fits", progress=True, ) plan = desi.plan_download(filter="dr1/**/selected-file.fits") @@ -308,9 +337,53 @@ without ``--dry-run``. Python follows the same steps: HTTP provides file transfer but does not define directory enumeration. Hallmark recognizes supported HTML indexes and directory landing pages automatically. -If a server supplies no usable listing, provide a published Hallmark catalog -or a more specific browsable URL; Hallmark cannot -infer hidden file URLs. +If a server supplies no usable listing, provide a published Hallmark catalog, +a more specific browsable URL, or a :doc:`backend plugin ` for its +API. Hallmark cannot infer hidden file URLs. + +7. Host catalogs separately from their data +------------------------------------------- + +A catalog's location does not set its payload location. For example, a team +can push the Git repository in ``.hm`` to GitHub while its data remote still +points to ``ssh://lab-data/srv/exports/lab/`` with ``auth: lab``. Collaborators +clone the existing metadata using their Git credentials: + +.. code-block:: bash + + hallmark clone 'https://github.com/example/lab-catalog.git' ./shared-lab + cd shared-lab + hallmark download --all --dry-run + +This clone needs no ``lab`` profile. Define the local profile before approving +a data transfer. The Git origin points to GitHub; the data remote named +``origin`` retains the SSH URL and profile reference recorded in the catalog. +Git fetch/pull operates on the metadata repository in ``.hm``. + +The same configuration can be published as a snapshot on a separate HTTPS +server. For example, publish ``config.yml``, ``meta.yml`` and referenced TSVs +under ``https://catalogs.example.org/lab/``. Its configuration can contain: + +.. code-block:: yaml + + remote: + - name: origin + url: ssh://lab-data/srv/exports/lab/ + auth: lab + +Then clone the snapshot: + +.. code-block:: bash + + hallmark clone 'https://catalogs.example.org/lab/' ./snapshot-lab + +Snapshot authentication uses the metadata server's settings. The SSH payload +URL and ``lab`` reference are preserved without resolving that profile during +cloning. For an SFTP snapshot, ``clone --auth catalog-reader`` selects its +metadata profile; it does not replace a data remote's profile. Snapshot +imports start local Git history and have no upstream Git history to pull. +These examples retain the usual local ``PATH/.hm`` layout while keeping +catalog hosting independent of payload hosting. .. _operational-behavior-and-troubleshooting: diff --git a/doc/usecase.rst b/doc/usecase.rst index 6a44479..b3d57c2 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -263,12 +263,12 @@ The local profile or defaults may use ``host_key_policy: accept-new`` to accept a host key on first contact. Changed host keys still cause an error. Repository configuration cannot enable this setting. -Clone a private catalog -~~~~~~~~~~~~~~~~~~~~~~~ +Initialize a private catalog +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Frank can prepare a catalog directly from the files on the server:: - hallmark clone ssh://campus/srv/export/ lab \ + hallmark init lab --from ssh://campus/srv/export/ \ --auth campus --fmt 'runs/run_{i:d}.h5' cd lab hallmark download --all --dry-run @@ -276,7 +276,7 @@ Frank can prepare a catalog directly from the files on the server:: This creates ``lab/.hm`` with only the matching run files in ``data.tsv``. Without a filter or format, the catalog contains all discovered files below -the supplied URL. No dataset contents are downloaded while cloning. +the supplied URL. Initialization downloads no dataset contents. The final command displays the transfer plan and asks Frank for confirmation. Both SSH and SFTP sources use structured SFTP directory enumeration. The server @@ -287,9 +287,10 @@ an apparently complete catalog. The generated data remote ``origin`` records the source URL and optional profile name. -Existing Git-hosted catalogs keep their recorded data remotes when cloned. -A filter on an existing Git catalog creates a local metadata commit while -preserving fetched history; it does not fetch the dataset's content objects. +Existing Git-hosted catalogs keep their complete catalog, history and +recorded data remotes when cloned. Clone filters select optional downloads +and require ``--with-download``; they leave Git HEAD and catalog rows unchanged. +The Git catalog can be hosted separately from its data servers. Published manifest checksums are recorded in the catalog. Missing checksums remain unknown; |hallmark|_ does not read dataset files locally or on the @@ -300,11 +301,11 @@ be cancelled. CyVerse and common HTTPS directory indexes are detected automatically:: - hallmark clone https://data.desi.lbl.gov/public/ desi --filter '**/*.fits' + hallmark init desi --from https://data.desi.lbl.gov/public/ --filter '**/*.fits' -A server with no usable listing must provide a published Hallmark catalog. -Discovery cannot find files whose URLs are absent from both the listing and -catalog. HTTP/SFTP catalog snapshots start local history; Git endpoints +A server with no usable listing needs a published Hallmark catalog or a +backend plugin that understands its API. See :doc:`backends` for the shared +interface and registration. Discovery cannot infer hidden file URLs. HTTP/SFTP catalog snapshots start local history; Git endpoints supply catalog history. Use ``--source-type git`` for ambiguous Git URLs. Frank can also inspect and approve a transfer in Python:: @@ -319,7 +320,8 @@ Frank can also inspect and approve a transfer in Python:: Plans are built from local metadata without contacting the server. Missing sizes and duration estimates are reported as unknown. During transfer, progress reports bytes and estimates remaining time when possible. -``build`` and ``build_repo`` remain deprecated compatibility interfaces. +``build`` and ``build_repo`` remain available but are deprecated in favor of +``init --from``. Local ``add``, ``commit`` and ``checkout`` retain their existing format and local-object requirements; preparing a remote catalog does not populate the local object store. diff --git a/mod/hallmark/__init__.py b/mod/hallmark/__init__.py index 1bfdf5e..cf30c11 100644 --- a/mod/hallmark/__init__.py +++ b/mod/hallmark/__init__.py @@ -18,5 +18,12 @@ from .repo import Repo from .paraframe import ParaFrame from .download_plan import DownloadItem, DownloadPlan +from .backends import ( + DataBackend, HttpBackend, SshBackend, CyVerseBackend, RemoteEntry, + register_backend, +) -__all__ = ["Repo", "ParaFrame", "DownloadItem", "DownloadPlan"] +__all__ = [ + "Repo", "ParaFrame", "DownloadItem", "DownloadPlan", "DataBackend", + "HttpBackend", "SshBackend", "CyVerseBackend", "RemoteEntry", "register_backend", +] diff --git a/mod/hallmark/backends.py b/mod/hallmark/backends.py new file mode 100644 index 0000000..e31d11d --- /dev/null +++ b/mod/hallmark/backends.py @@ -0,0 +1,97 @@ +"""Public data backends and registration for collaboration-specific adapters. + +Installed adapters declare the ``hallmark.backends`` entry-point group. Only +the selected adapter is imported when opening an operation; parsing remote +configuration and creating download plans never load adapter code. +""" + +from importlib import metadata +from threading import RLock + +from .transport import OperationContext +from .transport.base import ( + CapabilityError, DataBackend, DownloadError, RemoteConfigurationError, + RemoteEntry, RemoteObjectMissing, RemoteSpec, TransferCancelled, backend_name, +) +from .transport.cyverse import CyVerseBackend +from .transport.http import HttpBackend +from .transport.ssh import SshBackend + + +_registered = {"http": HttpBackend, "ssh": SshBackend, "cyverse": CyVerseBackend} +_loaded = {} +_lock = RLock() + + +def _entry_points(name): + """Find matching plugin metadata on Python 3.9 and newer versions.""" + entries = metadata.entry_points() + if hasattr(entries, "select"): + entries = entries.select(group="hallmark.backends") + else: + entries = entries.get("hallmark.backends", ()) + return [entry for entry in entries if entry.name == name] + + +def _validate_class(backend_class): + if (not isinstance(backend_class, type) + or not issubclass(backend_class, DataBackend) + or backend_class is DataBackend): + raise RemoteConfigurationError("Backend must be a DataBackend subclass") + return backend_class + + +def register_backend(name, backend_class): + """ + Register a backend class under a persistent configuration name. + + Args: + name (str): Unique lowercase name using letters, digits, underscores, + and hyphens, beginning with a letter. + backend_class (type[DataBackend]): Class constructed with an + OperationContext when selected by a data remote. + + Raises: + RemoteConfigurationError: If the name is invalid, already registered + or installed, or the class does not implement the base interface. + + Register before opening a repository operation. Distribution entry points + provide automatic registration in fresh Python processes and the CLI. + """ + name = backend_name(name) + _validate_class(backend_class) + with _lock: + if name in _registered or name in _loaded or _entry_points(name): + raise RemoteConfigurationError(f"Backend {name!r} is already registered") + _registered[name] = backend_class + + +def get_backend(name): + """Resolve a registered class, lazily importing a selected installed plugin.""" + name = backend_name(name) + with _lock: + entries = _entry_points(name) + if len(entries) > 1 or (entries and name in _registered): + raise RemoteConfigurationError(f"Duplicate backend name {name!r}") + if name in _registered: + return _registered[name] + if name in _loaded: + return _loaded[name] + if not entries: + raise RemoteConfigurationError( + f"Backend {name!r} is not installed or registered") + try: + backend_class = entries[0].load() + except Exception: + raise RemoteConfigurationError( + f"Unable to load backend {name!r}; check its installation") from None + _loaded[name] = _validate_class(backend_class) + return _loaded[name] + + +__all__ = [ + "DataBackend", "HttpBackend", "SshBackend", "CyVerseBackend", + "register_backend", "RemoteEntry", "RemoteSpec", "OperationContext", + "CapabilityError", "DownloadError", "RemoteConfigurationError", + "RemoteObjectMissing", "TransferCancelled", +] diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py index f373c8c..1790f35 100644 --- a/mod/hallmark/catalog.py +++ b/mod/hallmark/catalog.py @@ -1,4 +1,4 @@ -"""Create local catalogs from Git repositories and remote data directories.""" +"""Initialize remote datasets and clone existing Hallmark catalogs.""" from __future__ import annotations @@ -18,7 +18,8 @@ from .helper_functions import as_list_of_dicts from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name, row_to_path from .transport import OperationContext, RemoteSpec -from .transport.base import RemoteObjectMissing, literal_path +from .transport.base import (RemoteObjectMissing, literal_path, + thaw_backend_options) from .worktree import Worktree @@ -81,8 +82,8 @@ def _snapshot(context): """ Read and validate a published catalog without fetching dataset files. - Return None when required catalog files are absent or ``config.yml`` - does not describe a catalog. Invalid tables and metadata raise CloneError. + Return None when ``config.yml`` is absent. Once present, incomplete or + malformed catalog metadata raises CloneError rather than triggering fallback. """ try: config_text = context.read_text("config.yml") @@ -90,15 +91,16 @@ def _snapshot(context): return None try: config = yaml.safe_load(config_text) - except yaml.YAMLError: - return None + except yaml.YAMLError as exc: + raise CloneError("Invalid published catalog config.yml") from exc if not isinstance(config, dict) or "data" not in config: - return None + raise CloneError("Published catalog config.yml must contain a data mapping") try: meta_text = context.read_text("meta.yml") data_text = context.read_text("data.tsv") - except RemoteObjectMissing: - return None + except RemoteObjectMissing as exc: + raise CloneError( + "Published Hallmark catalog is missing required metadata") from exc try: meta = yaml.safe_load(meta_text) except yaml.YAMLError as exc: @@ -122,31 +124,6 @@ def _metadata_commit(repo, files, message): repo.state = repo.dothm.load() -def _filter_catalog(repo, *, filter=None, fmt=None): - """Filter catalog tables and file entries, then commit the selection.""" - if filter is None and fmt is None: - return - config = repo.state.config - names = _catalog_names(config) - for name in names: - frame = repo.dothm.load_tsv(name) - formats = _catalog_formats(config, name) - keep = [] - for _, row in frame.iterrows(): - path = _row_path(row, formats) - keep.append(path_matches(path, filter=filter, fmt=fmt)) - repo.dothm.dump_tsv(frame.loc[keep], name) - for section in ("data", "meta"): - values = as_list_of_dicts(config.get(section, [])) - if values is None: - continue - config[section] = [entry for entry in values - if not entry.get("file") or path_matches( - entry["file"], filter=filter, fmt=fmt)] - repo.dothm.dump_yml(config, "config") - _metadata_commit(repo, [*names, "config.yml"], "Filter local catalog") - - def _write_inventory(repo, source, entries, fmt): """Write discovered files and their source to a new catalog.""" columns = ["path", "checksum_algorithm", "checksum", "size_bytes", "mtime"] @@ -169,6 +146,10 @@ def _write_inventory(repo, source, entries, fmt): remote = {"name": "origin", "url": source.url} if source.auth: remote["auth"] = source.auth + if source.backend: + remote["backend"] = source.backend + if source.backend_options: + remote["backend_options"] = thaw_backend_options(source.backend_options) repo.dothm.dump_yml({"data": [data_spec], "remote": [remote]}, "config") repo.dothm.dump_yml({"source": source.url}, "meta") repo.dothm.dump_tsv(frame, "data", na_rep="") @@ -176,6 +157,59 @@ def _write_inventory(repo, source, entries, fmt): "Discover remote catalog") +def initialize_remote(cls, path, url, *, backend=None, backend_options=None, + auth=None, filter=None, fmt=None, progress=False): + """Discover a dataset and initialize its catalog without replacing local files.""" + destination = Path(path).expanduser().absolute() + if destination.is_symlink(): + raise DestinationExistsError(f"Destination is a symbolic link: {path}") + dothm_path, _ = cls.lwpaths(destination) + if dothm_path.exists() or dothm_path.is_symlink(): + raise DestinationExistsError( + f"Hallmark repository already exists: {dothm_path}") + if destination.exists() and not destination.is_dir(): + raise NotADirectoryError(f"Destination is not a directory: {path}") + path_matches("validation", filter=filter, fmt=fmt) + source = RemoteSpec.parse(str(url), auth, backend=backend, + backend_options=backend_options) + # Claim only the metadata directory. On failure, existing dataset files stay put. + missing_parents = [] + parent = dothm_path.parent + while not parent.exists(): + missing_parents.append(parent) + parent = parent.parent + created_parents = [] + owns_metadata = False + try: + for parent in reversed(missing_parents): + try: + parent.mkdir() + created_parents.append(parent) + except FileExistsError: + if not parent.is_dir(): + raise + try: + dothm_path.mkdir() + owns_metadata = True + except FileExistsError as exc: + raise DestinationExistsError( + f"Hallmark repository already exists: {dothm_path}") from exc + with OperationContext(source) as context: + entries = discover(context, filter=filter, fmt=fmt, progress=progress) + repo = cls.init(destination) + _write_inventory(repo, source, entries, fmt) + return repo + except BaseException: + if owns_metadata: + rmtree(dothm_path, ignore_errors=True) + for parent in reversed(created_parents): + try: + parent.rmdir() + except OSError: + pass + raise + + def _git_source(url, source_type): """Determine whether the selected source should be cloned with Git.""" if source_type == "git": @@ -183,33 +217,46 @@ def _git_source(url, source_type): if source_type != "auto": return False parsed = urlsplit(url) - return (not parsed.scheme or parsed.scheme in {"git", "file"} - or bool(re.match(r"^[^/]+@[^/]+:", url)) + scp_style = "://" not in url and re.match(r"^(?:[^/@:]+@)?[^/:]+:.+", url) + return (not parsed.scheme or parsed.scheme in {"git", "file", "ssh"} + or bool(scp_style) or (parsed.scheme != "sftp" and parsed.path.rstrip("/").endswith(".git"))) -def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, - source_type="auto", progress=False): +def _clone_git(cls, url, destination, display_path, auth): + """Clone a catalog's Git history without modifying its tracked metadata.""" + if auth is not None: + raise ValueError("Git cloning uses Git/SSH authentication, not auth profiles") + dothm_path, worktree_path = cls.lwpaths(destination) + local = Path(url).expanduser() + git_url = str(local / ".hm") if (local / ".hm").is_dir() else url + try: + Dothm.clone(git_url, dothm_path, display_path=display_path) + except CloneError as exc: + raise CloneError( + f"{exc}\nFor a raw dataset, use hallmark init PATH --from URL.") from exc + if worktree_path: + Worktree.init(worktree_path) + return cls(destination) + + +def clone_catalog(cls, url, path, *, auth=None, source_type="auto"): """ - Create a local catalog from Git, a published snapshot, or a data directory. + Clone an existing Git catalog or published HTTP/SFTP catalog snapshot. - An incomplete destination created by this call is removed on failure. - Existing destinations are rejected before any source access. + Existing destinations are rejected before source access. An incomplete + destination created by this call is removed on failure. Dataset discovery + belongs to ``Repo.init(from_url=...)``. Args: cls: Repository class used to initialize or open the result. - url (str): Catalog location or URL of the dataset directory. + url (str): Existing Git catalog or published snapshot location. path (Path | str): New repository destination. - auth (str, optional): Local SSH profile for data access. - filter (str | list[str], optional): Relative path glob or globs. - fmt (str, optional): Filename format used to select paths. - source_type (str): ``auto``, ``git``, ``directory``, or ``catalog``. - Defaults to automatic detection. - progress (bool | callable): Discovery progress display or callback. - Defaults to False. + auth (str, optional): Local profile for snapshot metadata access. + source_type (str): ``auto``, ``git``, or ``catalog``. Returns: - Repo: Repository containing catalog metadata without dataset files. + Repo: Repository containing complete catalog metadata without payloads. Raises: DestinationExistsError: If the destination already exists. @@ -217,10 +264,9 @@ def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, ValueError: If source options are invalid. DownloadError: If remote metadata cannot be read or validated. """ - if source_type not in {"auto", "git", "directory", "catalog"}: - raise ValueError("source_type must be auto, git, directory, or catalog") + if source_type not in {"auto", "git", "catalog"}: + raise ValueError("source_type must be auto, git, or catalog") url = str(url) - path_matches("validation", filter=filter, fmt=fmt) destination = Path(path).expanduser().absolute() if destination.exists() or destination.is_symlink(): raise DestinationExistsError( @@ -228,7 +274,6 @@ def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, is_git = _git_source(url, source_type) if is_git and auth is not None: raise ValueError("Git cloning uses Git/SSH authentication, not auth profiles") - # Create the destination exclusively so cleanup only removes our own directory. destination.parent.mkdir(parents=True, exist_ok=True) try: destination.mkdir() @@ -236,38 +281,23 @@ def clone_catalog(cls, url, path, *, auth=None, filter=None, fmt=None, raise DestinationExistsError(f"Destination already exists: {path}") from exc try: if is_git: - dothm_path, worktree_path = cls.lwpaths(destination) - local = Path(url).expanduser() - git_url = str(local / ".hm") if (local / ".hm").is_dir() else url - Dothm.clone(git_url, dothm_path, display_path=path) - if worktree_path: - Worktree.init(worktree_path) - repo = cls(destination) - _filter_catalog(repo, filter=filter, fmt=fmt) - return repo - + return _clone_git(cls, url, destination, path, auth) source = RemoteSpec.parse(url, auth) - snapshot = None with OperationContext(source) as context: - if source_type != "directory": + snapshot = _snapshot(context) + if snapshot is None and not source.root.rstrip("/").endswith(".hm"): + nested = RemoteSpec.parse(url.rstrip("/") + "/.hm/", auth) + with OperationContext(nested) as context: snapshot = _snapshot(context) - if snapshot is None and not source.root.rstrip("/").endswith(".hm"): - nested = RemoteSpec.parse(url.rstrip("/") + "/.hm/", auth) - with OperationContext(nested) as nested_context: - snapshot = _snapshot(nested_context) - if snapshot is None: - if source_type == "catalog": - raise CloneError("No published Hallmark catalog at this URL") - entries = discover(context, filter=filter, fmt=fmt, - progress=progress) + if snapshot is None: + if source_type == "auto" and source.scheme in {"http", "https"}: + return _clone_git(cls, url, destination, path, auth) + raise CloneError("No published Hallmark catalog at this URL; " + "use hallmark init PATH --from URL for a raw dataset") repo = cls.init(destination) - if snapshot is not None: - for name, contents in snapshot.items(): - (repo.dothm.path / name).write_text(contents, encoding="utf-8") - _metadata_commit(repo, snapshot, "Import published catalog snapshot") - _filter_catalog(repo, filter=filter, fmt=fmt) - else: - _write_inventory(repo, source, entries, fmt) + for name, contents in snapshot.items(): + (repo.dothm.path / name).write_text(contents, encoding="utf-8") + _metadata_commit(repo, snapshot, "Import published catalog snapshot") return repo except BaseException: rmtree(destination, ignore_errors=True) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 8794d34..050c0a9 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -27,6 +27,7 @@ from .helper_functions import validate_path_component from .repo_builder import build_repo from .downloader import DownloadError +from .discovery import path_matches from .error import CheckoutError, CloneError from .repo_config import normalize_tsv_name @@ -157,19 +158,59 @@ def hallmark(ctx): ctx.obj = Repo(".") +def _backend_options_file(path): + """Read a backend's configuration from a YAML mapping.""" + if path is None: + return None + with Path(path).open(encoding="utf-8") as handle: + options = yaml.safe_load(handle) + if not isinstance(options, dict): + raise ValueError("Backend options file must contain a YAML mapping") + return options + + @hallmark.command(short_help="Initialize a hallmark repository.") @click.argument("path") -def init(path): +@click.option("--from", "from_url", help="Discover a raw remote dataset.") +@click.option("--backend", help="Registered data backend name.") +@click.option("--backend-options", type=click.Path(exists=True, dir_okay=False), + help="YAML mapping of backend-specific options.") +@click.option("--auth", help="Optional local authentication profile for data access.") +@click.option("--filter", "filters", multiple=True, + help="Include paths matching a glob. May be repeated.") +@click.option("--fmt", help="Select paths and extract filename parameters.") +@click.option("--with-download", is_flag=True, + help="Review and approve a download after catalog creation.") +@click.option("--max-workers", type=click.IntRange(min=1), default=4, + show_default=True) +def init(path, from_url, backend, backend_options, auth, filters, fmt, + with_download, max_workers): """Initialize a hallmark repository at PATH. + Use --from URL to discover a remote dataset without downloading payloads. If PATH ends with `.hm`, a bare repository is created. Otherwise, a `.hm` directory is created inside PATH. """ - # attempt to initialize the hallmark repository at the specified path with _translate_cli_errors( - GitError, - prefix=("Failed to initialize hallmark repository " f'at "{path}"')): - Repo.init(path) + GitError, DownloadError, ValueError, OSError, yaml.YAMLError, + prefix=f'Failed to initialize hallmark repository at "{path}"'): + options = _backend_options_file(backend_options) + if with_download and not from_url: + raise ValueError("--with-download requires --from") + if with_download and Repo.lwpaths(path)[1] is None: + raise ValueError("Use a worktree destination for --with-download") + kwargs = dict(from_url=from_url, backend=backend, backend_options=options, + auth=auth, filter=filters or None, fmt=fmt, + progress=True, max_workers=max_workers) + if from_url is None and all(value is None for value in + (backend, options, auth, filters or None, fmt)): + repo = Repo.init(path) + else: + repo = Repo.init(path, **kwargs) + if from_url is not None: + click.echo(f'Successfully initialized "{path}"') + if with_download: + _run_download(repo, repo.plan_download(), max_workers=max_workers) @hallmark.command(short_help="Show information of the current directory.") @@ -279,14 +320,19 @@ def add(repo, encoding, inputs): @click.option("--remote-url") @click.option("--remote-auth", help="Local SSH profile name. An empty string removes the reference.") +@click.option("--remote-backend", help="Registered data backend name.") +@click.option("--remote-backend-options", type=click.Path(exists=True, dir_okay=False), + help="YAML mapping of backend-specific options.") @click.option("--encoding", "encodings", multiple=True) @click.pass_obj -def set_config(repo, fmt, remote_name, remote_url, remote_auth, encodings): +def set_config(repo, fmt, remote_name, remote_url, remote_auth, remote_backend, + remote_backend_options, encodings): """Update the current branch config.yml.""" # if no config changes are requested, raise a ClickException to inform the user if ( fmt is None and remote_name is None and remote_url is None - and remote_auth is None and not encodings): + and remote_auth is None and remote_backend is None + and remote_backend_options is None and not encodings): raise ClickException("No config changes requested.") encoding_updates = {} for item in encodings: @@ -298,13 +344,17 @@ def set_config(repo, fmt, remote_name, remote_url, remote_auth, encodings): encoding_updates[field.strip()] = regex # use the _translate_cli_errors context manager to handle specific exceptions - with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError): + with _translate_cli_errors(RuntimeError, ValueError, OSError, yaml.YAMLError): + options = _backend_options_file(remote_backend_options) repo.set_config( fmt=fmt, remote_name=remote_name, remote_url=remote_url, encoding_updates=encoding_updates or None, - **({"remote_auth": remote_auth} if remote_auth is not None else {})) + **({"remote_auth": remote_auth} if remote_auth is not None else {}), + **({"remote_backend": remote_backend} + if remote_backend is not None else {}), + **({"remote_backend_options": options} if options is not None else {})) click.echo("Updated hallmark config.") @@ -426,7 +476,7 @@ def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, _run_download(repo, plan, max_workers=max_workers) -@hallmark.command(short_help="Clone a catalog or discover a remote dataset.") +@hallmark.command(short_help="Clone an existing Hallmark catalog.") @click.argument("url") @click.argument("path") @click.option("--auth", help="Optional local SSH authentication profile.") @@ -434,11 +484,12 @@ def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, help="Select paths matching a glob. ** matches recursively. " "May be repeated.") @click.option("--fmt", - help="Select paths and extract parameters using a filename format.") + help="Select downloads using a filename format; " + "requires --with-download.") @click.option("--source-type", default="auto", show_default=True, - type=click.Choice(["auto", "git", "directory", "catalog"]), + type=click.Choice(["auto", "git", "catalog"]), help="Override automatic source detection.") -@click.option("--download", "fetch_data", is_flag=True, +@click.option("--with-download", "--download", "fetch_data", is_flag=True, help="Request a download after catalog creation, with confirmation.") @click.option("--no-fetch-data", is_flag=True, hidden=True) @click.option("--max-workers", type=click.IntRange(min=1), default=4, @@ -447,19 +498,25 @@ def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, max_workers, yes): """ - Clone a catalog or index a remote data directory at PATH. + Clone an existing Git catalog or published catalog snapshot at PATH. - Dataset files are not downloaded by default. Use --download to review - and approve a transfer after the catalog has been created. + Dataset files are not downloaded by default. Use --with-download to review + and approve a transfer. --filter and --fmt select downloads while preserving + the complete catalog and Git history. Use init --from for raw datasets. """ if fetch_data and no_fetch_data: raise ClickException("--download conflicts with --no-fetch-data") + if (filters or fmt is not None) and not fetch_data: + raise ClickException("--filter and --fmt require --with-download") + if fetch_data and Repo.lwpaths(path)[1] is None: + raise ClickException("Use a worktree destination for --with-download") if yes: click.echo("--yes is deprecated; downloads still require confirmation.", err=True) with _translate_cli_errors(DownloadError, GitError, ValueError): + path_matches("validation", filter=filters or None, fmt=fmt) try: - repo = Repo.clone(url, path, auth=auth, filter=filters or None, fmt=fmt, + repo = Repo.clone(url, path, auth=auth, source_type=source_type, progress=True, max_workers=max_workers) except CloneError as exc: @@ -469,11 +526,12 @@ def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, if fetch_data: if repo.worktree is None: raise ClickException("Use a worktree destination for --download") - _run_download(repo, repo.plan_download(), max_workers=max_workers) + plan = repo.plan_download(filter=filters or None, fmt=fmt) + _run_download(repo, plan, max_workers=max_workers) -@hallmark.command(short_help="Deprecated: use clone for remote datasets.") +@hallmark.command(short_help="Deprecated: use init --from for remote datasets.") @click.argument("directory") @click.argument("dataset_name") @click.option( @@ -507,7 +565,7 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, """ Build a catalog at DIRECTORY/DATASET_NAME.hm. - This command is deprecated. Use hallmark clone URL PATH for remote + This command is deprecated. Use hallmark init --from URL PATH for remote datasets, or hallmark init PATH to create a local repository. --dataset-url selects the exact discovery root; otherwise DATASET_NAME @@ -518,7 +576,7 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, provided, existing formats are reused or a generic path catalog is created. Dataset files are not downloaded during catalog creation. """ - click.echo("build is deprecated; use hallmark clone URL PATH.", err=True) + click.echo("build is deprecated; use hallmark init --from URL PATH.", err=True) if config_file and fmts: raise ClickException("Use only one of --config-file or --fmt, not both.") # validate the dataset name to ensure it is a valid path component diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py index 001f143..6d83fe5 100644 --- a/mod/hallmark/discovery.py +++ b/mod/hallmark/discovery.py @@ -4,20 +4,16 @@ import fnmatch import re -from collections import deque from dataclasses import replace from functools import lru_cache -from html.parser import HTMLParser from pathlib import PurePosixPath -from urllib.parse import quote, unquote, urljoin, urlsplit import parse from tqdm import tqdm from .helper_functions import CHECKSUM_ALGORITHMS_BY_STRENGTH, valid_checksum from .transport.base import ( - CapabilityError, DownloadError, RemoteEntry, RemoteObjectMissing, - literal_path, reject_controls, + DownloadError, RemoteEntry, RemoteObjectMissing, literal_path, ) @@ -75,150 +71,6 @@ def path_matches(path: str, filter=None, fmt: str | None = None) -> bool: return parser is None or parser.parse(path) is not None -class _IndexParser(HTMLParser): - """Read links and recognizable index markers without depending on layout.""" - - def __init__(self): - super().__init__(convert_charrefs=True) - self.links = [] - self.text = [] - self.row_kind = None - self.listing = False - self.has_table = False - self.has_form = False - self.current = None - self.in_anchor = False - self.cell_role = None - - def handle_starttag(self, tag, attrs): - attrs = dict(attrs) - if tag == "table": - self.has_table = True - elif tag == "form": - self.has_form = True - elif tag == "tr": - self.current = None - classes = attrs.get("class", "").split() - self.row_kind = None - if "object" in classes: - self.listing = True - self.row_kind = "directory" if "collection" in classes else "file" - elif tag == "td": - self.cell_role = attrs.get("class", "") - elif tag == "a" and "href" in attrs: - self.current = {"href": attrs["href"], "kind": self.row_kind, - "tail": "", "size": None, "mtime": None} - for field in ("size", "mtime"): - value = attrs.get(f"data-{field}") - if value is not None and value.isdigit(): - self.current[field] = int(value) - self.links.append(self.current) - self.in_anchor = True - - def handle_endtag(self, tag): - if tag == "tr": - self.row_kind = None - self.current = None - elif tag == "a": - self.in_anchor = False - elif tag == "td": - if self.current is not None: - self.current["tail"] += " " - self.cell_role = None - - def handle_data(self, data): - self.text.append(data) - if self.current is not None and not self.in_anchor: - self.current["tail"] += data - if self.cell_role in {"size", "bytes"}: - raw = data.strip().replace(",", "") - if raw.isdigit(): - self.current["size"] = int(raw) - - -def _origin(url): - """Return connection fields used to compare directory-listing URLs.""" - return (url.scheme, url.hostname, - url.port or (443 if url.scheme == "https" else 80), - url.username, url.password) - - -def _index_link(root_url, directory, href, kind): - """Resolve a link only when it stays beneath the supplied URL root.""" - reject_controls(href, "Index href") - root = urlsplit(root_url.rstrip("/") + "/") - current = urljoin(root.geturl(), quote(directory, safe="/")) - resolved = urlsplit(urljoin(current, href)) - if _origin(resolved) != _origin(root): - return None - # Sorting and navigation links do not identify dataset files. - if resolved.query or resolved.fragment: - return None - root_path = unquote(root.path, errors="strict") - path = unquote(resolved.path, errors="strict") - reject_controls(path, "Index path") - if "\\" in path or ".." in path.split("/"): - raise DownloadError("Remote index contains an unsafe path") - if not path.startswith(root_path): - return None - relative = path[len(root_path):] - if not relative: - return None - if any(part.lower() in {".hm", ".git"} for part in relative.split("/")): - return None - is_directory = kind == "directory" or resolved.path.endswith("/") - relative = literal_path(relative.rstrip("/")).as_posix() - return relative + ("/" if is_directory else ""), is_directory - - -def _parse_index(text, root_url, directory): - """Extract file and directory metadata from a supported HTML index.""" - parser = _IndexParser() - parser.feed(text) - links = [] - for entry in parser.links: - link = _index_link(root_url, directory, entry["href"], entry["kind"]) - if link is not None: - path, is_directory = link - size = entry["size"] - if size is None: - tail = re.search( - r"(?:\d{4}-\d\d-\d\d|\d\d-[A-Za-z]{3}-\d{4})" - r"\s+\d\d:\d\d(?::\d\d)?\s+(\d+)(?:\s|$)", entry["tail"]) - if tail: - size = int(tail.group(1)) - links.append((path, is_directory, size, entry["mtime"])) - markers = " ".join(parser.text).lower() - recognized = (parser.listing or "index of" in markers - or "directory listing" in markers or "parent directory" in markers - or any(is_dir for _, is_dir, _, _ in links)) - # CyVerse can emit a completely empty table without row-level markers. - empty_table = (parser.has_table and not parser.links - and not parser.has_form and not markers.strip()) - if not recognized and not empty_table: - raise CapabilityError( - "This URL exposes no usable directory listing. Use a browsable " - "directory URL or a published Hallmark catalog.") - return links - - -def _response_directory(context, requested): - """Use a redirected listing's canonical location when resolving child links.""" - transport = getattr(context, "transport", None) - final_url = getattr(transport, "text_urls", {}).get(requested) - if final_url is None: - return requested - root = urlsplit(context.remote.url.rstrip("/") + "/") - final = urlsplit(final_url) - if (_origin(root) == _origin(final) - and unquote(root.path).rstrip("/") == unquote(final.path).rstrip("/")): - return "" - link = _index_link(context.remote.url, "", final_url, "directory") - if link is None: - raise DownloadError("Redirected directory is outside the dataset root") - return link[0] - - def _manifest_algorithm(path): """Identify checksum manifests by their conventional filenames.""" name = PurePosixPath(path).name.lower() @@ -326,31 +178,10 @@ def add(entry): report() try: - if context.remote.scheme in {"ssh", "sftp"}: - for entry in context.transport.iter_entries(on_directory=report): - context.check_cancelled() - add(entry) - else: - queue, visited = deque([""]), set() - while queue: - directory = queue.popleft() - if directory in visited: - continue - visited.add(directory) - context.check_cancelled() - text = context.read_text(directory) - canonical = _response_directory(context, directory) - if canonical != directory and canonical in visited: - continue - visited.add(canonical) - links = _parse_index(text, context.remote.url, canonical) - for path, is_directory, size, mtime in links: - if is_directory: - if path not in visited: - queue.append(path) - else: - add(RemoteEntry(path=path, size=size, mtime=mtime)) - report(canonical) + context.transport.prepare() + for entry in context.transport.iter_entries(on_directory=report): + context.check_cancelled() + add(entry) _manifest_checksums(context, entries) return [entries[path] for path in sorted(entries) if path_matches(path, filter=filter, fmt=fmt)] diff --git a/mod/hallmark/download_plan.py b/mod/hallmark/download_plan.py index df1ef0e..47a9e6f 100644 --- a/mod/hallmark/download_plan.py +++ b/mod/hallmark/download_plan.py @@ -2,12 +2,15 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass, field from math import isfinite from pathlib import Path from typing import Optional, Union from urllib.parse import urlsplit, urlunsplit +from .transport.base import backend_name, freeze_backend_options + @dataclass(frozen=True) class DownloadItem: @@ -63,6 +66,8 @@ class DownloadPlan: remote_name (str, optional): Name of the selected data remote. estimated_bytes_per_second (float, optional): Positive rate supplied by the caller for duration estimates. + remote_backend (str, optional): Registered backend selected for transfer. + backend_options (mapping): Immutable backend configuration snapshot. """ items: tuple[DownloadItem, ...] @@ -71,6 +76,8 @@ class DownloadPlan: remote_auth: Optional[str] = None remote_name: Optional[str] = None estimated_bytes_per_second: Optional[float] = None + remote_backend: Optional[str] = None + backend_options: Mapping = field(default_factory=dict, repr=False, hash=False) def __post_init__(self) -> None: """Copy the selection and validate the source fields and rate.""" @@ -80,8 +87,13 @@ def __post_init__(self) -> None: if any(not isinstance(item, DownloadItem) for item in self.items): raise TypeError("items must contain DownloadItem values") if any(value is not None and not isinstance(value, str) for value in - (self.remote_url, self.remote_auth, self.remote_name)): + (self.remote_url, self.remote_auth, self.remote_name, + self.remote_backend)): raise TypeError("remote fields must be strings or None") + if self.remote_backend is not None: + backend_name(self.remote_backend) + object.__setattr__(self, "backend_options", + freeze_backend_options(self.backend_options)) rate = self.estimated_bytes_per_second if rate is not None and ( isinstance(rate, bool) or not isinstance(rate, (int, float)) @@ -134,6 +146,7 @@ def summary(self) -> str: parsed = urlsplit(self.remote_url) source = urlunsplit(parsed._replace( netloc=parsed.netloc.rsplit("@", 1)[-1], query="", fragment="")) + backend = f"\nBackend: {self.remote_backend}" if self.remote_backend else "" return (f"{self.file_count} file(s); {size}; estimated duration: {duration}" - f"\nSource: {source}" + f"\nSource: {source}{backend}" f"\nDestination: {self.output_path}") diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index 3259187..1a604b8 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -898,8 +898,11 @@ def plan_download( if items and (remote is None or not remote.get("url")): raise DownloadError("No remote URL is configured in config.yml") remote = remote or {} + source = None if remote.get("url"): - RemoteSpec.parse(remote["url"], remote.get("auth")) + source = RemoteSpec.parse( + remote["url"], remote.get("auth"), backend=remote.get("backend"), + backend_options=remote.get("backend_options")) for item in items: try: resolve_contained_path(output_root, item.relative_path, @@ -909,7 +912,9 @@ def plan_download( return DownloadPlan( tuple(items), remote.get("url"), output_root, remote_auth=remote.get("auth"), remote_name=remote.get("name"), - estimated_bytes_per_second=estimated_bytes_per_second) + estimated_bytes_per_second=estimated_bytes_per_second, + remote_backend=source.backend if source is not None else None, + backend_options=remote.get("backend_options")) def execute_download_plan( @@ -949,7 +954,9 @@ def execute_download_plan( return {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} if plan.output_path.resolve() != plan.output_path: raise DownloadError("Download destination changed since planning") - remote = RemoteSpec.parse(plan.remote_url, plan.remote_auth) + remote = RemoteSpec.parse( + plan.remote_url, plan.remote_auth, backend=plan.remote_backend, + backend_options=plan.backend_options) return _download_selected( remote, plan.output_path, [(item.relative_path, item.checksum) for item in plan.items], @@ -1020,7 +1027,9 @@ def download_remote_data( # if the remote URL is not configured, raise a DownloadError to indicate the issue if not remote_url: raise DownloadError("Remote URL not configured in config.yml") - remote_spec = RemoteSpec.parse(remote_url, remote_config.get("auth")) + remote_spec = RemoteSpec.parse( + remote_url, remote_config.get("auth"), backend=remote_config.get("backend"), + backend_options=remote_config.get("backend_options")) # If there are still no files selected for download if not selected_files: # return the results without attempting any downloads diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 666b4b6..b48b00a 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -176,16 +176,59 @@ def _populate_checksums(self, pf: ParaFrame) -> None: pf["sha1"] = [checksums[path] for path in full_paths] @classmethod - def init(cls, path: Union[Path, str]) -> "Repo": - ''' - Initialize a new hallmark repository. + def init(cls, path: Union[Path, str], *, from_url=None, backend=None, + backend_options=None, auth=None, filter=None, fmt=None, + download=False, approve=None, progress=False, max_workers=4) -> "Repo": + """ + Initialize a local repository, optionally discovering a remote dataset. + + Remote discovery reads listings and published checksums without fetching + payloads. Existing destination files are preserved; an existing ``.hm`` + is rejected before contacting the remote dataset. Args: - paht(paht|string): path to initialize a worktree or ``.hm`` repository. + path (Path | str): Worktree or bare ``.hm`` repository destination. + from_url (str, optional): Raw remote dataset to discover. + backend (str, optional): Registered data backend name. + backend_options (dict, optional): Backend-specific configuration. + auth (str, optional): Local authentication profile for data access. + filter (str | list[str], optional): Relative path glob or globs. + fmt (str, optional): Filename format for selection and parameters. + download (bool): Request approved downloads after discovery. + approve (callable, optional): Callback returning True to approve a plan. + progress (bool | callable): Discovery and download progress display. + max_workers (int): Maximum download workers. Defaults to 4. Returns: - Repo: newly created repository instance - ''' + Repo: Initialized repository, retained if download approval is declined. + + Raises: + DestinationExistsError: If remote initialization would replace a catalog. + ValueError: If remote options are supplied without ``from_url``. + DownloadError: If discovery fails or download approval is unavailable. + """ + from .catalog import initialize_remote + from .downloader import DownloadError, _require_positive_integer + + _require_positive_integer(max_workers, label="max_workers") + if from_url is not None: + if download and not callable(approve): + raise DownloadError( + "Downloading during init requires an approve(plan) callback") + if download and cls.lwpaths(path)[1] is None: + raise DownloadError( + "Initialize a worktree to download; " + "bare catalogs need an output path") + repo = initialize_remote( + cls, path, from_url, backend=backend, backend_options=backend_options, + auth=auth, filter=filter, fmt=fmt, progress=progress) + if download: + repo._download_after_creation( + approve=approve, max_workers=max_workers, progress=progress) + return repo + if any(value is not None for value in + (backend, backend_options, auth, filter, fmt, approve)) or download: + raise ValueError("Remote initialization options require from_url") dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path / "config.yml").write_text(Dothm.config_template(), @@ -197,6 +240,21 @@ def init(cls, path: Union[Path, str]) -> "Repo": Worktree.init(worktree_path) return cls(path) + def _download_after_creation(self, *, approve, max_workers, progress, + filter=None, fmt=None): + """Run an approved transfer after catalog creation has completed.""" + from .downloader import DownloadError + + plan = self.plan_download(filter=filter, fmt=fmt) + if plan.file_count and approve(plan) is True: + self.download_result = self.download( + plan, approved=True, max_workers=max_workers, progress=progress) + if self.download_result["failed"]: + details = "\n".join(self.download_result["errors"][:5]) + raise DownloadError( + f"Failed to download {self.download_result['failed']} " + f"file(s):\n{details}") + @classmethod def clone( cls, @@ -215,25 +273,25 @@ def clone( show_progress: Optional[bool] = None, ) -> "Repo": """ - Clone a Hallmark catalog or index a remote data directory. + Clone an existing Hallmark Git catalog or published snapshot. - Git sources retain their history. Published catalog snapshots and - discovered directories start new local history. Dataset files are + Git sources retain their history. Published catalog snapshots + start new local history. Dataset files are downloaded only when requested and approved after catalog creation. Declining approval leaves the catalog available without dataset files. Args: - url (str): Catalog location or URL of the dataset directory. + url (str): Existing Git catalog or published snapshot location. path (Path | str): New worktree or bare ``.hm`` repository path. - auth (str, optional): Local SSH profile for data access. Git - sources use Git's authentication configuration instead. - filter (str | list[str], optional): Relative path glob or globs. - A path must match at least one glob when provided. - fmt (str, optional): Filename format used to select paths and - extract parameters when discovering a directory. - source_type (str): ``auto``, ``git``, ``directory``, or ``catalog``. + auth (str, optional): Local profile for snapshot metadata access. + Git sources use Git's authentication configuration instead. + filter (str | list[str], optional): Download selection globs. + Requires ``download=True``; the catalog remains complete. + fmt (str, optional): Filename format used to select downloads. + Requires ``download=True``. + source_type (str): ``auto``, ``git``, or ``catalog``. Defaults to automatic source detection. - progress (bool): Show discovery and download progress. Defaults + progress (bool): Show download progress. Defaults to False. download (bool): Request downloads after preparing the catalog. Defaults to False and requires an approval callback when True. @@ -251,10 +309,11 @@ def clone( DestinationExistsError: If the destination already exists. CloneError: If a requested catalog is missing or invalid. ValueError: If source options are invalid or conflict. - DownloadError: If discovery or downloading fails, or a download + DownloadError: If metadata access or downloading fails, or a download is requested without a callback or worktree destination. """ from .catalog import clone_catalog + from .discovery import path_matches from .downloader import DownloadError, _require_positive_integer _require_positive_integer(max_workers, label="max_workers") @@ -264,24 +323,21 @@ def clone( download = fetch_data if show_progress is not None: progress = show_progress + if (filter is not None or fmt is not None) and not download: + raise ValueError("clone filter and fmt require download=True; " + "use init(from_url=...) to select a new catalog") if download and not callable(approve): raise DownloadError( "Downloading during clone requires an approve(plan) callback") if download and cls.lwpaths(path)[1] is None: raise DownloadError( "Clone a worktree to download; bare catalogs need an output path") - repo = clone_catalog(cls, url, path, auth=auth, filter=filter, fmt=fmt, - source_type=source_type, progress=progress) + path_matches("validation", filter=filter, fmt=fmt) + repo = clone_catalog(cls, url, path, auth=auth, source_type=source_type) if download: - plan = repo.plan_download() - if plan.file_count and approve(plan) is True: - repo.download_result = repo.download( - plan, approved=True, max_workers=max_workers, progress=progress) - if repo.download_result["failed"]: - details = "\n".join(repo.download_result["errors"][:5]) - raise DownloadError( - f"Failed to download {repo.download_result['failed']} " - f"file(s):\n{details}") + repo._download_after_creation( + approve=approve, max_workers=max_workers, progress=progress, + filter=filter, fmt=fmt) return repo def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, @@ -407,6 +463,8 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, remote_auth: Optional[str] = None, + remote_backend: Optional[str] = None, + remote_backend_options: Optional[dict] = None, ) -> dict: """ Update repository configuration values. @@ -417,6 +475,8 @@ def set_config( remote_url (str, optional): URL of the remote repository. remote_auth (str, optional): Local SSH profile name. An empty string removes the reference; None leaves it unchanged. + remote_backend (str, optional): Registered data backend name. + remote_backend_options (dict, optional): Backend-specific configuration. encoding_updates (dict[str, str], optional): Updates to encoding rules. Returns: @@ -428,7 +488,11 @@ def set_config( remote_name=remote_name, remote_url=remote_url, encoding_updates=encoding_updates, - **({"remote_auth": remote_auth} if remote_auth is not None else {})) + **({"remote_auth": remote_auth} if remote_auth is not None else {}), + **({"remote_backend": remote_backend} + if remote_backend is not None else {}), + **({"remote_backend_options": remote_backend_options} + if remote_backend_options is not None else {})) self.dothm.dump(self.state) return self.state.config diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index e04ada6..527f16d 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -16,7 +16,7 @@ from .repo import Repo from .transport import OperationContext, RemoteSpec from .transport.base import ( - CapabilityError, DownloadError, reject_controls) + CapabilityError, DownloadError, reject_controls, thaw_backend_options) from .fmt_detection import ( KNOWN_PROCESSING_STAGES, KNOWN_STATIC_FILE_STEMS) @@ -720,13 +720,15 @@ def build_repo( index_format: str | None = None, allow_remote_commands: bool = False, remote_hash: bool = False, + backend: str | None = None, + backend_options=None, ) -> "Repo": """ Build a remote catalog using the deprecated builder interface. - Prefer ``Repo.clone(url, path)`` for remote catalogs and ``Repo.init(path)`` - for local repositories. Discovery reads listings and published checksum - manifests without downloading dataset files. + Prefer ``Repo.init(path, from_url=url)`` for remote datasets. + Discovery reads listings and published checksum manifests without + downloading dataset files. Args: repo_path (Path): Destination repository path. @@ -749,6 +751,8 @@ def build_repo( allow_remote_commands (bool): Obsolete option, ignored when supplied. remote_hash (bool): Unsupported when True. Defaults to False; discovery does not compute checksums from dataset contents. + backend (str, optional): Registered discovery backend name. + backend_options (mapping, optional): Non-secret backend configuration. Returns: Repo: Repository containing the prepared catalog. @@ -761,7 +765,7 @@ def build_repo( DownloadError: If remote discovery fails. """ warnings.warn( - "build_repo is deprecated; use Repo.clone(url, path) or Repo.init(path)", + "build_repo is deprecated; use Repo.init(path, from_url=url)", DeprecationWarning, stacklevel=2) if remote_hash: raise CapabilityError( @@ -776,7 +780,8 @@ def build_repo( dataset_name = validate_path_component(dataset_name, label="dataset name") base_url = (dataset_url if dataset_url is not None else _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/")) - source = RemoteSpec.parse(base_url, dataset_auth) + source = RemoteSpec.parse( + base_url, dataset_auth, backend=backend, backend_options=backend_options) with OperationContext(source) as context: return _build_repo(repo_path, dataset_name, fmt_entries, config_file, remotes, overwrite, context) @@ -1054,8 +1059,19 @@ def _ensure_remote_files_listed(): remotes = [{"name": "origin"}] if source.remote.auth is not None: remotes[0]["auth"] = source.remote.auth + if source.remote.backend_options: + remotes[0]["backend_options"] = thaw_backend_options( + source.remote.backend_options) # create the final remotes list by adding the base_url to each remote entry final_remotes = [{"url": base_url, **remote} for remote in remotes] + default_backend = RemoteSpec.parse(base_url, source.remote.auth).backend + for remote in final_remotes: + if remote["url"] == base_url: + if source.remote.backend != default_backend: + remote.setdefault("backend", source.remote.backend) + if source.remote.backend_options: + remote.setdefault("backend_options", thaw_backend_options( + source.remote.backend_options)) repo.state.config["remote"] = final_remotes # if a meta file entry exists, add it to the config diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 799286d..c7ee874 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -16,7 +16,8 @@ as_list_of_dicts, coerce_fmt_value, normalize_nonempty_string, validate_path_component, validate_relative_path) -from .transport.base import RemoteSpec, profile_name, reject_controls +from .transport.base import ( + RemoteSpec, backend_name, profile_name, reject_controls, thaw_backend_options) def _update_remote_config( @@ -24,6 +25,8 @@ def _update_remote_config( remote_name: Optional[str], remote_url: Optional[str], remote_auth: Optional[str] = None, + remote_backend: Optional[str] = None, + remote_backend_options=None, ) -> None: """ Used by set_config. @@ -35,6 +38,10 @@ def _update_remote_config( remote_url (str, optional): New remote repository URL. remote_auth (str, optional): Local SSH profile name. An empty string removes the reference; None leaves it unchanged. + remote_backend (str, optional): Registered backend name. An empty string + restores automatic selection; None leaves it unchanged. + remote_backend_options (mapping, optional): Replace backend options. + An empty mapping clears options; None leaves them unchanged. Raises: ValueError: If the remote configuration is invalid or if the specified @@ -92,6 +99,13 @@ def _update_remote_config( selected.pop("auth", None) elif remote_auth is not None: selected["auth"] = profile_name(remote_auth) + if remote_backend == "": + selected.pop("backend", None) + elif remote_backend is not None: + selected["backend"] = backend_name(remote_backend) + if remote_backend_options is not None: + selected["backend_options"] = thaw_backend_options( + remote_backend_options) # normalize the remotes configuration to ensure it is a list of dictionaries normalized = normalize_remotes(remotes) # if preserve_list is True, store the normalized list; @@ -167,14 +181,23 @@ def normalize_remotes(remotes) -> list[dict]: else: raise ValueError(f"remote {index} must be a string or dictionary") - if set(entry) - {"name", "url", "auth"}: - raise ValueError("Remote entries support only name, url and auth fields") + if set(entry) - {"name", "url", "auth", "backend", "backend_options"}: + raise ValueError( + "Remote entries support only name, url, auth, backend and " + "backend_options fields") if "auth" in entry: profile_name(entry["auth"]) + if "backend" in entry: + entry["backend"] = backend_name(entry["backend"]) + if "backend_options" in entry: + entry["backend_options"] = thaw_backend_options( + entry["backend_options"]) if isinstance(entry.get("url"), str): reject_controls(entry["url"], "Remote URL") # allow names without URLs while configuring a data remote - RemoteSpec.parse(entry["url"], entry.get("auth")) + RemoteSpec.parse( + entry["url"], entry.get("auth"), backend=entry.get("backend"), + backend_options=entry.get("backend_options")) # for each required key ("name" and "url"), validate that it exists for key in ("name", "url"): # if the key is not present in the entry, skip to the next key @@ -341,6 +364,8 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, remote_auth: Optional[str] = None, + remote_backend: Optional[str] = None, + remote_backend_options=None, ) -> dict: """ Update the repository configuration. @@ -355,6 +380,10 @@ def set_config( remote_url (str, optional): Remote repository URL. remote_auth (str, optional): Local SSH profile name. An empty string removes the reference; None leaves it unchanged. + remote_backend (str, optional): Registered backend name. An empty string + restores automatic selection; None leaves it unchanged. + remote_backend_options (mapping, optional): Replace backend options. + An empty mapping clears options; None leaves them unchanged. encoding_updates (dict, optional): Encoding values to merge into the existing configuration. @@ -437,8 +466,11 @@ def set_config( config["data"][0] = updated_spec # update the remote when its name, URL, or profile reference changes - if remote_name is not None or remote_url is not None or remote_auth is not None: - _update_remote_config(config, remote_name, remote_url, remote_auth) + if any(value is not None for value in ( + remote_name, remote_url, remote_auth, + remote_backend, remote_backend_options)): + _update_remote_config(config, remote_name, remote_url, remote_auth, + remote_backend, remote_backend_options) repo.state.config = config return config @@ -533,4 +565,4 @@ def row_to_path(row, fmt: str) -> Path: # render the path using the format string and coerced values rendered_path = fmt.format(**values) # validate the rendered path to ensure it is a safe relative path - return validate_relative_path(rendered_path, label="formatted data path") \ No newline at end of file + return validate_relative_path(rendered_path, label="formatted data path") diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py index 029331a..fb12c4c 100644 --- a/mod/hallmark/transport/__init__.py +++ b/mod/hallmark/transport/__init__.py @@ -10,7 +10,7 @@ import requests from .auth import resolve_settings -from .base import RemoteEntry, RemoteSpec, TransferCancelled +from .base import DataBackend, RemoteEntry, RemoteSpec, TransferCancelled class OperationContext: @@ -40,14 +40,16 @@ def __init__( self._local = local() self._lock = Lock() self._sessions = [] - if remote.scheme in {"http", "https"}: - from .http import HttpTransport + from ..backends import get_backend - self.transport = HttpTransport(self) - else: - from .ssh import SshTransport - - self.transport = SshTransport(self) + try: + self.transport = get_backend(remote.backend)(self) + except BaseException: + self.cancelled.set() + for session in self._sessions: + session.close() + raise + self.backend = self.transport @contextmanager def executor(self, max_workers): @@ -99,13 +101,15 @@ def __enter__(self): return self def __exit__(self, exc_type, exc, tb): - if exc_type is not None: - self.cancel() try: - self.transport.close() + if exc_type is not None: + self.cancel() finally: - for session in self._sessions: - session.close() + try: + self.transport.close() + finally: + for session in self._sessions: + session.close() -__all__ = ["OperationContext", "RemoteEntry", "RemoteSpec"] +__all__ = ["DataBackend", "OperationContext", "RemoteEntry", "RemoteSpec"] diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py index 179ff83..60c6994 100644 --- a/mod/hallmark/transport/base.py +++ b/mod/hallmark/transport/base.py @@ -4,8 +4,10 @@ import ipaddress import re +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path +from types import MappingProxyType from urllib.parse import quote, unquote, urljoin, urlsplit, urlunsplit from ..error import HallmarkError @@ -32,6 +34,57 @@ class TransferCancelled(DownloadError): """Raised when a transfer is cancelled.""" +def backend_name(value): + """Validate a registered backend name, never an import path.""" + if not isinstance(value, str) or not re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", value): + raise RemoteConfigurationError( + "Backend name must match [a-z][a-z0-9_-]{0,63}") + return value + + +def freeze_backend_options(options=None): + """Copy YAML-compatible options into a deeply immutable mapping.""" + active = set() + + def freeze(value): + if isinstance(value, (Mapping, list, tuple)): + if id(value) in active: + raise RemoteConfigurationError("Backend options cannot contain cycles") + active.add(id(value)) + try: + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise RemoteConfigurationError( + "Backend option keys must be strings") + return MappingProxyType({key: freeze(item) + for key, item in value.items()}) + return tuple(freeze(item) for item in value) + finally: + active.remove(id(value)) + if value is None or isinstance(value, (str, bool, int, float)): + return value + raise RemoteConfigurationError( + "Backend options must contain YAML scalar, list, or mapping values") + + if options is None: + options = {} + if not isinstance(options, Mapping): + raise RemoteConfigurationError("Backend options must be a mapping") + return freeze(options) + + +def thaw_backend_options(options=None): + """Return an independent YAML-compatible copy of backend options.""" + def thaw(value): + if isinstance(value, Mapping): + return {key: thaw(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [thaw(item) for item in value] + return value + + return thaw(freeze_backend_options(options)) + + def reject_controls(value: str, label: str) -> None: """Reject control characters and text that cannot be encoded as UTF-8.""" try: @@ -124,6 +177,8 @@ class RemoteSpec: user (str, optional): Explicit SSH username. port (int, optional): Explicit port number. auth (str, optional): Local SSH authentication profile name. + backend (str): Registered backend name, resolved from the URL by default. + backend_options (Mapping): Deeply immutable backend configuration. """ url: str = field(repr=False) @@ -133,15 +188,33 @@ class RemoteSpec: user: str | None = field(default=None, repr=False) port: int | None = None auth: str | None = None + backend: str | None = None + backend_options: Mapping = field(default_factory=lambda: MappingProxyType({}), + repr=False, hash=False) + + def __post_init__(self): + selected = self.backend + if selected is None: + selected = "ssh" if self.scheme in {"ssh", "sftp"} else "http" + if selected == "http" and (self.host == "cyverse.org" + or self.host.endswith(".cyverse.org")): + selected = "cyverse" + object.__setattr__(self, "backend", backend_name(selected)) + object.__setattr__(self, "backend_options", + freeze_backend_options(self.backend_options)) @classmethod - def parse(cls, url: str, auth: str | None = None) -> RemoteSpec: + def parse(cls, url: str, auth: str | None = None, *, + backend=None, backend_options=None) -> RemoteSpec: """ Parse and validate a data-remote URL. Args: url (str): HTTP(S) URL or SSH/SFTP URL with an absolute dataset root. auth (str, optional): Local SSH profile name. Unsupported for HTTP(S). + backend (str, optional): Registered backend name. Defaults to URL + detection. Parsing does not load plugins or resolve profiles. + backend_options (Mapping, optional): Backend-specific configuration. Returns: RemoteSpec: Validated source description. @@ -205,7 +278,8 @@ def parse(cls, url: str, auth: str | None = None) -> RemoteSpec: "Auth profiles currently support SSH/SFTP only; " "HTTP retains Requests .netrc authentication" ) - return cls(url, scheme, host, root, user, port, auth) + return cls(url, scheme, host, root, user, port, auth, + backend, backend_options) def pathname(self, relative_path: str) -> str: """Append a literal relative path to the dataset root.""" @@ -229,9 +303,15 @@ def display(self) -> str: return f"{self.scheme}://{host}" -class Transport: +class DataBackend: """ - Common interface for internal data transports. + Common interface for dataset discovery and data transfer. + + Implementations receive shared cancellation state and thread-local HTTP + sessions through ``context``. Fetch may run concurrently: protect mutable + backend state, check ``context.check_cancelled()`` during long operations, + and release owned resources in ``close``. Authentication secrets belong in + local configuration, not persisted backend options. Args: context (OperationContext): Resources and settings for this operation. @@ -241,7 +321,7 @@ def __init__(self, context): self.context = context def prepare(self): - """Check connection requirements before starting transfers.""" + """Prepare for discovery or transfer; repeated calls must be safe.""" pass def fetch(self, relative_path, destination, *, chunk_size=8192): @@ -293,3 +373,11 @@ def read_text(self, relative_path, limit): def close(self): """Release connections and processes used by this transport.""" pass + + def cancel(self): + """Stop active backend work after the context's cancellation is set.""" + pass + + +# Existing transport imports remain valid for downstream integrations. +Transport = DataBackend diff --git a/mod/hallmark/transport/cyverse.py b/mod/hallmark/transport/cyverse.py new file mode 100644 index 0000000..f2940b0 --- /dev/null +++ b/mod/hallmark/transport/cyverse.py @@ -0,0 +1,40 @@ +"""CyVerse WebDAV HTML listing support, sharing the HTTP transfer backend.""" + +from html.parser import HTMLParser + +from .http import HttpBackend +from .index import _IndexParser, _parse_index + + +class CyVerseIndexParser(_IndexParser): + """Read CyVerse object and collection rows, including exact byte sizes.""" + + def handle_starttag(self, tag, attrs): + super().handle_starttag(tag, attrs) + if tag == "tr": + classes = dict(attrs).get("class", "").split() + if "object" in classes: + self.listing = True + self.row_kind = "directory" if "collection" in classes else "file" + + +def is_cyverse_index(text): + """Recognize CyVerse markup independently of its hosting domain.""" + class Probe(HTMLParser): + found = False + + def handle_starttag(self, tag, attrs): + if tag == "tr" and "object" in dict(attrs).get("class", "").split(): + self.found = True + + parser = Probe() + parser.feed(text) + return parser.found + + +class CyVerseBackend(HttpBackend): + """Discover CyVerse collections and transfer their files over HTTP(S).""" + + def _parse_index(self, text, directory): + return _parse_index(text, self.context.remote.url, directory, + parser_class=CyVerseIndexParser) diff --git a/mod/hallmark/transport/http.py b/mod/hallmark/transport/http.py index 14f3fed..8e16c44 100644 --- a/mod/hallmark/transport/http.py +++ b/mod/hallmark/transport/http.py @@ -2,20 +2,57 @@ from __future__ import annotations +from collections import deque from urllib.parse import unquote, urljoin, urlsplit import requests from ..helper_functions import REMOTE_REQUEST_TIMEOUT -from .base import DownloadError, RemoteObjectMissing, Transport, reject_controls +from .base import (DownloadError, RemoteConfigurationError, RemoteEntry, + RemoteObjectMissing, DataBackend, reject_controls) +from .index import _parse_index, _response_directory -class HttpTransport(Transport): +class HttpBackend(DataBackend): """Transfer files and metadata using reusable Requests sessions.""" def __init__(self, context): super().__init__(context) + if context.remote.scheme not in {"http", "https"}: + raise RemoteConfigurationError("HTTP backends require an HTTP(S) URL") self.text_urls = {} + def _parse_index(self, text, directory): + """Read generic indexes and recognize the CyVerse dialect automatically.""" + from .cyverse import CyVerseIndexParser, is_cyverse_index + + if is_cyverse_index(text): + return _parse_index(text, self.context.remote.url, directory, + parser_class=CyVerseIndexParser) + return _parse_index(text, self.context.remote.url, directory) + + def iter_entries(self, on_directory=None): + """Yield metadata from recursive listings without reading payloads.""" + queue, visited = deque([""]), set() + while queue: + directory = queue.popleft() + if directory in visited: + continue + visited.add(directory) + self.context.check_cancelled() + text = self.context.read_text(directory) + canonical = _response_directory(self.context, directory) + if canonical != directory and canonical in visited: + continue + visited.add(canonical) + for path, is_directory, size, mtime in self._parse_index(text, canonical): + if is_directory: + if path not in visited: + queue.append(path) + else: + yield RemoteEntry(path=path, size=size, mtime=mtime) + if on_directory is not None: + on_directory(canonical) + def _get(self, path): """Open a streaming response with the configured request timeout.""" return self.context.session().get( @@ -85,7 +122,6 @@ def fetch(self, relative_path, destination, *, chunk_size=8192): self.context.on_bytes(len(chunk)) except requests.RequestException as exc: raise self._error(exc) from None - def read_text(self, relative_path, limit): """Read metadata within the source root and byte limit.""" content = bytearray() @@ -104,3 +140,6 @@ def read_text(self, relative_path, limit): return text except requests.RequestException as exc: raise self._error(exc) from None + + +HttpTransport = HttpBackend diff --git a/mod/hallmark/transport/index.py b/mod/hallmark/transport/index.py new file mode 100644 index 0000000..5208f07 --- /dev/null +++ b/mod/hallmark/transport/index.py @@ -0,0 +1,149 @@ +"""Parse scoped HTML directory indexes for HTTP data backends.""" + +from __future__ import annotations + +import re +from html.parser import HTMLParser +from urllib.parse import quote, unquote, urljoin, urlsplit + +from .base import CapabilityError, DownloadError, literal_path, reject_controls + + +class _IndexParser(HTMLParser): + """Read links and recognizable index markers without depending on layout.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.links = [] + self.text = [] + self.row_kind = None + self.listing = False + self.has_table = False + self.has_form = False + self.current = None + self.in_anchor = False + self.cell_role = None + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "table": + self.has_table = True + elif tag == "form": + self.has_form = True + elif tag == "tr": + self.current = None + self.row_kind = None + elif tag == "td": + self.cell_role = attrs.get("class", "") + elif tag == "a" and "href" in attrs: + self.current = {"href": attrs["href"], "kind": self.row_kind, + "tail": "", "size": None, "mtime": None} + for field in ("size", "mtime"): + value = attrs.get(f"data-{field}") + if value is not None and value.isdigit(): + self.current[field] = int(value) + self.links.append(self.current) + self.in_anchor = True + + def handle_endtag(self, tag): + if tag == "tr": + self.row_kind = None + self.current = None + elif tag == "a": + self.in_anchor = False + elif tag == "td": + if self.current is not None: + self.current["tail"] += " " + self.cell_role = None + + def handle_data(self, data): + self.text.append(data) + if self.current is not None and not self.in_anchor: + self.current["tail"] += data + if self.cell_role in {"size", "bytes"}: + raw = data.strip().replace(",", "") + if raw.isdigit(): + self.current["size"] = int(raw) + + +def _origin(url): + """Return connection fields used to compare directory-listing URLs.""" + return (url.scheme, url.hostname, + url.port or (443 if url.scheme == "https" else 80), + url.username, url.password) + + +def _index_link(root_url, directory, href, kind): + """Resolve a link only when it stays beneath the supplied URL root.""" + reject_controls(href, "Index href") + root = urlsplit(root_url.rstrip("/") + "/") + current = urljoin(root.geturl(), quote(directory, safe="/")) + resolved = urlsplit(urljoin(current, href)) + if _origin(resolved) != _origin(root): + return None + # Sorting and navigation links do not identify dataset files. + if resolved.query or resolved.fragment: + return None + root_path = unquote(root.path, errors="strict") + path = unquote(resolved.path, errors="strict") + reject_controls(path, "Index path") + if "\\" in path or ".." in path.split("/"): + raise DownloadError("Remote index contains an unsafe path") + if not path.startswith(root_path): + return None + relative = path[len(root_path):] + if not relative: + return None + if any(part.lower() in {".hm", ".git"} for part in relative.split("/")): + return None + is_directory = kind == "directory" or resolved.path.endswith("/") + relative = literal_path(relative.rstrip("/")).as_posix() + return relative + ("/" if is_directory else ""), is_directory + + +def _parse_index(text, root_url, directory, parser_class=_IndexParser): + """Extract file and directory metadata from a supported HTML index.""" + parser = parser_class() + parser.feed(text) + links = [] + for entry in parser.links: + link = _index_link(root_url, directory, entry["href"], entry["kind"]) + if link is not None: + path, is_directory = link + size = entry["size"] + if size is None: + tail = re.search( + r"(?:\d{4}-\d\d-\d\d|\d\d-[A-Za-z]{3}-\d{4})" + r"\s+\d\d:\d\d(?::\d\d)?\s+(\d+)(?:\s|$)", entry["tail"]) + if tail: + size = int(tail.group(1)) + links.append((path, is_directory, size, entry["mtime"])) + markers = " ".join(parser.text).lower() + recognized = (parser.listing or "index of" in markers + or "directory listing" in markers or "parent directory" in markers + or any(is_dir for _, is_dir, _, _ in links)) + # CyVerse can emit a completely empty table without row-level markers. + empty_table = (parser.has_table and not parser.links + and not parser.has_form and not markers.strip()) + if not recognized and not empty_table: + raise CapabilityError( + "This URL exposes no usable directory listing. Use a browsable " + "directory URL or a published Hallmark catalog.") + return links + + +def _response_directory(context, requested): + """Use a redirected listing's canonical location when resolving child links.""" + transport = getattr(context, "transport", None) + final_url = getattr(transport, "text_urls", {}).get(requested) + if final_url is None: + return requested + root = urlsplit(context.remote.url.rstrip("/") + "/") + final = urlsplit(final_url) + if (_origin(root) == _origin(final) + and unquote(root.path).rstrip("/") == unquote(final.path).rstrip("/")): + return "" + link = _index_link(context.remote.url, "", final_url, "directory") + if link is None: + raise DownloadError("Redirected directory is outside the dataset root") + return link[0] diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index 9ef7d7b..e4f5d7e 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -19,7 +19,7 @@ DownloadError, RemoteConfigurationError, RemoteEntry, - Transport, + DataBackend, literal_path, reject_controls, ) @@ -48,7 +48,7 @@ def batch_argument(path): return "".join("\\" + char if char in special else char for char in text) -class SshTransport(Transport): +class SshBackend(DataBackend): """ Share one OpenSSH connection for SFTP transfers and metadata requests. @@ -59,6 +59,8 @@ class SshTransport(Transport): """ def __init__(self, context, *, ssh=None, sftp=None): super().__init__(context) + if context.remote.scheme not in {"ssh", "sftp"}: + raise RemoteConfigurationError("SSH backends require an SSH/SFTP URL") # Command overrides support offline tests and cannot come from repo YAML. self.ssh = ssh or ["ssh"] self.sftp = sftp or ["sftp"] @@ -416,3 +418,6 @@ def close(self): if self._socket_dir is not None: self._socket_dir.cleanup() self._socket_dir = None + + +SshTransport = SshBackend diff --git a/test/test_backend_config.py b/test/test_backend_config.py new file mode 100644 index 0000000..29a8873 --- /dev/null +++ b/test/test_backend_config.py @@ -0,0 +1,100 @@ +"""Persist backend settings without importing plugins or sharing mutable options.""" + +import pytest + +from hallmark import Repo +from hallmark import backends +from hallmark.backends import DataBackend, RemoteEntry, register_backend +from hallmark.repo_builder import build_repo +from hallmark.repo_config import normalize_remotes + + +def test_backend_config_roundtrip_and_selective_updates(tmp_path): + repo = Repo.init(tmp_path / "repo") + options = {"servers": [{"url": "https://data.test/", "prefix": "north"}]} + repo.set_config(remote_url="https://api.test/", remote_backend="survey", + remote_backend_options=options) + options["servers"][0]["prefix"] = "changed" + reopened = Repo(repo.worktree) + remote = reopened.state.config["remote"] + assert remote["backend"] == "survey" + assert remote["backend_options"]["servers"][0]["prefix"] == "north" + reopened.set_config(remote_name="archive") + assert Repo(repo.worktree).state.config["remote"]["backend"] == "survey" + reopened.set_config(remote_backend="", remote_backend_options={}) + remote = Repo(repo.worktree).state.config["remote"] + assert "backend" not in remote + assert remote["backend_options"] == {} + + +@pytest.mark.parametrize("update", [ + {"remote_backend": "arbitrary.module:Class"}, + {"remote_backend_options": ["not", "a", "mapping"]}, + {"remote_backend_options": {"nested": object()}}, +]) +def test_invalid_backend_update_preserves_existing_config(tmp_path, update): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="https://data.test/") + before = (repo.dothm.path / "config.yml").read_bytes() + with pytest.raises(ValueError): + repo.set_config(**update) + assert (repo.dothm.path / "config.yml").read_bytes() == before + assert "backend" not in repo.state.config["remote"] + + +def test_normalization_copies_nested_backend_options(): + source = {"name": "archive", "backend": "survey", + "backend_options": {"releases": [1, 2]}} + normalized = normalize_remotes(source) + source["backend_options"]["releases"].clear() + assert normalized[0]["backend_options"]["releases"] == [1, 2] + + +def test_builder_passes_backend_settings_to_discovery(monkeypatch, tmp_path): + captured = [] + + def build(*args): + captured.append(args[-1].remote) + + # No backend installation or network is required to inspect this hand-off. + class Context: + def __init__(self, remote): + self.remote = remote + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + monkeypatch.setattr("hallmark.repo_builder.OperationContext", Context) + monkeypatch.setattr("hallmark.repo_builder._build_repo", build) + with pytest.warns(DeprecationWarning, match="Repo.init"): + build_repo(tmp_path / "data.hm", "dataset", dataset_url="https://api.test/", + backend="survey", backend_options={"release": 3}) + assert captured[0].backend == "survey" + assert captured[0].backend_options["release"] == 3 + + +@pytest.mark.parametrize("mirror", [None, "https://mirror.test/data/"]) +def test_builder_persists_backend_only_for_its_data_source( + monkeypatch, tmp_path, mirror): + class SurveyBackend(DataBackend): + def iter_entries(self, on_directory=None): + assert self.context.remote.backend_options["release"] == 3 + yield RemoteEntry("tile.fits", size=12) + + monkeypatch.setattr(backends, "_registered", dict(backends._registered)) + monkeypatch.setattr(backends.metadata, "entry_points", lambda: {}) + register_backend("builder-survey", SurveyBackend) + with pytest.warns(DeprecationWarning): + repo = build_repo( + tmp_path / "survey.hm", "survey", dataset_url="https://api.test/", + backend="builder-survey", backend_options={"release": 3}, + remotes=[{"name": "mirror", "url": mirror}] if mirror else None) + remote = Repo(repo.dothm.path).state.config["remote"][0] + if mirror: + assert remote == {"name": "mirror", "url": mirror} + else: + assert remote["backend"] == "builder-survey" + assert remote["backend_options"] == {"release": 3} diff --git a/test/test_backends.py b/test/test_backends.py new file mode 100644 index 0000000..2b0d4e6 --- /dev/null +++ b/test/test_backends.py @@ -0,0 +1,334 @@ +"""Public backend registration and multi-server routing without public services.""" + +from contextlib import contextmanager +import hashlib +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +from pathlib import Path +from threading import Thread +from types import SimpleNamespace + +import pytest + +from hallmark import Repo, DataBackend, HttpBackend, SshBackend, CyVerseBackend +from hallmark import backends +from hallmark.discovery import discover +from hallmark.transport import OperationContext +from hallmark.transport.base import ( + CapabilityError, RemoteConfigurationError, RemoteEntry, RemoteSpec, Transport, + TransferCancelled, freeze_backend_options, thaw_backend_options, +) + + +@pytest.fixture(autouse=True) +def isolated_registry(monkeypatch): + monkeypatch.setattr(backends, "_registered", { + "http": HttpBackend, "ssh": SshBackend, "cyverse": CyVerseBackend}) + monkeypatch.setattr(backends, "_loaded", {}) + monkeypatch.setattr(backends.metadata, "entry_points", lambda: {}) + + +@pytest.mark.parametrize("url, expected", [ + ("https://lab.test/data/", HttpBackend), + ("http://lab.test/data/", HttpBackend), + ("ssh://lab/srv/data/", SshBackend), + ("sftp://lab/srv/data/", SshBackend), + ("https://data.cyverse.org/dav-anon/data/", CyVerseBackend), +]) +def test_builtin_detection(url, expected, monkeypatch, tmp_path): + monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + with OperationContext(RemoteSpec.parse(url)) as context: + assert type(context.backend) is expected + assert context.backend is context.transport + + +def test_explicit_selection_and_compatibility_aliases(): + from hallmark.transport.http import HttpTransport + from hallmark.transport.ssh import SshTransport + + assert Transport is DataBackend + assert HttpTransport is HttpBackend + assert SshTransport is SshBackend + with OperationContext(RemoteSpec.parse( + "https://data.cyverse.org/data/", backend="http")) as context: + assert type(context.backend) is HttpBackend + with OperationContext(RemoteSpec.parse( + "https://lab.test/data/", backend="cyverse")) as context: + assert type(context.backend) is CyVerseBackend + + +def test_backend_lifecycle_and_generic_discovery(tmp_path): + events = [] + + class SurveyBackend(DataBackend): + def iter_entries(self, on_directory=None): + on_directory("") + yield RemoteEntry("tile.fits", size=8) + + def prepare(self): + events.append("prepare") + + def fetch(self, relative_path, destination, *, chunk_size=8192): + self.context.check_cancelled() + destination.write_bytes(b"contents") + + def read_text(self, relative_path, limit): + assert limit == 20 + return "metadata" + + def cancel(self): + events.append("cancel") + + def close(self): + events.append("close") + + backends.register_backend("survey", SurveyBackend) + remote = RemoteSpec.parse("https://lab.test/", backend="survey") + with OperationContext(remote) as context: + context.text_limit = 20 + context.backend.prepare() + assert context.read_text("info") == "metadata" + assert discover(context) == [RemoteEntry("tile.fits", size=8)] + context.backend.fetch("tile.fits", tmp_path / "payload") + assert (tmp_path / "payload").read_bytes() == b"contents" + assert events == ["prepare", "prepare", "close"] + with pytest.raises(RuntimeError): + with OperationContext(remote): + raise RuntimeError("interrupted") + assert events[-2:] == ["cancel", "close"] + + +@pytest.mark.parametrize("name", ["", "Bad", "module.Backend", "bad/name", 12]) +def test_invalid_registry_names(name): + with pytest.raises(RemoteConfigurationError, match="Backend name"): + backends.register_backend(name, HttpBackend) + + +@pytest.mark.parametrize("backend_class", [ + object, object(), DataBackend, "module.Class", +]) +def test_invalid_registry_classes(backend_class): + with pytest.raises(RemoteConfigurationError, match="DataBackend subclass"): + backends.register_backend("invalid", backend_class) + + +def test_duplicate_registration_rejected(): + with pytest.raises(RemoteConfigurationError, match="already registered"): + backends.register_backend("http", HttpBackend) + backends.register_backend("survey", HttpBackend) + with pytest.raises(RemoteConfigurationError, match="already registered"): + backends.register_backend("survey", HttpBackend) + + +def test_plugin_loading_is_lazy_and_cached(monkeypatch): + loaded = [] + + class Entry: + name = "survey" + + def load(self): + loaded.append(self.name) + return HttpBackend + + monkeypatch.setattr(backends.metadata, "entry_points", + lambda: {"hallmark.backends": [Entry()]}) + remote = RemoteSpec.parse("https://lab.test/", backend="survey") + assert not loaded + for _ in range(2): + with OperationContext(remote) as context: + assert isinstance(context.backend, HttpBackend) + assert loaded == ["survey"] + with pytest.raises(RemoteConfigurationError, match="already registered"): + backends.register_backend("survey", HttpBackend) + + +def test_modern_entry_point_selection(monkeypatch): + entry = SimpleNamespace(name="survey", load=lambda: HttpBackend) + + class Entries: + def select(self, **kwargs): + assert kwargs == {"group": "hallmark.backends"} + return [entry] + + monkeypatch.setattr(backends.metadata, "entry_points", Entries) + assert backends.get_backend("survey") is HttpBackend + + +def test_duplicate_and_invalid_installed_backends(monkeypatch): + entry = SimpleNamespace(name="survey", load=lambda: object) + entries = [entry, entry] + monkeypatch.setattr(backends.metadata, "entry_points", + lambda: {"hallmark.backends": entries}) + with pytest.raises(RemoteConfigurationError, match="Duplicate backend"): + backends.get_backend("survey") + entries.pop() + with pytest.raises(RemoteConfigurationError, match="DataBackend subclass"): + backends.get_backend("survey") + entries[:] = [SimpleNamespace(name="http", load=lambda: HttpBackend)] + with pytest.raises(RemoteConfigurationError, match="Duplicate backend"): + backends.get_backend("http") + + +def test_missing_plugin_can_be_parsed_without_loading(monkeypatch): + def fail(): + raise AssertionError("Parsing must not inspect installed plugins") + + monkeypatch.setattr(backends.metadata, "entry_points", fail) + remote = RemoteSpec.parse("https://lab.test/", backend="uninstalled") + monkeypatch.setattr(backends.metadata, "entry_points", lambda: {}) + with pytest.raises(RemoteConfigurationError, match="not installed or registered"): + OperationContext(remote) + + +def test_plugin_load_failure_is_actionable_and_sanitized(monkeypatch): + def fail(): + raise ImportError("sensitive internal path") + + monkeypatch.setattr(backends.metadata, "entry_points", lambda: { + "hallmark.backends": [SimpleNamespace(name="survey", load=fail)]}) + with pytest.raises(RemoteConfigurationError, match="Unable to load backend") as exc: + backends.get_backend("survey") + assert "sensitive" not in str(exc.value) + + +def test_options_are_copied_and_deeply_immutable(): + options = {"routes": [{"names": ["first"]}], "enabled": True} + remote = RemoteSpec.parse("https://lab.test/", backend_options=options) + options["routes"][0]["names"].append("later") + assert remote.backend_options["routes"][0]["names"] == ("first",) + with pytest.raises(TypeError): + remote.backend_options["routes"][0]["names"] = () + assert thaw_backend_options(remote.backend_options) == { + "routes": [{"names": ["first"]}], "enabled": True} + assert freeze_backend_options() == {} + assert thaw_backend_options() == {} + + +@pytest.mark.parametrize("options", [[], {1: "bad"}, {"object": object()}]) +def test_invalid_options_are_rejected(options): + with pytest.raises(RemoteConfigurationError, match="Backend option"): + RemoteSpec.parse("https://lab.test/", backend_options=options) + + +def test_recursive_options_are_rejected(): + options = {} + options["cycle"] = options + with pytest.raises(RemoteConfigurationError, match="cycles"): + freeze_backend_options(options) + + +def test_base_capabilities(): + backend = DataBackend(SimpleNamespace()) + with pytest.raises(CapabilityError): + backend.iter_entries() + with pytest.raises(CapabilityError): + backend.read_text("index", 10) + with pytest.raises(NotImplementedError): + backend.fetch("file", Path("unused")) + + +def test_constructor_failure_closes_context_sessions(monkeypatch): + closed = [] + monkeypatch.setattr("hallmark.transport.requests.Session", lambda: SimpleNamespace( + close=lambda: closed.append(True))) + + class BrokenBackend(DataBackend): + def __init__(self, context): + super().__init__(context) + context.session() + raise RuntimeError("setup failed") + + backends.register_backend("broken", BrokenBackend) + with pytest.raises(RuntimeError, match="setup failed"): + OperationContext(RemoteSpec.parse("https://lab.test/", backend="broken")) + assert closed == [True] + + +def test_cancel_failure_still_closes_resources(): + closed = [] + + class BrokenBackend(DataBackend): + def cancel(self): + raise RuntimeError("cancel failed") + + def close(self): + closed.append(True) + + backends.register_backend("broken", BrokenBackend) + with pytest.raises(RuntimeError, match="cancel failed"): + with OperationContext(RemoteSpec.parse("https://lab.test/", backend="broken")): + raise ValueError("operation failed") + assert closed == [True] + + +@contextmanager +def local_server(root, requests): + class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(root), **kwargs) + + def log_message(self, *args): + pass + + def do_GET(self): + requests.append(self.path) + super().do_GET() + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/" + finally: + server.shutdown() + thread.join() + server.server_close() + + +def test_multi_server_example_uses_shared_discovery_and_verified_downloads(tmp_path): + path = Path(__file__).parents[1] / "demo" / "multi_server_backend.py" + module_spec = importlib.util.spec_from_file_location("example_backend", path) + example = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(example) + example.register() + requests = {"north": [], "south": []} + for name in requests: + root = tmp_path / name + root.mkdir() + payload = (name + " payload").encode() + (root / "file.fits").write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + (root / "sha256sums").write_text(digest + " file.fits\n") + with local_server(tmp_path / "north", requests["north"]) as north: + with local_server(tmp_path / "south", requests["south"]) as south: + options = {"routes": {"north": north, "south": south}} + repo = Repo.init(tmp_path / "repo", from_url="https://logical.test/dataset/", + backend="multi-server", backend_options=options, + filter="**/*.fits") + assert repo.state.data["path"].tolist() == [ + "north/file.fits", "south/file.fits"] + assert requests == {"north": ["/", "/sha256sums"], + "south": ["/", "/sha256sums"]} + repo = Repo(repo.worktree) + before = {name: list(paths) for name, paths in requests.items()} + plan = repo.plan_download() + assert requests == before + assert plan.remote_backend == "multi-server" + assert plan.backend_options["routes"] == options["routes"] + options["routes"]["north"] = south + repo.set_config(remote_backend_options={"routes": {"north": south}}) + result = repo.download(plan=plan, approved=True, + max_workers=2, progress=False) + assert result["succeeded"] == 2 and result["failed"] == 0 + for name in requests: + assert (repo.worktree / name / "file.fits").read_bytes() == ( + name + " payload").encode() + assert requests[name][-1] == "/file.fits" + remote = RemoteSpec.parse(plan.remote_url, + backend=plan.remote_backend, + backend_options=plan.backend_options) + with OperationContext(remote) as context: + context.cancel() + with pytest.raises(TransferCancelled): + next(context.backend.iter_entries()) diff --git a/test/test_catalog.py b/test/test_catalog.py index 9f86b19..5b67820 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -1,4 +1,4 @@ -"""Test catalog cloning without downloading dataset files or stored objects.""" +"""Test remote initialization and cloning without downloading dataset payloads.""" from pathlib import Path @@ -27,7 +27,7 @@ def read_text(context, path): return pages, reads -def test_directory_clone_is_recursive_filtered_and_payload_free( +def test_remote_init_is_recursive_filtered_and_payload_free( tmp_path, metadata_server): pages, reads = metadata_server root = "https://example.test/data/" @@ -35,10 +35,11 @@ def test_directory_clone_is_recursive_filtered_and_payload_free( 'note.txtsub/') pages[root + "sub/"] = '

Index of sub

data.h5' events = [] - repo = Repo.clone(root, tmp_path / "clone", filter="**/*.h5", - progress=events.append) + repo = Repo.init(tmp_path / "clone", from_url=root, filter="**/*.h5", + progress=events.append) assert repo.state.data["path"].tolist() == ["root.h5", "sub/data.h5"] - assert repo.state.config["remote"] == [{"name": "origin", "url": root}] + assert repo.state.config["remote"] == [ + {"name": "origin", "url": root, "backend": "http"}] assert repo.state.config["data"] == [{"db": "data.tsv"}] assert not (repo.worktree / "root.h5").exists() assert not any(url.endswith((".h5", ".txt")) for url in reads) @@ -48,20 +49,20 @@ def test_directory_clone_is_recursive_filtered_and_payload_free( assert events[-1]["directories"] == 2 -def test_url_only_clone_does_not_require_filename_inference(tmp_path, metadata_server): +def test_remote_init_does_not_require_filename_inference(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/odd/" pages[root] = '

Index of odd

odd name' - repo = Repo.clone(root, tmp_path / "clone") + repo = Repo.init(tmp_path / "clone", from_url=root) assert repo.state.data["path"].tolist() == ["odd name"] assert repo.plan_download().file_count == 1 -def test_empty_filtered_clone_is_valid(tmp_path, metadata_server): +def test_empty_filtered_init_is_valid(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/data/" pages[root] = '

Index of data

notes.txt' - repo = Repo.clone(root, tmp_path / "clone", fmt="run{run:d}.h5") + repo = Repo.init(tmp_path / "clone", from_url=root, fmt="run{run:d}.h5") assert repo.state.data.empty assert repo.plan_download().file_count == 0 @@ -71,7 +72,7 @@ def test_template_enriches_and_filters_inventory(tmp_path, metadata_server): root = "https://example.test/data/" pages[root] = ('

Index of data

run2.h5' 'notes.txt') - repo = Repo.clone(root, tmp_path / "clone", fmt="run{run:d}.h5") + repo = Repo.init(tmp_path / "clone", from_url=root, fmt="run{run:d}.h5") assert repo.state.data["path"].tolist() == ["run2.h5"] assert str(repo.state.data.iloc[0]["run"]) == "2" @@ -86,12 +87,14 @@ def test_published_snapshot_preserves_data_remote(tmp_path, metadata_server, nes "remote": [{"name": "origin", "url": "https://mirror.test/data/"}]}) pages[prefix + "meta.yml"] = "dataset: example\n" pages[prefix + "data.tsv"] = "path\tsize_bytes\na.h5\t12\nb.txt\t20\n" - repo = Repo.clone(root, tmp_path / "clone", filter="**/*.h5") - assert repo.state.data["path"].tolist() == ["a.h5"] - assert repo.plan_download().total_bytes == 12 + plans = [] + repo = Repo.clone(root, tmp_path / "clone", filter="**/*.h5", download=True, + approve=lambda plan: plans.append(plan) or False) + assert repo.state.data["path"].tolist() == ["a.h5", "b.txt"] + assert plans[0].total_bytes == 12 assert repo.plan_download().remote_url == "https://mirror.test/data/" assert not any(url.endswith("a.h5") for url in reads) - assert repo.dothm.head.commit.message.strip() == "Filter local catalog" + assert repo.dothm.head.commit.message.strip() == "Import published catalog snapshot" def test_filtered_git_clone_preserves_history_without_objects(tmp_path, monkeypatch): @@ -108,10 +111,14 @@ def fail(*args, **kwargs): raise AssertionError("Filtered metadata clone must not transfer data") monkeypatch.setattr("hallmark.downloader._fetch_file", fail) - repo = Repo.clone(str(source.dothm.path), tmp_path / "clone", filter="run1.h5") - assert repo.dothm.head.commit.parents[0].hexsha == head - assert len(repo.state.data) == 1 - assert repo.plan_download().items[0].relative_path == Path("run1.h5") + plans = [] + repo = Repo.clone(str(source.dothm.path), tmp_path / "clone", filter="run1.h5", + download=True, approve=lambda plan: plans.append(plan) or False) + assert repo.dothm.head.commit.hexsha == head + assert repo.dothm.index.diff("HEAD") == [] + assert len(repo.state.data) == 2 + assert plans[0].items[0].relative_path == Path("run1.h5") + assert plans[0].file_count == 1 assert not repo.objects.root.exists() assert not (repo.worktree / "run1.h5").exists() assert repo.dothm.remotes.origin.url == str(source.dothm.path) @@ -125,7 +132,7 @@ def test_clone_requires_callback_before_download_intent(tmp_path, metadata_serve assert not (tmp_path / "clone").exists() -def test_clone_refusal_keeps_catalog_without_payload( +def test_init_refusal_keeps_catalog_without_payload( tmp_path, metadata_server, monkeypatch): pages, _ = metadata_server root = "https://example.test/data/" @@ -140,7 +147,7 @@ def fail(*args, **kwargs): raise AssertionError("Refused transfer must not start") monkeypatch.setattr(Repo, "download", fail) - repo = Repo.clone(root, tmp_path / "clone", download=True, approve=decline) + repo = Repo.init(tmp_path / "clone", from_url=root, download=True, approve=decline) assert len(plans) == 1 assert plans[0].file_count == 1 assert repo.download_result is None @@ -154,8 +161,7 @@ def fail(*args, **kwargs): monkeypatch.setattr("hallmark.catalog.discover", fail) with pytest.raises(KeyboardInterrupt): - Repo.clone("https://example.test/data/", tmp_path / "clone", - source_type="directory") + Repo.init(tmp_path / "clone", from_url="https://example.test/data/") assert not (tmp_path / "clone").exists() @@ -191,11 +197,15 @@ def test_snapshot_filter_normalizes_legacy_db_names(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/data/" pages[root + "config.yml"] = ( - "data:\n- db: data\n fmt: 'item_{i}.dat'\n") + "data:\n- db: data\n fmt: 'item_{i}.dat'\n" + "remote:\n- name: origin\n url: https://data.test/files/\n") pages[root + "meta.yml"] = "{}\n" pages[root + "data.tsv"] = "i\n1\n2\n" - repo = Repo.clone(root, tmp_path / "clone", filter="item_1.dat") - assert repo.state.data["i"].astype(str).tolist() == ["1"] + plans = [] + repo = Repo.clone(root, tmp_path / "clone", filter="item_1.dat", download=True, + approve=lambda plan: plans.append(plan) or False) + assert repo.state.data["i"].astype(str).tolist() == ["1", "2"] + assert [item.relative_path for item in plans[0].items] == [Path("item_1.dat")] @pytest.mark.parametrize("table", [ @@ -211,3 +221,266 @@ def test_snapshot_validates_tables_without_filter(tmp_path, metadata_server, tab with pytest.raises((CloneError, ValueError)): Repo.clone(root, tmp_path / "clone") assert not (tmp_path / "clone").exists() + + +@pytest.mark.parametrize("failure", [KeyboardInterrupt, DownloadError]) +def test_remote_init_preserves_existing_directory_on_discovery_failure( + tmp_path, metadata_server, monkeypatch, failure): + destination = tmp_path / "existing" + destination.mkdir() + (destination / "keep.h5").write_bytes(b"existing scientific data") + + def fail(*args, **kwargs): + raise failure("interrupted") + + monkeypatch.setattr("hallmark.catalog.discover", fail) + with pytest.raises(failure): + Repo.init(destination, from_url="https://example.test/data/") + assert list(destination.iterdir()) == [destination / "keep.h5"] + assert (destination / "keep.h5").read_bytes() == b"existing scientific data" + + +def test_remote_init_preserves_existing_payload_and_rejects_existing_catalog( + tmp_path, metadata_server): + pages, reads = metadata_server + root = "https://example.test/data/" + pages[root] = '

Index of data

keep.h5' + destination = tmp_path / "existing" + destination.mkdir() + (destination / "keep.h5").write_bytes(b"local") + repo = Repo.init(destination, from_url=root) + assert repo.state.data["path"].tolist() == ["keep.h5"] + assert (destination / "keep.h5").read_bytes() == b"local" + head = repo.dothm.head.commit.hexsha + reads.clear() + with pytest.raises(DestinationExistsError): + Repo.init(destination, from_url=root) + assert reads == [] + assert Repo(destination).dothm.head.commit.hexsha == head + + +def test_remote_init_rejects_bare_symlink_before_access(tmp_path, metadata_server): + target = tmp_path / "existing" + target.mkdir() + (target / "keep").write_text("keep") + alias = tmp_path / "alias.hm" + alias.symlink_to(target, target_is_directory=True) + with pytest.raises(DestinationExistsError): + Repo.init(alias, from_url="https://example.test/data/") + assert metadata_server[1] == [] + assert list(target.iterdir()) == [target / "keep"] + + +def test_remote_init_cleans_only_created_metadata_on_write_failure( + tmp_path, metadata_server, monkeypatch): + destination = tmp_path / "existing" + destination.mkdir() + (destination / "keep").write_text("keep") + root = "https://example.test/data/" + metadata_server[0][root] = '

Index of data

' + + def fail(*args, **kwargs): + raise OSError("cannot write catalog") + + monkeypatch.setattr("hallmark.catalog._write_inventory", fail) + with pytest.raises(OSError, match="cannot write"): + Repo.init(destination, from_url=root) + assert list(destination.iterdir()) == [destination / "keep"] + + +@pytest.mark.parametrize("kwargs", [ + {"filter": "*.fits"}, {"fmt": "{name}.fits"}, {"source_type": "directory"}, + {"download": True, "approve": lambda plan: True, "fmt": "{name:invalid}"}, +]) +def test_clone_invalid_selection_fails_before_source_access( + tmp_path, metadata_server, kwargs): + with pytest.raises(ValueError): + Repo.clone("https://example.test/data/", tmp_path / "clone", **kwargs) + assert metadata_server[1] == [] + assert not (tmp_path / "clone").exists() + + +def test_raw_directory_clone_fails_with_init_guidance( + tmp_path, metadata_server, monkeypatch): + pages, reads = metadata_server + root = "https://example.test/data/" + pages[root] = '

Index of data

a.h5' + + def fail(*args, **kwargs): + raise CloneError("not a Git repository") + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(CloneError, match="hallmark init PATH --from URL"): + Repo.clone(root, tmp_path / "clone") + assert root not in reads + assert not (tmp_path / "clone").exists() + + +def test_suffixless_http_git_source_falls_back_after_absent_snapshots( + tmp_path, metadata_server, monkeypatch): + source = Repo.init(tmp_path / "source") + source.dothm.index.commit("Existing complete catalog") + original = type(source.dothm).clone + calls = [] + + def clone(url, destination, **kwargs): + calls.append(url) + return original(str(source.dothm.path), destination, **kwargs) + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", clone) + url = "https://git.example.test/group/dataset" + repo = Repo.clone(url, tmp_path / "clone") + assert calls == [url] + assert metadata_server[1] == [url + "/config.yml", url + "/.hm/config.yml"] + assert repo.dothm.head.commit.hexsha == source.dothm.head.commit.hexsha + + +@pytest.mark.parametrize("url", [ + "ssh://git.example.test/dataset", "git@example.test:group/dataset", + "example.test:group/dataset", "https://example.test/dataset.git", +]) +def test_git_sources_do_not_probe_snapshot_metadata(tmp_path, metadata_server, + monkeypatch, url): + calls = [] + + def fail(source, *args, **kwargs): + calls.append(source) + raise CloneError("test Git failure") + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(CloneError, match="test Git failure"): + Repo.clone(url, tmp_path / "clone") + assert calls == [url] + assert metadata_server[1] == [] + + +@pytest.mark.parametrize("config,meta,table", [ + ("data: [", "{}", "path\n"), + ("[]", "{}", "path\n"), + ("unrelated: config", "{}", "path\n"), + ("data: []", None, "path\n"), + ("data: []", "{}", None), +]) +def test_malformed_snapshot_never_falls_back_to_git( + tmp_path, metadata_server, monkeypatch, config, meta, table): + root = "https://example.test/catalog/" + for name, content in [("config.yml", config), ("meta.yml", meta), + ("data.tsv", table)]: + if content is not None: + metadata_server[0][root + name] = content + + def fail(*args, **kwargs): + raise AssertionError("Malformed metadata must not trigger Git fallback") + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(CloneError): + Repo.clone(root, tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_clone_does_not_require_payload_authentication( + tmp_path, metadata_server, monkeypatch): + root = "https://catalog.example.test/published/" + remote = {"name": "origin", "url": "ssh://private.example.test/science/", + "auth": "missing-local-profile", "backend": "ssh", + "backend_options": {"collection": ["release-1"]}} + metadata_server[0].update({ + root + "config.yml": yaml.safe_dump({"data": [{"db": "data.tsv"}], + "remote": [remote]}), + root + "meta.yml": "{}\n", root + "data.tsv": "path\na.fits\n"}) + repo = Repo.clone(root, tmp_path / "clone") + assert repo.state.config["remote"] == [remote] + assert all(url.startswith(root) for url in metadata_server[1]) + assert repo.plan_download().remote_auth == "missing-local-profile" + assert not repo.dothm.remotes + + +def test_init_download_requires_callback_before_source_access( + tmp_path, metadata_server): + with pytest.raises(DownloadError, match="approve"): + Repo.init(tmp_path / "new", from_url="https://example.test/data/", + download=True) + assert metadata_server[1] == [] + + +def test_empty_init_download_does_not_request_approval(tmp_path, metadata_server): + root = "https://example.test/empty/" + metadata_server[0][root] = '

Index of empty

' + + def fail(plan): + raise AssertionError("Empty plans require no confirmation") + + repo = Repo.init(tmp_path / "new", from_url=root, download=True, approve=fail) + assert repo.state.data.empty + assert repo.download_result is None + + +def test_init_downloads_only_after_approved_catalog_creation( + tmp_path, metadata_server, monkeypatch): + import requests + from mock_server import MockServer + + root = "https://example.test/data/" + metadata_server[0][root] = '

Index of data

a.fits' + server = MockServer(root) + server.add_file("a.fits", b"fits") + monkeypatch.setattr(requests, "Session", lambda: server) + destination = tmp_path / "new" + plans = [] + + def approve(plan): + assert Repo(destination).state.data["path"].tolist() == ["a.fits"] + assert not (destination / "a.fits").exists() + plans.append(plan) + return True + + repo = Repo.init(destination, from_url=root, download=True, approve=approve) + assert len(plans) == 1 + assert (destination / "a.fits").read_bytes() == b"fits" + assert repo.download_result["succeeded"] == 1 + + +def test_snapshot_access_failure_never_falls_back_to_git(tmp_path, monkeypatch): + def inaccessible(context, path): + raise DownloadError("Metadata server requires authentication") + + def fail(*args, **kwargs): + raise AssertionError("Access failure must not trigger Git fallback") + + monkeypatch.setattr(OperationContext, "read_text", inaccessible) + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(DownloadError, match="requires authentication"): + Repo.clone("https://catalog.example.test/data/", tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_download_uses_independent_payload_server( + tmp_path, metadata_server, monkeypatch): + import requests + from mock_server import MockServer + + catalog_url = "https://catalog.example.test/published/" + payload_url = "https://data.example.test/science/" + metadata_server[0].update({ + catalog_url + "config.yml": yaml.safe_dump({ + "data": [{"db": "data.tsv"}], + "remote": [{"name": "origin", "url": payload_url}]}), + catalog_url + "meta.yml": "{}\n", + catalog_url + "data.tsv": "path\tsize_bytes\na.fits\t4\n"}) + server = MockServer(payload_url) + server.add_file("a.fits", b"fits") + requests_made = [] + original_get = server.get + + def get(url, **kwargs): + requests_made.append(url) + return original_get(url, **kwargs) + + server.get = get + monkeypatch.setattr(requests, "Session", lambda: server) + repo = Repo.clone(catalog_url, tmp_path / "clone", download=True, + approve=lambda plan: plan.remote_url == payload_url) + assert requests_made == [payload_url + "a.fits"] + assert all(url.startswith(catalog_url) for url in metadata_server[1]) + assert (repo.worktree / "a.fits").read_bytes() == b"fits" + assert repo.download_result["succeeded"] == 1 diff --git a/test/test_cli.py b/test/test_cli.py index e465eee..f0d5a62 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -650,9 +650,11 @@ def fail_download(self, plan, **kwargs): def test_clone_cli_skips_download_when_no_remote_files(monkeypatch, tmp_path): repo = SimpleNamespace(worktree=tmp_path, - plan_download=lambda: _download_plan(0, tmp_path)) + plan_download=lambda **kwargs: _download_plan(0, tmp_path)) class FakeRepo: + lwpaths = Repo.lwpaths + @staticmethod def clone(url, path, **kwargs): return repo @@ -1349,25 +1351,29 @@ def reject_download(*args, **kwargs): assert "Download these files?" not in result.output -def test_clone_cli_forwards_discovery_options(monkeypatch, tmp_path): +def test_init_cli_forwards_discovery_options(monkeypatch, tmp_path): captured = {} + options = tmp_path / "backend.yml" + options.write_text("collection: latest\n") class FakeRepo: @staticmethod - def clone(url, path, **kwargs): - captured.update(url=url, path=path, **kwargs) + def init(path, **kwargs): + captured.update(path=path, **kwargs) return SimpleNamespace(worktree=Path(path)) monkeypatch.setattr(cli_module, "Repo", FakeRepo) result = CliRunner().invoke(hallmark, [ - "clone", "ssh://lab-data/export/", str(tmp_path / "target"), "--auth", "lab", + "init", str(tmp_path / "target"), "--from", "ssh://lab-data/export/", + "--auth", "lab", "--backend", "ssh", "--backend-options", str(options), "--filter", "**/*.fits", "--filter", "README*", "--fmt", "{name}.fits", - "--source-type", "directory", "--max-workers", "2"]) + "--max-workers", "2"]) assert result.exit_code == 0, result.output assert captured == { - "url": "ssh://lab-data/export/", "path": str(tmp_path / "target"), - "auth": "lab", "filter": ("**/*.fits", "README*"), "fmt": "{name}.fits", - "source_type": "directory", "progress": True, "max_workers": 2} + "from_url": "ssh://lab-data/export/", "path": str(tmp_path / "target"), + "auth": "lab", "backend": "ssh", "backend_options": {"collection": "latest"}, + "filter": ("**/*.fits", "README*"), "fmt": "{name}.fits", + "progress": True, "max_workers": 2} def test_clone_cli_rejects_conflicting_download_aliases(): @@ -1417,3 +1423,89 @@ def capture_get(url, **kwargs): assert requests_made == [] assert not (target / "tiny.fits").exists() assert not (target / "other.txt").exists() + + +@pytest.mark.parametrize("options", ["[]\n", "scalar\n", "options: [\n"]) +def test_init_cli_rejects_invalid_backend_options_before_access( + monkeypatch, tmp_path, options): + config = tmp_path / "options.yml" + config.write_text(options) + + def fail(*args, **kwargs): + raise AssertionError("Invalid options must be rejected before initialization") + + monkeypatch.setattr(Repo, "init", fail) + result = CliRunner().invoke(hallmark, [ + "init", str(tmp_path / "target"), "--from", "https://example.test/data/", + "--backend-options", str(config)]) + assert result.exit_code != 0 + assert "Error:" in result.output + assert not (tmp_path / "target").exists() + + +@pytest.mark.parametrize("arguments", [ + ["--filter", "*.fits"], ["--fmt", "{name}.fits"], + ["--with-download", "--fmt", "{name:invalid}"], +]) +def test_clone_cli_rejects_invalid_selection_before_source_access( + monkeypatch, arguments): + def fail(*args, **kwargs): + raise AssertionError("Invalid selection must fail before cloning") + + monkeypatch.setattr(Repo, "clone", fail) + result = CliRunner().invoke(hallmark, ["clone", "source", "target", *arguments]) + assert result.exit_code != 0 + assert "Error:" in result.output + + +def test_init_cli_success_does_not_echo_source_credentials(monkeypatch, tmp_path): + def initialize(path, **kwargs): + return SimpleNamespace(worktree=Path(path)) + + monkeypatch.setattr(Repo, "init", initialize) + result = CliRunner().invoke(hallmark, [ + "init", str(tmp_path / "target"), "--from", + "https://user:secret@example.test/data/?token=private"]) + assert result.exit_code == 0, result.output + assert "Successfully initialized" in result.output + assert "secret" not in result.output + assert "private" not in result.output + + +def test_set_config_cli_persists_backend_options(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + options = tmp_path / "backend.yml" + options.write_text("collection: [release-1, release-2]\n") + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, [ + "set-config", "--remote-url", "https://example.test/data/", + "--remote-backend", "http", "--remote-backend-options", str(options)]) + assert result.exit_code == 0, result.output + remote = Repo(repo.worktree).state.config["remote"] + assert remote["backend"] == "http" + assert remote["backend_options"] == {"collection": ["release-1", "release-2"]} + + +@pytest.mark.parametrize("answer", ["y\n", "n\n", ""]) +def test_init_cli_downloads_only_after_confirmation(monkeypatch, tmp_path, answer): + from hallmark.transport import OperationContext + from hallmark.transport.base import RemoteObjectMissing + from mock_server import MockServer + + def metadata(context, path): + if path == "": + return '

Index of data

a.fits' + raise RemoteObjectMissing("Missing metadata") + + server = MockServer("https://example.test/data/") + server.add_file("a.fits", b"fits") + monkeypatch.setattr(OperationContext, "read_text", metadata) + monkeypatch.setattr(requests, "Session", lambda: server) + destination = tmp_path / "target" + result = CliRunner().invoke(hallmark, [ + "init", str(destination), "--from", "https://example.test/data/", + "--with-download"], input=answer) + assert "Download these files? [y/N]" in result.output + assert Repo(destination).state.data["path"].tolist() == ["a.fits"] + assert (destination / "a.fits").exists() == (answer == "y\n") + assert (result.exit_code == 0) == (answer == "y\n") diff --git a/test/test_discovery.py b/test/test_discovery.py index d00939d..ad06cea 100644 --- a/test/test_discovery.py +++ b/test/test_discovery.py @@ -5,6 +5,7 @@ import pytest from hallmark.discovery import discover, path_matches +from hallmark.backends import HttpBackend from hallmark.transport.base import ( CapabilityError, DownloadError, RemoteEntry, RemoteSpec, TransferCancelled, ) @@ -16,6 +17,8 @@ def __init__(self, pages, url="https://data.test/public/"): self.remote = RemoteSpec.parse(url) self.pages = pages self.reads = [] + if self.remote.scheme in {"http", "https"}: + self.transport = HttpBackend(self) def read_text(self, path): self.reads.append(path) @@ -164,7 +167,7 @@ def test_redirected_listing_uses_final_directory_and_deduplicates_crawl(final_ur "alias/": index("file.fits", "nested/"), "release one/nested/": index("other.fits"), }) - source.transport = SimpleNamespace(text_urls={"alias/": final_url}) + source.transport.text_urls = {"alias/": final_url} entries = discover(source) assert [entry.path for entry in entries] == [ "release one/file.fits", "release one/nested/other.fits"] @@ -226,7 +229,7 @@ def entries(on_directory): on_directory("nested") yield RemoteEntry(path="nested/file.bin", size=20) - source.transport = SimpleNamespace(iter_entries=entries) + source.transport = SimpleNamespace(iter_entries=entries, prepare=lambda: None) snapshots = [] selected = discover(source, filter="**/*.fits", progress=snapshots.append) assert selected == [RemoteEntry(path="root.fits", size=12, mtime=123)] diff --git a/test/test_download_plan.py b/test/test_download_plan.py index d4d13da..7e43b7e 100644 --- a/test/test_download_plan.py +++ b/test/test_download_plan.py @@ -234,3 +234,48 @@ def test_bare_catalog_requires_explicit_output(catalog): catalog.worktree = None with pytest.raises(DownloadError, match="output_path"): plan_download(catalog) + + +def test_plan_pins_backend_and_nested_options_without_loading_plugin( + catalog, monkeypatch): + options = {"servers": [{"url": "https://original.test/"}]} + catalog.state.config["remote"].update( + backend="external-survey", backend_options=options) + + def reject_network(*args, **kwargs): + raise AssertionError("planning must not load or contact a backend") + + monkeypatch.setattr("hallmark.downloader.OperationContext", reject_network) + plan = plan_download(catalog, file_paths="nested/a.fits") + options["servers"][0]["url"] = "https://changed.test/" + catalog.state.config["remote"]["backend"] = "another-survey" + assert plan.remote_backend == "external-survey" + assert plan.backend_options["servers"][0]["url"] == "https://original.test/" + with pytest.raises(TypeError): + plan.backend_options["servers"][0]["url"] = "https://changed.test/" + with pytest.raises(TypeError): + plan.backend_options["extra"] = True + assert hash(plan) == hash(replace(plan)) + captured = [] + + def download(source, *args, **kwargs): + captured.append(source) + return {"succeeded": 1, "failed": 0, "total_bytes": 6, "errors": []} + + monkeypatch.setattr("hallmark.downloader._download_selected", download) + execute_download_plan(catalog, plan, approved=True) + assert captured[0].backend == "external-survey" + assert captured[0].backend_options["servers"][0]["url"] == \ + "https://original.test/" + + +def test_legacy_remote_pins_default_backend(catalog): + assert plan_download(catalog).remote_backend == "http" + + +def test_missing_plugin_can_be_planned_but_not_executed(catalog): + catalog.state.config["remote"]["backend"] = "missing-survey-plugin" + plan = plan_download(catalog, file_paths="nested/a.fits") + with pytest.raises(DownloadError, match="missing-survey-plugin"): + execute_download_plan(catalog, plan, approved=True) + assert not (catalog.worktree / "nested/a.fits").exists() diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py index 7115b5b..b6b59af 100644 --- a/test/test_ssh_integration.py +++ b/test/test_ssh_integration.py @@ -383,7 +383,7 @@ def test_sftp_only_discovers_metadata_and_fetches(ssh_server, tmp_path): @pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) @pytest.mark.parametrize("scheme", ["ssh", "sftp"]) -def test_sftp_only_clone_plans_then_requires_payload_approval( +def test_sftp_only_init_plans_then_requires_payload_approval( ssh_server, tmp_path, monkeypatch, scheme, ): root = ssh_server["root"] @@ -408,8 +408,9 @@ def decline(plan): plans.append(plan) return False - repo = Repo.clone( - ssh_server["url"].replace("ssh:", scheme + ":"), tmp_path / "clone", + repo = Repo.init( + tmp_path / "initialized", + from_url=ssh_server["url"].replace("ssh:", scheme + ":"), filter="**/*.fits", download=True, approve=decline, ) assert fetched == [] From 2e229cc9435dfbc755e96597996ae7e9cd2cc35d Mon Sep 17 00:00:00 2001 From: Dennis Date: Mon, 21 Sep 2026 21:01:02 -0700 Subject: [PATCH 15/29] Align remote workflow documentation and simplify examples --- README.rst | 8 +- demo/private_data_demo.md | 377 +++++++------------------ demo/scientific_workflows_cli.md | 251 ++++++++++++++++ demo/scientific_workflows_python.ipynb | 371 ++++++++++++++++++++++++ doc/api.rst | 22 +- doc/design.rst | 7 +- doc/index.rst | 6 +- doc/private_data.rst | 44 +-- doc/usecase.rst | 91 +++--- 9 files changed, 834 insertions(+), 343 deletions(-) create mode 100644 demo/scientific_workflows_cli.md create mode 100644 demo/scientific_workflows_python.ipynb diff --git a/README.rst b/README.rst index 9eaf72a..5671c9e 100644 --- a/README.rst +++ b/README.rst @@ -112,6 +112,10 @@ See `Data backends `_ for configuration and an extension examp Examples and configuration details are described in `Private data over SSH and SFTP `_. -For a sequential CLI and Python walkthrough, see the +For a sequential CLI walkthrough, see the `private data demo `_. It includes catalog -initialization, local authentication profiles and explicit download approval. +initialization, SSH configuration and explicit download approval. + +The `scientific CLI workflows `_ and +`scientific Python notebook `_ +follow four examples using EHT, DESI, Roman–Rubin and local DES data. diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md index c7b5e09..fb72ecf 100644 --- a/demo/private_data_demo.md +++ b/demo/private_data_demo.md @@ -1,317 +1,158 @@ # Private data demo: copyable instructions Run these instructions on your local machine using an existing data export. -The SSH alias uses the validation VM settings supplied for this demo. Replace -the example dataset root with your export path; the commands below have not -been run against that path on the VM. +The SSH configuration and dataset path are illustrative. Replace them with your +own export settings; this guide does not provision or contact a particular VM. -For a small walkthrough, the examples assume `runs/run_001.dat`, -`runs/run_002.dat`, a `README.md`, and optionally a publisher-provided `SHA256SUMS`. -The client needs this Hallmark checkout, Git, and OpenSSH `ssh`/`sftp` 9.6+. -The server needs SFTP, but does not require a login shell, Python, Hallmark, or Git. -Run the Bash blocks in order in one local terminal. +The export is assumed to contain `runs/run_001.dat`, `runs/run_002.dat`, a +`README.md`, and optionally a publisher-provided `SHA256SUMS`. The client needs +Hallmark, Git, and OpenSSH `ssh`/`sftp` 9.6+. The server needs SFTP, but does not +require a login shell, Python, Hallmark, or Git. Run the Bash blocks in order in +one terminal, starting in the Hallmark source checkout. -1. Install the checkout in your active Python environment. +1. Install the checkout in your active Python environment. `set -e` stops the +shell if a command fails or a download is declined, so later commands do not +continue after an unsuccessful transfer. ```bash -# Start in the Hallmark source checkout. +set -e python -m pip install -e . -ssh -V -hallmark init --help ``` -2. Add or update this entry in `~/.ssh/config`, preserving your other entries. -Replace the host, user, or identity when using a different server. +2. Add this illustrative entry to `~/.ssh/config`, preserving your other entries +and replacing the host, user, and identity with your export's settings. ```text Host lab-data - HostName 34.29.34.222 - User denniswu501 - IdentityFile ~/.ssh/id_ed25519 + HostName vm.example.org + User researcher + IdentityFile ~/.ssh/hallmark_vm IdentitiesOnly yes StrictHostKeyChecking yes ``` The verified host key must already be in `known_hosts`. Load an encrypted key -into your SSH agent first. Check SFTP access without an interactive login: +into your SSH agent first. SSH configuration alone is sufficient for this demo. +For optional named profiles, see the [private data guide](../doc/private_data.rst). +Each collaborator defines credentials locally; a catalog stores only the +profile name. -```bash -sftp -o BatchMode=yes -o StrictHostKeyChecking=yes -b /dev/null lab-data -``` - -3. Set the exact export root and create an isolated local workspace. The example -path is illustrative and may not exist on the validation VM. +3. Set the exact export root and create a new local workspace. Choose a different +workspace name when repeating the demo; the data-server path below is an example. ```bash export HM_DEMO_URL='ssh://lab-data/srv/exports/lab/' -HM_DEMO_WORKSPACE="$(mktemp -d "${TMPDIR:-/tmp}/hallmark-private-client.XXXXXX")" -export HM_DEMO_WORKSPACE +export HM_DEMO_WORKSPACE="$HOME/hallmark-private-client" +mkdir "$HM_DEMO_WORKSPACE" cd "$HM_DEMO_WORKSPACE" ``` -SSH configuration alone is enough. To demonstrate an optional local profile, -create this file outside the catalog: - -```bash -export HALLMARK_AUTH_FILE="$HM_DEMO_WORKSPACE/auth.yml" -cat > "$HALLMARK_AUTH_FILE" <<'YAML' -version: 1 -profiles: - demo: - hosts: [lab-data] - host_key_policy: strict - connect_timeout: 10 - transfer_timeout: 60 - max_sessions: 2 -YAML -chmod 600 "$HALLMARK_AUTH_FILE" -``` - -The profile uses the identity from SSH configuration. `hosts` matches the URL -alias `lab-data`, before SSH resolves its `HostName`. Each collaborator defines -their own local profile; catalog configuration stores only its name. - -4. Discover the two indexed runs and prepare a local `.hm`. +4. Discover the two indexed runs and prepare a local `.hm`. The format selects +run files and extracts their run numbers; it does not authorize a download. ```bash -hallmark init ./client --from "$HM_DEMO_URL" \ - --auth demo --fmt 'runs/run_{run:03d}.dat' -cat ./client/.hm/config.yml -cat ./client/.hm/data.tsv +hallmark init ./client --from "$HM_DEMO_URL" --fmt 'runs/run_{run:03d}.dat' +cd client ``` The destination can contain existing files when `.hm` is absent; those files are preserved. An existing `.hm` is rejected before contacting the data server. +Only listings and published metadata are read. Available sizes, modification +times, and published checksums are retained; missing checksums stay unknown. +Dataset files are not downloaded to calculate checksums. -Expected for the illustrative export: two run rows, with sizes and modification -times where available. Published manifest checksums are attached when present; -missing checksums stay unknown. The data remote is `origin` with `auth: demo`. -Only listings and catalog metadata are fetched. Dataset files are not read to -compute checksums, and no remote commands or server Python are required. +Omit `--fmt` to catalog every file below the dataset root. A filter such as +`--filter 'runs/*.dat'` selects relative paths without defining parameters. +`init --backend NAME --backend-options FILE` selects an installed data backend +and reads nonsecret options from a YAML mapping. See +[data backends](../doc/backends.rst) for registration and multiple-server routing. -Omit `--fmt` to recursively catalog every file below the URL, including supporting -files. Alternatively, `--filter '**/*.dat'` selects paths without defining -parameters. Filters and formats never authorize downloads. Discovery reports -completed directories and discovered files while its total remains unknown. - -5. Inspect the transfer plan before approving a download. +5. Review an offline plan for one run, then approve its transfer. The second +command displays the plan again and asks for confirmation; answer `y` only when +the source, destination, and selection are correct. The final check stops the +workflow if no selected input was downloaded. ```bash -cd "$HM_DEMO_WORKSPACE/client" -hallmark info -hallmark download --tsv data.tsv --dry-run hallmark download runs/run_001.dat --dry-run -``` - -A dry run contacts no server. It reports count, known bytes, unknown-size count, -source, and destination. The CLI reports duration as unknown; Python callers -can estimate it from a supplied transfer rate when all file sizes are known. -It does not validate credentials or remote-file existence. To download the -selected run, inspect the displayed plan and answer `y` at the prompt: - -```bash hallmark download runs/run_001.dat --max-workers 2 -cat runs/run_001.dat -``` - -Every nonempty CLI transfer prompts, including one file. Refusal or end-of-input -transfers nothing. `--yes` is deprecated and no longer bypasses approval. -Explicit file paths retain their catalog checksums. `--tsv data.tsv` selects -all rows in that table; `--all` selects all cataloged files. Overlapping paths -are deduplicated, and `--all` cannot be combined with explicit paths or TSVs. - -6. Use the SFTP URL alias and change a profile reference. - -```bash -hallmark set-config --remote-name origin \ - --remote-url "sftp://${HM_DEMO_URL#ssh://}" --remote-auth demo -hallmark download --filter 'runs/run_00[12].dat' --output ../via-sftp --dry-run - -# Return to SSH configuration alone, without a named profile. -hallmark set-config --remote-name origin --remote-auth '' -hallmark download runs/run_001.dat --output ../via-ssh-config --dry-run - -# Restore the profile for the Python examples. -hallmark set-config --remote-name origin --remote-auth demo -``` - -Both schemes use SFTP for discovery and transfer. SFTP-only accounts can complete -the whole workflow. Omit `--dry-run` when ready to inspect and approve a transfer. -Relative output paths use the current directory. A bare `.hm` catalog requires -an explicit `--output`. - -7. Inspect and approve a Python download. Stay in the `client` directory. -The script is saved locally so its approval prompt can read terminal input. - -```bash -cat > ../download_subset.py <<'PYTHON' -from hallmark import Repo -from hallmark.downloader import DownloadError - -repo = Repo(".") -plan = repo.plan_download( - output_path="../python-subset", file_paths=["runs/run_001.dat"], - remote_name="origin", -) -print(plan.summary()) -for item in plan.items: - print(item.relative_path, item.checksum, item.size_bytes) -if input("Download these files? [y/N] ").strip().lower() != "y": - raise SystemExit("Download declined") -try: - result = repo.download(plan, approved=True, max_workers=2, progress=True) -except DownloadError as exc: - raise SystemExit(f"Download could not start: {exc}") from exc -if result["failed"]: - raise SystemExit("\n".join(result["errors"])) -print(result) -PYTHON -python ../download_subset.py -``` - -The plan fixes its source, profile reference, backend settings, destination, -paths, and checksums. -Later catalog or configuration changes do not redirect it. Without -`approved=True`, nonempty Python transfers fail before connecting. Check the -returned `failed` count; successful files remain when another transfer fails. - -Files are written to temporary paths and published atomically after transfer and -any checksum verification. Failed files preserve an existing destination and -remove their temporary file. Repeating a download transfers the selection again. -Cancellation closes the connections and processes started for the download. -Byte progress shows an ETA when a total is known; unknown totals stay indeterminate. - -8. Request an approved download during initialization, using either interface. -The CLI prepares the catalog first and then displays its normal approval prompt: - -```bash -hallmark init ../init-and-download --from "$HM_DEMO_URL" \ - --auth demo --filter 'runs/run_001.dat' --with-download +test -f runs/run_001.dat ``` -Declining keeps the completed catalog available. Empty selections need no -approval. Python receives the completed plan through a callback: - -```bash -cat > ../init_with_approval.py <<'PYTHON' -import os -from hallmark import Repo - - -def approve(plan): - print(plan.summary()) - return input("Download these files? [y/N] ").strip().lower() == "y" - - -repo = Repo.init( - "../python-init", from_url=os.environ["HM_DEMO_URL"], auth="demo", - filter="runs/run_001.dat", download=True, approve=approve, progress=True, -) -print(repo.dothm.path) -PYTHON -python ../init_with_approval.py -``` - -9. Explore CyVerse and DESI with the same metadata-only workflow. These examples -use public endpoints, so no named auth profile is supplied. A broad URL can -require many directory-listing requests; use a specific subtree when appropriate. - -```bash -cd "$HM_DEMO_WORKSPACE" -hallmark init ./cyverse --from \ - 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/' \ - --filter '**/*.uvfits' -hallmark init ./desi --from 'https://data.desi.lbl.gov/public/' --filter '**/*.fits' - -cd "$HM_DEMO_WORKSPACE/cyverse" -hallmark download --all --dry-run -cd "$HM_DEMO_WORKSPACE/desi" -hallmark download --all --dry-run -``` +A dry run reports file count, known bytes, unknown-size count, source, and +destination without contacting the server. It does not check credentials or +remote-file existence. Every nonempty CLI transfer prompts, including one file. +Refusal transfers nothing and leaves the catalog available. Empty selections +need no confirmation. `--all` selects every cataloged file; `--tsv data.tsv` +selects that table. Neither flag supplies approval. -Choose actual catalog paths before downloading, for example -`hallmark download PATH-FROM-THE-CATALOG`. This displays a plan and prompts for -approval. A format such as `--fmt 'runs/run_{run:03d}.dat'` is a parameterized -path template; a filter such as `--filter '**/*.fits'` is a relative path glob. -Neither supplies approval. No `--index-format cyverse-html` option is needed. +Successful files are published only after transfer and any checksum verification. +A failed transfer preserves an existing destination and removes its temporary +file. Cancellation closes connections started for the operation. Repeating a +download transfers the selection again. -The equivalent Python preparation, from the workspace root, is: +6. Use the equivalent SFTP URL for this data remote. Changing the configuration +does not transfer files; the dry run shows the new source and another destination. ```bash -cd "$HM_DEMO_WORKSPACE" -python - <<'PYTHON' -from hallmark import Repo - -cyverse = Repo.init( - "cyverse-python", - from_url="https://data.cyverse.org/dav-anon/iplant/commons/" - "cyverse_curated/EHTC_FirstM87Results_Apr2019/", - filter="**/*.uvfits", progress=True, -) -desi = Repo.init( - "desi-python", from_url="https://data.desi.lbl.gov/public/", - filter="**/*.fits", progress=True, -) -for repo in (cyverse, desi): - plan = repo.plan_download() - print(plan.summary()) -PYTHON -``` - -For a subsequent selected download, use the inspect/approve sequence in step 7 -with `Repo("cyverse-python")` or `Repo("desi-python")` and an actual catalog path. -HTTPS download authentication retains Requests' `.netrc` and environment behavior. -A server with no usable index must expose a published Hallmark catalog or a -browsable directory; arbitrary HTTPS URLs cannot reveal hidden files. - -10. Reuse a catalog and finish the session. +hallmark set-config --remote-name origin \ + --remote-url "sftp://${HM_DEMO_URL#ssh://}" +hallmark download runs/run_001.dat --output ../via-sftp --dry-run +``` + +Both `ssh://` and `sftp://` use SFTP for discovery and data transfer. Relative +output paths use the current directory. A bare `.hm` catalog requires an +explicit `--output`. To set a named authentication profile, use +`hallmark set-config --remote-name origin --remote-auth PROFILE`; an empty +profile name returns to SSH configuration alone. + +7. Use the same review step in Python. The +[scientific workflows notebook](scientific_workflows_python.ipynb) demonstrates +`Repo.plan_download()` followed by explicit approval and +`repo.download(plan, approved=True)`. It stops on refusal or returned transfer +failures. A plan retains its source, profile reference, backend settings, +destination, paths, and checksums even if repository configuration later changes. + +8. To review a download as part of initialization, add `--with-download` to an +`init --from` command. The catalog is created before the approval prompt; +declining keeps it available. Python accepts `download=True` and an +`approve(plan)` callback. The [private data guide](../doc/private_data.rst) +contains both forms. + +9. Use the same discovery workflow for public datasets. The +[scientific CLI workflows](scientific_workflows_cli.md) select one EHT file from +its [UVFITS directory](https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/) +and one DESI redrock product from +[HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). +These narrow dataset roots avoid listing an entire archive. HTTPS requires a +usable directory index or a backend that can enumerate the dataset; arbitrary +URLs cannot reveal hidden files. + +10. Reuse the existing catalog without downloading dataset files. This local +clone retains the committed catalog and its Git history, including the original +SSH data URL. The uncommitted configuration change in step 6 remains local. ```bash cd "$HM_DEMO_WORKSPACE" hallmark clone ./client/.hm ./catalog-copy -hallmark init ./local-experiment -unset HALLMARK_AUTH_FILE ``` -A local or Git-hosted Hallmark repository retains its history. A published -HTTP/SFTP catalog directory is imported as a snapshot with new local history. -Use `--source-type git` to require a Git endpoint, or `--source-type catalog` -to require a published snapshot. Automatic HTTP(S) detection checks for -metadata at the root and in `.hm`, then tries Git if neither snapshot exists. -Git credentials are separate from data-remote profile references. Cloning -preserves the complete catalog and Git HEAD. To download selected files after -cloning, add `--with-download --filter PATTERN`; clone filters require download -intent and do not remove catalog rows. `--download` remains an alias. -Declining approval leaves the catalog available, and empty selections do not -prompt. Raw dataset URLs belong to `init --from`. - -`clone` downloads no dataset files by default, so `--no-fetch-data` is now only -a compatibility alias. Plain `init` creates a local repository; the older remote -`build` command is deprecated in favor of `init --from`. Discovery uses no -`--allow-remote-commands` or `--remote-hash` flag, and does not read dataset files -to compute checksums. -For local `add`/`commit` experiments, use a separate compatible one-format -repository; downloading a remote catalog does not populate the local object store. - -The catalog can also live on GitHub or an HTTPS metadata server while its -data remote remains on the SSH server. For example, after the team publishes -`.hm` as a Git repository, collaborators can run: - -```bash -hallmark clone https://github.com/example/lab-catalog.git ./shared-catalog -``` - -This uses Git credentials and preserves the stored data URL and profile -reference. A published snapshot at `https://catalogs.example.org/lab/` -can be cloned in the same way, starting local history. Cloning metadata alone -does not require the payload profile. - -Use `init --backend NAME --backend-options FILE` to select an installed data -backend and load its nonsecret options from a YAML mapping. Python accepts -`backend=NAME, backend_options=options`. See [data backends](../doc/backends.rst) -for registration and a multiple-server example. - -See [the private data guide](../doc/private_data.rst) for profile settings, -error handling and operational details. The auth-file override above lasts only -until `unset HALLMARK_AUTH_FILE`; existing catalog profile references still require -a matching local profile for later downloads. +`clone` copies existing catalogs; raw dataset URLs belong to `init --from`. +A Git catalog can live on GitHub while its data remains on the SSH server. A +published HTTP/SFTP catalog snapshot can likewise use a different data server; +snapshot imports start local history. Cloning metadata does not require the +configured data profile or dataset credentials. Git catalog authentication and +data authentication are independent. + +Clone preserves the complete catalog. Its `--filter` and `--fmt` options select +optional downloads and require `--with-download` (`--download` is an alias). +They do not remove catalog rows. Plain `init` creates a local repository; the +legacy remote `build` command is deprecated in favor of `init --from`. +Downloading files alone does not populate the local object store; the scientific +workflows show how to add and commit selected local inputs before branching. + +This demo passed on Python 3.9 and 3.13 using a disposable local SFTP export +with two 30-byte run files and published checksums. Discovery, one approved +30-byte transfer, refusal before transfer, configuration changes, and catalog +cloning passed on both versions. These checks do not establish availability or +credentials for any external server. Keep the workspace for inspection and +remove it manually when finished. diff --git a/demo/scientific_workflows_cli.md b/demo/scientific_workflows_cli.md new file mode 100644 index 0000000..466b4a8 --- /dev/null +++ b/demo/scientific_workflows_cli.md @@ -0,0 +1,251 @@ +# Four scientific workflows from the terminal + +Use selected EHT/CyVerse and DESI products, Roman–Rubin simulations on an +existing SFTP export, and a local DES Y3 product. The matching +[Python notebook](scientific_workflows_python.ipynb) includes a worktree example. +This guide uses branches; the current CLI has no worktree command. + +Run the setup, then one numbered workflow in order in the same Bash terminal. +Each workflow uses a new directory. The examples version a small +`analysis-settings.txt` file alongside one scientific input. Editing that file +records an analysis choice; it does not process or modify the scientific data. + +## Setup and disk budget + +Install the local Hallmark checkout in your active environment and create a new +workspace outside the source checkout. `set -e` stops the shell if a command +fails or a download is declined, so later versioning commands cannot continue +with a missing input. To repeat a workflow, choose a new workspace. + +```bash +set -e +python -m pip install -e . +export HM_ROOT="$HOME/hallmark-science-cli" +mkdir "$HM_ROOT" +``` + +Use your normal Git author configuration. No optional scientific-processing +packages are needed for these examples. Each remote workflow selects one file; +review its size before approving the transfer. Allow space for the downloaded +input and a stored baseline copy. Settings changes store only the changed +settings file; they do not duplicate an unchanged scientific input. + +`init --from` discovers a raw dataset and creates a catalog using directory +listings and published metadata. It does not download dataset files. Its +filters select catalog entries. `clone` copies an existing catalog; clone +filters select optional downloads and require `--with-download`. Every nonempty +CLI transfer displays its plan and prompts for approval. `--dry-run` prepares an +offline plan without transferring files. Unknown sizes remain unknown. + +Each example uses a working directory containing `.hm`. After download, +`hallmark add '{name}'` indexes the selected local input and settings file; +`commit` stores their contents so branch checkout can restore them. The catalog +URL and data URL can be hosted separately. See the +[private data guide](../doc/private_data.rst) for authentication and catalog +hosting, and [data backends](../doc/backends.rst) for backend extensions. + +## 1. CyVerse: EHT M87 visibility provenance + +The [EHT release](https://github.com/eventhorizontelescope/2019-D01-01) contains +2017 M87 observations. Select the April 5 low-band UVFITS product from the +release's UVFITS directory. An analysis might inspect positive-weight visibility +amplitudes before imaging; here we only record that intended quality check. + +### Catalog and approve a selected download + +Initialize the selected catalog, enter its working directory, and review the +offline download plan. Run the transfer only after checking the selection and +answering `y` at its prompt. Declining leaves the catalog available. +The final check stops the workflow if discovery selected no input file. + +```bash +hallmark init "$HM_ROOT/eht" --from \ + 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/' \ + --filter 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' +cd "$HM_ROOT/eht" +hallmark download --all --dry-run +hallmark download --all +test -f 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' +``` + +### Commit, annotate a branch, and recover + +Store the original input with an analysis setting, then record a different +quality-review choice on `eht-provenance`. Returning to `main` restores the +original settings; the final command prints `qa=positive-weights`. + +```bash +printf '%s\n' 'qa=positive-weights' > analysis-settings.txt +hallmark add '{name}' +hallmark commit -m 'Preserve EHT input and analysis settings' +hallmark checkout eht-provenance +printf '%s\n' 'qa=positive-weights-and-amplitude-review' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record EHT amplitude review setting' +hallmark checkout main +cat analysis-settings.txt +``` + +This changes neither calibration nor visibilities. A scientific amplitude check +would separately interpret the UVFITS weights and polarization layout. + +## 2. HTTPS: DESI spectrum inspection and quality selections + +Use the redrock catalog from +[DR1 Iron HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). +This route downloads redshift results, not the coadded spectra. A scientific +selection could combine warning flags, spectral type, and fit-quality +statistics; applying those cuts remains a separate analysis step. + +### Catalog and download by product + +Discover only `redrock-main-dark-23040.fits` in this HEALPixel, inspect the plan, +and approve its download. The selection excludes coadds and other products; +the final check stops the workflow if discovery selected no input file. + +```bash +hallmark init "$HM_ROOT/desi" --from \ + 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/' \ + --filter 'redrock-main-dark-23040.fits' +cd "$HM_ROOT/desi" +hallmark download --all --dry-run +hallmark download --all +test -f 'redrock-main-dark-23040.fits' +``` + +### Compare two selection branches + +Commit the input with an illustrative fit-quality threshold. Record a stricter +threshold on `desi-quality`, then return to `main`; the restored setting is +`min_deltachi2=25`. No target rows are filtered by these commands. + +```bash +printf '%s\n' 'min_deltachi2=25' > analysis-settings.txt +hallmark add '{name}' +hallmark commit -m 'Preserve DESI input and analysis settings' +hallmark checkout desi-quality +printf '%s\n' 'min_deltachi2=40' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record stricter DESI quality setting' +hallmark checkout main +cat analysis-settings.txt +``` + +## 3. SSH/SFTP: Roman–Rubin products over SFTP + +Assume an existing export contains the OpenUniverse2024 stellar catalog +`stars/pointsource_10307.parquet`. Select that one product; this example does not +provision a server. A science workflow could compare magnitude selections +before matching simulated sources between surveys. Here the catalog is kept +unchanged and only the intended magnitude limit is versioned. + +### Client authentication, catalog, and download + +Add this illustrative alias to `~/.ssh/config`, replacing the host, user, and +identity with your existing export's settings. The verified host key must +already be in `known_hosts`; load an encrypted identity into your SSH agent. +The client needs OpenSSH 9.6+ and the server needs SFTP. + +```text +Host lab-data + HostName vm.example.org + User researcher + IdentityFile ~/.ssh/hallmark_vm + IdentitiesOnly yes + StrictHostKeyChecking yes +``` + +Use the `stars/` directory as the dataset root so the selected product has a +simple local filename. Discover it, review the plan, and approve the transfer. +SSH configuration supplies authentication; no Hallmark profile is required. +The final check stops the workflow if discovery selected no input file. + +```bash +hallmark init "$HM_ROOT/roman-rubin" --from \ + 'sftp://lab-data/home/researcher/hallmark-exports/openuniverse2024/stars/' \ + --filter 'pointsource_10307.parquet' +cd "$HM_ROOT/roman-rubin" +hallmark download --all --dry-run +hallmark download --all +test -f 'pointsource_10307.parquet' +``` + +### Save and compare an analysis choice + +Store the selected input and an illustrative magnitude limit. Change that limit +on `roman-rubin-selection`, then return to `main`. The restored value is +`magnitude_limit=24`; the Parquet catalog has not been filtered or converted. + +```bash +printf '%s\n' 'magnitude_limit=24' > analysis-settings.txt +hallmark add '{name}' +hallmark commit -m 'Preserve Roman-Rubin input and analysis settings' +hallmark checkout roman-rubin-selection +printf '%s\n' 'magnitude_limit=23' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record Roman-Rubin magnitude setting' +hallmark checkout main +cat analysis-settings.txt +``` + +## +1. Local init: DES Y3 covariance-aware versions + +Use an existing local copy of +`2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits`, from the +[DES Y3 release](https://www.darkenergysurvey.org/des-year-3-cosmology-results-papers/). +This example copies that input into a new workspace and makes no network +request. A scientific angular cut must apply the same row selection to the +correlation-function tables and covariance matrix. Changing a settings file +alone does not perform that operation. + +### Prepare the local analysis product + +Replace the illustrative local path with your existing DES file. Initialize a +working directory and copy the input into it, leaving the source file intact. +There is no remote download to approve. + +```bash +export HM_DES_LOCAL='/path/to/2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits' +hallmark init "$HM_ROOT/des" +cd "$HM_ROOT/des" +cp "$HM_DES_LOCAL" . +``` + +### Compare angular selections with aligned covariance + +Commit the input and an illustrative minimum angle. Record a different angle +on `des-angular-cut`, then recover the original `theta_min_arcmin=2.5` setting +from `main`. Both branches retain the original FITS data and covariance. + +```bash +printf '%s\n' 'theta_min_arcmin=2.5' > analysis-settings.txt +hallmark add '{name}' +hallmark commit -m 'Preserve DES input and analysis settings' +hallmark checkout des-angular-cut +printf '%s\n' 'theta_min_arcmin=5' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record DES angular-cut setting' +hallmark checkout main +cat analysis-settings.txt +``` + +## Disk inspection, cleanup, and validation + +Use `du -sh "$HM_ROOT"` to inspect the workspace. Keep it while comparing +branches; remove it manually when finished. No example changes the original +remote data or the local DES source file. + +All four CLI workflows passed with tiny local HTTP/SFTP fixtures on Python 3.9 +and 3.13. Each full run downloaded 66 bytes across the three remote inputs and +copied a 21-byte local input. The checks covered discovery, published checksums, +approval, settings commits, and branch restoration. Declined downloads and +empty selections stopped before versioning. Notebook copies also passed from +both the repository root and `demo/` on both Python versions. +Separate Python 3.13 runs confirmed that CLI and notebook transfer failures +stop before versioning and leave the catalog available. + +The 95 CLI regression tests passed on each Python version. Bash syntax, notebook +structure and Python 3.9 syntax, local links, Ruff, and strict Sphinx checks +passed. These fixtures do not validate scientific file formats, analysis +choices, public-server availability, or public download sizes. No research +servers were contacted and no public datasets were downloaded. diff --git a/demo/scientific_workflows_python.ipynb b/demo/scientific_workflows_python.ipynb new file mode 100644 index 0000000..a10b33a --- /dev/null +++ b/demo/scientific_workflows_python.ipynb @@ -0,0 +1,371 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "science-00", + "metadata": {}, + "source": [ + "# Four scientific workflows with Hallmark\n", + "\n", + "EHT and DESI observations, Roman–Rubin simulations, and local DES Y3 products. The matching [CLI guide](scientific_workflows_cli.md) uses the same selected files; this notebook also shows one worktree.\n", + "\n", + "Run **Setup**, then one numbered workflow from start to finish. Each uses a separate directory and one explicitly selected input. Choose a new `ROOT` to repeat an example. `Repo.init(..., from_url=...)` discovers raw datasets without downloading files. `Repo.clone(...)` copies an existing catalog; its filters select optional downloads and preserve the complete catalog and history.\n", + "\n", + "Every transfer shows a plan and asks for approval. Declining or a failed transfer stops that workflow; do not continue to its versioning cell. The completed catalog remains available. The SSH example requires an existing export and your own access configuration.\n", + "\n", + "The examples version a small `analysis-settings.txt` file alongside each unchanged input. These settings record intended analysis choices; the cells do not process the scientific data. Detailed processing is described in prose." + ] + }, + { + "cell_type": "markdown", + "id": "science-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install this checkout with `python -m pip install -e .` and install JupyterLab in the same environment with `python -m pip install jupyterlab`. Select that environment's kernel. Git needs your normal author configuration. No additional scientific-processing packages are needed for these examples.\n", + "\n", + "Import Hallmark and the standard-library tools, then choose a new workspace under your home directory. The same setup works when Jupyter starts in the repository root or in `demo/`. Set `ROOT` before beginning a workflow. The source settings below select narrow dataset roots; replace `VM_URL` and `DES_LOCAL` with your existing export and local input when using those workflows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-02", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import shutil\n", + "from hallmark import Repo\n", + "\n", + "ROOT = (Path.home() / \"hallmark-science-python\").resolve()\n", + "ROOT.mkdir(exist_ok=True)\n", + "print(ROOT)\n", + "\n", + "EHT_URL = \"https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/\"\n", + "DESI_URL = \"https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/\"\n", + "VM_URL = \"sftp://lab-data/home/researcher/hallmark-exports/openuniverse2024/stars/\"\n", + "DES_FILE = \"2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits\"\n", + "DES_LOCAL = Path.home() / \"datasets\" / DES_FILE\n" + ] + }, + { + "cell_type": "markdown", + "id": "science-03", + "metadata": {}, + "source": [ + "This helper prints the plan and accepts only an explicit `y` before transferring files. An empty selection, a declined plan, or reported download failures raise an exception so the workflow stops. File sizes come from the catalog; a duration remains unknown without a supplied transfer rate." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-04", + "metadata": {}, + "outputs": [], + "source": [ + "def download_after_review(repo, plan):\n", + " print(plan.summary())\n", + " if not plan.items:\n", + " raise RuntimeError(\"No files selected; check the source and filter\")\n", + " if input(\"Download this selection? [y/N] \").strip().lower() != \"y\":\n", + " raise RuntimeError(\"Download declined; catalog remains available\")\n", + " result = repo.download(plan, approved=True, progress=True)\n", + " if result[\"failed\"]:\n", + " raise RuntimeError(result[\"errors\"])\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "id": "science-05", + "metadata": {}, + "source": [ + "## 1. CyVerse: EHT M87 visibility provenance\n", + "\n", + "Use one April 5 low-band UVFITS file from the [EHT M87 release](https://github.com/eventhorizontelescope/2019-D01-01). A scientific analysis could inspect positive-weight visibility amplitudes and compare the two frequency bands before imaging. This example records a QA choice without recalibrating visibilities or reconstructing an image.\n", + "\n", + "Discover that file under the release's `uvfits/` dataset root with the CyVerse backend. Review the plan and approve its download into the new worktree." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-06", + "metadata": {}, + "outputs": [], + "source": [ + "EHT_FILE = \"SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits\"\n", + "eht_dir = ROOT / \"eht\"\n", + "eht = Repo.init(eht_dir, from_url=EHT_URL, backend=\"cyverse\", filter=EHT_FILE)\n", + "download_after_review(eht, eht.plan_download(all_files=True))" + ] + }, + { + "cell_type": "markdown", + "id": "science-07", + "metadata": {}, + "source": [ + "Commit the input and QA settings as a baseline, then record another choice on `eht-provenance`. Switching back restores the original settings. The final checks confirm both the restoration and the unchanged input checksum. The `{name}` format tracks the two files at the worktree root." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-08", + "metadata": {}, + "outputs": [], + "source": [ + "eht_settings = eht_dir / \"analysis-settings.txt\"\n", + "eht_settings.write_text(\"qa=positive-weights\\n\", encoding=\"utf-8\")\n", + "eht.add(\"{name}\")\n", + "eht.commit(\"Preserve EHT input and QA settings\")\n", + "eht_base = eht.branches()[\"current\"]\n", + "eht_hash = Repo.checksum(eht_dir / EHT_FILE)\n", + "eht.checkout(\"eht-provenance\")\n", + "eht_settings.write_text(\"qa=positive-weights-and-amplitude-review\\n\", encoding=\"utf-8\")\n", + "eht.add(\".\")\n", + "eht.commit(\"Record amplitude-review choice\")\n", + "eht.checkout(eht_base)\n", + "assert eht_settings.read_text(encoding=\"utf-8\") == \"qa=positive-weights\\n\"\n", + "assert Repo.checksum(eht_dir / EHT_FILE) == eht_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-09", + "metadata": {}, + "source": [ + "A worktree lets you keep an alternative branch open alongside the baseline. Create its sibling directory, open it with `Repo(path)`, and commit another settings value there. The original worktree retains its baseline settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-10", + "metadata": {}, + "outputs": [], + "source": [ + "eht.add_worktree(\"eht-alternate-qa\")\n", + "eht_other = Repo(ROOT / \"eht-alternate-qa\")\n", + "other_settings = eht_other.worktree / \"analysis-settings.txt\"\n", + "other_settings.write_text(\"qa=band-consistency-review\\n\", encoding=\"utf-8\")\n", + "eht_other.add(\".\")\n", + "eht_other.commit(\"Record alternative QA settings in a worktree\")\n", + "assert eht_settings.read_text(encoding=\"utf-8\") == \"qa=positive-weights\\n\"" + ] + }, + { + "cell_type": "markdown", + "id": "science-11", + "metadata": {}, + "source": [ + "## 2. DESI: spectrum inspection and redshift-quality choices\n", + "\n", + "Select only `redrock-main-dark-23040.fits` from [DR1 Iron HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). A scientific analysis could compare redshift-quality thresholds after checking the redshift and fibermap records by TARGETID. Inspecting spectra would also require the corresponding coadd, which this example does not download.\n", + "\n", + "The settings below illustrate a threshold comparison, not official DESI LSS selection cuts. See the [redrock data model](https://desidatamodel.readthedocs.io/en/latest/DESI_SPECTRO_REDUX/SPECPROD/healpix/SURVEY/PROGRAM/PIXGROUP/PIXNUM/redrock-SURVEY-PROGRAM-PIXNUM.html) and [DR1 known issues](https://data.desi.lbl.gov/doc/releases/dr1/known-issues/) for scientific use.\n", + "\n", + "Initialize a catalog from the narrow HTTPS dataset root, review the selected redrock file, and approve its download." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-12", + "metadata": {}, + "outputs": [], + "source": [ + "DESI_FILE = \"redrock-main-dark-23040.fits\"\n", + "desi_dir = ROOT / \"desi\"\n", + "desi = Repo.init(desi_dir, from_url=DESI_URL, backend=\"http\", filter=DESI_FILE)\n", + "download_after_review(desi, desi.plan_download(all_files=True))" + ] + }, + { + "cell_type": "markdown", + "id": "science-13", + "metadata": {}, + "source": [ + "Commit the downloaded input and an illustrative DELTACHI2 threshold. Change only that threshold on `desi-quality`, commit it, then restore the baseline and check the unchanged input. No galaxy selection or FITS rewrite occurs here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-14", + "metadata": {}, + "outputs": [], + "source": [ + "desi_settings = desi_dir / \"analysis-settings.txt\"\n", + "desi_settings.write_text(\"min_deltachi2=25\\n\", encoding=\"utf-8\")\n", + "desi.add(\"{name}\")\n", + "desi.commit(\"Preserve DESI input and quality settings\")\n", + "desi_base = desi.branches()[\"current\"]\n", + "desi_hash = Repo.checksum(desi_dir / DESI_FILE)\n", + "desi.checkout(\"desi-quality\")\n", + "desi_settings.write_text(\"min_deltachi2=40\\n\", encoding=\"utf-8\")\n", + "desi.add(\".\")\n", + "desi.commit(\"Record stricter redshift-quality setting\")\n", + "desi.checkout(desi_base)\n", + "assert desi_settings.read_text(encoding=\"utf-8\") == \"min_deltachi2=25\\n\"\n", + "assert Repo.checksum(desi_dir / DESI_FILE) == desi_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-15", + "metadata": {}, + "source": [ + "## 3. SSH/SFTP: Roman–Rubin products\n", + "\n", + "Use the single `pointsource_10307.parquet` input from an existing export of the [OpenUniverse2024 shared-sky catalogs](https://irsa.ipac.caltech.edu/data/theory/openuniverse2024/roman/preview/roman_rubin_cats_v1.1.2_faint/). A scientific analysis could join source IDs with the separate flux table and compare Rubin color or magnitude selections. That additional table and the scientific processing are outside this example.\n", + "\n", + "The [CLI guide](scientific_workflows_cli.md#3-sshsftp-romanrubin-products-over-sftp) shows an illustrative `lab-data` SSH configuration. Replace its host, user, and key with your own settings and verify the server's host key. Set `VM_URL` to an export you already have permission to read; the notebook does not provision a VM or prepare its data.\n", + "\n", + "Discover the selected file under the export's `stars/` directory, then review and approve its SFTP download. Other workflows do not use this SSH connection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-16", + "metadata": {}, + "outputs": [], + "source": [ + "VM_FILE = \"pointsource_10307.parquet\"\n", + "vm_dir = ROOT / \"roman-rubin\"\n", + "vm = Repo.init(vm_dir, from_url=VM_URL, backend=\"ssh\", filter=VM_FILE)\n", + "download_after_review(vm, vm.plan_download(all_files=True))" + ] + }, + { + "cell_type": "markdown", + "id": "science-17", + "metadata": {}, + "source": [ + "Keep the simulation input unchanged and record an intended magnitude limit in the settings file. Commit a different limit on `roman-rubin-selection`, then return to the baseline and verify both the settings and input checksum. No simulated-source selection is applied." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-18", + "metadata": {}, + "outputs": [], + "source": [ + "vm_settings = vm_dir / \"analysis-settings.txt\"\n", + "vm_settings.write_text(\"magnitude_limit=24\\n\", encoding=\"utf-8\")\n", + "vm.add(\"{name}\")\n", + "vm.commit(\"Preserve Roman-Rubin input and selection settings\")\n", + "vm_base = vm.branches()[\"current\"]\n", + "vm_hash = Repo.checksum(vm_dir / VM_FILE)\n", + "vm.checkout(\"roman-rubin-selection\")\n", + "vm_settings.write_text(\"magnitude_limit=23\\n\", encoding=\"utf-8\")\n", + "vm.add(\".\")\n", + "vm.commit(\"Record brighter magnitude-limit setting\")\n", + "vm.checkout(vm_base)\n", + "assert vm_settings.read_text(encoding=\"utf-8\") == \"magnitude_limit=24\\n\"\n", + "assert Repo.checksum(vm_dir / VM_FILE) == vm_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-19", + "metadata": {}, + "source": [ + "## +1. Local init: DES Y3 covariance-aware analysis versions\n", + "\n", + "Start with your existing local `2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits` file. This workflow makes no server connection. See the [DES Y3 papers](https://www.darkenergysurvey.org/des-year-3-cosmology-results-papers/) for the published analysis.\n", + "\n", + "A scientific angular selection must apply the same retained rows to the data vector and both covariance axes, retaining the corresponding redshift-distribution information. The thresholds below only illustrate versioned settings; they are not the DES team's bin-dependent scale cuts and do not create a reduced likelihood.\n", + "\n", + "Set `DES_LOCAL` to your input. Copy it into a new experiment directory and initialize a local repository there, leaving the original file untouched." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-20", + "metadata": {}, + "outputs": [], + "source": [ + "assert DES_LOCAL.is_file(), \"Set DES_LOCAL to your existing DES input\"\n", + "des_dir = ROOT / \"des\"\n", + "des_dir.mkdir()\n", + "des_file = Path(shutil.copy2(DES_LOCAL, des_dir / DES_LOCAL.name))\n", + "des = Repo.init(des_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "science-21", + "metadata": {}, + "source": [ + "Commit the copied input and angular settings. Record a second threshold on `des-angular-cut`, then restore the baseline and verify that both input copies still have the original checksum. This changes the text settings only; no covariance or data-vector processing occurs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-22", + "metadata": {}, + "outputs": [], + "source": [ + "des_settings = des_dir / \"analysis-settings.txt\"\n", + "des_settings.write_text(\"theta_min_arcmin=2.5\\n\", encoding=\"utf-8\")\n", + "des.add(\"{name}\")\n", + "des.commit(\"Preserve DES input and angular settings\")\n", + "des_base = des.branches()[\"current\"]\n", + "des_hash = Repo.checksum(DES_LOCAL)\n", + "des.checkout(\"des-angular-cut\")\n", + "des_settings.write_text(\"theta_min_arcmin=5\\n\", encoding=\"utf-8\")\n", + "des.add(\".\")\n", + "des.commit(\"Record alternative angular threshold\")\n", + "des.checkout(des_base)\n", + "assert des_settings.read_text(encoding=\"utf-8\") == \"theta_min_arcmin=2.5\\n\"\n", + "assert Repo.checksum(des_file) == Repo.checksum(DES_LOCAL) == des_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-23", + "metadata": {}, + "source": [ + "## Inspect disk use and keep or remove your experiment\n", + "\n", + "The plan reports available transfer sizes before approval. Allow room for each downloaded input and its stored version; the EHT worktree also contains its own input copy. The DES workflow copies your existing input once and stores a version in `.hm/objects`. Changing these small settings files does not make new versions of the unchanged scientific inputs.\n", + "\n", + "Keep `ROOT` to retain the experiments and their history. To discard them, remove only the new experiment directories you created, including `eht-alternate-qa` if you ran that cell. Do not delete a shared dataset directory or remove only a worktree's `.hm` link." + ] + }, + { + "cell_type": "markdown", + "id": "science-24", + "metadata": {}, + "source": [ + "## Validation record\n", + "\n", + "All 11 code cells passed using tiny local HTTP/SFTP fixtures on Python 3.9 and 3.13, with a separate execution copy launched from the repository root and from `demo/` on each version. Each execution downloaded 66 bytes across the three remote inputs and copied a 21-byte local DES input. The checks covered discovery, published checksums, approval, settings commits, worktree isolation, input checksums, and branch recovery.\n", + "\n", + "Separate Python 3.13 runs confirmed that refusal and reported transfer failures stop before versioning while leaving the catalog available.\n", + "\n", + "The source notebook passes notebook-structure and Python 3.9 syntax checks, keeps its outputs cleared, and requires explicit download approval. Rendered headings and local links were checked. The matching [CLI guide](scientific_workflows_cli.md#disk-inspection-cleanup-and-validation) records the CLI and documentation checks. These fixtures do not validate scientific processing, public-server availability, or public download sizes; no research servers were contacted and no public datasets were downloaded.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/api.rst b/doc/api.rst index aceb8c1..38b34ea 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -97,7 +97,9 @@ Create a local repository:: Initialize a catalog from a remote dataset without downloading dataset files:: - hallmark init ./desi --from https://data.desi.lbl.gov/public/ --filter '**/*.fits' + hallmark init ./desi --from \ + https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/ \ + --filter 'redrock-main-dark-23040.fits' hallmark init ./lab --from ssh://lab-data/srv/data/ --fmt 'run{run:d}.h5' Use ``hallmark clone SOURCE PATH`` for an existing Hallmark Git repository @@ -117,23 +119,29 @@ Preview the selected files using the local catalog, then confirm a download:: Explicit paths and ``--tsv data.tsv`` also select files. Every nonempty transfer requires interactive confirmation, including transfers requested through ``init --with-download`` or ``clone --with-download``. Clone accepts its older -``--download`` alias. Clone filters and formats require download intent and -leave the complete catalog unchanged. The old ``--yes`` option no longer +``--download`` alias. Clone filters and formats require ``--with-download`` +and leave the complete catalog unchanged. The old ``--yes`` option no longer bypasses approval. A filter or filename format never authorizes a transfer. -Python uses the same plan and requires explicit approval:: +Python uses the same plan. First inspect the selected files:: from hallmark import Repo repo = Repo.init('lab', from_url='ssh://lab-data/srv/data/', progress=True) plan = repo.plan_download(filter='runs/**') print(plan.summary()) + +After reviewing the plan, approve the download and check for failures:: + result = repo.download(plan, approved=True, progress=True) + if result['failed']: + raise RuntimeError('\n'.join(result['errors'])) -Plans preserve their selected remote, backend settings, destination and checksums even when -repository configuration later changes. Size estimates require recorded file -sizes; duration estimates also require a supplied transfer rate. +Plans preserve their selected remote, backend settings, destination and +checksums even when repository configuration later changes. Size estimates +require recorded file sizes; duration estimates also require a supplied +transfer rate. .. automodule:: hallmark.download_plan :members: diff --git a/doc/design.rst b/doc/design.rst index 5e3e82b..f302dfc 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -77,8 +77,9 @@ Currently, |hallmark|_ has two built-in APIs: 1. Python API: the native API that all |hallmark|_ features are implemented in. ``State`` is the active object during the process lifetime. - ``Dothm`` and ``Worktree`` are optional depending on workflow (for - example, in-memory workflows may omit both). + ``Repo`` opens an on-disk repository. For an in-memory file index, + ``ParaFrame`` can discover and select files without ``Dothm`` or + ``Worktree``. 2. CLI: python features wrapped by ``click``. @@ -108,7 +109,7 @@ Repository ``Repo`` 2. bare repository::: - "repo.hm/" <--- Dothm, a standard or bare git repo + "repo.hm/" <--- Dothm, a standard git repo without a data worktree 3. shared repository (multiple worktrees)::: diff --git a/doc/index.rst b/doc/index.rst index e6e97d4..86da7d8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -10,9 +10,13 @@ For private datasets, see :doc:`private_data` for SSH setup, CLI workflows, Python examples and troubleshooting. Download the :download:`private data demo <../demo/private_data_demo.md>` -for a sequential CLI and Python walkthrough. See :doc:`backends` to add +for a sequential CLI walkthrough. See :doc:`backends` to add a data service or combine several servers under one catalog. +The :download:`scientific CLI workflows <../demo/scientific_workflows_cli.md>` +and :download:`scientific Python notebook <../demo/scientific_workflows_python.ipynb>` +follow four examples using EHT, DESI, Roman–Rubin and local DES data. + .. toctree:: :maxdepth: 2 :caption: Contents: diff --git a/doc/private_data.rst b/doc/private_data.rst index c5659cb..c325afc 100644 --- a/doc/private_data.rst +++ b/doc/private_data.rst @@ -152,13 +152,11 @@ Declining keeps the new catalog available; an empty selection needs no approval: --filter 'runs/run_001.h5' --with-download An existing catalog can be cloned from a local path, a Git endpoint, or an -HTTP/SFTP directory containing its metadata: +HTTP/SFTP directory containing its metadata. For example, copy this catalog: .. code-block:: bash hallmark clone ./.hm ../lab-copy - hallmark clone 'https://git.example.org/team/lab.git' ../lab-history - hallmark clone 'https://data.example.org/catalogs/lab.hm/' ../lab-snapshot Git endpoints preserve the complete catalog and history. Published HTTP/SFTP snapshots start new local history. A snapshot URL may name the metadata @@ -171,7 +169,7 @@ connection failures and malformed snapshots are reported rather than treated as dataset directories. Use ``init --from`` for raw datasets. Clone filters and formats select files to download, leaving the full catalog -and Git HEAD unchanged. They require ``--with-download`` (the older +and its history unchanged. They require ``--with-download`` (the older ``--download`` spelling remains an alias): .. code-block:: bash @@ -180,8 +178,8 @@ and Git HEAD unchanged. They require ``--with-download`` (the older --filter 'runs/run_001.h5' --with-download ``--no-fetch-data`` remains a compatibility alias for the default behavior -without payload downloads. Git authentication and data authentication are -independent; cloning a catalog does not require its payload credentials. +without dataset downloads. Git authentication and data authentication are +independent; cloning a catalog does not require credentials for its data. 4. Use a local authentication profile ------------------------------------- @@ -267,7 +265,8 @@ Inspect the plan, then approve the download: Without ``approved=True``, a nonempty transfer raises ``DownloadError`` before opening a connection. A plan freezes the selected files, source URL, profile reference, backend, backend options and destination. Later catalog or remote -configuration changes do not redirect it. ``plan_download(filter="**/*.h5")`` scopes the complete catalog; +configuration changes do not redirect it. ``plan_download(filter="**/*.h5")`` +selects matching files from the catalog; ``plan_download(output_path="subset", tsv_names=["data.tsv"])`` selects a TSV and destination. Planning makes no network requests. Known size totals and unknown-size counts are available as ``known_bytes`` and @@ -303,9 +302,11 @@ such as DESI. No CyVerse-specific index option is required: .. code-block:: bash hallmark init ./cyverse --from \ - 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/' \ - --filter '**/*.uvfits' - hallmark init ./desi --from 'https://data.desi.lbl.gov/public/' --filter '**/*.fits' + 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/' \ + --filter 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' + hallmark init ./desi --from \ + 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/' \ + --filter 'redrock-main-dark-23040.fits' Each URL is the full recursive discovery root. A broad root can require many listing requests even with a file filter; use a specific subtree when that is @@ -322,18 +323,19 @@ without ``--dry-run``. Python follows the same steps: cyverse = Repo.init( "cyverse-python", from_url="https://data.cyverse.org/dav-anon/iplant/commons/" - "cyverse_curated/EHTC_FirstM87Results_Apr2019/", - filter="**/*.uvfits", progress=True, + "cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/", + filter="SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits", progress=True, ) desi = Repo.init( - "desi-python", from_url="https://data.desi.lbl.gov/public/", - filter="**/*.fits", progress=True, + "desi-python", + from_url="https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/" + "healpix/main/dark/230/23040/", + filter="redrock-main-dark-23040.fits", progress=True, ) - plan = desi.plan_download(filter="dr1/**/selected-file.fits") + plan = desi.plan_download(all_files=True) print(plan.summary()) - # Replace the illustrative pattern with an actual catalog path, inspect, - # and authorize only the intended selection: - # result = desi.download(plan, approved=True, progress=True) + +Inspect this plan and approve it as in the Python download example above. HTTP provides file transfer but does not define directory enumeration. Hallmark recognizes supported HTML indexes and directory landing pages automatically. @@ -344,7 +346,7 @@ API. Hallmark cannot infer hidden file URLs. 7. Host catalogs separately from their data ------------------------------------------- -A catalog's location does not set its payload location. For example, a team +A catalog's location does not set its data location. For example, a team can push the Git repository in ``.hm`` to GitHub while its data remote still points to ``ssh://lab-data/srv/exports/lab/`` with ``auth: lab``. Collaborators clone the existing metadata using their Git credentials: @@ -377,13 +379,13 @@ Then clone the snapshot: hallmark clone 'https://catalogs.example.org/lab/' ./snapshot-lab -Snapshot authentication uses the metadata server's settings. The SSH payload +Snapshot authentication uses the metadata server's settings. The SSH data URL and ``lab`` reference are preserved without resolving that profile during cloning. For an SFTP snapshot, ``clone --auth catalog-reader`` selects its metadata profile; it does not replace a data remote's profile. Snapshot imports start local Git history and have no upstream Git history to pull. These examples retain the usual local ``PATH/.hm`` layout while keeping -catalog hosting independent of payload hosting. +catalog hosting independent of data hosting. .. _operational-behavior-and-troubleshooting: diff --git a/doc/usecase.rst b/doc/usecase.rst index b3d57c2..cbdee06 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -38,46 +38,48 @@ Dataset files are represented through staged ``sha1`` column in associated with each file in different rows. -2. CLI: Bare Repository with Worktree Ingest --------------------------------------------- +2. CLI: Bare Repository with Explicit Downloads +----------------------------------------------- -Bob prefers managing a bare hallmark repository for storage and -staging data from a linked worktree. -He initialized the bare repository and verify its mode:: +Bob prefers keeping the data index separate from downloaded files. +He initializes a bare |hallmark|_ catalog from a simulation export and +verifies its location:: - hallmark init --bare sim.hm + hallmark init sim.hm --from ssh://campus/srv/export/ \ + --fmt 'run{run:d}/frame{frame:d}.h5' cd sim.hm hallmark info -He then attaches a worktree, stages discovered files, and commits:: +The ``.hm`` suffix selects a bare catalog without a data worktree. +Bob reviews the selection and approves downloading to an explicit directory:: - hallmark worktree add /data/outputs - cd /data/outputs - hallmark add "run{run:d}/frame{frame:d}.h5" - hallmark status - hallmark diff - hallmark commit -m "Simulation snapshots" + hallmark download --all --output ../outputs --dry-run + hallmark download --all --output ../outputs -The commit updates the same bare repository that owns the linked -worktree. +The catalog stays in ``sim.hm``; the downloaded files go to ``outputs``. +For local ingest and versioning, Bob can initialize ``outputs`` as a +standard repository and follow Alice's workflow. -3. CLI: Branch-Isolated Analysis with Multiple Worktrees --------------------------------------------------------- +3. Python: Branch-Isolated Analysis with Multiple Worktrees +----------------------------------------------------------- Carol wants to manage data from multiple simultaneous observations without mixing data. -She creates a new branch and attach a second worktree to it:: +Starting from a committed standard repository, she creates a new branch +and attaches a second worktree through the Python API:: - hallmark branch obs2 - hallmark worktree add remote:/data/obs obs2 + from hallmark import Repo -She lists linked worktrees and continue on the new branch:: + repo = Repo("obs") + repo.add_worktree("obs2") + second = Repo(repo.worktree.parent / "obs2") - hallmark worktree list - hallmark status - hallmark add "{site}/{year:d}/{day:d}.fits" - hallmark commit -m "Observation ingest on branch obs2" +After adding observation files beneath ``obs2``, she stages and commits +them in that worktree:: + + second.add("{site}/{year:d}/{day:d}.fits") + second.commit("Observation ingest on branch obs2") Each worktree stays isolated by branch, so staged state and commits do not interfere. @@ -91,12 +93,10 @@ repository:: from hallmark import Repo - repo = Repo.open("obs") - info = repo.info() - print(info.local_path, info.worktree_path) + repo = Repo("obs") + print(repo.dothm.path, repo.worktree) - for y in range(2000,2026,5): - repo.add(f"{{site}}/{y}/{{day:d}}.fits") # escape {{ and }} + repo.add("{site}/{year:d}/{day:d}.fits") repo.commit("Nightly ingest") This workflow is suitable for finer control of data ingest. @@ -105,16 +105,17 @@ This workflow is suitable for finer control of data ingest. 5. Python: In-Memory State Workflows ------------------------------------ -Emma can use an memory-backed facade for data transformations that do -not require git operations:: +Emma can use ``ParaFrame`` to discover and select files without +creating a Git repository:: - from hallmark import Repo + from hallmark import ParaFrame - repo = Repo() - repo.add("data/{site}/{year:d}/{day:d}.fits") - repo.worktree("data_transformed/{year:d}/{day:d}/{site}.fits") + data = ParaFrame.parse("{site}/{year:d}/{day:d}.fits", base_path="data") + selected = data.filter(year=2025) + print(selected) -This is especially useful for data (re-)organization. +This keeps the file index in memory. Emma can pass the selected paths to +her analysis tools; filtering the index does not modify the data files. .. |hallmark| replace:: ``hallmark`` @@ -289,7 +290,7 @@ The generated data remote ``origin`` records the source URL and optional profile name. Existing Git-hosted catalogs keep their complete catalog, history and recorded data remotes when cloned. Clone filters select optional downloads -and require ``--with-download``; they leave Git HEAD and catalog rows unchanged. +and require ``--with-download``; they leave the catalog and history unchanged. The Git catalog can be hosted separately from its data servers. Published manifest checksums are recorded in the catalog. Missing checksums @@ -301,12 +302,15 @@ be cancelled. CyVerse and common HTTPS directory indexes are detected automatically:: - hallmark init desi --from https://data.desi.lbl.gov/public/ --filter '**/*.fits' + hallmark init desi --from \ + https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/ \ + --filter 'redrock-main-dark-23040.fits' A server with no usable listing needs a published Hallmark catalog or a backend plugin that understands its API. See :doc:`backends` for the shared -interface and registration. Discovery cannot infer hidden file URLs. HTTP/SFTP catalog snapshots start local history; Git endpoints -supply catalog history. Use ``--source-type git`` for ambiguous Git URLs. +interface and registration. Discovery cannot infer hidden file URLs. +HTTP/SFTP catalog snapshots start local history; Git endpoints supply catalog +history. Use ``--source-type git`` for ambiguous Git URLs. Frank can also inspect and approve a transfer in Python:: @@ -315,7 +319,12 @@ Frank can also inspect and approve a transfer in Python:: repo = Repo('lab') plan = repo.plan_download(filter='runs/run_1.h5') print(plan.summary()) + +After reviewing the plan, he approves the download and checks for failures:: + result = repo.download(plan, approved=True, progress=True) + if result['failed']: + raise RuntimeError('\n'.join(result['errors'])) Plans are built from local metadata without contacting the server. Missing sizes and duration estimates are reported as unknown. During transfer, From 7da898c94300ed9e0ff500b0414c3bd73da1c70b Mon Sep 17 00:00:00 2001 From: Dennis Date: Tue, 22 Sep 2026 22:03:26 -0700 Subject: [PATCH 16/29] Simplify data sources and review clone downloads by default Add named DESI releases and collections, automatic filename format detection, and catalog filters that retain unmatched format rows. Use standard OpenSSH configuration and reject obsolete Hallmark auth references. Keep initialization metadata-only, preserve complete cloned catalogs, and share immutable download planning and approval across default clone review and the interactive chooser. Cover migration, offline previews, cancellation, and SSH configuration with regression tests. --- mod/hallmark/__init__.py | 4 +- mod/hallmark/catalog.py | 93 +++-- mod/hallmark/cli.py | 422 ++++++++++++++------ mod/hallmark/discovery.py | 87 +++-- mod/hallmark/download_plan.py | 6 +- mod/hallmark/downloader.py | 30 +- mod/hallmark/repo.py | 178 +++------ mod/hallmark/repo_builder.py | 12 +- mod/hallmark/repo_config.py | 35 +- mod/hallmark/sources.py | 188 +++++++++ mod/hallmark/transport/__init__.py | 6 +- mod/hallmark/transport/auth.py | 152 -------- mod/hallmark/transport/base.py | 27 +- mod/hallmark/transport/settings.py | 17 + mod/hallmark/transport/ssh.py | 13 +- test/test_backends.py | 7 +- test/test_catalog.py | 82 ++-- test/test_cli.py | 132 +++---- test/test_discovery.py | 8 +- test/test_download_plan.py | 8 +- test/test_hallmark.py | 9 +- test/test_interactive_download.py | 599 +++++++++++++++++++++++++++++ test/test_sources.py | 410 ++++++++++++++++++++ test/test_ssh_integration.py | 58 ++- test/test_transport.py | 120 ++---- 25 files changed, 1883 insertions(+), 820 deletions(-) create mode 100644 mod/hallmark/sources.py delete mode 100644 mod/hallmark/transport/auth.py create mode 100644 mod/hallmark/transport/settings.py create mode 100644 test/test_interactive_download.py create mode 100644 test/test_sources.py diff --git a/mod/hallmark/__init__.py b/mod/hallmark/__init__.py index cf30c11..41adea9 100644 --- a/mod/hallmark/__init__.py +++ b/mod/hallmark/__init__.py @@ -15,6 +15,7 @@ """Hallmark python public API""" +from .sources import DataSource, SourceRelease, register_source from .repo import Repo from .paraframe import ParaFrame from .download_plan import DownloadItem, DownloadPlan @@ -24,6 +25,7 @@ ) __all__ = [ - "Repo", "ParaFrame", "DownloadItem", "DownloadPlan", "DataBackend", + "DataSource", "SourceRelease", "register_source", "Repo", "ParaFrame", + "DownloadItem", "DownloadPlan", "DataBackend", "HttpBackend", "SshBackend", "CyVerseBackend", "RemoteEntry", "register_backend", ] diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py index 1790f35..c6de35b 100644 --- a/mod/hallmark/catalog.py +++ b/mod/hallmark/catalog.py @@ -3,18 +3,20 @@ from __future__ import annotations import re +from dataclasses import replace from io import StringIO from pathlib import Path from shutil import rmtree -from urllib.parse import urlsplit +from urllib.parse import quote, urlsplit import pandas as pd -import parse import yaml -from .discovery import discover, path_matches +from .discovery import CATALOG_COLUMNS, discover, extraction_parser, path_matches +from .sources import resolve_source from .dothm import Dothm from .error import CloneError, DestinationExistsError +from .fmt_detection import detect_fmt from .helper_functions import as_list_of_dicts from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name, row_to_path from .transport import OperationContext, RemoteSpec @@ -124,41 +126,55 @@ def _metadata_commit(repo, files, message): repo.state = repo.dothm.load() -def _write_inventory(repo, source, entries, fmt): +def _write_inventory(repo, source, entries, format, provenance): """Write discovered files and their source to a new catalog.""" - columns = ["path", "checksum_algorithm", "checksum", "size_bytes", "mtime"] + columns = list(CATALOG_COLUMNS) + templates = [format] if format is not None else detect_fmt( + [entry.path for entry in entries]) + parsers, formats, fields = [], [], [] + for template in templates: + try: + parser, names = extraction_parser(template) + except ValueError: + if format is not None: + raise + # Inference can mistake literal braces for fields. Such filenames + # still belong in the catalog, without unsafe extraction columns. + continue + parsers.append(parser) + formats.append(template) + fields.extend(name for name in names if name not in fields) rows = [] for entry in entries: row = dict(zip(columns, (entry.path, entry.checksum_algorithm, entry.checksum, entry.size, entry.mtime))) - if fmt: - matched = parse.parse(fmt, entry.path) + for parser in parsers: + matched = parser.parse(entry.path) if matched is not None: row.update({key: value for key, value in matched.named.items() if key not in columns}) + # The detector returns formats in a stable order. A row uses + # the first matching template so overlapping guesses cannot + # overwrite one another's parameter values. + break rows.append(row) - frame = pd.DataFrame(rows) - if frame.empty: - frame = pd.DataFrame(columns=columns) - data_spec = {"db": "data.tsv"} - if fmt: - data_spec["fmt"] = fmt + frame = pd.DataFrame(rows, columns=columns + list(fields), dtype=object) + data_spec = {"db": "data.tsv", "extraction": { + "automatic": format is None, "formats": formats}} remote = {"name": "origin", "url": source.url} - if source.auth: - remote["auth"] = source.auth if source.backend: remote["backend"] = source.backend if source.backend_options: remote["backend_options"] = thaw_backend_options(source.backend_options) repo.dothm.dump_yml({"data": [data_spec], "remote": [remote]}, "config") - repo.dothm.dump_yml({"source": source.url}, "meta") + repo.dothm.dump_yml({"source": provenance}, "meta") repo.dothm.dump_tsv(frame, "data", na_rep="") _metadata_commit(repo, ["config.yml", "meta.yml", "data.tsv"], "Discover remote catalog") -def initialize_remote(cls, path, url, *, backend=None, backend_options=None, - auth=None, filter=None, fmt=None, progress=False): +def initialize_remote(cls, path, source, *, release=None, collections=None, + filter=None, format=None, progress=False): """Discover a dataset and initialize its catalog without replacing local files.""" destination = Path(path).expanduser().absolute() if destination.is_symlink(): @@ -169,9 +185,10 @@ def initialize_remote(cls, path, url, *, backend=None, backend_options=None, f"Hallmark repository already exists: {dothm_path}") if destination.exists() and not destination.is_dir(): raise NotADirectoryError(f"Destination is not a directory: {path}") - path_matches("validation", filter=filter, fmt=fmt) - source = RemoteSpec.parse(str(url), auth, backend=backend, - backend_options=backend_options) + path_matches("validation", filter=filter) + if format is not None: + extraction_parser(format) + source, roots, provenance = resolve_source(source, release, collections) # Claim only the metadata directory. On failure, existing dataset files stay put. missing_parents = [] parent = dothm_path.parent @@ -194,10 +211,19 @@ def initialize_remote(cls, path, url, *, backend=None, backend_options=None, except FileExistsError as exc: raise DestinationExistsError( f"Hallmark repository already exists: {dothm_path}") from exc - with OperationContext(source) as context: - entries = discover(context, filter=filter, fmt=fmt, progress=progress) + inventory = {} + for root in roots: + scoped = source if not root else RemoteSpec.parse( + source.url.rstrip("/") + "/" + quote(root, safe="/") + "/", + backend=source.backend, backend_options=source.backend_options) + with OperationContext(scoped) as context: + for entry in discover(context, filter=filter, progress=progress, + path_prefix=root): + path = root + "/" + entry.path if root else entry.path + inventory[path] = replace(entry, path=path) + entries = [inventory[path] for path in sorted(inventory)] repo = cls.init(destination) - _write_inventory(repo, source, entries, fmt) + _write_inventory(repo, source, entries, format, provenance) return repo except BaseException: if owns_metadata: @@ -223,10 +249,8 @@ def _git_source(url, source_type): or (parsed.scheme != "sftp" and parsed.path.rstrip("/").endswith(".git"))) -def _clone_git(cls, url, destination, display_path, auth): +def _clone_git(cls, url, destination, display_path): """Clone a catalog's Git history without modifying its tracked metadata.""" - if auth is not None: - raise ValueError("Git cloning uses Git/SSH authentication, not auth profiles") dothm_path, worktree_path = cls.lwpaths(destination) local = Path(url).expanduser() git_url = str(local / ".hm") if (local / ".hm").is_dir() else url @@ -240,19 +264,18 @@ def _clone_git(cls, url, destination, display_path, auth): return cls(destination) -def clone_catalog(cls, url, path, *, auth=None, source_type="auto"): +def clone_catalog(cls, url, path, *, source_type="auto"): """ Clone an existing Git catalog or published HTTP/SFTP catalog snapshot. Existing destinations are rejected before source access. An incomplete destination created by this call is removed on failure. Dataset discovery - belongs to ``Repo.init(from_url=...)``. + belongs to ``Repo.init(source=...)``. Args: cls: Repository class used to initialize or open the result. url (str): Existing Git catalog or published snapshot location. path (Path | str): New repository destination. - auth (str, optional): Local profile for snapshot metadata access. source_type (str): ``auto``, ``git``, or ``catalog``. Returns: @@ -272,8 +295,6 @@ def clone_catalog(cls, url, path, *, auth=None, source_type="auto"): raise DestinationExistsError( f"fatal: destination path '{path}' already exists and is not empty.") is_git = _git_source(url, source_type) - if is_git and auth is not None: - raise ValueError("Git cloning uses Git/SSH authentication, not auth profiles") destination.parent.mkdir(parents=True, exist_ok=True) try: destination.mkdir() @@ -281,17 +302,17 @@ def clone_catalog(cls, url, path, *, auth=None, source_type="auto"): raise DestinationExistsError(f"Destination already exists: {path}") from exc try: if is_git: - return _clone_git(cls, url, destination, path, auth) - source = RemoteSpec.parse(url, auth) + return _clone_git(cls, url, destination, path) + source = RemoteSpec.parse(url) with OperationContext(source) as context: snapshot = _snapshot(context) if snapshot is None and not source.root.rstrip("/").endswith(".hm"): - nested = RemoteSpec.parse(url.rstrip("/") + "/.hm/", auth) + nested = RemoteSpec.parse(url.rstrip("/") + "/.hm/") with OperationContext(nested) as context: snapshot = _snapshot(context) if snapshot is None: if source_type == "auto" and source.scheme in {"http", "https"}: - return _clone_git(cls, url, destination, path, auth) + return _clone_git(cls, url, destination, path) raise CloneError("No published Hallmark catalog at this URL; " "use hallmark init PATH --from URL for a raw dataset") repo = cls.init(destination) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 050c0a9..7ae5944 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -16,6 +16,8 @@ from contextlib import contextmanager from pathlib import Path +import shlex +import sys import click import requests @@ -26,8 +28,8 @@ from . import Repo from .helper_functions import validate_path_component from .repo_builder import build_repo -from .downloader import DownloadError -from .discovery import path_matches +from .downloader import DownloadError, _select_download_items, _select_remote_config +from .sources import get_source, list_sources from .error import CheckoutError, CloneError from .repo_config import normalize_tsv_name @@ -116,6 +118,36 @@ def _report_download_results(results: dict) -> None: raise ClickException(f"Failed to download {failed} file(s)") +def _show_download_plan(plan, *, show_paths=False): + """Display recorded sizes without querying the data server.""" + click.echo(plan.summary()) + if show_paths: + click.echo(f"Files with unknown size: {plan.unknown_size_count}") + for item in plan.items[:20]: + size = (f"{item.size_bytes:,} bytes" if item.size_bytes is not None + else "size unknown") + click.echo(f" {item.relative_path.as_posix()} ({size})") + if plan.file_count > 20: + click.echo(f" ... {plan.file_count - 20} more file(s)") + + +def _execute_download(repo, plan, *, max_workers): + """Execute the exact plan the user approved and report transfer failures.""" + results = repo.download(plan, approved=True, max_workers=max_workers, + progress=True) + _report_download_results(results) + + +def _confirm_download_plan(plan, *, decline_is_skip=False, show_paths=False): + """Display a plan and approve it explicitly, optionally treating refusal as skip.""" + _show_download_plan(plan, show_paths=show_paths) + if not plan.file_count: + click.echo("No files selected for download.") + return False + return click.confirm("Download these files?", default=False, + abort=not decline_is_skip) + + def _run_download(repo, plan, *, max_workers): """ Display a download plan, request approval, and report the results. @@ -130,14 +162,155 @@ def _run_download(repo, plan, *, max_workers): ClickException: If any file transfers fail. DownloadError: If the download cannot be started. """ - click.echo(plan.summary()) - if not plan.file_count: - click.echo("No files selected for download.") + if _confirm_download_plan(plan): + _execute_download(repo, plan, max_workers=max_workers) + + +def _download_output(repo, output): + """Prompt for the missing destination of a bare catalog without creating it.""" + if repo.worktree is None and output is None: + return click.prompt( + "Output directory (relative to the current directory)", + type=click.Path(file_okay=False)) + return output + + +def _choose_download_plan(repo, *, output, remote_name, dry_run): + """Return an approved plan, or None after skipping or an offline preview.""" + while True: + choice = click.prompt( + "Select files", type=click.Choice(["patterns", "all", "skip"], + case_sensitive=False), default="skip") + if choice == "skip": + return None + patterns = [] + if choice == "patterns": + click.echo("Enter one relative-path glob per line, without shell quotes. " + "Blank finishes; patterns are ORed. ** matches recursively.") + while True: + pattern = click.prompt("Pattern", default="", show_default=False) + if not pattern: + break + patterns.append(pattern) + if not patterns: + continue + output = _download_output(repo, output) + plan = repo.plan_download(output, all_files=choice == "all", + filter=patterns or None, remote_name=remote_name) + _show_download_plan(plan, show_paths=True) + if not plan.file_count: + click.echo("No cataloged files match. Choose another selection or skip.") + continue + if dry_run: + return None + action = click.prompt( + "Next action", type=click.Choice(["download", "change", "skip"], + case_sensitive=False), default="skip") + if action == "download": + return plan + if action == "skip": + return None + + +def _download_available(repo, remote_name=None): + """Explain an empty catalog or absent remote without contacting the server.""" + remote = _select_remote_config(repo, remote_name) + if not remote or not remote.get("url"): + click.echo("No data remote is configured; skipping download. " + "Configure one with hallmark set-config --remote-url URL.") + return False + if not _select_download_items(repo, all_files=True, catalog_only=True): + click.echo("The catalog is empty; nothing to download.") + return False + return True + + +def _interactive_download(repo, *, output=None, remote_name=None, max_workers=4, + dry_run=False): + """Select and approve cataloged files in a terminal, without network preflight.""" + if not sys.stdin.isatty(): + raise ClickException( + "--interactive requires a terminal. Use --filter 'PATTERN' --dry-run " + "to preview a selection; run in a terminal to approve a download.") + if not _download_available(repo, remote_name): return - click.confirm("Download these files?", default=False, abort=True) - results = repo.download(plan, approved=True, max_workers=max_workers, - progress=True) - _report_download_results(results) + click.echo("Catalog ready. Downloading dataset files can consume substantial " + "bandwidth and disk space. Nothing transfers until you approve " + "a plan. Size estimates use recorded catalog metadata.") + try: + plan = _choose_download_plan(repo, output=output, remote_name=remote_name, + dry_run=dry_run) + except click.Abort as exc: + # Click wraps both EOF and Ctrl+C in Abort. EOF skips this optional + # step; retain Click's interrupt behavior for Ctrl+C. + if not isinstance(exc.__context__, EOFError): + raise + plan = None + if plan is None: + click.echo("No download started; the catalog is unchanged.") + return + # Outside the prompt handler: transfer errors must never become a skip. + _execute_download(repo, plan, max_workers=max_workers) + + +def _offer_download(repo, *, filters=(), output=None, interactive=False): + """Review downloads after cloning, or print equivalent commands for later.""" + if not sys.stdin.isatty(): + click.echo("Warning: download approval is unavailable without " + "a terminal. The catalog is ready; download was skipped.", err=True) + try: + _download_available(repo) + except (DownloadError, ValueError) as exc: + # A skipped handoff cannot fail successful catalog creation, even + # when later downloading will require extra configuration. + click.echo(f"Before downloading: {exc}", err=True) + directory = repo.worktree if repo.worktree is not None else repo.dothm.path + arguments = ["hallmark", "download"] + if filters: + for pattern in filters: + arguments.extend(["--filter", pattern]) + else: + arguments.append("--all") + if output is not None: + # The example changes directory. Preserve --output's meaning relative + # to the original invocation, as plan_download does. + arguments.extend(["--output", str(Path(output).expanduser().resolve())]) + command = shlex.join(arguments) + needs_output = repo.worktree is None and output is None + if needs_output: + command += " --output '/path/to/downloads'" + click.echo("Preview this selection locally; use --filter 'PATTERN' " + "instead of --all to narrow it:") + click.echo(f" cd -- {shlex.quote(str(directory))}") + click.echo(f" {command} --dry-run") + click.echo("Run the download in a terminal; explicit approval is required:") + click.echo(f" {command}") + if needs_output: + click.echo("Replace /path/to/downloads with your output directory.") + return + with _translate_cli_errors(DownloadError, ValueError, OSError, + prefix="Catalog created; download failed"): + if interactive: + _interactive_download(repo, output=output) + return + if not _download_available(repo): + return + click.echo("Catalog ready. Review the recorded sizes before downloading; " + "transfers can consume substantial bandwidth and disk space.") + try: + output = _download_output(repo, output) + plan = repo.plan_download(output, all_files=True, filter=filters or None) + approved = _confirm_download_plan( + plan, decline_is_skip=True, show_paths=True) + except click.Abort as exc: + if not isinstance(exc.__context__, EOFError): + raise + approved = False + if not approved: + click.echo("No download started; the catalog is unchanged.") + return + # Keep transfer errors and interrupts outside the prompt's EOF handler. + _execute_download(repo, plan, max_workers=4) @click.group() @@ -150,67 +323,82 @@ def hallmark(ctx): manage data products in a complex workflow. """ # if the invoked subcommand is one of the commands that does not require a repo - if ctx.invoked_subcommand in [None, "init", "clone", "build"]: + if ctx.invoked_subcommand in [None, "init", "clone", "build", "sources"]: # return early without attempting to open a repository return # attempt to open the hallmark repository in the current directory - with _translate_cli_errors(GitError, prefix="Failed to open hallmark repository"): + with _translate_cli_errors( + GitError, ValueError, prefix="Failed to open hallmark repository"): ctx.obj = Repo(".") -def _backend_options_file(path): - """Read a backend's configuration from a YAML mapping.""" - if path is None: - return None - with Path(path).open(encoding="utf-8") as handle: - options = yaml.safe_load(handle) - if not isinstance(options, dict): - raise ValueError("Backend options file must contain a YAML mapping") - return options - - -@hallmark.command(short_help="Initialize a hallmark repository.") +@hallmark.command(short_help="List named data sources and available collections.") +@click.argument("name", required=False) +def sources(name): + """Show supported sources, releases and collection roots without crawling.""" + with _translate_cli_errors(ValueError): + descriptors = [get_source(name)] if name else list_sources() + for source in descriptors: + click.echo(f"{source.name}: {source.description}") + for release, definition in source.releases.items(): + click.echo(f" {release}: {definition.description or release}") + click.echo(f" {definition.url}") + if name: + for collection, roots in definition.collections.items(): + description = definition.collection_descriptions.get( + collection, "") + click.echo(f" {collection}: {description}") + for root in roots: + click.echo(f" {definition.url.rstrip('/')}/{root}/") + if name: + click.echo(" Omit --collection to catalog the entire release.") + + +@hallmark.command(short_help="Initialize a local repository or data catalog.") @click.argument("path") -@click.option("--from", "from_url", help="Discover a raw remote dataset.") -@click.option("--backend", help="Registered data backend name.") -@click.option("--backend-options", type=click.Path(exists=True, dir_okay=False), - help="YAML mapping of backend-specific options.") -@click.option("--auth", help="Optional local authentication profile for data access.") +@click.option("--from", "source", help="Named data source or raw dataset URL.") +@click.option("--release", + help="Release of a named source; prompted for in a terminal.") +@click.option("--collection", "collections", multiple=True, + help="Source collection to catalog. Repeat to include several.") @click.option("--filter", "filters", multiple=True, - help="Include paths matching a glob. May be repeated.") -@click.option("--fmt", help="Select paths and extract filename parameters.") -@click.option("--with-download", is_flag=True, - help="Review and approve a download after catalog creation.") -@click.option("--max-workers", type=click.IntRange(min=1), default=4, - show_default=True) -def init(path, from_url, backend, backend_options, auth, filters, fmt, - with_download, max_workers): - """Initialize a hallmark repository at PATH. - - Use --from URL to discover a remote dataset without downloading payloads. - If PATH ends with `.hm`, a bare repository is created. - Otherwise, a `.hm` directory is created inside PATH. + help="Catalog paths matching a glob. May be repeated.") +@click.option("--format", + help="Filename template overriding automatic detection. " + "Unmatched files remain cataloged.") +def init(path, source, release, collections, filters, format): + """Initialize a local repository or discover metadata without downloading files. + + --from accepts a source name such as desi or a dataset URL. Omit --collection + and --filter to catalog the whole release or URL root. Filename formats are + detected automatically; --format supplies an explicit template instead. + Format matching adds columns without filtering files. Use download separately + to select and approve transfers. Plain init creates an empty repository without + scanning existing local files. + A PATH ending in .hm creates a bare catalog; otherwise metadata lives in PATH/.hm. """ with _translate_cli_errors( GitError, DownloadError, ValueError, OSError, yaml.YAMLError, prefix=f'Failed to initialize hallmark repository at "{path}"'): - options = _backend_options_file(backend_options) - if with_download and not from_url: - raise ValueError("--with-download requires --from") - if with_download and Repo.lwpaths(path)[1] is None: - raise ValueError("Use a worktree destination for --with-download") - kwargs = dict(from_url=from_url, backend=backend, backend_options=options, - auth=auth, filter=filters or None, fmt=fmt, - progress=True, max_workers=max_workers) - if from_url is None and all(value is None for value in - (backend, options, auth, filters or None, fmt)): - repo = Repo.init(path) + if source is not None and "://" not in source and release is None: + descriptor = get_source(source) + if not sys.stdin.isatty(): + raise ValueError("--release is required in noninteractive use; " + f"choose from {', '.join(descriptor.releases)}") + release = click.prompt( + "Release", type=click.Choice(list(descriptor.releases))) + if (source is None and release is None and format is None + and not collections and not filters): + Repo.init(path) else: - repo = Repo.init(path, **kwargs) - if from_url is not None: + Repo.init(path, source=source, release=release, + collections=collections or None, filter=filters or None, + format=format, progress=True) + if source is not None: click.echo(f'Successfully initialized "{path}"') - if with_download: - _run_download(repo, repo.plan_download(), max_workers=max_workers) + click.echo("From this repository, use hallmark download --interactive " + "to choose files, or hallmark download --filter 'PATTERN' " + "--dry-run to preview a selection.") @hallmark.command(short_help="Show information of the current directory.") @@ -318,21 +506,14 @@ def add(repo, encoding, inputs): @click.option("--fmt") @click.option("--remote-name") @click.option("--remote-url") -@click.option("--remote-auth", - help="Local SSH profile name. An empty string removes the reference.") -@click.option("--remote-backend", help="Registered data backend name.") -@click.option("--remote-backend-options", type=click.Path(exists=True, dir_okay=False), - help="YAML mapping of backend-specific options.") @click.option("--encoding", "encodings", multiple=True) @click.pass_obj -def set_config(repo, fmt, remote_name, remote_url, remote_auth, remote_backend, - remote_backend_options, encodings): +def set_config(repo, fmt, remote_name, remote_url, encodings): """Update the current branch config.yml.""" # if no config changes are requested, raise a ClickException to inform the user if ( fmt is None and remote_name is None and remote_url is None - and remote_auth is None and remote_backend is None - and remote_backend_options is None and not encodings): + and not encodings): raise ClickException("No config changes requested.") encoding_updates = {} for item in encodings: @@ -345,16 +526,11 @@ def set_config(repo, fmt, remote_name, remote_url, remote_auth, remote_backend, # use the _translate_cli_errors context manager to handle specific exceptions with _translate_cli_errors(RuntimeError, ValueError, OSError, yaml.YAMLError): - options = _backend_options_file(remote_backend_options) repo.set_config( fmt=fmt, remote_name=remote_name, remote_url=remote_url, - encoding_updates=encoding_updates or None, - **({"remote_auth": remote_auth} if remote_auth is not None else {}), - **({"remote_backend": remote_backend} - if remote_backend is not None else {}), - **({"remote_backend_options": options} if options is not None else {})) + encoding_updates=encoding_updates or None) click.echo("Updated hallmark config.") @@ -434,7 +610,6 @@ def checkout(repo, target_branch): @click.option("--filter", "filters", multiple=True, help="Select paths matching a glob. ** matches recursively. " "May be repeated.") -@click.option("--fmt", help="Select paths matching a filename format.") @click.option("--remote", "remote_name", help="Name of the configured data remote to use.") @click.option("--output", type=click.Path(file_okay=False), @@ -443,20 +618,35 @@ def checkout(repo, target_branch): show_default=True) @click.option("--dry-run", is_flag=True, help="Show the download plan using only local catalog metadata.") +@click.option("--interactive", is_flag=True, + help="Choose cataloged files and review sizes before approval.") @click.option("-y", "--yes", is_flag=True, hidden=True) @click.pass_obj -def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, - output, max_workers, dry_run, yes): +def download(repo, files, tsv_names, download_all, filters, remote_name, + output, max_workers, dry_run, interactive, yes): """ Download selected files from a configured data remote. Preview the selection with --dry-run. Every nonempty download displays its plan and asks for confirmation before transferring files. + Use --interactive to choose patterns or all files in a terminal. """ + if interactive: + if files or tsv_names or download_all or filters: + raise ClickException( + "--interactive cannot be combined with paths, --filter, " + "--tsv, or --all") + if yes: + click.echo("--yes is deprecated; downloads still require confirmation.", + err=True) + with _translate_cli_errors(DownloadError, ValueError, OSError): + _interactive_download(repo, output=output, remote_name=remote_name, + max_workers=max_workers, dry_run=dry_run) + return if download_all and (files or tsv_names): raise ClickException("--all cannot be combined with file paths or --tsv") - if not files and not tsv_names and not download_all and not filters and not fmt: - raise ClickException("Provide file paths, --tsv, --all, --filter, or --fmt") + if not files and not tsv_names and not download_all and not filters: + raise ClickException("Provide file paths, --tsv, --all, or --filter") if repo.worktree is None and output is None: raise ClickException("--output is required when downloading from a bare repo") if yes: @@ -465,13 +655,9 @@ def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, with _translate_cli_errors(DownloadError, ValueError): plan = repo.plan_download( output, file_paths=files, tsv_names=tsv_names, all_files=download_all, - filter=filters or None, fmt=fmt, remote_name=remote_name) + filter=filters or None, remote_name=remote_name) if dry_run: - click.echo(plan.summary()) - for item in plan.items[:20]: - click.echo(f" {item.relative_path.as_posix()}") - if plan.file_count > 20: - click.echo(f" ... {plan.file_count - 20} more file(s)") + _show_download_plan(plan, show_paths=True) return _run_download(repo, plan, max_workers=max_workers) @@ -479,56 +665,43 @@ def download(repo, files, tsv_names, download_all, filters, fmt, remote_name, @hallmark.command(short_help="Clone an existing Hallmark catalog.") @click.argument("url") @click.argument("path") -@click.option("--auth", help="Optional local SSH authentication profile.") -@click.option("--filter", "filters", multiple=True, - help="Select paths matching a glob. ** matches recursively. " - "May be repeated.") -@click.option("--fmt", - help="Select downloads using a filename format; " - "requires --with-download.") @click.option("--source-type", default="auto", show_default=True, type=click.Choice(["auto", "git", "catalog"]), help="Override automatic source detection.") -@click.option("--with-download", "--download", "fetch_data", is_flag=True, - help="Request a download after catalog creation, with confirmation.") -@click.option("--no-fetch-data", is_flag=True, hidden=True) -@click.option("--max-workers", type=click.IntRange(min=1), default=4, - show_default=True) -@click.option("-y", "--yes", is_flag=True, hidden=True) -def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, - max_workers, yes): - """ - Clone an existing Git catalog or published catalog snapshot at PATH. - - Dataset files are not downloaded by default. Use --with-download to review - and approve a transfer. --filter and --fmt select downloads while preserving - the complete catalog and Git history. Use init --from for raw datasets. +@click.option("--no-download", is_flag=True, + help="Copy the complete catalog without reviewing or downloading files.") +@click.option("--filter", "filters", multiple=True, + help="Select downloads by relative-path glob; keep the full catalog. " + "May be repeated.") +@click.option("--interactive", is_flag=True, + help="Choose files interactively instead of reviewing the whole catalog.") +@click.option("--output", type=click.Path(file_okay=False), + help="Download directory. Defaults to the worktree; " + "prompted for bare repos.") +def clone(url, path, source_type, no_download, filters, interactive, output): + """Clone a complete Git catalog or published catalog snapshot at PATH. + + Copy the complete catalog first, then review a download plan and confirm + transfers. --filter selects downloads without trimming the catalog. + --no-download skips review; --interactive opens the selection chooser. + Without a terminal, keep the catalog and print download commands for later. + Use init --from for raw datasets. """ - if fetch_data and no_fetch_data: - raise ClickException("--download conflicts with --no-fetch-data") - if (filters or fmt is not None) and not fetch_data: - raise ClickException("--filter and --fmt require --with-download") - if fetch_data and Repo.lwpaths(path)[1] is None: - raise ClickException("Use a worktree destination for --with-download") - if yes: - click.echo("--yes is deprecated; downloads still require confirmation.", - err=True) + if no_download and (interactive or filters or output is not None): + raise ClickException( + "--no-download cannot be combined with --interactive, " + "--filter, or --output") + if interactive and filters: + raise ClickException("--interactive cannot be combined with --filter") with _translate_cli_errors(DownloadError, GitError, ValueError): - path_matches("validation", filter=filters or None, fmt=fmt) try: - repo = Repo.clone(url, path, auth=auth, - source_type=source_type, progress=True, - max_workers=max_workers) + repo = Repo.clone(url, path, source_type=source_type, progress=True) except CloneError as exc: click.echo(str(exc), err=True) raise SystemExit(1) from exc click.echo(f'Successfully cloned to "{path}"') - if fetch_data: - if repo.worktree is None: - raise ClickException("Use a worktree destination for --download") - plan = repo.plan_download(filter=filters or None, fmt=fmt) - _run_download(repo, plan, max_workers=max_workers) - + if not no_download: + _offer_download(repo, filters=filters, output=output, interactive=interactive) @hallmark.command(short_help="Deprecated: use init --from for remote datasets.") @@ -554,14 +727,13 @@ def clone(url, path, auth, filters, fmt, source_type, fetch_data, no_fetch_data, @click.option("--dataset-url", help="Exact dataset URL to search recursively. Recorded remotes " "do not change this root.") -@click.option("--dataset-auth", help="Local SSH profile for the dataset source.") @click.option("--index-format", type=click.Choice(["cyverse-html"]), hidden=True) @click.option("--allow-remote-commands", is_flag=True, hidden=True, help="Deprecated; discovery uses SFTP.") @click.option("--remote-hash", is_flag=True, hidden=True, help="Unsupported; discovery does not hash dataset files.") def build(directory, dataset_name, remotes, config_file, fmts, overwrite, - dataset_url, dataset_auth, index_format, allow_remote_commands, remote_hash): + dataset_url, index_format, allow_remote_commands, remote_hash): """ Build a catalog at DIRECTORY/DATASET_NAME.hm. @@ -620,7 +792,7 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, fmt_entries.append({"fmt": fmt, "db": db}) source_options = {key: value for key, value in { - "dataset_url": dataset_url, "dataset_auth": dataset_auth, + "dataset_url": dataset_url, "index_format": index_format, "allow_remote_commands": allow_remote_commands, "remote_hash": remote_hash}.items() if value is not None and value is not False} # build the hallmark repository with the specified parameters diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py index 6d83fe5..877b556 100644 --- a/mod/hallmark/discovery.py +++ b/mod/hallmark/discovery.py @@ -7,6 +7,7 @@ from dataclasses import replace from functools import lru_cache from pathlib import PurePosixPath +from string import Formatter import parse from tqdm import tqdm @@ -35,40 +36,46 @@ def match(path_index, pattern_index): return match(0, 0) -@lru_cache(maxsize=128) -def _format_parser(fmt): - """Compile and cache a case-sensitive filename format.""" - return parse.compile(fmt, case_sensitive=True) - - -def path_matches(path: str, filter=None, fmt: str | None = None) -> bool: - """ - Match a relative path against globs and a filename format. +CATALOG_COLUMNS = ("path", "checksum_algorithm", "checksum", "size_bytes", "mtime") - Args: - path (str): Path relative to the dataset root. - filter (str | list[str], optional): Globs to match. At least one - must match; ``**`` spans zero or more directories. - fmt (str, optional): Filename format that the complete path must match. - - Returns: - bool: True when the path satisfies every supplied selector. - Raises: - ValueError: If the filter or format is invalid. - """ - path = str(path) - parser = _format_parser(fmt) if fmt is not None else None - if filter is not None: - if not isinstance(filter, (str, list, tuple)): +@lru_cache(maxsize=128) +def extraction_parser(template): + """Validate a named filename template and return its parser and columns.""" + if not isinstance(template, str) or not template.strip(): + raise ValueError("format must be a nonempty filename template") + fields = [] + for _, name, _, conversion in Formatter().parse(template): + if name is None: + continue + if not name.isidentifier() or conversion is not None: raise ValueError( - "Path filters must be a glob string or a list of glob strings") - patterns = [filter] if isinstance(filter, str) else list(filter) - if not all(isinstance(pattern, str) for pattern in patterns): - raise ValueError("Path filters must be glob strings") - if not any(_glob_matches(path, pattern) for pattern in patterns): - return False - return parser is None or parser.parse(path) is not None + "Extraction fields must be simple names without conversions") + if name in {*CATALOG_COLUMNS, "sha1"}: + raise ValueError(f"Extraction field {name!r} is reserved catalog metadata") + if name not in fields: + fields.append(name) + if not fields: + raise ValueError("Extraction templates require at least one named field") + parser = parse.compile(template, case_sensitive=True) + # Force regular-expression compilation now, before accessing the source. + parser.parse("") + return parser, tuple(fields) + + +def path_matches(path: str, filter=None) -> bool: + """Match a relative path against any supplied case-sensitive inclusion glob. + + ``**`` spans zero or more directories. With no patterns every path matches. + """ + if filter is None: + return True + if not isinstance(filter, (str, list, tuple)): + raise ValueError("filter must be a glob string or a list of glob strings") + patterns = [filter] if isinstance(filter, str) else list(filter) + if not all(isinstance(pattern, str) and pattern for pattern in patterns): + raise ValueError("Inclusion patterns must be nonempty glob strings") + return any(_glob_matches(str(path), pattern) for pattern in patterns) def _manifest_algorithm(path): @@ -126,7 +133,8 @@ def _manifest_checksums(context, entries): previous, checksum_algorithm=algorithm, checksum=digest.lower()) -def discover(context, *, filter=None, fmt=None, progress=False) -> list[RemoteEntry]: +def discover(context, *, filter=None, progress=False, path_prefix="" + ) -> list[RemoteEntry]: """ Discover remote files and their published metadata recursively. @@ -136,7 +144,8 @@ def discover(context, *, filter=None, fmt=None, progress=False) -> list[RemoteEn Args: context (OperationContext): Source connection and cancellation state. filter (str | list[str], optional): Relative path glob or globs. - fmt (str, optional): Filename format that selected paths must match. + path_prefix (str): Release-relative root for collection selection. + Returned entry paths remain relative to the context root. progress (bool | callable): True displays a progress bar with an unknown total. A callback receives dictionaries containing ``directories``, ``files``, ``matched``, and ``current``. @@ -147,12 +156,16 @@ def discover(context, *, filter=None, fmt=None, progress=False) -> list[RemoteEn Unavailable sizes, modification times, and checksums remain None. Raises: - ValueError: If a filter or format is invalid. + ValueError: If an inclusion pattern is invalid. CapabilityError: If the source provides no supported listing. DownloadError: If discovery fails, paths are unsafe, or manifests conflict. """ # Validate selectors before contacting the source, even for an empty index. - path_matches("", filter=filter, fmt=fmt) + path_matches("", filter=filter) + def matches(path): + logical_path = path_prefix + "/" + path if path_prefix else path + return path_matches(logical_path, filter=filter) + entries = {} counts = {"directories": 0, "files": 0, "matched": 0, "current": ""} bar = tqdm(total=None, unit="dir", desc="Discovering", disable=not bool(progress)) \ @@ -174,7 +187,7 @@ def add(entry): if path not in entries: entries[path] = entry counts["files"] += 1 - counts["matched"] += int(path_matches(path, filter=filter, fmt=fmt)) + counts["matched"] += int(matches(path)) report() try: @@ -184,7 +197,7 @@ def add(entry): add(entry) _manifest_checksums(context, entries) return [entries[path] for path in sorted(entries) - if path_matches(path, filter=filter, fmt=fmt)] + if matches(path)] finally: if bar is not None: bar.close() diff --git a/mod/hallmark/download_plan.py b/mod/hallmark/download_plan.py index 47a9e6f..4ed35b5 100644 --- a/mod/hallmark/download_plan.py +++ b/mod/hallmark/download_plan.py @@ -62,7 +62,6 @@ class DownloadPlan: items (tuple[DownloadItem]): Selected files and their metadata. remote_url (str, optional): Data source URL; may be None for an empty plan. output_path (Path): Absolute destination directory. - remote_auth (str, optional): Local SSH profile name. remote_name (str, optional): Name of the selected data remote. estimated_bytes_per_second (float, optional): Positive rate supplied by the caller for duration estimates. @@ -73,7 +72,6 @@ class DownloadPlan: items: tuple[DownloadItem, ...] remote_url: Optional[str] = field(repr=False) output_path: Path - remote_auth: Optional[str] = None remote_name: Optional[str] = None estimated_bytes_per_second: Optional[float] = None remote_backend: Optional[str] = None @@ -87,7 +85,7 @@ def __post_init__(self) -> None: if any(not isinstance(item, DownloadItem) for item in self.items): raise TypeError("items must contain DownloadItem values") if any(value is not None and not isinstance(value, str) for value in - (self.remote_url, self.remote_auth, self.remote_name, + (self.remote_url, self.remote_name, self.remote_backend)): raise TypeError("remote fields must be strings or None") if self.remote_backend is not None: @@ -146,7 +144,7 @@ def summary(self) -> str: parsed = urlsplit(self.remote_url) source = urlunsplit(parsed._replace( netloc=parsed.netloc.rsplit("@", 1)[-1], query="", fragment="")) - backend = f"\nBackend: {self.remote_backend}" if self.remote_backend else "" + backend = f"\nTransport: {self.remote_backend}" if self.remote_backend else "" return (f"{self.file_count} file(s); {size}; estimated duration: {duration}" f"\nSource: {source}{backend}" f"\nDestination: {self.output_path}") diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index 1a604b8..dd458d7 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -630,6 +630,7 @@ def _select_download_items( file_paths: Sequence[str] = (), tsv_names: Sequence[str] = (), all_files: bool = False, + catalog_only: bool = False, ) -> list[DownloadItem]: """ Select files to download from a hallmark repository based on the provided @@ -697,8 +698,9 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], add_file(relative_path, _row_checksum(row), row) # Add explicitly requested file paths to the selected files. - for file_path in file_paths: - add_file(file_path) + if not catalog_only: + for file_path in file_paths: + add_file(file_path) # Organize data configuration entries by their TSV names for easier access. entries_by_tsv: dict[str, list[dict]] = {} for entry in data_config: @@ -789,6 +791,11 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], elif explicit_paths and not entries_by_tsv: add_frame(repo.state.data, data_config, explicit_only=True) + missing = explicit_paths - set(selected) + if missing: + raise DownloadError( + "Paths are not in the catalog: " + ", ".join(sorted(missing))) + return [DownloadItem(path, checksum, *metadata.get(path.as_posix(), (None, None))) for path, checksum in selected.values()] @@ -840,7 +847,6 @@ def plan_download( tsv_names: Optional[Sequence[str]] = None, all_files: bool = False, filter: Optional[Union[str, Sequence[str]]] = None, - fmt: Optional[str] = None, remote_name: Optional[str] = None, estimated_bytes_per_second: Optional[float] = None, ) -> DownloadPlan: @@ -859,7 +865,6 @@ def plan_download( all_files (bool): Select all configured files. Cannot be combined with explicit paths or TSVs. Defaults to False. filter (str | list[str], optional): Relative path glob or globs. - fmt (str, optional): Filename format that selected paths must match. remote_name (str, optional): Configured data remote to use. estimated_bytes_per_second (float, optional): Positive rate used to estimate duration when every file size is known. @@ -871,7 +876,7 @@ def plan_download( Raises: DownloadError: If the catalog, remote, selection, or destination is invalid. - ValueError: If a filter, format, or supplied rate is invalid. + ValueError: If an inclusion pattern or supplied rate is invalid. """ if output_path is None: output_path = getattr(repo, "worktree", None) @@ -889,11 +894,12 @@ def plan_download( raise DownloadError("all_files cannot be combined with paths or TSVs") items = _select_download_items( repo, file_paths=file_paths, tsv_names=tsv_names, - all_files=all_files or (not file_paths and not tsv_names)) - if filter is not None or fmt is not None: + all_files=all_files or (not file_paths and not tsv_names), catalog_only=True) + if filter is not None: from .discovery import path_matches + path_matches("validation", filter=filter) items = [item for item in items if path_matches( - item.relative_path.as_posix(), filter=filter, fmt=fmt)] + item.relative_path.as_posix(), filter=filter)] remote = _select_remote_config(repo, remote_name) if items and (remote is None or not remote.get("url")): raise DownloadError("No remote URL is configured in config.yml") @@ -901,7 +907,7 @@ def plan_download( source = None if remote.get("url"): source = RemoteSpec.parse( - remote["url"], remote.get("auth"), backend=remote.get("backend"), + remote["url"], backend=remote.get("backend"), backend_options=remote.get("backend_options")) for item in items: try: @@ -911,7 +917,7 @@ def plan_download( raise DownloadError(str(exc)) from exc return DownloadPlan( tuple(items), remote.get("url"), output_root, - remote_auth=remote.get("auth"), remote_name=remote.get("name"), + remote_name=remote.get("name"), estimated_bytes_per_second=estimated_bytes_per_second, remote_backend=source.backend if source is not None else None, backend_options=remote.get("backend_options")) @@ -955,7 +961,7 @@ def execute_download_plan( if plan.output_path.resolve() != plan.output_path: raise DownloadError("Download destination changed since planning") remote = RemoteSpec.parse( - plan.remote_url, plan.remote_auth, backend=plan.remote_backend, + plan.remote_url, backend=plan.remote_backend, backend_options=plan.backend_options) return _download_selected( remote, plan.output_path, @@ -1028,7 +1034,7 @@ def download_remote_data( if not remote_url: raise DownloadError("Remote URL not configured in config.yml") remote_spec = RemoteSpec.parse( - remote_url, remote_config.get("auth"), backend=remote_config.get("backend"), + remote_url, backend=remote_config.get("backend"), backend_options=remote_config.get("backend_options")) # If there are still no files selected for download if not selected_files: diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index b48b00a..8532f8e 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -47,6 +47,7 @@ from .repo_config import ( branch_encodings, branch_fmt, + normalize_remotes, row_to_path, set_config, single_data_fmt) @@ -97,6 +98,7 @@ def __init__(self, path: Union[Path, str]) -> None: self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) self.state = self.dothm.load() + normalize_remotes(self.state.config.get("remote")) self.paraframe_cls = ParaFrame self.download_result = None @@ -176,59 +178,43 @@ def _populate_checksums(self, pf: ParaFrame) -> None: pf["sha1"] = [checksums[path] for path in full_paths] @classmethod - def init(cls, path: Union[Path, str], *, from_url=None, backend=None, - backend_options=None, auth=None, filter=None, fmt=None, - download=False, approve=None, progress=False, max_workers=4) -> "Repo": - """ - Initialize a local repository, optionally discovering a remote dataset. + def init(cls, path: Union[Path, str], *, source=None, release=None, + collections=None, filter=None, format=None, progress=False) -> "Repo": + """Initialize a local repository or catalog a named source or raw URL. - Remote discovery reads listings and published checksums without fetching - payloads. Existing destination files are preserved; an existing ``.hm`` - is rejected before contacting the remote dataset. + Discovery reads listings and published checksums without fetching data. + Existing destination files are preserved; an existing catalog is rejected. Args: path (Path | str): Worktree or bare ``.hm`` repository destination. - from_url (str, optional): Raw remote dataset to discover. - backend (str, optional): Registered data backend name. - backend_options (dict, optional): Backend-specific configuration. - auth (str, optional): Local authentication profile for data access. - filter (str | list[str], optional): Relative path glob or globs. - fmt (str, optional): Filename format for selection and parameters. - download (bool): Request approved downloads after discovery. - approve (callable, optional): Callback returning True to approve a plan. - progress (bool | callable): Discovery and download progress display. - max_workers (int): Maximum download workers. Defaults to 4. + source (str, optional): Registered data source name or dataset URL. + release (str, optional): Explicit release for a named source. + collections (str | list[str], optional): Collections within the release. + Omission catalogs the entire release. + filter (str | list[str], optional): Relative path inclusion globs. + format (str, optional): Named filename template for extra columns, + overriding automatic detection from the filtered paths. + Unmatched files remain in the catalog. Detected templates and + their columns are combined in one table; literal paths remain + authoritative. No inferred template is required for cataloging. + progress (bool | callable): Display discovery progress or receive updates. Returns: - Repo: Initialized repository, retained if download approval is declined. + Repo: Initialized repository without downloaded data files. Raises: DestinationExistsError: If remote initialization would replace a catalog. - ValueError: If remote options are supplied without ``from_url``. - DownloadError: If discovery fails or download approval is unavailable. + ValueError: If source options or the extraction template are invalid. + DownloadError: If remote discovery fails. """ from .catalog import initialize_remote - from .downloader import DownloadError, _require_positive_integer - - _require_positive_integer(max_workers, label="max_workers") - if from_url is not None: - if download and not callable(approve): - raise DownloadError( - "Downloading during init requires an approve(plan) callback") - if download and cls.lwpaths(path)[1] is None: - raise DownloadError( - "Initialize a worktree to download; " - "bare catalogs need an output path") - repo = initialize_remote( - cls, path, from_url, backend=backend, backend_options=backend_options, - auth=auth, filter=filter, fmt=fmt, progress=progress) - if download: - repo._download_after_creation( - approve=approve, max_workers=max_workers, progress=progress) - return repo - if any(value is not None for value in - (backend, backend_options, auth, filter, fmt, approve)) or download: - raise ValueError("Remote initialization options require from_url") + + if source is not None: + return initialize_remote( + cls, path, source, release=release, collections=collections, + filter=filter, format=format, progress=progress) + if any(value is not None for value in (release, collections, filter, format)): + raise ValueError("Remote initialization options require source") dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path / "config.yml").write_text(Dothm.config_template(), @@ -240,114 +226,41 @@ def init(cls, path: Union[Path, str], *, from_url=None, backend=None, Worktree.init(worktree_path) return cls(path) - def _download_after_creation(self, *, approve, max_workers, progress, - filter=None, fmt=None): - """Run an approved transfer after catalog creation has completed.""" - from .downloader import DownloadError - - plan = self.plan_download(filter=filter, fmt=fmt) - if plan.file_count and approve(plan) is True: - self.download_result = self.download( - plan, approved=True, max_workers=max_workers, progress=progress) - if self.download_result["failed"]: - details = "\n".join(self.download_result["errors"][:5]) - raise DownloadError( - f"Failed to download {self.download_result['failed']} " - f"file(s):\n{details}") - @classmethod - def clone( - cls, - url: str, - path: Union[Path, str], - *, - auth: Optional[str] = None, - filter=None, - fmt: Optional[str] = None, - source_type: str = "auto", - progress: bool = False, - download: bool = False, - approve=None, - max_workers: int = 4, - fetch_data: Optional[bool] = None, - show_progress: Optional[bool] = None, - ) -> "Repo": - """ - Clone an existing Hallmark Git catalog or published snapshot. + def clone(cls, url: str, path: Union[Path, str], *, + source_type: str = "auto", progress: bool = False) -> "Repo": + """Clone a complete Hallmark Git catalog or published snapshot. - Git sources retain their history. Published catalog snapshots - start new local history. Dataset files are - downloaded only when requested and approved after catalog creation. - Declining approval leaves the catalog available without dataset files. + Git sources retain their history. Published snapshots start new history. + Dataset files are never downloaded; plan and execute transfers separately. Args: url (str): Existing Git catalog or published snapshot location. path (Path | str): New worktree or bare ``.hm`` repository path. - auth (str, optional): Local profile for snapshot metadata access. - Git sources use Git's authentication configuration instead. - filter (str | list[str], optional): Download selection globs. - Requires ``download=True``; the catalog remains complete. - fmt (str, optional): Filename format used to select downloads. - Requires ``download=True``. source_type (str): ``auto``, ``git``, or ``catalog``. - Defaults to automatic source detection. - progress (bool): Show download progress. Defaults - to False. - download (bool): Request downloads after preparing the catalog. - Defaults to False and requires an approval callback when True. - approve (callable, optional): Called with the completed download - plan. Return the Boolean True to approve the transfer. - max_workers (int): Maximum download workers. Defaults to 4. - fetch_data (bool, optional): Compatibility alias for ``download``. - show_progress (bool, optional): Compatibility alias for ``progress``. + progress (bool): Reserved for metadata progress reporting. Returns: - Repo: The cloned repository. ``download_result`` contains results - when an approved, nonempty download was performed. + Repo: Complete cloned catalog without downloaded data files. Raises: DestinationExistsError: If the destination already exists. CloneError: If a requested catalog is missing or invalid. - ValueError: If source options are invalid or conflict. - DownloadError: If metadata access or downloading fails, or a download - is requested without a callback or worktree destination. + ValueError: If source options are invalid. + DownloadError: If metadata access fails. """ from .catalog import clone_catalog - from .discovery import path_matches - from .downloader import DownloadError, _require_positive_integer - - _require_positive_integer(max_workers, label="max_workers") - if fetch_data is not None: - if download and not fetch_data: - raise ValueError("download and fetch_data conflict") - download = fetch_data - if show_progress is not None: - progress = show_progress - if (filter is not None or fmt is not None) and not download: - raise ValueError("clone filter and fmt require download=True; " - "use init(from_url=...) to select a new catalog") - if download and not callable(approve): - raise DownloadError( - "Downloading during clone requires an approve(plan) callback") - if download and cls.lwpaths(path)[1] is None: - raise DownloadError( - "Clone a worktree to download; bare catalogs need an output path") - path_matches("validation", filter=filter, fmt=fmt) - repo = clone_catalog(cls, url, path, auth=auth, source_type=source_type) - if download: - repo._download_after_creation( - approve=approve, max_workers=max_workers, progress=progress, - filter=filter, fmt=fmt) - return repo + + return clone_catalog(cls, url, path, source_type=source_type) def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, - all_files=False, filter=None, fmt=None, remote_name=None, + all_files=False, filter=None, remote_name=None, estimated_bytes_per_second=None): """ Plan a download using the local catalog without contacting a server. With no explicit paths or TSVs, select the complete catalog before - applying any filter or filename format. Planning does not approve + applying inclusion globs. Planning does not approve the transfer. Args: @@ -358,7 +271,6 @@ def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, all_files (bool): Select all configured files. Cannot be combined with explicit paths or TSVs. Defaults to False. filter (str | list[str], optional): Relative path glob or globs. - fmt (str, optional): Filename format that selected paths must match. remote_name (str, optional): Configured data remote to use. estimated_bytes_per_second (float, optional): Positive transfer rate for duration estimates. No rate is measured while planning. @@ -370,13 +282,13 @@ def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, Raises: DownloadError: If the catalog, remote, selection, or destination is invalid. - ValueError: If a filter, format, or supplied rate is invalid. + ValueError: If an inclusion pattern or supplied rate is invalid. """ from .downloader import plan_download return plan_download( self, output_path, file_paths=file_paths, tsv_names=tsv_names, - all_files=all_files, filter=filter, fmt=fmt, remote_name=remote_name, + all_files=all_files, filter=filter, remote_name=remote_name, estimated_bytes_per_second=estimated_bytes_per_second) def download(self, plan, *, approved=False, max_workers=4, progress=False): @@ -462,7 +374,6 @@ def set_config( remote_name: Optional[str] = None, remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, - remote_auth: Optional[str] = None, remote_backend: Optional[str] = None, remote_backend_options: Optional[dict] = None, ) -> dict: @@ -473,8 +384,6 @@ def set_config( fmt (str, optional): Data format specification. remote_name (str, optional): Name of the remote repository. remote_url (str, optional): URL of the remote repository. - remote_auth (str, optional): Local SSH profile name. An empty string - removes the reference; None leaves it unchanged. remote_backend (str, optional): Registered data backend name. remote_backend_options (dict, optional): Backend-specific configuration. encoding_updates (dict[str, str], optional): Updates to encoding rules. @@ -488,7 +397,6 @@ def set_config( remote_name=remote_name, remote_url=remote_url, encoding_updates=encoding_updates, - **({"remote_auth": remote_auth} if remote_auth is not None else {}), **({"remote_backend": remote_backend} if remote_backend is not None else {}), **({"remote_backend_options": remote_backend_options} diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index 527f16d..8ffab21 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -716,7 +716,6 @@ def build_repo( overwrite: bool = False, *, dataset_url: str | None = None, - dataset_auth: str | None = None, index_format: str | None = None, allow_remote_commands: bool = False, remote_hash: bool = False, @@ -726,7 +725,7 @@ def build_repo( """ Build a remote catalog using the deprecated builder interface. - Prefer ``Repo.init(path, from_url=url)`` for remote datasets. + Prefer ``Repo.init(path, source=url)`` for remote datasets. Discovery reads listings and published checksum manifests without downloading dataset files. @@ -745,7 +744,6 @@ def build_repo( Defaults to False; existing formats may be reused without it. dataset_url (str, optional): Exact discovery root. Defaults to the CyVerse curated-data directory for ``dataset_name``. - dataset_auth (str, optional): Local SSH profile for discovery. index_format (str, optional): Obsolete listing option. None, ``auto``, and ``cyverse-html`` are accepted; detection is automatic. allow_remote_commands (bool): Obsolete option, ignored when supplied. @@ -765,7 +763,7 @@ def build_repo( DownloadError: If remote discovery fails. """ warnings.warn( - "build_repo is deprecated; use Repo.init(path, from_url=url)", + "build_repo is deprecated; use Repo.init(path, source=url)", DeprecationWarning, stacklevel=2) if remote_hash: raise CapabilityError( @@ -781,7 +779,7 @@ def build_repo( base_url = (dataset_url if dataset_url is not None else _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/")) source = RemoteSpec.parse( - base_url, dataset_auth, backend=backend, backend_options=backend_options) + base_url, backend=backend, backend_options=backend_options) with OperationContext(source) as context: return _build_repo(repo_path, dataset_name, fmt_entries, config_file, remotes, overwrite, context) @@ -1057,14 +1055,12 @@ def _ensure_remote_files_listed(): if not remotes: # use a default remote named "origin" pointing to the base_url if none provided remotes = [{"name": "origin"}] - if source.remote.auth is not None: - remotes[0]["auth"] = source.remote.auth if source.remote.backend_options: remotes[0]["backend_options"] = thaw_backend_options( source.remote.backend_options) # create the final remotes list by adding the base_url to each remote entry final_remotes = [{"url": base_url, **remote} for remote in remotes] - default_backend = RemoteSpec.parse(base_url, source.remote.auth).backend + default_backend = RemoteSpec.parse(base_url).backend for remote in final_remotes: if remote["url"] == base_url: if source.remote.backend != default_backend: diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index c7ee874..c2f5b94 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -17,27 +17,24 @@ validate_path_component, validate_relative_path) from .transport.base import ( - RemoteSpec, backend_name, profile_name, reject_controls, thaw_backend_options) + RemoteSpec, backend_name, reject_controls, thaw_backend_options) def _update_remote_config( config: dict, remote_name: Optional[str], remote_url: Optional[str], - remote_auth: Optional[str] = None, remote_backend: Optional[str] = None, remote_backend_options=None, ) -> None: """ Used by set_config. - Update the repository data remote and its local profile reference. + Update the repository data remote and its transport configuration. Args: config (dict): The repository configuration dictionary. remote_name (str, optional): New remote repository name. remote_url (str, optional): New remote repository URL. - remote_auth (str, optional): Local SSH profile name. An empty string - removes the reference; None leaves it unchanged. remote_backend (str, optional): Registered backend name. An empty string restores automatic selection; None leaves it unchanged. remote_backend_options (mapping, optional): Replace backend options. @@ -95,10 +92,6 @@ def _update_remote_config( selected["name"] = remote_name if remote_url is not None: selected["url"] = remote_url - if remote_auth == "": - selected.pop("auth", None) - elif remote_auth is not None: - selected["auth"] = profile_name(remote_auth) if remote_backend == "": selected.pop("backend", None) elif remote_backend is not None: @@ -181,12 +174,15 @@ def normalize_remotes(remotes) -> list[dict]: else: raise ValueError(f"remote {index} must be a string or dictionary") - if set(entry) - {"name", "url", "auth", "backend", "backend_options"}: + if "auth" in entry: + raise ValueError( + "Hallmark auth profiles have been removed. Configure the host in " + "~/.ssh/config, use its alias in the remote URL, and remove the " + "obsolete auth field from config.yml.") + if set(entry) - {"name", "url", "backend", "backend_options"}: raise ValueError( - "Remote entries support only name, url, auth, backend and " + "Remote entries support only name, url, backend and " "backend_options fields") - if "auth" in entry: - profile_name(entry["auth"]) if "backend" in entry: entry["backend"] = backend_name(entry["backend"]) if "backend_options" in entry: @@ -196,7 +192,7 @@ def normalize_remotes(remotes) -> list[dict]: reject_controls(entry["url"], "Remote URL") # allow names without URLs while configuring a data remote RemoteSpec.parse( - entry["url"], entry.get("auth"), backend=entry.get("backend"), + entry["url"], backend=entry.get("backend"), backend_options=entry.get("backend_options")) # for each required key ("name" and "url"), validate that it exists for key in ("name", "url"): @@ -363,7 +359,6 @@ def set_config( remote_name: Optional[str] = None, remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, - remote_auth: Optional[str] = None, remote_backend: Optional[str] = None, remote_backend_options=None, ) -> dict: @@ -378,8 +373,6 @@ def set_config( fmt (str, optional): Filename format. remote_name (str, optional): Remote repository name. remote_url (str, optional): Remote repository URL. - remote_auth (str, optional): Local SSH profile name. An empty string - removes the reference; None leaves it unchanged. remote_backend (str, optional): Registered backend name. An empty string restores automatic selection; None leaves it unchanged. remote_backend_options (mapping, optional): Replace backend options. @@ -428,8 +421,6 @@ def set_config( if isinstance(remote_url, str): reject_controls(remote_url, "Remote URL") remote_url = normalize_nonempty_string(remote_url, label="remote_url") - if remote_auth not in (None, ""): - profile_name(remote_auth) # if a new format string or encoding updates are provided if fmt is not None or encoding_updates is not None: @@ -465,11 +456,11 @@ def set_config( # update the first entry in the "data" list of the configuration config["data"][0] = updated_spec - # update the remote when its name, URL, or profile reference changes + # update the remote when its name, URL, or transport configuration changes if any(value is not None for value in ( - remote_name, remote_url, remote_auth, + remote_name, remote_url, remote_backend, remote_backend_options)): - _update_remote_config(config, remote_name, remote_url, remote_auth, + _update_remote_config(config, remote_name, remote_url, remote_backend, remote_backend_options) repo.state.config = config diff --git a/mod/hallmark/sources.py b/mod/hallmark/sources.py new file mode 100644 index 0000000..552ba96 --- /dev/null +++ b/mod/hallmark/sources.py @@ -0,0 +1,188 @@ +"""Named data sources with explicit releases and collection directory roots.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib import metadata +from threading import RLock +from types import MappingProxyType + +from .transport.base import RemoteSpec, backend_name, literal_path + + +@dataclass(frozen=True) +class SourceRelease: + """A release URL and collection names mapped to relative directory roots. + + ``backend`` and ``backend_options`` are optional adapter configuration for + source authors. Collection paths and catalog paths use the release root. + """ + + url: str + collections: Mapping + description: str = "" + backend: str | None = None + backend_options: Mapping = field(default_factory=dict) + collection_descriptions: Mapping = field(default_factory=dict) + + def __post_init__(self): + remote = RemoteSpec.parse(self.url, backend=self.backend, + backend_options=self.backend_options) + collections = {} + for name, paths in self.collections.items(): + backend_name(name) + if isinstance(paths, str): + paths = (paths,) + paths = tuple(literal_path(path).as_posix() for path in paths) + if not paths: + raise ValueError("Collections must have at least one directory root") + collections[name] = paths + object.__setattr__(self, "collections", MappingProxyType(collections)) + object.__setattr__(self, "backend_options", remote.backend_options) + object.__setattr__(self, "collection_descriptions", + MappingProxyType(dict(self.collection_descriptions))) + + def remote(self): + """Return the validated transport specification without connecting.""" + return RemoteSpec.parse(self.url, backend=self.backend, + backend_options=self.backend_options) + + +@dataclass(frozen=True) +class DataSource: + """A named service with a description and explicit release definitions.""" + + name: str + description: str + releases: Mapping + + def __post_init__(self): + backend_name(self.name) + releases = dict(self.releases) + if not releases or any(not isinstance(value, SourceRelease) + for value in releases.values()): + raise ValueError("Sources require named SourceRelease definitions") + for name in releases: + backend_name(name) + object.__setattr__(self, "releases", MappingProxyType(releases)) + + +def _desi_release(release, production): + collections = { + "redshifts": (f"spectro/redux/{production}/zcatalog",), + "spectra": (f"spectro/redux/{production}/healpix", + f"spectro/redux/{production}/tiles"), + "vac": ("vac",), + } + if release == "dr1": + collections["lss"] = ("survey/catalogs/dr1/LSS",) + return SourceRelease( + f"https://data.desi.lbl.gov/public/{release}/", collections, + description="Early Data Release" if release == "edr" else "Data Release 1", + collection_descriptions={ + "redshifts": "Merged redshift catalogs (all published versions)", + "spectra": "HEALPixel and tile product directories", + "vac": "Value-added products", + **({"lss": "Large-scale-structure catalogs"} if release == "dr1" else {}), + }) + + +_registered = {"desi": DataSource( + "desi", "Dark Energy Spectroscopic Instrument public data", { + "edr": _desi_release("edr", "fuji"), + "dr1": _desi_release("dr1", "iron"), + })} +_loaded = {} +_lock = RLock() + + +def _entry_points(): + entries = metadata.entry_points() + if hasattr(entries, "select"): + return list(entries.select(group="hallmark.sources")) + return list(entries.get("hallmark.sources", ())) + + +def register_source(source: DataSource): + """Register a source for this process; installed plugins use hallmark.sources. + + Entry points expose a ``DataSource`` instance. Duplicate names are rejected. + Registration and source descriptions never contact a data service. + """ + if not isinstance(source, DataSource): + raise TypeError("Expected a DataSource") + with _lock: + if (source.name in _registered or source.name in _loaded + or any(item.name == source.name for item in _entry_points())): + raise ValueError(f"Duplicate data source {source.name!r}") + _registered[source.name] = source + + +def get_source(name: str) -> DataSource: + """Resolve a registered source or load its installed descriptor.""" + backend_name(name) + with _lock: + entries = [item for item in _entry_points() if item.name == name] + if len(entries) > 1 or (entries and name in _registered): + raise ValueError(f"Duplicate data source {name!r}") + if name in _registered: + return _registered[name] + if name in _loaded: + return _loaded[name] + if not entries: + raise ValueError(f"Unknown data source {name!r}; use hallmark sources") + try: + source = entries[0].load() + except Exception: + raise ValueError(f"Unable to load data source {name!r}") from None + if not isinstance(source, DataSource) or source.name != name: + raise ValueError( + f"Source plugin {name!r} must expose a matching DataSource") + _loaded[name] = source + return source + + +def list_sources(): + """Return source descriptors in name order without discovering files.""" + names = set(_registered) | {item.name for item in _entry_points()} + return [get_source(name) for name in sorted(names)] + + +def resolve_source(source, release=None, collections=None): + """Return a transport, crawl roots, and provenance for a name or raw URL.""" + if not isinstance(source, str) or not source.strip(): + raise ValueError("source must be a data source name or URL") + source = source.strip() + if "://" in source: + if release is not None or collections is not None: + raise ValueError("release and collections require a named data source") + remote = RemoteSpec.parse(source) + return remote, ("",), {"url": remote.url} + descriptor = get_source(source) + if release is None: + raise ValueError(f"Source {source!r} requires a release; " + f"choose from {', '.join(descriptor.releases)}") + if release not in descriptor.releases: + raise ValueError(f"Unknown release {release!r} for {source}; " + f"choose from {', '.join(descriptor.releases)}") + definition = descriptor.releases[release] + if isinstance(collections, str): + collections = [collections] + if collections is not None and not isinstance(collections, (list, tuple)): + raise ValueError("collections must be a name or a list of names") + selected = [] + for name in collections or (): + if not isinstance(name, str) or name not in definition.collections: + raise ValueError(f"Unknown collection {name!r} for {source}/{release}; " + f"choose from {', '.join(definition.collections)}") + if name not in selected: + selected.append(name) + roots = {path for name in selected for path in definition.collections[name]} + # Parent roots cover nested roots; do not crawl overlapping trees twice. + roots = tuple(path for path in sorted(roots) if not any( + path.startswith(parent + "/") for parent in roots if parent != path)) + return definition.remote(), roots or ("",), { + "name": source, "release": release, "collections": selected, + "url": definition.url, + } diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py index fb12c4c..bdedf48 100644 --- a/mod/hallmark/transport/__init__.py +++ b/mod/hallmark/transport/__init__.py @@ -9,7 +9,7 @@ import requests -from .auth import resolve_settings +from .settings import SSHSettings from .base import DataBackend, RemoteEntry, RemoteSpec, TransferCancelled @@ -21,7 +21,7 @@ class OperationContext: also cancels active work. Args: - remote (RemoteSpec): Data source and optional local profile name. + remote (RemoteSpec): Data source and transport configuration. output_root (Path | str, optional): Root used to validate download paths. """ @@ -31,7 +31,7 @@ def __init__( output_root=None, ): self.remote = remote - self.settings = resolve_settings(remote) + self.settings = SSHSettings() self.output_root = Path(output_root) if output_root is not None else None self.cancelled = Event() self.on_bytes = None diff --git a/mod/hallmark/transport/auth.py b/mod/hallmark/transport/auth.py deleted file mode 100644 index eac3dff..0000000 --- a/mod/hallmark/transport/auth.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Read local SSH settings for profile names stored in repository configuration.""" - -from __future__ import annotations - -import os -from dataclasses import dataclass, fields, replace -from pathlib import Path - -from ..helper_functions import load_yaml_file -from .base import ( - RemoteConfigurationError, - profile_name, - reject_controls, - ssh_host, - ssh_user, -) - - -@dataclass(frozen=True) -class SSHSettings: - """ - Local SSH settings shared by the connections in one operation. - - Timeouts are in seconds. ``transfer_timeout`` limits the total time - for each file, and ``max_sessions`` limits concurrent SFTP sessions. - Unset user, port, and identity values fall back to SSH configuration. - """ - user: str | None = None - port: int | None = None - identity_file: str | None = None - host_key_policy: str = "strict" - connect_timeout: int = 10 - transfer_timeout: int = 3600 - shutdown_timeout: int = 2 - max_sessions: int = 4 - - -def _settings(values): - """Validate local SSH options and normalize identity-file paths.""" - allowed = {item.name for item in fields(SSHSettings)} - if not isinstance(values, dict) or set(values) - allowed: - raise RemoteConfigurationError("Unsupported local SSH settings fields") - values = dict(values) - if "user" in values: - ssh_user(values["user"]) - for key in ( - "port", - "connect_timeout", - "transfer_timeout", - "shutdown_timeout", - "max_sessions", - ): - if key not in values: - continue - value = values[key] - if isinstance(value, bool) or not isinstance(value, int) or value < 1: - raise RemoteConfigurationError(f"SSH {key} must be a positive integer") - if key == "port" and value > 65535: - raise RemoteConfigurationError("SSH port must be at most 65535") - if values.get("host_key_policy", "strict") not in {"strict", "accept-new"}: - raise RemoteConfigurationError("host_key_policy must be strict or accept-new") - if "identity_file" in values: - value = values["identity_file"] - if not isinstance(value, str) or not value: - raise RemoteConfigurationError("identity_file must be a local path") - reject_controls(value, "identity_file") - # OpenSSH expands % tokens and environment substitutions inside paths. - if "%" in value or "$" in value: - raise RemoteConfigurationError("identity_file cannot contain expansions") - values["identity_file"] = str(Path(value).expanduser().absolute()) - return values - - -def resolve_settings(remote): - """ - Resolve local SSH settings before opening a connection. - - URL user and port values override a host-bound profile, which overrides - global timeout and concurrency defaults. Remaining values use SSH config. - - Args: - remote (RemoteSpec): Source URL and optional local profile name. - - Returns: - SSHSettings: Validated settings, or defaults for an HTTP source. - - Raises: - RemoteConfigurationError: If the auth file, settings, or host binding - is invalid, or a referenced profile is missing. - """ - settings = SSHSettings() - if remote.scheme not in {"ssh", "sftp"}: - return settings - configured = os.environ.get("HALLMARK_AUTH_FILE") - path = ( - Path(configured).expanduser() - if configured - else Path(os.environ.get("XDG_CONFIG_HOME", "~/.config")).expanduser() - / "hallmark" - / "auth.yml" - ) - if not path.exists() and not configured and remote.auth is None: - return replace(settings, user=remote.user, port=remote.port) - try: - document = load_yaml_file(path) - except Exception: - raise RemoteConfigurationError( - "Unable to read local Hallmark auth file" - ) from None - if ( - not isinstance(document, dict) - or set(document) - {"version", "profiles", "defaults"} - or type(document.get("version")) is not int - or document["version"] != 1 - ): - raise RemoteConfigurationError("Auth file must use version: 1") - defaults = _settings(document.get("defaults", {})) - # User, port, and identity settings require a profile bound to specific hosts. - if set(defaults) & {"user", "port", "identity_file"}: - raise RemoteConfigurationError("Global SSH defaults cannot select an identity") - settings = replace(settings, **defaults) - profiles = document.get("profiles", {}) - if not isinstance(profiles, dict): - raise RemoteConfigurationError("Auth profiles must be a mapping") - validated = {} - for name, profile in profiles.items(): - profile_name(name) - if not isinstance(profile, dict): - raise RemoteConfigurationError("Each auth profile must be a mapping") - hosts = profile.get("hosts") - if ( - not isinstance(hosts, list) - or not hosts - or any(not isinstance(host, str) for host in hosts) - ): - raise RemoteConfigurationError("Each auth profile requires hosts") - hosts = [ssh_host(host).lower() for host in hosts] - values = _settings({k: v for k, v in profile.items() if k != "hosts"}) - validated[name] = (hosts, values) - if remote.auth is not None: - if remote.auth not in validated: - raise RemoteConfigurationError(f"Unresolved auth profile {remote.auth!r}") - hosts, values = validated[remote.auth] - if remote.host.lower() not in hosts: - raise RemoteConfigurationError( - f"Auth profile {remote.auth!r} is not bound to {remote.host}" - ) - settings = replace(settings, **values) - # URL user and port override profile defaults after checking the host binding. - return replace( - settings, user=remote.user or settings.user, port=remote.port or settings.port - ) diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py index 60c6994..3b9729f 100644 --- a/mod/hallmark/transport/base.py +++ b/mod/hallmark/transport/base.py @@ -107,13 +107,6 @@ def literal_path(value) -> Path: raise RemoteConfigurationError(str(exc)) from None -def profile_name(value: str) -> str: - """Validate and return a local authentication profile name.""" - if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", value): - raise RemoteConfigurationError("Auth profile must match [A-Za-z0-9_-]{1,64}") - return value - - def ssh_host(value: str) -> str: # Aliases are not necessarily DNS names. Limit expansion tokens to characters # that cannot become shell syntax in a user's ProxyCommand/Match configuration. @@ -164,7 +157,7 @@ class RemoteEntry: @dataclass(frozen=True) class RemoteSpec: """ - A parsed data-remote URL and optional local authentication profile. + A parsed data-remote URL and transport configuration. Use ``parse`` to validate a URL. SSH roots are decoded once; catalog paths appended to the root are treated as literal filenames. @@ -176,7 +169,6 @@ class RemoteSpec: root (str): Dataset root, decoded for SSH and SFTP. user (str, optional): Explicit SSH username. port (int, optional): Explicit port number. - auth (str, optional): Local SSH authentication profile name. backend (str): Registered backend name, resolved from the URL by default. backend_options (Mapping): Deeply immutable backend configuration. """ @@ -187,7 +179,6 @@ class RemoteSpec: root: str = field(repr=False) user: str | None = field(default=None, repr=False) port: int | None = None - auth: str | None = None backend: str | None = None backend_options: Mapping = field(default_factory=lambda: MappingProxyType({}), repr=False, hash=False) @@ -204,30 +195,27 @@ def __post_init__(self): freeze_backend_options(self.backend_options)) @classmethod - def parse(cls, url: str, auth: str | None = None, *, + def parse(cls, url: str, *, backend=None, backend_options=None) -> RemoteSpec: """ Parse and validate a data-remote URL. Args: url (str): HTTP(S) URL or SSH/SFTP URL with an absolute dataset root. - auth (str, optional): Local SSH profile name. Unsupported for HTTP(S). backend (str, optional): Registered backend name. Defaults to URL - detection. Parsing does not load plugins or resolve profiles. + detection. Parsing does not load plugins. backend_options (Mapping, optional): Backend-specific configuration. Returns: RemoteSpec: Validated source description. Raises: - RemoteConfigurationError: If the URL or profile reference is invalid. + RemoteConfigurationError: If the URL is invalid. """ if not isinstance(url, str) or not url.strip(): raise RemoteConfigurationError("Remote URL must be a non-empty string") reject_controls(url, "Remote URL") url = url.strip() - if auth is not None: - profile_name(auth) try: parsed = urlsplit(url) host, port = parsed.hostname, parsed.port @@ -273,12 +261,7 @@ def parse(cls, url: str, auth: str | None = None, *, ) else: root, user = parsed.path, None - if auth is not None: - raise RemoteConfigurationError( - "Auth profiles currently support SSH/SFTP only; " - "HTTP retains Requests .netrc authentication" - ) - return cls(url, scheme, host, root, user, port, auth, + return cls(url, scheme, host, root, user, port, backend, backend_options) def pathname(self, relative_path: str) -> str: diff --git a/mod/hallmark/transport/settings.py b/mod/hallmark/transport/settings.py new file mode 100644 index 0000000..237d256 --- /dev/null +++ b/mod/hallmark/transport/settings.py @@ -0,0 +1,17 @@ +"""Internal resource limits for SSH transfers.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SSHSettings: + """ + Local SSH settings shared by the connections in one operation. + + Timeouts are in seconds. ``transfer_timeout`` limits the total time + for each file, and ``max_sessions`` limits concurrent SFTP sessions. + """ + connect_timeout: int = 10 + transfer_timeout: int = 3600 + shutdown_timeout: int = 2 + max_sessions: int = 4 diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index e4f5d7e..6b49ff3 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -76,10 +76,9 @@ def __init__(self, context, *, ssh=None, sftp=None): def _options(self, *, master=False): """Build noninteractive SSH options for the shared connection.""" settings = self.context.settings - policy = "yes" if settings.host_key_policy == "strict" else "accept-new" options = [ "BatchMode=yes", - f"StrictHostKeyChecking={policy}", + "StrictHostKeyChecking=yes", "ForwardAgent=no", "ForwardX11=no", "PermitLocalCommand=no", @@ -98,13 +97,11 @@ def _options(self, *, master=False): if not master: # A failed shared connection must not reauthenticate for each file. options.append("ProxyCommand=false") - if settings.user is not None: - options.append(f"User={settings.user}") - if settings.port is not None: - options.append(f"Port={settings.port}") + if self.context.remote.user is not None: + options.append(f"User={self.context.remote.user}") + if self.context.remote.port is not None: + options.append(f"Port={self.context.remote.port}") args = [part for option in options for part in ("-o", option)] - if settings.identity_file is not None: - args.extend(["-i", settings.identity_file, "-o", "IdentitiesOnly=yes"]) return args def _spawn(self, argv, **kwargs): diff --git a/test/test_backends.py b/test/test_backends.py index 2b0d4e6..ad2603e 100644 --- a/test/test_backends.py +++ b/test/test_backends.py @@ -303,8 +303,11 @@ def test_multi_server_example_uses_shared_discovery_and_verified_downloads(tmp_p with local_server(tmp_path / "north", requests["north"]) as north: with local_server(tmp_path / "south", requests["south"]) as south: options = {"routes": {"north": north, "south": south}} - repo = Repo.init(tmp_path / "repo", from_url="https://logical.test/dataset/", - backend="multi-server", backend_options=options, + from hallmark import DataSource, SourceRelease, register_source + register_source(DataSource("multi-example", "Multiple servers", { + "v1": SourceRelease("https://logical.test/dataset/", {}, + backend="multi-server", backend_options=options)})) + repo = Repo.init(tmp_path / "repo", source="multi-example", release="v1", filter="**/*.fits") assert repo.state.data["path"].tolist() == [ "north/file.fits", "south/file.fits"] diff --git a/test/test_catalog.py b/test/test_catalog.py index 5b67820..4e96c20 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -35,25 +35,28 @@ def test_remote_init_is_recursive_filtered_and_payload_free( 'note.txtsub/') pages[root + "sub/"] = '

Index of sub

data.h5' events = [] - repo = Repo.init(tmp_path / "clone", from_url=root, filter="**/*.h5", + repo = Repo.init(tmp_path / "clone", source=root, filter="**/*.h5", progress=events.append) assert repo.state.data["path"].tolist() == ["root.h5", "sub/data.h5"] assert repo.state.config["remote"] == [ {"name": "origin", "url": root, "backend": "http"}] - assert repo.state.config["data"] == [{"db": "data.tsv"}] + assert repo.state.config["data"] == [{"db": "data.tsv", "extraction": { + "automatic": True, "formats": []}}] assert not (repo.worktree / "root.h5").exists() assert not any(url.endswith((".h5", ".txt")) for url in reads) plan = repo.plan_download() assert plan.file_count == 2 assert plan.unknown_size_count == 2 assert events[-1]["directories"] == 2 + assert events[-1]["matched"] == 2 + assert events[-1]["files"] == 3 def test_remote_init_does_not_require_filename_inference(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/odd/" pages[root] = '

Index of odd

odd name' - repo = Repo.init(tmp_path / "clone", from_url=root) + repo = Repo.init(tmp_path / "clone", source=root) assert repo.state.data["path"].tolist() == ["odd name"] assert repo.plan_download().file_count == 1 @@ -62,19 +65,25 @@ def test_empty_filtered_init_is_valid(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/data/" pages[root] = '

Index of data

notes.txt' - repo = Repo.init(tmp_path / "clone", from_url=root, fmt="run{run:d}.h5") - assert repo.state.data.empty - assert repo.plan_download().file_count == 0 + repo = Repo.init(tmp_path / "clone", source=root, format="run{run:d}.h5") + assert repo.state.data["path"].tolist() == ["notes.txt"] + assert "run" in repo.state.data.columns + assert repo.state.data.iloc[0]["run"] == "" + assert repo.plan_download().file_count == 1 -def test_template_enriches_and_filters_inventory(tmp_path, metadata_server): +def test_template_enriches_without_filtering_inventory(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/data/" pages[root] = ('

Index of data

run2.h5' 'notes.txt') - repo = Repo.init(tmp_path / "clone", from_url=root, fmt="run{run:d}.h5") - assert repo.state.data["path"].tolist() == ["run2.h5"] - assert str(repo.state.data.iloc[0]["run"]) == "2" + repo = Repo.init(tmp_path / "clone", source=root, format="run{run:d}.h5") + assert repo.state.data["path"].tolist() == ["notes.txt", "run2.h5"] + assert repo.state.data.iloc[0]["run"] == "" + assert float(repo.state.data.iloc[1]["run"]) == 2 + assert repo.state.config["data"] == [ + {"db": "data.tsv", "extraction": { + "automatic": False, "formats": ["run{run:d}.h5"]}}] @pytest.mark.parametrize("nested", [False, True]) @@ -88,8 +97,8 @@ def test_published_snapshot_preserves_data_remote(tmp_path, metadata_server, nes pages[prefix + "meta.yml"] = "dataset: example\n" pages[prefix + "data.tsv"] = "path\tsize_bytes\na.h5\t12\nb.txt\t20\n" plans = [] - repo = Repo.clone(root, tmp_path / "clone", filter="**/*.h5", download=True, - approve=lambda plan: plans.append(plan) or False) + repo = Repo.clone(root, tmp_path / "clone") + plans.append(repo.plan_download(filter="**/*.h5")) assert repo.state.data["path"].tolist() == ["a.h5", "b.txt"] assert plans[0].total_bytes == 12 assert repo.plan_download().remote_url == "https://mirror.test/data/" @@ -112,8 +121,8 @@ def fail(*args, **kwargs): monkeypatch.setattr("hallmark.downloader._fetch_file", fail) plans = [] - repo = Repo.clone(str(source.dothm.path), tmp_path / "clone", filter="run1.h5", - download=True, approve=lambda plan: plans.append(plan) or False) + repo = Repo.clone(str(source.dothm.path), tmp_path / "clone") + plans.append(repo.plan_download(filter="run1.h5")) assert repo.dothm.head.commit.hexsha == head assert repo.dothm.index.diff("HEAD") == [] assert len(repo.state.data) == 2 @@ -126,7 +135,7 @@ def fail(*args, **kwargs): def test_clone_requires_callback_before_download_intent(tmp_path, metadata_server): _, reads = metadata_server - with pytest.raises(DownloadError, match="approve"): + with pytest.raises(TypeError, match="download"): Repo.clone("https://example.test/data/", tmp_path / "clone", download=True) assert reads == [] assert not (tmp_path / "clone").exists() @@ -147,7 +156,8 @@ def fail(*args, **kwargs): raise AssertionError("Refused transfer must not start") monkeypatch.setattr(Repo, "download", fail) - repo = Repo.init(tmp_path / "clone", from_url=root, download=True, approve=decline) + repo = Repo.init(tmp_path / "clone", source=root) + decline(repo.plan_download()) assert len(plans) == 1 assert plans[0].file_count == 1 assert repo.download_result is None @@ -161,7 +171,7 @@ def fail(*args, **kwargs): monkeypatch.setattr("hallmark.catalog.discover", fail) with pytest.raises(KeyboardInterrupt): - Repo.init(tmp_path / "clone", from_url="https://example.test/data/") + Repo.init(tmp_path / "clone", source="https://example.test/data/") assert not (tmp_path / "clone").exists() @@ -202,8 +212,8 @@ def test_snapshot_filter_normalizes_legacy_db_names(tmp_path, metadata_server): pages[root + "meta.yml"] = "{}\n" pages[root + "data.tsv"] = "i\n1\n2\n" plans = [] - repo = Repo.clone(root, tmp_path / "clone", filter="item_1.dat", download=True, - approve=lambda plan: plans.append(plan) or False) + repo = Repo.clone(root, tmp_path / "clone") + plans.append(repo.plan_download(filter="item_1.dat")) assert repo.state.data["i"].astype(str).tolist() == ["1", "2"] assert [item.relative_path for item in plans[0].items] == [Path("item_1.dat")] @@ -235,7 +245,7 @@ def fail(*args, **kwargs): monkeypatch.setattr("hallmark.catalog.discover", fail) with pytest.raises(failure): - Repo.init(destination, from_url="https://example.test/data/") + Repo.init(destination, source="https://example.test/data/") assert list(destination.iterdir()) == [destination / "keep.h5"] assert (destination / "keep.h5").read_bytes() == b"existing scientific data" @@ -248,13 +258,13 @@ def test_remote_init_preserves_existing_payload_and_rejects_existing_catalog( destination = tmp_path / "existing" destination.mkdir() (destination / "keep.h5").write_bytes(b"local") - repo = Repo.init(destination, from_url=root) + repo = Repo.init(destination, source=root) assert repo.state.data["path"].tolist() == ["keep.h5"] assert (destination / "keep.h5").read_bytes() == b"local" head = repo.dothm.head.commit.hexsha reads.clear() with pytest.raises(DestinationExistsError): - Repo.init(destination, from_url=root) + Repo.init(destination, source=root) assert reads == [] assert Repo(destination).dothm.head.commit.hexsha == head @@ -266,7 +276,7 @@ def test_remote_init_rejects_bare_symlink_before_access(tmp_path, metadata_serve alias = tmp_path / "alias.hm" alias.symlink_to(target, target_is_directory=True) with pytest.raises(DestinationExistsError): - Repo.init(alias, from_url="https://example.test/data/") + Repo.init(alias, source="https://example.test/data/") assert metadata_server[1] == [] assert list(target.iterdir()) == [target / "keep"] @@ -284,7 +294,7 @@ def fail(*args, **kwargs): monkeypatch.setattr("hallmark.catalog._write_inventory", fail) with pytest.raises(OSError, match="cannot write"): - Repo.init(destination, from_url=root) + Repo.init(destination, source=root) assert list(destination.iterdir()) == [destination / "keep"] @@ -294,7 +304,7 @@ def fail(*args, **kwargs): ]) def test_clone_invalid_selection_fails_before_source_access( tmp_path, metadata_server, kwargs): - with pytest.raises(ValueError): + with pytest.raises((TypeError, ValueError)): Repo.clone("https://example.test/data/", tmp_path / "clone", **kwargs) assert metadata_server[1] == [] assert not (tmp_path / "clone").exists() @@ -382,7 +392,7 @@ def test_snapshot_clone_does_not_require_payload_authentication( tmp_path, metadata_server, monkeypatch): root = "https://catalog.example.test/published/" remote = {"name": "origin", "url": "ssh://private.example.test/science/", - "auth": "missing-local-profile", "backend": "ssh", + "backend": "ssh", "backend_options": {"collection": ["release-1"]}} metadata_server[0].update({ root + "config.yml": yaml.safe_dump({"data": [{"db": "data.tsv"}], @@ -391,14 +401,14 @@ def test_snapshot_clone_does_not_require_payload_authentication( repo = Repo.clone(root, tmp_path / "clone") assert repo.state.config["remote"] == [remote] assert all(url.startswith(root) for url in metadata_server[1]) - assert repo.plan_download().remote_auth == "missing-local-profile" + assert repo.plan_download().remote_backend == "ssh" assert not repo.dothm.remotes def test_init_download_requires_callback_before_source_access( tmp_path, metadata_server): - with pytest.raises(DownloadError, match="approve"): - Repo.init(tmp_path / "new", from_url="https://example.test/data/", + with pytest.raises(TypeError, match="download"): + Repo.init(tmp_path / "new", source="https://example.test/data/", download=True) assert metadata_server[1] == [] @@ -410,7 +420,8 @@ def test_empty_init_download_does_not_request_approval(tmp_path, metadata_server def fail(plan): raise AssertionError("Empty plans require no confirmation") - repo = Repo.init(tmp_path / "new", from_url=root, download=True, approve=fail) + repo = Repo.init(tmp_path / "new", source=root) + assert repo.plan_download().file_count == 0 assert repo.state.data.empty assert repo.download_result is None @@ -434,7 +445,10 @@ def approve(plan): plans.append(plan) return True - repo = Repo.init(destination, from_url=root, download=True, approve=approve) + repo = Repo.init(destination, source=root) + plan = repo.plan_download() + assert approve(plan) + repo.download(plan, approved=True) assert len(plans) == 1 assert (destination / "a.fits").read_bytes() == b"fits" assert repo.download_result["succeeded"] == 1 @@ -478,8 +492,10 @@ def get(url, **kwargs): server.get = get monkeypatch.setattr(requests, "Session", lambda: server) - repo = Repo.clone(catalog_url, tmp_path / "clone", download=True, - approve=lambda plan: plan.remote_url == payload_url) + repo = Repo.clone(catalog_url, tmp_path / "clone") + plan = repo.plan_download() + assert plan.remote_url == payload_url + repo.download(plan, approved=True) assert requests_made == [payload_url + "a.fits"] assert all(url.startswith(catalog_url) for url in metadata_server[1]) assert (repo.worktree / "a.fits").read_bytes() == b"fits" diff --git a/test/test_cli.py b/test/test_cli.py index f0d5a62..f83426f 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -559,7 +559,7 @@ def test_clone_existing_destination_fails_with_plain_git_stderr(): (target / "placeholder.txt").write_text("test\n", encoding="utf-8") result = runner.invoke( hallmark, - ["clone", "--no-fetch-data", str(source / ".hm"), str(target)]) + ["clone", str(source / ".hm"), str(target)]) assert result.exit_code != 0, \ f"Expected non-zero exit code for clone, got {result.exit_code}" @@ -617,7 +617,7 @@ def test_clone_copies_committed_hallmark_state(): GitRepo(str(source / ".hm")).index.commit("commit initial hallmark state") result = runner.invoke( hallmark, - ["clone", "--no-fetch-data", str(source / ".hm"), "target"]) + ["clone", str(source / ".hm"), "target"]) assert result.exit_code == 0, \ f"Expected exit code 0 for clone, got {result.exit_code}" @@ -633,24 +633,16 @@ def test_clone_copies_committed_hallmark_state(): "Expected data.tsv to exist in target after clone" -def test_clone_reports_download_error_cleanly(monkeypatch, tmp_path): - source = _local_cli_catalog(tmp_path / "source") - - def fail_download(self, plan, **kwargs): - raise DownloadError("Remote download failed") - - monkeypatch.setattr(Repo, "download", fail_download) +def test_clone_rejects_removed_download_option(tmp_path): result = CliRunner().invoke(hallmark, [ - "clone", str(source.dothm.path), str(tmp_path / "target"), "--download"], - input="y\n") - assert result.exit_code != 0 - assert "Error: Remote download failed" in result.output - assert "Download these files? [y/N]" in result.output + "clone", "source", str(tmp_path / "target"), "--download"]) + assert result.exit_code == 2 + assert "No such option" in result.output + assert not (tmp_path / "target").exists() def test_clone_cli_skips_download_when_no_remote_files(monkeypatch, tmp_path): - repo = SimpleNamespace(worktree=tmp_path, - plan_download=lambda **kwargs: _download_plan(0, tmp_path)) + repo = Repo.init(tmp_path / "empty") class FakeRepo: lwpaths = Repo.lwpaths @@ -661,39 +653,19 @@ def clone(url, path, **kwargs): monkeypatch.setattr(cli_module, "Repo", FakeRepo) result = CliRunner().invoke(hallmark, [ - "clone", "source", "target", "--download"]) + "clone", "source", "target"]) assert result.exit_code == 0, result.output assert 'Successfully cloned to "target"' in result.output - assert "No files selected for download." in result.output assert "Download these files?" not in result.output -@pytest.mark.parametrize("max_workers", [0, -1]) -def test_clone_rejects_nonpositive_max_workers(max_workers): - """ - Test that the hallmark CLI 'clone' command rejects non-positive values for the - --max-workers option. This test verifies that the clone command fails with an - appropriate error message when a non-positive value is provided. - Args: - max_workers: The non-positive value to test for the --max-workers option. - """ - result = CliRunner().invoke( - hallmark,[ - "clone", - "--max-workers", - str(max_workers), - "source", - "target"]) - - assert result.exit_code != 0, f"Expected non-zero exit code for clone with \ - non-positive max workers, got {result.exit_code}" - assert "Invalid value for '--max-workers'" in result.output, \ - f"Expected error message about non-positive max workers, got: {result.output}" - assert "x>=1" in result.output, \ - f"Expected error message about non-positive max workers, got: {result.output}" - +@pytest.mark.parametrize("max_workers", [0, -1, 4]) +def test_clone_rejects_removed_max_workers(max_workers): + result = CliRunner().invoke(hallmark, [ + "clone", "source", "target", "--max-workers", str(max_workers)]) + assert result.exit_code == 2 + assert "No such option" in result.output -### build tests ### def test_build_cli_parses_fmts_remotes_and_db_suffix(monkeypatch): """ @@ -1154,7 +1126,7 @@ def test_build_cli_reports_missing_config_yml_in_directory(tmp_path): @pytest.mark.parametrize( "arguments, message", [ - (["download"], "Provide file paths, --tsv, --all, --filter, or --fmt"), + (["download"], "Provide file paths, --tsv, --all, or --filter"), (["download", "file.dat", "--all"], "--all cannot be combined"), (["download", "--tsv", "data", "--all"], "--all cannot be combined")]) def test_download_cli_rejects_invalid_selection_combinations( @@ -1242,7 +1214,6 @@ def test_download_cli_reports_empty_selection(monkeypatch): _install_repo(monkeypatch) result = CliRunner().invoke(hallmark, ["download", "--all"]) assert result.exit_code == 0, result.output - assert "No files selected for download." in result.output assert "Download these files?" not in result.output @@ -1264,12 +1235,12 @@ def fake_download(actual_plan, **kwargs): repo.download = fake_download result = CliRunner().invoke(hallmark, [ "download", "nested/file.dat", "--remote", "mirror", "--max-workers", "2", - "--filter", "**/*.dat", "--fmt", "nested/{name}.dat"], input="y\n") + "--filter", "**/*.dat"], input="y\n") assert result.exit_code == 0, result.output assert "Successfully downloaded 1 files (1.0 MB)" in result.output assert captured["plan"] == (None, { "file_paths": ("nested/file.dat",), "tsv_names": (), "all_files": False, - "filter": ("**/*.dat",), "fmt": "nested/{name}.dat", "remote_name": "mirror"}) + "filter": ("**/*.dat",), "remote_name": "mirror"}) assert captured["download"][0] is plan assert captured["download"][1] == { "max_workers": 2, "progress": True, "approved": True} @@ -1333,12 +1304,13 @@ def reject_download(*args, **kwargs): assert "--yes is deprecated" in result.output -@pytest.mark.parametrize("arguments", [[], ["--no-fetch-data"]]) -def test_clone_cli_defaults_to_metadata_only(monkeypatch, tmp_path, arguments): +@pytest.mark.parametrize("arguments", [[], ["--no-download"]]) +def test_clone_cli_skips_download_without_terminal_or_when_disabled( + monkeypatch, tmp_path, arguments): source = _local_cli_catalog(tmp_path / "source") def reject_download(*args, **kwargs): - raise AssertionError("default clone must not plan or download payloads") + raise AssertionError("noninteractive or disabled download must not run") monkeypatch.setattr(Repo, "plan_download", reject_download) monkeypatch.setattr(Repo, "download", reject_download) @@ -1353,34 +1325,27 @@ def reject_download(*args, **kwargs): def test_init_cli_forwards_discovery_options(monkeypatch, tmp_path): captured = {} - options = tmp_path / "backend.yml" - options.write_text("collection: latest\n") - class FakeRepo: - @staticmethod - def init(path, **kwargs): - captured.update(path=path, **kwargs) - return SimpleNamespace(worktree=Path(path)) + def initialize(path, **kwargs): + captured.update(path=path, **kwargs) - monkeypatch.setattr(cli_module, "Repo", FakeRepo) + monkeypatch.setattr(Repo, "init", initialize) result = CliRunner().invoke(hallmark, [ - "init", str(tmp_path / "target"), "--from", "ssh://lab-data/export/", - "--auth", "lab", "--backend", "ssh", "--backend-options", str(options), - "--filter", "**/*.fits", "--filter", "README*", "--fmt", "{name}.fits", - "--max-workers", "2"]) + "init", str(tmp_path / "target"), "--from", "desi", "--release", "dr1", + "--collection", "redshifts", "--filter", "**/*.fits", + "--filter", "README*", "--format", "{name}.fits"]) assert result.exit_code == 0, result.output assert captured == { - "from_url": "ssh://lab-data/export/", "path": str(tmp_path / "target"), - "auth": "lab", "backend": "ssh", "backend_options": {"collection": "latest"}, - "filter": ("**/*.fits", "README*"), "fmt": "{name}.fits", - "progress": True, "max_workers": 2} + "source": "desi", "path": str(tmp_path / "target"), "release": "dr1", + "collections": ("redshifts",), "filter": ("**/*.fits", "README*"), + "format": "{name}.fits", "progress": True} def test_clone_cli_rejects_conflicting_download_aliases(): result = CliRunner().invoke(hallmark, [ - "clone", "source", "target", "--download", "--no-fetch-data"]) + "clone", "source", "target", "--no-fetch-data"]) assert result.exit_code != 0 - assert "--download conflicts with --no-fetch-data" in result.output + assert "No such option" in result.output @pytest.mark.parametrize("during_clone", [False, True]) @@ -1404,8 +1369,11 @@ def capture_get(url, **kwargs): monkeypatch.setattr(requests, "Session", lambda: server) if during_clone: target = tmp_path / "target" - arguments = ["clone", str(source.dothm.path), str(target), - "--filter", "*.fits", "--download"] + cloned = CliRunner().invoke(hallmark, [ + "clone", str(source.dothm.path), str(target)]) + assert cloned.exit_code == 0, cloned.output + monkeypatch.chdir(target) + arguments = ["download", "--filter", "*.fits"] else: target = source.worktree monkeypatch.chdir(target) @@ -1444,7 +1412,7 @@ def fail(*args, **kwargs): @pytest.mark.parametrize("arguments", [ - ["--filter", "*.fits"], ["--fmt", "{name}.fits"], + ["--interactive", "--filter", "*.fits"], ["--fmt", "{name}.fits"], ["--with-download", "--fmt", "{name:invalid}"], ]) def test_clone_cli_rejects_invalid_selection_before_source_access( @@ -1472,18 +1440,13 @@ def initialize(path, **kwargs): assert "private" not in result.output -def test_set_config_cli_persists_backend_options(monkeypatch, tmp_path): +@pytest.mark.parametrize("option", ["--remote-backend", "--remote-backend-options"]) +def test_set_config_cli_rejects_backend_flags(monkeypatch, tmp_path, option): repo = Repo.init(tmp_path / "repo") - options = tmp_path / "backend.yml" - options.write_text("collection: [release-1, release-2]\n") monkeypatch.chdir(repo.worktree) - result = CliRunner().invoke(hallmark, [ - "set-config", "--remote-url", "https://example.test/data/", - "--remote-backend", "http", "--remote-backend-options", str(options)]) - assert result.exit_code == 0, result.output - remote = Repo(repo.worktree).state.config["remote"] - assert remote["backend"] == "http" - assert remote["backend_options"] == {"collection": ["release-1", "release-2"]} + result = CliRunner().invoke(hallmark, ["set-config", option, "unused"]) + assert result.exit_code == 2 + assert "No such option" in result.output @pytest.mark.parametrize("answer", ["y\n", "n\n", ""]) @@ -1503,8 +1466,11 @@ def metadata(context, path): monkeypatch.setattr(requests, "Session", lambda: server) destination = tmp_path / "target" result = CliRunner().invoke(hallmark, [ - "init", str(destination), "--from", "https://example.test/data/", - "--with-download"], input=answer) + "init", str(destination), "--from", "https://example.test/data/"]) + assert result.exit_code == 0, result.output + assert "Download these files?" not in result.output + monkeypatch.chdir(destination) + result = CliRunner().invoke(hallmark, ["download", "--all"], input=answer) assert "Download these files? [y/N]" in result.output assert Repo(destination).state.data["path"].tolist() == ["a.fits"] assert (destination / "a.fits").exists() == (answer == "y\n") diff --git a/test/test_discovery.py b/test/test_discovery.py index ad06cea..e6a8f40 100644 --- a/test/test_discovery.py +++ b/test/test_discovery.py @@ -142,11 +142,9 @@ def test_recursive_globs(path, pattern, matched): assert path_matches(path, filter=pattern) is matched -def test_exact_format_and_filter_are_both_required(): - assert path_matches("sample_001.fits", fmt="sample_{id:03d}.fits") - assert not path_matches("sample_bad.fits", fmt="sample_{id:03d}.fits") - assert not path_matches("Sample_001.fits", fmt="sample_{id:03d}.fits") - assert not path_matches("sample_001.fits", filter="*.txt", fmt="sample_{id}.fits") +def test_globs_are_case_sensitive(): + assert path_matches("sample_001.fits", filter="sample_*.fits") + assert not path_matches("Sample_001.fits", filter="sample_*.fits") @pytest.mark.parametrize("invalid", [12, {"*.fits": True}, iter(["*.fits"])]) diff --git a/test/test_download_plan.py b/test/test_download_plan.py index 7e43b7e..b4b52d5 100644 --- a/test/test_download_plan.py +++ b/test/test_download_plan.py @@ -57,12 +57,12 @@ def reject_network(*args, **kwargs): def test_explicit_paths_retain_catalog_checksums_and_metadata(catalog): - plan = plan_download(catalog, file_paths=["nested/a.fits", "unlisted.bin"]) - assert plan.file_count == 2 + plan = plan_download(catalog, file_paths=["nested/a.fits"]) + assert plan.file_count == 1 assert plan.items[0].checksum == ("sha256", sha256(b"abcdef").hexdigest()) assert plan.items[0].size_bytes == 6 - assert plan.items[1].checksum is None - assert plan.items[1].size_bytes is None + with pytest.raises(DownloadError, match="not in the catalog"): + plan_download(catalog, file_paths=["unlisted.bin"]) assert select_download_files(catalog, file_paths=["nested/a.fits"]) == [ (Path("nested/a.fits"), plan.items[0].checksum)] diff --git a/test/test_hallmark.py b/test/test_hallmark.py index 1e615ff..5629f37 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -1296,8 +1296,10 @@ def fake_download_file(context, relative_path, destination, sha1, chunk_size): monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download_file) - clone = Repo.clone(str(source.dothm.path), tmp_path / "clone", - download=True, approve=lambda plan: plan.file_count == 1) + clone = Repo.clone(str(source.dothm.path), tmp_path / "clone") + plan = clone.plan_download() + assert plan.file_count == 1 + clone.download(plan, approved=True) assert captured == { "url": "https://example.com/data/a0_i0.h5", @@ -1347,7 +1349,7 @@ def test_repo_clone_removes_incomplete_destination(tmp_path): destination = tmp_path / "clone" with pytest.raises(CloneError, match="missing required file"): - Repo.clone(str(source.dothm.path), destination, fetch_data=False) + Repo.clone(str(source.dothm.path), destination) assert not destination.exists(), f"Expected incomplete clone destination \ {destination} to be removed after failed clone attempt" @@ -2770,7 +2772,6 @@ def test_checkout_remote_branch(tmp_path): clone = Repo.clone( str(source.dothm.path), tmp_path / "clone", - fetch_data=False, ) # The cloned repository should have the tracked file in its worktree. diff --git a/test/test_interactive_download.py b/test/test_interactive_download.py new file mode 100644 index 0000000..cb99299 --- /dev/null +++ b/test/test_interactive_download.py @@ -0,0 +1,599 @@ +"""Interactive downloads and default clone review after metadata creation.""" + +from contextlib import contextmanager +import importlib +from pathlib import Path +import shlex +import sys +from unittest.mock import patch +from urllib.parse import quote + +from click import termui +from click.testing import CliRunner +import pandas as pd +import pytest + +from hallmark import Repo +from hallmark.cli import hallmark +from hallmark.transport import OperationContext +from mock_server import MockServer + +cli = importlib.import_module("hallmark.cli") +ROOT = "https://example.test/data/" +README = "README with space,comma.md" + + +class TerminalRunner(CliRunner): + """Model a real terminal, including EOF on Click 8.1 / Python 3.9.""" + + @contextmanager + def isolation(self, *args, **kwargs): + with super().isolation(*args, **kwargs) as streams: + with patch.object(sys.stdin, "isatty", return_value=True), patch.object( + termui, "visible_prompt_func", input): + yield streams + + +@pytest.fixture +def catalog(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "source catalog") + repo.state.config = { + "data": [{"db": "data.tsv", "extraction": { + "automatic": False, "formats": ["runs/run_{run:03d}.h5"]}}], + "remote": [{"name": "origin", "url": ROOT}]} + repo.state.data = pd.DataFrame([ + {"path": "runs/run_001.h5", "size_bytes": 4, "run": "001"}, + {"path": "runs/run_002.h5", "size_bytes": None, "run": "002"}, + {"path": README, "size_bytes": 5, "run": ""}, + ]) + repo.dothm.dump(repo.state) + repo.dothm.index.add(["config.yml", "data.tsv"]) + repo.dothm.index.commit("Catalog fixture") + server = MockServer(ROOT) + for name in ("runs/run_001.h5", "runs/run_002.h5", README, "omitted.bin"): + server.add_file(quote(name, safe="/"), b"notes" if name == README else b"data") + transfers = [] + original = server.get + + def get(url, **kwargs): + transfers.append(url) + return original(url, **kwargs) + + server.get = get + monkeypatch.setattr("requests.Session", lambda: server) + monkeypatch.chdir(repo.worktree) + return repo, transfers + + +def metadata(repo): + return {name: (repo.dothm.path / name).read_bytes() + for name in ("data.tsv", "config.yml", "meta.yml")} + + +@pytest.mark.parametrize("answer", ["\n", "skip\n", "", "patterns\n", "all\n", + "all\n\n", "all\nskip\n"]) +def test_skip_and_eof_never_transfer(catalog, answer): + repo, transfers = catalog + before = metadata(repo) + result = TerminalRunner().invoke(hallmark, ["download", "--interactive"], + input=answer) + assert result.exit_code == 0, result.output + assert "No download started" in result.output + assert transfers == [] + assert metadata(Repo(repo.worktree)) == before + + +def test_patterns_are_raw_ored_and_approval_executes_displayed_plan( + catalog, monkeypatch): + repo, transfers = catalog + before = metadata(repo) + shown, executed = [], [] + display, execute = cli._show_download_plan, Repo.download + + def show(plan, **kwargs): + assert transfers == [] + shown.append(plan) + display(plan, **kwargs) + + def download(self, plan, **kwargs): + assert kwargs == {"approved": True, "max_workers": 2, "progress": True} + assert plan is shown[-1] + executed.append(plan) + return execute(self, plan, **kwargs) + + monkeypatch.setattr(cli, "_show_download_plan", show) + monkeypatch.setattr(Repo, "download", download) + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--max-workers", "2"], + input=f"patterns\nruns/*001.h5\n{README}\n\ndownload\n") + assert result.exit_code == 0, result.output + assert "2 file(s); 9 bytes" in result.output + assert f"{README} (5 bytes)" in result.output + assert "Files with unknown size: 0" in result.output + assert len(transfers) == 2 + assert len(executed) == 1 + assert (repo.worktree / README).read_bytes() == b"notes" + assert not (repo.worktree / "runs/run_002.h5").exists() + assert metadata(Repo(repo.worktree)) == before + + +def test_revision_does_not_approve_previous_selection(catalog): + repo, transfers = catalog + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive"], + input="all\nchange\npatterns\nruns/*001.h5\n\ndownload\n") + assert result.exit_code == 0, result.output + assert "3 file(s)" in result.output and "1 file(s); 4 bytes" in result.output + assert "1 file(s) with unknown size" in result.output + assert "runs/run_002.h5 (size unknown)" in result.output + assert transfers == [ROOT + "runs/run_001.h5"] + assert not (repo.worktree / README).exists() + + +def test_empty_patterns_and_no_matches_return_to_selection(catalog): + _, transfers = catalog + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive"], + input="patterns\n\npatterns\nomitted*\n\nall\nskip\n") + assert result.exit_code == 0, result.output + assert "No cataloged files match" in result.output + assert "3 file(s)" in result.output + assert transfers == [] + + +def test_interactive_dry_run_and_remote_override(catalog, monkeypatch, tmp_path): + repo, transfers = catalog + repo.set_config(remote_name="mirror", remote_url="https://mirror.test/") + destination = tmp_path / "uncreated downloads" + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--dry-run", "--remote", "mirror", + "--output", str(destination)], input="all\n") + assert result.exit_code == 0, result.output + assert "https://mirror.test/" in result.output + assert f"Destination: {destination}" in result.output + assert "Next action" not in result.output + assert not destination.exists() + assert transfers == [] + + +def test_preview_truncates_paths_but_not_totals(catalog): + repo, transfers = catalog + repo.state.data = pd.DataFrame([ + {"path": f"file-{number:03d}.dat", "size_bytes": number} + for number in range(25)]) + repo.dothm.dump(repo.state) + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--dry-run"], input="all\n") + assert result.exit_code == 0, result.output + assert "25 file(s); 300 bytes" in result.output + assert "file-019.dat (19 bytes)" in result.output + assert "file-020.dat" not in result.output + assert "... 5 more file(s)" in result.output + assert transfers == [] + + +@pytest.mark.parametrize("selectors", [["runs/run_001.h5"], ["--all"], + ["--filter", "*.h5"], ["--tsv", "data.tsv"]]) +def test_interactive_rejects_explicit_selectors(catalog, selectors): + _, transfers = catalog + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", *selectors]) + assert result.exit_code != 0 + assert "--interactive cannot be combined" in result.output + assert transfers == [] + + +def test_standalone_interactive_requires_terminal(catalog): + _, transfers = catalog + result = CliRunner().invoke(hallmark, ["download", "--interactive"]) + assert result.exit_code != 0 + assert "requires a terminal" in result.output + assert "--dry-run" in result.output + assert "Select files" not in result.output + assert transfers == [] + + +@pytest.mark.parametrize("kind", ["empty", "no-remote"]) +def test_unavailable_download_skips_without_prompt(catalog, kind): + repo, transfers = catalog + if kind == "empty": + repo.state.data = repo.state.data.iloc[:0] + else: + repo.state.config.pop("remote") + repo.dothm.dump(repo.state) + result = TerminalRunner().invoke(hallmark, ["download", "--interactive"]) + assert result.exit_code == 0, result.output + message = "catalog is empty" if kind == "empty" else "No data remote" + assert message in result.output + assert "Select files" not in result.output + assert transfers == [] + + +def init_pages(monkeypatch): + pages = { + ROOT: '

Index of data

runs/' + 'README.mdomitted.bin', + ROOT + "runs/": '

Index of runs

' + 'run_001.h5' + 'run_002.h5', + } + monkeypatch.setattr(OperationContext, "read_text", + lambda context, path: pages[context.remote.url + path]) + + +def test_bare_catalog_prompts_validates_output_without_creating_it( + catalog, tmp_path, monkeypatch): + source, transfers = catalog + repo = Repo.clone(str(source.dothm.path), tmp_path / "bare.hm") + monkeypatch.chdir(repo.dothm.path) + not_directory = tmp_path / "file" + not_directory.write_text("keep") + destination = tmp_path / "download output" + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--dry-run"], + input=f"all\n{not_directory}\n{destination}\n") + assert result.exit_code == 0, result.output + assert "is a file" in result.output + assert f"Destination: {destination}" in result.output + assert not destination.exists() + assert not_directory.read_text() == "keep" + assert transfers == [] + + +@pytest.mark.parametrize("stage", ["selection", "patterns", "output", "approval"]) +def test_interrupt_is_not_a_successful_skip(catalog, tmp_path, monkeypatch, stage): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if stage == "output" else "copy") + responses = iter({"selection": [], "patterns": ["patterns"], + "output": ["all"], "approval": ["all"]}[stage]) + choose = cli._choose_download_plan + + def interrupted(prompt): + try: + return next(responses) + except StopIteration: + raise KeyboardInterrupt() from None + + def interruptible_choice(repo, **kwargs): + with patch.object(termui, "visible_prompt_func", interrupted): + return choose(repo, **kwargs) + + monkeypatch.setattr(cli, "_choose_download_plan", interruptible_choice) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), + "--interactive"]) + assert result.exit_code != 0 + assert "Aborted" in result.output + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +def test_all_download_requires_approval_even_with_legacy_yes(catalog): + repo, transfers = catalog + result = TerminalRunner().invoke(hallmark, ["download", "--interactive", "--yes"], + input="all\ndownload\n") + assert result.exit_code == 0, result.output + assert "still require confirmation" in result.output + assert "Next action" in result.output + assert len(transfers) == 3 + assert all((repo.worktree / name).exists() + for name in ("runs/run_001.h5", "runs/run_002.h5", README)) + + +def test_transfer_interrupt_is_not_caught_as_prompt_eof(catalog, monkeypatch): + repo, transfers = catalog + before = metadata(repo) + + def interrupt(self, plan, **kwargs): + raise KeyboardInterrupt() + + monkeypatch.setattr(Repo, "download", interrupt) + result = TerminalRunner().invoke(hallmark, ["download", "--interactive"], + input="all\ndownload\n") + assert result.exit_code != 0 + assert "Aborted" in result.output + assert "No download started" not in result.output + assert metadata(Repo(repo.worktree)) == before + assert transfers == [] + + +@pytest.mark.parametrize("bare", [False, True]) +@pytest.mark.parametrize("options", [[], ["--interactive"], + ["--filter", "runs/*001.h5", "--filter", README], + ["--output", "../output 'files'"]]) +def test_noninteractive_clone_keeps_catalog_and_prints_equivalent_commands( + catalog, tmp_path, bare, options): + source, transfers = catalog + destination = tmp_path / ("new 'catalog'.hm" if bare else "new 'catalog'") + cwd = Path.cwd() + result = CliRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), *options], + input="y\n") + assert result.exit_code == 0, result.output + target = Repo(destination) + assert "download was skipped" in result.output + assert "Download these files?" not in result.output + location = target.dothm.path if bare else target.worktree + assert "cd -- " + shlex.quote(str(location)) in result.output + commands = [line.strip() for line in result.output.splitlines() + if line.startswith(" hallmark download")] + assert len(commands) == 2 + preview, execute = map(shlex.split, commands) + assert preview == execute + ["--dry-run"] + if "--filter" in options: + assert execute[2:6] == options + else: + assert execute[2] == "--all" + if "--output" in options: + assert execute[-2:] == ["--output", str((cwd / options[1]).resolve())] + elif bare: + assert execute[-2:] == ["--output", "/path/to/downloads"] + else: + assert "--output" not in execute + assert "explicit approval is required" in result.output + assert Path.cwd() == cwd + assert transfers == [] + assert metadata(target) == metadata(source) + assert target.dothm.head.commit.hexsha == source.dothm.head.commit.hexsha + + +@pytest.mark.parametrize("options,input_text,expected", [ + ([], "y\n", ["runs/run_001.h5", "runs/run_002.h5", README]), + ([], "n\n", []), + ([], "\n", []), + ([], "", []), + (["--filter", "runs/*001.h5", "--filter", README], "y\n", + ["runs/run_001.h5", README]), + (["--filter", "omitted*"], "", []), + (["--interactive"], "patterns\nruns/*001.h5\n\ndownload\n", ["runs/run_001.h5"]), + (["--interactive"], "skip\n", []), +]) +def test_clone_reviews_only_download_selection_after_complete_copy( + catalog, tmp_path, monkeypatch, options, input_text, expected): + source, transfers = catalog + destination = tmp_path / "new catalog" + before = metadata(source) + shown, executed = [], [] + show, download = cli._show_download_plan, Repo.download + + def show_after_creation(plan, **kwargs): + assert transfers == [] + assert metadata(Repo(destination)) == before + assert (Repo(destination).dothm.head.commit.hexsha + == source.dothm.head.commit.hexsha) + assert plan.output_path == destination + shown.append(plan) + show(plan, **kwargs) + + def execute(self, plan, **kwargs): + assert plan is shown[-1] + assert kwargs == {"approved": True, "max_workers": 4, "progress": True} + executed.append(plan) + return download(self, plan, **kwargs) + + monkeypatch.setattr(cli, "_show_download_plan", show_after_creation) + monkeypatch.setattr(Repo, "download", execute) + initial_cwd = Path.cwd() + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), *options], + input=input_text) + assert result.exit_code == 0, result.output + assert result.output.index("Successfully") < result.output.index("Catalog ready") + if "--interactive" not in options: + assert "Select files" not in result.output + assert metadata(Repo(destination)) == before + assert Path.cwd() == initial_cwd + assert set(transfers) == {ROOT + quote(name, safe="/") for name in expected} + assert bool(executed) == bool(expected) + assert Repo(destination).plan_download().file_count == 3 + assert not (destination / "omitted.bin").exists() + if not options: + assert "3 file(s); 9 bytes known; 1 file(s) with unknown size" in result.output + assert "runs/run_002.h5 (size unknown)" in result.output + + +@pytest.mark.parametrize("bare", [False, True]) +def test_no_download_disables_review_and_planning(catalog, tmp_path, monkeypatch, bare): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if bare else "copy") + + def unexpected(*args, **kwargs): + raise AssertionError("No download planning or review was requested") + + monkeypatch.setattr(cli, "_offer_download", unexpected) + monkeypatch.setattr(Repo, "plan_download", unexpected) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), "--no-download"]) + assert result.exit_code == 0, result.output + assert "Download these files?" not in result.output + assert "download was skipped" not in result.output + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +@pytest.mark.parametrize("options", [ + ["--no-download", "--interactive"], ["--no-download", "--filter", "*.fits"], + ["--no-download", "--output", "output"], ["--interactive", "--filter", "*.fits"], + ["--with-download"], +]) +def test_clone_invalid_options_fail_before_source_access( + tmp_path, monkeypatch, options): + def unexpected(*args, **kwargs): + raise AssertionError("Must reject options before cloning") + + monkeypatch.setattr(Repo, "clone", unexpected) + destination = tmp_path / "new" + result = TerminalRunner().invoke(hallmark, ["clone", "source", str(destination), + *options]) + assert result.exit_code != 0 + assert "Error:" in result.output + assert not isinstance(result.exception, AssertionError) + assert not destination.exists() + + +@pytest.mark.parametrize("failure", ["setup", "partial", "interrupt"]) +def test_clone_download_failure_preserves_catalog( + catalog, tmp_path, monkeypatch, failure): + source, _ = catalog + destination = tmp_path / "created" + + def fail(self, plan, **kwargs): + if failure == "setup": + raise cli.DownloadError("fixture access denied") + if failure == "interrupt": + raise KeyboardInterrupt() + return {"succeeded": 1, "failed": 1, "total_bytes": 4, + "errors": ["fixture error"]} + + monkeypatch.setattr(Repo, "download", fail) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination)], input="y\n") + assert result.exit_code != 0 + assert "Successfully cloned" in result.output + assert "Failed to clone" not in result.output + assert "No download started" not in result.output + assert metadata(Repo(destination)) == metadata(source) + assert Repo(destination).dothm.head.commit.hexsha == source.dothm.head.commit.hexsha + + +@pytest.mark.parametrize("provided", [False, True]) +@pytest.mark.parametrize("bare", [False, True]) +def test_clone_output_and_bare_prompt(catalog, tmp_path, bare, provided): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if bare else "copy") + output = tmp_path / "downloads" + arguments = ["clone", str(source.dothm.path), str(destination), + "--filter", "runs/*001.h5"] + if provided: + arguments += ["--output", str(output)] + answer = (str(output) + "\n" if bare and not provided else "") + "y\n" + result = TerminalRunner().invoke(hallmark, arguments, input=answer) + assert result.exit_code == 0, result.output + assert ("Output directory" in result.output) == (bare and not provided) + target = output if bare or provided else destination + assert (target / "runs/run_001.h5").read_bytes() == b"data" + assert transfers == [ROOT + "runs/run_001.h5"] + assert metadata(Repo(destination)) == metadata(source) + + +@pytest.mark.parametrize("interrupt", [False, True]) +@pytest.mark.parametrize("bare", [False, True]) +def test_clone_eof_or_interrupt_before_transfer(catalog, tmp_path, monkeypatch, + bare, interrupt): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if bare else "copy") + if interrupt: + def stop(*args, **kwargs): + raise KeyboardInterrupt() + # Default review uses click.confirm for approval and click.prompt for output. + monkeypatch.setattr(cli, "_confirm_download_plan", stop) + if bare: + monkeypatch.setattr(cli, "_download_output", stop) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination)], input="") + assert (result.exit_code != 0) == interrupt + assert ("No download started" in result.output) != interrupt + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +@pytest.mark.parametrize("kind", ["empty", "no-remote"]) +def test_clone_without_downloadable_data_completes(catalog, tmp_path, kind): + source, transfers = catalog + if kind == "empty": + source.state.data = source.state.data.iloc[:0] + else: + source.state.config.pop("remote") + source.dothm.dump(source.state) + source.dothm.index.add(["config.yml", "data.tsv"]) + source.dothm.index.commit("Unavailable download") + destination = tmp_path / "copy" + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination)]) + assert result.exit_code == 0, result.output + message = "catalog is empty" if kind == "empty" else "No data remote" + assert message in result.output + assert "Download these files?" not in result.output + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +@pytest.mark.parametrize("remote", [False, True]) +def test_init_never_scans_local_files_or_offers_downloads( + catalog, tmp_path, monkeypatch, remote): + _, transfers = catalog + destination = tmp_path / "initialized" + destination.mkdir() + local = destination / "existing.h5" + local.write_bytes(b"untouched") + arguments = ["init", str(destination)] + if remote: + init_pages(monkeypatch) + arguments += ["--from", ROOT, "--filter", "runs/*.h5", + "--format", "runs/run_{run:03d}.h5"] + else: + def no_remote_access(*args, **kwargs): + raise AssertionError("Local init must not contact a remote") + monkeypatch.setattr(OperationContext, "__enter__", no_remote_access) + + def no_download(*args, **kwargs): + raise AssertionError("Init must not offer, plan, or execute downloads") + + monkeypatch.setattr(cli, "_offer_download", no_download) + monkeypatch.setattr(Repo, "plan_download", no_download) + monkeypatch.setattr(Repo, "download", no_download) + result = TerminalRunner().invoke(hallmark, arguments) + assert result.exit_code == 0, result.output + repo = Repo(destination) + assert local.read_bytes() == b"untouched" + assert transfers == [] + assert "Download these files?" not in result.output + assert "Select files" not in result.output + if remote: + assert "hallmark download --interactive" in result.output + assert repo.state.data["path"].tolist() == [ + "runs/run_001.h5", "runs/run_002.h5"] + else: + assert repo.state.data.empty + assert not list((repo.dothm.path / "objects").rglob("*")) + + +def test_failed_clone_never_reviews_download(catalog, monkeypatch): + repo, transfers = catalog + + def unexpected(*args, **kwargs): + raise AssertionError("No handoff before successful creation") + + monkeypatch.setattr(cli, "_offer_download", unexpected) + result = TerminalRunner().invoke(hallmark, ["clone", str(repo.dothm.path), + str(repo.worktree)]) + assert result.exit_code != 0 + assert not isinstance(result.exception, AssertionError) + assert transfers == [] + + +def test_snapshot_clone_reviews_downloads_from_saved_remote( + catalog, tmp_path, monkeypatch): + source, transfers = catalog + snapshot = metadata(source) + snapshot_url = "https://example.test/published/" + reads = [] + + def read(context, path): + assert context.remote.url == snapshot_url + reads.append(path) + return snapshot[path].decode() + + monkeypatch.setattr(OperationContext, "read_text", read) + destination = tmp_path / "snapshot" + result = TerminalRunner().invoke( + hallmark, ["clone", snapshot_url, str(destination), "--source-type", "catalog", + "--filter", "runs/*001.h5"], input="y\n") + assert result.exit_code == 0, result.output + assert set(reads) == {"config.yml", "meta.yml", "data.tsv"} + assert transfers == [ROOT + "runs/run_001.h5"] + target = Repo(destination) + assert metadata(target) == snapshot + assert target.plan_download().file_count == 3 + message = target.dothm.head.commit.message.strip() + assert message == "Import published catalog snapshot" diff --git a/test/test_sources.py b/test/test_sources.py new file mode 100644 index 0000000..ab4106d --- /dev/null +++ b/test/test_sources.py @@ -0,0 +1,410 @@ +"""Named-source and extraction workflows using only in-memory metadata fixtures.""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml +from click.testing import CliRunner + +from hallmark import DataSource, Repo, SourceRelease, register_source +from hallmark import sources +from hallmark.cli import hallmark +from hallmark.sources import get_source, list_sources, resolve_source +from hallmark.transport import OperationContext +from hallmark.transport.base import DownloadError, RemoteObjectMissing + + +def index(*paths): + return '

Index of data

' + ''.join( + f'{path}' for path in paths) + + +@pytest.fixture +def pages(monkeypatch): + content, reads = {}, [] + + def read(context, path): + url = context.remote.url.rstrip("/") + "/" + path + reads.append(url) + if url not in content: + raise RemoteObjectMissing(f"Unexpected request: {url}") + return content[url] + + monkeypatch.setattr(OperationContext, "read_text", read) + return content, reads + + +@pytest.fixture +def registry(monkeypatch): + monkeypatch.setattr(sources, "_registered", dict(sources._registered)) + monkeypatch.setattr(sources, "_loaded", {}) + monkeypatch.setattr(sources, "_entry_points", lambda: []) + + +def test_desi_definitions_are_explicit_and_offline(pages): + descriptor = get_source("desi") + assert list(descriptor.releases) == ["edr", "dr1"] + assert descriptor.releases["edr"].collections["redshifts"] == ( + "spectro/redux/fuji/zcatalog",) + assert "lss" not in descriptor.releases["edr"].collections + assert descriptor.releases["dr1"].collections["lss"] == ( + "survey/catalogs/dr1/LSS",) + assert "desi" in [item.name for item in list_sources()] + result = CliRunner().invoke(hallmark, ["sources", "desi"]) + assert result.exit_code == 0, result.output + assert "spectro/redux/iron/zcatalog/" in result.output + assert "entire release" in result.output + assert pages[1] == [] + + +def test_collection_union_crawls_only_selected_roots(tmp_path, pages): + content, reads = pages + root = "https://data.desi.lbl.gov/public/dr1/" + zcat = "spectro/redux/iron/zcatalog/" + content[root + zcat] = index("zall.fits", "notes.txt") + content[root + "vac/"] = index("extra.fits") + events = [] + repo = Repo.init(tmp_path / "desi", source="desi", release="dr1", + collections=["redshifts", "vac", "redshifts"], + filter="**/*.fits", progress=events.append) + assert any(event["files"] == 2 and event["matched"] == 1 for event in events) + expected = [zcat + "zall.fits", "vac/extra.fits"] + assert repo.state.data["path"].tolist() == expected + assert reads == [root + zcat, root + "vac/"] + stored = yaml.safe_load((repo.dothm.path / "meta.yml").read_text())["source"] + assert stored == {"name": "desi", "release": "dr1", "url": root, + "collections": ["redshifts", "vac"]} + repo = Repo(repo.worktree) + plan = repo.plan_download(filter="**/zall.fits") + assert plan.remote_url == root + assert [item.relative_path.as_posix() for item in plan.items] == expected[:1] + assert reads == [root + zcat, root + "vac/"] + with pytest.raises(DownloadError, match="not in the catalog"): + repo.plan_download(file_paths=["spectro/omitted.fits"]) + + +def test_omitting_collection_includes_entire_release(tmp_path, pages): + root = "https://data.desi.lbl.gov/public/edr/" + pages[0].update({root: index("LICENSE.md", "spectro/", "target/"), + root + "spectro/": index("spectra.fits"), + root + "target/": index("targets.fits")}) + repo = Repo.init(tmp_path / "all", source="desi", release="edr") + assert repo.state.data["path"].tolist() == [ + "LICENSE.md", "spectro/spectra.fits", "target/targets.fits"] + assert not (repo.worktree / "LICENSE.md").exists() + + +@pytest.mark.parametrize("options, message", [ + ({"source": "desi"}, "requires a release"), + ({"source": "desi", "release": "dr2"}, "Unknown release"), + ({"source": "desi", "release": "edr", "collections": ["lss"]}, + "Unknown collection"), + ({"source": "unknown"}, "Unknown data source"), + ({"source": "https://test/data/", "release": "dr1"}, "named data source"), + ({"source": "https://test/data/", "collections": []}, "named data source"), + ({"filter": "*.fits"}, "require source"), +]) +def test_invalid_source_selection_precedes_io(tmp_path, pages, options, message): + destination = tmp_path / "catalog" + with pytest.raises(ValueError, match=message): + Repo.init(destination, **options) + assert pages[1] == [] + assert not destination.exists() + + +def test_cli_release_prompt_and_cancellation(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(Repo, "init", lambda path, **kw: calls.append((path, kw))) + runner = CliRunner() + # Click replaces stdin during invocation, so set isatty on the replacement. + original = get_source + cancel = False + + def terminal_source(name): + import sys + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + if cancel: + # Click 8.1's runner returns empty strings forever at EOF. Model a + # real terminal's EOF instead; CliRunner restores this hook on exit. + from click import termui + + def eof_input(prompt): + raise EOFError() + + termui.visible_prompt_func = eof_input + return original(name) + + monkeypatch.setattr("hallmark.cli.get_source", terminal_source) + destination = str(tmp_path / "repo") + result = runner.invoke(hallmark, ["init", destination, "--from", "desi"], + input="invalid\ndr1\n") + assert result.exit_code == 0, result.output + assert calls[0][1]["release"] == "dr1" + assert calls[0][1]["collections"] is None + calls.clear() + cancel = True + result = runner.invoke(hallmark, ["init", destination, "--from", "desi"], input="") + assert result.exit_code != 0 + assert calls == [] + assert not Path(destination).exists() + + +def test_cli_requires_release_without_terminal(tmp_path, pages): + result = CliRunner().invoke(hallmark, [ + "init", str(tmp_path / "repo"), "--from", "desi"], input="dr1\n") + assert result.exit_code != 0 + assert "--release is required in noninteractive use" in result.output + assert not (tmp_path / "repo").exists() + assert pages[1] == [] + + +@pytest.mark.parametrize("template", [ + "{", "{run:invalid}", "{path}", "{checksum}", "{sha1}", + "{size_bytes}", "{mtime}", "{checksum_algorithm}", "{}", "{0}", + "{a.b}", "{a[0]}", "{run!r}", "constant", "", +]) +def test_invalid_extraction_precedes_io(tmp_path, pages, template): + with pytest.raises(ValueError): + Repo.init(tmp_path / "invalid", source="https://test/data/", format=template) + assert pages[1] == [] + assert not (tmp_path / "invalid").exists() + + +def test_extraction_retains_unmatched_files_through_reload_and_download( + tmp_path, pages, monkeypatch): + from mock_server import MockServer + + root = "https://test/data/" + pages[0][root] = index("run_001.h5", "RUN_002.h5", "README.md", "omit.bin") + repo = Repo.init(tmp_path / "repo", source=root, + filter=["*.h5", "README*"], format="run_{run:03d}.h5") + before = (repo.dothm.path / "data.tsv").read_bytes() + repo = Repo(repo.worktree) + rows = repo.state.data.set_index("path") + assert set(rows.index) == {"run_001.h5", "RUN_002.h5", "README.md"} + assert float(rows.loc["run_001.h5", "run"]) == 1 + assert rows.loc["README.md", "run"] == "" + assert rows.loc["RUN_002.h5", "run"] == "" + server = MockServer(root) + server.add_file("README.md", b"notes") + monkeypatch.setattr("requests.Session", lambda: server) + plan = repo.plan_download(filter="README*") + assert plan.file_count == 1 + result = repo.download(plan, approved=True) + assert result["succeeded"] == 1 + assert (repo.worktree / "README.md").read_bytes() == b"notes" + assert (repo.dothm.path / "data.tsv").read_bytes() == before + # Configuration changes and a metadata clone preserve extraction and literal paths. + repo.set_config(remote_name="archive") + repo.dothm.index.add(["config.yml"]) + repo.dothm.index.commit("Rename data remote") + clone = Repo.clone(str(repo.dothm.path), tmp_path / "copy") + assert clone.state.config["data"] == repo.state.config["data"] + assert clone.plan_download().file_count == 3 + assert not (clone.worktree / "README.md").exists() + + +def test_empty_catalog_retains_extraction_columns(tmp_path, pages): + root = "https://test/empty/" + pages[0][root] = index() + repo = Repo.init(tmp_path / "empty", source=root, format="run_{run:d}.h5") + assert "run" in repo.state.data.columns + assert repo.state.data.empty + assert repo.plan_download().file_count == 0 + + +def test_source_registration_and_overlapping_roots(registry): + release = SourceRelease("https://test/", {"both": ["a", "a/b"], "one": "a"}) + descriptor = DataSource("example", "Example source", {"v1": release}) + register_source(descriptor) + _, roots, provenance = resolve_source("example", "v1", ["both", "one"]) + assert roots == ("a",) + assert provenance["collections"] == ["both", "one"] + with pytest.raises(ValueError, match="Duplicate"): + register_source(descriptor) + with pytest.raises(TypeError): + descriptor.releases["v2"] = release + + +def test_source_plugin_validation(registry, monkeypatch): + descriptor = DataSource("plugin", "A plugin", {"v1": SourceRelease( + "https://test/", {})}) + entry = SimpleNamespace(name="plugin", load=lambda: descriptor) + monkeypatch.setattr(sources, "_entry_points", lambda: [entry]) + assert get_source("plugin") is descriptor + assert get_source("plugin") is descriptor + monkeypatch.setattr(sources, "_entry_points", lambda: [entry, entry]) + with pytest.raises(ValueError, match="Duplicate"): + get_source("plugin") + + +@pytest.mark.parametrize("operation", ["open", "git-clone", "snapshot-clone"]) +def test_old_auth_references_require_explicit_migration(tmp_path, pages, operation): + repo = Repo.init(tmp_path / "legacy") + config = {"data": [{"db": "data.tsv"}], "remote": [{ + "name": "origin", "url": "sftp://lab-data/export/", "auth": "old"}]} + repo.dothm.dump_yml(config, "config") + repo.dothm.index.add(["config.yml"]) + repo.dothm.index.commit("Legacy profile reference") + destination = tmp_path / "copy" + if operation == "snapshot-clone": + root = "https://test/catalog/" + pages[0].update({root + "config.yml": yaml.safe_dump(config), + root + "meta.yml": "{}\n", root + "data.tsv": "path\n"}) + with pytest.raises(ValueError, match="~/.ssh/config"): + if operation == "open": + Repo(repo.worktree) + elif operation == "git-clone": + Repo.clone(str(repo.dothm.path), destination) + else: + Repo.clone(root, destination, source_type="catalog") + assert not destination.exists() + assert yaml.safe_load((repo.dothm.path / "config.yml").read_text()) == config + + +@pytest.mark.parametrize("option", [ + "--auth", "--backend", "--backend-options", "--include", "--extract", "--fmt", + "--max-workers", "--with-download", +]) +def test_init_removed_options_have_no_alias(tmp_path, pages, option): + destination = tmp_path / "new" + result = CliRunner().invoke(hallmark, ["init", str(destination), option]) + assert result.exit_code == 2 + assert "No such option" in result.output + assert not destination.exists() + assert pages[1] == [] + + +@pytest.mark.parametrize("kwargs", [ + {"from_url": "https://test/"}, {"auth": "old"}, {"backend": "http"}, + {"include": "*"}, {"extract": "{name}"}, {"fmt": "{name}"}, {"download": True}, +]) +def test_init_removed_python_arguments_have_no_shim(tmp_path, kwargs): + with pytest.raises(TypeError): + Repo.init(tmp_path / "new", **kwargs) + assert not (tmp_path / "new").exists() + + +@pytest.mark.parametrize("option", ["--release", "--format"]) +def test_empty_remote_option_does_not_silently_initialize_locally(tmp_path, option): + destination = tmp_path / "new" + result = CliRunner().invoke(hallmark, ["init", str(destination), option, ""]) + assert result.exit_code != 0 + assert "require source" in result.output + assert not destination.exists() + + +def test_automatic_formats_preserve_mixed_catalog_and_downloads( + tmp_path, pages, monkeypatch): + from mock_server import MockServer + + root = "https://test/data/" + paths = ["run_001.h5", "run_002.h5", "sgra_20170406_nustar.fits", + "sgra_20170411_chandra.fits", "README.md", "archive.tar.gz"] + pages[0][root] = index(*paths) + repo = Repo.init(tmp_path / "auto", source=root) + assert pages[1] == [root] # Detection uses filenames, never payloads. + stored = repo.state.config["data"][0] + assert "fmt" not in stored + assert stored["extraction"] == { + "automatic": True, + "formats": ["run_{scan}.h5", "sgra_{date}_{p0}.fits"], + } + repo = Repo(repo.worktree) + rows = repo.state.data.set_index("path") + assert set(rows.index) == set(paths) + assert str(rows.loc["run_001.h5", "scan"]).zfill(3) == "001" + assert str(rows.loc["sgra_20170406_nustar.fits", "date"]) == "20170406" + assert rows.loc["sgra_20170406_nustar.fits", "p0"] == "nustar" + for path in ["README.md", "archive.tar.gz"]: + assert all(rows.loc[path, field] == "" for field in ("scan", "date", "p0")) + assert rows.loc["run_001.h5", "date"] == "" + assert rows.loc["sgra_20170406_nustar.fits", "scan"] == "" + + before = (repo.dothm.path / "data.tsv").read_bytes() + server = MockServer(root) + for path in ["run_001.h5", "README.md"]: + server.add_file(path, b"tiny fixture") + monkeypatch.setattr("requests.Session", lambda: server) + plan = repo.plan_download(filter=["run_001.h5", "README*"]) + assert {item.relative_path.as_posix() for item in plan.items} == { + "run_001.h5", "README.md"} + assert repo.download(plan, approved=True)["succeeded"] == 2 + assert (repo.dothm.path / "data.tsv").read_bytes() == before + repo.set_config(remote_name="archive") + repo.dothm.index.add(["config.yml"]) + repo.dothm.index.commit("Rename remote") + clone = Repo.clone(str(repo.dothm.path), tmp_path / "copy") + assert clone.state.config["data"] == [stored] + assert clone.plan_download().file_count == len(paths) + assert (clone.dothm.path / "data.tsv").read_bytes() == before + assert not (clone.worktree / "run_001.h5").exists() + + +def test_explicit_format_bypasses_detection(tmp_path, pages, monkeypatch): + def unexpected_detection(*args, **kwargs): + raise AssertionError("Explicit format must override inference") + + monkeypatch.setattr("hallmark.catalog.detect_fmt", unexpected_detection) + root = "https://test/data/" + pages[0][root] = index("run_001.h5", "run_002.h5", "README.md") + repo = Repo.init(tmp_path / "explicit", source=root, format="run_{run:03d}.h5") + assert "run" in repo.state.data.columns + assert "scan" not in repo.state.data.columns + assert len(repo.state.data) == 3 + assert repo.state.config["data"][0]["extraction"] == { + "automatic": False, "formats": ["run_{run:03d}.h5"]} + + +@pytest.mark.parametrize("patterns,expected", [ + (["run_001.h5", "README*"], ["README.md", "run_001.h5"]), + ("*.absent", []), +]) +def test_filter_precedes_automatic_detection(tmp_path, pages, patterns, expected): + root = "https://test/data/" + pages[0][root] = index("run_001.h5", "run_002.h5", "README.md") + repo = Repo.init(tmp_path / "filtered", source=root, filter=patterns) + assert repo.state.data["path"].tolist() == expected + # A single run has no sibling from which to infer a varying field. + assert repo.state.config["data"][0]["extraction"]["formats"] == [] + assert "scan" not in repo.state.data.columns + assert repo.plan_download().file_count == len(expected) + assert pages[1] == [root] + + +def test_inferred_reserved_fields_do_not_replace_metadata(tmp_path, pages): + root = "https://test/data/" + pages[0][root] = index("{path}_001.h5", "{path}_002.h5", "README.md") + repo = Repo.init(tmp_path / "literal", source=root) + assert set(repo.state.data["path"]) == { + "{path}_001.h5", "{path}_002.h5", "README.md"} + assert repo.state.config["data"][0]["extraction"]["formats"] == [] + assert repo.plan_download().file_count == 3 + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_cli_filter_and_format_workflow(tmp_path, pages, explicit): + root = "https://test/data/" + pages[0][root] = index("run_001.h5", "run_002.h5", "README.md", "omit.bin") + destination = tmp_path / "cli" + arguments = ["init", str(destination), "--from", root, + "--filter", "*.h5", "--filter", "README*"] + if explicit: + arguments += ["--format", "run_{run:03d}.h5"] + result = CliRunner().invoke(hallmark, arguments) + assert result.exit_code == 0, result.output + repo = Repo(destination) + assert set(repo.state.data["path"]) == {"run_001.h5", "run_002.h5", "README.md"} + assert ("run" if explicit else "scan") in repo.state.data.columns + assert pages[1] == [root] + + +@pytest.mark.parametrize("option", ["--include", "--extract", "--format", "--fmt"]) +def test_download_rejects_removed_and_initialization_only_flags( + tmp_path, monkeypatch, option): + repo = Repo.init(tmp_path / "repo") + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["download", option, "value"]) + assert result.exit_code == 2 + assert "No such option" in result.output diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py index b6b59af..e59cbf3 100644 --- a/test/test_ssh_integration.py +++ b/test/test_ssh_integration.py @@ -14,7 +14,6 @@ from click.testing import CliRunner import pytest -import yaml from hallmark import Repo from hallmark.cli import hallmark @@ -203,27 +202,20 @@ def test_host_trust_and_auth_fail_preflight(ssh_server, tmp_path, trust): assert not (tmp_path / "download").exists() -def test_simultaneous_profiles_have_separate_masters(ssh_server, monkeypatch): +def test_simultaneous_ssh_aliases_have_separate_masters(ssh_server): server = ssh_server (server["root"] / "item").write_bytes(b"data") - auth = server["base"] / "auth.yml" - auth.write_text( - yaml.safe_dump( - { - "version": 1, - "profiles": { - name: { - "hosts": ["hm-test"], - "identity_file": str(server["base"] / key), - } - for name, key in [("one", "client"), ("two", "second")] - }, - } - ) - ) - monkeypatch.setenv("HALLMARK_AUTH_FILE", str(auth)) - with OperationContext(RemoteSpec.parse(server["url"], "one")) as one: - with OperationContext(RemoteSpec.parse(server["url"], "two")) as two: + original = server["config"].read_text() + aliases = "".join( + f"Host {alias}\n HostName 127.0.0.1\n" + f" IdentityFile {server['base'] / key}\n" + for alias, key in [("one", "client"), ("two", "second")]) + server["config"].write_text(aliases + original.replace( + f" IdentityFile {server['base'] / 'client'}\n", "")) + first_url = server["url"].replace("hm-test", "one") + second_url = server["url"].replace("hm-test", "two") + with OperationContext(RemoteSpec.parse(first_url)) as one: + with OperationContext(RemoteSpec.parse(second_url)) as two: with ThreadPoolExecutor(2) as pool: list(pool.map(lambda context: context.transport.prepare(), [one, two])) assert one.transport._socket != two.transport._socket @@ -236,7 +228,7 @@ def test_simultaneous_profiles_have_separate_masters(ssh_server, monkeypatch): assert first.poll() is not None -def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): +def test_build_manifest_download_clone_workflow(ssh_server, tmp_path, monkeypatch): root = ssh_server["root"] (root / "nested").mkdir() (root / "nested/item_1.dat").write_bytes(b"science") @@ -290,15 +282,18 @@ def test_build_manifest_download_clone_workflow(ssh_server, tmp_path): single.dothm.index.commit("Remove deliberately invalid test entry") clone = Repo.clone( str(single.dothm.path), tmp_path / "python-clone", - download=True, approve=lambda plan: True, ) + clone.download(clone.plan_download(), approved=True) assert (clone.worktree / "nested/item_1.dat").read_bytes() == b"science" result = CliRunner().invoke( hallmark, - ["clone", str(single.dothm.path), str(tmp_path / "cli-clone"), "--download"], + ["clone", str(single.dothm.path), str(tmp_path / "cli-clone")], input="y\n", ) assert result.exit_code == 0, result.output + monkeypatch.chdir(tmp_path / "cli-clone") + result = CliRunner().invoke(hallmark, ["download", "--all"], input="y\n") + assert result.exit_code == 0, result.output assert (tmp_path / "cli-clone/nested/item_1.dat").read_bytes() == b"science" @@ -410,9 +405,10 @@ def decline(plan): repo = Repo.init( tmp_path / "initialized", - from_url=ssh_server["url"].replace("ssh:", scheme + ":"), - filter="**/*.fits", download=True, approve=decline, + source=ssh_server["url"].replace("ssh:", scheme + ":"), + filter="**/*.fits", ) + decline(repo.plan_download()) assert fetched == [] assert repo.state.data["path"].tolist() == ["nested/science.fits"] assert len(plans) == 1 @@ -440,9 +436,9 @@ def test_disabled_subsystem_fails(ssh_server, tmp_path): def test_local_session_limit(ssh_server, tmp_path, monkeypatch): for i in range(4): (ssh_server["root"] / str(i)).write_bytes(b"data") - auth = tmp_path / "auth.yml" - auth.write_text("version: 1\ndefaults:\n max_sessions: 1\n") - monkeypatch.setenv("HALLMARK_AUTH_FILE", str(auth)) + from hallmark.transport.settings import SSHSettings + monkeypatch.setattr("hallmark.transport.SSHSettings", + lambda: SSHSettings(max_sessions=1)) repo = Repo.init(tmp_path / "repo") repo.set_config(remote_url=ssh_server["url"]) result = download_remote_data( @@ -455,7 +451,7 @@ def test_local_session_limit(ssh_server, tmp_path, monkeypatch): assert result["failed"] == 0 -def test_profile_source_recording_and_explicit_output( +def test_ssh_config_source_recording_and_explicit_output( ssh_server, tmp_path, monkeypatch ): auth = tmp_path / "auth.yml" @@ -466,16 +462,14 @@ def test_profile_source_recording_and_explicit_output( "label", [], dataset_url=ssh_server["url"], - dataset_auth="lab", ) - assert repo.state.config["remote"][0]["auth"] == "lab" + assert "auth" not in repo.state.config["remote"][0] explicit = build_repo( tmp_path / "explicit", "label", [], remotes=[{"name": "mirror", "url": "https://example.test/data"}], dataset_url=ssh_server["url"], - dataset_auth="lab", ) assert explicit.state.config["remote"] == [ {"name": "mirror", "url": "https://example.test/data"} diff --git a/test/test_transport.py b/test/test_transport.py index 798bf5a..5093c15 100644 --- a/test/test_transport.py +++ b/test/test_transport.py @@ -12,16 +12,15 @@ import traceback from urllib.parse import quote +import pandas as pd import pytest import requests -import yaml from hallmark import Repo from hallmark.downloader import _download_file, download_remote_data from hallmark.repo_builder import _manifest_matches, build_repo from hallmark.repo_config import normalize_remotes from hallmark.transport import OperationContext, RemoteSpec -from hallmark.transport.auth import resolve_settings from hallmark.transport.base import ( CapabilityError, DownloadError, @@ -108,83 +107,28 @@ def test_controls_rejected_at_all_path_boundaries(path, tmp_path): _download_file("ssh://campus/" + path, tmp_path / "out") -def write_auth(monkeypatch, tmp_path, profiles, defaults=None): - """Write a local authentication file for a transport test.""" - path = tmp_path / "auth.yml" - path.write_text( - yaml.safe_dump({"version": 1, "profiles": profiles, "defaults": defaults or {}}) - ) - monkeypatch.setenv("HALLMARK_AUTH_FILE", str(path)) - return path - - -def test_profile_precedence_binding_and_isolation(monkeypatch, tmp_path): - write_auth( - monkeypatch, - tmp_path, - { - "one": { - "hosts": ["campus"], - "user": "alice", - "port": 2200, - "identity_file": "~/key-one", - }, - "two": { - "hosts": ["campus"], - "user": "bob", - "port": 2201, - "identity_file": "~/key-two", - }, - }, - {"transfer_timeout": 77}, - ) - first = resolve_settings(RemoteSpec.parse("ssh://carol@campus:2222/data", "one")) - second = resolve_settings(RemoteSpec.parse("sftp://campus/data", "two")) - assert (first.user, first.port, first.transfer_timeout) == ("carol", 2222, 77) - assert (second.user, second.port) == ("bob", 2201) - assert first.identity_file != second.identity_file - with pytest.raises(RemoteConfigurationError, match="not bound"): - resolve_settings(RemoteSpec.parse("ssh://other/data", "one")) - with pytest.raises(RemoteConfigurationError, match="Unresolved"): - resolve_settings(RemoteSpec.parse("ssh://campus/data", "missing")) - - -@pytest.mark.parametrize( - "profile", - [ - {"hosts": ["campus"], "password": "secret"}, - {"hosts": ["campus"], "ssh_options": {"ProxyCommand": "anything"}}, - {"hosts": ["campus"], "port": True}, - {"hosts": ["campus"], "host_key_policy": "no"}, - {"hosts": ["campus"], "identity_file": "%d/key"}, - {"hosts": []}, - {"hosts": ["$(id)"]}, - ], -) -def test_invalid_profiles_fail_preflight(monkeypatch, tmp_path, profile): - write_auth(monkeypatch, tmp_path, {"campus": profile}) - with pytest.raises(RemoteConfigurationError): - OperationContext(RemoteSpec.parse("ssh://campus/data", "campus")) - - -def test_profile_xdg_and_removal(monkeypatch, tmp_path): +def test_obsolete_auth_files_and_environment_are_ignored(monkeypatch, tmp_path): folder = tmp_path / "config" / "hallmark" folder.mkdir(parents=True) - (folder / "auth.yml").write_text( - "version: 1\nprofiles:\n lab:\n hosts: [campus]\n user: alice\n" - ) - assert ( - resolve_settings(RemoteSpec.parse("ssh://campus/data", "lab")).user == "alice" - ) - repo = Repo.init(tmp_path / "repo") - repo.set_config(remote_url="ssh://campus/data", remote_auth="lab") - assert repo.state.config["remote"]["auth"] == "lab" - repo.set_config(remote_auth="") - assert "auth" not in Repo(repo.worktree).state.config["remote"] - before = repo.state.config.copy() - with pytest.raises(ValueError): - repo.set_config(remote_auth="invalid.name", fmt="changed-{x}") - assert repo.state.config == before + obsolete = folder / "auth.yml" + obsolete.write_text("invalid: [yaml") + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(obsolete)) + with OperationContext(RemoteSpec.parse("ssh://carol@campus:2222/data")) as ctx: + options = ctx.transport._options() + assert "User=carol" in options + assert "Port=2222" in options + assert "-i" not in options + with OperationContext(RemoteSpec.parse("sftp://campus/data")) as ctx: + options = ctx.transport._options() + assert not any(option.startswith(("User=", "Port=")) for option in options) + assert "-i" not in options + assert obsolete.read_text() == "invalid: [yaml" + + +@pytest.mark.parametrize("auth", ["lab", None, 3]) +def test_stored_auth_profiles_report_migration(auth): + with pytest.raises(ValueError, match="~/.ssh/config"): + normalize_remotes({"url": "ssh://campus/data", "auth": auth}) @pytest.mark.parametrize( @@ -578,7 +522,9 @@ def test_cli_dry_run_does_not_resolve_auth_or_start_clients(monkeypatch, tmp_pat from hallmark.cli import hallmark repo = Repo.init(tmp_path / "repo") - repo.set_config(remote_url="ssh://campus/data", remote_auth="missing") + repo.set_config(remote_url="ssh://campus/data") + repo.state.data = pd.DataFrame({"path": ["item.dat"]}) + repo.dothm.dump(repo.state) monkeypatch.chdir(repo.worktree) def fail(*args, **kwargs): @@ -590,22 +536,15 @@ def fail(*args, **kwargs): assert "item.dat" in result.output -def test_cli_profile_roundtrip(monkeypatch, tmp_path): +def test_cli_rejects_removed_auth_flag(monkeypatch, tmp_path): from click.testing import CliRunner from hallmark.cli import hallmark repo = Repo.init(tmp_path / "repo") monkeypatch.chdir(repo.worktree) - runner = CliRunner() - result = runner.invoke( - hallmark, - ["set-config", "--remote-url", "ssh://campus/data", "--remote-auth", "lab"], - ) - assert result.exit_code == 0, result.output - assert Repo(repo.worktree).state.config["remote"]["auth"] == "lab" - result = runner.invoke(hallmark, ["set-config", "--remote-auth", ""]) - assert result.exit_code == 0, result.output - assert "auth" not in Repo(repo.worktree).state.config["remote"] + result = CliRunner().invoke(hallmark, ["set-config", "--remote-auth", "lab"]) + assert result.exit_code == 2 + assert "No such option" in result.output def test_http_manifest_auth_failure_is_not_optional(monkeypatch): @@ -722,14 +661,11 @@ def test_build_cli_passes_source_controls(monkeypatch, tmp_path): "run_{i}.h5=data.tsv", "--dataset-url", "ssh://campus/data", - "--dataset-auth", - "lab", "--allow-remote-commands", "--remote-hash", ], ) assert result.exit_code == 0, result.output assert calls[0]["dataset_url"] == "ssh://campus/data" - assert calls[0]["dataset_auth"] == "lab" assert calls[0]["allow_remote_commands"] is True assert calls[0]["remote_hash"] is True From 17a3cb59da1cbaa086e6761e53a8608e223f992b Mon Sep 17 00:00:00 2001 From: Dennis Date: Tue, 22 Sep 2026 22:03:53 -0700 Subject: [PATCH 17/29] Document source selection and approved download workflows Align the README and guides with named sources, automatic formats, OpenSSH configuration, metadata-only initialization, and default clone download review. Explain Python approval, adapter registration, and migration from removed options. --- README.rst | 21 +- doc/api.rst | 55 ++-- doc/backends.rst | 111 ++++---- doc/private_data.rst | 617 +++++++++++++++++-------------------------- doc/usecase.rst | 81 +----- 5 files changed, 369 insertions(+), 516 deletions(-) diff --git a/README.rst b/README.rst index 5671c9e..377134e 100644 --- a/README.rst +++ b/README.rst @@ -95,15 +95,28 @@ is required. Omit the filter to catalog everything beneath the supplied URL. Every dataset transfer requires approval. Python callers can inspect ``repo.plan_download()`` and then execute ``repo.download(plan, approved=True)``. -Optional authentication profiles remain local and can be selected with -``init --auth PROFILE`` or ``set-config --remote-auth PROFILE``. Add -``--with-download`` to ``init`` to review and approve a transfer after discovery. +CLI users can run ``hallmark download --interactive`` to choose path globs or +all cataloged files, review recorded sizes, then download, change the selection, +or skip. Initialization never offers downloads. CLI cloning copies the complete +catalog, then displays a download plan and requests confirmation by default. +Use ``clone --interactive`` for the chooser, or ``clone --no-download`` to copy +only the catalog. Without a terminal, clone keeps the catalog, skips downloading, +and prints commands to use later. +SSH and SFTP use standard ``~/.ssh/config`` aliases and the SSH agent. +Use ``hallmark sources desi`` to inspect named sources, releases and collections. +For example, ``hallmark init desi --from desi --release dr1 --collection redshifts`` +creates a catalog for that collection. Omit the collection to index the whole +release. ``init --filter`` selects catalog entries. Filename formats are detected +automatically from those paths; ``--format`` supplies an explicit template instead. +Both retain unmatched files. Download selection belongs to the +separate ``download --filter`` command. The older remote ``build`` command is deprecated in favor of ``init --from``. Use ``hallmark clone CATALOG PATH`` for an existing Git-hosted ``.hm`` or a published HTTP/SFTP catalog snapshot. Catalogs can live on GitHub or another server while their data remotes point elsewhere. Git clones preserve the full -catalog and its history; clone filters select optional downloads only. +catalog and its history. ``clone --filter`` narrows the planned downloads without +removing catalog entries. No payloads transfer until the user approves the plan. The public ``DataBackend`` interface supports generic HTTP/SSH, CyVerse, and registered plugins for collaboration-specific APIs and multiple data servers. diff --git a/doc/api.rst b/doc/api.rst index 38b34ea..dabed36 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -100,7 +100,13 @@ Initialize a catalog from a remote dataset without downloading dataset files:: hallmark init ./desi --from \ https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/ \ --filter 'redrock-main-dark-23040.fits' - hallmark init ./lab --from ssh://lab-data/srv/data/ --fmt 'run{run:d}.h5' + hallmark init ./lab --from ssh://lab-data/srv/data/ --filter 'run*.h5' --format 'run{run:d}.h5' + +``--filter`` selects catalog entries. Without ``--format``, filename templates +are detected automatically from the selected paths. An explicit ``--format`` +overrides detection. Both retain unmatched files, with empty parameter values; +if no template can be inferred, paths and available metadata are still cataloged. +Python uses the same ``filter=`` and ``format=`` arguments to ``Repo.init``. Use ``hallmark clone SOURCE PATH`` for an existing Hallmark Git repository or published HTTP/SFTP snapshot. Git clones retain the full catalog and its @@ -116,19 +122,28 @@ Preview the selected files using the local catalog, then confirm a download:: hallmark download --all --dry-run hallmark download --filter 'runs/**' -Explicit paths and ``--tsv data.tsv`` also select files. Every nonempty transfer -requires interactive confirmation, including transfers requested through -``init --with-download`` or ``clone --with-download``. Clone accepts its older -``--download`` alias. Clone filters and formats require ``--with-download`` -and leave the complete catalog unchanged. The old ``--yes`` option no longer -bypasses approval. -A filter or filename format never authorizes a transfer. +Explicit cataloged paths and ``--tsv data.tsv`` also select files. Every nonempty +transfer requires interactive confirmation. Initialization only creates the +repository or discovers its catalog; it never offers downloads. CLI cloning +copies the complete catalog, then reviews a download plan by default. Use +``clone --filter`` to narrow that plan, ``clone --interactive`` for a chooser, +or ``clone --no-download`` to skip review. ``clone --output`` specifies the download +directory; bare catalogs prompt for one if omitted. These download options cannot +be combined with ``--no-download``; ``--interactive`` also conflicts with ``--filter``. +``download --filter`` narrows the saved catalog without changing it. +The old download ``--yes`` option does not bypass approval. + +Use ``hallmark download --interactive`` to choose patterns or all cataloged files, +review recorded sizes, then download, revise, or skip. Add ``--dry-run`` to stop at +the preview. Without a terminal, clone keeps the catalog and prints commands for +later; standalone ``download --interactive`` reports an error. Python +initialization and cloning remain metadata-only and never open this CLI chooser. Python uses the same plan. First inspect the selected files:: from hallmark import Repo - repo = Repo.init('lab', from_url='ssh://lab-data/srv/data/', progress=True) + repo = Repo.init('lab', source='ssh://lab-data/srv/data/', progress=True) plan = repo.plan_download(filter='runs/**') print(plan.summary()) @@ -164,17 +179,8 @@ Dataset builders and data remotes .. autofunction:: hallmark.repo_builder.list_remote_files -Associate a data remote with a local SSH authentication profile:: - - repo.set_config(remote_name="campus", remote_auth="campus") - -Remove the profile reference:: - - repo.set_config(remote_name="campus", remote_auth="") - -The repository stores the profile name. The SSH settings remain in the -local authentication file. Passing ``remote_auth=None`` leaves the -reference unchanged. +SSH/SFTP uses standard ``~/.ssh/config`` and the SSH agent. Set a remote URL +containing the configured host alias; no Hallmark authentication profile is used. The legacy ``download_remote_data`` function also requires ``approved=True``. Both download APIs return a dictionary containing ``succeeded``, @@ -187,3 +193,12 @@ and checksum verification. See :doc:`private_data` for CLI and Python examples, and :ref:`private-transport-reference` for authentication settings and supported server configurations. + +Named data sources +------------------ + +List sources with ``hallmark sources`` and release/collection roots with +``hallmark sources desi``. See :doc:`private_data` for selection and migration. + +.. automodule:: hallmark.sources + :members: DataSource, SourceRelease, register_source, get_source, list_sources diff --git a/doc/backends.rst b/doc/backends.rst index 842d0a7..f937abe 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -18,52 +18,53 @@ implementations: * ``CyVerseBackend`` extends HTTP support with CyVerse directory and metadata parsing. -Without an explicit backend, Hallmark selects one from the data URL. Explicit -selection takes precedence. For example: +Users select data sources with ``--from NAME`` or provide a raw dataset URL. +For raw URLs, Hallmark selects the transport automatically. Source authors can +specify an adapter and its nonsecret configuration in a release definition; +users do not need backend flags or an options file. -.. code-block:: bash +Define a source and register it for the current Python process: - hallmark init ./observations --from 'https://data.example.org/export/' \ - --backend http --filter '**/*.fits' +.. code-block:: python + + from hallmark import DataSource, SourceRelease, Repo, register_source -For an installed plugin, pass its registered name and an optional YAML mapping: + survey = DataSource("survey", "Example survey", { + "dr1": SourceRelease( + "https://survey.example.org/release/", + {"redshifts": ("catalogs/redshifts",)}, + ), + }) + register_source(survey) + repo = Repo.init("survey", source="survey", release="dr1", + collections=["redshifts"], filter="**/*.fits") -.. code-block:: bash +Collection roots are relative directories, and catalog paths remain relative +to the release URL. An omitted collection catalogs the entire release. To +expose this descriptor to other Python processes and the CLI, publish the +``survey`` instance in the plugin's ``pyproject.toml``: - hallmark init ./survey --from 'https://survey.example.org/release/' \ - --backend survey --backend-options ./survey-options.yml +.. code-block:: toml -``Repo.init(..., from_url=url, backend="survey", backend_options=options)`` -accepts the same settings in Python. The data remote stores ``backend`` and -``backend_options`` alongside its ``name``, ``url`` and optional ``auth``: + [project.entry-points."hallmark.sources"] + survey = "my_survey.sources:survey" -.. code-block:: yaml +Users can then run ``hallmark sources survey`` to see available releases and +roots, or ``hallmark init ./survey --from survey --release dr1``. Source +registration and listing must not contact the service. Source names are unique +across registered and installed descriptors. - remote: - - name: origin - url: https://survey.example.org/release/ - backend: survey - backend_options: - release: dr1 - -Options contain ordinary configuration that can be shared with the catalog. -Keep tokens, passwords and private keys outside them. The built-in SSH backend -uses local auth profiles; HTTP retains Requests' authentication and environment -behavior. A plugin documents how it obtains any additional credentials locally. -The example ``release`` option above is interpreted by the plugin, not Hallmark. - -Update an existing remote in Python with -``repo.set_config(remote_backend="survey", remote_backend_options=options)``. -Passing ``remote_backend=""`` returns to URL-based selection; passing -``remote_backend_options={}`` clears the options. ``None`` leaves the respective -setting unchanged. Configurations without backend fields remain valid. +The saved data remote retains its resolved URL and transport ``backend`` and +``backend_options``. Existing catalog transport metadata remains supported. +The developer API ``repo.set_config(remote_backend=..., +remote_backend_options=...)`` can update this configuration; it has no CLI +counterpart. Options contain shareable configuration, never credentials. +SSH uses ``~/.ssh/config`` and its agent; HTTP retains Requests authentication. ``repo.plan_download()`` reads the local catalog without opening a connection. -The plan captures its selected backend name and deeply immutable options, in -addition to its source URL, profile reference, destination and file metadata. -Changes to repository settings or the caller's original options mapping do not -redirect an approved plan. Install or register the selected plugin before -executing a transfer that uses it. +The plan captures its backend and immutable options, source URL, destination +and file metadata. Later configuration changes cannot redirect an approved +plan. Install the selected transport plugin before executing a transfer. Writing a backend ----------------- @@ -151,14 +152,15 @@ transfer: .. code-block:: python - from hallmark import Repo, register_backend + from hallmark import DataSource, SourceRelease, Repo, register_backend, register_source from my_survey.backend import SurveyBackend register_backend("survey", SurveyBackend) - repo = Repo.init( - "survey", from_url="https://survey.example.org/release/", - backend="survey", - ) + register_source(DataSource("survey", "Manifest survey", { + "dr1": SourceRelease("https://survey.example.org/release/", {}, + backend="survey"), + })) + repo = Repo.init("survey", source="survey", release="dr1") For CLI use and other collaborators, publish the class as an entry point in the plugin package's ``pyproject.toml``: @@ -179,8 +181,8 @@ Mapping one catalog across servers ---------------------------------- The :download:`multiple-server backend example <../demo/multi_server_backend.py>` -combines HTTP roots beneath logical prefixes. For example, the options file -can contain: +combines HTTP roots beneath logical prefixes. Its source definition supplies +the following adapter options: .. code-block:: yaml @@ -198,24 +200,23 @@ From the source checkout, register the demonstration backend explicitly: .. code-block:: python - from hallmark import Repo + from hallmark import DataSource, SourceRelease, Repo, register_source from demo.multi_server_backend import register register() - repo = Repo.init( - "combined", - from_url="https://north.example.org/export/", - backend="multi-server", - backend_options={"routes": { - "north": "https://north.example.org/export/", - "south": "https://south.example.org/export/", - }}, - ) + register_source(DataSource("combined", "Two data servers", { + "v1": SourceRelease("https://north.example.org/export/", {}, + backend="multi-server", backend_options={"routes": { + "north": "https://north.example.org/export/", + "south": "https://south.example.org/export/", + }}), + })) + repo = Repo.init("combined", source="combined", release="v1") plan = repo.plan_download(filter="north/**/*.fits") print(plan.summary()) The example is tested with two disposable local HTTP servers. The URLs above -are illustrative. Collaboration-specific adapters for DESI, LSST, Roman, JWST -or Euclid can implement their own manifests, directory conventions or API +are illustrative. Collaboration-specific adapters for LSST, Roman, JWST or +Euclid can implement their own manifests, directory conventions or API routing through the same contract; those service-specific plugins are not included with Hallmark. diff --git a/doc/private_data.rst b/doc/private_data.rst index c325afc..80e58f9 100644 --- a/doc/private_data.rst +++ b/doc/private_data.rst @@ -1,443 +1,322 @@ -.. _private-data: +Data sources and private data +============================= -Private data over SSH and SFTP -============================== +``hallmark init`` creates a local repository or catalogs a named source or raw +dataset URL. It reads directory listings and published checksum metadata without +downloading dataset files or offering transfers. ``hallmark clone`` copies a +complete existing catalog, then reviews a download plan by default. Use +``clone --no-download`` to skip review. ``hallmark download`` selects and approves +transfers from an existing catalog. -|hallmark|_ can discover a remote dataset, prepare a local catalog, and download -selected files after approval. ``init --from`` creates a new ``.hm`` from a -remote dataset; ``clone`` copies an existing Hallmark Git catalog or published -catalog snapshot. Both leave dataset files on their servers by default. -A **data remote** identifies the dataset location; it is independent of the -Git remote or snapshot URL used to share the catalog. +1. Choose a source +------------------ -The SSH examples use the alias ``lab-data`` and the absolute dataset root -``/srv/exports/lab/``. Replace the hostname, username, identity path and root -with your own values. Run Hallmark locally. The server needs SFTP access, but -does not require Hallmark, Git, Python or a login shell. +List supported sources, then inspect DESI releases and collection roots. These +commands read source definitions without crawling a dataset: -1. Configure and check SSH access ---------------------------------- - -Install the checkout with ``python -m pip install -e .``. The client requires -system OpenSSH ``ssh`` and ``sftp`` version 9.6 or newer, with connection -multiplexing available. Linux and macOS have integration-test jobs; Windows -SSH transport is unsupported. +.. code-block:: bash -Add an entry to your local ``~/.ssh/config``: + hallmark sources + hallmark sources desi -.. code-block:: text +DESI supports EDR and DR1. Its collections are ``redshifts`` (merged redshift +catalogs), ``spectra`` (HEALPixel and tile product directories), and ``vac`` +(value-added products). DR1 also provides ``lss`` (large-scale-structure +catalogs). The mappings follow the `DESI data organization +`_. - Host lab-data - HostName data.example.org - User researcher - IdentityFile ~/.ssh/id_ed25519 - IdentitiesOnly yes - StrictHostKeyChecking yes - -Have your administrator provision the server's verified host key in -``known_hosts``, or verify its fingerprint through a trusted channel before -accepting it. Load an encrypted key into your SSH agent before running Hallmark. -Check that SFTP works without an interactive prompt: +Create a catalog for one collection in an explicit release: .. code-block:: bash - ssh -V - sftp -o BatchMode=yes -b /dev/null lab-data + hallmark init ./desi --from desi --release dr1 --collection redshifts -Hallmark uses noninteractive authentication and strict host-key checking. It -cannot answer password, MFA or initial hardware-key prompts. Ordinary SSH -configuration, including ``ProxyJump``, remains available. +Repeat ``--collection`` to include several collections. Without it, Hallmark +catalogs the entire release, including spectra, ancillary products and root +files. Collection selection limits the directories traversed. Paths remain +relative to the release root even when a single collection is selected. +The source name, release, collections and resolved URL are saved in metadata. +Downloads use that resolved URL and the saved catalog without rediscovery. -2. Initialize a catalog from remote data ----------------------------------------- +``hallmark init ./desi --from desi`` prompts for a release in a terminal. +Scripts and Python must provide a release explicitly. Cancelling the prompt +creates no destination. No release is selected implicitly. -Suppose the export contains ``runs/run_001.h5``, ``runs/run_002.h5`` and -``README.md``. From an empty local workspace, run: +Raw URLs remain available for a specific subtree or another service: .. code-block:: bash - hallmark init ./lab --from 'ssh://lab-data/srv/exports/lab/' \ - --fmt 'runs/run_{run:03d}.h5' - -This creates ``./lab/.hm`` and catalogs only the matching run files. The URL -is the exact discovery root. A filename format selects paths and extracts -parameters; it does not approve a download. A glob filter can select paths -without defining parameters: + hallmark init ./observations --from 'https://data.example.org/export/' \ + --filter '**/*.fits' -.. code-block:: bash - - hallmark init ./lab-h5 --from 'sftp://lab-data/srv/exports/lab/' \ - --filter '**/*.h5' - -Without ``--filter`` or ``--fmt``, discovery recursively covers every directory -beneath the URL and catalogs all discovered files. A filter still traverses -directories needed to find matching files. Discovery shows an indeterminate -progress bar with counts of completed directories and discovered files. -The percentage and completion time remain unknown until a total is available. - -Both URL schemes use structured SFTP directory enumeration and file transfers. -SFTP-only accounts work for discovery as well as downloading. No -``--allow-remote-commands`` or ``--remote-hash`` option is needed. Discovery -reads listings and published checksum manifests without reading dataset files -to compute checksums. Missing checksums remain unknown. For reproducible -catalogs, publish an immutable export with a SHA-256 manifest, for example -`` runs/run_001.h5`` records in ``SHA256SUMS``. - -The generated data remote is named ``origin`` and points to the source URL. -The destination may already contain files, provided it has no ``.hm``; -initialization preserves those files. An existing ``.hm`` is rejected before -network access. Use ``hallmark init PATH`` without ``--from`` for a local -repository. The older remote ``build`` command and ``build_repo`` API remain -available but are deprecated in favor of ``init --from``. - -Select a built-in backend with ``--backend http``, ``ssh``, or ``cyverse``, or -name an installed plugin. Without an explicit choice, Hallmark selects a -backend from the URL. ``--backend-options FILE`` reads a YAML mapping of -nonsecret backend settings. These settings are saved with the data remote; -credentials belong in local authentication configuration. See :doc:`backends` -for the plugin interface and a multiple-server example. - -3. Preview, approve and download --------------------------------- +The URL is the exact recursive root. HTTP sources need a supported browsable +index; HTTP file access alone does not provide directory enumeration. +``--release`` and ``--collection`` apply only to named sources. Public HTTP +sources need no credentials; private HTTP retains Requests' local ``.netrc`` +and environment behavior. See :doc:`backends` for source and transport plugins. -Continue from the same workspace: +2. Configure SSH or SFTP access +------------------------------- -.. code-block:: bash +Use standard OpenSSH configuration. Add your server alias to ``~/.ssh/config``: - cd lab - hallmark info - hallmark download --tsv data.tsv --dry-run - hallmark download runs/run_001.h5 - -The dry run reads only the local catalog. It reports file count, known bytes, -the number of unknown file sizes, destination and source. The CLI reports -duration as unknown. Python callers can supply a transfer rate to estimate -duration when all file sizes are known. The dry run does not test credentials, -host trust or remote-file existence. +.. code-block:: text -Every nonempty CLI download displays its plan and asks -``Download these files? [y/N]``. Answer ``y`` to authorize the displayed -transfer. Refusal, a blank answer or end-of-input transfers no dataset files. -The deprecated ``--yes`` option is accepted but does not bypass confirmation. -During transfer, byte progress and measured throughput provide an ETA when the -total is known; unknown totals remain indeterminate. + Host lab-data + HostName data.example.org + User researcher + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes + StrictHostKeyChecking yes -Choose a scope from inside ``lab``: +Use your actual host, user and key. Provision the server's verified host key in +``known_hosts`` and load an encrypted key into your SSH agent. Check that SFTP +works without an interactive password or MFA prompt: .. code-block:: bash - hallmark download --filter 'runs/run_00[12].h5' --dry-run - hallmark download --all --output ./payload + ssh -V + sftp -o BatchMode=yes -b /dev/null lab-data -Explicit paths use their recorded catalog checksums when available, just like -``--tsv`` and ``--all``. A path absent from the catalog can be requested, but -its size and checksum are unknown. Files without a usable publisher checksum -can be downloaded, but their contents cannot be checked against the catalog. +Hallmark requires OpenSSH 9.6 or newer with connection multiplexing. Linux and +macOS are supported; Windows SSH transport is unsupported. Both ``ssh://`` and +``sftp://`` use structured SFTP enumeration and transfers. The server needs no +Hallmark, Git, Python or login shell. SSH configuration supplies identities, +users, ports and proxies, including ``ProxyJump``; explicit URL user/port values +take precedence. Hallmark enforces noninteractive access and strict host-key +checking. There is no Hallmark authentication file or profile-saving prompt. -``--tsv`` can be repeated and combined with explicit paths; overlapping entries -are deduplicated. ``--all`` cannot be combined with either selection mode. -Relative output paths are relative to the current directory. Without -``--output``, downloads go to the repository worktree; bare ``.hm`` repositories -require an explicit output directory. Repeating a download transfers the -selection again and replaces destinations only after successful transfer and -any recorded checksum verification. +3. Select catalog entries and extract fields +-------------------------------------------- -To request downloads during initialization, add ``--with-download``. Catalog -preparation finishes first, then the same plan and approval prompt appear. -Declining keeps the new catalog available; an empty selection needs no approval: +Suppose the source contains ``runs/run_001.h5``, ``runs/run_002.h5`` and +``README.md``. Include both run files and the README, extracting the integer +``run`` field only from matching filenames: .. code-block:: bash - hallmark init ../lab-one --from 'ssh://lab-data/srv/exports/lab/' \ - --filter 'runs/run_001.h5' --with-download + hallmark init ./lab --from 'sftp://lab-data/srv/exports/lab/' \ + --filter 'runs/*.h5' --filter 'README*' \ + --format 'runs/run_{run:03d}.h5' + +``--filter`` selects catalog entries; repeated globs form a union. Matching +is case-sensitive, ``*`` stays within a path segment, and ``**`` spans zero or +more directories. Quote patterns to prevent shell expansion. Globs filter the +inventory; they do not currently prune directory traversal. Narrow the URL or +choose collections when you want to limit the crawl. + +Without ``--format``, Hallmark automatically detects filename templates from the +filtered paths using its existing format detector. Multiple detected templates +contribute columns to one catalog; each path uses the first matching template +in the detector's stable order. Inferred names are heuristic, for example +``runs/run_001.h5`` and ``runs/run_002.h5`` yield ``runs/run_{scan}.h5``. +Detection uses filenames only and does not inspect dataset contents. Static +metadata and archive files can be ignored during inference but remain cataloged. +If no usable template is found, the catalog contains paths and metadata alone. + +``--format`` overrides detection with one explicit template, allowing you to +choose field names and types. It adds named fields without excluding files. +In the example above, the README remains in +the catalog with an empty ``run`` value. Extraction fields cannot overwrite +reserved metadata columns. Their columns exist even if no filename matches. +Inferred templates that would overwrite metadata or cannot be parsed are skipped; +invalid explicit templates are rejected before discovery. Templates are recorded +under ``extraction`` in the data configuration, separately from the local indexing +``fmt`` setting. Literal catalog paths determine downloads, even if detection +found no template. Reloading, cloning, and downloading do not repeat detection. + +Without ``--filter``, every discovered file within the selected scope is +cataloged. Discovery preserves available sizes, times and published checksums; +unknown values remain unknown. It never downloads payloads to compute hashes. +Publish SHA-256 manifests for reproducible verification where possible. + +The destination may already contain files if it has no ``.hm``; those files are +preserved. An existing catalog is rejected before remote access. A destination +ending in ``.hm`` creates a bare catalog. Plain ``hallmark init PATH`` creates a +local repository without cataloging local files automatically. + +4. Preview and approve downloads +-------------------------------- -An existing catalog can be cloned from a local path, a Git endpoint, or an -HTTP/SFTP directory containing its metadata. For example, copy this catalog: +From the new worktree, preview one run using only local catalog metadata, then +request the same transfer: .. code-block:: bash - hallmark clone ./.hm ../lab-copy - -Git endpoints preserve the complete catalog and history. Published HTTP/SFTP -snapshots start new local history. A snapshot URL may name the metadata -directory itself or its parent containing ``.hm``. Use ``--source-type git`` -to require Git or ``--source-type catalog`` to require a snapshot. Automatic -selection treats local paths, SCP-style addresses, Git/file/SSH URLs and URLs -ending in ``.git`` as Git sources. SFTP URLs select snapshots. Other HTTP(S) -URLs are checked for a snapshot before Git is attempted. Authentication errors, -connection failures and malformed snapshots are reported rather than treated -as dataset directories. Use ``init --from`` for raw datasets. - -Clone filters and formats select files to download, leaving the full catalog -and its history unchanged. They require ``--with-download`` (the older -``--download`` spelling remains an alias): + cd lab + hallmark download --filter 'runs/run_001.h5' --dry-run + hallmark download --filter 'runs/run_001.h5' + +The download command displays the file count, known bytes, unknown-size count, +source and destination, then asks for confirmation. Declining leaves the catalog +available. ``--filter`` selects only cataloged files; explicit paths cannot add +files omitted during initialization. Downloads do not change catalog membership. +Use ``--all`` for the whole catalog or ``--tsv data.tsv`` for a configured table. +A bare catalog requires ``--output DIRECTORY`` with explicit selectors; the +interactive chooser can prompt for it. ``--max-workers`` defaults to 4 +and must be positive. Internal SSH session limits may further cap concurrency. + +Dataset transfers require approval even for one file. The legacy download +``--yes`` option does not bypass confirmation. To choose files interactively +instead of supplying selectors on the command line, run: .. code-block:: bash - hallmark clone ./.hm ../lab-selected \ - --filter 'runs/run_001.h5' --with-download - -``--no-fetch-data`` remains a compatibility alias for the default behavior -without dataset downloads. Git authentication and data authentication are -independent; cloning a catalog does not require credentials for its data. + hallmark download --interactive -4. Use a local authentication profile -------------------------------------- +The chooser warns about bandwidth and disk use, then offers ``patterns``, +``all``, or ``skip`` (the default). Enter one relative-path glob per line without +shell quotes; a blank line finishes the list. Spaces and commas remain part of +the pattern. Patterns are ORed, case-sensitive, and support recursive ``**``. +Finishing an empty list or matching no files returns to selection. -Profiles are optional when SSH configuration already provides access. They let -each collaborator supply credentials locally while the catalog records only a -profile name. Add the following to ``~/.config/hallmark/auth.yml``, merging it -with any existing profiles: +The preview shows the source, destination, file count, known bytes, unknown-size +count, and up to 20 paths with recorded sizes. It uses local metadata only; +unknown sizes remain unknown. Choose ``download`` to approve that exact plan, +``change`` to select again and review a new plan, or ``skip`` (the default). +``--interactive --dry-run`` previews the selection and exits without offering +execution. ``--output``, ``--remote`` and ``--max-workers`` are supported with +``--interactive``; paths, ``--filter``, ``--tsv`` and ``--all`` cannot be combined +with it. Bare ``hallmark download`` still requires selectors or ``--interactive``. -.. code-block:: yaml - - version: 1 - profiles: - lab: - hosts: [lab-data] - user: researcher - identity_file: ~/.ssh/id_ed25519 - host_key_policy: strict - connect_timeout: 10 - transfer_timeout: 3600 - shutdown_timeout: 2 - max_sessions: 4 - -Keep this file outside the repository and restrict it to your account, for -example with ``chmod 600 ~/.config/hallmark/auth.yml``. Hallmark reads -``$HALLMARK_AUTH_FILE`` when set, otherwise -``$XDG_CONFIG_HOME/hallmark/auth.yml``, otherwise the path above. Every client -using a catalog's profile reference must define that profile locally. - -Associate an existing data remote with a profile, or select one when -initializing a catalog from a remote dataset: +CLI cloning first copies the complete catalog, then shows a download plan and +asks for confirmation. All cataloged files are selected unless ``--filter`` +narrows the planned transfers. These examples use separate destinations: .. code-block:: bash - hallmark set-config --remote-name origin --remote-auth lab - hallmark download --remote origin --tsv data.tsv --dry-run - hallmark init ../lab-profile --from 'ssh://lab-data/srv/exports/lab/' --auth lab - -The new catalog's ``origin`` remote records ``auth: lab``. Clear a reference with -``hallmark set-config --remote-name origin --remote-auth ''`` to use SSH -configuration alone. With ``init``, ``--auth`` selects data access. With -``clone``, it selects access to an SSH/SFTP catalog snapshot; cloned data -remotes retain their own profile references. Git cloning uses Git's own -authentication configuration. - -``hosts`` binds the profile to the URL's alias or literal hostname before -``HostName`` resolution. This example binds to ``lab-data``, not -``data.example.org``. Explicit URL user/port values override profile settings; -unset values use SSH configuration. Passwords, tokens and arbitrary -SSH options are not valid profile fields. ``transfer_timeout`` is the total -per-file time budget in seconds, not an idle timeout. Effective SSH download -concurrency is the smaller of ``--max-workers`` and local ``max_sessions`` -(both default to 4). + hallmark clone ./lab/.hm ./lab-copy-selected --filter 'runs/run_001.h5' + hallmark clone ./lab/.hm ./lab-copy-interactive --interactive + +The initialization filter controls catalog membership. Clone filters and the +chooser select only downloads, retaining the complete catalog and its history. +``clone --interactive`` opens the chooser instead of the default plan and cannot +be combined with ``--filter``. ``clone --no-download`` skips review entirely and +cannot be combined with ``--interactive``, ``--filter``, or ``--output``. +``clone --output DIRECTORY`` changes the download destination; a bare catalog +prompts for one if omitted. Declining or EOF during clone's review exits +successfully and leaves the catalog available. Ctrl+C or a transfer failure exits +unsuccessfully while preserving it. Empty catalogs and catalogs without a data +remote skip review with an explanation. + +Without an interactive terminal, clone still copies the catalog, warns that +approval is unavailable, and prints repository-specific preview and download +commands reflecting its filters and output directory. Later transfers still +require explicit approval. Standalone +``download --interactive`` requires a terminal. Existing source requirements, +such as an explicit DESI release in noninteractive use, still apply. +Download worker limits belong on ``download``; ``init`` and ``clone`` do not +accept them. Python initialization and cloning remain metadata-only. 5. Use the Python API --------------------- -These examples use fresh local destinations. Like the CLI, ``Repo.init`` -with ``from_url`` discovers metadata without downloading dataset files: +Create a catalog and inspect a plan before transferring selected data: .. code-block:: python from hallmark import Repo repo = Repo.init( - "lab-python", from_url="ssh://lab-data/srv/exports/lab/", - auth="lab", # Omit when SSH configuration already supplies access. - fmt="runs/run_{run:03d}.h5", + "lab-python", source="sftp://lab-data/srv/exports/lab/", + filter=["runs/*.h5", "README*"], format="runs/run_{run:03d}.h5", progress=True, ) - plan = repo.plan_download(file_paths=["runs/run_001.h5"]) + plan = repo.plan_download(filter="runs/run_001.h5") print(plan.summary()) - print(plan.items) # Paths, checksums, optional size_bytes and mtime. -Inspect the plan, then approve the download: +After reviewing the plan, approve it explicitly and check the results: .. code-block:: python - result = repo.download(plan, approved=True, progress=True) + result = repo.download(plan, approved=True, max_workers=4, progress=True) if result["failed"]: - raise RuntimeError("\n".join(result["errors"])) - -Without ``approved=True``, a nonempty transfer raises ``DownloadError`` before -opening a connection. A plan freezes the selected files, source URL, profile -reference, backend, backend options and destination. Later catalog or remote -configuration changes do not redirect it. ``plan_download(filter="**/*.h5")`` -selects matching files from the catalog; -``plan_download(output_path="subset", tsv_names=["data.tsv"])`` selects a TSV -and destination. Planning makes no network requests. Known size totals and -unknown-size counts are available as ``known_bytes`` and -``unknown_size_count``. ``estimated_seconds`` stays ``None`` unless all sizes -are known and ``estimated_bytes_per_second`` was supplied when planning. - -For downloads during initialization or cloning, supply a callback that reviews -the completed plan and returns ``True`` to approve: - -.. code-block:: python - - def approve(plan): - print(plan.summary()) - return input("Download these files? [y/N] ").strip().lower() == "y" - - repo = Repo.init( - "lab-with-download", from_url="ssh://lab-data/srv/exports/lab/", - filter="runs/run_001.h5", download=True, approve=approve, progress=True, - ) - -``repo.download`` returns ``succeeded``, ``failed``, ``total_bytes`` and -``errors``. Inspect ``failed`` because successful files remain when another -file fails. Setup failures raise ``hallmark.downloader.DownloadError``. -``repo.set_config(remote_auth="")`` removes a profile reference; ``None`` -leaves it unchanged. - -6. Discover public HTTPS collections ------------------------------------- - -The same workflow supports CyVerse and ordinary browsable HTTPS collections -such as DESI. No CyVerse-specific index option is required: - -.. code-block:: bash - - hallmark init ./cyverse --from \ - 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/' \ - --filter 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' - hallmark init ./desi --from \ - 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/' \ - --filter 'redrock-main-dark-23040.fits' - -Each URL is the full recursive discovery root. A broad root can require many -listing requests even with a file filter; use a specific subtree when that is -your intended scope. Public collections need no auth profile. Private HTTP -access retains Requests' normal ``.netrc`` and environment behavior; Hallmark's -named auth profiles apply to SSH/SFTP. - -From either initialized catalog, inspect and approve a selected subset using -``hallmark download --filter PATTERN --dry-run`` and then the same command -without ``--dry-run``. Python follows the same steps: - -.. code-block:: python - - cyverse = Repo.init( - "cyverse-python", - from_url="https://data.cyverse.org/dav-anon/iplant/commons/" - "cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/", - filter="SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits", progress=True, - ) - desi = Repo.init( - "desi-python", - from_url="https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/" - "healpix/main/dark/230/23040/", - filter="redrock-main-dark-23040.fits", progress=True, - ) - plan = desi.plan_download(all_files=True) - print(plan.summary()) - -Inspect this plan and approve it as in the Python download example above. - -HTTP provides file transfer but does not define directory enumeration. Hallmark -recognizes supported HTML indexes and directory landing pages automatically. -If a server supplies no usable listing, provide a published Hallmark catalog, -a more specific browsable URL, or a :doc:`backend plugin ` for its -API. Hallmark cannot infer hidden file URLs. - -7. Host catalogs separately from their data -------------------------------------------- - -A catalog's location does not set its data location. For example, a team -can push the Git repository in ``.hm`` to GitHub while its data remote still -points to ``ssh://lab-data/srv/exports/lab/`` with ``auth: lab``. Collaborators -clone the existing metadata using their Git credentials: - -.. code-block:: bash - - hallmark clone 'https://github.com/example/lab-catalog.git' ./shared-lab - cd shared-lab - hallmark download --all --dry-run - -This clone needs no ``lab`` profile. Define the local profile before approving -a data transfer. The Git origin points to GitHub; the data remote named -``origin`` retains the SSH URL and profile reference recorded in the catalog. -Git fetch/pull operates on the metadata repository in ``.hm``. - -The same configuration can be published as a snapshot on a separate HTTPS -server. For example, publish ``config.yml``, ``meta.yml`` and referenced TSVs -under ``https://catalogs.example.org/lab/``. Its configuration can contain: + raise RuntimeError("; ".join(result["errors"])) -.. code-block:: yaml +``Repo.init("desi", source="desi", release="dr1", collections=["redshifts"])`` +uses the same named source. Python never prompts for releases. Plans freeze +paths, checksums, transport options, source and destination; subsequent catalog +configuration changes do not redirect them. ``estimated_bytes_per_second`` can +provide a duration estimate when every selected file has a known size. - remote: - - name: origin - url: ssh://lab-data/srv/exports/lab/ - auth: lab +6. Share a catalog +------------------ -Then clone the snapshot: +Clone an existing catalog without download review, then select downloads separately: .. code-block:: bash - hallmark clone 'https://catalogs.example.org/lab/' ./snapshot-lab + hallmark clone ./lab/.hm ./lab-copy --no-download + cd lab-copy + hallmark download --filter 'runs/run_001.h5' --dry-run + hallmark download --filter 'runs/run_001.h5' -Snapshot authentication uses the metadata server's settings. The SSH data -URL and ``lab`` reference are preserved without resolving that profile during -cloning. For an SFTP snapshot, ``clone --auth catalog-reader`` selects its -metadata profile; it does not replace a data remote's profile. Snapshot -imports start local Git history and have no upstream Git history to pull. -These examples retain the usual local ``PATH/.hm`` layout while keeping -catalog hosting independent of data hosting. +Git clones preserve the full catalog and Git history. Published HTTP/SFTP +snapshots start new local history. Use ``--source-type git`` or +``--source-type catalog`` to override source detection. The catalog can be +hosted independently from the dataset, for example GitHub metadata with an SFTP +data remote. Metadata cloning does not connect to that data remote. Each client +configures its own SSH alias and credentials for later transfers. -.. _operational-behavior-and-troubleshooting: +Downloads alone do not populate the local object store. Use a local repository +and the normal ``add`` and ``commit`` workflow to version downloaded inputs. +Local filename-format indexing retains its existing behavior. -Download behavior and troubleshooting -------------------------------------- +7. Migrate older commands +------------------------- -Files are downloaded to temporary paths and published atomically after transfer -and any recorded checksum verification. A failed transfer preserves an existing -destination and removes its temporary file. Successful files remain available -when another file fails; a multi-file download is not a single transaction. -The CLI exits nonzero on transfer failures. Cancellation stops the active -operation and closes the SSH connections and processes it started. +This interface change intentionally provides no argument aliases. Existing +catalogs retain their local formats and transport metadata. Stored authentication +profile references produce an actionable migration error. -.. list-table:: Common failures +.. list-table:: Interface migration :header-rows: 1 - :widths: 30 70 - - * - Symptom - - Action - * - SSH connection check fails - - Check the trusted host key, loaded identity, username, server availability - and OpenSSH version. For profile-only settings, pass the same - identity/user/port to the manual SFTP check. - * - Profile unresolved or not bound to the host - - Check the auth-file path, profile name and URL host against ``hosts``. - Another collaborator's local profile is not shared with the catalog. - * - No supported remote listing - - Use a browsable collection root or published catalog. Confirm that an - SFTP account can enumerate the requested directories. - * - Checksum mismatch - - Check whether the export changed since catalog creation. Reconcile the - catalog and source against a trusted version before retrying. - * - Transfer timeout or session limit - - Adjust the local per-file time budget or lower concurrency to match - the server's capacity. - -Data URLs require absolute roots, for example -``ssh://researcher@lab-data:2222/srv/exports/lab/``. SCP shorthand is unsupported -for data directories. Percent-encode reserved characters in URL roots; catalog -paths are literal relative names. Hallmark rejects traversal and observed -destination symlinks; server permissions remain the access-control boundary. - -Discovered catalogs can use path rows and published catalogs can contain -multiple TSVs. Existing local ``add``/``commit``/``checkout`` workflows still -require a compatible one-format ``data.tsv`` repository; downloading does not -populate the local object store. For local experiments, initialize a separate -repository, copy in the downloaded files, configure their filename format, -then add and commit them. - -See :ref:`private-transport-reference` for profile settings, HTTP compatibility -and disposable transport tests, and :doc:`api` for function signatures. - -.. |hallmark| replace:: ``hallmark`` - -.. _hallmark: https://github.com/l6a/hallmark + :widths: 35 65 + + * - Previous interface + - Replacement + * - ``init --backend`` / ``--backend-options`` + - ``--from NAME --release NAME --collection NAME``; source authors configure adapters. + * - ``init --include`` + - ``init --filter`` selects catalog entries. + * - ``init --extract`` / ``--fmt`` + - ``--format`` overrides automatic detection; use ``--filter`` for selection. + * - ``download --include`` / ``--fmt`` + - ``download --filter`` with path globs. + * - ``init --with-download`` + - Initialize metadata, then run ``download`` separately. + * - ``clone --with-download`` / ``--download`` + - Plain ``clone`` reviews downloads by default; ``--interactive`` opens the chooser. + * - ``clone --no-fetch-data`` or relying on catalog-only CLI defaults + - ``clone --no-download`` copies the complete catalog without download review. + * - Clone filtering + - ``clone --filter`` selects payloads only; the catalog remains complete. + * - ``--auth``, ``--remote-auth``, ``--dataset-auth`` + - Configure ``~/.ssh/config`` and use its host alias in the URL. + * - ``Repo.init(from_url=...)`` + - ``Repo.init(source=...)``. + * - Python ``include=`` / ``extract=`` / remote ``fmt=`` + - ``filter=`` and initialization-only ``format=``. + * - Python clone/init download arguments and approval callbacks + - ``repo.plan_download()`` followed by ``repo.download(plan, approved=True)``. + +Remove obsolete ``auth`` fields from catalog ``config.yml`` after configuring +the host in SSH configuration. Hallmark neither reads nor deletes +``~/.config/hallmark/auth.yml``; ``HALLMARK_AUTH_FILE`` is ignored. Python profile +arguments and CLI backend-setting options have been removed. Existing +``backend`` and ``backend_options`` catalog metadata remain supported for +transport plugins. The legacy remote builder remains deprecated. + +8. Transfer failures and limits +------------------------------- + +Files are downloaded to temporary paths and published atomically after checksum +verification. Successful files remain if another transfer fails. Cancelling +stops owned SSH processes and waits for workers; HTTP checks cancellation +between chunks and remains subject to request timeouts. Missing checksums cannot +verify content. Setup and authentication failures stop the operation. + +If SSH fails, check the alias with the SFTP command above, trusted host keys, +agent access and server permissions. If HTTP discovery fails, check that the +URL publishes a supported directory index. Large roots can take many listing +requests even when the final catalog contains few matching files. diff --git a/doc/usecase.rst b/doc/usecase.rst index cbdee06..01fa3d9 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -46,7 +46,7 @@ He initializes a bare |hallmark|_ catalog from a simulation export and verifies its location:: hallmark init sim.hm --from ssh://campus/srv/export/ \ - --fmt 'run{run:d}/frame{frame:d}.h5' + --format 'run{run:d}/frame{frame:d}.h5' cd sim.hm hallmark info @@ -202,67 +202,13 @@ call may wait for its 10-second connection timeout or 30-second read timeout; a response that keeps sending small amounts of data can delay cancellation further. -Local profiles -~~~~~~~~~~~~~~ - -Frank can keep SSH settings in a local authentication profile and -record its name in the repository. -|hallmark|_ reads profiles from ``$HALLMARK_AUTH_FILE`` when set, -otherwise from ``$XDG_CONFIG_HOME/hallmark/auth.yml``, or from -``~/.config/hallmark/auth.yml`` if neither variable is set. -Profiles apply to SSH and SFTP connections. - -For example, his authentication file can contain: - -.. code-block:: yaml - - version: 1 - defaults: - connect_timeout: 10 - transfer_timeout: 3600 - shutdown_timeout: 2 - max_sessions: 4 - profiles: - campus: - hosts: [campus] - user: researcher - port: 22 - identity_file: ~/.ssh/id_ed25519 - host_key_policy: strict - -He associates the profile with the data remote:: - - hallmark set-config --remote-name campus --remote-auth campus - -He can remove the reference later:: - - hallmark set-config --remote-name campus --remote-auth '' - -The Python equivalents use ``repo.set_config`` with -``remote_name="campus"`` and ``remote_auth="campus"`` or ``remote_auth=""``. -Passing ``None`` leaves the profile reference unchanged. -The repository stores only the profile name; identity settings remain local. - -Profile names must match ``[A-Za-z0-9_-]{1,64}``. -The ``hosts`` list contains the aliases or hostnames used in data URLs, -before SSH resolves ``HostName``. -A missing profile or a host outside this list causes an error before -connecting. A username or port in the URL overrides the profile setting; -remaining values come from SSH configuration and OpenSSH defaults. - -Profiles accept the fields shown above and may override the four -timeout and concurrency defaults. Global defaults cannot set ``hosts``, -``user``, ``port``, or ``identity_file``. -Passwords, tokens, arbitrary SSH options, executable paths, and SSH -configuration paths are not supported profile fields. - -``transfer_timeout`` limits the total transfer time for each file in -seconds, so large datasets may need a higher value. -``max_sessions`` limits concurrent transfers within an operation to -respect the server's session limit. -The local profile or defaults may use ``host_key_policy: accept-new`` -to accept a host key on first contact. Changed host keys still cause -an error. Repository configuration cannot enable this setting. +SSH configuration +~~~~~~~~~~~~~~~~~ + +Frank keeps connection settings in ``~/.ssh/config``. Hallmark passes the URL's +host alias to OpenSSH, which resolves the user, port, identity and any proxy. +Explicit URL user and port values take precedence. Hallmark does not read a +separate authentication file or prompt to create one. Initialize a private catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -270,13 +216,13 @@ Initialize a private catalog Frank can prepare a catalog directly from the files on the server:: hallmark init lab --from ssh://campus/srv/export/ \ - --auth campus --fmt 'runs/run_{i:d}.h5' + --filter 'runs/run_*.h5' --format 'runs/run_{i:d}.h5' cd lab hallmark download --all --dry-run hallmark download --all This creates ``lab/.hm`` with only the matching run files in ``data.tsv``. -Without a filter or format, the catalog contains all discovered files below +Without an inclusion glob, the catalog contains all discovered files below the supplied URL. Initialization downloads no dataset contents. The final command displays the transfer plan and asks Frank for confirmation. @@ -286,11 +232,10 @@ both discovery and downloads. Symlinks and special files are skipped. Authentication and permission failures stop discovery rather than producing an apparently complete catalog. -The generated data remote ``origin`` records the source URL and optional -profile name. +The generated data remote ``origin`` records the source URL. Existing Git-hosted catalogs keep their complete catalog, history and -recorded data remotes when cloned. Clone filters select optional downloads -and require ``--with-download``; they leave the catalog and history unchanged. +recorded data remotes when cloned. Select and approve transfers afterward with +``download --filter``; downloads leave the catalog and history unchanged. The Git catalog can be hosted separately from its data servers. Published manifest checksums are recorded in the catalog. Missing checksums From 4d00814ef3d358698904d88556c239c5ddb46f75 Mon Sep 17 00:00:00 2001 From: Dennis Date: Tue, 22 Sep 2026 22:04:18 -0700 Subject: [PATCH 18/29] Update private data demos and clear notebook outputs Use source and format arguments, demonstrate interactive download selection and catalog-only cloning, and adapt the multiple-server example to source registration. Keep validated notebooks free of generated outputs. --- demo/demo_cli.ipynb | 184 +------ demo/demo_python.ipynb | 640 ++----------------------- demo/multi_server_backend.py | 5 +- demo/private_data_demo.md | 60 ++- demo/scientific_workflows_cli.md | 16 +- demo/scientific_workflows_python.ipynb | 10 +- 6 files changed, 117 insertions(+), 798 deletions(-) diff --git a/demo/demo_cli.ipynb b/demo/demo_cli.ipynb index 152fc11..0953881 100644 --- a/demo/demo_cli.ipynb +++ b/demo/demo_cli.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -51,40 +51,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== .hm directory ===\n", - " config.yml\n", - " data.tsv\n", - " meta.yml\n", - "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n", - "\n", - "=== initial config.yml ===\n", - "# Edit this file only if your branch needs regex substitutions or a preset remote.\n", - "# For simple names, you can just run: hallmark add \"a{a}_i{i}.h5\"\n", - "data:\n", - " -\n", - " # fmt: \"{release}_{source}_{year}_{doy:03d}_{band}.uvfits\"\n", - " encoding:\n", - " # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "remote:\n", - " # name: origin\n", - " # url: https://example.com/path/to/data/\n", - "\n", - "=== objects stored so far ===\n", - " 0\n" - ] - } - ], + "outputs": [], "source": [ "echo \"=== .hm directory ===\"\n", "ls repo1/.hm/\n", @@ -107,24 +80,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "dot-hallmark repo: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm\"\n", - "hallmark worktree: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1\"\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark info && popd\n" ] @@ -138,33 +100,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files in repo1/:\n", - " a0.75_i0.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0.75_i120.h5\n", - " a0_i0.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - " a0_i120.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1\n", "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", @@ -182,71 +124,26 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Changes to be committed\n", - " a0.75_i0.h5\n", - "a0.75_i120.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0_i0.h5\n", - " a0_i120.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd\n" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== config.yml after add ===\n", - "data:\n", - "- fmt: a{a}_i{i}.h5\n", - " encoding: null\n", - "remote: null\n", - "\n", - "=== data.tsv after add ===\n", - "sha1\ta\ti\n", - "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", - "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", - "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", - "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", - "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", - "18682593a012503b71935017dbc044665254a23c\t0\t0\n", - "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", - "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", - "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", - "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" - ] - } - ], + "outputs": [], "source": [ "echo \"=== config.yml after add ===\"\n", "cat repo1/.hm/config.yml\n", @@ -257,45 +154,26 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Committed staged state changes.\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark commit -m \"add main dataset\" && popd\n" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after commit:\n", - " 10\n" - ] - } - ], + "outputs": [], "source": [ "echo \"Objects stored after commit:\"\n", "find repo1/.hm/objects -type f | wc -l\n" @@ -310,33 +188,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "commit 7c035e02716b9d3ee6ccecb9c0abb7769a9dd96a\n", - "Author: Ram Adithya \n", - "Date: Mon Apr 20 19:28:53 2026 -0700\n", - "\n", - " add main dataset\n", - "\n", - "commit d709e546105c75dad9b43b71f543151974d6d531\n", - "Author: Ram Adithya \n", - "Date: Mon Apr 20 19:28:50 2026 -0700\n", - "\n", - " Initial commit: local `.hm` repository\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark log && popd\n" ] @@ -804,9 +662,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 10. Clone a hallmark repository\n", - "\n", - "This clone demo uses the default CLI behavior, which clones the hallmark repo and then downloads the configured data files. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" + "This clone demo uses `--no-download` to copy the complete Hallmark catalog without download review. By default, CLI clone shows a plan for all cataloged files and requests confirmation after copying metadata. `clone --filter` narrows only the payload selection, while `clone --interactive` opens the patterns/all/skip chooser. Without a terminal, clone keeps the catalog and prints download commands for later. Run `hallmark download --all` or `hallmark download --interactive` separately when ready. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" ] }, { @@ -819,7 +675,7 @@ }, "outputs": [], "source": [ - "hallmark clone \"https://github.com/l6a/hallmark-demo-repo.hm.git\" hallmark-demo-clone\n" + "hallmark clone \"https://github.com/l6a/hallmark-demo-repo.hm.git\" hallmark-demo-clone --no-download\n" ] }, { diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index da3c0b1..9c34a7a 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -79,33 +79,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/astro_se_int/hallmark/demo/repo1 ~/astro_se_int/hallmark/demo\n", - "Files in repo1/:\n", - "a0_i0.h5\n", - "a0_i120.h5\n", - "a0_i30.h5\n", - "a0_i60.h5\n", - "a0_i90.h5\n", - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", - "~/astro_se_int/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "pushd repo1\n", @@ -124,123 +104,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i30.h50.7530.0
3a0.75_i60.h50.7560.0
4a0.75_i90.h50.7590.0
5a0_i0.h50.000.0
6a0_i120.h50.00120.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "1 a0.75_i120.h5 0.75 120.0\n", - "2 a0.75_i30.h5 0.75 30.0\n", - "3 a0.75_i60.h5 0.75 60.0\n", - "4 a0.75_i90.h5 0.75 90.0\n", - "5 a0_i0.h5 0.00 0.0\n", - "6 a0_i120.h5 0.00 120.0\n", - "7 a0_i30.h5 0.00 30.0\n", - "8 a0_i60.h5 0.00 60.0\n", - "9 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "paraframe = repo1.add(\"a{a}_i{i}.h5\")\n", "paraframe" @@ -255,112 +125,52 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'main',\n", - " 'staged': {'state': ['config.yml', 'data.tsv', 'meta.yml'],\n", - " 'added': ['a0.75_i0.h5',\n", - " 'a0.75_i120.h5',\n", - " 'a0.75_i30.h5',\n", - " 'a0.75_i60.h5',\n", - " 'a0.75_i90.h5',\n", - " 'a0_i0.h5',\n", - " 'a0_i120.h5',\n", - " 'a0_i30.h5',\n", - " 'a0_i60.h5',\n", - " 'a0_i90.h5'],\n", - " 'modified': [],\n", - " 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "repo1.status()" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "repo1.commit(\"add main dataset\")\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'main',\n", - " 'staged': {'state': [], 'added': [], 'modified': [], 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "repo1.status()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after commit:\n", - " 10\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "echo \"Objects stored after commit:\"\n", @@ -376,31 +186,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "commit d3da52e3a3077ce58995037957d74d1316fadde8\n", - "Author: Nayera Abdessalam \n", - "Date: Fri May 1 09:27:14 2026 -0700\n", - "\n", - " add main dataset\n", - "\n", - "commit ecb6fc7ffa7fd6b61d59a580d131c829caf942a8\n", - "Author: Nayera Abdessalam \n", - "Date: Fri May 1 09:27:14 2026 -0700\n", - "\n", - " Initial commit: local `.hm` repository\n" - ] - } - ], + "outputs": [], "source": [ "print(repo1.log())\n" ] @@ -416,27 +208,13 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'experiment',\n", - " 'staged': {'state': [], 'added': [], 'modified': [], 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# add(\".\") rebuilds the manifest from files that currently exist\n", "\n", @@ -464,24 +242,13 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Check back to main to see that the manifest has been correctly rebuilt\n", "repo1.checkout(\"main\")" @@ -489,24 +256,13 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'current': 'main', 'names': ['experiment', 'main']}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Show available branches\n", "repo1.branches()" @@ -514,31 +270,13 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout back to main:\n", - "repo1/a0_i0.h5\n", - "repo1/a0_i120.h5\n", - "repo1/a0_i30.h5\n", - "repo1/a0_i60.h5\n", - "repo1/a0_i90.h5\n", - "repo1/a0.75_i0.h5\n", - "repo1/a0.75_i120.h5\n", - "repo1/a0.75_i30.h5\n", - "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "echo \"Files after checkout back to main:\"\n", @@ -561,88 +299,13 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
5a0_i0.h50.00.0
6a0_i120.h50.0120.0
7a0_i30.h50.030.0
8a0_i60.h50.060.0
9a0_i90.h50.090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "5 a0_i0.h5 0.0 0.0\n", - "6 a0_i120.h5 0.0 120.0\n", - "7 a0_i30.h5 0.0 30.0\n", - "8 a0_i60.h5 0.0 60.0\n", - "9 a0_i90.h5 0.0 90.0" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Single Parameter Filter\n", "paraframe(a=0)" @@ -650,95 +313,13 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
2a0.75_i30.h50.7530.0
5a0_i0.h50.000.0
6a0_i120.h50.00120.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "2 a0.75_i30.h5 0.75 30.0\n", - "5 a0_i0.h5 0.00 0.0\n", - "6 a0_i120.h5 0.00 120.0\n", - "7 a0_i30.h5 0.00 30.0\n", - "8 a0_i60.h5 0.00 60.0\n", - "9 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Or Filter\n", "paraframe(a=0, i=30)" @@ -746,60 +327,13 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
7a0_i30.h50.030.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "7 a0_i30.h5 0.0 30.0" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# And Filter\n", "paraframe(a=0)(i=30)" @@ -807,95 +341,13 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
2a0.75_i30.h50.7530.0
3a0.75_i60.h50.7560.0
4a0.75_i90.h50.7590.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "2 a0.75_i30.h5 0.75 30.0\n", - "3 a0.75_i60.h5 0.75 60.0\n", - "4 a0.75_i90.h5 0.75 90.0\n", - "7 a0_i30.h5 0.00 30.0\n", - "8 a0_i60.h5 0.00 60.0\n", - "9 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Masking Filter retains Pandas DataFrame functionality\n", "paraframe[(30 <= paraframe.i) & (paraframe.i <= 100)]" @@ -910,39 +362,19 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 8/8 [00:03<00:00, 2.26file/s]\n" - ] - }, - { - "ename": "DownloadError", - "evalue": "Failed to download 8 file(s):\n - Checksum mismatch for SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_hi_hops_netcal_StokesI.uvfits.part\n - ... 3 more error(s)", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mDownloadError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[27]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mhallmark\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01merror\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DestinationExistsError\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m clone = Repo.clone(\u001b[33m'\u001b[39m\u001b[33mhttps://github.com/l6a/hallmark-demo-repo.hm.git\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mhallmark-demo-clone\u001b[39m\u001b[33m\"\u001b[39m, show_progress=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 6\u001b[39m clone_result = {\n\u001b[32m 7\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdot_hallmark_repo\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(clone.dothm.path),\n\u001b[32m 8\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mhallmark_worktree\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(clone.worktree),\n\u001b[32m 9\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mbranches\u001b[39m\u001b[33m\"\u001b[39m: clone.branches(),\n\u001b[32m 10\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdata_rows\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mlen\u001b[39m(clone.state.data),\n\u001b[32m 11\u001b[39m }\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m DestinationExistsError \u001b[38;5;28;01mas\u001b[39;00m exc:\n", - "\u001b[36mFile \u001b[39m\u001b[32m:23\u001b[39m, in \u001b[36mclone\u001b[39m\u001b[34m(cls, url, path, fetch_data, max_workers, show_progress)\u001b[39m\n", - "\u001b[31mDownloadError\u001b[39m: Failed to download 8 file(s):\n - Checksum mismatch for SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_hi_hops_netcal_StokesI.uvfits.part\n - ... 3 more error(s)" - ] - } - ], + "outputs": [], "source": [ "# Demo: clone a remote demo repository\n", "from hallmark.error import DestinationExistsError\n", "\n", "try:\n", - " clone = Repo.clone('https://github.com/l6a/hallmark-demo-repo.hm.git', \"hallmark-demo-clone\", show_progress=True)\n", + " clone = Repo.clone('https://github.com/l6a/hallmark-demo-repo.hm.git', \"hallmark-demo-clone\", progress=True)\n", " clone_result = {\n", " \"dot_hallmark_repo\": str(clone.dothm.path),\n", " \"hallmark_worktree\": str(clone.worktree),\n", diff --git a/demo/multi_server_backend.py b/demo/multi_server_backend.py index d5633a5..5832a2d 100644 --- a/demo/multi_server_backend.py +++ b/demo/multi_server_backend.py @@ -1,7 +1,8 @@ """Example backend joining independent HTTP roots into one logical dataset. -Call ``register()`` before ``Repo.init(..., backend="multi-server", -backend_options={"routes": {"north": north_url, "south": south_url}})``. +Call ``register()`` and define a ``DataSource`` release with +``backend="multi-server"`` and ``backend_options={"routes": ...}``, then call +``Repo.init(..., source=source_name, release=release_name)``. An installed plugin instead exposes ``MultiServerBackend`` through the ``hallmark.backends`` entry-point group. No servers are contacted on import. """ diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md index fb72ecf..0a54749 100644 --- a/demo/private_data_demo.md +++ b/demo/private_data_demo.md @@ -47,11 +47,12 @@ mkdir "$HM_DEMO_WORKSPACE" cd "$HM_DEMO_WORKSPACE" ``` -4. Discover the two indexed runs and prepare a local `.hm`. The format selects -run files and extracts their run numbers; it does not authorize a download. +4. Discover the two indexed runs and prepare a local `.hm`. The filter selects +run files; the explicit format extracts their run numbers instead of inferring +a template. Initialization does not authorize a download. ```bash -hallmark init ./client --from "$HM_DEMO_URL" --fmt 'runs/run_{run:03d}.dat' +hallmark init ./client --from "$HM_DEMO_URL" --filter 'runs/*.dat' --format 'runs/run_{run:03d}.dat' cd client ``` @@ -61,11 +62,12 @@ Only listings and published metadata are read. Available sizes, modification times, and published checksums are retained; missing checksums stay unknown. Dataset files are not downloaded to calculate checksums. -Omit `--fmt` to catalog every file below the dataset root. A filter such as -`--filter 'runs/*.dat'` selects relative paths without defining parameters. -`init --backend NAME --backend-options FILE` selects an installed data backend -and reads nonsecret options from a YAML mapping. See -[data backends](../doc/backends.rst) for registration and multiple-server routing. +Omit `--filter` to catalog every file below the dataset root. Omit `--format` +to detect filename templates automatically from the selected paths. An explicit +format overrides detection; either way, unmatched files remain cataloged. +Use `hallmark sources` +to inspect named sources; `--release` and `--collection` select their contents. +See [data backends](../doc/backends.rst) for source registration and adapter routing. 5. Review an offline plan for one run, then approve its transfer. The second command displays the plan again and asks for confirmation; answer `y` only when @@ -101,22 +103,37 @@ hallmark download runs/run_001.dat --output ../via-sftp --dry-run Both `ssh://` and `sftp://` use SFTP for discovery and data transfer. Relative output paths use the current directory. A bare `.hm` catalog requires an -explicit `--output`. To set a named authentication profile, use -`hallmark set-config --remote-name origin --remote-auth PROFILE`; an empty -profile name returns to SSH configuration alone. +explicit `--output`, or the interactive chooser prompts for a directory. +Authentication uses `~/.ssh/config` and the SSH agent; Hallmark has no +authentication-profile file or flags. 7. Use the same review step in Python. The [scientific workflows notebook](scientific_workflows_python.ipynb) demonstrates `Repo.plan_download()` followed by explicit approval and `repo.download(plan, approved=True)`. It stops on refusal or returned transfer -failures. A plan retains its source, profile reference, backend settings, +failures. A plan retains its source, backend settings, destination, paths, and checksums even if repository configuration later changes. -8. To review a download as part of initialization, add `--with-download` to an -`init --from` command. The catalog is created before the approval prompt; -declining keeps it available. Python accepts `download=True` and an -`approve(plan)` callback. The [private data guide](../doc/private_data.rst) -contains both forms. +8. Plain initialization and cloning do not download payloads. To choose files +interactively from the completed catalog, run the following command. Choose +`patterns` and enter one raw relative-path glob per line, ending with a blank +line, or choose `all`. Review recorded sizes, then choose `download`, `change`, +or `skip` (the default). Unknown file sizes remain unknown. + +```bash +hallmark download --interactive +``` + +Add `--dry-run` to stop after the preview. Initialization never offers downloads. +CLI cloning reviews a download plan after copying the complete catalog by default; +use `clone --filter` to narrow transfers or `clone --interactive` for this chooser. +Use `clone --no-download` to skip review. Without a terminal, clone keeps the +catalog, skips downloading, and prints commands reflecting the selected filters +and output directory. Declining or EOF during clone's review preserves the catalog +and exits successfully; an interrupt or transfer failure preserves it but exits +with an error. Python continues to use `repo.plan_download()` and then +`repo.download(plan, approved=True)`. The +[private data guide](../doc/private_data.rst) describes both interfaces. 9. Use the same discovery workflow for public datasets. The [scientific CLI workflows](scientific_workflows_cli.md) select one EHT file from @@ -133,19 +150,18 @@ SSH data URL. The uncommitted configuration change in step 6 remains local. ```bash cd "$HM_DEMO_WORKSPACE" -hallmark clone ./client/.hm ./catalog-copy +hallmark clone ./client/.hm ./catalog-copy --no-download ``` `clone` copies existing catalogs; raw dataset URLs belong to `init --from`. A Git catalog can live on GitHub while its data remains on the SSH server. A published HTTP/SFTP catalog snapshot can likewise use a different data server; snapshot imports start local history. Cloning metadata does not require the -configured data profile or dataset credentials. Git catalog authentication and +dataset credentials. Git catalog authentication and data authentication are independent. -Clone preserves the complete catalog. Its `--filter` and `--fmt` options select -optional downloads and require `--with-download` (`--download` is an alias). -They do not remove catalog rows. Plain `init` creates a local repository; the +Clone preserves the complete catalog. `download --filter` selects transfers +without removing catalog rows. Plain `init` creates a local repository; the legacy remote `build` command is deprecated in favor of `init --from`. Downloading files alone does not populate the local object store; the scientific workflows show how to add and commit selected local inputs before branching. diff --git a/demo/scientific_workflows_cli.md b/demo/scientific_workflows_cli.md index 466b4a8..b0b72ca 100644 --- a/demo/scientific_workflows_cli.md +++ b/demo/scientific_workflows_cli.md @@ -32,11 +32,23 @@ settings file; they do not duplicate an unchanged scientific input. `init --from` discovers a raw dataset and creates a catalog using directory listings and published metadata. It does not download dataset files. Its -filters select catalog entries. `clone` copies an existing catalog; clone -filters select optional downloads and require `--with-download`. Every nonempty +`--filter` globs select catalog entries. Filename templates are detected from +those paths automatically; `--format` overrides detection without excluding +unmatched files. A single selected file may have no inferable template and is +still cataloged. `clone` copies a complete existing +catalog. Select transfers separately with `download --filter`. Every nonempty CLI transfer displays its plan and prompts for approval. `--dry-run` prepares an offline plan without transferring files. Unknown sizes remain unknown. +For interactive selection, use `hallmark download --interactive`. It accepts +one raw glob per line or an all-files choice, shows recorded sizes, and lets you +download, change the selection, or skip. Initialization never offers downloads. +CLI cloning copies the complete catalog and then reviews a download plan by +default. Use `clone --interactive` for the chooser, `clone --filter` to narrow +transfers, or `clone --no-download` for metadata alone. Without a terminal, clone +keeps the catalog and prints commands for downloading later. +The examples below keep explicit selections so their intended inputs are clear. + Each example uses a working directory containing `.hm`. After download, `hallmark add '{name}'` indexes the selected local input and settings file; `commit` stores their contents so branch checkout can restore them. The catalog diff --git a/demo/scientific_workflows_python.ipynb b/demo/scientific_workflows_python.ipynb index a10b33a..991a41c 100644 --- a/demo/scientific_workflows_python.ipynb +++ b/demo/scientific_workflows_python.ipynb @@ -9,7 +9,9 @@ "\n", "EHT and DESI observations, Roman–Rubin simulations, and local DES Y3 products. The matching [CLI guide](scientific_workflows_cli.md) uses the same selected files; this notebook also shows one worktree.\n", "\n", - "Run **Setup**, then one numbered workflow from start to finish. Each uses a separate directory and one explicitly selected input. Choose a new `ROOT` to repeat an example. `Repo.init(..., from_url=...)` discovers raw datasets without downloading files. `Repo.clone(...)` copies an existing catalog; its filters select optional downloads and preserve the complete catalog and history.\n", + "Run **Setup**, then one numbered workflow from start to finish. Each uses a separate directory and one explicitly selected input. Choose a new `ROOT` to repeat an example. `Repo.init(..., source=...)` discovers raw datasets without downloading files. `Repo.clone(...)` copies an existing catalog; it preserves the complete catalog and history. Use `plan_download(filter=...)` and approve transfers separately.\n", + "\n", + "`Repo.init(..., filter=...)` selects catalog entries. Filename templates are detected automatically from those paths; `format=...` overrides detection. Unmatched files remain cataloged. Selecting only one input may leave too little information to infer a template; its literal path and available metadata are still recorded.\n", "\n", "Every transfer shows a plan and asks for approval. Declining or a failed transfer stops that workflow; do not continue to its versioning cell. The completed catalog remains available. The SSH example requires an existing export and your own access configuration.\n", "\n", @@ -98,7 +100,7 @@ "source": [ "EHT_FILE = \"SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits\"\n", "eht_dir = ROOT / \"eht\"\n", - "eht = Repo.init(eht_dir, from_url=EHT_URL, backend=\"cyverse\", filter=EHT_FILE)\n", + "eht = Repo.init(eht_dir, source=EHT_URL, filter=EHT_FILE)\n", "download_after_review(eht, eht.plan_download(all_files=True))" ] }, @@ -179,7 +181,7 @@ "source": [ "DESI_FILE = \"redrock-main-dark-23040.fits\"\n", "desi_dir = ROOT / \"desi\"\n", - "desi = Repo.init(desi_dir, from_url=DESI_URL, backend=\"http\", filter=DESI_FILE)\n", + "desi = Repo.init(desi_dir, source=DESI_URL, filter=DESI_FILE)\n", "download_after_review(desi, desi.plan_download(all_files=True))" ] }, @@ -236,7 +238,7 @@ "source": [ "VM_FILE = \"pointsource_10307.parquet\"\n", "vm_dir = ROOT / \"roman-rubin\"\n", - "vm = Repo.init(vm_dir, from_url=VM_URL, backend=\"ssh\", filter=VM_FILE)\n", + "vm = Repo.init(vm_dir, source=VM_URL, filter=VM_FILE)\n", "download_after_review(vm, vm.plan_download(all_files=True))" ] }, From 293917443a4658a61c3a80a73d113bd43e16d484 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:14:02 -0700 Subject: [PATCH 19/29] Store each data entry's catalog in its own TSV Prepare branches for several filename templates. The first entry keeps data.tsv, so single-template repositories are unchanged; later entries name their own TSV. Rows are classified as local (committed as objects) or catalog-only (remote entries, path catalogs and builder tables). Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/catalog.py | 19 +- mod/hallmark/dothm.py | 51 +++++- mod/hallmark/repo_config.py | 328 ++++++++++++++++++++++++++++++---- mod/hallmark/repo_manifest.py | 198 ++++++++++++++++++-- mod/hallmark/repo_state.py | 21 ++- mod/hallmark/state.py | 94 ++++++++-- test/test_catalog_entries.py | 180 +++++++++++++++++++ 7 files changed, 802 insertions(+), 89 deletions(-) create mode 100644 test/test_catalog_entries.py diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py index c6de35b..ee18fd6 100644 --- a/mod/hallmark/catalog.py +++ b/mod/hallmark/catalog.py @@ -18,7 +18,8 @@ from .error import CloneError, DestinationExistsError from .fmt_detection import detect_fmt from .helper_functions import as_list_of_dicts -from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name, row_to_path +from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name +from .repo_manifest import row_relative_path from .transport import OperationContext, RemoteSpec from .transport.base import (RemoteObjectMissing, literal_path, thaw_backend_options) @@ -49,18 +50,10 @@ def _catalog_formats(config, name): def _row_path(row, formats): """Resolve a catalog row to one literal relative path.""" - value = row.get("path") - if value is not None and not pd.isna(value) and str(value): - return literal_path(str(value)).as_posix() - paths = [] - for template in formats: - try: - paths.append(row_to_path(row, template).as_posix()) - except (ValueError, KeyError): - continue - if len(set(paths)) != 1: - raise CloneError("Cannot resolve a catalog row to a unique file path") - return paths[0] + try: + return row_relative_path(row, formats).as_posix() + except ValueError as exc: + raise CloneError(str(exc)) from exc def _validate_snapshot(files, config): diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 2ea03c5..fba45bd 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -27,7 +27,8 @@ from .error import CloneError, DothmError from .helper_functions import ( atomic_output_path, load_yaml_file, validate_path_component) -from .state import State +from .repo_config import catalog_table_names +from .state import DEFAULT_DB, State class _HallmarkYamlDumper(yaml.Dumper): @@ -85,7 +86,8 @@ class Dothm(Repo): """Local ``.hm`` storage backend. The backend version controls the hallmark ``State`` database files - (``config.yml``, ``meta.yml``, ``data.tsv``) on-disk. + (``config.yml``, ``meta.yml``, ``data.tsv`` and the catalog TSVs of + additional data entries) on-disk. It is itself a git worktree. """ def _storage_path(self, stem: Union[Path, str], suffix: str) -> Path: @@ -154,7 +156,6 @@ def config_template() -> str: # name: origin # url: https://example.com/path/to/data/ # SSH URL example: ssh://campus/srv/export/ (requires a trusted host key) - # auth: campus # Optional local profile name. Credentials stay local. """ @classmethod @@ -196,16 +197,52 @@ def link(self, path: Union[Path, str], branch: Optional[str] = None): return Dothm(path) def load(self) -> State: - return State( - config = self.load_yml("config"), + config = self.load_yml("config") + state = State( + config = config, meta = self.load_yml("meta"), data = self.load_tsv("data")) + # additional data entries keep their catalogs in their own TSVs + for name in catalog_table_names(config): + if name != DEFAULT_DB: + state.tables[name] = self._load_optional_tsv(name) + return state def dump(self, state: State) -> None: + # write catalogs before the config that references them + names = [DEFAULT_DB] + self.dump_tsv(state.data, "data") + for name, frame in state.tables.items(): + if (name in state.changed_tables + or not self._storage_path(name, ".tsv").exists()): + self.dump_tsv(frame, name) + names.append(name) + for name in sorted(state.removed_tables - set(state.tables) - {DEFAULT_DB}): + self.index.remove([name], ignore_unmatch=True) + self._storage_path(name, ".tsv").unlink(missing_ok=True) self.dump_yml(state.config, "config") self.dump_yml(state.meta, "meta") - self.dump_tsv(state.data, "data") - self.index.add(["config.yml", "meta.yml", "data.tsv"]) + self.index.add(["config.yml", "meta.yml", *names]) + state.changed_tables.clear() + state.removed_tables.clear() + + def _load_optional_tsv(self, stem: Union[Path, str]) -> pd.DataFrame: + """ + Used by load. + Load a catalog TSV, returning an empty table when it has not been written. + + Args: + stem (Union[Path, str]): The TSV file name. + + Returns: + pd.DataFrame: The table, or an empty table with a ``sha1`` column. + """ + if not self._storage_path(stem, ".tsv").is_file(): + return State().data.copy() + try: + return self.load_tsv(stem) + except pd.errors.EmptyDataError: + return State().data.copy() def load_yml(self, stem: Union[Path, str]) -> dict: """ diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index c2f5b94..7428f2e 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -7,14 +7,17 @@ from __future__ import annotations +import re from copy import deepcopy +from dataclasses import dataclass, field from pathlib import Path from string import Formatter -from typing import Dict, Optional +from typing import Dict, Iterable, Optional from .helper_functions import ( as_list_of_dicts, coerce_fmt_value, normalize_nonempty_string, validate_path_component, validate_relative_path) +from .state import DEFAULT_DB from .transport.base import ( RemoteSpec, backend_name, reject_controls, thaw_backend_options) @@ -292,12 +295,254 @@ def single_data_fmt(config: dict) -> Optional[str]: return fmt.strip() -def get_or_create_branch_data_spec(config: dict) -> dict: +@dataclass(frozen=True) +class CatalogEntry: + """ + A data entry that owns a catalog table. + + Attributes: + index (int): Position of the entry in the config ``data`` list. + fmt (str, optional): Filename template relative to the worktree, or to + ``url`` for a remote entry. Builder path catalogs may omit it. + db (str): TSV name of the entry's catalog table. + url (str, optional): Remote directory the template is relative to. + Entries without a URL describe files in the worktree. + backend (str, optional): Registered transport backend for ``url``. + backend_options (dict, optional): Backend-specific configuration. + """ + + index: int + fmt: Optional[str] + db: str + url: Optional[str] = None + backend: Optional[str] = None + backend_options: Optional[dict] = field(default=None, hash=False, compare=False) + + @property + def is_remote(self) -> bool: + """True when the entry catalogs files at a remote URL.""" + return bool(self.url) + + +def _data_entry_list(config) -> list: + """Return the config ``data`` section as a list, or an empty list.""" + if not isinstance(config, dict): + return [] + return as_list_of_dicts(config.get("data")) or [] + + +def is_placeholder(entry) -> bool: + """ + Return True for a data entry that does not catalog files yet, such as the + ``encoding`` placeholder written by ``hallmark init``. + """ + return isinstance(entry, dict) and not {"fmt", "url", "file", "db"} & set(entry) + + +def _entry_fmt(entry: dict) -> Optional[str]: + """Return an entry's stripped template, or None if it has none.""" + fmt = entry.get("fmt") + return fmt.strip() if isinstance(fmt, str) and fmt.strip() else None + + +def catalog_entries(config) -> list[CatalogEntry]: + """ + Return the data entries that own catalog tables, in config order. + + Placeholders and static ``file`` entries are skipped. An entry without an + explicit ``db`` uses ``data.tsv``. + + Args: + config (dict): The repository configuration dictionary. + + Returns: + list[CatalogEntry]: The catalog entries. + + Raises: + ValueError: If an entry names an invalid TSV. + """ + entries = [] + for index, entry in enumerate(_data_entry_list(config)): + if not isinstance(entry, dict): + continue + fmt = _entry_fmt(entry) + if fmt is None and not entry.get("db"): + continue + url = entry.get("url") + entries.append(CatalogEntry( + index=index, + fmt=fmt, + db=normalize_tsv_name(entry["db"]) if entry.get("db") else DEFAULT_DB, + url=url if isinstance(url, str) and url else None, + backend=entry.get("backend"), + backend_options=entry.get("backend_options"))) + return entries + + +def catalog_table_names(config) -> list[str]: + """ + Return the TSV names referenced by catalog entries, skipping invalid names. + + Used when loading a repository, before any operation validates the + configuration. + + Args: + config (dict): The repository configuration dictionary. + + Returns: + list[str]: Unique TSV names in config order. + """ + names = [] + for entry in _data_entry_list(config): + if not isinstance(entry, dict): + continue + if entry.get("db"): + try: + name = normalize_tsv_name(entry["db"]) + except ValueError: + continue + elif _entry_fmt(entry) is not None: + name = DEFAULT_DB + else: + continue + if name not in names: + names.append(name) + return names + + +def find_entry(config, fmt: str) -> Optional[CatalogEntry]: + """ + Return the catalog entry whose template is ``fmt``. + + Args: + config (dict): The repository configuration dictionary. + fmt (str): Filename template. + + Returns: + CatalogEntry | None: The matching entry, or None. + """ + return next((entry for entry in catalog_entries(config) if entry.fmt == fmt), + None) + + +def next_db_name(config, used: Iterable[str] = ()) -> str: + """ + Choose the TSV name for a new data entry. + + The first entry uses ``data.tsv``. Later entries use ``data-N.tsv`` with a + number larger than any in the config or in ``used``, so a removed table's + history is not reused by a different template. + + Args: + config (dict): The repository configuration dictionary. + used (Iterable[str]): TSV names used previously, such as in history. + + Returns: + str: The TSV name for the new entry. + """ + entries = catalog_entries(config) + if all(entry.db != DEFAULT_DB for entry in entries): + return DEFAULT_DB + numbers = [1] + for name in [*(entry.db for entry in entries), *used]: + match = re.fullmatch(r"data-(\d+)\.tsv", str(name), flags=re.IGNORECASE) + if match: + numbers.append(int(match.group(1))) + return f"data-{max(numbers) + 1}.tsv" + + +def add_entry(config: dict, entry: dict) -> int: """ - Ensure the configuration contains a valid data specification. + Add a data entry to the configuration. - If the ``data`` entry is missing or malformed, it is initialized with - a single empty dictionary. + The first entry fills the placeholder written by ``hallmark init`` so that + single-template repositories keep their existing layout. Later entries are + appended. + + Args: + config (dict): The repository configuration dictionary, updated in place. + entry (dict): The new data entry. + + Returns: + int: The entry's position in the config ``data`` list. + + Raises: + ValueError: If the config ``data`` section is malformed. + """ + data = config.get("data") + entries = [] if data is None else as_list_of_dicts(data) + if entries is None: + raise ValueError('config "data" must be a mapping or list of mappings') + entries = list(entries) + config["data"] = entries + if not catalog_entries(config): + for index, existing in enumerate(entries): + if is_placeholder(existing): + merged = dict(entry) + for key, value in existing.items(): + # a remote entry has no use for an empty encoding placeholder + if key in merged or (entry.get("url") and value is None): + continue + merged[key] = value + entries[index] = merged + return index + entries.append(entry) + return len(entries) - 1 + + +def remove_entry(config: dict, index: int) -> dict: + """ + Remove a data entry from the configuration. + + Args: + config (dict): The repository configuration dictionary, updated in place. + index (int): The entry's position in the config ``data`` list. + + Returns: + dict: The removed entry. + """ + entries = list(_data_entry_list(config)) + removed = entries.pop(index) + config["data"] = entries + return removed + + +def sole_data_spec(config: dict, *, create: bool = False) -> Optional[dict]: + """ + Return the data entry that single-template settings apply to. + + This is the only catalog entry, or the placeholder when there is none. + + Args: + config (dict): The repository configuration dictionary. + create (bool): Add an empty entry when none exists. Defaults to False. + + Returns: + dict | None: The data entry, or None when none exists and ``create`` + is False. + + Raises: + RuntimeError: If the branch has more than one catalog entry. + """ + entries = catalog_entries(config) + if len(entries) > 1: + raise RuntimeError( + "this branch tracks several templates; use hallmark add TEMPLATE or " + "hallmark rm --cached TEMPLATE instead of changing config.yml") + data = _data_entry_list(config) + if entries: + return data[entries[0].index] + placeholder = next((entry for entry in data if is_placeholder(entry)), None) + if placeholder is not None or not create: + return placeholder + config["data"] = [*data, {}] + return config["data"][-1] + + +def get_or_create_branch_data_spec(config: dict) -> dict: + """ + Ensure the configuration contains a data specification for single-template + settings and return it. Args: config (dict): Repository configuration. @@ -305,20 +550,13 @@ def get_or_create_branch_data_spec(config: dict) -> dict: Returns: dict: The branch data specification. """ - # get the "data" section from the configuration - data = config.get("data") - # if "data" already defines exactly one dictionary entry, use it as-is - if isinstance(data, list) and len(data) == 1 and isinstance(data[0], dict): - return data[0] - # otherwise (missing, empty, or malformed), initialize it with a single empty dict - config["data"] = [{}] - return config["data"][0] + return sole_data_spec(config, create=True) def require_branch_data_spec(repo) -> dict: """ - Return the branch data specification. Raises RuntimeError if - the configuration does not define exactly one entry under ``data``. + Return the branch data specification. Raises RuntimeError if the + configuration defines no data entry or several catalog entries. Args: repo: Repository object. @@ -326,32 +564,45 @@ def require_branch_data_spec(repo) -> dict: Returns: dict: The branch data specification. """ - # call the helper function to get the single data specification from the config - spec = _data_spec_or_none(repo.state.config) - # raise a RuntimeError if there is no valid single data specification + spec = sole_data_spec(repo.state.config) if spec is None: raise RuntimeError( - 'branch config must define exactly one entry under "data" in config.yml') + 'branch config must define an entry under "data" in config.yml') return spec def branch_fmt(repo) -> str: """ - Return the configured filename format. Raises RuntimeError if - no valid format string is defined. + Return the template of a single-template branch. Raises RuntimeError if + no valid template is defined. Args: repo: Repository object. Returns: - str: The format string stored in ``data[0].fmt``. + str: The format string of the branch's data entry. """ return normalize_nonempty_string( require_branch_data_spec(repo).get("fmt"), - label="branch data[0].fmt", + label="branch data fmt", exception_type=RuntimeError) +def branch_encodings(repo) -> list[dict]: + """ + Return the filename encodings of a single-template branch. + + Args: + repo: Repository object. + + Returns: + list[dict]: A list containing the encoding specification, or an + empty list if no encodings are defined. + """ + spec = require_branch_data_spec(repo) + return [spec] if isinstance(spec.get("encoding"), dict) else [] + + def set_config( repo, *, @@ -424,8 +675,12 @@ def set_config( # if a new format string or encoding updates are provided if fmt is not None or encoding_updates is not None: - # ensure that the "data" section has exactly one entry and retrieve it - spec = get_or_create_branch_data_spec(config) + # retrieve the only data entry; several templates cannot share settings + spec = sole_data_spec(config, create=True) + if fmt is not None and spec.get("url"): + raise ValueError( + "cannot change the template of a remote entry; run " + "hallmark add URL/TEMPLATE instead") updated_spec = {} # if a new format string is provided, update the "fmt" key in the spec if fmt is not None: @@ -453,8 +708,10 @@ def set_config( # exclude "fmt" and "encoding" keys since they are already handled if key not in {"fmt", "encoding"}: updated_spec[key] = value - # update the first entry in the "data" list of the configuration - config["data"][0] = updated_spec + # replace the entry at its position in the "data" list + index = next(position for position, entry in enumerate(config["data"]) + if entry is spec) + config["data"][index] = updated_spec # update the remote when its name, URL, or transport configuration changes if any(value is not None for value in ( @@ -467,19 +724,22 @@ def set_config( return config -def branch_encodings(repo) -> list[dict]: +def entry_encodings(spec: Optional[dict], fmt: str) -> list[dict]: """ - Return the configured filename encodings. + Return the filename encodings that apply to a template. Args: - repo: Repository object. + spec (dict, optional): The data entry or placeholder holding the + encoding rules. + fmt (str): The template being added. Returns: - list[dict]: A list containing the encoding specification, or an - empty list if no encodings are defined. + list[dict]: A list containing the encoding specification for ``fmt``, + or an empty list if no encodings are defined. """ - spec = require_branch_data_spec(repo) - return [spec] if isinstance(spec.get("encoding"), dict) else [] + if not isinstance(spec, dict) or not isinstance(spec.get("encoding"), dict): + return [] + return [{**spec, "fmt": fmt}] def fmt_fields(fmt: str) -> list[str]: diff --git a/mod/hallmark/repo_manifest.py b/mod/hallmark/repo_manifest.py index 0c27019..66a4e7b 100644 --- a/mod/hallmark/repo_manifest.py +++ b/mod/hallmark/repo_manifest.py @@ -1,12 +1,14 @@ from __future__ import annotations +from dataclasses import dataclass from pathlib import Path from collections.abc import Iterator import pandas as pd from .helper_functions import safe_str -from .repo_config import fmt_fields, row_to_path, single_data_fmt +from .repo_config import CatalogEntry, catalog_entries, fmt_fields, row_to_path +from .transport.base import literal_path def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: @@ -48,19 +50,184 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: return pd.DataFrame(rows, columns=columns) +@dataclass(frozen=True) +class CatalogRow: + """ + One cataloged file. + + Attributes: + path (str): POSIX path relative to the worktree and the data source. + record (dict): The catalog row. + db (str): TSV name of the catalog table. + entries (tuple[CatalogEntry, ...]): Data entries sharing the table. + local (bool): True when the file's content is tracked in the worktree + and object store; False for catalog-only rows such as remote files. + """ + + path: str + record: dict + db: str + entries: tuple + local: bool + + +def _cell_text(value) -> str: + """Return a catalog cell as text, treating missing markers as empty.""" + if value is None or (not isinstance(value, str) and pd.isna(value)): + return "" + text = str(value).strip() + return "" if text.lower() in {"none", "nan", ""} else text + + +def row_relative_path(row, formats) -> Path: + """ + Resolve a catalog row to one literal relative path. + + A ``path`` column is authoritative. Otherwise exactly one template must + render the row. + + Args: + row: Catalog row mapping. + formats (list[str]): Templates of the data entries sharing the table. + + Returns: + Path: The relative file path. + + Raises: + ValueError: If the row cannot be resolved to a unique safe path. + """ + value = row.get("path") + if _cell_text(value): + return Path(literal_path(str(value)).as_posix()) + paths = set() + for template in formats: + try: + paths.add(row_to_path(row, template).as_posix()) + except (ValueError, KeyError): + continue + if len(paths) != 1: + raise ValueError("Cannot resolve a catalog row to a unique file path") + return Path(paths.pop()) + + +def table_groups(config) -> dict[str, tuple[CatalogEntry, ...]]: + """ + Group catalog entries by the TSV that holds their rows. + + Args: + config (dict): The repository configuration dictionary. + + Returns: + dict[str, tuple[CatalogEntry, ...]]: Entries keyed by TSV name. + """ + groups: dict[str, list[CatalogEntry]] = {} + for entry in catalog_entries(config): + groups.setdefault(entry.db, []).append(entry) + return {name: tuple(entries) for name, entries in groups.items()} + + +def is_local_table(entries, frame: pd.DataFrame) -> bool: + """ + Return True when a table's rows describe worktree files stored as objects. + + Remote entries, path catalogs, and builder tables shared by several + templates are catalog-only: their files are not committed as objects, + restored by checkout, or checked in the worktree. + + Args: + entries (tuple[CatalogEntry, ...]): Data entries sharing the table. + frame (pd.DataFrame): The catalog table. + + Returns: + bool: True for a local table. + """ + return (len(entries) == 1 and entries[0].fmt is not None + and not entries[0].is_remote + and "sha1" in frame.columns and "path" not in frame.columns) + + +def iter_catalog_rows(state, *, local: bool | None = None) -> Iterator[CatalogRow]: + """ + Iterate over the files cataloged by every data entry. + + Args: + state: Repository state. + local (bool, optional): True for local rows only, False for + catalog-only rows only, or None for all rows. + + Yields: + CatalogRow: Each cataloged file. + + Raises: + ValueError: If a row cannot be resolved to a safe relative path. + """ + for name, entries in table_groups(state.config).items(): + frame = state.table(name) + if frame.empty: + continue + local_table = is_local_table(entries, frame) + if local is not None and local_table != local: + continue + formats = [entry.fmt for entry in entries if entry.fmt] + for record in frame.to_dict(orient="records"): + if local_table: + path = row_to_path(record, formats[0]) + else: + path = row_relative_path(record, formats) + yield CatalogRow(path.as_posix(), record, name, entries, local_table) + + +def row_fingerprint(record) -> str: + """ + Summarize the recorded content of a catalog row. + + The SHA-1 is preferred, then a published checksum, then size and + modification time. + + Args: + record: Catalog row mapping. + + Returns: + str: A comparable fingerprint. + """ + sha1 = _cell_text(record.get("sha1")) + if sha1: + return sha1.lower() + checksum = _cell_text(record.get("checksum")) + if checksum: + algorithm = _cell_text(record.get("checksum_algorithm")) or "unknown" + return f"{algorithm.lower()}:{checksum.lower()}" + size = _cell_text(record.get("size_bytes")) + mtime = _cell_text(record.get("mtime")) + return f"size:{size};mtime:{mtime}" + + +def catalog_map(state) -> dict[str, CatalogRow]: + """ + Map every cataloged path to its catalog row. + + Args: + state: Repository state. + + Returns: + dict[str, CatalogRow]: Rows keyed by relative POSIX path. + """ + return {row.path: row for row in iter_catalog_rows(state)} + + def iter_manifest_entries( state, *, fmt: str | None = None, ) -> Iterator[tuple[Path, str]]: """ - Iterate over the manifest entries in the repository state. + Iterate over the local manifest entries in the repository state. Args: state: Repository state. - fmt (str | None): Optional filename format to use for parsing paths. - If not provided, the format will be determined from the repository - configuration. + fmt (str | None): Optional filename format applied to ``data.tsv``. + If not provided, the local catalog entries of the repository + configuration are used. Returns: Iterator[tuple[Path, str]]: An iterator over tuples containing the @@ -69,16 +236,13 @@ def iter_manifest_entries( Yields: tuple[Path, str]: Tuples of the relative file path and its SHA-1 checksum. """ + if fmt is None: + for row in iter_catalog_rows(state, local=True): + yield Path(row.path), str(row.record["sha1"]) + return # If the repository state has no data, return immediately if state.data.empty: return - # If no format is provided, determine it from the repository configuration. - if fmt is None: - # get the single data format from the repository configuration - fmt = single_data_fmt(state.config) - # if the format is still None, return immediately - if fmt is None: - return # convert each record in the repository state to a dictionary for record in state.data.to_dict(orient="records"): # yield a tuple containing the relative file path and its SHA-1 checksum @@ -87,13 +251,13 @@ def iter_manifest_entries( def manifest_map(state, *, fmt: str | None = None) -> dict[str, str]: """ - Create a mapping from file paths to SHA-1 checksums. + Create a mapping from local file paths to SHA-1 checksums. Args: state: Repository state. - fmt (str | None): Optional filename format to use for parsing paths. - If not provided, the format will be determined from the repository - configuration. + fmt (str | None): Optional filename format applied to ``data.tsv``. + If not provided, the local catalog entries of the repository + configuration are used. Returns: dict[str, str]: Dictionary mapping relative file paths to their @@ -101,4 +265,4 @@ def manifest_map(state, *, fmt: str | None = None) -> dict[str, str]: """ # call iter_manifest_entries to get an iterator of (path, checksum) tuples return {path.as_posix(): checksum - for path, checksum in iter_manifest_entries(state, fmt=fmt)} \ No newline at end of file + for path, checksum in iter_manifest_entries(state, fmt=fmt)} diff --git a/mod/hallmark/repo_state.py b/mod/hallmark/repo_state.py index 340a021..c24ef4a 100644 --- a/mod/hallmark/repo_state.py +++ b/mod/hallmark/repo_state.py @@ -10,7 +10,8 @@ from .helper_functions import load_yaml from .objects import Objects -from .state import State +from .repo_config import catalog_table_names +from .state import DEFAULT_DB, State def _load_revision_yaml( repo, @@ -75,10 +76,13 @@ def _copy_current_state(repo, *, include_data: bool) -> State: """ # if include_data is True, copy current data; otherwise, create an empty DataFrame data = (repo.state.data.copy() if include_data else State().data.copy()) + tables = ({name: frame.copy() for name, frame in repo.state.tables.items()} + if include_data else {}) # new State object with deep copies of current config and meta, and the copied data return State( - config=deepcopy(repo.state.config), meta=deepcopy(repo.state.meta), data=data) + config=deepcopy(repo.state.config), meta=deepcopy(repo.state.meta), data=data, + tables=tables) def _load_revision_state(repo, revision: str) -> State: @@ -97,10 +101,21 @@ def _load_revision_state(repo, revision: str) -> State: data_text = repo.dothm.git.show(f"{revision}:data.tsv") # Load the config.yml and meta.yml content from the specified revision, # falling back to current state if not found - return State( + state = State( config=_load_revision_yaml(repo, revision, "config.yml", repo.state.config), meta=_load_revision_yaml(repo, revision, "meta.yml", repo.state.meta), data=_parse_data_tsv(data_text)) + # load the catalogs of additional data entries named by that revision's config + for name in catalog_table_names(state.config): + if name == DEFAULT_DB: + continue + try: + state.tables[name] = _parse_data_tsv( + repo.dothm.git.show(f"{revision}:{name}")) + # a catalog staged but never committed is empty at this revision + except GitCommandError: + state.tables[name] = State().data.copy() + return state def load_branch_data(repo, branch: str) -> State: diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 78c72d2..9b55cb5 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -19,6 +19,10 @@ # Define the default columns for the state DataFrame COLUMNS = ["sha1"] +# The catalog table of the first data entry; other entries name their own TSV. +DEFAULT_DB = "data.tsv" +# Remote file metadata columns. They describe a row but do not identify it. +METADATA_COLUMNS = ("checksum_algorithm", "checksum", "size_bytes", "mtime") def _normalized_state_data(frame: pd.DataFrame) -> pd.DataFrame: @@ -50,6 +54,9 @@ def _normalized_state_data(frame: pd.DataFrame) -> pd.DataFrame: # normalize the data by retaining only the relevant columns normalized = frame.loc[:, columns].copy() + # remote rows without a published SHA-1 store an empty value, not NaN + normalized["sha1"] = normalized["sha1"].map( + lambda value: "" if pd.isna(value) else value) for column in parameter_columns: # normalize non-checksum columns to string type, # replacing NaN values with empty strings @@ -67,8 +74,12 @@ class State: Attributes: config: Repository configuration values. meta: Repository metadata. - data: Tabular file index containing indexed object checksums - (``sha1``) and associated metadata. + data: Tabular file index of the default catalog (``data.tsv``), + containing indexed object checksums (``sha1``) and associated + metadata. + tables: Catalog tables of additional data entries, keyed by TSV name. + changed_tables: Additional tables modified since they were last written. + removed_tables: Additional tables to delete when the state is written. """ config: dict = field(default_factory=dict) @@ -76,31 +87,83 @@ class State: data: pd.DataFrame = field( default_factory=lambda: pd.DataFrame(columns=COLUMNS) ) + tables: dict = field(default_factory=dict) + changed_tables: set = field(default_factory=set, repr=False, compare=False) + removed_tables: set = field(default_factory=set, repr=False, compare=False) - def update(self, pf) -> None: + def table(self, db: str = DEFAULT_DB) -> pd.DataFrame: """ - Merge ``ParaFrame`` rows into the state database. + Return a catalog table, or an empty table if it has no rows yet. + + Args: + db (str): TSV name of the table. Defaults to ``data.tsv``. + + Returns: + pd.DataFrame: The catalog table. + """ + if db == DEFAULT_DB: + return self.data + frame = self.tables.get(db) + return frame if frame is not None else pd.DataFrame(columns=COLUMNS) + + def set_table(self, db: str, frame: pd.DataFrame) -> None: + """ + Replace a catalog table and mark it for writing. + + Args: + db (str): TSV name of the table. + frame (pd.DataFrame): The new table contents. + """ + if db == DEFAULT_DB: + self.data = frame + return + self.tables[db] = frame + self.changed_tables.add(db) + self.removed_tables.discard(db) + + def drop_table(self, db: str) -> None: + """ + Remove a catalog table. ``data.tsv`` is emptied rather than deleted, + because every Hallmark repository contains it. + + Args: + db (str): TSV name of the table. + """ + if db == DEFAULT_DB: + self.data = pd.DataFrame(columns=COLUMNS) + return + self.tables.pop(db, None) + self.changed_tables.discard(db) + self.removed_tables.add(db) + + def update(self, pf, db: str = DEFAULT_DB) -> None: + """ + Merge ``ParaFrame`` rows into a catalog table. Existing rows with matching keys are updated, while new rows are - appended. + appended. Checksums and remote file metadata are not part of the key. Args: pf (ParaFrame): ``ParaFrame`` containing rows to add or update. + db (str): TSV name of the table. Defaults to ``data.tsv``. Returns: None. """ + current = self.table(db) if pf.empty: - # create empty DataFrame with the same columns as the existing state data. - columns = (self.data.columns if len(self.data.columns) else COLUMNS) + # create empty DataFrame with the same columns as the existing table. + columns = (current.columns if len(current.columns) else COLUMNS) incoming = pd.DataFrame(columns=columns) # if the provided ParaFrame is not empty, normalize its data else: incoming = _normalized_state_data(pf) # Merge the incoming rows with the existing state. - merged = pd.concat([self.data, incoming], ignore_index=True, sort=False) + merged = pd.concat([current, incoming], ignore_index=True, sort=False) - key_columns = [column for column in merged.columns if column != "sha1"] + value_columns = [column for column in merged.columns if column != "sha1"] + key_columns = [column for column in value_columns + if column not in METADATA_COLUMNS] if key_columns: # Remove duplicate entries while keeping the most recent row. deduped = merged.drop_duplicates(subset=key_columns, keep="last") @@ -109,22 +172,23 @@ def update(self, pf) -> None: deduped = merged.tail(1) # reset the index of the deduplicated DataFrame - # retain only the "sha1" and key columns. - self.data = ( - deduped.loc[:, ["sha1", *key_columns]].reset_index(drop=True)) + # retain the "sha1" column followed by the other columns. + self.set_table( + db, deduped.loc[:, ["sha1", *value_columns]].reset_index(drop=True)) - def replace(self, pf) -> None: + def replace(self, pf, db: str = DEFAULT_DB) -> None: """ - Replace the contents of the state database. + Replace the contents of a catalog table. Existing rows are discarded and replaced with the rows from the provided ``ParaFrame``. Args: pf (ParaFrame): ``ParaFrame`` containing the replacement rows. + db (str): TSV name of the table. Defaults to ``data.tsv``. Returns: None. """ # call the normalization function to ensure consistent data types and structure - self.data = _normalized_state_data(pf) + self.set_table(db, _normalized_state_data(pf)) diff --git a/test/test_catalog_entries.py b/test/test_catalog_entries.py new file mode 100644 index 0000000..23fcf36 --- /dev/null +++ b/test/test_catalog_entries.py @@ -0,0 +1,180 @@ +"""Test catalogs with several data entries, each stored in its own TSV.""" + +from pathlib import Path + +import pandas as pd +import pytest +import yaml + +from hallmark import Repo +from hallmark.dothm import Dothm +from hallmark.repo_config import ( + add_entry, catalog_entries, catalog_table_names, next_db_name, remove_entry, + set_config, sole_data_spec) +from hallmark.repo_manifest import ( + catalog_map, iter_catalog_rows, manifest_map, row_fingerprint) +from hallmark.repo_state import load_head_state +from hallmark.state import State + + +def _template_config(): + return yaml.safe_load(Dothm.config_template()) + + +def test_first_entry_fills_the_init_placeholder(): + config = _template_config() + assert catalog_entries(config) == [] + assert add_entry(config, {"fmt": "a{a}.h5"}) == 0 + assert config["data"] == [{"fmt": "a{a}.h5", "encoding": None}] + assert add_entry(config, {"fmt": "{run}.dat", "db": "data-2.tsv"}) == 1 + assert [entry.db for entry in catalog_entries(config)] == [ + "data.tsv", "data-2.tsv"] + + +def test_remote_entry_drops_the_empty_encoding_placeholder(): + config = _template_config() + add_entry(config, {"fmt": "{src}.h5", "url": "https://example.test/er2/"}) + assert config["data"] == [ + {"fmt": "{src}.h5", "url": "https://example.test/er2/"}] + assert catalog_entries(config)[0].is_remote + + +def test_catalog_entries_skip_placeholders_and_static_files(): + config = {"data": [{"encoding": None}, {"file": "README.md", "md5": "0" * 32}, + {"fmt": "a{a}.h5", "db": "a"}, {"db": "paths"}]} + entries = catalog_entries(config) + assert [(entry.index, entry.fmt, entry.db) for entry in entries] == [ + (2, "a{a}.h5", "a.tsv"), (3, None, "paths.tsv")] + assert catalog_table_names(config) == ["a.tsv", "paths.tsv"] + + +def test_catalog_table_names_skip_invalid_names(): + assert catalog_table_names({"data": [{"fmt": "x{a}", "db": "../x"}]}) == [] + + +def test_next_db_name_never_reuses_a_number(): + config = {"data": [{"fmt": "a{a}.h5"}]} + assert next_db_name({"data": []}) == "data.tsv" + assert next_db_name(config) == "data-2.tsv" + assert next_db_name(config, used=["data-2.tsv", "DATA-7.TSV"]) == "data-8.tsv" + add_entry(config, {"fmt": "{b}.dat", "db": "data-3.tsv"}) + remove_entry(config, 0) + # data.tsv always exists, so a later first entry takes it back + assert next_db_name(config) == "data.tsv" + + +def test_single_template_settings_refuse_several_entries(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "a{a}.h5"}, + {"fmt": "{b}.dat", "db": "data-2.tsv"}] + with pytest.raises(RuntimeError, match="several templates"): + set_config(repo, fmt="c{c}.h5") + with pytest.raises(RuntimeError, match="several templates"): + sole_data_spec(repo.state.config) + + +def test_set_config_refuses_to_change_a_remote_template(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "{a}.h5", "url": "https://example.test/"}] + with pytest.raises(ValueError, match="remote entry"): + set_config(repo, fmt="{b}.h5") + + +def test_set_config_updates_the_sole_entry_in_place(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"file": "README.md"}, {"fmt": "a{a}.h5"}] + set_config(repo, fmt="b{b}.h5", encoding_updates={"b": "x(.*)"}) + assert repo.state.config["data"] == [ + {"file": "README.md"}, {"fmt": "b{b}.h5", "encoding": {"b": "x(.*)"}}] + + +def test_tables_round_trip_through_dump_load_and_history(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [ + {"fmt": "a{a}.h5", "encoding": None}, + {"fmt": "{src}.h5", "db": "data-2.tsv", "url": "https://example.test/"}] + repo.state.replace(pd.DataFrame({"sha1": ["1" * 40], "a": ["0"]})) + remote = pd.DataFrame({"sha1": [None], "checksum_algorithm": ["md5"], + "checksum": ["f" * 32], "size_bytes": ["7"], + "mtime": [""], "src": ["M87"]}) + repo.state.replace(remote, db="data-2.tsv") + repo.dothm.dump(repo.state) + repo.dothm.index.commit("Two templates") + + reopened = Repo(tmp_path / "repo") + assert reopened.state.data["a"].tolist() == ["0"] + table = reopened.state.table("data-2.tsv") + assert table.to_dict("records") == [{ + "sha1": "", "checksum_algorithm": "md5", "checksum": "f" * 32, + "size_bytes": "7", "mtime": "", "src": "M87"}] + head = load_head_state(reopened) + assert head.table("data-2.tsv")["src"].tolist() == ["M87"] + + +def test_a_configured_table_that_was_never_written_loads_empty(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"db": "later.tsv"}] + repo.dothm.dump_yml(repo.state.config, "config") + assert Repo(tmp_path / "repo").state.table("later.tsv").empty + + +def test_dropped_tables_are_deleted_and_unstaged(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "a{a}.h5"}, + {"fmt": "{b}.dat", "db": "data-2.tsv"}] + repo.state.replace(pd.DataFrame({"sha1": ["2" * 40], "b": ["1"]}), + db="data-2.tsv") + repo.dothm.dump(repo.state) + assert (repo.dothm.path / "data-2.tsv").is_file() + remove_entry(repo.state.config, 1) + repo.state.drop_table("data-2.tsv") + repo.dothm.dump(repo.state) + assert not (repo.dothm.path / "data-2.tsv").exists() + assert ("data-2.tsv", 0) not in repo.dothm.index.entries + + +def test_update_ignores_metadata_columns_when_matching_rows(): + state = State() + old = pd.DataFrame({"sha1": [""], "checksum": ["a" * 32], "size_bytes": ["1"], + "name": ["x"]}) + new = old.assign(checksum="b" * 32, size_bytes="2") + state.update(old) + state.update(new) + assert state.data.to_dict("records") == [ + {"sha1": "", "checksum": "b" * 32, "size_bytes": "2", "name": "x"}] + + +def test_rows_of_remote_path_and_shared_tables_are_catalog_only(): + state = State(config={"data": [ + {"fmt": "a{a}.h5"}, + {"fmt": "{src}.h5", "db": "remote.tsv", "url": "https://example.test/"}, + {"fmt": "x{x}.fits", "db": "built.tsv"}, + {"fmt": "y{y}.fits", "db": "built.tsv"}]}, + data=pd.DataFrame({"sha1": ["1" * 40], "a": ["0"]})) + state.tables["remote.tsv"] = pd.DataFrame({"sha1": [""], "src": ["M87"]}) + state.tables["built.tsv"] = pd.DataFrame({ + "path": ["x1.fits", "y2.fits"], "checksum": ["None", "c" * 32], + "x": ["1", "None"], "y": ["None", "2"]}) + rows = {row.path: row.local for row in iter_catalog_rows(state)} + assert rows == {"a0.h5": True, "M87.h5": False, "x1.fits": False, + "y2.fits": False} + assert manifest_map(state) == {"a0.h5": "1" * 40} + assert set(catalog_map(state)) == set(rows) + + +def test_row_fingerprint_prefers_sha1_then_checksum_then_size(): + assert row_fingerprint({"sha1": "ABC", "checksum": "d"}) == "abc" + assert row_fingerprint({"sha1": "", "checksum_algorithm": "MD5", + "checksum": "D"}) == "md5:d" + assert row_fingerprint({"checksum": "None", "size_bytes": "3", + "mtime": None}) == "size:3;mtime:" + + +def test_single_template_repository_layout_is_unchanged(tmp_path): + repo = Repo.init(tmp_path / "repo") + (repo.worktree / "a1.h5").write_text("one\n", encoding="utf-8") + repo.add("a{a}.h5") + assert sorted(path.name for path in repo.dothm.path.glob("*.tsv")) == [ + "data.tsv"] + assert Path(repo.dothm.path / "config.yml").read_text(encoding="utf-8") == ( + "data:\n- fmt: a{a}.h5\n encoding: null\nremote: null\n") From 0548d7ab3ac729bf5a313f94c197c2cae107359e Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:18:21 -0700 Subject: [PATCH 20/29] Handle every template in status, commit and checkout Commit stores objects for local templates only; remote and other catalog-only rows are committed as metadata. Status summarizes their staged changes per template and no longer reports downloaded or absent copies. Checkout restores and removes local files only, keeping a local file when the target catalogs identical content remotely and refusing to overwrite a different downloaded copy. Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/cli.py | 8 ++ mod/hallmark/repo.py | 157 ++++++++++++++++++++++------------ mod/hallmark/repo_config.py | 8 +- mod/hallmark/repo_worktree.py | 18 ++-- test/test_catalog_entries.py | 149 ++++++++++++++++++++++++++++++++ test/test_hallmark.py | 8 +- 6 files changed, 278 insertions(+), 70 deletions(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 7ae5944..780129c 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -437,6 +437,13 @@ def emit_section(title, entries, fg): for path in paths: click.echo(" " + click.style(f"{label}: {path}", fg=fg)) + catalog_lines = [] + for change in staged.get("catalog", []): + source = f' from {change["url"]}' if change["url"] else "" + catalog_lines.append( + f'{", ".join(change["templates"])}{source}: ' + f'{change["added"]} new, {change["modified"]} modified, ' + f'{change["deleted"]} removed') emit_section( "Changes to be committed:", [ @@ -444,6 +451,7 @@ def emit_section(title, entries, fg): ("new file", staged["added"]), ("modified", staged["modified"]), ("deleted", staged["deleted"]), + ("catalog", catalog_lines), ], "green", ) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 8532f8e..5a880a0 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -28,7 +28,9 @@ from .worktree import Worktree from .objects import Objects from .paraframe import ParaFrame -from .repo_manifest import manifest_frame_from_pf, manifest_map, iter_manifest_entries +from .repo_manifest import ( + catalog_map, iter_catalog_rows, manifest_frame_from_pf, row_fingerprint, + table_groups) from .repo_state import ( load_branch_data, load_head_state, find_remote_branch, fetch_missing_objects_from_remote) @@ -48,9 +50,43 @@ branch_encodings, branch_fmt, normalize_remotes, - row_to_path, - set_config, - single_data_fmt) + set_config) + +def _catalog_changes(head_state: State, state: State) -> list[dict]: + """ + Used by status. + Summarize staged changes to catalog-only files for each catalog table. + + Args: + head_state (State): The committed state. + state (State): The staged state. + + Returns: + list[dict]: For each changed table, its ``templates``, ``url`` and + counts of ``added``, ``modified`` and ``deleted`` rows. + """ + def rows_by_table(source): + tables: dict[str, dict[str, str]] = {} + for row in iter_catalog_rows(source, local=False): + tables.setdefault(row.db, {})[row.path] = row_fingerprint(row.record) + return tables + + head_tables, staged_tables = rows_by_table(head_state), rows_by_table(state) + groups = {**table_groups(head_state.config), **table_groups(state.config)} + changes = [] + for name, entries in groups.items(): + head, staged = head_tables.get(name, {}), staged_tables.get(name, {}) + counts = { + "added": sum(path not in head for path in staged), + "modified": sum(path in head and head[path] != fingerprint + for path, fingerprint in staged.items()), + "deleted": sum(path not in staged for path in head)} + if any(counts.values()): + changes.append({ + "templates": [entry.fmt or name for entry in entries], + "url": entries[0].url, **counts}) + return changes + @dataclass(init=False) class Repo: @@ -409,19 +445,29 @@ def status(self) -> dict[str, object]: Return repository status information. Includes staged changes, worktree modifications, deletions, and untracked files. + Local files are listed individually. Changes to catalog-only files, + such as those of a remote template, are summarized per template. + Their absent or downloaded copies are never reported as deleted, + modified, or untracked. + Args: self: Repository instance. Returns: dict[str, object]: Status summary including: - branch (str) - - staged changes (dict) + - staged changes (dict), including per-template ``catalog`` + summaries - worktree changes (dict) - untracked files (list[str]) """ head_state = load_head_state(self) - head_map = manifest_map(head_state) - staged_map = manifest_map(self.state) + head_rows = catalog_map(head_state) + staged_rows = catalog_map(self.state) + head_map = {path: row_fingerprint(row.record) + for path, row in head_rows.items() if row.local} + staged_map = {path: row_fingerprint(row.record) + for path, row in staged_rows.items() if row.local} state_changes = sorted({ diff.a_path or diff.b_path for diff in self.dothm.index.diff("HEAD") @@ -437,18 +483,20 @@ def status(self) -> dict[str, object]: worktree_modified: list[str] = [] worktree_deleted: list[str] = [] - staged_paths = set(staged_map) # If the repository has a worktree, check for modified and missing tracked files if self.worktree is not None: - worktree_modified, worktree_deleted = worktree_changes(self, staged_map) + local_checksums = {path: str(row.record["sha1"]) + for path, row in staged_rows.items() if row.local} + worktree_modified, worktree_deleted = worktree_changes( + self, local_checksums) worktree_root = Path(self.worktree) # generator that yields relative paths of all files in the worktree worktree_files = (full_path.relative_to(worktree_root).as_posix() for full_path in iter_repository_files(worktree_root)) - # filter out staged paths from the worktree files + # cataloged files, including downloaded copies, are not untracked untracked = sorted(path for path in worktree_files - if path not in staged_paths) + if path not in staged_rows) else: untracked = [] @@ -459,6 +507,7 @@ def status(self) -> dict[str, object]: "added": staged_added, "modified": staged_modified, "deleted": staged_deleted, + "catalog": _catalog_changes(head_state, self.state), }, "worktree": { "modified": sorted(worktree_modified), @@ -539,27 +588,19 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: if (not allow_empty and not self.dothm.index.diff("HEAD")): # return early since there are no changes to commit return False - # get the current format string and the HEAD state of the repository - current_fmt = branch_fmt(self) head_state = load_head_state(self) - # get the format string of the HEAD state for comparison - head_fmt = single_data_fmt(head_state.config) - # head entries are the set of (path, sha1) tuples from the HEAD state - head_entries: set[tuple[Path, str]] = set() - - if head_fmt == current_fmt: - # populate head_entries with the paths and checksums from the HEAD state - head_entries = {(relative_path, checksum.lower()) - for relative_path, checksum - in iter_manifest_entries(head_state, fmt=head_fmt)} + # head entries are the set of (path, sha1) tuples of HEAD's local files + head_entries = {(row.path, str(row.record["sha1"]).lower()) + for row in iter_catalog_rows(head_state, local=True)} # list of tuples containing (full path, expected sha1) for stored files files_to_store: list[tuple[Path, str]] = [] - # for each entry in the current manifest - for relative_path, checksum in iter_manifest_entries( - self.state, fmt=current_fmt): + # for each local file in the current catalog; catalog-only files, such + # as remote files, are committed as metadata without objects + for row in iter_catalog_rows(self.state, local=True): + relative_path = row.path # get the expected SHA1 checksum for the file - expected_sha1 = checksum.lower() + expected_sha1 = str(row.record["sha1"]).lower() # current_entry is a tuple of (relative path, expected sha1) for this file current_entry = (relative_path, expected_sha1) # if the manifest entry is not in the HEAD entries @@ -639,18 +680,24 @@ def checkout(self, target_branch: str) -> bool: create_new_branch = not has_local and not has_remote current_tracked = tracked_paths(self) target_state = load_branch_data(self, target_branch) - # Get the data format string for the target branch configuration - target_fmt = single_data_fmt(target_state.config) - if target_fmt is None: - # Raise an error if the target branch does not meet the expected criteria - raise CheckoutError( - "checkout currently supports only repositories with " - "one data format and data.tsv") - # list of (relative path, sha1) tuples for all entries in the target branch - target_entries_raw = list(iter_manifest_entries(target_state, fmt=target_fmt)) + # checkout restores and removes local files only; catalog-only files, + # such as downloaded copies of remote files, are left in place + try: + target_rows = list(iter_catalog_rows(target_state)) + current_rows = list(iter_catalog_rows(self.state)) + except ValueError as exc: + raise CheckoutError(str(exc)) from exc + # list of (relative path, sha1) tuples for the target branch's local files + target_entries_raw = [(Path(row.path), str(row.record["sha1"])) + for row in target_rows if row.local] # get the set of relative paths for all tracked files in the target branch target_tracked = {path for path, _ in target_entries_raw} + # the target's catalog-only files and the current branch's downloaded copies + target_catalog_sha = {Path(row.path): str(row.record.get("sha1", "")).lower() + for row in target_rows if not row.local} + current_catalog_only = {Path(row.path) for row in current_rows + if not row.local} # try to identify any missing objects in the target branch that are not present # in the object store @@ -707,18 +754,26 @@ def checkout(self, target_branch: str) -> bool: for (rel_path, full_path, expected_sha1) in conflict_candidates: # if the computed checksum does not match the expected checksum if (conflict_checksums[full_path] != expected_sha1): + if rel_path in current_catalog_only: + raise CheckoutError( + f'target tracked path "{rel_path}" conflicts with ' + "a downloaded copy cataloged by the current branch") raise CheckoutError( f'target tracked path "{rel_path}" ' "already exists as an untracked file") # Store the name of the currently active branch before switching original_branch = self.dothm.active_branch.name - # Get the current format string from the branch configuration - current_fmt = branch_fmt(self) - # Create a mapping of current tracked paths to their SHA1 checksums + # Create a mapping of current local tracked paths to their SHA1 checksums current_sha_by_path = { - row_to_path(row, current_fmt): str(row["sha1"]).lower() - for _, row in self.state.data.iterrows()} + Path(row.path): str(row.record["sha1"]).lower() + for row in current_rows if row.local} + # a local file that the target catalogs remotely with the same content + # stays in place instead of being removed and downloaded again + kept_paths = { + path for path in current_tracked - target_tracked + if target_catalog_sha.get(path) + and target_catalog_sha[path] == current_sha_by_path.get(path)} # tuples of (relative path, sha1) for all files in the target branch target_entries = [(path, sha1.lower()) for path, sha1 in target_entries_raw] # changed_target_entries are the files in the target branch that have different @@ -789,7 +844,7 @@ def remove_empty_parents(path: Path) -> None: # paths that are either currently tracked but not in the target branch # or paths that are tracked but have changed from the current branch affected_paths = sorted((current_tracked - target_tracked - ) | changed_target_paths, + - kept_paths) | changed_target_paths, key=lambda path: (len(path.parts), path.as_posix()), reverse=True) for relative_path in affected_paths: # absolute path in the worktree for the affected relative path @@ -920,13 +975,10 @@ def add_worktree(self, target_branch: str) -> bool: else: target_state = load_head_state(self) - # get the target fmt from the target branch configuration - target_fmt = single_data_fmt(target_state.config) - # if target branch does not have exactly one data format, raise a RuntimeError - if target_fmt is None: - raise RuntimeError( - "add_worktree requires exactly one data entry " - "with a non-empty fmt") + # the new worktree receives the target's local files; catalog-only + # files, such as remote files, are downloaded separately + target_local = [(row.path, str(row.record["sha1"])) + for row in iter_catalog_rows(target_state, local=True)] # if the target worktree does not exist if not target_dothm.exists(): @@ -934,7 +986,7 @@ def add_worktree(self, target_branch: str) -> bool: # the object store try: missing_objects = self.objects.missing( - row["sha1"] for _, row in target_state.data.iterrows()) + sha1 for _, sha1 in target_local) # if a ValueError occurs during object existence check, raise cleanly except ValueError as exc: raise FileNotFoundError(str(exc)) from exc @@ -963,8 +1015,7 @@ def add_worktree(self, target_branch: str) -> bool: # iterate over the target state data and restore files from the object store try: - for _, row in target_state.data.iterrows(): - rel_path = row_to_path(row, target_fmt) + for rel_path, sha1 in target_local: # resolve relative path to an absolute path in the target worktree destination = resolve_contained_path( target, @@ -973,7 +1024,7 @@ def add_worktree(self, target_branch: str) -> bool: # ensure the parent directory exists before restoring the file destination.parent.mkdir(parents=True, exist_ok=True) # restore the file from the object store using its SHA-1 checksum - self.objects.restore(row["sha1"], destination) + self.objects.restore(sha1, destination) # if any exception occurs during file restoration, attempt a clean up except Exception as exc: diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 7428f2e..47e7a2c 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -359,12 +359,16 @@ def catalog_entries(config) -> list[CatalogEntry]: list[CatalogEntry]: The catalog entries. Raises: - ValueError: If an entry names an invalid TSV. + ValueError: If the data section is malformed or an entry names an + invalid TSV. """ + data = config.get("data") if isinstance(config, dict) else None + if data is not None and as_list_of_dicts(data) is None: + raise ValueError('config "data" must be a mapping or list of mappings') entries = [] for index, entry in enumerate(_data_entry_list(config)): if not isinstance(entry, dict): - continue + raise ValueError(f"config data entry {index} must be a mapping") fmt = _entry_fmt(entry) if fmt is None and not entry.get("db"): continue diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index 254d2c1..c4fd74f 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -5,7 +5,6 @@ from .error import CheckoutError from .helper_functions import ( SymlinkPathError, resolve_contained_path, validate_relative_path) -from .repo_config import branch_fmt from .repo_manifest import manifest_map, iter_manifest_entries @@ -85,13 +84,12 @@ def tracked_paths(repo) -> set[Path]: repo (Repo): Repository object. Returns: - set[Path]: Paths of all files tracked in the current - repository state. + set[Path]: Paths of all local files tracked in the current + repository state. Catalog-only files, such as remote files, are + not managed in the worktree and are excluded. ''' - # call branch_fmt to get the filename format from the repository configuration - fmt = branch_fmt(repo) - # use iter_manifest_entries to iterate over the manifest entries and collect paths - return {path for path, _ in iter_manifest_entries(repo.state, fmt=fmt)} + # use iter_manifest_entries to iterate over the local entries and collect paths + return {path for path, _ in iter_manifest_entries(repo.state)} def worktree_changes(repo, expected_checksums: dict[str, str] @@ -168,10 +166,8 @@ def ensure_clean_tracked_files(repo) -> None: if repo.worktree is None: raise CheckoutError("cannot checkout without a worktree") - # determine the filename format from the repository configuration - fmt = branch_fmt(repo) - # get the expected checksums for all tracked files in the repository - expected_checksums = manifest_map(repo.state, fmt=fmt) + # get the expected checksums for all local tracked files in the repository + expected_checksums = manifest_map(repo.state) # try to get the modified and missing tracked files using worktree_changes try: modified, missing = worktree_changes(repo, expected_checksums) diff --git a/test/test_catalog_entries.py b/test/test_catalog_entries.py index 23fcf36..c5c6d91 100644 --- a/test/test_catalog_entries.py +++ b/test/test_catalog_entries.py @@ -178,3 +178,152 @@ def test_single_template_repository_layout_is_unchanged(tmp_path): "data.tsv"] assert Path(repo.dothm.path / "config.yml").read_text(encoding="utf-8") == ( "data:\n- fmt: a{a}.h5\n encoding: null\nremote: null\n") + + +REMOTE = "https://example.test/er2/" + + +def _write(repo, name, text): + path = repo.worktree / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return repo.checksum(path) + + +def _commit_local(repo, files, message="local data"): + """Track files as the branch's only template and commit their contents.""" + repo.state.config["data"] = [{"fmt": "{name}.h5", "encoding": None}] + repo.state.replace(pd.DataFrame({ + "sha1": [_write(repo, f"{name}.h5", text) for name, text in files.items()], + "name": list(files)})) + repo.dothm.dump(repo.state) + repo.commit(message) + + +def _catalog_remote(repo, rows, *, keep_local=False): + """Catalog remote files by name, with their published SHA-1 or ''.""" + entries = [{"fmt": "{name}.h5", "db": "data-2.tsv", "url": REMOTE}] + if keep_local: + entries.insert(0, {"fmt": "{run}.dat", "encoding": None}) + else: + repo.state.drop_table("data.tsv") + repo.state.config["data"] = entries + repo.state.replace(pd.DataFrame({ + "sha1": list(rows.values()), "checksum_algorithm": ["md5"] * len(rows), + "checksum": [str(index) * 32 for index, _ in enumerate(rows)], + "size_bytes": ["4"] * len(rows), "mtime": [""] * len(rows), + "name": list(rows)}), db="data-2.tsv") + repo.dothm.dump(repo.state) + + +def test_commit_stores_objects_only_for_local_entries(tmp_path): + repo = Repo.init(tmp_path / "repo") + run_sha = _write(repo, "1.dat", "run\n") + _catalog_remote(repo, {"M87": ""}, keep_local=True) + repo.state.replace(pd.DataFrame({"sha1": [run_sha], "run": ["1"]})) + repo.dothm.dump(repo.state) + assert repo.commit("mixed templates") + assert repo.objects.contains(run_sha) + assert len([path for path in repo.objects.root.rglob("*") if path.is_file()]) == 1 + assert repo.status()["staged"]["catalog"] == [] + + +def test_status_summarizes_remote_changes_and_ignores_downloads(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": "", "SGRA": "", "OJ287": ""}) + repo.commit("remote catalog") + _write(repo, "M87.h5", "downloaded\n") + snapshot = repo.status() + assert snapshot["untracked"] == [] + assert snapshot["worktree"] == {"modified": [], "deleted": []} + + table = repo.state.table("data-2.tsv") + table = table[table["name"] != "OJ287"].copy() + table.loc[table["name"] == "SGRA", "checksum"] = "e" * 32 + table.loc[len(table)] = ["", "md5", "f" * 32, "4", "", "3C279"] + repo.state.set_table("data-2.tsv", table) + repo.dothm.dump(repo.state) + assert repo.status()["staged"]["catalog"] == [{ + "templates": ["{name}.h5"], "url": REMOTE, + "added": 1, "modified": 1, "deleted": 1}] + + +def test_cli_status_prints_remote_catalog_summaries(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": "", "SGRA": ""}) + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["status"]) + assert result.exit_code == 0, result.output + assert f"catalog: {{name}}.h5 from {REMOTE}: 2 new, 0 modified, 0 removed" \ + in result.output + + +def test_checkout_removes_local_files_a_remote_branch_catalogs_differently( + tmp_path): + repo = Repo.init(tmp_path / "repo") + _commit_local(repo, {"M87": "local\n"}) + repo.checkout("remote") + _catalog_remote(repo, {"M87": ""}) + repo.commit("catalog the archive copy") + repo.checkout("main") + repo.checkout("remote") + assert not (repo.worktree / "M87.h5").exists() + repo.checkout("main") + assert (repo.worktree / "M87.h5").read_text(encoding="utf-8") == "local\n" + + +def test_checkout_keeps_local_files_a_remote_branch_catalogs_identically( + tmp_path): + repo = Repo.init(tmp_path / "repo") + _commit_local(repo, {"M87": "same\n"}) + sha1 = repo.checksum(repo.worktree / "M87.h5") + repo.checkout("remote") + _catalog_remote(repo, {"M87": sha1}) + repo.commit("catalog the archive copy") + repo.checkout("main") + repo.checkout("remote") + assert (repo.worktree / "M87.h5").read_text(encoding="utf-8") == "same\n" + + +def test_checkout_rejects_a_different_downloaded_copy_of_a_local_file(tmp_path): + repo = Repo.init(tmp_path / "repo") + _commit_local(repo, {"M87": "local\n"}) + repo.checkout("remote") + _catalog_remote(repo, {"M87": ""}) + repo.commit("catalog the archive copy") + _write(repo, "M87.h5", "downloaded\n") + with pytest.raises(Exception, match="downloaded copy"): + repo.checkout("main") + assert repo.dothm.active_branch.name == "remote" + _write(repo, "M87.h5", "local\n") + assert repo.checkout("main") + + +def test_checkout_between_remote_branches_leaves_downloads_alone(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + repo.commit("first archive") + _write(repo, "M87.h5", "downloaded\n") + repo.checkout("other") + _catalog_remote(repo, {"SGRA": ""}) + repo.commit("second archive") + repo.checkout("main") + assert (repo.worktree / "M87.h5").read_text(encoding="utf-8") == "downloaded\n" + assert repo.status()["untracked"] == [] + + +def test_add_worktree_restores_only_local_files(tmp_path): + repo = Repo.init(tmp_path / "main") + run_sha = _write(repo, "1.dat", "run\n") + _catalog_remote(repo, {"M87": ""}, keep_local=True) + repo.state.replace(pd.DataFrame({"sha1": [run_sha], "run": ["1"]})) + repo.dothm.dump(repo.state) + repo.commit("mixed templates") + assert repo.add_worktree("experiment") + experiment = tmp_path / "experiment" + assert (experiment / "1.dat").read_text(encoding="utf-8") == "run\n" + assert not (experiment / "M87.h5").exists() + assert Repo(experiment).state.table("data-2.tsv")["name"].tolist() == ["M87"] diff --git a/test/test_hallmark.py b/test/test_hallmark.py index 5629f37..b1b2e76 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -755,6 +755,7 @@ def test_repo_status_reports_staged_worktree_and_untracked_changes(tmp_path): "added": [], "modified": [], "deleted": [], + "catalog": [], } assert snapshot["worktree"]["modified"] == ["a0_i0.h5"] assert snapshot["worktree"]["deleted"] == ["a0_i30.h5"] @@ -2234,16 +2235,15 @@ def test_add_worktree_rejects_invalid_data_config_before_creation(tmp_path): Args: tmp_path: pytest fixture that provides a temporary directory for the test. Raises: - RuntimeError: If the repository's data configuration is invalid - (not exactly one entry). + ValueError: If the repository's data configuration is malformed. """ repo = Repo.init(tmp_path / "repo") - repo.state.config["data"] = [] + repo.state.config["data"] = "not a list of entries" repo.dothm.dump(repo.state) repo.dothm.index.commit("invalid data configuration") destination = tmp_path / "experiment" - with pytest.raises(RuntimeError, match="requires exactly one data entry"): + with pytest.raises(ValueError, match="must be a mapping or list of mappings"): repo.add_worktree("experiment") assert not destination.exists(), \ f"Expected destination {destination} to not exist, but it does." From 4b49dd0fe909d7390b579c56ac35ea121ba46048 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:23:37 -0700 Subject: [PATCH 21/29] Track several templates per branch and add rm --cached A new template now becomes another data entry instead of replacing the catalog, and must match at least one file that no other template tracks. "add ." rescans every local template within the current directory and keeps rows outside it. "add -n" previews matches without hashing, and "rm --cached TEMPLATE" stops tracking a template without deleting files. Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/cli.py | 40 ++++- mod/hallmark/helper_functions.py | 4 +- mod/hallmark/repo.py | 280 +++++++++++++++++++++++++++---- mod/hallmark/repo_config.py | 61 ++++--- test/test_catalog_entries.py | 168 +++++++++++++++++++ test/test_cli.py | 8 +- test/test_hallmark.py | 17 +- 7 files changed, 496 insertions(+), 82 deletions(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 780129c..b75705e 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -483,22 +483,26 @@ def emit_section(title, entries, fg): default=False, show_default=True, help="Enable regex-based encoding rules from config.yml.") +@click.option("-n", "--dry-run", is_flag=True, + help="List the matching files without staging them.") @click.argument("inputs", nargs=-1, required=True) @click.pass_obj -def add(repo, encoding, inputs): +def add(repo, encoding, dry_run, inputs): """Add files to the hallmark index. - `hallmark add [--regex] FORMAT` uses the branch format string workflow. - `hallmark add "."` rebuilds the manifest from current files that match - the branch `fmt` in `config.yml`. - Explicit path inputs such as shell-expanded `*` are not supported yet - with the parameter-based manifest format. + `hallmark add [--regex] FORMAT` stages the files matching a filename + template such as `a{a}_i{i}.h5`. Each new template is tracked alongside + the existing ones; `hallmark rm --cached FORMAT` stops tracking one. + A new template must match at least one file. + `hallmark add "."` rescans every tracked template within the current + directory. Explicit path inputs such as shell-expanded `*` are not + supported yet with the parameter-based manifest format. """ # attempt to add the specified files to the hallmark index, handling any errors with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError): # if there is only one input, use the add method for a single input if len(inputs) == 1: - pf = repo.add(inputs[0], encoding) + pf = repo.add(inputs[0], encoding, dry_run=dry_run) # oterhwise, use the add_paths method for multiple inputs else: pf = repo.add_paths(list(inputs)) @@ -506,10 +510,30 @@ def add(repo, encoding, inputs): if pf.empty: click.echo("No files matched the format string.") else: - click.echo("Changes to be committed") + click.echo("Would add" if dry_run else "Changes to be committed") click.echo(pf.path.to_string(index=False, header=False)) +@hallmark.command(short_help="Stop tracking a template.") +@click.option("--cached", is_flag=True, + help="Remove the template from the catalog and keep its files.") +@click.argument("template") +@click.pass_obj +def rm(repo, cached, template): + """Stop tracking TEMPLATE, like `git rm --cached`. + + The template's catalog is removed from the index; files are left in + place. TEMPLATE is the tracked filename template, or the full URL + template of a remote one. + """ + if not cached: + raise ClickException( + "hallmark rm requires --cached; files are never deleted") + with _translate_cli_errors(RuntimeError, ValueError, GitError): + entry = repo.rm_cached(template) + click.echo(f"Stopped tracking {entry.fmt or entry.db}; files were left in place.") + + @hallmark.command("set-config", short_help="Update hallmark branch config.") @click.option("--fmt") @click.option("--remote-name") diff --git a/mod/hallmark/helper_functions.py b/mod/hallmark/helper_functions.py index 38b4217..1680984 100644 --- a/mod/hallmark/helper_functions.py +++ b/mod/hallmark/helper_functions.py @@ -270,8 +270,8 @@ def normalize_nonempty_string( ) -> str: """ Used in downloader by _select_download_files and download_remote_data, repo by - _validate_branch_name, add, and commit, repo_config by normalize_remotes, - branch_fmt, and set_config. + _validate_branch_name, add, and commit, repo_config by normalize_remotes + and set_config. Normalize a string by stripping whitespace and ensuring it is non-empty. Args: diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 5a880a0..7e5dfa9 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -23,35 +23,85 @@ from typing import Dict, List, Optional, Tuple, Union from git.exc import GitCommandError +import pandas as pd + from .dothm import Dothm from .state import State from .worktree import Worktree from .objects import Objects from .paraframe import ParaFrame from .repo_manifest import ( - catalog_map, iter_catalog_rows, manifest_frame_from_pf, row_fingerprint, - table_groups) + catalog_map, is_local_table, iter_catalog_rows, manifest_frame_from_pf, + row_fingerprint, table_groups) from .repo_state import ( load_branch_data, load_head_state, find_remote_branch, fetch_missing_objects_from_remote) from .error import CheckoutError, DestinationExistsError, DothmError from .helper_functions import ( FILE_IO_CHUNK_SIZE, + as_list_of_dicts, chdir, iter_repository_files, normalize_nonempty_string, resolve_contained_path) from .repo_worktree import ( + effective_cwd, ensure_clean_tracked_files, - filtered_paraframe, tracked_paths, worktree_changes) from .repo_config import ( - branch_encodings, - branch_fmt, + CatalogEntry, + DEFAULT_DB, + add_entry, + catalog_entries, + check_template_fields, + entry_encodings, + find_entry, + is_placeholder, + next_db_name, normalize_remotes, + remove_entry, + row_to_path, set_config) +def _encoding_spec(config: dict) -> Optional[dict]: + """ + Used by add. + Return the entry holding encoding rules for a template not tracked yet: + the placeholder, or the only catalog entry. + """ + entries = as_list_of_dicts(config.get("data")) or [] + placeholder = next((entry for entry in entries if is_placeholder(entry)), None) + if placeholder is not None: + return placeholder + catalog = catalog_entries(config) + return entries[catalog[0].index] if len(catalog) == 1 else None + + +def _row_within(record: dict, fmt: str, within_root) -> bool: + """ + Used by add. + Return True if a row's file lies within the rescanned subtree, or if the + row no longer renders with its template and should be replaced. + """ + try: + return within_root(row_to_path(record, fmt).as_posix()) + except (KeyError, ValueError): + return True + + +def _entry_names(entry: CatalogEntry) -> set[str]: + """ + Used by rm_cached. + Return the spellings that identify an entry: its template, and for a remote + entry its full URL template. + """ + names = {entry.fmt} if entry.fmt else {entry.db} + if entry.url and entry.fmt: + names.add(entry.url.rstrip("/") + "/" + entry.fmt) + return names + + def _catalog_changes(head_state: State, state: State) -> list[dict]: """ Used by status. @@ -516,15 +566,26 @@ def status(self) -> dict[str, object]: "untracked": untracked, } - def add(self, fmt: str, encoding: bool = False) -> ParaFrame: + def add(self, fmt: str, encoding: bool = False, *, + dry_run: bool = False) -> ParaFrame: ''' - Stage files or updated repository indecing from the worktree. + Stage the files matching a template, or rescan the tracked templates. + + A new template becomes another data entry with its own catalog table; + existing templates are kept. Adding a tracked template again merges + its current files. ``"."`` rescans every local template within the + current directory, dropping rows of files that no longer exist there. Args: fmt (string): Format string or "." for full directory scan. encoding (boolean): Whether to apply encoding rules. + dry_run (boolean): List matching files without hashing or staging. Returns: paraframe Parsed and filtered file index (without checksums). + Raises: + ValueError: If the template is invalid, matches no files when new, + or matches files tracked by another template. + RuntimeError: If the repository has no worktree. ''' if self.worktree is None: raise RuntimeError( @@ -532,45 +593,196 @@ def add(self, fmt: str, encoding: bool = False) -> ParaFrame: # Normalize the format string to ensure it is a non-empty string fmt = normalize_nonempty_string(fmt, label="format") - # "." means rescan the whole worktree using the already-configured format - rescanning = fmt == "." - # use the current branch format; otherwise, use the provided format - if rescanning: - resolved_fmt = branch_fmt(self) - previous_fmt = resolved_fmt - else: - resolved_fmt = fmt - try: - previous_fmt = branch_fmt(self) - except RuntimeError: - previous_fmt = None + # "." means rescan the worktree using the already-configured templates + if fmt == ".": + return self._rescan_local(encoding=encoding, dry_run=dry_run) + + check_template_fields(fmt) + config = self.state.config + entry = find_entry(config, fmt) + if entry is not None and entry.is_remote: + raise ValueError( + f"template {fmt!r} is tracked from a remote URL; run " + f"hallmark rm --cached {fmt!r} before adding local files") + encodings = None + if encoding: + spec = (config["data"][entry.index] if entry is not None + else _encoding_spec(config)) + encodings = entry_encodings(spec, fmt) # with the working directory set to the worktree, parse files into a ParaFrame with chdir(self.worktree): pf = ParaFrame.parse( - resolved_fmt, + fmt, base_path=self.worktree, - encodings=branch_encodings(self) if encoding else None, + encodings=encodings, encoding=encoding) - # if rescanning, filter to include only files that match the configured format - if rescanning: - pf = filtered_paraframe(self, pf) + if dry_run: + return pf + if entry is None: + # like git add, a new pathspec must match something + if pf.empty: + raise ValueError(f"template {fmt!r} did not match any files") + self._check_ownership(fmt, pf["path"]) # Compute checksums for all files in the ParaFrame in parallel self._populate_checksums(pf) - - manifest = manifest_frame_from_pf(pf, resolved_fmt) - # if not rescanning, update the repository configuration with the new format - if not rescanning: - set_config(self, fmt=resolved_fmt) - # an explicit fmt replaces only if the format actually changed - if rescanning or previous_fmt != resolved_fmt: - self.state.replace(manifest) - # if the format is unchanged, update the existing state with new entries + manifest = manifest_frame_from_pf(pf, fmt) + if entry is None: + db = next_db_name(config, used=self._used_table_names()) + add_entry(config, {"fmt": fmt, **({"db": db} if db != DEFAULT_DB else {})}) + self.state.replace(manifest, db=db) + # adding a tracked template again merges its current files else: - self.state.update(manifest) + self.state.update(manifest, db=entry.db) self.dothm.dump(self.state) # return a ParaFrame without the "sha1" column for display purposes return pf.drop(columns=["sha1"], errors="ignore") + def _rescan_local(self, *, encoding: bool, dry_run: bool) -> ParaFrame: + """ + Used by add. + Rescan every local template within the current directory's subtree. + + Rows of files outside the subtree are kept. Files cataloged by a remote + template, such as downloaded copies, are skipped. + + Args: + encoding (bool): Apply the encoding rules of templates defining them. + dry_run (bool): List matching files without hashing or staging. + + Returns: + ParaFrame: The matching files of every local template. + + Raises: + RuntimeError: If no local template is tracked. + ValueError: If a file matches more than one local template. + """ + config = self.state.config + local = [entries[0] for name, entries in table_groups(config).items() + if is_local_table(entries, self.state.table(name))] + if not local: + raise RuntimeError( + "no local templates to rescan; run hallmark add TEMPLATE first") + with_encodings = [entry for entry in local + if entry_encodings(config["data"][entry.index], entry.fmt)] + if encoding and not with_encodings: + raise ValueError("no encoding rules are configured; use hallmark " + "set-config --encoding FIELD=REGEX") + worktree = Path(self.worktree).resolve() + relative_root = effective_cwd(self).relative_to(worktree) + catalog_only = {row.path for row in iter_catalog_rows(self.state, local=False)} + + def within_root(path) -> bool: + return relative_root == Path(".") or relative_root in Path(path).parents + + scans, claimed = [], {} + for entry in local: + use_encoding = encoding and entry in with_encodings + with chdir(self.worktree): + pf = ParaFrame.parse( + entry.fmt, base_path=self.worktree, + encodings=(entry_encodings(config["data"][entry.index], entry.fmt) + if use_encoding else None), + encoding=use_encoding) + if not pf.empty: + pf = pf[pf["path"].map(within_root) & ~pf["path"].isin(catalog_only)] + for path in pf["path"] if not pf.empty else (): + if path in claimed: + raise ValueError( + f'"{path}" matches templates {claimed[path]!r} and ' + f"{entry.fmt!r}; use more specific templates") + claimed[path] = entry.fmt + scans.append((entry, pf)) + frames = [pf for _, pf in scans if not pf.empty] + combined = (ParaFrame(pd.concat(frames, ignore_index=True)) if frames + else ParaFrame(columns=["path"])) + if dry_run: + return combined + for entry, pf in scans: + self._populate_checksums(pf) + manifest = manifest_frame_from_pf(pf, entry.fmt) + if relative_root != Path("."): + # keep rows of files outside the rescanned subtree + kept = [record for record in + self.state.table(entry.db).to_dict(orient="records") + if not _row_within(record, entry.fmt, within_root)] + manifest = pd.concat([pd.DataFrame(kept, columns=manifest.columns), + manifest], ignore_index=True) + self.state.replace(manifest, db=entry.db) + self.dothm.dump(self.state) + return combined.drop(columns=["sha1"], errors="ignore") + + def _check_ownership(self, fmt: str, paths) -> None: + """ + Used by add. + Reject a new template that matches files tracked by another template. + + Args: + fmt (str): The new template. + paths: Relative POSIX paths the template matches. + + Raises: + ValueError: If any path is already cataloged. + """ + owned = catalog_map(self.state) + overlap = sorted(path for path in set(paths) if path in owned) + if overlap: + shown = ", ".join(overlap[:5]) + more = f" and {len(overlap) - 5} more" if len(overlap) > 5 else "" + raise ValueError( + f"template {fmt!r} matches files tracked by another template " + f"({shown}{more}); use a more specific template or run " + "hallmark rm --cached TEMPLATE first") + + def _used_table_names(self) -> set[str]: + """ + Used by add. + Return catalog TSV names present now or at any point in history, so a + new template never reuses the history of a removed one. + """ + names = {path.name for path in self.dothm.path.glob("*.tsv")} + try: + history = self.dothm.git.log( + "--all", "--format=", "--name-only", "--", "*.tsv") + # a repository without commits has no history to consult + except GitCommandError: + history = "" + names.update(line.strip() for line in history.splitlines() if line.strip()) + return names + + def rm_cached(self, template: str) -> CatalogEntry: + ''' + Stop tracking a template without deleting any files, like + ``git rm --cached``. + + Args: + template (str): The tracked template, or a remote template's full + URL template. + + Returns: + CatalogEntry: The removed entry. + + Raises: + ValueError: If the template is not tracked or shares its catalog + table with another template. + ''' + template = normalize_nonempty_string(template, label="template") + entries = catalog_entries(self.state.config) + entry = next((candidate for candidate in entries + if template in _entry_names(candidate)), None) + if entry is None: + tracked = ", ".join(repr(candidate.fmt) for candidate in entries + if candidate.fmt) or "none" + raise ValueError( + f"template {template!r} is not tracked; tracked templates: {tracked}") + if any(other.db == entry.db for other in entries if other is not entry): + raise ValueError( + f"template {entry.fmt!r} shares its catalog table {entry.db} with " + "another template; edit config.yml to remove it") + remove_entry(self.state.config, entry.index) + self.state.drop_table(entry.db) + self.dothm.dump(self.state) + return entry + def commit(self, msg: str, allow_empty: bool = False) -> bool: ''' Commit staged changes to the repository. Raises ValueError if commit diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 47e7a2c..b4e86ce 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -17,7 +17,7 @@ from .helper_functions import ( as_list_of_dicts, coerce_fmt_value, normalize_nonempty_string, validate_path_component, validate_relative_path) -from .state import DEFAULT_DB +from .state import DEFAULT_DB, METADATA_COLUMNS from .transport.base import ( RemoteSpec, backend_name, reject_controls, thaw_backend_options) @@ -295,6 +295,33 @@ def single_data_fmt(config: dict) -> Optional[str]: return fmt.strip() +# Template fields that would collide with catalog columns. The checksum +# algorithm names are read as digests when a download plan is built. +RESERVED_FIELDS = frozenset( + {"path", "sha1", "md5", "sha256", "sha512", *METADATA_COLUMNS}) + + +def check_template_fields(fmt: str) -> list[str]: + """ + Validate the field names of a filename template. + + Args: + fmt (str): Filename template. + + Returns: + list[str]: Unique field names in the order they appear. + + Raises: + ValueError: If a field name is reserved catalog metadata. + """ + fields = fmt_fields(fmt) + reserved = [name for name in fields if name in RESERVED_FIELDS] + if reserved: + raise ValueError( + f"Template field {reserved[0]!r} is reserved catalog metadata") + return fields + + @dataclass(frozen=True) class CatalogEntry: """ @@ -575,38 +602,6 @@ def require_branch_data_spec(repo) -> dict: return spec -def branch_fmt(repo) -> str: - """ - Return the template of a single-template branch. Raises RuntimeError if - no valid template is defined. - - Args: - repo: Repository object. - - Returns: - str: The format string of the branch's data entry. - """ - return normalize_nonempty_string( - require_branch_data_spec(repo).get("fmt"), - label="branch data fmt", - exception_type=RuntimeError) - - -def branch_encodings(repo) -> list[dict]: - """ - Return the filename encodings of a single-template branch. - - Args: - repo: Repository object. - - Returns: - list[dict]: A list containing the encoding specification, or an - empty list if no encodings are defined. - """ - spec = require_branch_data_spec(repo) - return [spec] if isinstance(spec.get("encoding"), dict) else [] - - def set_config( repo, *, diff --git a/test/test_catalog_entries.py b/test/test_catalog_entries.py index c5c6d91..724de1e 100644 --- a/test/test_catalog_entries.py +++ b/test/test_catalog_entries.py @@ -327,3 +327,171 @@ def test_add_worktree_restores_only_local_files(tmp_path): assert (experiment / "1.dat").read_text(encoding="utf-8") == "run\n" assert not (experiment / "M87.h5").exists() assert Repo(experiment).state.table("data-2.tsv")["name"].tolist() == ["M87"] + + +def test_templates_accumulate_and_round_trip_through_checkout(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + _write(repo, "runs/run_1.dat", "dat\n") + repo.add("{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + assert [entry.db for entry in catalog_entries(repo.state.config)] == [ + "data.tsv", "data-2.tsv"] + repo.commit("two templates") + repo.checkout("empty") + repo.rm_cached("{src}_{day}.h5") + repo.rm_cached("runs/run_{run}.dat") + repo.commit("nothing tracked") + repo.checkout("main") + assert (repo.worktree / "M87_095.h5").read_text(encoding="utf-8") == "h5\n" + assert (repo.worktree / "runs/run_1.dat").read_text(encoding="utf-8") == "dat\n" + assert repo.status()["untracked"] == [] + + +def test_new_template_must_match_files_and_leaves_config_unchanged(tmp_path): + repo = Repo.init(tmp_path / "repo") + before = (repo.dothm.path / "config.yml").read_text(encoding="utf-8") + with pytest.raises(ValueError, match="did not match any files"): + repo.add("{src}.fits") + assert (repo.dothm.path / "config.yml").read_text(encoding="utf-8") == before + + +def test_new_template_cannot_claim_files_of_another_template(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + repo.add("{src}_{day}.h5") + with pytest.raises(ValueError, match=r"tracked by another template \(M87_095.h5\)"): + repo.add("{name}.h5") + assert len(catalog_entries(repo.state.config)) == 1 + + +def test_reserved_fields_are_rejected(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "a.h5", "a\n") + for fmt in ("{path}.h5", "{sha256}.h5", "{size_bytes}.h5"): + with pytest.raises(ValueError, match="reserved catalog metadata"): + repo.add(fmt) + + +def test_local_add_refuses_a_template_tracked_remotely(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + _write(repo, "M87.h5", "downloaded\n") + with pytest.raises(ValueError, match="rm --cached"): + repo.add("{name}.h5") + + +def test_dry_run_lists_matches_without_hashing_or_staging(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + staged = {key: entry.binsha for key, entry in repo.dothm.index.entries.items()} + monkeypatch.setattr(Repo, "checksum_many", + lambda paths: pytest.fail("dry run must not hash")) + result = repo.add("{src}_{day}.h5", dry_run=True) + assert result["path"].tolist() == ["M87_095.h5"] + assert catalog_entries(repo.state.config) == [] + assert {key: entry.binsha + for key, entry in repo.dothm.index.entries.items()} == staged + + +def test_dot_rescans_every_local_template_and_skips_downloads(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + _write(repo, "runs/run_1.dat", "dat\n") + repo.add("{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + _write(repo, "SGRA_096.h5", "new\n") + (repo.worktree / "runs/run_1.dat").unlink() + result = repo.add(".") + assert sorted(result["path"]) == ["M87_095.h5", "SGRA_096.h5"] + assert repo.state.data["src"].tolist() == ["M87", "SGRA"] + assert repo.state.table("data-2.tsv").empty + + +def test_dot_from_a_subdirectory_keeps_rows_outside_it(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "repo") + _write(repo, "a/1.dat", "a1\n") + _write(repo, "b/1.dat", "b1\n") + repo.add("{group}/{run}.dat") + (repo.worktree / "a/1.dat").unlink() + _write(repo, "a/2.dat", "a2\n") + monkeypatch.chdir(repo.worktree / "a") + assert repo.add(".")["path"].tolist() == ["a/2.dat"] + rows = repo.state.data.sort_values("group")[["group", "run"]] + assert rows.values.tolist() == [["a", "2"], ["b", "1"]] + + +def test_dot_rejects_a_file_matching_two_local_templates(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "x.h5", "x\n") + _write(repo, "y_1.dat", "y\n") + repo.add("{name}.h5") + repo.add("{name}_{run}.dat") + repo.state.config["data"][1]["fmt"] = "{name}_{run}.{ext}" + _write(repo, "z_2.h5", "z\n") + with pytest.raises(ValueError, match="matches templates"): + repo.add(".") + + +def test_rm_cached_untracks_a_template_and_keeps_its_files(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + _write(repo, "runs/run_1.dat", "dat\n") + repo.add("{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + repo.commit("two templates") + removed = repo.rm_cached("runs/run_{run}.dat") + assert removed.db == "data-2.tsv" + assert not (repo.dothm.path / "data-2.tsv").exists() + assert (repo.worktree / "runs/run_1.dat").exists() + assert repo.status()["staged"]["deleted"] == ["runs/run_1.dat"] + repo.commit("drop runs") + repo.add("runs/run_{run}.dat") + # the removed table's history is never reused by a new template + assert find_db(repo, "runs/run_{run}.dat") == "data-3.tsv" + + +def find_db(repo, fmt): + return next(entry.db for entry in catalog_entries(repo.state.config) + if entry.fmt == fmt) + + +def test_rm_cached_accepts_a_remote_url_template_and_reports_unknown_ones(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + with pytest.raises(ValueError, + match=r"not tracked; tracked templates: '\{name\}.h5'"): + repo.rm_cached("{src}.h5") + repo.rm_cached(REMOTE + "{name}.h5") + assert catalog_entries(repo.state.config) == [] + + +def test_rm_cached_refuses_tables_shared_by_builder_templates(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "x{x}.fits", "db": "built.tsv"}, + {"fmt": "y{y}.fits", "db": "built.tsv"}] + with pytest.raises(ValueError, match="shares its catalog table"): + repo.rm_cached("x{x}.fits") + + +def test_cli_add_dry_run_and_rm_cached(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + monkeypatch.chdir(repo.worktree) + runner = CliRunner() + preview = runner.invoke(hallmark, ["add", "-n", "{src}_{day}.h5"]) + assert preview.exit_code == 0, preview.output + assert preview.output.splitlines() == ["Would add", "M87_095.h5"] + assert runner.invoke(hallmark, ["add", "{src}_{day}.h5"]).exit_code == 0 + refused = runner.invoke(hallmark, ["rm", "{src}_{day}.h5"]) + assert refused.exit_code != 0 + assert "requires --cached" in refused.output + removed = runner.invoke(hallmark, ["rm", "--cached", "{src}_{day}.h5"]) + assert removed.exit_code == 0, removed.output + assert "files were left in place" in removed.output + missing = runner.invoke(hallmark, ["add", "{src}.fits"]) + assert missing.exit_code != 0 + assert "did not match any files" in missing.output diff --git a/test/test_cli.py b/test/test_cli.py index f83426f..407b29a 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -277,18 +277,20 @@ def test_cli_add_regex_flag(monkeypatch): with chdir("repo"): called = {} - def fake_add(self, fmt, encoding=False): + def fake_add(self, fmt, encoding=False, *, dry_run=False): """Fake add method to capture arguments passed to Repo.add.""" called["fmt"] = fmt called["encoding"] = encoding + called["dry_run"] = dry_run return ParaFrame([{"path": "am0.5_i30.h5"}]) monkeypatch.setattr("hallmark.cli.Repo.add", fake_add) result = runner.invoke(hallmark, ["add", "--regex", "."]) assert result.exit_code == 0, f"Expected exit code 0 for add with \ '--regex', got {result.exit_code}" - assert called == {"fmt": ".", "encoding": True}, f"Expected add to be \ - called with fmt='.' and encoding=True, got {called}" + assert called == {"fmt": ".", "encoding": True, "dry_run": False}, \ + f"Expected add to be called with fmt='.' and encoding=True, \ + got {called}" def test_cli_status(): diff --git a/test/test_hallmark.py b/test/test_hallmark.py index b1b2e76..bad452c 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -353,7 +353,7 @@ def test_repo_add_pattern_keeps_deleted_manifest_rows(tmp_path): ] -def test_repo_add_pattern_replaces_manifest_when_fmt_changes(tmp_path): +def test_repo_add_new_pattern_appends_a_template(tmp_path): repo = Repo.init(tmp_path / "repo") _write_files(repo.worktree, ["a0.4_i30_w3.h5", "b0.4_i30_w3.h5"]) @@ -362,8 +362,18 @@ def test_repo_add_pattern_replaces_manifest_when_fmt_changes(tmp_path): repo.checkout("experiment") repo.add("b{a}_i{i}_w{w}.h5") - assert repo.state.config["data"] == [{"fmt": "b{a}_i{i}_w{w}.h5", "encoding": None}] + assert repo.state.config["data"] == [ + {"fmt": "a{a}_i{i}_w{w}.h5", "encoding": None}, + {"fmt": "b{a}_i{i}_w{w}.h5", "db": "data-2.tsv"}] assert repo.state.data.to_dict(orient="records") == [ + { + "sha1": Repo.checksum(repo.worktree / "a0.4_i30_w3.h5"), + "a": "0.4", + "i": "30", + "w": "3", + } + ] + assert repo.state.table("data-2.tsv").to_dict(orient="records") == [ { "sha1": Repo.checksum(repo.worktree / "b0.4_i30_w3.h5"), "a": "0.4", @@ -940,6 +950,7 @@ def test_checkout_rebuilds_worktree_for_branch_specific_nested_fmt(tmp_path): repo.commit("main data") repo.checkout("experiment") + repo.rm_cached("main/a{a}_i{i}.h5") (repo.worktree / "exp" / "run1").mkdir(parents=True) _write_files(repo.worktree, ["exp/run1/b0_i0.h5"]) repo.add("exp/run{run}/b{a}_i{i}.h5") @@ -1029,6 +1040,7 @@ def test_checkout_rejects_symlink_destination_escape(tmp_path): repo.add("main/a{a}_i{i}.h5") repo.commit("main data") repo.checkout("experiment") + repo.rm_cached("main/a{a}_i{i}.h5") (repo.worktree / "main/a0_i0.h5").unlink() (repo.worktree / "exp").mkdir() _write_files(repo.worktree, ["exp/a1_i45.h5"]) @@ -1164,6 +1176,7 @@ def test_checkout_rejects_directory_at_target_file_path(tmp_path): repo.add("data_{number}.txt") repo.commit("main data") repo.checkout("experiment") + repo.rm_cached("data_{number}.txt") data_path.unlink() experiment_path = (repo.worktree / "experiment_1.txt") experiment_path.write_text("experiment\n", encoding="utf-8") From fe09f10610cb65043614d161cc7a69b6ef124a38 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:29:31 -0700 Subject: [PATCH 22/29] Download each template's files from its own source Download items now carry a source: the named --remote when given, otherwise the template's URL, otherwise the default data remote. A plan can span several sources; its summary lists each one, and execution checks every source before transferring files from each in turn. Files without any configured source are skipped and counted, or rejected when requested explicitly. Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/cli.py | 6 +- mod/hallmark/download_plan.py | 120 +++++++++++++++++-- mod/hallmark/downloader.py | 211 ++++++++++++++++++++++++---------- mod/hallmark/repo_config.py | 6 +- test/test_catalog_entries.py | 110 ++++++++++++++++++ 5 files changed, 373 insertions(+), 80 deletions(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index b75705e..2dd367a 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -31,7 +31,7 @@ from .downloader import DownloadError, _select_download_items, _select_remote_config from .sources import get_source, list_sources from .error import CheckoutError, CloneError -from .repo_config import normalize_tsv_name +from .repo_config import catalog_entries, normalize_tsv_name # use a context manager to translate application errors into clean Click errors @@ -215,7 +215,9 @@ def _choose_download_plan(repo, *, output, remote_name, dry_run): def _download_available(repo, remote_name=None): """Explain an empty catalog or absent remote without contacting the server.""" remote = _select_remote_config(repo, remote_name) - if not remote or not remote.get("url"): + template_urls = remote_name is None and any( + entry.is_remote for entry in catalog_entries(repo.state.config)) + if not template_urls and (not remote or not remote.get("url")): click.echo("No data remote is configured; skipping download. " "Configure one with hallmark set-config --remote-url URL.") return False diff --git a/mod/hallmark/download_plan.py b/mod/hallmark/download_plan.py index 4ed35b5..3008ace 100644 --- a/mod/hallmark/download_plan.py +++ b/mod/hallmark/download_plan.py @@ -9,7 +9,54 @@ from typing import Optional, Union from urllib.parse import urlsplit, urlunsplit -from .transport.base import backend_name, freeze_backend_options +from .transport.base import RemoteSpec, backend_name, freeze_backend_options + + +def _redacted_url(url: str) -> str: + """Return a URL without credentials, query parameters, or fragments.""" + parsed = urlsplit(url) + return urlunsplit(parsed._replace( + netloc=parsed.netloc.rsplit("@", 1)[-1], query="", fragment="")) + + +@dataclass(frozen=True) +class DownloadSource: + """ + The data location files are downloaded from. + + Attributes: + url (str): Directory URL that cataloged paths are relative to. + name (str, optional): Name of the configured data remote, if any. + backend (str, optional): Registered backend selected for transfer. + backend_options (mapping): Immutable backend configuration snapshot. + """ + + url: str = field(repr=False) + name: Optional[str] = None + backend: Optional[str] = None + backend_options: Mapping = field(default_factory=dict, repr=False, hash=False) + + def __post_init__(self) -> None: + """Validate the source fields and freeze the backend options.""" + if not isinstance(self.url, str) or not self.url: + raise TypeError("source url must be a nonempty string") + if any(value is not None and not isinstance(value, str) + for value in (self.name, self.backend)): + raise TypeError("source name and backend must be strings or None") + if self.backend is not None: + backend_name(self.backend) + object.__setattr__(self, "backend_options", + freeze_backend_options(self.backend_options)) + + @property + def display(self) -> str: + """The URL without credentials, query parameters, or fragments.""" + return _redacted_url(self.url) + + def remote(self) -> RemoteSpec: + """Parse the source into a remote specification for transfer.""" + return RemoteSpec.parse(self.url, backend=self.backend, + backend_options=self.backend_options) @dataclass(frozen=True) @@ -18,16 +65,19 @@ class DownloadItem: A file and its catalog metadata at the time a download is planned. Attributes: - relative_path (Path): File path relative to the data remote. + relative_path (Path): File path relative to the data source. checksum (str | tuple, optional): Digest or (algorithm, digest) pair. size_bytes (int, optional): Recorded file size; None if unknown. mtime (str, optional): Recorded modification time; None if unknown. + source (DownloadSource, optional): Where the file is downloaded from; + None uses the plan's source. """ relative_path: Path checksum: Optional[Union[str, tuple[str, str]]] = None size_bytes: Optional[int] = None mtime: Optional[str] = None + source: Optional[DownloadSource] = None def __post_init__(self) -> None: """Normalize the path and validate optional file metadata.""" @@ -48,25 +98,33 @@ def __post_init__(self) -> None: raise ValueError("size_bytes must be a nonnegative integer or None") if self.mtime is not None and not isinstance(self.mtime, str): raise ValueError("mtime must be a string or None") + if self.source is not None and not isinstance(self.source, DownloadSource): + raise TypeError("source must be a DownloadSource or None") @dataclass(frozen=True) class DownloadPlan: """ - An immutable selection of files, source, and destination for a download. + An immutable selection of files, sources, and destination for a download. Creating a plan does not approve a transfer. Sizes come from the catalog; a duration estimate requires every file size and a supplied transfer rate. + Files may come from several sources, such as templates added from + different URLs. The plan-level source fields describe the source of items + without their own source; plans with one source fill them in. Attributes: items (tuple[DownloadItem]): Selected files and their metadata. - remote_url (str, optional): Data source URL; may be None for an empty plan. + remote_url (str, optional): Data source URL; None for an empty plan or + one whose items come from several sources. output_path (Path): Absolute destination directory. remote_name (str, optional): Name of the selected data remote. estimated_bytes_per_second (float, optional): Positive rate supplied by the caller for duration estimates. remote_backend (str, optional): Registered backend selected for transfer. backend_options (mapping): Immutable backend configuration snapshot. + unsourced_count (int): Cataloged files skipped because no data source + is configured for them. """ items: tuple[DownloadItem, ...] @@ -76,6 +134,7 @@ class DownloadPlan: estimated_bytes_per_second: Optional[float] = None remote_backend: Optional[str] = None backend_options: Mapping = field(default_factory=dict, repr=False, hash=False) + unsourced_count: int = 0 def __post_init__(self) -> None: """Copy the selection and validate the source fields and rate.""" @@ -92,6 +151,10 @@ def __post_init__(self) -> None: backend_name(self.remote_backend) object.__setattr__(self, "backend_options", freeze_backend_options(self.backend_options)) + if (isinstance(self.unsourced_count, bool) + or not isinstance(self.unsourced_count, int) + or self.unsourced_count < 0): + raise ValueError("unsourced_count must be a nonnegative integer") rate = self.estimated_bytes_per_second if rate is not None and ( isinstance(rate, bool) or not isinstance(rate, (int, float)) @@ -99,6 +162,28 @@ def __post_init__(self) -> None: ): raise ValueError("estimated_bytes_per_second must be a positive rate") + @property + def default_source(self) -> Optional[DownloadSource]: + """The plan-level source, or None when the plan has none.""" + if not self.remote_url: + return None + return DownloadSource(self.remote_url, self.remote_name, + self.remote_backend, self.backend_options) + + def source_for(self, item: DownloadItem) -> Optional[DownloadSource]: + """Return the source an item is downloaded from.""" + return item.source or self.default_source + + @property + def sources(self) -> tuple[DownloadSource, ...]: + """Distinct sources of the selected files, in plan order.""" + sources = [] + for item in self.items: + source = self.source_for(item) + if source is not None and source not in sources: + sources.append(source) + return tuple(sources) + @property def file_count(self) -> int: """Number of files selected for download.""" @@ -139,12 +224,23 @@ def summary(self) -> str: size += f" known; {self.unknown_size_count} file(s) with unknown size" duration = ("unknown" if self.estimated_seconds is None else f"about {self.estimated_seconds:.1f} seconds") - source = "" - if self.remote_url: - parsed = urlsplit(self.remote_url) - source = urlunsplit(parsed._replace( - netloc=parsed.netloc.rsplit("@", 1)[-1], query="", fragment="")) - backend = f"\nTransport: {self.remote_backend}" if self.remote_backend else "" + sources = self.sources + if len(sources) > 1: + counts = {source: 0 for source in sources} + for item in self.items: + counts[self.source_for(item)] += 1 + lines = "".join( + f"\n {source.display} ({count} file(s)" + + (f"; transport {source.backend}" if source.backend else "") + ")" + for source, count in counts.items()) + origin = f"\nSources:{lines}" + else: + source = (sources[0] if sources else self.default_source) + origin = f"\nSource: {source.display if source else ''}" + if source is not None and source.backend: + origin += f"\nTransport: {source.backend}" + skipped = (f"\nSkipped {self.unsourced_count} cataloged file(s) without a " + "data source" if self.unsourced_count else "") return (f"{self.file_count} file(s); {size}; estimated duration: {duration}" - f"\nSource: {source}{backend}" - f"\nDestination: {self.output_path}") + f"{origin}" + f"\nDestination: {self.output_path}{skipped}") diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index dd458d7..336821b 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -15,7 +15,7 @@ from .transport import OperationContext, RemoteSpec from .transport.base import DownloadError, literal_path -from .download_plan import DownloadItem, DownloadPlan +from .download_plan import DownloadItem, DownloadPlan, DownloadSource from .helper_functions import ( CHECKSUM_ALGORITHMS_BY_STRENGTH, SUPPORTED_CHECKSUM_ALGORITHMS, @@ -26,6 +26,7 @@ resolve_contained_path, valid_checksum) from .repo_config import ( + catalog_entries, normalize_remotes, normalize_tsv_name, row_to_path) @@ -631,34 +632,43 @@ def _select_download_items( tsv_names: Sequence[str] = (), all_files: bool = False, catalog_only: bool = False, + resolve_source=None, ) -> list[DownloadItem]: """ Select files to download from a hallmark repository based on the provided file paths, TSV names, and the repository's configuration. + With no paths or TSVs, every cataloged file is selected. + Args: repo: The hallmark repository object. file_paths: A sequence of specific file paths to download. tsv_names: A sequence of TSV names to download files from. all_files: If True, include all files from the repository's configuration. + catalog_only: If True, explicit paths must be cataloged. + resolve_source: Optional callable receiving the data entries of a + catalog table (empty for static files) and returning the + ``DownloadSource`` of its files, or None if none is configured. Returns: - list[DownloadItem]: Relative paths, checksums, and available file metadata. + list[DownloadItem]: Relative paths, checksums, available file metadata, + and sources when ``resolve_source`` is given. """ # Get the repository configuration, defaulting to an empty dictionary if not found. config = _repository_config(repo) - # get the data configuration entries from the "data" section of the config - data_config = _config_section_entries(config, "data") # dictionary to store selected files with relative paths and optional checksums. selected: dict[str, tuple[Path, Optional[ChecksumSpec]]] = {} metadata = {} + sources: dict[str, Optional[DownloadSource]] = {} explicit_paths = {_safe_remote_path(path).as_posix() for path in file_paths} + all_files = all_files or (not file_paths and not tsv_names) def add_file( value: Union[str, Path], expected_checksum: Optional[ChecksumSpec] = None, row=None, + source: Optional[DownloadSource] = None, ) -> None: """Add a file and its catalog metadata, checking for conflicting records.""" # Resolve the input value to a safe relative path @@ -666,22 +676,26 @@ def add_file( # Merge the selected file into the dictionary of selected files, ensuring no # conflicting checksums exist. _merge_selected_file(selected, relative_path, expected_checksum) + key = relative_path.as_posix() + if source is not None: + if sources.get(key) not in (None, source): + raise DownloadError(f"Conflicting sources for {key!r}") + sources[key] = source if row is not None: size = _catalog_size(row.get("size_bytes")) modified = row.get("mtime") modified = (None if modified is None or pd.isna(modified) or str(modified).strip() == "" else str(modified)) - previous_size, previous_modified = metadata.get( - relative_path.as_posix(), (None, None)) + previous_size, previous_modified = metadata.get(key, (None, None)) if size is not None and previous_size not in (None, size): raise DownloadError( f"Conflicting sizes for {relative_path.as_posix()!r}") - metadata[relative_path.as_posix()] = ( + metadata[key] = ( size if size is not None else previous_size, modified if modified is not None else previous_modified) def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], - *, explicit_only=False) -> None: + *, explicit_only=False, source=None) -> None: """Add matching catalog rows, retaining their checksums and file metadata.""" # if the DataFrame is None or empty, return early without adding any files if frame is None or frame.empty: @@ -695,23 +709,28 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], # add the resolved remote path and its checksum to the selected files relative_path = _safe_remote_path(_resolve_remote_path(row, fmt_entries)) if not explicit_only or relative_path.as_posix() in explicit_paths: - add_file(relative_path, _row_checksum(row), row) + add_file(relative_path, _row_checksum(row), row, source) # Add explicitly requested file paths to the selected files. if not catalog_only: for file_path in file_paths: add_file(file_path) - # Organize data configuration entries by their TSV names for easier access. - entries_by_tsv: dict[str, list[dict]] = {} - for entry in data_config: - # Path catalogs need no parameterized filename format. - if not entry.get("db"): - continue - # download_tsv_name will normalize and validate the TSV name - tsv_name = _download_tsv_name(entry["db"]) - # Add the entry to the list of entries for the corresponding TSV name - entries_by_tsv.setdefault(tsv_name, []).append(entry) - + # Group data entries by the TSV holding their rows. Entries without an + # explicit "db" share data.tsv, whose rows are loaded in the repository state. + try: + catalog = catalog_entries(config) + except ValueError as exc: + raise DownloadError(str(exc)) from exc + raw_entries = as_list_of_dicts(config.get("data")) or [] + tables: dict[str, list] = {} + for entry in catalog: + tables.setdefault(entry.db, []).append(entry) + in_state = {name for name, entries in tables.items() + if not any(raw_entries[entry.index].get("db") for entry in entries)} + # rows of an unreferenced data.tsv are cataloged by their path column + if "data.tsv" not in tables and not repo.state.data.empty: + tables["data.tsv"] = [] + in_state.add("data.tsv") requested_tsvs = [] seen_tsvs = set() @@ -725,24 +744,31 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], # if the all_files flag is set, include all TSVs in the requested list and seen set if all_files: - for tsv_name in entries_by_tsv: + for tsv_name in tables: if tsv_name not in seen_tsvs: requested_tsvs.append(tsv_name) seen_tsvs.add(tsv_name) lookup_tsvs = list(requested_tsvs) if explicit_paths: - lookup_tsvs.extend(name for name in entries_by_tsv + lookup_tsvs.extend(name for name in tables if name not in requested_tsvs) for tsv_name in lookup_tsvs: - fmt_entries = entries_by_tsv.get(tsv_name) + entries = tables.get(tsv_name) # if there are no format entries for the requested TSV - if fmt_entries is None: - configured = ", ".join(entries_by_tsv) or "" + if entries is None: + configured = ", ".join(tables) or "" # raise an error indicating that the TSV is not configured in the repository raise DownloadError( f"TSV {tsv_name!r} is not configured. " f"Configured TSVs: {configured}") + fmt_entries = [raw_entries[entry.index] for entry in entries] + source = resolve_source(entries) if resolve_source is not None else None + explicit_only = tsv_name not in requested_tsvs + if tsv_name in in_state: + add_frame(repo.state.data, fmt_entries, explicit_only=explicit_only, + source=source) + continue # Construct the path to the TSV file in the repository's dothm directory. tsv_path = repo.dothm.path / tsv_name @@ -758,8 +784,8 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], keep_default_na=False, chunksize=TSV_READ_CHUNK_SIZE) for frame in frames: - add_frame(frame, fmt_entries, - explicit_only=tsv_name not in requested_tsvs) + add_frame(frame, fmt_entries, explicit_only=explicit_only, + source=source) # skip empty TSV files without raising an error except pd.errors.EmptyDataError: continue @@ -768,6 +794,7 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], raise DownloadError(f"Unable to read TSV {tsv_path}: {exc}") from exc if all_files or explicit_paths: + static_source = resolve_source(()) if resolve_source is not None else None # for each section ("data" and "meta") in the repository configuration for section_name in ("data", "meta"): # iterate through each entry in the section @@ -777,29 +804,61 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], # if a file path is specified in the entry, add it to the selected files if file_path and (all_files or _safe_remote_path(file_path).as_posix() in explicit_paths): - add_file(file_path, _entry_checksum(entry), entry) - - # Determine whether to use legacy data formats based on the presence of file paths, - # TSV names, and all_files flag. - use_legacy_data = (not file_paths and not tsv_names and not all_files - ) or (all_files and not entries_by_tsv) - if use_legacy_data: - # legacy_formats are entries in the data configuration that have a "fmt" key - legacy_formats = [entry for entry in data_config if entry.get("fmt")] - # add files from legacy formats to the selected files - add_frame(repo.state.data, legacy_formats) - elif explicit_paths and not entries_by_tsv: - add_frame(repo.state.data, data_config, explicit_only=True) + add_file(file_path, _entry_checksum(entry), entry, static_source) missing = explicit_paths - set(selected) if missing: raise DownloadError( "Paths are not in the catalog: " + ", ".join(sorted(missing))) - return [DownloadItem(path, checksum, *metadata.get(path.as_posix(), (None, None))) + return [DownloadItem(path, checksum, *metadata.get(path.as_posix(), (None, None)), + source=sources.get(path.as_posix())) for path, checksum in selected.values()] +def _download_source(remote: Optional[dict]) -> Optional[DownloadSource]: + """ + Used by _source_resolver. + Validate a configured data remote and describe it as a download source. + """ + if not remote or not remote.get("url"): + return None + spec = RemoteSpec.parse(remote["url"], backend=remote.get("backend"), + backend_options=remote.get("backend_options")) + return DownloadSource(remote["url"], remote.get("name"), spec.backend, + remote.get("backend_options") or {}) + + +def _source_resolver(repo, remote_name: Optional[str] = None): + """ + Used by plan_download. + Return a function choosing the download source of a catalog table. + + A named remote overrides every source. Otherwise files come from their + data entry's URL, or from the default configured remote. + """ + override = (_download_source(_select_remote_config(repo, remote_name)) + if remote_name is not None else None) + default = [] + + def resolve(entries) -> Optional[DownloadSource]: + if override is not None: + return override + entry = next((entry for entry in entries if entry.url), None) + if entry is not None: + spec = RemoteSpec.parse(entry.url, backend=entry.backend, + backend_options=entry.backend_options) + return DownloadSource(entry.url, None, spec.backend, + entry.backend_options or {}) + # the default remote is resolved only when a table needs it, since + # several unnamed choices are an error only then + if not default: + default.append(_download_source(_select_remote_config(repo))) + return default[0] + + return resolve + + def _catalog_size(value) -> Optional[int]: """Read a recorded size without treating missing or malformed sizes as zero.""" if value is None or isinstance(value, bool) or pd.isna(value): @@ -894,33 +953,45 @@ def plan_download( raise DownloadError("all_files cannot be combined with paths or TSVs") items = _select_download_items( repo, file_paths=file_paths, tsv_names=tsv_names, - all_files=all_files or (not file_paths and not tsv_names), catalog_only=True) + all_files=all_files or (not file_paths and not tsv_names), catalog_only=True, + resolve_source=_source_resolver(repo, remote_name)) if filter is not None: from .discovery import path_matches path_matches("validation", filter=filter) items = [item for item in items if path_matches( item.relative_path.as_posix(), filter=filter)] - remote = _select_remote_config(repo, remote_name) - if items and (remote is None or not remote.get("url")): - raise DownloadError("No remote URL is configured in config.yml") - remote = remote or {} - source = None - if remote.get("url"): - source = RemoteSpec.parse( - remote["url"], backend=remote.get("backend"), - backend_options=remote.get("backend_options")) + unsourced = [item for item in items if item.source is None] + if unsourced and (file_paths or tsv_names or len(unsourced) == len(items)): + # explicitly requested files, or a selection with nothing downloadable + if len(unsourced) == len(items): + raise DownloadError("No remote URL is configured in config.yml") + shown = ", ".join(item.relative_path.as_posix() for item in unsourced[:5]) + raise DownloadError(f"No data source is configured for: {shown}") + items = [item for item in items if item.source is not None] for item in items: try: resolve_contained_path(output_root, item.relative_path, label="download destination") except ValueError as exc: raise DownloadError(str(exc)) from exc + sources = list(dict.fromkeys(item.source for item in items)) + if not sources and remote_name is None: + # describe the default source even when nothing is selected + try: + sources = [source for source in ( + _download_source(_select_remote_config(repo)),) if source] + except DownloadError: + sources = [] + elif not sources: + sources = [_download_source(_select_remote_config(repo, remote_name))] + source = sources[0] if len(sources) == 1 else None return DownloadPlan( - tuple(items), remote.get("url"), output_root, - remote_name=remote.get("name"), + tuple(items), source.url if source else None, output_root, + remote_name=source.name if source else None, estimated_bytes_per_second=estimated_bytes_per_second, - remote_backend=source.backend if source is not None else None, - backend_options=remote.get("backend_options")) + remote_backend=source.backend if source else None, + backend_options=source.backend_options if source else {}, + unsourced_count=len(unsourced)) def execute_download_plan( @@ -960,14 +1031,28 @@ def execute_download_plan( return {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} if plan.output_path.resolve() != plan.output_path: raise DownloadError("Download destination changed since planning") - remote = RemoteSpec.parse( - plan.remote_url, backend=plan.remote_backend, - backend_options=plan.backend_options) - return _download_selected( - remote, plan.output_path, - [(item.relative_path, item.checksum) for item in plan.items], - max_workers=max_workers, show_progress=show_progress, - byte_progress=True, total_bytes=plan.total_bytes) + groups: dict = {} + for item in plan.items: + source = plan.source_for(item) + if source is None: + raise DownloadError( + f"No data source for {item.relative_path.as_posix()!r}") + groups.setdefault(source, []).append(item) + # validate every source before transferring any file + remotes = {source: source.remote() for source in groups} + results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + for source, items in groups.items(): + sizes = [item.size_bytes for item in items] + group = _download_selected( + remotes[source], plan.output_path, + [(item.relative_path, item.checksum) for item in items], + max_workers=max_workers, show_progress=show_progress, + byte_progress=True, + total_bytes=None if None in sizes else sum(sizes)) + for key in ("succeeded", "failed", "total_bytes"): + results[key] += group[key] + results["errors"].extend(group["errors"]) + return results def download_remote_data( diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index b4e86ce..79cc293 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -376,8 +376,8 @@ def catalog_entries(config) -> list[CatalogEntry]: """ Return the data entries that own catalog tables, in config order. - Placeholders and static ``file`` entries are skipped. An entry without an - explicit ``db`` uses ``data.tsv``. + Placeholders, static ``file`` entries and entries that are not mappings + are skipped. An entry without an explicit ``db`` uses ``data.tsv``. Args: config (dict): The repository configuration dictionary. @@ -395,7 +395,7 @@ def catalog_entries(config) -> list[CatalogEntry]: entries = [] for index, entry in enumerate(_data_entry_list(config)): if not isinstance(entry, dict): - raise ValueError(f"config data entry {index} must be a mapping") + continue fmt = _entry_fmt(entry) if fmt is None and not entry.get("db"): continue diff --git a/test/test_catalog_entries.py b/test/test_catalog_entries.py index 724de1e..6086b20 100644 --- a/test/test_catalog_entries.py +++ b/test/test_catalog_entries.py @@ -495,3 +495,113 @@ def test_cli_add_dry_run_and_rm_cached(tmp_path, monkeypatch): missing = runner.invoke(hallmark, ["add", "{src}.fits"]) assert missing.exit_code != 0 assert "did not match any files" in missing.output + + +MIRROR = "https://mirror.test/export/" +SIMS = "https://sims.test/runs/" + + +def _two_archives(repo, *, local=True): + """Catalog a local run, archive images, and simulation files.""" + entries = [{"fmt": "{name}.h5", "db": "data-2.tsv", "url": REMOTE}, + {"fmt": "{run}.dat", "db": "data-3.tsv", "url": SIMS}] + if local: + entries.insert(0, {"fmt": "notes_{n}.txt", "encoding": None}) + repo.state.replace(pd.DataFrame({ + "sha1": [_write(repo, "notes_1.txt", "note\n")], "n": ["1"]})) + repo.state.config["data"] = entries + remote_columns = {"sha1": "", "checksum_algorithm": "", "checksum": "", + "mtime": ""} + repo.state.replace(pd.DataFrame([ + {**remote_columns, "size_bytes": "5", "name": "M87"}, + {**remote_columns, "size_bytes": "6", "name": "SGRA"}]), db="data-2.tsv") + repo.state.replace(pd.DataFrame([ + {**remote_columns, "size_bytes": "7", "run": "001"}]), db="data-3.tsv") + repo.dothm.dump(repo.state) + + +def test_plan_downloads_each_template_from_its_own_url(tmp_path): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo, local=False) + plan = repo.plan_download(all_files=True) + assert {item.relative_path.as_posix(): item.source.url + for item in plan.items} == { + "M87.h5": REMOTE, "SGRA.h5": REMOTE, "001.dat": SIMS} + assert [source.url for source in plan.sources] == [REMOTE, SIMS] + assert plan.remote_url is None + assert plan.total_bytes == 18 + summary = plan.summary() + assert f"Sources:\n {REMOTE} (2 file(s); transport http)" in summary + assert f"\n {SIMS} (1 file(s); transport http)" in summary + + +def test_plan_with_one_source_keeps_the_plan_level_fields(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + plan = repo.plan_download() + assert plan.remote_url == REMOTE + assert plan.remote_backend == "http" + assert f"Source: {REMOTE}\nTransport: http" in plan.summary() + + +def test_local_templates_use_the_default_remote_and_skip_without_one(tmp_path): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo) + plan = repo.plan_download(all_files=True) + assert plan.file_count == 3 + assert plan.unsourced_count == 1 + assert "Skipped 1 cataloged file(s) without a data source" in plan.summary() + with pytest.raises(Exception, match="No data source is configured for: notes"): + repo.plan_download(file_paths=["notes_1.txt", "M87.h5"]) + + repo.set_config(remote_url=MIRROR) + plan = repo.plan_download(all_files=True) + assert {item.relative_path.as_posix(): item.source.url + for item in plan.items}["notes_1.txt"] == MIRROR + assert plan.unsourced_count == 0 + + +def test_named_remote_overrides_every_source(tmp_path): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo) + repo.set_config(remote_name="mirror", remote_url=MIRROR) + plan = repo.plan_download(all_files=True, remote_name="mirror") + assert {item.source.url for item in plan.items} == {MIRROR} + assert plan.remote_name == "mirror" + + +def test_nothing_downloadable_reports_the_missing_remote(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "a1.h5", "one\n") + repo.add("a{a}.h5") + with pytest.raises(Exception, match="No remote URL is configured"): + repo.plan_download(all_files=True) + + +def test_execute_downloads_each_source_in_turn(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo, local=False) + fetched = [] + + def fake_fetch(context, relative_path, destination, checksum, chunk_size): + fetched.append(context.remote.file_url(relative_path.as_posix())) + return 1 + + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_fetch) + result = repo.download(repo.plan_download(all_files=True), approved=True) + assert result == {"succeeded": 3, "failed": 0, "total_bytes": 3, "errors": []} + assert sorted(fetched) == [ + REMOTE + "M87.h5", REMOTE + "SGRA.h5", SIMS + "001.dat"] + + +def test_cli_download_dry_run_lists_every_source(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _two_archives(repo, local=False) + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["download", "--all", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "3 file(s); 18 bytes" in result.output + assert "Sources:" in result.output and SIMS in result.output From 2eb1635f44c6b20919ef0aa6ddf6dfb449a35f33 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:35:19 -0700 Subject: [PATCH 23/29] Catalog remote files with URL templates in hallmark add hallmark add 'https://host/ER2/{src}_{day}.h5' records the URL up to the first templated segment on a new data entry and catalogs the matching remote files with their sizes and published checksums, without downloading them. Listings skip directories the template cannot match. Adding the URL template again syncs it with the remote directory; a template that matches nothing, conflicts with a tracked template, or carries credentials is rejected. Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/__init__.py | 4 +- mod/hallmark/cli.py | 37 ++++- mod/hallmark/discovery.py | 180 ++++++++++++++++++++- mod/hallmark/repo.py | 31 +++- mod/hallmark/repo_remote.py | 233 +++++++++++++++++++++++++++ mod/hallmark/sources.py | 19 +++ mod/hallmark/transport/__init__.py | 3 + mod/hallmark/transport/base.py | 5 + mod/hallmark/transport/http.py | 3 +- mod/hallmark/transport/ssh.py | 4 +- test/test_remote_add.py | 246 +++++++++++++++++++++++++++++ test/test_sftp_metadata.py | 28 ++++ 12 files changed, 779 insertions(+), 14 deletions(-) create mode 100644 mod/hallmark/repo_remote.py create mode 100644 test/test_remote_add.py diff --git a/mod/hallmark/__init__.py b/mod/hallmark/__init__.py index 41adea9..86bb763 100644 --- a/mod/hallmark/__init__.py +++ b/mod/hallmark/__init__.py @@ -18,7 +18,7 @@ from .sources import DataSource, SourceRelease, register_source from .repo import Repo from .paraframe import ParaFrame -from .download_plan import DownloadItem, DownloadPlan +from .download_plan import DownloadItem, DownloadPlan, DownloadSource from .backends import ( DataBackend, HttpBackend, SshBackend, CyVerseBackend, RemoteEntry, register_backend, @@ -26,6 +26,6 @@ __all__ = [ "DataSource", "SourceRelease", "register_source", "Repo", "ParaFrame", - "DownloadItem", "DownloadPlan", "DataBackend", + "DownloadItem", "DownloadPlan", "DownloadSource", "DataBackend", "HttpBackend", "SshBackend", "CyVerseBackend", "RemoteEntry", "register_backend", ] diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 2dd367a..5d19865 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -31,6 +31,7 @@ from .downloader import DownloadError, _select_download_items, _select_remote_config from .sources import get_source, list_sources from .error import CheckoutError, CloneError +from .discovery import is_remote_url from .repo_config import catalog_entries, normalize_tsv_name @@ -496,19 +497,31 @@ def add(repo, encoding, dry_run, inputs): template such as `a{a}_i{i}.h5`. Each new template is tracked alongside the existing ones; `hallmark rm --cached FORMAT` stops tracking one. A new template must match at least one file. - `hallmark add "."` rescans every tracked template within the current + `hallmark add "."` rescans every local template within the current directory. Explicit path inputs such as shell-expanded `*` are not supported yet with the parameter-based manifest format. + + `hallmark add 'URL/{field}...'` catalogs remote files matching a URL + template, such as `https://host/ER2/{src}_{day}.h5`, without downloading + them. The URL up to the first segment with a field is recorded as the + template's source. Adding it again syncs the catalog with the remote + directory. Use `hallmark ls-remote URL` to find templates. """ + remote = len(inputs) == 1 and is_remote_url(inputs[0]) # attempt to add the specified files to the hallmark index, handling any errors - with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError): + with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError, + DownloadError): # if there is only one input, use the add method for a single input if len(inputs) == 1: - pf = repo.add(inputs[0], encoding, dry_run=dry_run) + pf = repo.add(inputs[0], encoding, dry_run=dry_run, + **({"progress": True} if remote else {})) # oterhwise, use the add_paths method for multiple inputs else: pf = repo.add_paths(list(inputs)) + if remote: + _report_remote_add(pf, dry_run=dry_run) + return if pf.empty: click.echo("No files matched the format string.") else: @@ -516,6 +529,24 @@ def add(repo, encoding, dry_run, inputs): click.echo(pf.path.to_string(index=False, header=False)) +def _report_remote_add(pf, *, dry_run): + """Summarize a remote catalog update without listing every file.""" + paths = pf["path"].tolist() + click.echo("Would add" if dry_run else "Changes to be committed") + for path in paths[:20]: + click.echo(path) + if len(paths) > 20: + click.echo(f"... {len(paths) - 20} more file(s)") + mismatched = pf.attrs.get("mismatched", 0) + if mismatched: + click.echo(f"Skipped {mismatched} listed file(s) whose names the template " + "cannot re-create exactly.") + if not dry_run: + click.echo(f"Cataloged {len(paths)} remote file(s); nothing was downloaded. " + "Run hallmark commit -m MESSAGE, then hallmark download to " + "fetch files.") + + @hallmark.command(short_help="Stop tracking a template.") @click.option("--cached", is_flag=True, help="Remove the template from the catalog and keep its files.") diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py index 877b556..3eb4a10 100644 --- a/mod/hallmark/discovery.py +++ b/mod/hallmark/discovery.py @@ -8,13 +8,16 @@ from functools import lru_cache from pathlib import PurePosixPath from string import Formatter +from typing import Optional +from urllib.parse import quote, unquote, urlsplit, urlunsplit import parse from tqdm import tqdm from .helper_functions import CHECKSUM_ALGORITHMS_BY_STRENGTH, valid_checksum +from .repo_config import RESERVED_FIELDS from .transport.base import ( - DownloadError, RemoteEntry, RemoteObjectMissing, literal_path, + DownloadError, RemoteEntry, RemoteObjectMissing, literal_path, reject_controls, ) @@ -51,7 +54,7 @@ def extraction_parser(template): if not name.isidentifier() or conversion is not None: raise ValueError( "Extraction fields must be simple names without conversions") - if name in {*CATALOG_COLUMNS, "sha1"}: + if name in {*CATALOG_COLUMNS, *RESERVED_FIELDS}: raise ValueError(f"Extraction field {name!r} is reserved catalog metadata") if name not in fields: fields.append(name) @@ -63,6 +66,168 @@ def extraction_parser(template): return parser, tuple(fields) +def _template_parts(segment: str): + """Split one template segment into literal text and field definitions.""" + try: + return list(Formatter().parse(segment)) + except ValueError as exc: + raise ValueError(f"Invalid filename template segment {segment!r}: {exc}") \ + from None + + +def _escape_braces(text: str) -> str: + return text.replace("{", "{{").replace("}", "}}") + + +def is_remote_url(value) -> bool: + """Return True for a string that names a remote location, such as a URL.""" + return isinstance(value, str) and "://" in value + + +def split_url_template(url: str) -> tuple[str, Optional[str]]: + """ + Split a URL template at the first path segment containing a field. + + The part before that segment is the remote directory; the rest is a + filename template relative to it. Literal parts of the template are + percent-decoded once, so ``%20`` names a space and ``%7B`` a literal brace. + A URL without fields names one file, unless it ends in ``/``. + + Args: + url (str): URL such as ``https://host/ER2/{src}_{day}.h5``. + + Returns: + tuple[str, str | None]: The directory URL, ending in ``/``, and the + template; the template is None for a directory URL. + + Raises: + ValueError: If the URL has credentials, a query or fragment, fields + outside its path, or an invalid template. + """ + if not is_remote_url(url): + raise ValueError("Expected a URL such as https://host/path/{field}.ext") + reject_controls(url, "URL") + parsed = urlsplit(url) + if any(brace in parsed.scheme + parsed.netloc for brace in "{}"): + raise ValueError("Template fields are allowed only in the URL path") + if "?" in url or "#" in url: + raise ValueError("Remote URLs must not contain a query or fragment") + if parsed.password is not None or ( + parsed.username is not None and parsed.scheme in {"http", "https"}): + raise ValueError( + "Remote URLs must not contain credentials; configure them locally, " + "for example in ~/.netrc or ~/.ssh/config") + segments = parsed.path.split("/") + index = next((position for position, segment in enumerate(segments) + if any(name is not None + for _, name, _, _ in _template_parts(segment))), None) + if index is None: + if not parsed.path.strip("/") or parsed.path.endswith("/"): + return url, None + index = len(segments) - 1 + base = urlunsplit((parsed.scheme, parsed.netloc, + "/".join(segments[:index]) + "/", "", "")) + decoded = [] + for segment in segments[index:]: + text = [] + for literal, name, spec, conversion in _template_parts(segment): + text.append(_escape_braces(unquote(literal))) + if name is not None: + if conversion is not None: + raise ValueError("Template fields cannot use conversions") + text.append("{" + name + (f":{spec}" if spec else "") + "}") + decoded.append("".join(text)) + return base, "/".join(decoded) + + +def template_url(base: str, template: str) -> str: + """Join a directory URL and a template, percent-encoding literal text.""" + segments = [] + for segment in template.split("/"): + text = [] + for literal, name, spec, _ in _template_parts(segment): + text.append(quote(literal, safe="")) + if name is not None: + text.append("{" + name + (f":{spec}" if spec else "") + "}") + segments.append("".join(text)) + return base.rstrip("/") + "/" + "/".join(segments) + + +class TemplateMatcher: + """ + Match relative paths against a filename template, one segment at a time. + + Fields never span ``/``, and a path segment starting with ``.`` matches + only a template segment starting with ``.``, as with local glob patterns. + + Args: + template (str): Filename template such as ``{src}/{day}.h5``. + + Raises: + ValueError: If the template is invalid or uses reserved field names. + """ + + def __init__(self, template: str): + if not isinstance(template, str) or not template.strip(): + raise ValueError("format must be a nonempty filename template") + segments = template.split("/") + if any(segment in {"", ".", ".."} for segment in segments): + raise ValueError( + f"Template {template!r} must be a relative path naming files") + self.template = template + self.fields = [] + self._parsers, self._hidden, globs = [], [], [] + for segment in segments: + stripped, pattern = [], [] + for literal, name, _, conversion in _template_parts(segment): + stripped.append(_escape_braces(literal)) + pattern.append(re.sub(r"([\[\]*?])", r"[\1]", literal)) + if name is None: + continue + if not name.isidentifier() or conversion is not None: + raise ValueError( + "Template fields must be simple names without conversions") + if name in RESERVED_FIELDS: + raise ValueError( + f"Template field {name!r} is reserved catalog metadata") + if name not in self.fields: + self.fields.append(name) + stripped.append("{" + name + "}") + pattern.append("*") + self._parsers.append(parse.compile("".join(stripped), case_sensitive=True)) + self._hidden.append(segment.startswith(".")) + globs.append(re.sub(r"\*+", "*", "".join(pattern))) + self.glob = "/".join(globs) + + def _match_segment(self, index: int, part: str) -> Optional[dict]: + if part.startswith(".") and not self._hidden[index]: + return None + result = self._parsers[index].parse(part) + return None if result is None else dict(result.named) + + def match(self, path: str) -> Optional[dict]: + """Return the field values of a matching path, or None.""" + parts = path.split("/") + if len(parts) != len(self._parsers): + return None + values = {} + for index, part in enumerate(parts): + named = self._match_segment(index, part) + if named is None: + return None + for name, value in named.items(): + if values.setdefault(name, value) != value: + return None + return values + + def may_contain(self, directory: str) -> bool: + """Return True if files under a relative directory could match.""" + parts = [part for part in directory.split("/") if part] + return len(parts) < len(self._parsers) and all( + self._match_segment(index, part) is not None + for index, part in enumerate(parts)) + + def path_matches(path: str, filter=None) -> bool: """Match a relative path against any supplied case-sensitive inclusion glob. @@ -133,8 +298,8 @@ def _manifest_checksums(context, entries): previous, checksum_algorithm=algorithm, checksum=digest.lower()) -def discover(context, *, filter=None, progress=False, path_prefix="" - ) -> list[RemoteEntry]: +def discover(context, *, filter=None, progress=False, path_prefix="", + descend=None) -> list[RemoteEntry]: """ Discover remote files and their published metadata recursively. @@ -146,6 +311,10 @@ def discover(context, *, filter=None, progress=False, path_prefix="" filter (str | list[str], optional): Relative path glob or globs. path_prefix (str): Release-relative root for collection selection. Returned entry paths remain relative to the context root. + descend (callable, optional): Predicate receiving a relative directory + path ending in ``/``; listings skip directories for which it + returns False. Checksum manifests in skipped directories are + not read. progress (bool | callable): True displays a progress bar with an unknown total. A callback receives dictionaries containing ``directories``, ``files``, ``matched``, and ``current``. @@ -190,6 +359,8 @@ def add(entry): counts["matched"] += int(matches(path)) report() + previous_descend = getattr(context, "descend", None) + context.descend = descend try: context.transport.prepare() for entry in context.transport.iter_entries(on_directory=report): @@ -199,5 +370,6 @@ def add(entry): return [entries[path] for path in sorted(entries) if matches(path)] finally: + context.descend = previous_descend if bar is not None: bar.close() diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 7e5dfa9..8e88175 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -29,6 +29,7 @@ from .state import State from .worktree import Worktree from .objects import Objects +from .discovery import is_remote_url from .paraframe import ParaFrame from .repo_manifest import ( catalog_map, is_local_table, iter_catalog_rows, manifest_frame_from_pf, @@ -567,7 +568,8 @@ def status(self) -> dict[str, object]: } def add(self, fmt: str, encoding: bool = False, *, - dry_run: bool = False) -> ParaFrame: + dry_run: bool = False, progress=False, backend: Optional[str] = None, + backend_options: Optional[dict] = None) -> ParaFrame: ''' Stage the files matching a template, or rescan the tracked templates. @@ -576,17 +578,40 @@ def add(self, fmt: str, encoding: bool = False, *, its current files. ``"."`` rescans every local template within the current directory, dropping rows of files that no longer exist there. + A URL template such as ``https://host/ER2/{src}_{day}.h5`` catalogs + the matching remote files without downloading them. The URL up to the + first segment with a field is recorded on the data entry. Adding the + URL template again syncs it with the remote directory. + Args: - fmt (string): Format string or "." for full directory scan. + fmt (string): Format string, URL template, or "." for full + directory scan. encoding (boolean): Whether to apply encoding rules. dry_run (boolean): List matching files without hashing or staging. + progress (bool | callable): Display remote discovery progress or + receive updates. + backend (str, optional): Registered backend for a URL template. + backend_options (dict, optional): Backend configuration for a URL + template. Returns: paraframe Parsed and filtered file index (without checksums). Raises: ValueError: If the template is invalid, matches no files when new, or matches files tracked by another template. - RuntimeError: If the repository has no worktree. + RuntimeError: If a local template is added to a repository without + a worktree. + DownloadError: If remote discovery fails. ''' + if is_remote_url(fmt): + if encoding: + raise ValueError("--regex encoding rules apply to local files only") + from .repo_remote import add_remote_template + + return add_remote_template( + self, fmt, dry_run=dry_run, progress=progress, backend=backend, + backend_options=backend_options) + if backend is not None or backend_options is not None: + raise ValueError("backend options apply to URL templates only") if self.worktree is None: raise RuntimeError( "cannot add files in a bare repository without a worktree") diff --git a/mod/hallmark/repo_remote.py b/mod/hallmark/repo_remote.py new file mode 100644 index 0000000..5009a60 --- /dev/null +++ b/mod/hallmark/repo_remote.py @@ -0,0 +1,233 @@ +"""Catalog remote files that match a URL template, without downloading them.""" + +from __future__ import annotations + +from typing import Optional + +import pandas as pd + +from .discovery import ( + TemplateMatcher, discover, split_url_template, template_url) +from .fmt_detection import detect_fmt +from .paraframe import ParaFrame +from .repo_config import ( + DEFAULT_DB, add_entry, find_entry, next_db_name, row_to_path) +from .repo_manifest import catalog_map +from .transport import OperationContext, RemoteSpec +from .transport.base import thaw_backend_options + +# Columns of a remote catalog table, before the template fields. +REMOTE_COLUMNS = ("sha1", "checksum_algorithm", "checksum", "size_bytes", "mtime") + + +def _remote(base: str, backend=None, backend_options=None) -> RemoteSpec: + """Parse a directory URL, inheriting a registered source's backend.""" + if backend is None: + from .sources import backend_for_url + + inherited = backend_for_url(base) + if inherited is not None: + backend, options = inherited + backend_options = options if backend_options is None else backend_options + return RemoteSpec.parse(base, backend=backend, backend_options=backend_options) + + +def _require_template(url: str) -> tuple[str, str]: + """Split a URL template, pointing directory URLs to ls-remote.""" + base, template = split_url_template(url) + if template is None: + raise ValueError( + f"{base} is a directory; run hallmark ls-remote {base} to see " + "suggested templates, then add URL/TEMPLATE") + return base, template + + +def match_remote_template(url: str, *, progress=False, backend=None, + backend_options=None): + """ + List the remote files matching a URL template without downloading them. + + Only directories the template can match are listed. Published checksum + manifests in those directories provide checksums. + + Args: + url (str): URL template such as ``https://host/ER2/{src}_{day}.h5``. + progress (bool | callable): Display discovery progress or receive updates. + backend (str, optional): Registered backend for the URL. + backend_options (dict, optional): Backend-specific configuration. + + Returns: + tuple[RemoteSpec, str, pd.DataFrame, int]: The remote directory, the + template, catalog rows sorted by path (with a ``path`` column), and + the number of listed files that matched the pattern but not the + template's exact formatting. + + Raises: + ValueError: If the URL or template is invalid. + DownloadError: If discovery fails. + """ + base, template = _require_template(url) + matcher = TemplateMatcher(template) + remote = _remote(base, backend, backend_options) + with OperationContext(remote) as context: + entries = discover(context, filter=matcher.glob, progress=progress, + descend=matcher.may_contain) + rows, mismatched = [], 0 + for entry in entries: + values = matcher.match(entry.path) + if values is None: + continue + # a file must be re-creatable from its fields, as local rows are + try: + rendered = row_to_path(values, template).as_posix() + except (KeyError, TypeError, ValueError): + rendered = None + if rendered != entry.path: + mismatched += 1 + continue + published_sha1 = (entry.checksum if entry.checksum_algorithm == "sha1" + else "") + rows.append({ + "path": entry.path, "sha1": published_sha1 or "", + "checksum_algorithm": entry.checksum_algorithm or "", + "checksum": entry.checksum or "", + "size_bytes": "" if entry.size is None else str(entry.size), + "mtime": "" if entry.mtime is None else str(entry.mtime), + **values}) + frame = pd.DataFrame( + rows, columns=["path", *REMOTE_COLUMNS, *matcher.fields], dtype=object) + return remote, template, frame, mismatched + + +def add_remote_template(repo, url: str, *, dry_run=False, progress=False, + backend=None, backend_options=None) -> ParaFrame: + """ + Catalog the remote files matching a URL template as a data entry. + + A new template becomes another data entry recording the directory URL. + Adding a tracked URL template again syncs it: new files are added, + changed files updated, and files no longer listed are dropped. + + Args: + repo: Repository object. + url (str): URL template such as ``https://host/ER2/{src}_{day}.h5``. + dry_run (bool): List the matching files without staging them. + progress (bool | callable): Display discovery progress or receive updates. + backend (str, optional): Registered backend recorded for the URL. + backend_options (dict, optional): Backend-specific configuration. + + Returns: + ParaFrame: The matching files with their ``path`` and field values. + ``attrs["mismatched"]`` counts listed files skipped because the + template cannot re-create their names exactly. + + Raises: + ValueError: If the URL is invalid, matches nothing, conflicts with a + tracked template, or matches files tracked by another template. + DownloadError: If discovery fails. + """ + base, template = _require_template(url) + TemplateMatcher(template) + config = repo.state.config + entry = find_entry(config, template) + if entry is not None and (entry.url or "").rstrip("/") != base.rstrip("/"): + where = "the worktree" if not entry.url else entry.url + raise ValueError( + f"template {template!r} is already tracked from {where}; run " + f"hallmark rm --cached {template!r} first") + if entry is not None and backend is None: + backend, backend_options = entry.backend, entry.backend_options + remote, template, frame, mismatched = match_remote_template( + url, progress=progress, backend=backend, backend_options=backend_options) + if frame.empty: + detail = (f"; {mismatched} listed file(s) fit the pattern but not the " + "template's formatting" if mismatched else "") + raise ValueError( + f"template {template!r} did not match any files at {base}{detail}") + result = ParaFrame(frame[["path", *frame.columns[len(REMOTE_COLUMNS) + 1:]]]) + result.attrs["mismatched"] = mismatched + if dry_run: + return result + owned = catalog_map(repo.state) + overlap = sorted(path for path in frame["path"] + if path in owned and owned[path].db != getattr(entry, "db", None)) + if overlap: + shown = ", ".join(overlap[:5]) + more = f" and {len(overlap) - 5} more" if len(overlap) > 5 else "" + raise ValueError( + f"template {template!r} matches files tracked by another template " + f"({shown}{more}); use a more specific template or run " + "hallmark rm --cached TEMPLATE first") + if entry is None: + db = next_db_name(config, used=repo._used_table_names()) + spec = {"fmt": template, "url": base} + # record a backend chosen explicitly or inherited from a source release + if remote.backend != RemoteSpec.parse(base).backend: + spec["backend"] = remote.backend + if remote.backend_options: + spec["backend_options"] = thaw_backend_options(remote.backend_options) + if db != DEFAULT_DB: + spec["db"] = db + add_entry(config, spec) + else: + db = entry.db + # a remote template is synced as a whole, like git add + repo.state.replace(frame.drop(columns=["path"]), db=db) + repo.dothm.dump(repo.state) + return result + + +def suggest_templates(url: str, *, progress=False, backend=None, + backend_options=None) -> tuple[str, list[tuple[str, int]], int]: + """ + List a remote directory and suggest URL templates for its files. + + Args: + url (str): Directory URL. + progress (bool | callable): Display discovery progress or receive updates. + backend (str, optional): Registered backend for the URL. + backend_options (dict, optional): Backend-specific configuration. + + Returns: + tuple[str, list[tuple[str, int]], int]: The directory URL, suggested + URL templates with the number of files each matches (most first), and + the number of files listed. + + Raises: + ValueError: If the URL has template fields or is otherwise invalid. + DownloadError: If discovery fails. + """ + base, template = split_url_template(url.rstrip("/") + "/") + if template is not None: + raise ValueError("ls-remote takes a directory URL without template fields") + remote = _remote(base, backend, backend_options) + with OperationContext(remote) as context: + paths = [entry.path for entry in discover(context, progress=progress)] + suggestions = [] + for candidate in detect_fmt(paths): + try: + matcher = TemplateMatcher(candidate) + # inference can mistake literal braces for fields or pick reserved names + except ValueError: + continue + count = 0 + for path in paths: + values = matcher.match(path) + if values is None: + continue + try: + if row_to_path(values, candidate).as_posix() == path: + count += 1 + except (KeyError, TypeError, ValueError): + continue + if count: + suggestions.append((template_url(base, candidate), count)) + suggestions.sort(key=lambda item: (-item[1], item[0])) + return base, suggestions, len(paths) + + +def remote_url_template(entry) -> Optional[str]: + """Return a remote entry's full URL template, or None for a local entry.""" + if not entry.url or not entry.fmt: + return None + return template_url(entry.url, entry.fmt) diff --git a/mod/hallmark/sources.py b/mod/hallmark/sources.py index 552ba96..5739862 100644 --- a/mod/hallmark/sources.py +++ b/mod/hallmark/sources.py @@ -149,6 +149,25 @@ def list_sources(): return [get_source(name) for name in sorted(names)] +def backend_for_url(url: str): + """ + Return the backend a registered source release declares for a URL. + + Args: + url (str): Remote directory URL. + + Returns: + tuple[str, Mapping] | None: The backend name and options of the release + whose URL contains ``url``, or None if no such release declares one. + """ + for source in list_sources(): + for release in source.releases.values(): + root = release.url.rstrip("/") + "/" + if release.backend and (url.rstrip("/") + "/").startswith(root): + return release.backend, release.backend_options + return None + + def resolve_source(source, release=None, collections=None): """Return a transport, crawl roots, and provenance for a name or raw URL.""" if not isinstance(source, str) or not source.strip(): diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py index bdedf48..22ca24b 100644 --- a/mod/hallmark/transport/__init__.py +++ b/mod/hallmark/transport/__init__.py @@ -35,6 +35,9 @@ def __init__( self.output_root = Path(output_root) if output_root is not None else None self.cancelled = Event() self.on_bytes = None + # Optional predicate receiving a relative directory path ending in "/". + # Listings skip directories for which it returns False. + self.descend = None self.text_limit = 16 * 1024 * 1024 self.listing_timeout = 60 self._local = local() diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py index 3b9729f..e29cbc5 100644 --- a/mod/hallmark/transport/base.py +++ b/mod/hallmark/transport/base.py @@ -296,6 +296,11 @@ class DataBackend: and release owned resources in ``close``. Authentication secrets belong in local configuration, not persisted backend options. + Recursive listings should skip a subdirectory when ``context.descend`` is + set and returns False for its relative path (ending in ``/``). Discovery + sets it to avoid listing directories a filename template cannot match; + backends that ignore it remain correct, only slower. + Args: context (OperationContext): Resources and settings for this operation. """ diff --git a/mod/hallmark/transport/http.py b/mod/hallmark/transport/http.py index 8e16c44..ad21cad 100644 --- a/mod/hallmark/transport/http.py +++ b/mod/hallmark/transport/http.py @@ -32,6 +32,7 @@ def _parse_index(self, text, directory): def iter_entries(self, on_directory=None): """Yield metadata from recursive listings without reading payloads.""" + descend = getattr(self.context, "descend", None) queue, visited = deque([""]), set() while queue: directory = queue.popleft() @@ -46,7 +47,7 @@ def iter_entries(self, on_directory=None): visited.add(canonical) for path, is_directory, size, mtime in self._parse_index(text, canonical): if is_directory: - if path not in visited: + if path not in visited and (descend is None or descend(path)): queue.append(path) else: yield RemoteEntry(path=path, size=size, mtime=mtime) diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py index 6b49ff3..7cc10ef 100644 --- a/mod/hallmark/transport/ssh.py +++ b/mod/hallmark/transport/ssh.py @@ -352,6 +352,7 @@ def iter_entries(self, on_directory=None): DownloadError: If discovery fails or a path escapes the dataset root. CapabilityError: If the server omits required file types. """ + descend = getattr(self.context, "descend", None) with self._metadata() as session: root = session.realpath(self.context.remote.root).rstrip("/") or "/" reject_controls(root, "SFTP root") @@ -395,7 +396,8 @@ def iter_entries(self, on_directory=None): if mode is None: raise CapabilityError("SFTP server omitted file type") if stat.S_ISDIR(mode): - pending.append((path + "/", absolute)) + if descend is None or descend(path + "/"): + pending.append((path + "/", absolute)) elif stat.S_ISREG(mode): entry = RemoteEntry(path, attrs["size"], attrs["mtime"]) yield entry diff --git a/test/test_remote_add.py b/test/test_remote_add.py new file mode 100644 index 0000000..0ca185f --- /dev/null +++ b/test/test_remote_add.py @@ -0,0 +1,246 @@ +"""Test cataloging remote files with URL templates in hallmark add.""" + +import hashlib + +import pytest +import requests +from click.testing import CliRunner + +from hallmark import Repo +from hallmark.cli import hallmark +from hallmark.discovery import TemplateMatcher, split_url_template +from hallmark.repo_config import catalog_entries +from hallmark.transport import OperationContext +from hallmark.transport.base import RemoteObjectMissing + +ROOT = "https://archive.test/2026MOVIE/ER2/" +FILES = {"M87_095.h5": b"m87-095", "M87_096.h5": b"m87-096", + "SGRA_095.h5": b"sgra-095"} + + +def _index(entries): + """Render an Apache-style index with sizes for files and directories.""" + links = "".join( + f'{name}' if name.endswith("/") + else f'{name}' + for name, size in entries) + return f"

Index of listing

{links}" + + +@pytest.fixture +def archive(monkeypatch): + """Serve an archive listing with a checksum manifest; record listing reads.""" + pages, reads = {}, [] + + def read_text(context, path): + url = context.remote.url.rstrip("/") + "/" + path + reads.append(url) + if url not in pages: + raise RemoteObjectMissing("No such metadata object") + return pages[url] + + monkeypatch.setattr(OperationContext, "read_text", read_text) + sums = "".join(f"{hashlib.md5(data).hexdigest()} {name}\n" + for name, data in FILES.items()) + pages[ROOT] = _index([*((name, len(data)) for name, data in FILES.items()), + ("md5sums.txt", len(sums)), ("README.md", 5), + ("calibration/", None), (".hidden_1.h5", 3)]) + pages[ROOT + "md5sums.txt"] = sums + pages[ROOT + "calibration/"] = _index([("M87_cal.h5", 4)]) + return pages, reads + + +@pytest.fixture +def repo(tmp_path): + return Repo.init(tmp_path / "repo") + + +def test_split_url_template_separates_directory_and_template(): + assert split_url_template(ROOT + "{src}_{day}.h5") == (ROOT, "{src}_{day}.h5") + assert split_url_template("https://h/a%20b/{run}/x%7B%25.dat") == ( + "https://h/a%20b/", "{run}/x{{%.dat") + assert split_url_template("ssh://campus/srv/export/{run}.h5") == ( + "ssh://campus/srv/export/", "{run}.h5") + assert split_url_template(ROOT) == (ROOT, None) + assert split_url_template(ROOT + "README.md") == (ROOT, "README.md") + + +@pytest.mark.parametrize("url, message", [ + ("https://user:secret@h/d/{a}.h5", "credentials"), + ("https://token@h/d/{a}.h5", "credentials"), + ("https://h/d/{a}.h5?sig=1", "query or fragment"), + ("https://h/d/{a}.h5#part", "query or fragment"), + ("https://{host}/d/{a}.h5", "only in the URL path"), + ("https://h/d/{a!r}.h5", "conversions"), + ("https://h/d/{a.h5", "Invalid filename template"), +]) +def test_split_url_template_rejects_unsafe_urls(url, message): + with pytest.raises(ValueError, match=message): + split_url_template(url) + + +def test_template_matcher_keeps_fields_within_segments(): + matcher = TemplateMatcher("{src}/{day}_[x].h5") + assert matcher.glob == "*/*_[[]x[]].h5" + assert matcher.match("M87/095_[x].h5") == {"src": "M87", "day": "095"} + assert matcher.match("M87/sub/095_[x].h5") is None + assert matcher.match(".cache/095_[x].h5") is None + assert matcher.may_contain("M87/") + assert not matcher.may_contain("M87/095/") + assert TemplateMatcher("{a}{b}.h5").glob == "*.h5" + with pytest.raises(ValueError, match="reserved"): + TemplateMatcher("{size_bytes}.h5") + with pytest.raises(ValueError, match="naming files"): + TemplateMatcher("{src}/") + + +def test_add_catalogs_matching_remote_files_without_payloads(repo, archive): + pages, reads = archive + result = repo.add(ROOT + "{src}_{day}.h5") + assert result["path"].tolist() == sorted(FILES) + assert repo.state.config["data"] == [{"fmt": "{src}_{day}.h5", "url": ROOT}] + rows = repo.state.data.to_dict("records") + assert rows[0] == { + "sha1": "", "checksum_algorithm": "md5", + "checksum": hashlib.md5(FILES["M87_095.h5"]).hexdigest(), + "size_bytes": "7", "mtime": "", "src": "M87", "day": "095"} + # only the root and the manifest were read; subdirectories were pruned + assert reads == [ROOT, ROOT + "md5sums.txt"] + assert not any((repo.worktree / name).exists() for name in FILES) + reopened = Repo(repo.worktree) + assert reopened.state.data.to_dict("records") == rows + + +def test_remote_catalog_commits_without_objects_and_status_stays_clean( + repo, archive): + repo.add(ROOT + "{src}_{day}.h5") + snapshot = repo.status() + assert snapshot["staged"]["catalog"] == [{ + "templates": ["{src}_{day}.h5"], "url": ROOT, + "added": 3, "modified": 0, "deleted": 0}] + assert repo.commit("Add remote EHT data") + assert not any(path.is_file() for path in repo.objects.root.rglob("*")) + snapshot = repo.status() + assert snapshot["staged"]["catalog"] == [] + assert snapshot["worktree"] == {"modified": [], "deleted": []} + assert snapshot["untracked"] == [] + + +def test_download_verifies_published_checksums(repo, archive, monkeypatch): + from mock_server import MockServer + + repo.add(ROOT + "{src}_{day}.h5") + repo.commit("Add remote EHT data") + server = MockServer(ROOT) + for name, data in FILES.items(): + server.add_file(name, data) + monkeypatch.setattr(requests, "Session", lambda: server) + plan = repo.plan_download() + assert plan.total_bytes == sum(map(len, FILES.values())) + result = repo.download(plan, approved=True) + assert result["succeeded"] == 3 and result["failed"] == 0 + assert (repo.worktree / "SGRA_095.h5").read_bytes() == b"sgra-095" + snapshot = repo.status() + assert snapshot["untracked"] == [] + assert snapshot["worktree"] == {"modified": [], "deleted": []} + + +def test_readding_a_remote_template_syncs_removals(repo, archive): + pages, _ = archive + repo.add(ROOT + "{src}_{day}.h5") + repo.commit("Add remote EHT data") + pages[ROOT] = _index([("M87_095.h5", 7), ("M87_097.h5", 9)]) + del pages[ROOT + "md5sums.txt"] + repo.add(ROOT + "{src}_{day}.h5") + assert repo.state.data["day"].tolist() == ["095", "097"] + assert repo.status()["staged"]["catalog"] == [{ + "templates": ["{src}_{day}.h5"], "url": ROOT, + "added": 1, "modified": 1, "deleted": 2}] + + +def test_no_match_is_an_error_and_changes_nothing(repo, archive): + before = (repo.dothm.path / "config.yml").read_text(encoding="utf-8") + with pytest.raises(ValueError, match="did not match any files"): + repo.add(ROOT + "{src}.fits") + assert (repo.dothm.path / "config.yml").read_text(encoding="utf-8") == before + + +def test_files_the_template_cannot_recreate_are_counted(repo, archive): + with pytest.raises(ValueError, match="3 listed file.s. fit the pattern"): + repo.add(ROOT + "{src}_{day:02d}.h5") + pages, _ = archive + pages[ROOT] = _index([("M87_7.h5", 1), ("M87_07.h5", 1)]) + result = repo.add(ROOT + "{src}_{run:02d}.h5") + assert result["path"].tolist() == ["M87_07.h5"] + assert result.attrs["mismatched"] == 1 + + +def test_template_segments_and_dry_run(repo, archive): + pages, reads = archive + pages[ROOT + "calibration/"] = _index([("M87_cal.h5", 4)]) + preview = repo.add(ROOT + "calibration/{src}_cal.h5", dry_run=True) + assert preview["path"].tolist() == ["M87_cal.h5"] + assert catalog_entries(repo.state.config) == [] + repo.add(ROOT + "{kind}/{src}_cal.h5") + assert repo.state.data.to_dict("records")[0]["kind"] == "calibration" + + +def test_conflicting_templates_are_rejected(repo, archive): + (repo.worktree / "local_1.h5").write_text("x\n", encoding="utf-8") + repo.add(ROOT + "{src}_{day}.h5") + with pytest.raises(ValueError, match="tracked by another template"): + repo.add(ROOT + "{name}.h5") + with pytest.raises(ValueError, match="already tracked from"): + repo.add("https://mirror.test/ER2/{src}_{day}.h5") + with pytest.raises(ValueError, match="local files only"): + repo.add(ROOT + "{src}_{day}.h5", encoding=True) + repo.add("local_{n}.h5") + with pytest.raises(ValueError, match="already tracked from the worktree"): + repo.add(ROOT + "local_{n}.h5") + + +def test_remote_and_local_templates_share_a_branch(repo, archive): + (repo.worktree / "runs").mkdir() + (repo.worktree / "runs/run_1.dat").write_text("run\n", encoding="utf-8") + repo.add(ROOT + "{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + assert [(entry.db, entry.url) for entry in catalog_entries(repo.state.config)] \ + == [("data.tsv", ROOT), ("data-2.tsv", None)] + assert repo.commit("remote images and local runs") + assert len([path for path in repo.objects.root.rglob("*") + if path.is_file()]) == 1 + assert repo.add(".")["path"].tolist() == ["runs/run_1.dat"] + + +def test_directory_url_points_to_ls_remote(repo, archive): + with pytest.raises(ValueError, match="hallmark ls-remote"): + repo.add(ROOT) + + +def test_remote_add_works_in_a_bare_repository(tmp_path, archive): + bare = Repo.init(tmp_path / "catalog.hm") + bare.add(ROOT + "{src}_{day}.h5") + assert bare.commit("bare catalog") + assert bare.plan_download(tmp_path / "out").file_count == 3 + + +def test_explicit_backend_is_recorded_on_the_entry(repo, archive): + repo.add(ROOT + "{src}_{day}.h5", backend="cyverse") + assert repo.state.config["data"][0]["backend"] == "cyverse" + assert repo.plan_download().remote_backend == "cyverse" + + +def test_cli_add_url_summarizes_and_hides_credentials(repo, archive, monkeypatch): + monkeypatch.chdir(repo.worktree) + runner = CliRunner() + result = runner.invoke(hallmark, ["add", ROOT + "{src}_{day}.h5"]) + assert result.exit_code == 0, result.output + assert "Cataloged 3 remote file(s); nothing was downloaded." in result.output + assert "SGRA_095.h5" in result.output + secret = runner.invoke( + hallmark, ["add", "https://user:hunter2@archive.test/ER2/{a}.h5"]) + assert secret.exit_code != 0 + assert "hunter2" not in secret.output + directory = runner.invoke(hallmark, ["add", ROOT]) + assert directory.exit_code != 0 + assert "ls-remote" in directory.output diff --git a/test/test_sftp_metadata.py b/test/test_sftp_metadata.py index 42a992c..80fae34 100644 --- a/test/test_sftp_metadata.py +++ b/test/test_sftp_metadata.py @@ -229,3 +229,31 @@ def test_wrong_response_cannot_be_treated_as_file_attributes( with context.transport._metadata() as session: session.lstat("/data") assert not context.transport._processes + + +def test_sftp_listing_skips_directories_rejected_by_descend(monkeypatch): + listed = [] + + class Session: + def realpath(self, path): + return path + + def lstat(self, path): + return {"size": 4, "mode": stat.S_IFDIR, "mtime": 1} + + def iterdir(self, path): + listed.append(path) + if path == "/data": + yield "keep", {"size": None, "mode": stat.S_IFDIR, "mtime": None} + yield "skip", {"size": None, "mode": stat.S_IFDIR, "mtime": None} + else: + yield "file", {"size": 4, "mode": stat.S_IFREG, "mtime": 1} + + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + monkeypatch.setattr( + context.transport, "_metadata", lambda: nullcontext(Session()), + ) + context.descend = lambda directory: directory == "keep/" + entries = list(context.transport.iter_entries()) + assert [entry.path for entry in entries] == ["keep/file"] + assert listed == ["/data", "/data/keep"] From 62f984b8f03388254574c747a8972f2133a0b739 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:41:27 -0700 Subject: [PATCH 24/29] Replace init --from with ls-remote and URL templates in add init now only creates an empty repository, like git init. Remote datasets are cataloged with hallmark add 'URL/{field}...', and the new read-only ls-remote command lists a remote directory and suggests templates using filename-format detection. Named sources still list release and collection URLs, and supply their backend to templates under those URLs. Co-Authored-By: Claude Opus 5.5 (1M context) --- demo/multi_server_backend.py | 6 +- mod/hallmark/catalog.py | 133 +----------- mod/hallmark/cli.py | 91 ++++---- mod/hallmark/discovery.py | 34 +-- mod/hallmark/repo.py | 36 +--- mod/hallmark/repo_builder.py | 6 +- mod/hallmark/sources.py | 39 ---- test/test_backends.py | 10 +- test/test_catalog.py | 226 +++----------------- test/test_cli.py | 65 ++---- test/test_interactive_download.py | 25 +-- test/test_sources.py | 342 +++++++++--------------------- test/test_ssh_integration.py | 23 +- 13 files changed, 234 insertions(+), 802 deletions(-) diff --git a/demo/multi_server_backend.py b/demo/multi_server_backend.py index 5832a2d..668fbb3 100644 --- a/demo/multi_server_backend.py +++ b/demo/multi_server_backend.py @@ -1,8 +1,10 @@ """Example backend joining independent HTTP roots into one logical dataset. Call ``register()`` and define a ``DataSource`` release with -``backend="multi-server"`` and ``backend_options={"routes": ...}``, then call -``Repo.init(..., source=source_name, release=release_name)``. +``backend="multi-server"`` and ``backend_options={"routes": ...}``. Then +``repo.add(release_url + "{site}/file.fits")`` catalogs files under the release +URL with this backend; ``Repo.add(..., backend=..., backend_options=...)`` +selects it without registering a source. An installed plugin instead exposes ``MultiServerBackend`` through the ``hallmark.backends`` entry-point group. No servers are contacted on import. """ diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py index ee18fd6..1f758a5 100644 --- a/mod/hallmark/catalog.py +++ b/mod/hallmark/catalog.py @@ -1,28 +1,23 @@ -"""Initialize remote datasets and clone existing Hallmark catalogs.""" +"""Clone existing Hallmark catalogs from Git or published snapshots.""" from __future__ import annotations import re -from dataclasses import replace from io import StringIO from pathlib import Path from shutil import rmtree -from urllib.parse import quote, urlsplit +from urllib.parse import urlsplit import pandas as pd import yaml -from .discovery import CATALOG_COLUMNS, discover, extraction_parser, path_matches -from .sources import resolve_source from .dothm import Dothm from .error import CloneError, DestinationExistsError -from .fmt_detection import detect_fmt from .helper_functions import as_list_of_dicts from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name from .repo_manifest import row_relative_path from .transport import OperationContext, RemoteSpec -from .transport.base import (RemoteObjectMissing, literal_path, - thaw_backend_options) +from .transport.base import RemoteObjectMissing, literal_path from .worktree import Worktree @@ -119,116 +114,6 @@ def _metadata_commit(repo, files, message): repo.state = repo.dothm.load() -def _write_inventory(repo, source, entries, format, provenance): - """Write discovered files and their source to a new catalog.""" - columns = list(CATALOG_COLUMNS) - templates = [format] if format is not None else detect_fmt( - [entry.path for entry in entries]) - parsers, formats, fields = [], [], [] - for template in templates: - try: - parser, names = extraction_parser(template) - except ValueError: - if format is not None: - raise - # Inference can mistake literal braces for fields. Such filenames - # still belong in the catalog, without unsafe extraction columns. - continue - parsers.append(parser) - formats.append(template) - fields.extend(name for name in names if name not in fields) - rows = [] - for entry in entries: - row = dict(zip(columns, (entry.path, entry.checksum_algorithm, - entry.checksum, entry.size, entry.mtime))) - for parser in parsers: - matched = parser.parse(entry.path) - if matched is not None: - row.update({key: value for key, value in matched.named.items() - if key not in columns}) - # The detector returns formats in a stable order. A row uses - # the first matching template so overlapping guesses cannot - # overwrite one another's parameter values. - break - rows.append(row) - frame = pd.DataFrame(rows, columns=columns + list(fields), dtype=object) - data_spec = {"db": "data.tsv", "extraction": { - "automatic": format is None, "formats": formats}} - remote = {"name": "origin", "url": source.url} - if source.backend: - remote["backend"] = source.backend - if source.backend_options: - remote["backend_options"] = thaw_backend_options(source.backend_options) - repo.dothm.dump_yml({"data": [data_spec], "remote": [remote]}, "config") - repo.dothm.dump_yml({"source": provenance}, "meta") - repo.dothm.dump_tsv(frame, "data", na_rep="") - _metadata_commit(repo, ["config.yml", "meta.yml", "data.tsv"], - "Discover remote catalog") - - -def initialize_remote(cls, path, source, *, release=None, collections=None, - filter=None, format=None, progress=False): - """Discover a dataset and initialize its catalog without replacing local files.""" - destination = Path(path).expanduser().absolute() - if destination.is_symlink(): - raise DestinationExistsError(f"Destination is a symbolic link: {path}") - dothm_path, _ = cls.lwpaths(destination) - if dothm_path.exists() or dothm_path.is_symlink(): - raise DestinationExistsError( - f"Hallmark repository already exists: {dothm_path}") - if destination.exists() and not destination.is_dir(): - raise NotADirectoryError(f"Destination is not a directory: {path}") - path_matches("validation", filter=filter) - if format is not None: - extraction_parser(format) - source, roots, provenance = resolve_source(source, release, collections) - # Claim only the metadata directory. On failure, existing dataset files stay put. - missing_parents = [] - parent = dothm_path.parent - while not parent.exists(): - missing_parents.append(parent) - parent = parent.parent - created_parents = [] - owns_metadata = False - try: - for parent in reversed(missing_parents): - try: - parent.mkdir() - created_parents.append(parent) - except FileExistsError: - if not parent.is_dir(): - raise - try: - dothm_path.mkdir() - owns_metadata = True - except FileExistsError as exc: - raise DestinationExistsError( - f"Hallmark repository already exists: {dothm_path}") from exc - inventory = {} - for root in roots: - scoped = source if not root else RemoteSpec.parse( - source.url.rstrip("/") + "/" + quote(root, safe="/") + "/", - backend=source.backend, backend_options=source.backend_options) - with OperationContext(scoped) as context: - for entry in discover(context, filter=filter, progress=progress, - path_prefix=root): - path = root + "/" + entry.path if root else entry.path - inventory[path] = replace(entry, path=path) - entries = [inventory[path] for path in sorted(inventory)] - repo = cls.init(destination) - _write_inventory(repo, source, entries, format, provenance) - return repo - except BaseException: - if owns_metadata: - rmtree(dothm_path, ignore_errors=True) - for parent in reversed(created_parents): - try: - parent.rmdir() - except OSError: - pass - raise - - def _git_source(url, source_type): """Determine whether the selected source should be cloned with Git.""" if source_type == "git": @@ -251,7 +136,8 @@ def _clone_git(cls, url, destination, display_path): Dothm.clone(git_url, dothm_path, display_path=display_path) except CloneError as exc: raise CloneError( - f"{exc}\nFor a raw dataset, use hallmark init PATH --from URL.") from exc + f"{exc}\nFor a raw dataset, use hallmark init PATH, then " + "hallmark add 'URL/{field}...'.") from exc if worktree_path: Worktree.init(worktree_path) return cls(destination) @@ -262,8 +148,8 @@ def clone_catalog(cls, url, path, *, source_type="auto"): Clone an existing Git catalog or published HTTP/SFTP catalog snapshot. Existing destinations are rejected before source access. An incomplete - destination created by this call is removed on failure. Dataset discovery - belongs to ``Repo.init(source=...)``. + destination created by this call is removed on failure. Raw datasets are + cataloged with ``Repo.add(url_template)`` in a new repository. Args: cls: Repository class used to initialize or open the result. @@ -306,8 +192,9 @@ def clone_catalog(cls, url, path, *, source_type="auto"): if snapshot is None: if source_type == "auto" and source.scheme in {"http", "https"}: return _clone_git(cls, url, destination, path) - raise CloneError("No published Hallmark catalog at this URL; " - "use hallmark init PATH --from URL for a raw dataset") + raise CloneError("No published Hallmark catalog at this URL; for a " + "raw dataset, use hallmark init PATH, then " + "hallmark add 'URL/{field}...'") repo = cls.init(destination) for name, contents in snapshot.items(): (repo.dothm.path / name).write_text(contents, encoding="utf-8") diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 5d19865..7eac9bb 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -326,7 +326,8 @@ def hallmark(ctx): manage data products in a complex workflow. """ # if the invoked subcommand is one of the commands that does not require a repo - if ctx.invoked_subcommand in [None, "init", "clone", "build", "sources"]: + if ctx.invoked_subcommand in [ + None, "init", "clone", "build", "sources", "ls-remote"]: # return early without attempting to open a repository return # attempt to open the hallmark repository in the current directory @@ -354,54 +355,49 @@ def sources(name): for root in roots: click.echo(f" {definition.url.rstrip('/')}/{root}/") if name: - click.echo(" Omit --collection to catalog the entire release.") + click.echo(" Run hallmark ls-remote URL to find filename templates, " + "then hallmark add 'URL/TEMPLATE'.") -@hallmark.command(short_help="Initialize a local repository or data catalog.") +@hallmark.command(short_help="Initialize an empty local repository.") @click.argument("path") -@click.option("--from", "source", help="Named data source or raw dataset URL.") -@click.option("--release", - help="Release of a named source; prompted for in a terminal.") -@click.option("--collection", "collections", multiple=True, - help="Source collection to catalog. Repeat to include several.") -@click.option("--filter", "filters", multiple=True, - help="Catalog paths matching a glob. May be repeated.") -@click.option("--format", - help="Filename template overriding automatic detection. " - "Unmatched files remain cataloged.") -def init(path, source, release, collections, filters, format): - """Initialize a local repository or discover metadata without downloading files. - - --from accepts a source name such as desi or a dataset URL. Omit --collection - and --filter to catalog the whole release or URL root. Filename formats are - detected automatically; --format supplies an explicit template instead. - Format matching adds columns without filtering files. Use download separately - to select and approve transfers. Plain init creates an empty repository without - scanning existing local files. +def init(path): + """Initialize an empty hallmark repository at PATH. + + Plain init creates an empty repository without scanning existing files. + Catalog files afterwards with hallmark add: a filename template such as + `a{a}_i{i}.h5` tracks local files, and a URL template such as + `https://host/ER2/{src}_{day}.h5` catalogs remote files without + downloading them. A PATH ending in .hm creates a bare catalog; otherwise metadata lives in PATH/.hm. """ with _translate_cli_errors( - GitError, DownloadError, ValueError, OSError, yaml.YAMLError, + GitError, ValueError, OSError, yaml.YAMLError, prefix=f'Failed to initialize hallmark repository at "{path}"'): - if source is not None and "://" not in source and release is None: - descriptor = get_source(source) - if not sys.stdin.isatty(): - raise ValueError("--release is required in noninteractive use; " - f"choose from {', '.join(descriptor.releases)}") - release = click.prompt( - "Release", type=click.Choice(list(descriptor.releases))) - if (source is None and release is None and format is None - and not collections and not filters): - Repo.init(path) - else: - Repo.init(path, source=source, release=release, - collections=collections or None, filter=filters or None, - format=format, progress=True) - if source is not None: - click.echo(f'Successfully initialized "{path}"') - click.echo("From this repository, use hallmark download --interactive " - "to choose files, or hallmark download --filter 'PATTERN' " - "--dry-run to preview a selection.") + Repo.init(path) + + +@hallmark.command("ls-remote", + short_help="List a remote directory and suggest templates.") +@click.argument("url") +def ls_remote(url): + """List the files at a remote directory URL and suggest URL templates. + + Nothing is downloaded and no repository is changed or required. Each + suggestion is a command that catalogs the matching files. + """ + from .repo_remote import suggest_templates + + with _translate_cli_errors(DownloadError, ValueError): + base, suggestions, total = suggest_templates(url, progress=True) + click.echo(f"{total} file(s) at {base}") + if not suggestions: + click.echo("No filename templates detected. Write one with {field} " + "placeholders and preview it with hallmark add -n 'URL/TEMPLATE'.") + return + click.echo("Suggested templates (files matched):") + for template, count in suggestions: + click.echo(f" hallmark add {shlex.quote(template)} ({count})") @hallmark.command(short_help="Show information of the current directory.") @@ -750,7 +746,7 @@ def clone(url, path, source_type, no_download, filters, interactive, output): transfers. --filter selects downloads without trimming the catalog. --no-download skips review; --interactive opens the selection chooser. Without a terminal, keep the catalog and print download commands for later. - Use init --from for raw datasets. + For raw datasets, use init, then add a URL template. """ if no_download and (interactive or filters or output is not None): raise ClickException( @@ -769,7 +765,7 @@ def clone(url, path, source_type, no_download, filters, interactive, output): _offer_download(repo, filters=filters, output=output, interactive=interactive) -@hallmark.command(short_help="Deprecated: use init --from for remote datasets.") +@hallmark.command(short_help="Deprecated: use init, then add a URL template.") @click.argument("directory") @click.argument("dataset_name") @click.option( @@ -802,8 +798,8 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, """ Build a catalog at DIRECTORY/DATASET_NAME.hm. - This command is deprecated. Use hallmark init --from URL PATH for remote - datasets, or hallmark init PATH to create a local repository. + This command is deprecated. Use hallmark init PATH, then + hallmark add 'URL/{field}...' to catalog remote datasets. --dataset-url selects the exact discovery root; otherwise DATASET_NAME identifies a dataset beneath the CyVerse curated-data directory. @@ -813,7 +809,8 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite, provided, existing formats are reused or a generic path catalog is created. Dataset files are not downloaded during catalog creation. """ - click.echo("build is deprecated; use hallmark init --from URL PATH.", err=True) + click.echo("build is deprecated; use hallmark init PATH, then " + "hallmark add 'URL/{field}...'.", err=True) if config_file and fmts: raise ClickException("Use only one of --config-file or --fmt, not both.") # validate the dataset name to ensure it is a valid path component diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py index 3eb4a10..53f062a 100644 --- a/mod/hallmark/discovery.py +++ b/mod/hallmark/discovery.py @@ -39,33 +39,6 @@ def match(path_index, pattern_index): return match(0, 0) -CATALOG_COLUMNS = ("path", "checksum_algorithm", "checksum", "size_bytes", "mtime") - - -@lru_cache(maxsize=128) -def extraction_parser(template): - """Validate a named filename template and return its parser and columns.""" - if not isinstance(template, str) or not template.strip(): - raise ValueError("format must be a nonempty filename template") - fields = [] - for _, name, _, conversion in Formatter().parse(template): - if name is None: - continue - if not name.isidentifier() or conversion is not None: - raise ValueError( - "Extraction fields must be simple names without conversions") - if name in {*CATALOG_COLUMNS, *RESERVED_FIELDS}: - raise ValueError(f"Extraction field {name!r} is reserved catalog metadata") - if name not in fields: - fields.append(name) - if not fields: - raise ValueError("Extraction templates require at least one named field") - parser = parse.compile(template, case_sensitive=True) - # Force regular-expression compilation now, before accessing the source. - parser.parse("") - return parser, tuple(fields) - - def _template_parts(segment: str): """Split one template segment into literal text and field definitions.""" try: @@ -298,7 +271,7 @@ def _manifest_checksums(context, entries): previous, checksum_algorithm=algorithm, checksum=digest.lower()) -def discover(context, *, filter=None, progress=False, path_prefix="", +def discover(context, *, filter=None, progress=False, descend=None) -> list[RemoteEntry]: """ Discover remote files and their published metadata recursively. @@ -309,8 +282,6 @@ def discover(context, *, filter=None, progress=False, path_prefix="", Args: context (OperationContext): Source connection and cancellation state. filter (str | list[str], optional): Relative path glob or globs. - path_prefix (str): Release-relative root for collection selection. - Returned entry paths remain relative to the context root. descend (callable, optional): Predicate receiving a relative directory path ending in ``/``; listings skip directories for which it returns False. Checksum manifests in skipped directories are @@ -332,8 +303,7 @@ def discover(context, *, filter=None, progress=False, path_prefix="", # Validate selectors before contacting the source, even for an empty index. path_matches("", filter=filter) def matches(path): - logical_path = path_prefix + "/" + path if path_prefix else path - return path_matches(logical_path, filter=filter) + return path_matches(path, filter=filter) entries = {} counts = {"directories": 0, "files": 0, "matched": 0, "current": ""} diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 8e88175..d6b6498 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -265,43 +265,19 @@ def _populate_checksums(self, pf: ParaFrame) -> None: pf["sha1"] = [checksums[path] for path in full_paths] @classmethod - def init(cls, path: Union[Path, str], *, source=None, release=None, - collections=None, filter=None, format=None, progress=False) -> "Repo": - """Initialize a local repository or catalog a named source or raw URL. + def init(cls, path: Union[Path, str]) -> "Repo": + """Initialize an empty local repository. - Discovery reads listings and published checksums without fetching data. - Existing destination files are preserved; an existing catalog is rejected. + Catalog local or remote files afterwards with ``add``; a URL template + such as ``https://host/ER2/{src}_{day}.h5`` catalogs remote files + without downloading them. Args: path (Path | str): Worktree or bare ``.hm`` repository destination. - source (str, optional): Registered data source name or dataset URL. - release (str, optional): Explicit release for a named source. - collections (str | list[str], optional): Collections within the release. - Omission catalogs the entire release. - filter (str | list[str], optional): Relative path inclusion globs. - format (str, optional): Named filename template for extra columns, - overriding automatic detection from the filtered paths. - Unmatched files remain in the catalog. Detected templates and - their columns are combined in one table; literal paths remain - authoritative. No inferred template is required for cataloging. - progress (bool | callable): Display discovery progress or receive updates. Returns: - Repo: Initialized repository without downloaded data files. - - Raises: - DestinationExistsError: If remote initialization would replace a catalog. - ValueError: If source options or the extraction template are invalid. - DownloadError: If remote discovery fails. + Repo: The initialized repository. """ - from .catalog import initialize_remote - - if source is not None: - return initialize_remote( - cls, path, source, release=release, collections=collections, - filter=filter, format=format, progress=progress) - if any(value is not None for value in (release, collections, filter, format)): - raise ValueError("Remote initialization options require source") dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path / "config.yml").write_text(Dothm.config_template(), diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index 8ffab21..1fbc598 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -725,7 +725,8 @@ def build_repo( """ Build a remote catalog using the deprecated builder interface. - Prefer ``Repo.init(path, source=url)`` for remote datasets. + Prefer ``Repo.init(path)`` followed by ``Repo.add(url_template)`` for + remote datasets. Discovery reads listings and published checksum manifests without downloading dataset files. @@ -763,7 +764,8 @@ def build_repo( DownloadError: If remote discovery fails. """ warnings.warn( - "build_repo is deprecated; use Repo.init(path, source=url)", + "build_repo is deprecated; use Repo.init(path) and " + "Repo.add(url_template)", DeprecationWarning, stacklevel=2) if remote_hash: raise CapabilityError( diff --git a/mod/hallmark/sources.py b/mod/hallmark/sources.py index 5739862..ef247a1 100644 --- a/mod/hallmark/sources.py +++ b/mod/hallmark/sources.py @@ -166,42 +166,3 @@ def backend_for_url(url: str): if release.backend and (url.rstrip("/") + "/").startswith(root): return release.backend, release.backend_options return None - - -def resolve_source(source, release=None, collections=None): - """Return a transport, crawl roots, and provenance for a name or raw URL.""" - if not isinstance(source, str) or not source.strip(): - raise ValueError("source must be a data source name or URL") - source = source.strip() - if "://" in source: - if release is not None or collections is not None: - raise ValueError("release and collections require a named data source") - remote = RemoteSpec.parse(source) - return remote, ("",), {"url": remote.url} - descriptor = get_source(source) - if release is None: - raise ValueError(f"Source {source!r} requires a release; " - f"choose from {', '.join(descriptor.releases)}") - if release not in descriptor.releases: - raise ValueError(f"Unknown release {release!r} for {source}; " - f"choose from {', '.join(descriptor.releases)}") - definition = descriptor.releases[release] - if isinstance(collections, str): - collections = [collections] - if collections is not None and not isinstance(collections, (list, tuple)): - raise ValueError("collections must be a name or a list of names") - selected = [] - for name in collections or (): - if not isinstance(name, str) or name not in definition.collections: - raise ValueError(f"Unknown collection {name!r} for {source}/{release}; " - f"choose from {', '.join(definition.collections)}") - if name not in selected: - selected.append(name) - roots = {path for name in selected for path in definition.collections[name]} - # Parent roots cover nested roots; do not crawl overlapping trees twice. - roots = tuple(path for path in sorted(roots) if not any( - path.startswith(parent + "/") for parent in roots if parent != path)) - return definition.remote(), roots or ("",), { - "name": source, "release": release, "collections": selected, - "url": definition.url, - } diff --git a/test/test_backends.py b/test/test_backends.py index ad2603e..939b961 100644 --- a/test/test_backends.py +++ b/test/test_backends.py @@ -307,10 +307,12 @@ def test_multi_server_example_uses_shared_discovery_and_verified_downloads(tmp_p register_source(DataSource("multi-example", "Multiple servers", { "v1": SourceRelease("https://logical.test/dataset/", {}, backend="multi-server", backend_options=options)})) - repo = Repo.init(tmp_path / "repo", source="multi-example", release="v1", - filter="**/*.fits") - assert repo.state.data["path"].tolist() == [ - "north/file.fits", "south/file.fits"] + repo = Repo.init(tmp_path / "repo") + # the template inherits the release's backend and routes + repo.add("https://logical.test/dataset/{site}/file.fits") + assert repo.state.data["site"].tolist() == ["north", "south"] + assert repo.state.data["checksum_algorithm"].tolist() == [ + "sha256", "sha256"] assert requests == {"north": ["/", "/sha256sums"], "south": ["/", "/sha256sums"]} repo = Repo(repo.worktree) diff --git a/test/test_catalog.py b/test/test_catalog.py index 4e96c20..d853c46 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -1,4 +1,4 @@ -"""Test remote initialization and cloning without downloading dataset payloads.""" +"""Test cloning catalogs and cataloging remote files without downloading payloads.""" from pathlib import Path @@ -27,63 +27,37 @@ def read_text(context, path): return pages, reads -def test_remote_init_is_recursive_filtered_and_payload_free( - tmp_path, metadata_server): - pages, reads = metadata_server - root = "https://example.test/data/" - pages[root] = ('

Index of /data/

root.h5' - 'note.txtsub/') - pages[root + "sub/"] = '

Index of sub

data.h5' - events = [] - repo = Repo.init(tmp_path / "clone", source=root, filter="**/*.h5", - progress=events.append) - assert repo.state.data["path"].tolist() == ["root.h5", "sub/data.h5"] - assert repo.state.config["remote"] == [ - {"name": "origin", "url": root, "backend": "http"}] - assert repo.state.config["data"] == [{"db": "data.tsv", "extraction": { - "automatic": True, "formats": []}}] - assert not (repo.worktree / "root.h5").exists() - assert not any(url.endswith((".h5", ".txt")) for url in reads) - plan = repo.plan_download() - assert plan.file_count == 2 - assert plan.unknown_size_count == 2 - assert events[-1]["directories"] == 2 - assert events[-1]["matched"] == 2 - assert events[-1]["files"] == 3 - - -def test_remote_init_does_not_require_filename_inference(tmp_path, metadata_server): +def test_remote_add_decodes_a_literal_file_url(tmp_path, metadata_server): pages, _ = metadata_server root = "https://example.test/odd/" pages[root] = '

Index of odd

odd name' - repo = Repo.init(tmp_path / "clone", source=root) - assert repo.state.data["path"].tolist() == ["odd name"] - assert repo.plan_download().file_count == 1 + repo = Repo.init(tmp_path / "clone") + repo.add(root + "odd%20name") + assert repo.state.config["data"][0] == {"fmt": "odd name", "url": root} + assert repo.plan_download().items[0].relative_path == Path("odd name") -def test_empty_filtered_init_is_valid(tmp_path, metadata_server): - pages, _ = metadata_server - root = "https://example.test/data/" - pages[root] = '

Index of data

notes.txt' - repo = Repo.init(tmp_path / "clone", source=root, format="run{run:d}.h5") - assert repo.state.data["path"].tolist() == ["notes.txt"] - assert "run" in repo.state.data.columns - assert repo.state.data.iloc[0]["run"] == "" - assert repo.plan_download().file_count == 1 +@pytest.mark.parametrize("failure", [KeyboardInterrupt, DownloadError]) +def test_failed_remote_discovery_leaves_the_catalog_unchanged( + tmp_path, metadata_server, monkeypatch, failure): + repo = Repo.init(tmp_path / "existing") + (repo.worktree / "keep.h5").write_bytes(b"existing scientific data") + repo.add("{name}.h5") + stored = {path.name: path.read_bytes() + for path in repo.dothm.path.iterdir() if path.is_file()} + staged = {key: entry.binsha for key, entry in repo.dothm.index.entries.items()} + def fail(*args, **kwargs): + raise failure("interrupted") -def test_template_enriches_without_filtering_inventory(tmp_path, metadata_server): - pages, _ = metadata_server - root = "https://example.test/data/" - pages[root] = ('

Index of data

run2.h5' - 'notes.txt') - repo = Repo.init(tmp_path / "clone", source=root, format="run{run:d}.h5") - assert repo.state.data["path"].tolist() == ["notes.txt", "run2.h5"] - assert repo.state.data.iloc[0]["run"] == "" - assert float(repo.state.data.iloc[1]["run"]) == 2 - assert repo.state.config["data"] == [ - {"db": "data.tsv", "extraction": { - "automatic": False, "formats": ["run{run:d}.h5"]}}] + monkeypatch.setattr("hallmark.repo_remote.discover", fail) + with pytest.raises(failure): + repo.add("https://example.test/data/{run}.fits") + assert {path.name: path.read_bytes() + for path in repo.dothm.path.iterdir() if path.is_file()} == stored + assert {key: entry.binsha + for key, entry in repo.dothm.index.entries.items()} == staged + assert (repo.worktree / "keep.h5").read_bytes() == b"existing scientific data" @pytest.mark.parametrize("nested", [False, True]) @@ -141,40 +115,6 @@ def test_clone_requires_callback_before_download_intent(tmp_path, metadata_serve assert not (tmp_path / "clone").exists() -def test_init_refusal_keeps_catalog_without_payload( - tmp_path, metadata_server, monkeypatch): - pages, _ = metadata_server - root = "https://example.test/data/" - pages[root] = '

Index of data

a.h5' - plans = [] - - def decline(plan): - plans.append(plan) - return False - - def fail(*args, **kwargs): - raise AssertionError("Refused transfer must not start") - - monkeypatch.setattr(Repo, "download", fail) - repo = Repo.init(tmp_path / "clone", source=root) - decline(repo.plan_download()) - assert len(plans) == 1 - assert plans[0].file_count == 1 - assert repo.download_result is None - assert not (repo.worktree / "a.h5").exists() - - -def test_interrupted_discovery_removes_incomplete_destination( - tmp_path, metadata_server, monkeypatch): - def fail(*args, **kwargs): - raise KeyboardInterrupt() - - monkeypatch.setattr("hallmark.catalog.discover", fail) - with pytest.raises(KeyboardInterrupt): - Repo.init(tmp_path / "clone", source="https://example.test/data/") - assert not (tmp_path / "clone").exists() - - def test_existing_destination_is_never_changed(tmp_path, metadata_server): destination = tmp_path / "clone" destination.mkdir() @@ -233,71 +173,6 @@ def test_snapshot_validates_tables_without_filter(tmp_path, metadata_server, tab assert not (tmp_path / "clone").exists() -@pytest.mark.parametrize("failure", [KeyboardInterrupt, DownloadError]) -def test_remote_init_preserves_existing_directory_on_discovery_failure( - tmp_path, metadata_server, monkeypatch, failure): - destination = tmp_path / "existing" - destination.mkdir() - (destination / "keep.h5").write_bytes(b"existing scientific data") - - def fail(*args, **kwargs): - raise failure("interrupted") - - monkeypatch.setattr("hallmark.catalog.discover", fail) - with pytest.raises(failure): - Repo.init(destination, source="https://example.test/data/") - assert list(destination.iterdir()) == [destination / "keep.h5"] - assert (destination / "keep.h5").read_bytes() == b"existing scientific data" - - -def test_remote_init_preserves_existing_payload_and_rejects_existing_catalog( - tmp_path, metadata_server): - pages, reads = metadata_server - root = "https://example.test/data/" - pages[root] = '

Index of data

keep.h5' - destination = tmp_path / "existing" - destination.mkdir() - (destination / "keep.h5").write_bytes(b"local") - repo = Repo.init(destination, source=root) - assert repo.state.data["path"].tolist() == ["keep.h5"] - assert (destination / "keep.h5").read_bytes() == b"local" - head = repo.dothm.head.commit.hexsha - reads.clear() - with pytest.raises(DestinationExistsError): - Repo.init(destination, source=root) - assert reads == [] - assert Repo(destination).dothm.head.commit.hexsha == head - - -def test_remote_init_rejects_bare_symlink_before_access(tmp_path, metadata_server): - target = tmp_path / "existing" - target.mkdir() - (target / "keep").write_text("keep") - alias = tmp_path / "alias.hm" - alias.symlink_to(target, target_is_directory=True) - with pytest.raises(DestinationExistsError): - Repo.init(alias, source="https://example.test/data/") - assert metadata_server[1] == [] - assert list(target.iterdir()) == [target / "keep"] - - -def test_remote_init_cleans_only_created_metadata_on_write_failure( - tmp_path, metadata_server, monkeypatch): - destination = tmp_path / "existing" - destination.mkdir() - (destination / "keep").write_text("keep") - root = "https://example.test/data/" - metadata_server[0][root] = '

Index of data

' - - def fail(*args, **kwargs): - raise OSError("cannot write catalog") - - monkeypatch.setattr("hallmark.catalog._write_inventory", fail) - with pytest.raises(OSError, match="cannot write"): - Repo.init(destination, source=root) - assert list(destination.iterdir()) == [destination / "keep"] - - @pytest.mark.parametrize("kwargs", [ {"filter": "*.fits"}, {"fmt": "{name}.fits"}, {"source_type": "directory"}, {"download": True, "approve": lambda plan: True, "fmt": "{name:invalid}"}, @@ -320,7 +195,7 @@ def fail(*args, **kwargs): raise CloneError("not a Git repository") monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) - with pytest.raises(CloneError, match="hallmark init PATH --from URL"): + with pytest.raises(CloneError, match="hallmark init PATH, then hallmark add"): Repo.clone(root, tmp_path / "clone") assert root not in reads assert not (tmp_path / "clone").exists() @@ -405,55 +280,6 @@ def test_snapshot_clone_does_not_require_payload_authentication( assert not repo.dothm.remotes -def test_init_download_requires_callback_before_source_access( - tmp_path, metadata_server): - with pytest.raises(TypeError, match="download"): - Repo.init(tmp_path / "new", source="https://example.test/data/", - download=True) - assert metadata_server[1] == [] - - -def test_empty_init_download_does_not_request_approval(tmp_path, metadata_server): - root = "https://example.test/empty/" - metadata_server[0][root] = '

Index of empty

' - - def fail(plan): - raise AssertionError("Empty plans require no confirmation") - - repo = Repo.init(tmp_path / "new", source=root) - assert repo.plan_download().file_count == 0 - assert repo.state.data.empty - assert repo.download_result is None - - -def test_init_downloads_only_after_approved_catalog_creation( - tmp_path, metadata_server, monkeypatch): - import requests - from mock_server import MockServer - - root = "https://example.test/data/" - metadata_server[0][root] = '

Index of data

a.fits' - server = MockServer(root) - server.add_file("a.fits", b"fits") - monkeypatch.setattr(requests, "Session", lambda: server) - destination = tmp_path / "new" - plans = [] - - def approve(plan): - assert Repo(destination).state.data["path"].tolist() == ["a.fits"] - assert not (destination / "a.fits").exists() - plans.append(plan) - return True - - repo = Repo.init(destination, source=root) - plan = repo.plan_download() - assert approve(plan) - repo.download(plan, approved=True) - assert len(plans) == 1 - assert (destination / "a.fits").read_bytes() == b"fits" - assert repo.download_result["succeeded"] == 1 - - def test_snapshot_access_failure_never_falls_back_to_git(tmp_path, monkeypatch): def inaccessible(context, path): raise DownloadError("Metadata server requires authentication") diff --git a/test/test_cli.py b/test/test_cli.py index 407b29a..04bd05e 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -1325,22 +1325,13 @@ def reject_download(*args, **kwargs): assert "Download these files?" not in result.output -def test_init_cli_forwards_discovery_options(monkeypatch, tmp_path): - captured = {} - - def initialize(path, **kwargs): - captured.update(path=path, **kwargs) - - monkeypatch.setattr(Repo, "init", initialize) - result = CliRunner().invoke(hallmark, [ - "init", str(tmp_path / "target"), "--from", "desi", "--release", "dr1", - "--collection", "redshifts", "--filter", "**/*.fits", - "--filter", "README*", "--format", "{name}.fits"]) +def test_init_cli_creates_only_a_local_repository(monkeypatch, tmp_path): + captured = [] + monkeypatch.setattr(Repo, "init", lambda *args, **kwargs: captured.append( + (args, kwargs))) + result = CliRunner().invoke(hallmark, ["init", str(tmp_path / "target")]) assert result.exit_code == 0, result.output - assert captured == { - "source": "desi", "path": str(tmp_path / "target"), "release": "dr1", - "collections": ("redshifts",), "filter": ("**/*.fits", "README*"), - "format": "{name}.fits", "progress": True} + assert captured == [((str(tmp_path / "target"),), {})] def test_clone_cli_rejects_conflicting_download_aliases(): @@ -1395,24 +1386,6 @@ def capture_get(url, **kwargs): assert not (target / "other.txt").exists() -@pytest.mark.parametrize("options", ["[]\n", "scalar\n", "options: [\n"]) -def test_init_cli_rejects_invalid_backend_options_before_access( - monkeypatch, tmp_path, options): - config = tmp_path / "options.yml" - config.write_text(options) - - def fail(*args, **kwargs): - raise AssertionError("Invalid options must be rejected before initialization") - - monkeypatch.setattr(Repo, "init", fail) - result = CliRunner().invoke(hallmark, [ - "init", str(tmp_path / "target"), "--from", "https://example.test/data/", - "--backend-options", str(config)]) - assert result.exit_code != 0 - assert "Error:" in result.output - assert not (tmp_path / "target").exists() - - @pytest.mark.parametrize("arguments", [ ["--interactive", "--filter", "*.fits"], ["--fmt", "{name}.fits"], ["--with-download", "--fmt", "{name:invalid}"], @@ -1428,20 +1401,6 @@ def fail(*args, **kwargs): assert "Error:" in result.output -def test_init_cli_success_does_not_echo_source_credentials(monkeypatch, tmp_path): - def initialize(path, **kwargs): - return SimpleNamespace(worktree=Path(path)) - - monkeypatch.setattr(Repo, "init", initialize) - result = CliRunner().invoke(hallmark, [ - "init", str(tmp_path / "target"), "--from", - "https://user:secret@example.test/data/?token=private"]) - assert result.exit_code == 0, result.output - assert "Successfully initialized" in result.output - assert "secret" not in result.output - assert "private" not in result.output - - @pytest.mark.parametrize("option", ["--remote-backend", "--remote-backend-options"]) def test_set_config_cli_rejects_backend_flags(monkeypatch, tmp_path, option): repo = Repo.init(tmp_path / "repo") @@ -1452,7 +1411,7 @@ def test_set_config_cli_rejects_backend_flags(monkeypatch, tmp_path, option): @pytest.mark.parametrize("answer", ["y\n", "n\n", ""]) -def test_init_cli_downloads_only_after_confirmation(monkeypatch, tmp_path, answer): +def test_remote_add_downloads_only_after_confirmation(monkeypatch, tmp_path, answer): from hallmark.transport import OperationContext from hallmark.transport.base import RemoteObjectMissing from mock_server import MockServer @@ -1467,13 +1426,15 @@ def metadata(context, path): monkeypatch.setattr(OperationContext, "read_text", metadata) monkeypatch.setattr(requests, "Session", lambda: server) destination = tmp_path / "target" - result = CliRunner().invoke(hallmark, [ - "init", str(destination), "--from", "https://example.test/data/"]) + result = CliRunner().invoke(hallmark, ["init", str(destination)]) assert result.exit_code == 0, result.output - assert "Download these files?" not in result.output monkeypatch.chdir(destination) + result = CliRunner().invoke( + hallmark, ["add", "https://example.test/data/{name}.fits"]) + assert result.exit_code == 0, result.output + assert "Download these files?" not in result.output result = CliRunner().invoke(hallmark, ["download", "--all"], input=answer) assert "Download these files? [y/N]" in result.output - assert Repo(destination).state.data["path"].tolist() == ["a.fits"] + assert Repo(destination).state.data["name"].tolist() == ["a"] assert (destination / "a.fits").exists() == (answer == "y\n") assert (result.exit_code == 0) == (answer == "y\n") diff --git a/test/test_interactive_download.py b/test/test_interactive_download.py index cb99299..4aca17c 100644 --- a/test/test_interactive_download.py +++ b/test/test_interactive_download.py @@ -519,40 +519,41 @@ def test_clone_without_downloadable_data_completes(catalog, tmp_path, kind): @pytest.mark.parametrize("remote", [False, True]) -def test_init_never_scans_local_files_or_offers_downloads( +def test_init_and_remote_add_never_scan_local_files_or_offer_downloads( catalog, tmp_path, monkeypatch, remote): _, transfers = catalog destination = tmp_path / "initialized" destination.mkdir() local = destination / "existing.h5" local.write_bytes(b"untouched") - arguments = ["init", str(destination)] - if remote: - init_pages(monkeypatch) - arguments += ["--from", ROOT, "--filter", "runs/*.h5", - "--format", "runs/run_{run:03d}.h5"] - else: + if not remote: def no_remote_access(*args, **kwargs): raise AssertionError("Local init must not contact a remote") monkeypatch.setattr(OperationContext, "__enter__", no_remote_access) def no_download(*args, **kwargs): - raise AssertionError("Init must not offer, plan, or execute downloads") + raise AssertionError("Init and add must not offer, plan, or execute downloads") monkeypatch.setattr(cli, "_offer_download", no_download) monkeypatch.setattr(Repo, "plan_download", no_download) monkeypatch.setattr(Repo, "download", no_download) - result = TerminalRunner().invoke(hallmark, arguments) + result = TerminalRunner().invoke(hallmark, ["init", str(destination)]) assert result.exit_code == 0, result.output + if remote: + init_pages(monkeypatch) + monkeypatch.chdir(destination) + result = TerminalRunner().invoke( + hallmark, ["add", ROOT + "{group}/run_{run:03d}.h5"]) + assert result.exit_code == 0, result.output repo = Repo(destination) assert local.read_bytes() == b"untouched" assert transfers == [] assert "Download these files?" not in result.output assert "Select files" not in result.output if remote: - assert "hallmark download --interactive" in result.output - assert repo.state.data["path"].tolist() == [ - "runs/run_001.h5", "runs/run_002.h5"] + assert "then hallmark download" in result.output + assert repo.state.data[["group", "run"]].values.tolist() == [ + ["runs", "001"], ["runs", "002"]] else: assert repo.state.data.empty assert not list((repo.dothm.path / "objects").rglob("*")) diff --git a/test/test_sources.py b/test/test_sources.py index ab4106d..ea2ebbd 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -1,6 +1,5 @@ -"""Named-source and extraction workflows using only in-memory metadata fixtures.""" +"""Named sources, template suggestions and URL templates with in-memory listings.""" -from pathlib import Path from types import SimpleNamespace import pytest @@ -10,9 +9,11 @@ from hallmark import DataSource, Repo, SourceRelease, register_source from hallmark import sources from hallmark.cli import hallmark -from hallmark.sources import get_source, list_sources, resolve_source +from hallmark.discovery import TemplateMatcher +from hallmark.repo_remote import suggest_templates +from hallmark.sources import backend_for_url, get_source, list_sources from hallmark.transport import OperationContext -from hallmark.transport.base import DownloadError, RemoteObjectMissing +from hallmark.transport.base import RemoteObjectMissing def index(*paths): @@ -54,179 +55,92 @@ def test_desi_definitions_are_explicit_and_offline(pages): result = CliRunner().invoke(hallmark, ["sources", "desi"]) assert result.exit_code == 0, result.output assert "spectro/redux/iron/zcatalog/" in result.output - assert "entire release" in result.output + assert "hallmark ls-remote URL" in result.output assert pages[1] == [] -def test_collection_union_crawls_only_selected_roots(tmp_path, pages): +def test_collection_urls_from_sources_feed_add(tmp_path, pages): content, reads = pages - root = "https://data.desi.lbl.gov/public/dr1/" - zcat = "spectro/redux/iron/zcatalog/" - content[root + zcat] = index("zall.fits", "notes.txt") - content[root + "vac/"] = index("extra.fits") - events = [] - repo = Repo.init(tmp_path / "desi", source="desi", release="dr1", - collections=["redshifts", "vac", "redshifts"], - filter="**/*.fits", progress=events.append) - assert any(event["files"] == 2 and event["matched"] == 1 for event in events) - expected = [zcat + "zall.fits", "vac/extra.fits"] - assert repo.state.data["path"].tolist() == expected - assert reads == [root + zcat, root + "vac/"] - stored = yaml.safe_load((repo.dothm.path / "meta.yml").read_text())["source"] - assert stored == {"name": "desi", "release": "dr1", "url": root, - "collections": ["redshifts", "vac"]} - repo = Repo(repo.worktree) - plan = repo.plan_download(filter="**/zall.fits") - assert plan.remote_url == root - assert [item.relative_path.as_posix() for item in plan.items] == expected[:1] - assert reads == [root + zcat, root + "vac/"] - with pytest.raises(DownloadError, match="not in the catalog"): - repo.plan_download(file_paths=["spectro/omitted.fits"]) - - -def test_omitting_collection_includes_entire_release(tmp_path, pages): - root = "https://data.desi.lbl.gov/public/edr/" - pages[0].update({root: index("LICENSE.md", "spectro/", "target/"), - root + "spectro/": index("spectra.fits"), - root + "target/": index("targets.fits")}) - repo = Repo.init(tmp_path / "all", source="desi", release="edr") - assert repo.state.data["path"].tolist() == [ - "LICENSE.md", "spectro/spectra.fits", "target/targets.fits"] - assert not (repo.worktree / "LICENSE.md").exists() - - -@pytest.mark.parametrize("options, message", [ - ({"source": "desi"}, "requires a release"), - ({"source": "desi", "release": "dr2"}, "Unknown release"), - ({"source": "desi", "release": "edr", "collections": ["lss"]}, - "Unknown collection"), - ({"source": "unknown"}, "Unknown data source"), - ({"source": "https://test/data/", "release": "dr1"}, "named data source"), - ({"source": "https://test/data/", "collections": []}, "named data source"), - ({"filter": "*.fits"}, "require source"), -]) -def test_invalid_source_selection_precedes_io(tmp_path, pages, options, message): - destination = tmp_path / "catalog" - with pytest.raises(ValueError, match=message): - Repo.init(destination, **options) - assert pages[1] == [] - assert not destination.exists() - - -def test_cli_release_prompt_and_cancellation(tmp_path, monkeypatch): - calls = [] - monkeypatch.setattr(Repo, "init", lambda path, **kw: calls.append((path, kw))) - runner = CliRunner() - # Click replaces stdin during invocation, so set isatty on the replacement. - original = get_source - cancel = False - - def terminal_source(name): - import sys - monkeypatch.setattr(sys.stdin, "isatty", lambda: True) - if cancel: - # Click 8.1's runner returns empty strings forever at EOF. Model a - # real terminal's EOF instead; CliRunner restores this hook on exit. - from click import termui - - def eof_input(prompt): - raise EOFError() - - termui.visible_prompt_func = eof_input - return original(name) - - monkeypatch.setattr("hallmark.cli.get_source", terminal_source) - destination = str(tmp_path / "repo") - result = runner.invoke(hallmark, ["init", destination, "--from", "desi"], - input="invalid\ndr1\n") - assert result.exit_code == 0, result.output - assert calls[0][1]["release"] == "dr1" - assert calls[0][1]["collections"] is None - calls.clear() - cancel = True - result = runner.invoke(hallmark, ["init", destination, "--from", "desi"], input="") - assert result.exit_code != 0 - assert calls == [] - assert not Path(destination).exists() - - -def test_cli_requires_release_without_terminal(tmp_path, pages): - result = CliRunner().invoke(hallmark, [ - "init", str(tmp_path / "repo"), "--from", "desi"], input="dr1\n") - assert result.exit_code != 0 - assert "--release is required in noninteractive use" in result.output - assert not (tmp_path / "repo").exists() - assert pages[1] == [] + zcatalog = "https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/zcatalog/" + content[zcatalog] = index("zall-pix-iron.fits", "zall-tilecumulative-iron.fits", + "notes.txt", "v1/") + repo = Repo.init(tmp_path / "desi") + repo.add(zcatalog + "zall-{kind}-iron.fits") + assert repo.state.data["kind"].tolist() == ["pix", "tilecumulative"] + # the template has one segment, so subdirectories are never listed + assert reads == [zcatalog] + plan = repo.plan_download(filter="*pix*") + assert plan.remote_url == zcatalog + assert [item.relative_path.as_posix() for item in plan.items] == [ + "zall-pix-iron.fits"] @pytest.mark.parametrize("template", [ - "{", "{run:invalid}", "{path}", "{checksum}", "{sha1}", - "{size_bytes}", "{mtime}", "{checksum_algorithm}", "{}", "{0}", - "{a.b}", "{a[0]}", "{run!r}", "constant", "", + "{", "{path}", "{checksum}", "{sha1}", "{md5}", "{size_bytes}", "{mtime}", + "{checksum_algorithm}", "{}", "{0}", "{a.b}", "{a[0]}", "{run!r}", "", + "{run}/", ]) -def test_invalid_extraction_precedes_io(tmp_path, pages, template): +def test_invalid_url_templates_are_rejected_before_access(tmp_path, pages, template): + repo = Repo.init(tmp_path / "repo") with pytest.raises(ValueError): - Repo.init(tmp_path / "invalid", source="https://test/data/", format=template) + repo.add("https://test/data/" + template) assert pages[1] == [] - assert not (tmp_path / "invalid").exists() -def test_extraction_retains_unmatched_files_through_reload_and_download( +def test_remote_template_survives_reload_rename_and_clone( tmp_path, pages, monkeypatch): from mock_server import MockServer root = "https://test/data/" pages[0][root] = index("run_001.h5", "RUN_002.h5", "README.md", "omit.bin") - repo = Repo.init(tmp_path / "repo", source=root, - filter=["*.h5", "README*"], format="run_{run:03d}.h5") + repo = Repo.init(tmp_path / "repo") + repo.add(root + "run_{run:03d}.h5") + repo.commit("Catalog runs") before = (repo.dothm.path / "data.tsv").read_bytes() repo = Repo(repo.worktree) - rows = repo.state.data.set_index("path") - assert set(rows.index) == {"run_001.h5", "RUN_002.h5", "README.md"} - assert float(rows.loc["run_001.h5", "run"]) == 1 - assert rows.loc["README.md", "run"] == "" - assert rows.loc["RUN_002.h5", "run"] == "" + # templates are case-sensitive, and other files are not cataloged + assert repo.state.data["run"].tolist() == ["001"] server = MockServer(root) - server.add_file("README.md", b"notes") + server.add_file("run_001.h5", b"run one") monkeypatch.setattr("requests.Session", lambda: server) - plan = repo.plan_download(filter="README*") - assert plan.file_count == 1 - result = repo.download(plan, approved=True) - assert result["succeeded"] == 1 - assert (repo.worktree / "README.md").read_bytes() == b"notes" + plan = repo.plan_download() + assert repo.download(plan, approved=True)["succeeded"] == 1 + assert (repo.worktree / "run_001.h5").read_bytes() == b"run one" assert (repo.dothm.path / "data.tsv").read_bytes() == before - # Configuration changes and a metadata clone preserve extraction and literal paths. - repo.set_config(remote_name="archive") - repo.dothm.index.add(["config.yml"]) - repo.dothm.index.commit("Rename data remote") + # Configuring a mirror and cloning the catalog preserve the template source. + repo.set_config(remote_name="archive", remote_url="https://mirror.test/data/") + repo.commit("Add a mirror") clone = Repo.clone(str(repo.dothm.path), tmp_path / "copy") assert clone.state.config["data"] == repo.state.config["data"] - assert clone.plan_download().file_count == 3 - assert not (clone.worktree / "README.md").exists() - - -def test_empty_catalog_retains_extraction_columns(tmp_path, pages): - root = "https://test/empty/" - pages[0][root] = index() - repo = Repo.init(tmp_path / "empty", source=root, format="run_{run:d}.h5") - assert "run" in repo.state.data.columns - assert repo.state.data.empty - assert repo.plan_download().file_count == 0 + assert clone.plan_download().remote_url == root + assert clone.plan_download(remote_name="archive").remote_url == ( + "https://mirror.test/data/") + assert not (clone.worktree / "run_001.h5").exists() -def test_source_registration_and_overlapping_roots(registry): - release = SourceRelease("https://test/", {"both": ["a", "a/b"], "one": "a"}) +def test_source_registration_and_backend_lookup(registry): + release = SourceRelease("https://test/", {"both": ["a", "a/b"], "one": "a"}, + backend="cyverse") descriptor = DataSource("example", "Example source", {"v1": release}) register_source(descriptor) - _, roots, provenance = resolve_source("example", "v1", ["both", "one"]) - assert roots == ("a",) - assert provenance["collections"] == ["both", "one"] + assert backend_for_url("https://test/a/b/")[0] == "cyverse" + assert backend_for_url("https://other.test/") is None with pytest.raises(ValueError, match="Duplicate"): register_source(descriptor) with pytest.raises(TypeError): descriptor.releases["v2"] = release +def test_add_inherits_the_backend_of_a_registered_release(tmp_path, pages, registry): + root = "https://test/release/" + register_source(DataSource("example", "Example source", {"v1": SourceRelease( + root, {}, backend="cyverse")})) + pages[0][root + "runs/"] = index("run_1.h5") + repo = Repo.init(tmp_path / "repo") + repo.add(root + "runs/run_{run}.h5") + assert repo.state.config["data"][0]["backend"] == "cyverse" + + def test_source_plugin_validation(registry, monkeypatch): descriptor = DataSource("plugin", "A plugin", {"v1": SourceRelease( "https://test/", {})}) @@ -265,11 +179,12 @@ def test_old_auth_references_require_explicit_migration(tmp_path, pages, operati @pytest.mark.parametrize("option", [ "--auth", "--backend", "--backend-options", "--include", "--extract", "--fmt", - "--max-workers", "--with-download", + "--max-workers", "--with-download", "--from", "--release", "--collection", + "--filter", "--format", ]) def test_init_removed_options_have_no_alias(tmp_path, pages, option): destination = tmp_path / "new" - result = CliRunner().invoke(hallmark, ["init", str(destination), option]) + result = CliRunner().invoke(hallmark, ["init", str(destination), option, "x"]) assert result.exit_code == 2 assert "No such option" in result.output assert not destination.exists() @@ -279,6 +194,8 @@ def test_init_removed_options_have_no_alias(tmp_path, pages, option): @pytest.mark.parametrize("kwargs", [ {"from_url": "https://test/"}, {"auth": "old"}, {"backend": "http"}, {"include": "*"}, {"extract": "{name}"}, {"fmt": "{name}"}, {"download": True}, + {"source": "https://test/"}, {"release": "dr1"}, {"collections": ["a"]}, + {"filter": "*"}, {"format": "{name}"}, ]) def test_init_removed_python_arguments_have_no_shim(tmp_path, kwargs): with pytest.raises(TypeError): @@ -286,121 +203,52 @@ def test_init_removed_python_arguments_have_no_shim(tmp_path, kwargs): assert not (tmp_path / "new").exists() -@pytest.mark.parametrize("option", ["--release", "--format"]) -def test_empty_remote_option_does_not_silently_initialize_locally(tmp_path, option): - destination = tmp_path / "new" - result = CliRunner().invoke(hallmark, ["init", str(destination), option, ""]) - assert result.exit_code != 0 - assert "require source" in result.output - assert not destination.exists() - - -def test_automatic_formats_preserve_mixed_catalog_and_downloads( - tmp_path, pages, monkeypatch): - from mock_server import MockServer - +def test_ls_remote_suggests_templates_from_filenames(tmp_path, pages): root = "https://test/data/" paths = ["run_001.h5", "run_002.h5", "sgra_20170406_nustar.fits", "sgra_20170411_chandra.fits", "README.md", "archive.tar.gz"] pages[0][root] = index(*paths) - repo = Repo.init(tmp_path / "auto", source=root) + base, suggestions, total = suggest_templates(root) assert pages[1] == [root] # Detection uses filenames, never payloads. - stored = repo.state.config["data"][0] - assert "fmt" not in stored - assert stored["extraction"] == { - "automatic": True, - "formats": ["run_{scan}.h5", "sgra_{date}_{p0}.fits"], - } - repo = Repo(repo.worktree) - rows = repo.state.data.set_index("path") - assert set(rows.index) == set(paths) - assert str(rows.loc["run_001.h5", "scan"]).zfill(3) == "001" - assert str(rows.loc["sgra_20170406_nustar.fits", "date"]) == "20170406" - assert rows.loc["sgra_20170406_nustar.fits", "p0"] == "nustar" - for path in ["README.md", "archive.tar.gz"]: - assert all(rows.loc[path, field] == "" for field in ("scan", "date", "p0")) - assert rows.loc["run_001.h5", "date"] == "" - assert rows.loc["sgra_20170406_nustar.fits", "scan"] == "" - - before = (repo.dothm.path / "data.tsv").read_bytes() - server = MockServer(root) - for path in ["run_001.h5", "README.md"]: - server.add_file(path, b"tiny fixture") - monkeypatch.setattr("requests.Session", lambda: server) - plan = repo.plan_download(filter=["run_001.h5", "README*"]) - assert {item.relative_path.as_posix() for item in plan.items} == { - "run_001.h5", "README.md"} - assert repo.download(plan, approved=True)["succeeded"] == 2 - assert (repo.dothm.path / "data.tsv").read_bytes() == before - repo.set_config(remote_name="archive") - repo.dothm.index.add(["config.yml"]) - repo.dothm.index.commit("Rename remote") - clone = Repo.clone(str(repo.dothm.path), tmp_path / "copy") - assert clone.state.config["data"] == [stored] - assert clone.plan_download().file_count == len(paths) - assert (clone.dothm.path / "data.tsv").read_bytes() == before - assert not (clone.worktree / "run_001.h5").exists() - - -def test_explicit_format_bypasses_detection(tmp_path, pages, monkeypatch): - def unexpected_detection(*args, **kwargs): - raise AssertionError("Explicit format must override inference") - - monkeypatch.setattr("hallmark.catalog.detect_fmt", unexpected_detection) - root = "https://test/data/" - pages[0][root] = index("run_001.h5", "run_002.h5", "README.md") - repo = Repo.init(tmp_path / "explicit", source=root, format="run_{run:03d}.h5") - assert "run" in repo.state.data.columns - assert "scan" not in repo.state.data.columns - assert len(repo.state.data) == 3 - assert repo.state.config["data"][0]["extraction"] == { - "automatic": False, "formats": ["run_{run:03d}.h5"]} - - -@pytest.mark.parametrize("patterns,expected", [ - (["run_001.h5", "README*"], ["README.md", "run_001.h5"]), - ("*.absent", []), -]) -def test_filter_precedes_automatic_detection(tmp_path, pages, patterns, expected): - root = "https://test/data/" - pages[0][root] = index("run_001.h5", "run_002.h5", "README.md") - repo = Repo.init(tmp_path / "filtered", source=root, filter=patterns) - assert repo.state.data["path"].tolist() == expected - # A single run has no sibling from which to infer a varying field. - assert repo.state.config["data"][0]["extraction"]["formats"] == [] - assert "scan" not in repo.state.data.columns - assert repo.plan_download().file_count == len(expected) - assert pages[1] == [root] + assert (base, total) == (root, len(paths)) + assert suggestions == [(root + "run_{scan}.h5", 2), + (root + "sgra_{date}_{p0}.fits", 2)] + repo = Repo.init(tmp_path / "repo") + repo.add(suggestions[1][0]) + assert repo.state.data[["date", "p0"]].values.tolist() == [ + ["20170406", "nustar"], ["20170411", "chandra"]] -def test_inferred_reserved_fields_do_not_replace_metadata(tmp_path, pages): +def test_ls_remote_skips_suggestions_with_reserved_or_literal_braces(pages): root = "https://test/data/" pages[0][root] = index("{path}_001.h5", "{path}_002.h5", "README.md") - repo = Repo.init(tmp_path / "literal", source=root) - assert set(repo.state.data["path"]) == { - "{path}_001.h5", "{path}_002.h5", "README.md"} - assert repo.state.config["data"][0]["extraction"]["formats"] == [] - assert repo.plan_download().file_count == 3 + _, suggestions, total = suggest_templates(root) + assert total == 3 + for template, _ in suggestions: + TemplateMatcher(template[len(root):]) -@pytest.mark.parametrize("explicit", [False, True]) -def test_cli_filter_and_format_workflow(tmp_path, pages, explicit): +def test_cli_ls_remote_then_add_workflow(tmp_path, pages, monkeypatch): root = "https://test/data/" pages[0][root] = index("run_001.h5", "run_002.h5", "README.md", "omit.bin") - destination = tmp_path / "cli" - arguments = ["init", str(destination), "--from", root, - "--filter", "*.h5", "--filter", "README*"] - if explicit: - arguments += ["--format", "run_{run:03d}.h5"] - result = CliRunner().invoke(hallmark, arguments) - assert result.exit_code == 0, result.output - repo = Repo(destination) - assert set(repo.state.data["path"]) == {"run_001.h5", "run_002.h5", "README.md"} - assert ("run" if explicit else "scan") in repo.state.data.columns - assert pages[1] == [root] - - -@pytest.mark.parametrize("option", ["--include", "--extract", "--format", "--fmt"]) + runner = CliRunner() + listing = runner.invoke(hallmark, ["ls-remote", root]) + assert listing.exit_code == 0, listing.output + assert "4 file(s) at https://test/data/" in listing.output + assert f"hallmark add '{root}run_{{scan}}.h5' (2)" in listing.output + assert runner.invoke(hallmark, ["init", str(tmp_path / "cli")]).exit_code == 0 + monkeypatch.chdir(tmp_path / "cli") + preview = runner.invoke(hallmark, ["add", "-n", root + "run_{run:03d}.h5"]) + assert preview.exit_code == 0, preview.output + assert preview.stdout.splitlines()[:3] == ["Would add", "run_001.h5", + "run_002.h5"] + assert not Repo(tmp_path / "cli").state.config["data"][0].get("fmt") + added = runner.invoke(hallmark, ["add", root + "run_{run:03d}.h5"]) + assert added.exit_code == 0, added.output + assert set(Repo(tmp_path / "cli").state.data["run"]) == {"001", "002"} + + +@pytest.mark.parametrize("option", ["--extract", "--format", "--fmt"]) def test_download_rejects_removed_and_initialization_only_flags( tmp_path, monkeypatch, option): repo = Repo.init(tmp_path / "repo") diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py index e59cbf3..d48d8ac 100644 --- a/test/test_ssh_integration.py +++ b/test/test_ssh_integration.py @@ -378,12 +378,13 @@ def test_sftp_only_discovers_metadata_and_fetches(ssh_server, tmp_path): @pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) @pytest.mark.parametrize("scheme", ["ssh", "sftp"]) -def test_sftp_only_init_plans_then_requires_payload_approval( +def test_sftp_only_add_plans_then_requires_payload_approval( ssh_server, tmp_path, monkeypatch, scheme, ): root = ssh_server["root"] - (root / "nested").mkdir() + (root / "nested" / "deeper").mkdir(parents=True) (root / "nested" / "science.fits").write_bytes(b"science") + (root / "nested" / "deeper" / "skipped.fits").write_bytes(b"deep") (root / "notes.txt").write_bytes(b"notes") fetched = [] fetch = SshTransport._fetch @@ -392,25 +393,22 @@ def record_fetch(self, path, destination, file_limit=None): fetched.append(path) return fetch(self, path, destination, file_limit) - def reject_git_probe(*args, **kwargs): - raise AssertionError("SFTP directory detection must not invoke remote Git") - monkeypatch.setattr(SshTransport, "_fetch", record_fetch) - monkeypatch.setattr("hallmark.catalog.Dothm.clone", reject_git_probe) plans = [] def decline(plan): plans.append(plan) return False - repo = Repo.init( - tmp_path / "initialized", - source=ssh_server["url"].replace("ssh:", scheme + ":"), - filter="**/*.fits", - ) + repo = Repo.init(tmp_path / "initialized") + url = ssh_server["url"].replace("ssh:", scheme + ":").rstrip("/") + repo.add(url + "/{group}/{name}.fits") + assert repo.commit("Catalog remote science files") + assert not any(path.is_file() for path in repo.objects.root.rglob("*")) decline(repo.plan_download()) assert fetched == [] - assert repo.state.data["path"].tolist() == ["nested/science.fits"] + assert repo.state.data[["group", "name"]].values.tolist() == [ + ["nested", "science"]] assert len(plans) == 1 assert plans[0].total_bytes == 7 assert not (repo.worktree / "nested").exists() @@ -421,6 +419,7 @@ def decline(plan): assert result["succeeded"] == 1 assert fetched == ["nested/science.fits"] assert (repo.worktree / "nested/science.fits").read_bytes() == b"science" + assert repo.status()["untracked"] == [] @pytest.mark.parametrize("ssh_server", ["no-sftp"], indirect=True) From f76412dda063d5d5468512d0a1fb09e3187767c7 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:43:53 -0700 Subject: [PATCH 25/29] Rename download and clone --filter to --include git clone --filter selects which objects a partial clone fetches, so a path glob option of the same name is misleading. --include follows git-annex and matches the include= keyword now used by plan_download and discovery. Neither spelling exists on main, so no alias is kept. Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/cli.py | 47 ++++++++++++++++--------------- mod/hallmark/discovery.py | 18 ++++++------ mod/hallmark/downloader.py | 13 +++++---- mod/hallmark/repo.py | 6 ++-- mod/hallmark/repo_remote.py | 2 +- test/test_catalog.py | 6 ++-- test/test_cli.py | 12 ++++---- test/test_discovery.py | 16 +++++------ test/test_download_plan.py | 4 +-- test/test_interactive_download.py | 18 ++++++------ test/test_sources.py | 4 +-- 11 files changed, 74 insertions(+), 72 deletions(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 7eac9bb..0c7cac6 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -197,7 +197,7 @@ def _choose_download_plan(repo, *, output, remote_name, dry_run): continue output = _download_output(repo, output) plan = repo.plan_download(output, all_files=choice == "all", - filter=patterns or None, remote_name=remote_name) + include=patterns or None, remote_name=remote_name) _show_download_plan(plan, show_paths=True) if not plan.file_count: click.echo("No cataloged files match. Choose another selection or skip.") @@ -233,7 +233,7 @@ def _interactive_download(repo, *, output=None, remote_name=None, max_workers=4, """Select and approve cataloged files in a terminal, without network preflight.""" if not sys.stdin.isatty(): raise ClickException( - "--interactive requires a terminal. Use --filter 'PATTERN' --dry-run " + "--interactive requires a terminal. Use --include 'PATTERN' --dry-run " "to preview a selection; run in a terminal to approve a download.") if not _download_available(repo, remote_name): return @@ -256,7 +256,7 @@ def _interactive_download(repo, *, output=None, remote_name=None, max_workers=4, _execute_download(repo, plan, max_workers=max_workers) -def _offer_download(repo, *, filters=(), output=None, interactive=False): +def _offer_download(repo, *, includes=(), output=None, interactive=False): """Review downloads after cloning, or print equivalent commands for later.""" if not sys.stdin.isatty(): click.echo("Warning: download approval is unavailable without " @@ -269,9 +269,9 @@ def _offer_download(repo, *, filters=(), output=None, interactive=False): click.echo(f"Before downloading: {exc}", err=True) directory = repo.worktree if repo.worktree is not None else repo.dothm.path arguments = ["hallmark", "download"] - if filters: - for pattern in filters: - arguments.extend(["--filter", pattern]) + if includes: + for pattern in includes: + arguments.extend(["--include", pattern]) else: arguments.append("--all") if output is not None: @@ -282,7 +282,7 @@ def _offer_download(repo, *, filters=(), output=None, interactive=False): needs_output = repo.worktree is None and output is None if needs_output: command += " --output '/path/to/downloads'" - click.echo("Preview this selection locally; use --filter 'PATTERN' " + click.echo("Preview this selection locally; use --include 'PATTERN' " "instead of --all to narrow it:") click.echo(f" cd -- {shlex.quote(str(directory))}") click.echo(f" {command} --dry-run") @@ -302,7 +302,7 @@ def _offer_download(repo, *, filters=(), output=None, interactive=False): "transfers can consume substantial bandwidth and disk space.") try: output = _download_output(repo, output) - plan = repo.plan_download(output, all_files=True, filter=filters or None) + plan = repo.plan_download(output, all_files=True, include=includes or None) approved = _confirm_download_plan( plan, decline_is_skip=True, show_paths=True) except click.Abort as exc: @@ -668,7 +668,7 @@ def checkout(repo, target_branch): help="Select a catalog TSV. May be repeated.") @click.option("--all", "download_all", is_flag=True, help="Select all cataloged files.") -@click.option("--filter", "filters", multiple=True, +@click.option("--include", "includes", multiple=True, help="Select paths matching a glob. ** matches recursively. " "May be repeated.") @click.option("--remote", "remote_name", @@ -683,7 +683,7 @@ def checkout(repo, target_branch): help="Choose cataloged files and review sizes before approval.") @click.option("-y", "--yes", is_flag=True, hidden=True) @click.pass_obj -def download(repo, files, tsv_names, download_all, filters, remote_name, +def download(repo, files, tsv_names, download_all, includes, remote_name, output, max_workers, dry_run, interactive, yes): """ Download selected files from a configured data remote. @@ -693,9 +693,9 @@ def download(repo, files, tsv_names, download_all, filters, remote_name, Use --interactive to choose patterns or all files in a terminal. """ if interactive: - if files or tsv_names or download_all or filters: + if files or tsv_names or download_all or includes: raise ClickException( - "--interactive cannot be combined with paths, --filter, " + "--interactive cannot be combined with paths, --include, " "--tsv, or --all") if yes: click.echo("--yes is deprecated; downloads still require confirmation.", @@ -706,8 +706,8 @@ def download(repo, files, tsv_names, download_all, filters, remote_name, return if download_all and (files or tsv_names): raise ClickException("--all cannot be combined with file paths or --tsv") - if not files and not tsv_names and not download_all and not filters: - raise ClickException("Provide file paths, --tsv, --all, or --filter") + if not files and not tsv_names and not download_all and not includes: + raise ClickException("Provide file paths, --tsv, --all, or --include") if repo.worktree is None and output is None: raise ClickException("--output is required when downloading from a bare repo") if yes: @@ -716,7 +716,7 @@ def download(repo, files, tsv_names, download_all, filters, remote_name, with _translate_cli_errors(DownloadError, ValueError): plan = repo.plan_download( output, file_paths=files, tsv_names=tsv_names, all_files=download_all, - filter=filters or None, remote_name=remote_name) + include=includes or None, remote_name=remote_name) if dry_run: _show_download_plan(plan, show_paths=True) return @@ -731,7 +731,7 @@ def download(repo, files, tsv_names, download_all, filters, remote_name, help="Override automatic source detection.") @click.option("--no-download", is_flag=True, help="Copy the complete catalog without reviewing or downloading files.") -@click.option("--filter", "filters", multiple=True, +@click.option("--include", "includes", multiple=True, help="Select downloads by relative-path glob; keep the full catalog. " "May be repeated.") @click.option("--interactive", is_flag=True, @@ -739,21 +739,21 @@ def download(repo, files, tsv_names, download_all, filters, remote_name, @click.option("--output", type=click.Path(file_okay=False), help="Download directory. Defaults to the worktree; " "prompted for bare repos.") -def clone(url, path, source_type, no_download, filters, interactive, output): +def clone(url, path, source_type, no_download, includes, interactive, output): """Clone a complete Git catalog or published catalog snapshot at PATH. Copy the complete catalog first, then review a download plan and confirm - transfers. --filter selects downloads without trimming the catalog. + transfers. --include selects downloads without trimming the catalog. --no-download skips review; --interactive opens the selection chooser. Without a terminal, keep the catalog and print download commands for later. For raw datasets, use init, then add a URL template. """ - if no_download and (interactive or filters or output is not None): + if no_download and (interactive or includes or output is not None): raise ClickException( "--no-download cannot be combined with --interactive, " - "--filter, or --output") - if interactive and filters: - raise ClickException("--interactive cannot be combined with --filter") + "--include, or --output") + if interactive and includes: + raise ClickException("--interactive cannot be combined with --include") with _translate_cli_errors(DownloadError, GitError, ValueError): try: repo = Repo.clone(url, path, source_type=source_type, progress=True) @@ -762,7 +762,8 @@ def clone(url, path, source_type, no_download, filters, interactive, output): raise SystemExit(1) from exc click.echo(f'Successfully cloned to "{path}"') if not no_download: - _offer_download(repo, filters=filters, output=output, interactive=interactive) + _offer_download(repo, includes=includes, output=output, + interactive=interactive) @hallmark.command(short_help="Deprecated: use init, then add a URL template.") diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py index 53f062a..e0648f4 100644 --- a/mod/hallmark/discovery.py +++ b/mod/hallmark/discovery.py @@ -201,16 +201,16 @@ def may_contain(self, directory: str) -> bool: for index, part in enumerate(parts)) -def path_matches(path: str, filter=None) -> bool: +def path_matches(path: str, include=None) -> bool: """Match a relative path against any supplied case-sensitive inclusion glob. ``**`` spans zero or more directories. With no patterns every path matches. """ - if filter is None: + if include is None: return True - if not isinstance(filter, (str, list, tuple)): - raise ValueError("filter must be a glob string or a list of glob strings") - patterns = [filter] if isinstance(filter, str) else list(filter) + if not isinstance(include, (str, list, tuple)): + raise ValueError("include must be a glob string or a list of glob strings") + patterns = [include] if isinstance(include, str) else list(include) if not all(isinstance(pattern, str) and pattern for pattern in patterns): raise ValueError("Inclusion patterns must be nonempty glob strings") return any(_glob_matches(str(path), pattern) for pattern in patterns) @@ -271,7 +271,7 @@ def _manifest_checksums(context, entries): previous, checksum_algorithm=algorithm, checksum=digest.lower()) -def discover(context, *, filter=None, progress=False, +def discover(context, *, include=None, progress=False, descend=None) -> list[RemoteEntry]: """ Discover remote files and their published metadata recursively. @@ -281,7 +281,7 @@ def discover(context, *, filter=None, progress=False, Args: context (OperationContext): Source connection and cancellation state. - filter (str | list[str], optional): Relative path glob or globs. + include (str | list[str], optional): Relative path glob or globs. descend (callable, optional): Predicate receiving a relative directory path ending in ``/``; listings skip directories for which it returns False. Checksum manifests in skipped directories are @@ -301,9 +301,9 @@ def discover(context, *, filter=None, progress=False, DownloadError: If discovery fails, paths are unsafe, or manifests conflict. """ # Validate selectors before contacting the source, even for an empty index. - path_matches("", filter=filter) + path_matches("", include=include) def matches(path): - return path_matches(path, filter=filter) + return path_matches(path, include=include) entries = {} counts = {"directories": 0, "files": 0, "matched": 0, "current": ""} diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index 336821b..98e7995 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -905,7 +905,7 @@ def plan_download( file_paths: Optional[Sequence[str]] = None, tsv_names: Optional[Sequence[str]] = None, all_files: bool = False, - filter: Optional[Union[str, Sequence[str]]] = None, + include: Optional[Union[str, Sequence[str]]] = None, remote_name: Optional[str] = None, estimated_bytes_per_second: Optional[float] = None, ) -> DownloadPlan: @@ -913,7 +913,8 @@ def plan_download( Plan a download using local catalog metadata. No network requests are made. With no explicit paths or TSVs, select - the complete catalog before applying filters. Missing sizes remain unknown. + the complete catalog before applying inclusion globs. Missing sizes remain + unknown. Args: repo: The hallmark repository object. @@ -923,7 +924,7 @@ def plan_download( tsv_names (sequence[str], optional): Catalog TSVs to select. all_files (bool): Select all configured files. Cannot be combined with explicit paths or TSVs. Defaults to False. - filter (str | list[str], optional): Relative path glob or globs. + include (str | list[str], optional): Relative path glob or globs. remote_name (str, optional): Configured data remote to use. estimated_bytes_per_second (float, optional): Positive rate used to estimate duration when every file size is known. @@ -955,11 +956,11 @@ def plan_download( repo, file_paths=file_paths, tsv_names=tsv_names, all_files=all_files or (not file_paths and not tsv_names), catalog_only=True, resolve_source=_source_resolver(repo, remote_name)) - if filter is not None: + if include is not None: from .discovery import path_matches - path_matches("validation", filter=filter) + path_matches("validation", include=include) items = [item for item in items if path_matches( - item.relative_path.as_posix(), filter=filter)] + item.relative_path.as_posix(), include=include)] unsourced = [item for item in items if item.source is None] if unsourced and (file_paths or tsv_names or len(unsourced) == len(items)): # explicitly requested files, or a selection with nothing downloadable diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index d6b6498..b33d921 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -317,7 +317,7 @@ def clone(cls, url: str, path: Union[Path, str], *, return clone_catalog(cls, url, path, source_type=source_type) def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, - all_files=False, filter=None, remote_name=None, + all_files=False, include=None, remote_name=None, estimated_bytes_per_second=None): """ Plan a download using the local catalog without contacting a server. @@ -333,7 +333,7 @@ def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, tsv_names (sequence[str], optional): Catalog TSVs to select. all_files (bool): Select all configured files. Cannot be combined with explicit paths or TSVs. Defaults to False. - filter (str | list[str], optional): Relative path glob or globs. + include (str | list[str], optional): Relative path glob or globs. remote_name (str, optional): Configured data remote to use. estimated_bytes_per_second (float, optional): Positive transfer rate for duration estimates. No rate is measured while planning. @@ -351,7 +351,7 @@ def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, return plan_download( self, output_path, file_paths=file_paths, tsv_names=tsv_names, - all_files=all_files, filter=filter, remote_name=remote_name, + all_files=all_files, include=include, remote_name=remote_name, estimated_bytes_per_second=estimated_bytes_per_second) def download(self, plan, *, approved=False, max_workers=4, progress=False): diff --git a/mod/hallmark/repo_remote.py b/mod/hallmark/repo_remote.py index 5009a60..a0ad7a5 100644 --- a/mod/hallmark/repo_remote.py +++ b/mod/hallmark/repo_remote.py @@ -70,7 +70,7 @@ def match_remote_template(url: str, *, progress=False, backend=None, matcher = TemplateMatcher(template) remote = _remote(base, backend, backend_options) with OperationContext(remote) as context: - entries = discover(context, filter=matcher.glob, progress=progress, + entries = discover(context, include=matcher.glob, progress=progress, descend=matcher.may_contain) rows, mismatched = [], 0 for entry in entries: diff --git a/test/test_catalog.py b/test/test_catalog.py index d853c46..8d7f795 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -72,7 +72,7 @@ def test_published_snapshot_preserves_data_remote(tmp_path, metadata_server, nes pages[prefix + "data.tsv"] = "path\tsize_bytes\na.h5\t12\nb.txt\t20\n" plans = [] repo = Repo.clone(root, tmp_path / "clone") - plans.append(repo.plan_download(filter="**/*.h5")) + plans.append(repo.plan_download(include="**/*.h5")) assert repo.state.data["path"].tolist() == ["a.h5", "b.txt"] assert plans[0].total_bytes == 12 assert repo.plan_download().remote_url == "https://mirror.test/data/" @@ -96,7 +96,7 @@ def fail(*args, **kwargs): monkeypatch.setattr("hallmark.downloader._fetch_file", fail) plans = [] repo = Repo.clone(str(source.dothm.path), tmp_path / "clone") - plans.append(repo.plan_download(filter="run1.h5")) + plans.append(repo.plan_download(include="run1.h5")) assert repo.dothm.head.commit.hexsha == head assert repo.dothm.index.diff("HEAD") == [] assert len(repo.state.data) == 2 @@ -153,7 +153,7 @@ def test_snapshot_filter_normalizes_legacy_db_names(tmp_path, metadata_server): pages[root + "data.tsv"] = "i\n1\n2\n" plans = [] repo = Repo.clone(root, tmp_path / "clone") - plans.append(repo.plan_download(filter="item_1.dat")) + plans.append(repo.plan_download(include="item_1.dat")) assert repo.state.data["i"].astype(str).tolist() == ["1", "2"] assert [item.relative_path for item in plans[0].items] == [Path("item_1.dat")] diff --git a/test/test_cli.py b/test/test_cli.py index 04bd05e..29e56ed 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -1128,7 +1128,7 @@ def test_build_cli_reports_missing_config_yml_in_directory(tmp_path): @pytest.mark.parametrize( "arguments, message", [ - (["download"], "Provide file paths, --tsv, --all, or --filter"), + (["download"], "Provide file paths, --tsv, --all, or --include"), (["download", "file.dat", "--all"], "--all cannot be combined"), (["download", "--tsv", "data", "--all"], "--all cannot be combined")]) def test_download_cli_rejects_invalid_selection_combinations( @@ -1237,12 +1237,12 @@ def fake_download(actual_plan, **kwargs): repo.download = fake_download result = CliRunner().invoke(hallmark, [ "download", "nested/file.dat", "--remote", "mirror", "--max-workers", "2", - "--filter", "**/*.dat"], input="y\n") + "--include", "**/*.dat"], input="y\n") assert result.exit_code == 0, result.output assert "Successfully downloaded 1 files (1.0 MB)" in result.output assert captured["plan"] == (None, { "file_paths": ("nested/file.dat",), "tsv_names": (), "all_files": False, - "filter": ("**/*.dat",), "remote_name": "mirror"}) + "include": ("**/*.dat",), "remote_name": "mirror"}) assert captured["download"][0] is plan assert captured["download"][1] == { "max_workers": 2, "progress": True, "approved": True} @@ -1366,11 +1366,11 @@ def capture_get(url, **kwargs): "clone", str(source.dothm.path), str(target)]) assert cloned.exit_code == 0, cloned.output monkeypatch.chdir(target) - arguments = ["download", "--filter", "*.fits"] + arguments = ["download", "--include", "*.fits"] else: target = source.worktree monkeypatch.chdir(target) - arguments = ["download", "--filter", "*.fits"] + arguments = ["download", "--include", "*.fits"] result = CliRunner().invoke(hallmark, arguments, input=answer) assert "1 file(s); 4 bytes" in result.output assert "Download these files? [y/N]" in result.output @@ -1387,7 +1387,7 @@ def capture_get(url, **kwargs): @pytest.mark.parametrize("arguments", [ - ["--interactive", "--filter", "*.fits"], ["--fmt", "{name}.fits"], + ["--interactive", "--include", "*.fits"], ["--fmt", "{name}.fits"], ["--with-download", "--fmt", "{name:invalid}"], ]) def test_clone_cli_rejects_invalid_selection_before_source_access( diff --git a/test/test_discovery.py b/test/test_discovery.py index e6a8f40..f524ff5 100644 --- a/test/test_discovery.py +++ b/test/test_discovery.py @@ -139,19 +139,19 @@ def test_encoded_unsafe_paths_are_rejected(href): ("root.txt", ["**/*.fits", "**/*.txt"], True), ]) def test_recursive_globs(path, pattern, matched): - assert path_matches(path, filter=pattern) is matched + assert path_matches(path, include=pattern) is matched def test_globs_are_case_sensitive(): - assert path_matches("sample_001.fits", filter="sample_*.fits") - assert not path_matches("Sample_001.fits", filter="sample_*.fits") + assert path_matches("sample_001.fits", include="sample_*.fits") + assert not path_matches("Sample_001.fits", include="sample_*.fits") @pytest.mark.parametrize("invalid", [12, {"*.fits": True}, iter(["*.fits"])]) def test_invalid_filter_types_fail_before_source_access(invalid): source = Source({"": index("file.fits")}) with pytest.raises(ValueError, match="glob string"): - discover(source, filter=invalid) + discover(source, include=invalid) assert source.reads == [] @@ -175,7 +175,7 @@ def test_redirected_listing_uses_final_directory_and_deduplicates_crawl(final_ur def test_filter_traverses_unmatched_directories_and_excludes_other_payloads(): source = Source({"": index("root.fits", "nested/", "notes.txt"), "nested/": index("other.fits", "other.bin")}) - assert [entry.path for entry in discover(source, filter="**/*.fits")] == [ + assert [entry.path for entry in discover(source, include="**/*.fits")] == [ "nested/other.fits", "root.fits"] assert source.reads == ["", "nested/"] @@ -186,7 +186,7 @@ def test_only_conventional_checksum_metadata_is_read(): "": index("data.fits", "sha256sums.txt", "custom_checksum_payload.bin"), "sha256sums.txt": f"{digest} data.fits\n", }) - entries = discover(source, filter="*.fits") + entries = discover(source, include="*.fits") assert entries == [RemoteEntry(path="data.fits", checksum_algorithm="sha256", checksum=digest)] assert source.reads == ["", "sha256sums.txt"] @@ -205,7 +205,7 @@ def test_nested_manifest_accepts_published_directory_prefixed_paths(): "release/project/": index("sha256sums", "file.fits"), "release/project/sha256sums": "a" * 64 + " project/file.fits\n", }) - assert discover(source, filter="**/*.fits")[0].checksum == "a" * 64 + assert discover(source, include="**/*.fits")[0].checksum == "a" * 64 def test_conflicting_manifests_fail(): @@ -229,7 +229,7 @@ def entries(on_directory): source.transport = SimpleNamespace(iter_entries=entries, prepare=lambda: None) snapshots = [] - selected = discover(source, filter="**/*.fits", progress=snapshots.append) + selected = discover(source, include="**/*.fits", progress=snapshots.append) assert selected == [RemoteEntry(path="root.fits", size=12, mtime=123)] assert snapshots[-1] == {"directories": 2, "files": 2, "matched": 1, "current": "nested"} diff --git a/test/test_download_plan.py b/test/test_download_plan.py index b4b52d5..5a6462f 100644 --- a/test/test_download_plan.py +++ b/test/test_download_plan.py @@ -76,7 +76,7 @@ def test_sizes_survive_nullable_numeric_tsv_serialization(catalog): def test_plan_filters_without_authorizing_download(catalog): - plan = plan_download(catalog, filter="**/*.fits") + plan = plan_download(catalog, include="**/*.fits") assert [item.relative_path.as_posix() for item in plan.items] == [ "nested/a.fits", "empty.fits"] assert plan.total_bytes == 6 @@ -225,7 +225,7 @@ def test_execution_rejects_changed_output_root(catalog, tmp_path): def test_empty_plan_needs_no_approval(catalog): - plan = plan_download(catalog, filter="**/*.absent") + plan = plan_download(catalog, include="**/*.absent") assert execute_download_plan(catalog, plan) == { "succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} diff --git a/test/test_interactive_download.py b/test/test_interactive_download.py index 4aca17c..d8a1879 100644 --- a/test/test_interactive_download.py +++ b/test/test_interactive_download.py @@ -173,7 +173,7 @@ def test_preview_truncates_paths_but_not_totals(catalog): @pytest.mark.parametrize("selectors", [["runs/run_001.h5"], ["--all"], - ["--filter", "*.h5"], ["--tsv", "data.tsv"]]) + ["--include", "*.h5"], ["--tsv", "data.tsv"]]) def test_interactive_rejects_explicit_selectors(catalog, selectors): _, transfers = catalog result = TerminalRunner().invoke( @@ -299,7 +299,7 @@ def interrupt(self, plan, **kwargs): @pytest.mark.parametrize("bare", [False, True]) @pytest.mark.parametrize("options", [[], ["--interactive"], - ["--filter", "runs/*001.h5", "--filter", README], + ["--include", "runs/*001.h5", "--include", README], ["--output", "../output 'files'"]]) def test_noninteractive_clone_keeps_catalog_and_prints_equivalent_commands( catalog, tmp_path, bare, options): @@ -320,7 +320,7 @@ def test_noninteractive_clone_keeps_catalog_and_prints_equivalent_commands( assert len(commands) == 2 preview, execute = map(shlex.split, commands) assert preview == execute + ["--dry-run"] - if "--filter" in options: + if "--include" in options: assert execute[2:6] == options else: assert execute[2] == "--all" @@ -342,9 +342,9 @@ def test_noninteractive_clone_keeps_catalog_and_prints_equivalent_commands( ([], "n\n", []), ([], "\n", []), ([], "", []), - (["--filter", "runs/*001.h5", "--filter", README], "y\n", + (["--include", "runs/*001.h5", "--include", README], "y\n", ["runs/run_001.h5", README]), - (["--filter", "omitted*"], "", []), + (["--include", "omitted*"], "", []), (["--interactive"], "patterns\nruns/*001.h5\n\ndownload\n", ["runs/run_001.h5"]), (["--interactive"], "skip\n", []), ]) @@ -412,8 +412,8 @@ def unexpected(*args, **kwargs): @pytest.mark.parametrize("options", [ - ["--no-download", "--interactive"], ["--no-download", "--filter", "*.fits"], - ["--no-download", "--output", "output"], ["--interactive", "--filter", "*.fits"], + ["--no-download", "--interactive"], ["--no-download", "--include", "*.fits"], + ["--no-download", "--output", "output"], ["--interactive", "--include", "*.fits"], ["--with-download"], ]) def test_clone_invalid_options_fail_before_source_access( @@ -463,7 +463,7 @@ def test_clone_output_and_bare_prompt(catalog, tmp_path, bare, provided): destination = tmp_path / ("copy.hm" if bare else "copy") output = tmp_path / "downloads" arguments = ["clone", str(source.dothm.path), str(destination), - "--filter", "runs/*001.h5"] + "--include", "runs/*001.h5"] if provided: arguments += ["--output", str(output)] answer = (str(output) + "\n" if bare and not provided else "") + "y\n" @@ -589,7 +589,7 @@ def read(context, path): destination = tmp_path / "snapshot" result = TerminalRunner().invoke( hallmark, ["clone", snapshot_url, str(destination), "--source-type", "catalog", - "--filter", "runs/*001.h5"], input="y\n") + "--include", "runs/*001.h5"], input="y\n") assert result.exit_code == 0, result.output assert set(reads) == {"config.yml", "meta.yml", "data.tsv"} assert transfers == [ROOT + "runs/run_001.h5"] diff --git a/test/test_sources.py b/test/test_sources.py index ea2ebbd..cbabed1 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -69,7 +69,7 @@ def test_collection_urls_from_sources_feed_add(tmp_path, pages): assert repo.state.data["kind"].tolist() == ["pix", "tilecumulative"] # the template has one segment, so subdirectories are never listed assert reads == [zcatalog] - plan = repo.plan_download(filter="*pix*") + plan = repo.plan_download(include="*pix*") assert plan.remote_url == zcatalog assert [item.relative_path.as_posix() for item in plan.items] == [ "zall-pix-iron.fits"] @@ -248,7 +248,7 @@ def test_cli_ls_remote_then_add_workflow(tmp_path, pages, monkeypatch): assert set(Repo(tmp_path / "cli").state.data["run"]) == {"001", "002"} -@pytest.mark.parametrize("option", ["--extract", "--format", "--fmt"]) +@pytest.mark.parametrize("option", ["--extract", "--format", "--fmt", "--filter"]) def test_download_rejects_removed_and_initialization_only_flags( tmp_path, monkeypatch, option): repo = Repo.init(tmp_path / "repo") From 9d5e7dcafaaa18faa03232c64207a64635e6abf8 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:54:15 -0700 Subject: [PATCH 26/29] Hide empty sections in hallmark status emit_section tested whether its list of labeled path lists was empty, which it never is, so status printed a bare "Changes not staged for commit:" heading even when nothing had changed. Co-Authored-By: Claude Opus 5.5 (1M context) --- mod/hallmark/cli.py | 2 +- test/test_catalog_entries.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 0c7cac6..ce1b0a0 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -428,7 +428,7 @@ def status(repo): untracked = snapshot["untracked"] def emit_section(title, entries, fg): - if not entries: + if not any(paths for _, paths in entries): return click.echo("") click.secho(title, fg=fg) diff --git a/test/test_catalog_entries.py b/test/test_catalog_entries.py index 6086b20..c3dfb57 100644 --- a/test/test_catalog_entries.py +++ b/test/test_catalog_entries.py @@ -605,3 +605,18 @@ def test_cli_download_dry_run_lists_every_source(tmp_path, monkeypatch): assert result.exit_code == 0, result.output assert "3 file(s); 18 bytes" in result.output assert "Sources:" in result.output and SIMS in result.output + + +def test_cli_status_omits_headers_of_empty_sections(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + monkeypatch.chdir(repo.worktree) + staged = CliRunner().invoke(hallmark, ["status"]) + assert "Changes to be committed:" in staged.output + assert "Changes not staged for commit:" not in staged.output + repo.commit("remote catalog") + clean = CliRunner().invoke(hallmark, ["status"]) + assert clean.output == "On branch main\n\nnothing to commit, working tree clean\n" From a30ede2a6ce51e4cae7e642e1ade0868098e5858 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 12:55:00 -0700 Subject: [PATCH 27/29] Document remote templates in add and update the demos Rewrite the remote workflows in the README, user guide, API and backend pages, and demos as init, add URL/TEMPLATE, commit, download. Describe ls-remote, template matching and sync, several templates per branch, rm --cached, per-template download sources, and the migration from init --from and --filter. The demos catalog each remote input by URL and version only the local settings file; their validation notes record this revision's runs against local HTTP fixtures and a loopback SFTP export. Fix two docstrings that broke the strict Sphinx build. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.rst | 62 ++--- demo/demo_cli.ipynb | 2 +- demo/private_data_demo.md | 102 +++++---- demo/scientific_workflows_cli.md | 137 +++++------ demo/scientific_workflows_python.ipynb | 50 ++-- doc/api.rst | 67 +++--- doc/backends.rst | 65 +++--- doc/private_data.rst | 305 +++++++++++++------------ doc/usecase.rst | 83 +++++-- mod/hallmark/repo.py | 3 +- mod/hallmark/repo_worktree.py | 2 +- 11 files changed, 489 insertions(+), 389 deletions(-) diff --git a/README.rst b/README.rst index 377134e..5fe44c3 100644 --- a/README.rst +++ b/README.rst @@ -74,49 +74,59 @@ Or install from source for development purposes:: Private data remotes -------------------- -|hallmark|_ can download data products from HTTP(S), SSH, and SFTP servers. -A data remote specifies where the files are stored, while a Git remote -is used to share the history of the data index. +|hallmark|_ can catalog and download data products stored on HTTP(S), SSH, and +SFTP servers. A data source specifies where the files are stored, while a Git +remote is used to share the history of the data index. For a private server, |hallmark|_ uses OpenSSH 9.6 or newer with a trusted host key and key-based authentication that does not require a prompt. -Initialize a catalog from a remote dataset, then select files to download:: +Add remote files to a repository with a URL template, then select files to +download:: - hallmark init campus --from ssh://campus/srv/export/ --filter '**/*.h5' - cd campus + hallmark init eht + cd eht + hallmark add 'https://archive.example.org/2026MOVIE/ER2/{src}_{day}.h5' + hallmark commit -m 'Add remote EHT data' hallmark download --all --dry-run hallmark download --all -Remote initialization discovers files without downloading dataset contents. +``add`` lists the remote directory and catalogs the matching files, with their +sizes and published checksums, without downloading dataset contents. The URL +up to the first path segment containing a ``{field}`` is recorded as the +template's source; the rest is the filename template, and its fields become +catalog columns. Adding the same URL template again syncs the catalog with the +server, including files that were removed. A branch can track several +templates, local or remote, such as ``{run}.dat`` files in the worktree and +``.h5`` files on a server; ``hallmark rm --cached TEMPLATE`` stops tracking +one. Remote files are committed as catalog entries, not as local objects. + +Use ``hallmark ls-remote URL`` to list a directory and print suggested +templates, and ``hallmark add -n 'URL/TEMPLATE'`` to preview the matches. CyVerse and ordinary browsable HTTPS directories, including DESI, use the same -workflow. -SSH discovery also works with SFTP-only accounts; no server shell or Python -is required. Omit the filter to catalog everything beneath the supplied URL. +workflow. SSH listing also works with SFTP-only accounts; no server shell or +Python is required. Every dataset transfer requires approval. Python callers can inspect ``repo.plan_download()`` and then execute ``repo.download(plan, approved=True)``. CLI users can run ``hallmark download --interactive`` to choose path globs or all cataloged files, review recorded sizes, then download, change the selection, -or skip. Initialization never offers downloads. CLI cloning copies the complete -catalog, then displays a download plan and requests confirmation by default. -Use ``clone --interactive`` for the chooser, or ``clone --no-download`` to copy -only the catalog. Without a terminal, clone keeps the catalog, skips downloading, -and prints commands to use later. +or skip. ``init`` and ``add`` never offer downloads. CLI cloning copies the +complete catalog, then displays a download plan and requests confirmation by +default. Use ``clone --interactive`` for the chooser, or ``clone --no-download`` +to copy only the catalog. Without a terminal, clone keeps the catalog, skips +downloading, and prints commands to use later. SSH and SFTP use standard ``~/.ssh/config`` aliases and the SSH agent. -Use ``hallmark sources desi`` to inspect named sources, releases and collections. -For example, ``hallmark init desi --from desi --release dr1 --collection redshifts`` -creates a catalog for that collection. Omit the collection to index the whole -release. ``init --filter`` selects catalog entries. Filename formats are detected -automatically from those paths; ``--format`` supplies an explicit template instead. -Both retain unmatched files. Download selection belongs to the -separate ``download --filter`` command. -The older remote ``build`` command is deprecated in favor of ``init --from``. +Use ``hallmark sources desi`` to list the release and collection URLs of a +named source for use with ``ls-remote`` and ``add``. Download selection belongs +to the separate ``download --include`` command. +The older remote ``build`` command is deprecated in favor of ``add``. Use ``hallmark clone CATALOG PATH`` for an existing Git-hosted ``.hm`` or a published HTTP/SFTP catalog snapshot. Catalogs can live on GitHub or another -server while their data remotes point elsewhere. Git clones preserve the full -catalog and its history. ``clone --filter`` narrows the planned downloads without -removing catalog entries. No payloads transfer until the user approves the plan. +server while their data sources point elsewhere. Git clones preserve the full +catalog and its history. ``clone --include`` narrows the planned downloads +without removing catalog entries. No payloads transfer until the user approves +the plan. The public ``DataBackend`` interface supports generic HTTP/SSH, CyVerse, and registered plugins for collaboration-specific APIs and multiple data servers. diff --git a/demo/demo_cli.ipynb b/demo/demo_cli.ipynb index 0953881..b4d3ed4 100644 --- a/demo/demo_cli.ipynb +++ b/demo/demo_cli.ipynb @@ -662,7 +662,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This clone demo uses `--no-download` to copy the complete Hallmark catalog without download review. By default, CLI clone shows a plan for all cataloged files and requests confirmation after copying metadata. `clone --filter` narrows only the payload selection, while `clone --interactive` opens the patterns/all/skip chooser. Without a terminal, clone keeps the catalog and prints download commands for later. Run `hallmark download --all` or `hallmark download --interactive` separately when ready. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" + "This clone demo uses `--no-download` to copy the complete Hallmark catalog without download review. By default, CLI clone shows a plan for all cataloged files and requests confirmation after copying metadata. `clone --include` narrows only the payload selection, while `clone --interactive` opens the patterns/all/skip chooser. Without a terminal, clone keeps the catalog and prints download commands for later. Run `hallmark download --all` or `hallmark download --interactive` separately when ready. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" ] }, { diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md index 0a54749..49aaf11 100644 --- a/demo/private_data_demo.md +++ b/demo/private_data_demo.md @@ -32,10 +32,9 @@ Host lab-data ``` The verified host key must already be in `known_hosts`. Load an encrypted key -into your SSH agent first. SSH configuration alone is sufficient for this demo. -For optional named profiles, see the [private data guide](../doc/private_data.rst). -Each collaborator defines credentials locally; a catalog stores only the -profile name. +into your SSH agent first. SSH configuration alone is sufficient for this demo; +see the [private data guide](../doc/private_data.rst) for details. Each +collaborator configures SSH access locally; a catalog stores only the URL. 3. Set the exact export root and create a new local workspace. Choose a different workspace name when repeating the demo; the data-server path below is an example. @@ -47,27 +46,31 @@ mkdir "$HM_DEMO_WORKSPACE" cd "$HM_DEMO_WORKSPACE" ``` -4. Discover the two indexed runs and prepare a local `.hm`. The filter selects -run files; the explicit format extracts their run numbers instead of inferring -a template. Initialization does not authorize a download. +4. Create a local repository and catalog the two runs with a URL template. +The template selects the run files and extracts their run numbers; `{group}` +keeps the `runs` directory in the local layout. Cataloging does not authorize +a download. ```bash -hallmark init ./client --from "$HM_DEMO_URL" --filter 'runs/*.dat' --format 'runs/run_{run:03d}.dat' +hallmark init ./client cd client +hallmark ls-remote "$HM_DEMO_URL" +hallmark add "${HM_DEMO_URL}{group}/run_{run:03d}.dat" +hallmark commit -m 'Catalog lab runs' ``` -The destination can contain existing files when `.hm` is absent; those files -are preserved. An existing `.hm` is rejected before contacting the data server. -Only listings and published metadata are read. Available sizes, modification -times, and published checksums are retained; missing checksums stay unknown. -Dataset files are not downloaded to calculate checksums. +`ls-remote` lists the export and suggests templates without changing the +repository. `add` records the export root as the template's source and lists +only the directories the template can match. Only listings and published +metadata are read. Available sizes, modification times, and published checksums +are retained; missing checksums stay unknown. Dataset files are not downloaded +to calculate checksums, and remote files are committed as catalog entries only. -Omit `--filter` to catalog every file below the dataset root. Omit `--format` -to detect filename templates automatically from the selected paths. An explicit -format overrides detection; either way, unmatched files remain cataloged. -Use `hallmark sources` -to inspect named sources; `--release` and `--collection` select their contents. -See [data backends](../doc/backends.rst) for source registration and adapter routing. +The README does not match the template and is not cataloged; add another +template to track it, as in `hallmark add "${HM_DEMO_URL}README.md"`. Running the +same `add` command later syncs the catalog with the export. Use `hallmark +sources` to find the URLs of named sources. See +[data backends](../doc/backends.rst) for source registration and adapter routing. 5. Review an offline plan for one run, then approve its transfer. The second command displays the plan again and asks for confirmation; answer `y` only when @@ -92,13 +95,15 @@ A failed transfer preserves an existing destination and removes its temporary file. Cancellation closes connections started for the operation. Repeating a download transfers the selection again. -6. Use the equivalent SFTP URL for this data remote. Changing the configuration -does not transfer files; the dry run shows the new source and another destination. +6. Configure the equivalent SFTP URL as a data remote and select it with +`--remote`, which downloads every selected file from that remote instead of the +template's source. Changing the configuration does not transfer files; the dry +run shows the new source and another destination. ```bash -hallmark set-config --remote-name origin \ +hallmark set-config --remote-name via-sftp \ --remote-url "sftp://${HM_DEMO_URL#ssh://}" -hallmark download runs/run_001.dat --output ../via-sftp --dry-run +hallmark download runs/run_001.dat --remote via-sftp --output ../via-sftp --dry-run ``` Both `ssh://` and `sftp://` use SFTP for discovery and data transfer. Relative @@ -114,7 +119,7 @@ authentication-profile file or flags. failures. A plan retains its source, backend settings, destination, paths, and checksums even if repository configuration later changes. -8. Plain initialization and cloning do not download payloads. To choose files +8. `init`, `add` and cloning do not download payloads. To choose files interactively from the completed catalog, run the following command. Choose `patterns` and enter one raw relative-path glob per line, ending with a blank line, or choose `all`. Review recorded sizes, then choose `download`, `change`, @@ -124,25 +129,26 @@ or `skip` (the default). Unknown file sizes remain unknown. hallmark download --interactive ``` -Add `--dry-run` to stop after the preview. Initialization never offers downloads. -CLI cloning reviews a download plan after copying the complete catalog by default; -use `clone --filter` to narrow transfers or `clone --interactive` for this chooser. -Use `clone --no-download` to skip review. Without a terminal, clone keeps the -catalog, skips downloading, and prints commands reflecting the selected filters -and output directory. Declining or EOF during clone's review preserves the catalog +Add `--dry-run` to stop after the preview. `init` and `add` never offer +downloads. CLI cloning reviews a download plan after copying the complete +catalog by default; use `clone --include` to narrow transfers or +`clone --interactive` for this chooser. Use `clone --no-download` to skip +review. Without a terminal, clone keeps the catalog, skips downloading, and +prints commands reflecting the selected globs and output directory. Declining or EOF during clone's review preserves the catalog and exits successfully; an interrupt or transfer failure preserves it but exits with an error. Python continues to use `repo.plan_download()` and then `repo.download(plan, approved=True)`. The [private data guide](../doc/private_data.rst) describes both interfaces. -9. Use the same discovery workflow for public datasets. The -[scientific CLI workflows](scientific_workflows_cli.md) select one EHT file from +9. Use the same `add` workflow for public datasets. The +[scientific CLI workflows](scientific_workflows_cli.md) catalog one EHT file from its [UVFITS directory](https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/) and one DESI redrock product from [HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). -These narrow dataset roots avoid listing an entire archive. HTTPS requires a -usable directory index or a backend that can enumerate the dataset; arbitrary -URLs cannot reveal hidden files. +A template lists only the directories it can match, so a URL deep in an archive +avoids listing the entire archive. HTTPS requires a usable directory index or a +backend that can enumerate the dataset; arbitrary URLs cannot reveal hidden +files. 10. Reuse the existing catalog without downloading dataset files. This local clone retains the committed catalog and its Git history, including the original @@ -153,22 +159,22 @@ cd "$HM_DEMO_WORKSPACE" hallmark clone ./client/.hm ./catalog-copy --no-download ``` -`clone` copies existing catalogs; raw dataset URLs belong to `init --from`. +`clone` copies existing catalogs; raw dataset URLs belong to `hallmark add`. A Git catalog can live on GitHub while its data remains on the SSH server. A published HTTP/SFTP catalog snapshot can likewise use a different data server; snapshot imports start local history. Cloning metadata does not require the dataset credentials. Git catalog authentication and data authentication are independent. -Clone preserves the complete catalog. `download --filter` selects transfers -without removing catalog rows. Plain `init` creates a local repository; the -legacy remote `build` command is deprecated in favor of `init --from`. -Downloading files alone does not populate the local object store; the scientific -workflows show how to add and commit selected local inputs before branching. - -This demo passed on Python 3.9 and 3.13 using a disposable local SFTP export -with two 30-byte run files and published checksums. Discovery, one approved -30-byte transfer, refusal before transfer, configuration changes, and catalog -cloning passed on both versions. These checks do not establish availability or -credentials for any external server. Keep the workspace for inspection and -remove it manually when finished. +Clone preserves the complete catalog. `download --include` selects transfers +without removing catalog rows. `init` creates an empty repository; the legacy +remote `build` command is deprecated in favor of `add`. Downloading files does +not populate the local object store; the scientific workflows show how to +version local settings beside cataloged inputs before branching. + +For this revision, steps 4, 5, 6 and 10 were run on Python 3.13 against a +disposable loopback SFTP-only export with two 30-byte run files and a published +`SHA256SUMS`. Cataloging, one approved 30-byte transfer, a declined transfer, +the mirror dry run, and catalog cloning passed. These checks do not establish +availability or credentials for any external server. Keep the workspace for +inspection and remove it manually when finished. diff --git a/demo/scientific_workflows_cli.md b/demo/scientific_workflows_cli.md index b0b72ca..bffc208 100644 --- a/demo/scientific_workflows_cli.md +++ b/demo/scientific_workflows_cli.md @@ -27,34 +27,37 @@ mkdir "$HM_ROOT" Use your normal Git author configuration. No optional scientific-processing packages are needed for these examples. Each remote workflow selects one file; review its size before approving the transfer. Allow space for the downloaded -input and a stored baseline copy. Settings changes store only the changed -settings file; they do not duplicate an unchanged scientific input. - -`init --from` discovers a raw dataset and creates a catalog using directory -listings and published metadata. It does not download dataset files. Its -`--filter` globs select catalog entries. Filename templates are detected from -those paths automatically; `--format` overrides detection without excluding -unmatched files. A single selected file may have no inferable template and is -still cataloged. `clone` copies a complete existing -catalog. Select transfers separately with `download --filter`. Every nonempty -CLI transfer displays its plan and prompts for approval. `--dry-run` prepares an -offline plan without transferring files. Unknown sizes remain unknown. +input. A remote input is cataloged by its source and checksum rather than +copied into the object store, and settings changes store only the changed +settings file. + +`hallmark add URL` catalogs remote files using directory listings and +published metadata. It does not download dataset files. A URL template such as +`.../uvfits/SR1_M87_2017_{day}_lo_hops_netcal_StokesI.uvfits` catalogs every +matching file and records `day` as a column; the examples below name one file +exactly, without fields, so each catalogs a single input. `hallmark ls-remote +URL` lists a directory and suggests templates, and `hallmark add -n` previews +the matches. `clone` copies a complete existing catalog. Select transfers +separately with `download --include`. Every nonempty CLI transfer displays its +plan and prompts for approval. `--dry-run` prepares an offline plan without +transferring files. Unknown sizes remain unknown. For interactive selection, use `hallmark download --interactive`. It accepts one raw glob per line or an all-files choice, shows recorded sizes, and lets you -download, change the selection, or skip. Initialization never offers downloads. -CLI cloning copies the complete catalog and then reviews a download plan by -default. Use `clone --interactive` for the chooser, `clone --filter` to narrow -transfers, or `clone --no-download` for metadata alone. Without a terminal, clone -keeps the catalog and prints commands for downloading later. +download, change the selection, or skip. `init` and `add` never offer +downloads. CLI cloning copies the complete catalog and then reviews a download +plan by default. Use `clone --interactive` for the chooser, `clone --include` +to narrow transfers, or `clone --no-download` for metadata alone. Without a +terminal, clone keeps the catalog and prints commands for downloading later. The examples below keep explicit selections so their intended inputs are clear. -Each example uses a working directory containing `.hm`. After download, -`hallmark add '{name}'` indexes the selected local input and settings file; -`commit` stores their contents so branch checkout can restore them. The catalog -URL and data URL can be hosted separately. See the -[private data guide](../doc/private_data.rst) for authentication and catalog -hosting, and [data backends](../doc/backends.rst) for backend extensions. +Each example uses a working directory containing `.hm`. The remote input and +the local `analysis-settings.txt` are two templates in the same branch. `commit` +stores the settings file's contents so branch checkout can restore it; checkout +leaves the downloaded input in place. The catalog URL and data URL can be +hosted separately. See the [private data guide](../doc/private_data.rst) for +authentication and catalog hosting, and [data backends](../doc/backends.rst) +for backend extensions. ## 1. CyVerse: EHT M87 visibility provenance @@ -65,16 +68,18 @@ amplitudes before imaging; here we only record that intended quality check. ### Catalog and approve a selected download -Initialize the selected catalog, enter its working directory, and review the -offline download plan. Run the transfer only after checking the selection and +Create a repository, catalog the selected product, and review the offline +download plan. Run the transfer only after checking the selection and answering `y` at its prompt. Declining leaves the catalog available. -The final check stops the workflow if discovery selected no input file. +`add` stops the workflow if the listing does not contain the product, and the +final check stops it if the file was not downloaded. ```bash -hallmark init "$HM_ROOT/eht" --from \ - 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/' \ - --filter 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' +hallmark init "$HM_ROOT/eht" cd "$HM_ROOT/eht" +hallmark add \ + 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' +hallmark commit -m 'Catalog EHT input' hallmark download --all --dry-run hallmark download --all test -f 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' @@ -82,14 +87,14 @@ test -f 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' ### Commit, annotate a branch, and recover -Store the original input with an analysis setting, then record a different +Store an analysis setting beside the cataloged input, then record a different quality-review choice on `eht-provenance`. Returning to `main` restores the original settings; the final command prints `qa=positive-weights`. ```bash printf '%s\n' 'qa=positive-weights' > analysis-settings.txt -hallmark add '{name}' -hallmark commit -m 'Preserve EHT input and analysis settings' +hallmark add 'analysis-settings.txt' +hallmark commit -m 'Record EHT analysis settings' hallmark checkout eht-provenance printf '%s\n' 'qa=positive-weights-and-amplitude-review' > analysis-settings.txt hallmark add . @@ -111,15 +116,16 @@ statistics; applying those cuts remains a separate analysis step. ### Catalog and download by product -Discover only `redrock-main-dark-23040.fits` in this HEALPixel, inspect the plan, -and approve its download. The selection excludes coadds and other products; -the final check stops the workflow if discovery selected no input file. +Catalog only `redrock-main-dark-23040.fits` in this HEALPixel, inspect the +plan, and approve its download. The selection excludes coadds and other +products; the final check stops the workflow if the file was not downloaded. ```bash -hallmark init "$HM_ROOT/desi" --from \ - 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/' \ - --filter 'redrock-main-dark-23040.fits' +hallmark init "$HM_ROOT/desi" cd "$HM_ROOT/desi" +hallmark add \ + 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/redrock-main-dark-23040.fits' +hallmark commit -m 'Catalog DESI input' hallmark download --all --dry-run hallmark download --all test -f 'redrock-main-dark-23040.fits' @@ -127,14 +133,14 @@ test -f 'redrock-main-dark-23040.fits' ### Compare two selection branches -Commit the input with an illustrative fit-quality threshold. Record a stricter +Commit an illustrative fit-quality threshold. Record a stricter threshold on `desi-quality`, then return to `main`; the restored setting is `min_deltachi2=25`. No target rows are filtered by these commands. ```bash printf '%s\n' 'min_deltachi2=25' > analysis-settings.txt -hallmark add '{name}' -hallmark commit -m 'Preserve DESI input and analysis settings' +hallmark add 'analysis-settings.txt' +hallmark commit -m 'Record DESI analysis settings' hallmark checkout desi-quality printf '%s\n' 'min_deltachi2=40' > analysis-settings.txt hallmark add . @@ -167,16 +173,17 @@ Host lab-data StrictHostKeyChecking yes ``` -Use the `stars/` directory as the dataset root so the selected product has a -simple local filename. Discover it, review the plan, and approve the transfer. -SSH configuration supplies authentication; no Hallmark profile is required. -The final check stops the workflow if discovery selected no input file. +The URL's `stars/` directory becomes the template's source, so the selected +product has a simple local filename. Catalog it, review the plan, and approve +the transfer. SSH configuration supplies authentication; no Hallmark profile is +required. The final check stops the workflow if the file was not downloaded. ```bash -hallmark init "$HM_ROOT/roman-rubin" --from \ - 'sftp://lab-data/home/researcher/hallmark-exports/openuniverse2024/stars/' \ - --filter 'pointsource_10307.parquet' +hallmark init "$HM_ROOT/roman-rubin" cd "$HM_ROOT/roman-rubin" +hallmark add \ + 'sftp://lab-data/home/researcher/hallmark-exports/openuniverse2024/stars/pointsource_10307.parquet' +hallmark commit -m 'Catalog Roman-Rubin input' hallmark download --all --dry-run hallmark download --all test -f 'pointsource_10307.parquet' @@ -184,14 +191,14 @@ test -f 'pointsource_10307.parquet' ### Save and compare an analysis choice -Store the selected input and an illustrative magnitude limit. Change that limit +Store an illustrative magnitude limit beside the cataloged input. Change that limit on `roman-rubin-selection`, then return to `main`. The restored value is `magnitude_limit=24`; the Parquet catalog has not been filtered or converted. ```bash printf '%s\n' 'magnitude_limit=24' > analysis-settings.txt -hallmark add '{name}' -hallmark commit -m 'Preserve Roman-Rubin input and analysis settings' +hallmark add 'analysis-settings.txt' +hallmark commit -m 'Record Roman-Rubin analysis settings' hallmark checkout roman-rubin-selection printf '%s\n' 'magnitude_limit=23' > analysis-settings.txt hallmark add . @@ -247,17 +254,17 @@ Use `du -sh "$HM_ROOT"` to inspect the workspace. Keep it while comparing branches; remove it manually when finished. No example changes the original remote data or the local DES source file. -All four CLI workflows passed with tiny local HTTP/SFTP fixtures on Python 3.9 -and 3.13. Each full run downloaded 66 bytes across the three remote inputs and -copied a 21-byte local input. The checks covered discovery, published checksums, -approval, settings commits, and branch restoration. Declined downloads and -empty selections stopped before versioning. Notebook copies also passed from -both the repository root and `demo/` on both Python versions. -Separate Python 3.13 runs confirmed that CLI and notebook transfer failures -stop before versioning and leave the catalog available. - -The 95 CLI regression tests passed on each Python version. Bash syntax, notebook -structure and Python 3.9 syntax, local links, Ruff, and strict Sphinx checks -passed. These fixtures do not validate scientific file formats, analysis -choices, public-server availability, or public download sizes. No research -servers were contacted and no public datasets were downloaded. +For this revision, the command blocks above were run in order on Python 3.13 +against tiny local HTTP fixtures that mirror the EHT, DESI and export paths, +including published MD5 and SHA-256 manifests. The SFTP workflow's commands +were run over HTTP; SFTP cataloging and transfer are covered by the OpenSSH +integration tests with a loopback server. Each remote workflow cataloged and +downloaded one input, stored only its settings files as objects, and restored +the baseline setting on `main`. Declined downloads stop the shell before +versioning. + +The complete test suite, including the OpenSSH integration tests, Ruff, and a +strict Sphinx build passed; CI runs the suite on Python 3.9 through 3.14. These +fixtures do not validate scientific file formats, analysis choices, +public-server availability, or public download sizes. No research servers were +contacted and no public datasets were downloaded. diff --git a/demo/scientific_workflows_python.ipynb b/demo/scientific_workflows_python.ipynb index 991a41c..217e4e3 100644 --- a/demo/scientific_workflows_python.ipynb +++ b/demo/scientific_workflows_python.ipynb @@ -9,13 +9,13 @@ "\n", "EHT and DESI observations, Roman–Rubin simulations, and local DES Y3 products. The matching [CLI guide](scientific_workflows_cli.md) uses the same selected files; this notebook also shows one worktree.\n", "\n", - "Run **Setup**, then one numbered workflow from start to finish. Each uses a separate directory and one explicitly selected input. Choose a new `ROOT` to repeat an example. `Repo.init(..., source=...)` discovers raw datasets without downloading files. `Repo.clone(...)` copies an existing catalog; it preserves the complete catalog and history. Use `plan_download(filter=...)` and approve transfers separately.\n", + "Run **Setup**, then one numbered workflow from start to finish. Each uses a separate directory and one explicitly selected input. Choose a new `ROOT` to repeat an example. `repo.add(url)` catalogs remote files without downloading them. `Repo.clone(...)` copies an existing catalog; it preserves the complete catalog and history. Use `plan_download(include=...)` and approve transfers separately.\n", "\n", - "`Repo.init(..., filter=...)` selects catalog entries. Filename templates are detected automatically from those paths; `format=...` overrides detection. Unmatched files remain cataloged. Selecting only one input may leave too little information to infer a template; its literal path and available metadata are still recorded.\n", + "A URL template such as `.../uvfits/SR1_M87_2017_{day}_lo_hops_netcal_StokesI.uvfits` catalogs every matching file and records its fields as columns. These examples name one file exactly, without fields, so each catalogs a single input with its size and any published checksum.\n", "\n", "Every transfer shows a plan and asks for approval. Declining or a failed transfer stops that workflow; do not continue to its versioning cell. The completed catalog remains available. The SSH example requires an existing export and your own access configuration.\n", "\n", - "The examples version a small `analysis-settings.txt` file alongside each unchanged input. These settings record intended analysis choices; the cells do not process the scientific data. Detailed processing is described in prose." + "The examples version a small `analysis-settings.txt` file as a local template beside each unchanged input. These settings record intended analysis choices; the cells do not process the scientific data. Detailed processing is described in prose." ] }, { @@ -88,7 +88,7 @@ "\n", "Use one April 5 low-band UVFITS file from the [EHT M87 release](https://github.com/eventhorizontelescope/2019-D01-01). A scientific analysis could inspect positive-weight visibility amplitudes and compare the two frequency bands before imaging. This example records a QA choice without recalibrating visibilities or reconstructing an image.\n", "\n", - "Discover that file under the release's `uvfits/` dataset root with the CyVerse backend. Review the plan and approve its download into the new worktree." + "Catalog that file by its URL in the release's `uvfits/` directory; the CyVerse backend is selected from the URL. Review the plan and approve its download into the new worktree." ] }, { @@ -100,7 +100,9 @@ "source": [ "EHT_FILE = \"SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits\"\n", "eht_dir = ROOT / \"eht\"\n", - "eht = Repo.init(eht_dir, source=EHT_URL, filter=EHT_FILE)\n", + "eht = Repo.init(eht_dir)\n", + "eht.add(EHT_URL + EHT_FILE)\n", + "eht.commit(\"Catalog EHT input\")\n", "download_after_review(eht, eht.plan_download(all_files=True))" ] }, @@ -109,7 +111,7 @@ "id": "science-07", "metadata": {}, "source": [ - "Commit the input and QA settings as a baseline, then record another choice on `eht-provenance`. Switching back restores the original settings. The final checks confirm both the restoration and the unchanged input checksum. The `{name}` format tracks the two files at the worktree root." + "Track the QA settings as a local template beside the cataloged input and commit them as a baseline, then record another choice on `eht-provenance`. Switching back restores the original settings; checkout leaves the downloaded input in place. The final checks confirm both the restoration and the unchanged input checksum." ] }, { @@ -121,8 +123,8 @@ "source": [ "eht_settings = eht_dir / \"analysis-settings.txt\"\n", "eht_settings.write_text(\"qa=positive-weights\\n\", encoding=\"utf-8\")\n", - "eht.add(\"{name}\")\n", - "eht.commit(\"Preserve EHT input and QA settings\")\n", + "eht.add(\"analysis-settings.txt\")\n", + "eht.commit(\"Record EHT QA settings\")\n", "eht_base = eht.branches()[\"current\"]\n", "eht_hash = Repo.checksum(eht_dir / EHT_FILE)\n", "eht.checkout(\"eht-provenance\")\n", @@ -139,7 +141,7 @@ "id": "science-09", "metadata": {}, "source": [ - "A worktree lets you keep an alternative branch open alongside the baseline. Create its sibling directory, open it with `Repo(path)`, and commit another settings value there. The original worktree retains its baseline settings." + "A worktree lets you keep an alternative branch open alongside the baseline. Create its sibling directory, open it with `Repo(path)`, and commit another settings value there. The original worktree retains its baseline settings. The new worktree restores the tracked settings file; the remote input is cataloged there but not downloaded." ] }, { @@ -169,7 +171,7 @@ "\n", "The settings below illustrate a threshold comparison, not official DESI LSS selection cuts. See the [redrock data model](https://desidatamodel.readthedocs.io/en/latest/DESI_SPECTRO_REDUX/SPECPROD/healpix/SURVEY/PROGRAM/PIXGROUP/PIXNUM/redrock-SURVEY-PROGRAM-PIXNUM.html) and [DR1 known issues](https://data.desi.lbl.gov/doc/releases/dr1/known-issues/) for scientific use.\n", "\n", - "Initialize a catalog from the narrow HTTPS dataset root, review the selected redrock file, and approve its download." + "Catalog the redrock file by its HTTPS URL, review the plan, and approve its download." ] }, { @@ -181,7 +183,9 @@ "source": [ "DESI_FILE = \"redrock-main-dark-23040.fits\"\n", "desi_dir = ROOT / \"desi\"\n", - "desi = Repo.init(desi_dir, source=DESI_URL, filter=DESI_FILE)\n", + "desi = Repo.init(desi_dir)\n", + "desi.add(DESI_URL + DESI_FILE)\n", + "desi.commit(\"Catalog DESI input\")\n", "download_after_review(desi, desi.plan_download(all_files=True))" ] }, @@ -190,7 +194,7 @@ "id": "science-13", "metadata": {}, "source": [ - "Commit the downloaded input and an illustrative DELTACHI2 threshold. Change only that threshold on `desi-quality`, commit it, then restore the baseline and check the unchanged input. No galaxy selection or FITS rewrite occurs here." + "Commit an illustrative DELTACHI2 threshold beside the cataloged input. Change only that threshold on `desi-quality`, commit it, then restore the baseline and check the unchanged input. No galaxy selection or FITS rewrite occurs here." ] }, { @@ -202,8 +206,8 @@ "source": [ "desi_settings = desi_dir / \"analysis-settings.txt\"\n", "desi_settings.write_text(\"min_deltachi2=25\\n\", encoding=\"utf-8\")\n", - "desi.add(\"{name}\")\n", - "desi.commit(\"Preserve DESI input and quality settings\")\n", + "desi.add(\"analysis-settings.txt\")\n", + "desi.commit(\"Record DESI quality settings\")\n", "desi_base = desi.branches()[\"current\"]\n", "desi_hash = Repo.checksum(desi_dir / DESI_FILE)\n", "desi.checkout(\"desi-quality\")\n", @@ -226,7 +230,7 @@ "\n", "The [CLI guide](scientific_workflows_cli.md#3-sshsftp-romanrubin-products-over-sftp) shows an illustrative `lab-data` SSH configuration. Replace its host, user, and key with your own settings and verify the server's host key. Set `VM_URL` to an export you already have permission to read; the notebook does not provision a VM or prepare its data.\n", "\n", - "Discover the selected file under the export's `stars/` directory, then review and approve its SFTP download. Other workflows do not use this SSH connection." + "Catalog the selected file by its URL in the export's `stars/` directory, then review and approve its SFTP download. Other workflows do not use this SSH connection." ] }, { @@ -238,7 +242,9 @@ "source": [ "VM_FILE = \"pointsource_10307.parquet\"\n", "vm_dir = ROOT / \"roman-rubin\"\n", - "vm = Repo.init(vm_dir, source=VM_URL, filter=VM_FILE)\n", + "vm = Repo.init(vm_dir)\n", + "vm.add(VM_URL + VM_FILE)\n", + "vm.commit(\"Catalog Roman-Rubin input\")\n", "download_after_review(vm, vm.plan_download(all_files=True))" ] }, @@ -259,8 +265,8 @@ "source": [ "vm_settings = vm_dir / \"analysis-settings.txt\"\n", "vm_settings.write_text(\"magnitude_limit=24\\n\", encoding=\"utf-8\")\n", - "vm.add(\"{name}\")\n", - "vm.commit(\"Preserve Roman-Rubin input and selection settings\")\n", + "vm.add(\"analysis-settings.txt\")\n", + "vm.commit(\"Record Roman-Rubin selection settings\")\n", "vm_base = vm.branches()[\"current\"]\n", "vm_hash = Repo.checksum(vm_dir / VM_FILE)\n", "vm.checkout(\"roman-rubin-selection\")\n", @@ -337,7 +343,7 @@ "source": [ "## Inspect disk use and keep or remove your experiment\n", "\n", - "The plan reports available transfer sizes before approval. Allow room for each downloaded input and its stored version; the EHT worktree also contains its own input copy. The DES workflow copies your existing input once and stores a version in `.hm/objects`. Changing these small settings files does not make new versions of the unchanged scientific inputs.\n", + "The plan reports available transfer sizes before approval. Allow room for each downloaded input; remote inputs are cataloged by URL and checksum rather than stored again in `.hm/objects`. The DES workflow copies your existing input once and stores a version in `.hm/objects`. Changing these small settings files does not make new versions of the unchanged scientific inputs.\n", "\n", "Keep `ROOT` to retain the experiments and their history. To discard them, remove only the new experiment directories you created, including `eht-alternate-qa` if you ran that cell. Do not delete a shared dataset directory or remove only a worktree's `.hm` link." ] @@ -349,11 +355,9 @@ "source": [ "## Validation record\n", "\n", - "All 11 code cells passed using tiny local HTTP/SFTP fixtures on Python 3.9 and 3.13, with a separate execution copy launched from the repository root and from `demo/` on each version. Each execution downloaded 66 bytes across the three remote inputs and copied a 21-byte local DES input. The checks covered discovery, published checksums, approval, settings commits, worktree isolation, input checksums, and branch recovery.\n", + "For this revision, all 11 code cells were run in order on Python 3.13 against tiny local HTTP fixtures that mirror the EHT, DESI and export paths, with the SFTP export's URL served over HTTP and each approval prompt answered `y`. The checks covered cataloging, published checksums, approval, settings commits, worktree isolation, input checksums, and branch recovery. SFTP cataloging and transfer are covered by the OpenSSH integration tests with a loopback server.\n", "\n", - "Separate Python 3.13 runs confirmed that refusal and reported transfer failures stop before versioning while leaving the catalog available.\n", - "\n", - "The source notebook passes notebook-structure and Python 3.9 syntax checks, keeps its outputs cleared, and requires explicit download approval. Rendered headings and local links were checked. The matching [CLI guide](scientific_workflows_cli.md#disk-inspection-cleanup-and-validation) records the CLI and documentation checks. These fixtures do not validate scientific processing, public-server availability, or public download sizes; no research servers were contacted and no public datasets were downloaded.\n" + "The source notebook passes nbformat validation, keeps its outputs cleared, and requires explicit download approval. The matching [CLI guide](scientific_workflows_cli.md#disk-inspection-cleanup-and-validation) records the CLI and documentation checks. These fixtures do not validate scientific processing, public-server availability, or public download sizes; no research servers were contacted and no public datasets were downloaded." ] } ], diff --git a/doc/api.rst b/doc/api.rst index dabed36..9765913 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -95,24 +95,26 @@ Create a local repository:: hallmark init ./local-project -Initialize a catalog from a remote dataset without downloading dataset files:: +Catalog remote files with a URL template, without downloading dataset files:: - hallmark init ./desi --from \ - https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/ \ - --filter 'redrock-main-dark-23040.fits' - hallmark init ./lab --from ssh://lab-data/srv/data/ --filter 'run*.h5' --format 'run{run:d}.h5' + hallmark add 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/{pixel}/redrock-main-dark-{pixel}.fits' + hallmark add 'ssh://lab-data/srv/data/run{run:d}.h5' + hallmark commit -m 'Catalog remote data' -``--filter`` selects catalog entries. Without ``--format``, filename templates -are detected automatically from the selected paths. An explicit ``--format`` -overrides detection. Both retain unmatched files, with empty parameter values; -if no template can be inferred, paths and available metadata are still cataloged. -Python uses the same ``filter=`` and ``format=`` arguments to ``Repo.init``. +The URL up to the first path segment containing a ``{field}`` is recorded as +the template's source; the rest is matched against the listed files, one path +segment at a time, and its fields become catalog columns. Each template is +tracked beside the others; adding one again syncs it with the server, and +``hallmark rm --cached TEMPLATE`` stops tracking it. ``hallmark add -n`` previews +the matches, and ``hallmark ls-remote URL`` lists a directory and suggests +templates. Python uses ``repo.add(url_template)``. Use ``hallmark clone SOURCE PATH`` for an existing Hallmark Git repository or published HTTP/SFTP snapshot. Git clones retain the full catalog and its history; snapshots start new local history. Use ``--source-type git`` or ``--source-type catalog`` to override automatic detection. The older ``build`` -command remains available but is deprecated in favor of ``init --from``. +command remains available but is deprecated in favor of ``add`` with a URL +template. Downloading remote data ----------------------- @@ -120,31 +122,35 @@ Downloading remote data Preview the selected files using the local catalog, then confirm a download:: hallmark download --all --dry-run - hallmark download --filter 'runs/**' + hallmark download --include 'runs/**' Explicit cataloged paths and ``--tsv data.tsv`` also select files. Every nonempty -transfer requires interactive confirmation. Initialization only creates the -repository or discovers its catalog; it never offers downloads. CLI cloning -copies the complete catalog, then reviews a download plan by default. Use -``clone --filter`` to narrow that plan, ``clone --interactive`` for a chooser, -or ``clone --no-download`` to skip review. ``clone --output`` specifies the download -directory; bare catalogs prompt for one if omitted. These download options cannot -be combined with ``--no-download``; ``--interactive`` also conflicts with ``--filter``. -``download --filter`` narrows the saved catalog without changing it. +transfer requires interactive confirmation. ``init`` and ``add`` only create the +repository or catalog files; they never offer downloads. Each remote template's +files download from its source, so one plan can span several servers; +``--remote NAME`` selects a configured data remote for every file instead. CLI +cloning copies the complete catalog, then reviews a download plan by default. +Use ``clone --include`` to narrow that plan, ``clone --interactive`` for a +chooser, or ``clone --no-download`` to skip review. ``clone --output`` specifies +the download directory; bare catalogs prompt for one if omitted. These download +options cannot be combined with ``--no-download``; ``--interactive`` also +conflicts with ``--include``. ``download --include`` narrows the saved catalog +without changing it. The old download ``--yes`` option does not bypass approval. Use ``hallmark download --interactive`` to choose patterns or all cataloged files, review recorded sizes, then download, revise, or skip. Add ``--dry-run`` to stop at the preview. Without a terminal, clone keeps the catalog and prints commands for -later; standalone ``download --interactive`` reports an error. Python -initialization and cloning remain metadata-only and never open this CLI chooser. +later; standalone ``download --interactive`` reports an error. Python ``add`` +and cloning remain metadata-only and never open this CLI chooser. Python uses the same plan. First inspect the selected files:: from hallmark import Repo - repo = Repo.init('lab', source='ssh://lab-data/srv/data/', progress=True) - plan = repo.plan_download(filter='runs/**') + repo = Repo.init('lab') + repo.add('ssh://lab-data/srv/data/{group}/run{run:d}.h5', progress=True) + plan = repo.plan_download(include='runs/**') print(plan.summary()) After reviewing the plan, approve the download and check for failures:: @@ -153,7 +159,7 @@ After reviewing the plan, approve the download and check for failures:: if result['failed']: raise RuntimeError('\n'.join(result['errors'])) -Plans preserve their selected remote, backend settings, destination and +Plans preserve their selected sources, backend settings, destination and checksums even when repository configuration later changes. Size estimates require recorded file sizes; duration estimates also require a supplied transfer rate. @@ -179,8 +185,8 @@ Dataset builders and data remotes .. autofunction:: hallmark.repo_builder.list_remote_files -SSH/SFTP uses standard ``~/.ssh/config`` and the SSH agent. Set a remote URL -containing the configured host alias; no Hallmark authentication profile is used. +SSH/SFTP uses standard ``~/.ssh/config`` and the SSH agent. Use URLs containing +the configured host alias; no Hallmark authentication profile is used. The legacy ``download_remote_data`` function also requires ``approved=True``. Both download APIs return a dictionary containing ``succeeded``, @@ -197,8 +203,11 @@ supported server configurations. Named data sources ------------------ -List sources with ``hallmark sources`` and release/collection roots with -``hallmark sources desi``. See :doc:`private_data` for selection and migration. +List sources with ``hallmark sources`` and release/collection URLs with +``hallmark sources desi``, then catalog files beneath them with ``ls-remote`` and +``add``. See :doc:`private_data` for templates and migration. + +.. autofunction:: hallmark.repo_remote.suggest_templates .. automodule:: hallmark.sources :members: DataSource, SourceRelease, register_source, get_source, list_sources diff --git a/doc/backends.rst b/doc/backends.rst index f937abe..d31a409 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -18,10 +18,10 @@ implementations: * ``CyVerseBackend`` extends HTTP support with CyVerse directory and metadata parsing. -Users select data sources with ``--from NAME`` or provide a raw dataset URL. -For raw URLs, Hallmark selects the transport automatically. Source authors can -specify an adapter and its nonsecret configuration in a release definition; -users do not need backend flags or an options file. +Users catalog files with ``hallmark add 'URL/TEMPLATE'``, and Hallmark selects +the transport from the URL automatically. Source authors can specify an adapter +and its nonsecret configuration in a release definition; templates added +beneath the release URL use it, so users need no backend flags or options file. Define a source and register it for the current Python process: @@ -36,35 +36,38 @@ Define a source and register it for the current Python process: ), }) register_source(survey) - repo = Repo.init("survey", source="survey", release="dr1", - collections=["redshifts"], filter="**/*.fits") + repo = Repo.init("survey") + repo.add("https://survey.example.org/release/catalogs/redshifts/{name}.fits") -Collection roots are relative directories, and catalog paths remain relative -to the release URL. An omitted collection catalogs the entire release. To -expose this descriptor to other Python processes and the CLI, publish the -``survey`` instance in the plugin's ``pyproject.toml``: +Collection roots are relative directories below the release URL. To expose +this descriptor to other Python processes and the CLI, publish the ``survey`` +instance in the plugin's ``pyproject.toml``: .. code-block:: toml [project.entry-points."hallmark.sources"] survey = "my_survey.sources:survey" -Users can then run ``hallmark sources survey`` to see available releases and -roots, or ``hallmark init ./survey --from survey --release dr1``. Source -registration and listing must not contact the service. Source names are unique -across registered and installed descriptors. - -The saved data remote retains its resolved URL and transport ``backend`` and -``backend_options``. Existing catalog transport metadata remains supported. -The developer API ``repo.set_config(remote_backend=..., -remote_backend_options=...)`` can update this configuration; it has no CLI -counterpart. Options contain shareable configuration, never credentials. -SSH uses ``~/.ssh/config`` and its agent; HTTP retains Requests authentication. +Users can then run ``hallmark sources survey`` to see the URLs of available +releases and collections, and ``hallmark ls-remote`` and ``hallmark add`` with +those URLs. Source registration and listing must not contact the service. +Source names are unique across registered and installed descriptors. + +A template's data entry records its URL and, when the URL alone would select a +different transport, its ``backend`` and ``backend_options``. Python +``repo.add(url_template, backend=..., backend_options=...)`` selects them +explicitly. Data remotes configured with ``set-config`` keep their own +transport settings; the developer API ``repo.set_config(remote_backend=..., +remote_backend_options=...)`` updates them and has no CLI counterpart. +Existing catalog transport metadata remains supported. Options contain +shareable configuration, never credentials. SSH uses ``~/.ssh/config`` and its +agent; HTTP retains Requests authentication. ``repo.plan_download()`` reads the local catalog without opening a connection. -The plan captures its backend and immutable options, source URL, destination -and file metadata. Later configuration changes cannot redirect an approved -plan. Install the selected transport plugin before executing a transfer. +The plan captures each source's backend and immutable options, URL, the +destination and file metadata. Later configuration changes cannot redirect an +approved plan. Install the selected transport plugins before executing a +transfer. Writing a backend ----------------- @@ -89,6 +92,10 @@ operation has been cancelled. - Yield ``RemoteEntry`` records with literal logical paths relative to the dataset root. Report completed directories through the optional callback. Missing sizes, timestamps and published checksums remain ``None``. + When ``context.descend`` is set, skip each subdirectory for which it + returns False; it receives the relative path ending in ``/``. ``add`` + uses it so templates list only directories they can match. Ignoring it + is correct but slower. * - ``read_text(relative_path, limit)`` - Read metadata within the supplied byte limit and return decoded text. Reject oversized responses and propagate access or connection errors. @@ -147,7 +154,7 @@ For example, an HTTP service can publish a small ``manifest.json`` array with if on_directory is not None: on_directory("") -Register the class for the current Python process before initialization or +Register the class for the current Python process before cataloging or transfer: .. code-block:: python @@ -160,7 +167,8 @@ transfer: "dr1": SourceRelease("https://survey.example.org/release/", {}, backend="survey"), })) - repo = Repo.init("survey", source="survey", release="dr1") + repo = Repo.init("survey") + repo.add("https://survey.example.org/release/{field}/{name}.fits") For CLI use and other collaborators, publish the class as an entry point in the plugin package's ``pyproject.toml``: @@ -211,8 +219,9 @@ From the source checkout, register the demonstration backend explicitly: "south": "https://south.example.org/export/", }}), })) - repo = Repo.init("combined", source="combined", release="v1") - plan = repo.plan_download(filter="north/**/*.fits") + repo = Repo.init("combined") + repo.add("https://north.example.org/export/{site}/run_{run}.fits") + plan = repo.plan_download(include="north/**/*.fits") print(plan.summary()) The example is tested with two disposable local HTTP servers. The URLs above diff --git a/doc/private_data.rst b/doc/private_data.rst index 80e58f9..5337e7a 100644 --- a/doc/private_data.rst +++ b/doc/private_data.rst @@ -1,18 +1,18 @@ Data sources and private data ============================= -``hallmark init`` creates a local repository or catalogs a named source or raw -dataset URL. It reads directory listings and published checksum metadata without -downloading dataset files or offering transfers. ``hallmark clone`` copies a -complete existing catalog, then reviews a download plan by default. Use -``clone --no-download`` to skip review. ``hallmark download`` selects and approves -transfers from an existing catalog. +``hallmark add`` catalogs remote files that match a URL template. It reads +directory listings and published checksum metadata without downloading dataset +files or offering transfers. ``hallmark ls-remote`` lists a remote directory +and suggests templates. ``hallmark clone`` copies a complete existing catalog, +then reviews a download plan by default. Use ``clone --no-download`` to skip +review. ``hallmark download`` selects and approves transfers from a catalog. 1. Choose a source ------------------ List supported sources, then inspect DESI releases and collection roots. These -commands read source definitions without crawling a dataset: +commands read source definitions without listing a dataset: .. code-block:: bash @@ -25,35 +25,31 @@ catalogs), ``spectra`` (HEALPixel and tile product directories), and ``vac`` catalogs). The mappings follow the `DESI data organization `_. -Create a catalog for one collection in an explicit release: +``hallmark sources desi`` prints the URL of each release and collection. List +a collection to see its files and suggested templates, then add one of the +suggested commands to a repository: .. code-block:: bash - hallmark init ./desi --from desi --release dr1 --collection redshifts + hallmark init ./desi + cd desi + hallmark ls-remote https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/zcatalog/ + hallmark add 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/{pixel}/redrock-main-dark-{pixel}.fits' + hallmark commit -m 'Catalog DR1 redshifts' -Repeat ``--collection`` to include several collections. Without it, Hallmark -catalogs the entire release, including spectra, ancillary products and root -files. Collection selection limits the directories traversed. Paths remain -relative to the release root even when a single collection is selected. -The source name, release, collections and resolved URL are saved in metadata. -Downloads use that resolved URL and the saved catalog without rediscovery. - -``hallmark init ./desi --from desi`` prompts for a release in a terminal. -Scripts and Python must provide a release explicitly. Cancelling the prompt -creates no destination. No release is selected implicitly. - -Raw URLs remain available for a specific subtree or another service: +The source name only helps you find URLs; the catalog records the URL itself. +A release that declares a transport backend supplies it to templates beneath +its URL. Any browsable URL works the same way: .. code-block:: bash - hallmark init ./observations --from 'https://data.example.org/export/' \ - --filter '**/*.fits' + hallmark add 'https://data.example.org/export/{site}/{night}.fits' -The URL is the exact recursive root. HTTP sources need a supported browsable -index; HTTP file access alone does not provide directory enumeration. -``--release`` and ``--collection`` apply only to named sources. Public HTTP -sources need no credentials; private HTTP retains Requests' local ``.netrc`` -and environment behavior. See :doc:`backends` for source and transport plugins. +HTTP sources need a supported browsable index; HTTP file access alone does not +provide directory listings. Public HTTP sources need no credentials; private +HTTP retains Requests' local ``.netrc`` and environment behavior. URLs with +credentials, queries or fragments are rejected, because the URL is committed +with the catalog. See :doc:`backends` for source and transport plugins. 2. Configure SSH or SFTP access ------------------------------- @@ -80,81 +76,100 @@ works without an interactive password or MFA prompt: Hallmark requires OpenSSH 9.6 or newer with connection multiplexing. Linux and macOS are supported; Windows SSH transport is unsupported. Both ``ssh://`` and -``sftp://`` use structured SFTP enumeration and transfers. The server needs no +``sftp://`` use structured SFTP listings and transfers. The server needs no Hallmark, Git, Python or login shell. SSH configuration supplies identities, users, ports and proxies, including ``ProxyJump``; explicit URL user/port values take precedence. Hallmark enforces noninteractive access and strict host-key checking. There is no Hallmark authentication file or profile-saving prompt. -3. Select catalog entries and extract fields --------------------------------------------- +3. Write a template +------------------- -Suppose the source contains ``runs/run_001.h5``, ``runs/run_002.h5`` and -``README.md``. Include both run files and the README, extracting the integer -``run`` field only from matching filenames: +Suppose the server contains ``runs/run_001.h5``, ``runs/run_002.h5`` and +``README.md``. Preview the run files, then catalog them and commit: .. code-block:: bash - hallmark init ./lab --from 'sftp://lab-data/srv/exports/lab/' \ - --filter 'runs/*.h5' --filter 'README*' \ - --format 'runs/run_{run:03d}.h5' - -``--filter`` selects catalog entries; repeated globs form a union. Matching -is case-sensitive, ``*`` stays within a path segment, and ``**`` spans zero or -more directories. Quote patterns to prevent shell expansion. Globs filter the -inventory; they do not currently prune directory traversal. Narrow the URL or -choose collections when you want to limit the crawl. - -Without ``--format``, Hallmark automatically detects filename templates from the -filtered paths using its existing format detector. Multiple detected templates -contribute columns to one catalog; each path uses the first matching template -in the detector's stable order. Inferred names are heuristic, for example -``runs/run_001.h5`` and ``runs/run_002.h5`` yield ``runs/run_{scan}.h5``. -Detection uses filenames only and does not inspect dataset contents. Static -metadata and archive files can be ignored during inference but remain cataloged. -If no usable template is found, the catalog contains paths and metadata alone. - -``--format`` overrides detection with one explicit template, allowing you to -choose field names and types. It adds named fields without excluding files. -In the example above, the README remains in -the catalog with an empty ``run`` value. Extraction fields cannot overwrite -reserved metadata columns. Their columns exist even if no filename matches. -Inferred templates that would overwrite metadata or cannot be parsed are skipped; -invalid explicit templates are rejected before discovery. Templates are recorded -under ``extraction`` in the data configuration, separately from the local indexing -``fmt`` setting. Literal catalog paths determine downloads, even if detection -found no template. Reloading, cloning, and downloading do not repeat detection. - -Without ``--filter``, every discovered file within the selected scope is -cataloged. Discovery preserves available sizes, times and published checksums; -unknown values remain unknown. It never downloads payloads to compute hashes. -Publish SHA-256 manifests for reproducible verification where possible. - -The destination may already contain files if it has no ``.hm``; those files are -preserved. An existing catalog is rejected before remote access. A destination -ending in ``.hm`` creates a bare catalog. Plain ``hallmark init PATH`` creates a -local repository without cataloging local files automatically. + hallmark init ./lab + cd lab + hallmark add -n 'sftp://lab-data/srv/exports/lab/runs/run_{run}.h5' + hallmark add 'sftp://lab-data/srv/exports/lab/runs/run_{run}.h5' + hallmark commit -m 'Catalog lab runs' + +The URL is split at the first path segment containing a ``{field}``: +``sftp://lab-data/srv/exports/lab/runs/`` becomes the template's source and +``run_{run}.h5`` its filename template. Files are cataloged, and later +downloaded, at paths relative to the source, here ``run_001.h5``. Put a +directory in the template to keep it in the local layout: +``sftp://lab-data/srv/exports/lab/{group}/run_{run}.h5`` catalogs +``runs/run_001.h5``. + +Matching follows local ``hallmark add``: + +- Each field matches text within one path segment and never spans ``/``. +- Matching is case-sensitive. Hidden files and directories match only a + template segment that starts with ``.``. +- A file is cataloged only if the template re-creates its name exactly, so + ``run_{run:03d}.h5`` matches ``run_001.h5`` but not ``run_1.h5``. ``add`` + reports how many listed files fit the pattern but not its formatting. +- Field names cannot be reserved catalog columns such as ``path``, + ``checksum`` or ``size_bytes``. +- Literal parts of the URL are percent-decoded once: ``%20`` names a space and + ``%7B`` a literal brace. + +Only directories the template can match are listed, so a template deep in a +large tree reads few listings. A template that matches nothing is an error and +changes nothing. The catalog records each file's field values, size, +modification time and published checksum; unknown values remain unknown. +Hallmark never downloads payloads to compute hashes. Publish SHA-256 manifests +for reproducible verification where possible. Remote files are committed as +catalog entries, not as local objects. + +Run the same ``add`` command again to sync the template with the server. New +files are added, changed sizes or checksums are updated, and files that are no +longer listed are removed from the catalog. ``hallmark status`` summarizes +these changes per template before you commit them. + +When you do not know the layout, ``hallmark ls-remote URL`` lists the directory +and prints suggested templates, with the number of files each matches. +Suggestions come from Hallmark's filename-format detector and are heuristic: +``run_001.h5`` and ``run_002.h5`` yield ``run_{scan}.h5``. Rename fields before +adding a suggestion. ``ls-remote`` lists the whole tree below the URL and never +changes a repository. + +A branch can track several templates. Each has its own source and catalog +table, and local templates, such as ``reduced/{run}.dat``, can sit beside remote +ones. A file belongs to only one template. ``hallmark rm --cached TEMPLATE`` +stops tracking a template and leaves its files in place. ``hallmark init PATH`` +creates an empty repository; a PATH ending in ``.hm`` creates a bare catalog. 4. Preview and approve downloads -------------------------------- -From the new worktree, preview one run using only local catalog metadata, then +From the worktree, preview one run using only local catalog metadata, then request the same transfer: .. code-block:: bash - cd lab - hallmark download --filter 'runs/run_001.h5' --dry-run - hallmark download --filter 'runs/run_001.h5' + hallmark download --include 'run_001.h5' --dry-run + hallmark download --include 'run_001.h5' The download command displays the file count, known bytes, unknown-size count, -source and destination, then asks for confirmation. Declining leaves the catalog -available. ``--filter`` selects only cataloged files; explicit paths cannot add -files omitted during initialization. Downloads do not change catalog membership. -Use ``--all`` for the whole catalog or ``--tsv data.tsv`` for a configured table. -A bare catalog requires ``--output DIRECTORY`` with explicit selectors; the -interactive chooser can prompt for it. ``--max-workers`` defaults to 4 -and must be positive. Internal SSH session limits may further cap concurrency. +sources and destination, then asks for confirmation. Declining leaves the +catalog available. ``--include`` selects only cataloged files; explicit paths +cannot add files that no template catalogs. Downloads do not change catalog +membership. Use ``--all`` for the whole catalog or ``--tsv data.tsv`` for one +template's table. A bare catalog requires ``--output DIRECTORY`` with explicit +selectors; the interactive chooser can prompt for it. ``--max-workers`` +defaults to 4 and must be positive. Internal SSH session limits may further cap +concurrency. + +Files of each remote template download from its source; files of local +templates download from the default data remote, if one is configured. A plan +can therefore span several servers, and its preview lists each source with its +file count. ``--remote NAME`` downloads every selected file from a data remote +configured with ``set-config`` instead. Selected files without any source are +skipped and counted. Dataset transfers require approval even for one file. The legacy download ``--yes`` option does not bypass confirmation. To choose files interactively @@ -170,59 +185,58 @@ shell quotes; a blank line finishes the list. Spaces and commas remain part of the pattern. Patterns are ORed, case-sensitive, and support recursive ``**``. Finishing an empty list or matching no files returns to selection. -The preview shows the source, destination, file count, known bytes, unknown-size -count, and up to 20 paths with recorded sizes. It uses local metadata only; -unknown sizes remain unknown. Choose ``download`` to approve that exact plan, -``change`` to select again and review a new plan, or ``skip`` (the default). -``--interactive --dry-run`` previews the selection and exits without offering -execution. ``--output``, ``--remote`` and ``--max-workers`` are supported with -``--interactive``; paths, ``--filter``, ``--tsv`` and ``--all`` cannot be combined -with it. Bare ``hallmark download`` still requires selectors or ``--interactive``. +The preview shows the sources, destination, file count, known bytes, +unknown-size count, and up to 20 paths with recorded sizes. It uses local +metadata only; unknown sizes remain unknown. Choose ``download`` to approve +that exact plan, ``change`` to select again and review a new plan, or ``skip`` +(the default). ``--interactive --dry-run`` previews the selection and exits +without offering execution. ``--output``, ``--remote`` and ``--max-workers`` are +supported with ``--interactive``; paths, ``--include``, ``--tsv`` and ``--all`` +cannot be combined with it. Bare ``hallmark download`` still requires +selectors or ``--interactive``. CLI cloning first copies the complete catalog, then shows a download plan and -asks for confirmation. All cataloged files are selected unless ``--filter`` +asks for confirmation. All cataloged files are selected unless ``--include`` narrows the planned transfers. These examples use separate destinations: .. code-block:: bash - hallmark clone ./lab/.hm ./lab-copy-selected --filter 'runs/run_001.h5' + hallmark clone ./lab/.hm ./lab-copy-selected --include 'run_001.h5' hallmark clone ./lab/.hm ./lab-copy-interactive --interactive -The initialization filter controls catalog membership. Clone filters and the -chooser select only downloads, retaining the complete catalog and its history. +Templates control catalog membership. Clone ``--include`` globs and the chooser +select only downloads, retaining the complete catalog and its history. ``clone --interactive`` opens the chooser instead of the default plan and cannot -be combined with ``--filter``. ``clone --no-download`` skips review entirely and -cannot be combined with ``--interactive``, ``--filter``, or ``--output``. +be combined with ``--include``. ``clone --no-download`` skips review entirely and +cannot be combined with ``--interactive``, ``--include``, or ``--output``. ``clone --output DIRECTORY`` changes the download destination; a bare catalog prompts for one if omitted. Declining or EOF during clone's review exits successfully and leaves the catalog available. Ctrl+C or a transfer failure exits unsuccessfully while preserving it. Empty catalogs and catalogs without a data -remote skip review with an explanation. +source skip review with an explanation. Without an interactive terminal, clone still copies the catalog, warns that approval is unavailable, and prints repository-specific preview and download -commands reflecting its filters and output directory. Later transfers still -require explicit approval. Standalone -``download --interactive`` requires a terminal. Existing source requirements, -such as an explicit DESI release in noninteractive use, still apply. -Download worker limits belong on ``download``; ``init`` and ``clone`` do not -accept them. Python initialization and cloning remain metadata-only. +commands reflecting its ``--include`` globs and output directory. Later +transfers still require explicit approval. Standalone ``download --interactive`` +requires a terminal. Download worker limits belong on ``download``; ``init``, +``add`` and ``clone`` do not accept them. Python ``add`` and cloning remain +metadata-only. 5. Use the Python API --------------------- -Create a catalog and inspect a plan before transferring selected data: +Catalog remote files and inspect a plan before transferring selected data: .. code-block:: python from hallmark import Repo - repo = Repo.init( - "lab-python", source="sftp://lab-data/srv/exports/lab/", - filter=["runs/*.h5", "README*"], format="runs/run_{run:03d}.h5", - progress=True, - ) - plan = repo.plan_download(filter="runs/run_001.h5") + repo = Repo.init("lab-python") + repo.add("sftp://lab-data/srv/exports/lab/runs/run_{run:03d}.h5", + progress=True) + repo.commit("Catalog lab runs") + plan = repo.plan_download(include="run_001.h5") print(plan.summary()) After reviewing the plan, approve it explicitly and check the results: @@ -233,11 +247,13 @@ After reviewing the plan, approve it explicitly and check the results: if result["failed"]: raise RuntimeError("; ".join(result["errors"])) -``Repo.init("desi", source="desi", release="dr1", collections=["redshifts"])`` -uses the same named source. Python never prompts for releases. Plans freeze -paths, checksums, transport options, source and destination; subsequent catalog -configuration changes do not redirect them. ``estimated_bytes_per_second`` can -provide a duration estimate when every selected file has a known size. +``repo.add(url_template, dry_run=True)`` returns the matching files without +staging them. ``repo.add(url_template, backend=..., backend_options=...)`` +records a transport plugin for the template. Plans freeze paths, checksums, +sources, transport options and destination; subsequent catalog configuration +changes do not redirect them. ``plan.sources`` lists the servers a plan +contacts. ``estimated_bytes_per_second`` can provide a duration estimate when +every selected file has a known size. 6. Share a catalog ------------------ @@ -248,19 +264,19 @@ Clone an existing catalog without download review, then select downloads separat hallmark clone ./lab/.hm ./lab-copy --no-download cd lab-copy - hallmark download --filter 'runs/run_001.h5' --dry-run - hallmark download --filter 'runs/run_001.h5' + hallmark download --include 'run_001.h5' --dry-run + hallmark download --include 'run_001.h5' Git clones preserve the full catalog and Git history. Published HTTP/SFTP snapshots start new local history. Use ``--source-type git`` or ``--source-type catalog`` to override source detection. The catalog can be -hosted independently from the dataset, for example GitHub metadata with an SFTP -data remote. Metadata cloning does not connect to that data remote. Each client +hosted independently from the dataset, for example GitHub metadata with SFTP +data sources. Metadata cloning does not connect to those sources. Each client configures its own SSH alias and credentials for later transfers. -Downloads alone do not populate the local object store. Use a local repository -and the normal ``add`` and ``commit`` workflow to version downloaded inputs. -Local filename-format indexing retains its existing behavior. +Downloads do not populate the local object store. To version downloaded files +locally, run ``hallmark rm --cached`` on the remote template and add the same +template as a local one. 7. Migrate older commands ------------------------- @@ -275,28 +291,34 @@ profile references produce an actionable migration error. * - Previous interface - Replacement + * - ``init --from URL`` / ``--release`` / ``--collection`` + - ``hallmark init PATH``, then ``hallmark add 'URL/TEMPLATE'``; find URLs + with ``hallmark sources`` and ``hallmark ls-remote``. + * - ``init --filter`` / ``--include`` + - The template selects files; ``hallmark add -n`` previews the matches. + * - ``init --format`` / ``--extract`` / ``--fmt`` + - The template's fields; ``hallmark ls-remote`` suggests templates. * - ``init --backend`` / ``--backend-options`` - - ``--from NAME --release NAME --collection NAME``; source authors configure adapters. - * - ``init --include`` - - ``init --filter`` selects catalog entries. - * - ``init --extract`` / ``--fmt`` - - ``--format`` overrides automatic detection; use ``--filter`` for selection. - * - ``download --include`` / ``--fmt`` - - ``download --filter`` with path globs. + - ``Repo.add(url_template, backend=..., backend_options=...)``; source + authors can declare a backend on a release. + * - ``download --filter`` / ``--fmt`` + - ``download --include`` with path globs. + * - ``clone --filter`` + - ``clone --include`` selects payloads only; the catalog remains complete. * - ``init --with-download`` - - Initialize metadata, then run ``download`` separately. + - Add a template, then run ``download`` separately. * - ``clone --with-download`` / ``--download`` - Plain ``clone`` reviews downloads by default; ``--interactive`` opens the chooser. * - ``clone --no-fetch-data`` or relying on catalog-only CLI defaults - ``clone --no-download`` copies the complete catalog without download review. - * - Clone filtering - - ``clone --filter`` selects payloads only; the catalog remains complete. + * - Adding a different template to replace a branch's catalog + - The template is added beside the others; ``hallmark rm --cached`` removes one. * - ``--auth``, ``--remote-auth``, ``--dataset-auth`` - Configure ``~/.ssh/config`` and use its host alias in the URL. - * - ``Repo.init(from_url=...)`` - - ``Repo.init(source=...)``. - * - Python ``include=`` / ``extract=`` / remote ``fmt=`` - - ``filter=`` and initialization-only ``format=``. + * - ``Repo.init(source=...)`` / ``Repo.init(from_url=...)`` + - ``Repo.init(path)`` followed by ``repo.add(url_template)``. + * - Python ``filter=`` + - ``include=``. * - Python clone/init download arguments and approval callbacks - ``repo.plan_download()`` followed by ``repo.download(plan, approved=True)``. @@ -317,6 +339,7 @@ between chunks and remains subject to request timeouts. Missing checksums cannot verify content. Setup and authentication failures stop the operation. If SSH fails, check the alias with the SFTP command above, trusted host keys, -agent access and server permissions. If HTTP discovery fails, check that the -URL publishes a supported directory index. Large roots can take many listing -requests even when the final catalog contains few matching files. +agent access and server permissions. If listing fails, check that the URL +publishes a supported directory index. ``ls-remote`` lists the whole tree below +its URL, which can take many requests for a large root; ``add`` lists only the +directories its template can match. diff --git a/doc/usecase.rst b/doc/usecase.rst index 01fa3d9..a644733 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -42,15 +42,19 @@ associated with each file in different rows. ----------------------------------------------- Bob prefers keeping the data index separate from downloaded files. -He initializes a bare |hallmark|_ catalog from a simulation export and -verifies its location:: +He initializes a bare |hallmark|_ catalog, adds a simulation export from a +campus server, and verifies its location:: - hallmark init sim.hm --from ssh://campus/srv/export/ \ - --format 'run{run:d}/frame{frame:d}.h5' + hallmark init sim.hm cd sim.hm + hallmark add 'ssh://campus/srv/export/run{run:d}/frame{frame:d}.h5' + hallmark commit -m "Catalog simulation export" hallmark info The ``.hm`` suffix selects a bare catalog without a data worktree. +``add`` records ``ssh://campus/srv/export/`` as the source of the template +``run{run:d}/frame{frame:d}.h5`` and catalogs the matching files without +downloading them. Bob reviews the selection and approves downloading to an explicit directory:: hallmark download --all --output ../outputs --dry-run @@ -210,32 +214,62 @@ host alias to OpenSSH, which resolves the user, port, identity and any proxy. Explicit URL user and port values take precedence. Hallmark does not read a separate authentication file or prompt to create one. -Initialize a private catalog -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Catalog files on the server +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Frank can prepare a catalog directly from the files on the server:: +Frank can catalog the run files directly on the server:: - hallmark init lab --from ssh://campus/srv/export/ \ - --filter 'runs/run_*.h5' --format 'runs/run_{i:d}.h5' + hallmark init lab cd lab + hallmark ls-remote ssh://campus/srv/export/runs/ + hallmark add 'ssh://campus/srv/export/runs/run_{i:d}.h5' + hallmark commit -m "Catalog lab runs" hallmark download --all --dry-run hallmark download --all -This creates ``lab/.hm`` with only the matching run files in ``data.tsv``. -Without an inclusion glob, the catalog contains all discovered files below -the supplied URL. Initialization downloads no dataset contents. +``ls-remote`` lists the directory and suggests templates without changing the +repository. ``add`` records ``ssh://campus/srv/export/runs/``, the URL up to +the first path segment with a field, as the template's source, and catalogs +only the files matching ``run_{i:d}.h5``. They download to ``lab/run_1.h5`` +and so on; to keep the ``runs`` directory in the local layout, make it part of +the template, as in ``ssh://campus/srv/export/{group}/run_{i:d}.h5``. +Fields never span a ``/``, and only directories the template can match are +listed. A template that matches nothing is an error, and a file is cataloged +only if the template re-creates its name exactly; ``hallmark add -n`` previews +the matches. No dataset contents are downloaded. The final command displays the transfer plan and asks Frank for confirmation. Both SSH and SFTP sources use structured SFTP directory enumeration. The server needs no login shell, Python or Hallmark, and SFTP-only accounts work for -both discovery and downloads. Symlinks and special files are skipped. -Authentication and permission failures stop discovery rather than producing +both listing and downloads. Symlinks and special files are skipped. +Authentication and permission failures stop listing rather than producing an apparently complete catalog. -The generated data remote ``origin`` records the source URL. +When the server changes, Frank runs the same ``add`` command again. It syncs +the template's catalog: new files are added, changed sizes or checksums are +updated, and files no longer on the server are dropped. ``hallmark status`` +summarizes these catalog changes per template before he commits them. + +Frank keeps his reduced products in the worktree beside the server data. A +branch can track several templates, local and remote:: + + hallmark add 'reduced/run_{i:d}.dat' + hallmark commit -m "Add reduced runs" + +Local templates are committed as objects, so ``checkout`` restores their +files. Remote templates are committed as catalog entries only: checkout never +removes or restores downloaded copies, and ``status`` does not report missing +or downloaded copies as changes. After switching branches, downloading again +replaces a stale copy. ``hallmark rm --cached TEMPLATE`` stops tracking a +template and leaves its files in place. + +The data entry records the template's source URL. Data remotes configured +with ``set-config`` act as mirrors: ``download --remote NAME`` downloads every +selected file from that remote instead. Local templates download from the +default data remote, when one is configured. Existing Git-hosted catalogs keep their complete catalog, history and -recorded data remotes when cloned. Select and approve transfers afterward with -``download --filter``; downloads leave the catalog and history unchanged. +recorded sources when cloned. Select and approve transfers afterward with +``download --include``; downloads leave the catalog and history unchanged. The Git catalog can be hosted separately from its data servers. Published manifest checksums are recorded in the catalog. Missing checksums @@ -247,9 +281,8 @@ be cancelled. CyVerse and common HTTPS directory indexes are detected automatically:: - hallmark init desi --from \ - https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/ \ - --filter 'redrock-main-dark-23040.fits' + hallmark add \ + 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/{pixel}/redrock-main-dark-{pixel}.fits' A server with no usable listing needs a published Hallmark catalog or a backend plugin that understands its API. See :doc:`backends` for the shared @@ -262,7 +295,7 @@ Frank can also inspect and approve a transfer in Python:: from hallmark import Repo repo = Repo('lab') - plan = repo.plan_download(filter='runs/run_1.h5') + plan = repo.plan_download(include='run_1.h5') print(plan.summary()) After reviewing the plan, he approves the download and checks for failures:: @@ -275,10 +308,10 @@ Plans are built from local metadata without contacting the server. Missing sizes and duration estimates are reported as unknown. During transfer, progress reports bytes and estimates remaining time when possible. ``build`` and ``build_repo`` remain available but are deprecated in favor of -``init --from``. -Local ``add``, ``commit`` and ``checkout`` retain their existing format and -local-object requirements; preparing a remote catalog does not populate the -local object store. +``add`` with a URL template. +Cataloging remote files does not populate the local object store. To version +downloaded files locally, stop tracking the remote template with +``hallmark rm --cached`` and add the same template as a local one. Transport validation ~~~~~~~~~~~~~~~~~~~~ diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index b33d921..bff0316 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -483,8 +483,7 @@ def status(self) -> dict[str, object]: Returns: dict[str, object]: Status summary including: - branch (str) - - staged changes (dict), including per-template ``catalog`` - summaries + - staged changes (dict), with per-template ``catalog`` summaries - worktree changes (dict) - untracked files (list[str]) """ diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index c4fd74f..fedf7d7 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -16,7 +16,7 @@ def effective_cwd(repo) -> Path: Args: repo (repo): repository object Returns: - path: The current working directory if it is inside the repository + Path: The current working directory if it is inside the repository worktree; otherwise, the worktree root. ''' if repo.worktree is None: From 53e85f56589ee64cb698e5f12374d10865236aec Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 13:18:55 -0700 Subject: [PATCH 28/29] Fix ls-remote CLI test on Python 3.9 Python 3.9 installs Click 8.1, whose CliRunner mixes stderr into stdout, so the discovery progress bar preceded the "Would add" preview. Locate the preview instead of assuming it is the first line. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/test_sources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_sources.py b/test/test_sources.py index cbabed1..33e77fc 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -240,8 +240,10 @@ def test_cli_ls_remote_then_add_workflow(tmp_path, pages, monkeypatch): monkeypatch.chdir(tmp_path / "cli") preview = runner.invoke(hallmark, ["add", "-n", root + "run_{run:03d}.h5"]) assert preview.exit_code == 0, preview.output - assert preview.stdout.splitlines()[:3] == ["Would add", "run_001.h5", - "run_002.h5"] + # Click 8.1, used on Python 3.9, mixes the stderr progress bar into stdout + lines = preview.stdout.splitlines() + start = lines.index("Would add") + assert lines[start:start + 3] == ["Would add", "run_001.h5", "run_002.h5"] assert not Repo(tmp_path / "cli").state.config["data"][0].get("fmt") added = runner.invoke(hallmark, ["add", root + "run_{run:03d}.h5"]) assert added.exit_code == 0, added.output From 0d31c6a6b3e767e36bbee7f79e570f49fa745fd5 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 24 Sep 2026 14:39:16 -0700 Subject: [PATCH 29/29] upd gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ad0c224..c4260be 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ _build/ !/demo/scientific_workflows_cli.md !/demo/scientific_workflows_python.ipynb /cli_test_repos/ +doc/temp.md