diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16556b2..9b31994 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,9 +5,9 @@ name: CI on: push: - branches: [ "main" ] + branches: [ "main", "dev" ] pull_request: - branches: [ "main" ] + branches: [ "main", "dev" ] jobs: build: @@ -35,3 +35,24 @@ jobs: - name: Test with pytest run: | pytest + + ssh-transport: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + - name: Install Linux SSH server + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y openssh-server + - name: Install Hallmark and tests + run: python -m pip install -e . pytest + - name: Exercise disposable OpenSSH server and client + env: + HALLMARK_RUN_SSH_TESTS: "1" + run: python -m pytest test/test_ssh_integration.py test/test_transport.py -q diff --git a/.gitignore b/.gitignore index c1b977c..c4260be 100644 --- a/.gitignore +++ b/.gitignore @@ -17,5 +17,10 @@ __pycache__/ # Ignore Sphinx build output by default _build/ -/demo/ -/cli_test_repos/ \ No newline at end of file +/demo/* +!/demo/private_data_demo.md +!/demo/multi_server_backend.py +!/demo/scientific_workflows_cli.md +!/demo/scientific_workflows_python.ipynb +/cli_test_repos/ +doc/temp.md diff --git a/README.rst b/README.rst index 9df3b17..5fe44c3 100644 --- a/README.rst +++ b/README.rst @@ -46,7 +46,7 @@ Features: ------------- Examples of using ``ParaFrame`` with Python API or Command Line Interface (CLI) -can be found in the Jupyter Notebook tutorials in the ``demos`` folder. +can be found in the Jupyter Notebook tutorials in the ``demo`` folder. ``Installation`` ----------------- @@ -69,4 +69,76 @@ Or install from source for development purposes:: .. _hallmark: https://github.com/l6a/hallmark .. _yukon: https://github.com/l6a/yukon .. _banyan: https://github.com/l6a/banyan -.. _eht: https://eventhorizontelescope.org \ No newline at end of file +.. _eht: https://eventhorizontelescope.org + +Private data remotes +-------------------- + +|hallmark|_ can catalog and download data products stored on HTTP(S), SSH, and +SFTP servers. A data source specifies where the files are stored, while a Git +remote is used to share the history of the data index. +For a private server, |hallmark|_ uses OpenSSH 9.6 or newer with a trusted +host key and key-based authentication that does not require a prompt. + +Add remote files to a repository with a URL template, then select files to +download:: + + hallmark init eht + cd eht + hallmark add 'https://archive.example.org/2026MOVIE/ER2/{src}_{day}.h5' + hallmark commit -m 'Add remote EHT data' + hallmark download --all --dry-run + hallmark download --all + +``add`` lists the remote directory and catalogs the matching files, with their +sizes and published checksums, without downloading dataset contents. The URL +up to the first path segment containing a ``{field}`` is recorded as the +template's source; the rest is the filename template, and its fields become +catalog columns. Adding the same URL template again syncs the catalog with the +server, including files that were removed. A branch can track several +templates, local or remote, such as ``{run}.dat`` files in the worktree and +``.h5`` files on a server; ``hallmark rm --cached TEMPLATE`` stops tracking +one. Remote files are committed as catalog entries, not as local objects. + +Use ``hallmark ls-remote URL`` to list a directory and print suggested +templates, and ``hallmark add -n 'URL/TEMPLATE'`` to preview the matches. +CyVerse and ordinary browsable HTTPS directories, including DESI, use the same +workflow. SSH listing also works with SFTP-only accounts; no server shell or +Python is required. + +Every dataset transfer requires approval. Python callers can inspect +``repo.plan_download()`` and then execute ``repo.download(plan, approved=True)``. +CLI users can run ``hallmark download --interactive`` to choose path globs or +all cataloged files, review recorded sizes, then download, change the selection, +or skip. ``init`` and ``add`` never offer downloads. CLI cloning copies the +complete catalog, then displays a download plan and requests confirmation by +default. Use ``clone --interactive`` for the chooser, or ``clone --no-download`` +to copy only the catalog. Without a terminal, clone keeps the catalog, skips +downloading, and prints commands to use later. +SSH and SFTP use standard ``~/.ssh/config`` aliases and the SSH agent. +Use ``hallmark sources desi`` to list the release and collection URLs of a +named source for use with ``ls-remote`` and ``add``. Download selection belongs +to the separate ``download --include`` command. +The older remote ``build`` command is deprecated in favor of ``add``. + +Use ``hallmark clone CATALOG PATH`` for an existing Git-hosted ``.hm`` or a +published HTTP/SFTP catalog snapshot. Catalogs can live on GitHub or another +server while their data sources point elsewhere. Git clones preserve the full +catalog and its history. ``clone --include`` narrows the planned downloads +without removing catalog entries. No payloads transfer until the user approves +the plan. + +The public ``DataBackend`` interface supports generic HTTP/SSH, CyVerse, and +registered plugins for collaboration-specific APIs and multiple data servers. +See `Data backends `_ for configuration and an extension example. + +Examples and configuration details are described in +`Private data over SSH and SFTP `_. + +For a sequential CLI walkthrough, see the +`private data demo `_. It includes catalog +initialization, SSH configuration and explicit download approval. + +The `scientific CLI workflows `_ and +`scientific Python notebook `_ +follow four examples using EHT, DESI, Roman–Rubin and local DES data. diff --git a/demo/demo_cli.ipynb b/demo/demo_cli.ipynb index 152fc11..b4d3ed4 100644 --- a/demo/demo_cli.ipynb +++ b/demo/demo_cli.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -51,40 +51,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== .hm directory ===\n", - " config.yml\n", - " data.tsv\n", - " meta.yml\n", - "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n", - "\n", - "=== initial config.yml ===\n", - "# Edit this file only if your branch needs regex substitutions or a preset remote.\n", - "# For simple names, you can just run: hallmark add \"a{a}_i{i}.h5\"\n", - "data:\n", - " -\n", - " # fmt: \"{release}_{source}_{year}_{doy:03d}_{band}.uvfits\"\n", - " encoding:\n", - " # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "remote:\n", - " # name: origin\n", - " # url: https://example.com/path/to/data/\n", - "\n", - "=== objects stored so far ===\n", - " 0\n" - ] - } - ], + "outputs": [], "source": [ "echo \"=== .hm directory ===\"\n", "ls repo1/.hm/\n", @@ -107,24 +80,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "dot-hallmark repo: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm\"\n", - "hallmark worktree: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1\"\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark info && popd\n" ] @@ -138,33 +100,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files in repo1/:\n", - " a0.75_i0.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0.75_i120.h5\n", - " a0_i0.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - " a0_i120.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1\n", "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", @@ -182,71 +124,26 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Changes to be committed\n", - " a0.75_i0.h5\n", - "a0.75_i120.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0_i0.h5\n", - " a0_i120.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd\n" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== config.yml after add ===\n", - "data:\n", - "- fmt: a{a}_i{i}.h5\n", - " encoding: null\n", - "remote: null\n", - "\n", - "=== data.tsv after add ===\n", - "sha1\ta\ti\n", - "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", - "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", - "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", - "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", - "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", - "18682593a012503b71935017dbc044665254a23c\t0\t0\n", - "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", - "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", - "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", - "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" - ] - } - ], + "outputs": [], "source": [ "echo \"=== config.yml after add ===\"\n", "cat repo1/.hm/config.yml\n", @@ -257,45 +154,26 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Committed staged state changes.\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark commit -m \"add main dataset\" && popd\n" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after commit:\n", - " 10\n" - ] - } - ], + "outputs": [], "source": [ "echo \"Objects stored after commit:\"\n", "find repo1/.hm/objects -type f | wc -l\n" @@ -310,33 +188,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "commit 7c035e02716b9d3ee6ccecb9c0abb7769a9dd96a\n", - "Author: Ram Adithya \n", - "Date: Mon Apr 20 19:28:53 2026 -0700\n", - "\n", - " add main dataset\n", - "\n", - "commit d709e546105c75dad9b43b71f543151974d6d531\n", - "Author: Ram Adithya \n", - "Date: Mon Apr 20 19:28:50 2026 -0700\n", - "\n", - " Initial commit: local `.hm` repository\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1 && hallmark log && popd\n" ] @@ -804,9 +662,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 10. Clone a hallmark repository\n", - "\n", - "This clone demo uses the default CLI behavior, which clones the hallmark repo and then downloads the configured data files. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" + "This clone demo uses `--no-download` to copy the complete Hallmark catalog without download review. By default, CLI clone shows a plan for all cataloged files and requests confirmation after copying metadata. `clone --include` narrows only the payload selection, while `clone --interactive` opens the patterns/all/skip chooser. Without a terminal, clone keeps the catalog and prints download commands for later. Run `hallmark download --all` or `hallmark download --interactive` separately when ready. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" ] }, { @@ -819,7 +675,7 @@ }, "outputs": [], "source": [ - "hallmark clone \"https://github.com/l6a/hallmark-demo-repo.hm.git\" hallmark-demo-clone\n" + "hallmark clone \"https://github.com/l6a/hallmark-demo-repo.hm.git\" hallmark-demo-clone --no-download\n" ] }, { diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index da3c0b1..9c34a7a 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -79,33 +79,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/astro_se_int/hallmark/demo/repo1 ~/astro_se_int/hallmark/demo\n", - "Files in repo1/:\n", - "a0_i0.h5\n", - "a0_i120.h5\n", - "a0_i30.h5\n", - "a0_i60.h5\n", - "a0_i90.h5\n", - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", - "~/astro_se_int/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "pushd repo1\n", @@ -124,123 +104,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i30.h50.7530.0
3a0.75_i60.h50.7560.0
4a0.75_i90.h50.7590.0
5a0_i0.h50.000.0
6a0_i120.h50.00120.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "1 a0.75_i120.h5 0.75 120.0\n", - "2 a0.75_i30.h5 0.75 30.0\n", - "3 a0.75_i60.h5 0.75 60.0\n", - "4 a0.75_i90.h5 0.75 90.0\n", - "5 a0_i0.h5 0.00 0.0\n", - "6 a0_i120.h5 0.00 120.0\n", - "7 a0_i30.h5 0.00 30.0\n", - "8 a0_i60.h5 0.00 60.0\n", - "9 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "paraframe = repo1.add(\"a{a}_i{i}.h5\")\n", "paraframe" @@ -255,112 +125,52 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'main',\n", - " 'staged': {'state': ['config.yml', 'data.tsv', 'meta.yml'],\n", - " 'added': ['a0.75_i0.h5',\n", - " 'a0.75_i120.h5',\n", - " 'a0.75_i30.h5',\n", - " 'a0.75_i60.h5',\n", - " 'a0.75_i90.h5',\n", - " 'a0_i0.h5',\n", - " 'a0_i120.h5',\n", - " 'a0_i30.h5',\n", - " 'a0_i60.h5',\n", - " 'a0_i90.h5'],\n", - " 'modified': [],\n", - " 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "repo1.status()" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "repo1.commit(\"add main dataset\")\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'main',\n", - " 'staged': {'state': [], 'added': [], 'modified': [], 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "repo1.status()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after commit:\n", - " 10\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "echo \"Objects stored after commit:\"\n", @@ -376,31 +186,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "commit d3da52e3a3077ce58995037957d74d1316fadde8\n", - "Author: Nayera Abdessalam \n", - "Date: Fri May 1 09:27:14 2026 -0700\n", - "\n", - " add main dataset\n", - "\n", - "commit ecb6fc7ffa7fd6b61d59a580d131c829caf942a8\n", - "Author: Nayera Abdessalam \n", - "Date: Fri May 1 09:27:14 2026 -0700\n", - "\n", - " Initial commit: local `.hm` repository\n" - ] - } - ], + "outputs": [], "source": [ "print(repo1.log())\n" ] @@ -416,27 +208,13 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'experiment',\n", - " 'staged': {'state': [], 'added': [], 'modified': [], 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# add(\".\") rebuilds the manifest from files that currently exist\n", "\n", @@ -464,24 +242,13 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Check back to main to see that the manifest has been correctly rebuilt\n", "repo1.checkout(\"main\")" @@ -489,24 +256,13 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'current': 'main', 'names': ['experiment', 'main']}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Show available branches\n", "repo1.branches()" @@ -514,31 +270,13 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout back to main:\n", - "repo1/a0_i0.h5\n", - "repo1/a0_i120.h5\n", - "repo1/a0_i30.h5\n", - "repo1/a0_i60.h5\n", - "repo1/a0_i90.h5\n", - "repo1/a0.75_i0.h5\n", - "repo1/a0.75_i120.h5\n", - "repo1/a0.75_i30.h5\n", - "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "echo \"Files after checkout back to main:\"\n", @@ -561,88 +299,13 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
5a0_i0.h50.00.0
6a0_i120.h50.0120.0
7a0_i30.h50.030.0
8a0_i60.h50.060.0
9a0_i90.h50.090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "5 a0_i0.h5 0.0 0.0\n", - "6 a0_i120.h5 0.0 120.0\n", - "7 a0_i30.h5 0.0 30.0\n", - "8 a0_i60.h5 0.0 60.0\n", - "9 a0_i90.h5 0.0 90.0" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Single Parameter Filter\n", "paraframe(a=0)" @@ -650,95 +313,13 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
2a0.75_i30.h50.7530.0
5a0_i0.h50.000.0
6a0_i120.h50.00120.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "2 a0.75_i30.h5 0.75 30.0\n", - "5 a0_i0.h5 0.00 0.0\n", - "6 a0_i120.h5 0.00 120.0\n", - "7 a0_i30.h5 0.00 30.0\n", - "8 a0_i60.h5 0.00 60.0\n", - "9 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Or Filter\n", "paraframe(a=0, i=30)" @@ -746,60 +327,13 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
7a0_i30.h50.030.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "7 a0_i30.h5 0.0 30.0" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# And Filter\n", "paraframe(a=0)(i=30)" @@ -807,95 +341,13 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
2a0.75_i30.h50.7530.0
3a0.75_i60.h50.7560.0
4a0.75_i90.h50.7590.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "2 a0.75_i30.h5 0.75 30.0\n", - "3 a0.75_i60.h5 0.75 60.0\n", - "4 a0.75_i90.h5 0.75 90.0\n", - "7 a0_i30.h5 0.00 30.0\n", - "8 a0_i60.h5 0.00 60.0\n", - "9 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Masking Filter retains Pandas DataFrame functionality\n", "paraframe[(30 <= paraframe.i) & (paraframe.i <= 100)]" @@ -910,39 +362,19 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 8/8 [00:03<00:00, 2.26file/s]\n" - ] - }, - { - "ename": "DownloadError", - "evalue": "Failed to download 8 file(s):\n - Checksum mismatch for SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_hi_hops_netcal_StokesI.uvfits.part\n - ... 3 more error(s)", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mDownloadError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[27]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mhallmark\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01merror\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DestinationExistsError\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m clone = Repo.clone(\u001b[33m'\u001b[39m\u001b[33mhttps://github.com/l6a/hallmark-demo-repo.hm.git\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mhallmark-demo-clone\u001b[39m\u001b[33m\"\u001b[39m, show_progress=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 6\u001b[39m clone_result = {\n\u001b[32m 7\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdot_hallmark_repo\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(clone.dothm.path),\n\u001b[32m 8\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mhallmark_worktree\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(clone.worktree),\n\u001b[32m 9\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mbranches\u001b[39m\u001b[33m\"\u001b[39m: clone.branches(),\n\u001b[32m 10\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdata_rows\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mlen\u001b[39m(clone.state.data),\n\u001b[32m 11\u001b[39m }\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m DestinationExistsError \u001b[38;5;28;01mas\u001b[39;00m exc:\n", - "\u001b[36mFile \u001b[39m\u001b[32m:23\u001b[39m, in \u001b[36mclone\u001b[39m\u001b[34m(cls, url, path, fetch_data, max_workers, show_progress)\u001b[39m\n", - "\u001b[31mDownloadError\u001b[39m: Failed to download 8 file(s):\n - Checksum mismatch for SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_hi_hops_netcal_StokesI.uvfits.part\n - ... 3 more error(s)" - ] - } - ], + "outputs": [], "source": [ "# Demo: clone a remote demo repository\n", "from hallmark.error import DestinationExistsError\n", "\n", "try:\n", - " clone = Repo.clone('https://github.com/l6a/hallmark-demo-repo.hm.git', \"hallmark-demo-clone\", show_progress=True)\n", + " clone = Repo.clone('https://github.com/l6a/hallmark-demo-repo.hm.git', \"hallmark-demo-clone\", progress=True)\n", " clone_result = {\n", " \"dot_hallmark_repo\": str(clone.dothm.path),\n", " \"hallmark_worktree\": str(clone.worktree),\n", diff --git a/demo/multi_server_backend.py b/demo/multi_server_backend.py new file mode 100644 index 0000000..668fbb3 --- /dev/null +++ b/demo/multi_server_backend.py @@ -0,0 +1,97 @@ +"""Example backend joining independent HTTP roots into one logical dataset. + +Call ``register()`` and define a ``DataSource`` release with +``backend="multi-server"`` and ``backend_options={"routes": ...}``. Then +``repo.add(release_url + "{site}/file.fits")`` catalogs files under the release +URL with this backend; ``Repo.add(..., backend=..., backend_options=...)`` +selects it without registering a source. +An installed plugin instead exposes ``MultiServerBackend`` through the +``hallmark.backends`` entry-point group. No servers are contacted on import. +""" + +from collections.abc import Mapping +from contextlib import ExitStack +from dataclasses import replace + +from hallmark.backends import ( + DataBackend, OperationContext, RemoteConfigurationError, + RemoteObjectMissing, RemoteSpec, register_backend, +) +from hallmark.transport.base import literal_path + + +class MultiServerBackend(DataBackend): + """Map each first path segment to a separate HTTP(S) server root. + + Child contexts provide thread-local sessions. Their cancellation event is + shared with the parent; cleanup closes every child's sessions. Credentials + use Requests' usual local .netrc configuration for each server. + """ + + def __init__(self, context): + super().__init__(context) + routes = context.remote.backend_options.get("routes") + if not isinstance(routes, Mapping) or not routes: + raise RemoteConfigurationError("Multi-server options require routes") + specs = {} + for prefix, url in routes.items(): + if literal_path(prefix).as_posix() != prefix or "/" in prefix: + raise RemoteConfigurationError("Route names must be one path segment") + spec = RemoteSpec.parse(url, backend="http") + if spec.scheme not in {"http", "https"}: + raise RemoteConfigurationError("Example routes require HTTP(S) URLs") + specs[prefix] = spec + self._stack = ExitStack() + self._children = {} + try: + for prefix, spec in specs.items(): + child = self._stack.enter_context( + OperationContext(spec, context.output_root)) + child.cancelled = context.cancelled + self._children[prefix] = child + except BaseException: + self._stack.close() + raise + + def _route(self, relative_path): + path = literal_path(relative_path).as_posix() + prefix, separator, suffix = path.partition("/") + if not separator or prefix not in self._children: + raise RemoteObjectMissing("No route for this logical dataset path") + return self._children[prefix], suffix + + def prepare(self): + for child in self._children.values(): + child.transport.prepare() + + def iter_entries(self, on_directory=None): + for prefix, child in self._children.items(): + self.context.check_cancelled() + + def report(directory, prefix=prefix): + if on_directory is not None: + on_directory(prefix + "/" + directory) + + for entry in child.transport.iter_entries(on_directory=report): + yield replace(entry, path=prefix + "/" + entry.path) + + def read_text(self, relative_path, limit): + child, path = self._route(relative_path) + return child.transport.read_text(path, limit) + + def fetch(self, relative_path, destination, *, chunk_size=8192): + child, path = self._route(relative_path) + child.on_bytes = self.context.on_bytes + child.transport.fetch(path, destination, chunk_size=chunk_size) + + def cancel(self): + for child in self._children.values(): + child.cancel() + + def close(self): + self._stack.close() + + +def register(): + """Register this example in the current process before using it.""" + register_backend("multi-server", MultiServerBackend) diff --git a/demo/private_data_demo.md b/demo/private_data_demo.md new file mode 100644 index 0000000..49aaf11 --- /dev/null +++ b/demo/private_data_demo.md @@ -0,0 +1,180 @@ +# Private data demo: copyable instructions + +Run these instructions on your local machine using an existing data export. +The SSH configuration and dataset path are illustrative. Replace them with your +own export settings; this guide does not provision or contact a particular VM. + +The export is assumed to contain `runs/run_001.dat`, `runs/run_002.dat`, a +`README.md`, and optionally a publisher-provided `SHA256SUMS`. The client needs +Hallmark, Git, and OpenSSH `ssh`/`sftp` 9.6+. The server needs SFTP, but does not +require a login shell, Python, Hallmark, or Git. Run the Bash blocks in order in +one terminal, starting in the Hallmark source checkout. + +1. Install the checkout in your active Python environment. `set -e` stops the +shell if a command fails or a download is declined, so later commands do not +continue after an unsuccessful transfer. + +```bash +set -e +python -m pip install -e . +``` + +2. Add this illustrative entry to `~/.ssh/config`, preserving your other entries +and replacing the host, user, and identity with your export's settings. + +```text +Host lab-data + HostName vm.example.org + User researcher + IdentityFile ~/.ssh/hallmark_vm + IdentitiesOnly yes + StrictHostKeyChecking yes +``` + +The verified host key must already be in `known_hosts`. Load an encrypted key +into your SSH agent first. SSH configuration alone is sufficient for this demo; +see the [private data guide](../doc/private_data.rst) for details. Each +collaborator configures SSH access locally; a catalog stores only the URL. + +3. Set the exact export root and create a new local workspace. Choose a different +workspace name when repeating the demo; the data-server path below is an example. + +```bash +export HM_DEMO_URL='ssh://lab-data/srv/exports/lab/' +export HM_DEMO_WORKSPACE="$HOME/hallmark-private-client" +mkdir "$HM_DEMO_WORKSPACE" +cd "$HM_DEMO_WORKSPACE" +``` + +4. Create a local repository and catalog the two runs with a URL template. +The template selects the run files and extracts their run numbers; `{group}` +keeps the `runs` directory in the local layout. Cataloging does not authorize +a download. + +```bash +hallmark init ./client +cd client +hallmark ls-remote "$HM_DEMO_URL" +hallmark add "${HM_DEMO_URL}{group}/run_{run:03d}.dat" +hallmark commit -m 'Catalog lab runs' +``` + +`ls-remote` lists the export and suggests templates without changing the +repository. `add` records the export root as the template's source and lists +only the directories the template can match. Only listings and published +metadata are read. Available sizes, modification times, and published checksums +are retained; missing checksums stay unknown. Dataset files are not downloaded +to calculate checksums, and remote files are committed as catalog entries only. + +The README does not match the template and is not cataloged; add another +template to track it, as in `hallmark add "${HM_DEMO_URL}README.md"`. Running the +same `add` command later syncs the catalog with the export. Use `hallmark +sources` to find the URLs of named sources. See +[data backends](../doc/backends.rst) for source registration and adapter routing. + +5. Review an offline plan for one run, then approve its transfer. The second +command displays the plan again and asks for confirmation; answer `y` only when +the source, destination, and selection are correct. The final check stops the +workflow if no selected input was downloaded. + +```bash +hallmark download runs/run_001.dat --dry-run +hallmark download runs/run_001.dat --max-workers 2 +test -f runs/run_001.dat +``` + +A dry run reports file count, known bytes, unknown-size count, source, and +destination without contacting the server. It does not check credentials or +remote-file existence. Every nonempty CLI transfer prompts, including one file. +Refusal transfers nothing and leaves the catalog available. Empty selections +need no confirmation. `--all` selects every cataloged file; `--tsv data.tsv` +selects that table. Neither flag supplies approval. + +Successful files are published only after transfer and any checksum verification. +A failed transfer preserves an existing destination and removes its temporary +file. Cancellation closes connections started for the operation. Repeating a +download transfers the selection again. + +6. Configure the equivalent SFTP URL as a data remote and select it with +`--remote`, which downloads every selected file from that remote instead of the +template's source. Changing the configuration does not transfer files; the dry +run shows the new source and another destination. + +```bash +hallmark set-config --remote-name via-sftp \ + --remote-url "sftp://${HM_DEMO_URL#ssh://}" +hallmark download runs/run_001.dat --remote via-sftp --output ../via-sftp --dry-run +``` + +Both `ssh://` and `sftp://` use SFTP for discovery and data transfer. Relative +output paths use the current directory. A bare `.hm` catalog requires an +explicit `--output`, or the interactive chooser prompts for a directory. +Authentication uses `~/.ssh/config` and the SSH agent; Hallmark has no +authentication-profile file or flags. + +7. Use the same review step in Python. The +[scientific workflows notebook](scientific_workflows_python.ipynb) demonstrates +`Repo.plan_download()` followed by explicit approval and +`repo.download(plan, approved=True)`. It stops on refusal or returned transfer +failures. A plan retains its source, backend settings, +destination, paths, and checksums even if repository configuration later changes. + +8. `init`, `add` and cloning do not download payloads. To choose files +interactively from the completed catalog, run the following command. Choose +`patterns` and enter one raw relative-path glob per line, ending with a blank +line, or choose `all`. Review recorded sizes, then choose `download`, `change`, +or `skip` (the default). Unknown file sizes remain unknown. + +```bash +hallmark download --interactive +``` + +Add `--dry-run` to stop after the preview. `init` and `add` never offer +downloads. CLI cloning reviews a download plan after copying the complete +catalog by default; use `clone --include` to narrow transfers or +`clone --interactive` for this chooser. Use `clone --no-download` to skip +review. Without a terminal, clone keeps the catalog, skips downloading, and +prints commands reflecting the selected globs and output directory. Declining or EOF during clone's review preserves the catalog +and exits successfully; an interrupt or transfer failure preserves it but exits +with an error. Python continues to use `repo.plan_download()` and then +`repo.download(plan, approved=True)`. The +[private data guide](../doc/private_data.rst) describes both interfaces. + +9. Use the same `add` workflow for public datasets. The +[scientific CLI workflows](scientific_workflows_cli.md) catalog one EHT file from +its [UVFITS directory](https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/) +and one DESI redrock product from +[HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). +A template lists only the directories it can match, so a URL deep in an archive +avoids listing the entire archive. HTTPS requires a usable directory index or a +backend that can enumerate the dataset; arbitrary URLs cannot reveal hidden +files. + +10. Reuse the existing catalog without downloading dataset files. This local +clone retains the committed catalog and its Git history, including the original +SSH data URL. The uncommitted configuration change in step 6 remains local. + +```bash +cd "$HM_DEMO_WORKSPACE" +hallmark clone ./client/.hm ./catalog-copy --no-download +``` + +`clone` copies existing catalogs; raw dataset URLs belong to `hallmark add`. +A Git catalog can live on GitHub while its data remains on the SSH server. A +published HTTP/SFTP catalog snapshot can likewise use a different data server; +snapshot imports start local history. Cloning metadata does not require the +dataset credentials. Git catalog authentication and +data authentication are independent. + +Clone preserves the complete catalog. `download --include` selects transfers +without removing catalog rows. `init` creates an empty repository; the legacy +remote `build` command is deprecated in favor of `add`. Downloading files does +not populate the local object store; the scientific workflows show how to +version local settings beside cataloged inputs before branching. + +For this revision, steps 4, 5, 6 and 10 were run on Python 3.13 against a +disposable loopback SFTP-only export with two 30-byte run files and a published +`SHA256SUMS`. Cataloging, one approved 30-byte transfer, a declined transfer, +the mirror dry run, and catalog cloning passed. These checks do not establish +availability or credentials for any external server. Keep the workspace for +inspection and remove it manually when finished. diff --git a/demo/scientific_workflows_cli.md b/demo/scientific_workflows_cli.md new file mode 100644 index 0000000..bffc208 --- /dev/null +++ b/demo/scientific_workflows_cli.md @@ -0,0 +1,270 @@ +# Four scientific workflows from the terminal + +Use selected EHT/CyVerse and DESI products, Roman–Rubin simulations on an +existing SFTP export, and a local DES Y3 product. The matching +[Python notebook](scientific_workflows_python.ipynb) includes a worktree example. +This guide uses branches; the current CLI has no worktree command. + +Run the setup, then one numbered workflow in order in the same Bash terminal. +Each workflow uses a new directory. The examples version a small +`analysis-settings.txt` file alongside one scientific input. Editing that file +records an analysis choice; it does not process or modify the scientific data. + +## Setup and disk budget + +Install the local Hallmark checkout in your active environment and create a new +workspace outside the source checkout. `set -e` stops the shell if a command +fails or a download is declined, so later versioning commands cannot continue +with a missing input. To repeat a workflow, choose a new workspace. + +```bash +set -e +python -m pip install -e . +export HM_ROOT="$HOME/hallmark-science-cli" +mkdir "$HM_ROOT" +``` + +Use your normal Git author configuration. No optional scientific-processing +packages are needed for these examples. Each remote workflow selects one file; +review its size before approving the transfer. Allow space for the downloaded +input. A remote input is cataloged by its source and checksum rather than +copied into the object store, and settings changes store only the changed +settings file. + +`hallmark add URL` catalogs remote files using directory listings and +published metadata. It does not download dataset files. A URL template such as +`.../uvfits/SR1_M87_2017_{day}_lo_hops_netcal_StokesI.uvfits` catalogs every +matching file and records `day` as a column; the examples below name one file +exactly, without fields, so each catalogs a single input. `hallmark ls-remote +URL` lists a directory and suggests templates, and `hallmark add -n` previews +the matches. `clone` copies a complete existing catalog. Select transfers +separately with `download --include`. Every nonempty CLI transfer displays its +plan and prompts for approval. `--dry-run` prepares an offline plan without +transferring files. Unknown sizes remain unknown. + +For interactive selection, use `hallmark download --interactive`. It accepts +one raw glob per line or an all-files choice, shows recorded sizes, and lets you +download, change the selection, or skip. `init` and `add` never offer +downloads. CLI cloning copies the complete catalog and then reviews a download +plan by default. Use `clone --interactive` for the chooser, `clone --include` +to narrow transfers, or `clone --no-download` for metadata alone. Without a +terminal, clone keeps the catalog and prints commands for downloading later. +The examples below keep explicit selections so their intended inputs are clear. + +Each example uses a working directory containing `.hm`. The remote input and +the local `analysis-settings.txt` are two templates in the same branch. `commit` +stores the settings file's contents so branch checkout can restore it; checkout +leaves the downloaded input in place. The catalog URL and data URL can be +hosted separately. See the [private data guide](../doc/private_data.rst) for +authentication and catalog hosting, and [data backends](../doc/backends.rst) +for backend extensions. + +## 1. CyVerse: EHT M87 visibility provenance + +The [EHT release](https://github.com/eventhorizontelescope/2019-D01-01) contains +2017 M87 observations. Select the April 5 low-band UVFITS product from the +release's UVFITS directory. An analysis might inspect positive-weight visibility +amplitudes before imaging; here we only record that intended quality check. + +### Catalog and approve a selected download + +Create a repository, catalog the selected product, and review the offline +download plan. Run the transfer only after checking the selection and +answering `y` at its prompt. Declining leaves the catalog available. +`add` stops the workflow if the listing does not contain the product, and the +final check stops it if the file was not downloaded. + +```bash +hallmark init "$HM_ROOT/eht" +cd "$HM_ROOT/eht" +hallmark add \ + 'https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' +hallmark commit -m 'Catalog EHT input' +hallmark download --all --dry-run +hallmark download --all +test -f 'SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits' +``` + +### Commit, annotate a branch, and recover + +Store an analysis setting beside the cataloged input, then record a different +quality-review choice on `eht-provenance`. Returning to `main` restores the +original settings; the final command prints `qa=positive-weights`. + +```bash +printf '%s\n' 'qa=positive-weights' > analysis-settings.txt +hallmark add 'analysis-settings.txt' +hallmark commit -m 'Record EHT analysis settings' +hallmark checkout eht-provenance +printf '%s\n' 'qa=positive-weights-and-amplitude-review' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record EHT amplitude review setting' +hallmark checkout main +cat analysis-settings.txt +``` + +This changes neither calibration nor visibilities. A scientific amplitude check +would separately interpret the UVFITS weights and polarization layout. + +## 2. HTTPS: DESI spectrum inspection and quality selections + +Use the redrock catalog from +[DR1 Iron HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). +This route downloads redshift results, not the coadded spectra. A scientific +selection could combine warning flags, spectral type, and fit-quality +statistics; applying those cuts remains a separate analysis step. + +### Catalog and download by product + +Catalog only `redrock-main-dark-23040.fits` in this HEALPixel, inspect the +plan, and approve its download. The selection excludes coadds and other +products; the final check stops the workflow if the file was not downloaded. + +```bash +hallmark init "$HM_ROOT/desi" +cd "$HM_ROOT/desi" +hallmark add \ + 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/redrock-main-dark-23040.fits' +hallmark commit -m 'Catalog DESI input' +hallmark download --all --dry-run +hallmark download --all +test -f 'redrock-main-dark-23040.fits' +``` + +### Compare two selection branches + +Commit an illustrative fit-quality threshold. Record a stricter +threshold on `desi-quality`, then return to `main`; the restored setting is +`min_deltachi2=25`. No target rows are filtered by these commands. + +```bash +printf '%s\n' 'min_deltachi2=25' > analysis-settings.txt +hallmark add 'analysis-settings.txt' +hallmark commit -m 'Record DESI analysis settings' +hallmark checkout desi-quality +printf '%s\n' 'min_deltachi2=40' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record stricter DESI quality setting' +hallmark checkout main +cat analysis-settings.txt +``` + +## 3. SSH/SFTP: Roman–Rubin products over SFTP + +Assume an existing export contains the OpenUniverse2024 stellar catalog +`stars/pointsource_10307.parquet`. Select that one product; this example does not +provision a server. A science workflow could compare magnitude selections +before matching simulated sources between surveys. Here the catalog is kept +unchanged and only the intended magnitude limit is versioned. + +### Client authentication, catalog, and download + +Add this illustrative alias to `~/.ssh/config`, replacing the host, user, and +identity with your existing export's settings. The verified host key must +already be in `known_hosts`; load an encrypted identity into your SSH agent. +The client needs OpenSSH 9.6+ and the server needs SFTP. + +```text +Host lab-data + HostName vm.example.org + User researcher + IdentityFile ~/.ssh/hallmark_vm + IdentitiesOnly yes + StrictHostKeyChecking yes +``` + +The URL's `stars/` directory becomes the template's source, so the selected +product has a simple local filename. Catalog it, review the plan, and approve +the transfer. SSH configuration supplies authentication; no Hallmark profile is +required. The final check stops the workflow if the file was not downloaded. + +```bash +hallmark init "$HM_ROOT/roman-rubin" +cd "$HM_ROOT/roman-rubin" +hallmark add \ + 'sftp://lab-data/home/researcher/hallmark-exports/openuniverse2024/stars/pointsource_10307.parquet' +hallmark commit -m 'Catalog Roman-Rubin input' +hallmark download --all --dry-run +hallmark download --all +test -f 'pointsource_10307.parquet' +``` + +### Save and compare an analysis choice + +Store an illustrative magnitude limit beside the cataloged input. Change that limit +on `roman-rubin-selection`, then return to `main`. The restored value is +`magnitude_limit=24`; the Parquet catalog has not been filtered or converted. + +```bash +printf '%s\n' 'magnitude_limit=24' > analysis-settings.txt +hallmark add 'analysis-settings.txt' +hallmark commit -m 'Record Roman-Rubin analysis settings' +hallmark checkout roman-rubin-selection +printf '%s\n' 'magnitude_limit=23' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record Roman-Rubin magnitude setting' +hallmark checkout main +cat analysis-settings.txt +``` + +## +1. Local init: DES Y3 covariance-aware versions + +Use an existing local copy of +`2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits`, from the +[DES Y3 release](https://www.darkenergysurvey.org/des-year-3-cosmology-results-papers/). +This example copies that input into a new workspace and makes no network +request. A scientific angular cut must apply the same row selection to the +correlation-function tables and covariance matrix. Changing a settings file +alone does not perform that operation. + +### Prepare the local analysis product + +Replace the illustrative local path with your existing DES file. Initialize a +working directory and copy the input into it, leaving the source file intact. +There is no remote download to approve. + +```bash +export HM_DES_LOCAL='/path/to/2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits' +hallmark init "$HM_ROOT/des" +cd "$HM_ROOT/des" +cp "$HM_DES_LOCAL" . +``` + +### Compare angular selections with aligned covariance + +Commit the input and an illustrative minimum angle. Record a different angle +on `des-angular-cut`, then recover the original `theta_min_arcmin=2.5` setting +from `main`. Both branches retain the original FITS data and covariance. + +```bash +printf '%s\n' 'theta_min_arcmin=2.5' > analysis-settings.txt +hallmark add '{name}' +hallmark commit -m 'Preserve DES input and analysis settings' +hallmark checkout des-angular-cut +printf '%s\n' 'theta_min_arcmin=5' > analysis-settings.txt +hallmark add . +hallmark commit -m 'Record DES angular-cut setting' +hallmark checkout main +cat analysis-settings.txt +``` + +## Disk inspection, cleanup, and validation + +Use `du -sh "$HM_ROOT"` to inspect the workspace. Keep it while comparing +branches; remove it manually when finished. No example changes the original +remote data or the local DES source file. + +For this revision, the command blocks above were run in order on Python 3.13 +against tiny local HTTP fixtures that mirror the EHT, DESI and export paths, +including published MD5 and SHA-256 manifests. The SFTP workflow's commands +were run over HTTP; SFTP cataloging and transfer are covered by the OpenSSH +integration tests with a loopback server. Each remote workflow cataloged and +downloaded one input, stored only its settings files as objects, and restored +the baseline setting on `main`. Declined downloads stop the shell before +versioning. + +The complete test suite, including the OpenSSH integration tests, Ruff, and a +strict Sphinx build passed; CI runs the suite on Python 3.9 through 3.14. These +fixtures do not validate scientific file formats, analysis choices, +public-server availability, or public download sizes. No research servers were +contacted and no public datasets were downloaded. diff --git a/demo/scientific_workflows_python.ipynb b/demo/scientific_workflows_python.ipynb new file mode 100644 index 0000000..217e4e3 --- /dev/null +++ b/demo/scientific_workflows_python.ipynb @@ -0,0 +1,377 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "science-00", + "metadata": {}, + "source": [ + "# Four scientific workflows with Hallmark\n", + "\n", + "EHT and DESI observations, Roman–Rubin simulations, and local DES Y3 products. The matching [CLI guide](scientific_workflows_cli.md) uses the same selected files; this notebook also shows one worktree.\n", + "\n", + "Run **Setup**, then one numbered workflow from start to finish. Each uses a separate directory and one explicitly selected input. Choose a new `ROOT` to repeat an example. `repo.add(url)` catalogs remote files without downloading them. `Repo.clone(...)` copies an existing catalog; it preserves the complete catalog and history. Use `plan_download(include=...)` and approve transfers separately.\n", + "\n", + "A URL template such as `.../uvfits/SR1_M87_2017_{day}_lo_hops_netcal_StokesI.uvfits` catalogs every matching file and records its fields as columns. These examples name one file exactly, without fields, so each catalogs a single input with its size and any published checksum.\n", + "\n", + "Every transfer shows a plan and asks for approval. Declining or a failed transfer stops that workflow; do not continue to its versioning cell. The completed catalog remains available. The SSH example requires an existing export and your own access configuration.\n", + "\n", + "The examples version a small `analysis-settings.txt` file as a local template beside each unchanged input. These settings record intended analysis choices; the cells do not process the scientific data. Detailed processing is described in prose." + ] + }, + { + "cell_type": "markdown", + "id": "science-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install this checkout with `python -m pip install -e .` and install JupyterLab in the same environment with `python -m pip install jupyterlab`. Select that environment's kernel. Git needs your normal author configuration. No additional scientific-processing packages are needed for these examples.\n", + "\n", + "Import Hallmark and the standard-library tools, then choose a new workspace under your home directory. The same setup works when Jupyter starts in the repository root or in `demo/`. Set `ROOT` before beginning a workflow. The source settings below select narrow dataset roots; replace `VM_URL` and `DES_LOCAL` with your existing export and local input when using those workflows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-02", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import shutil\n", + "from hallmark import Repo\n", + "\n", + "ROOT = (Path.home() / \"hallmark-science-python\").resolve()\n", + "ROOT.mkdir(exist_ok=True)\n", + "print(ROOT)\n", + "\n", + "EHT_URL = \"https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/\"\n", + "DESI_URL = \"https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/\"\n", + "VM_URL = \"sftp://lab-data/home/researcher/hallmark-exports/openuniverse2024/stars/\"\n", + "DES_FILE = \"2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits\"\n", + "DES_LOCAL = Path.home() / \"datasets\" / DES_FILE\n" + ] + }, + { + "cell_type": "markdown", + "id": "science-03", + "metadata": {}, + "source": [ + "This helper prints the plan and accepts only an explicit `y` before transferring files. An empty selection, a declined plan, or reported download failures raise an exception so the workflow stops. File sizes come from the catalog; a duration remains unknown without a supplied transfer rate." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-04", + "metadata": {}, + "outputs": [], + "source": [ + "def download_after_review(repo, plan):\n", + " print(plan.summary())\n", + " if not plan.items:\n", + " raise RuntimeError(\"No files selected; check the source and filter\")\n", + " if input(\"Download this selection? [y/N] \").strip().lower() != \"y\":\n", + " raise RuntimeError(\"Download declined; catalog remains available\")\n", + " result = repo.download(plan, approved=True, progress=True)\n", + " if result[\"failed\"]:\n", + " raise RuntimeError(result[\"errors\"])\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "id": "science-05", + "metadata": {}, + "source": [ + "## 1. CyVerse: EHT M87 visibility provenance\n", + "\n", + "Use one April 5 low-band UVFITS file from the [EHT M87 release](https://github.com/eventhorizontelescope/2019-D01-01). A scientific analysis could inspect positive-weight visibility amplitudes and compare the two frequency bands before imaging. This example records a QA choice without recalibrating visibilities or reconstructing an image.\n", + "\n", + "Catalog that file by its URL in the release's `uvfits/` directory; the CyVerse backend is selected from the URL. Review the plan and approve its download into the new worktree." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-06", + "metadata": {}, + "outputs": [], + "source": [ + "EHT_FILE = \"SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits\"\n", + "eht_dir = ROOT / \"eht\"\n", + "eht = Repo.init(eht_dir)\n", + "eht.add(EHT_URL + EHT_FILE)\n", + "eht.commit(\"Catalog EHT input\")\n", + "download_after_review(eht, eht.plan_download(all_files=True))" + ] + }, + { + "cell_type": "markdown", + "id": "science-07", + "metadata": {}, + "source": [ + "Track the QA settings as a local template beside the cataloged input and commit them as a baseline, then record another choice on `eht-provenance`. Switching back restores the original settings; checkout leaves the downloaded input in place. The final checks confirm both the restoration and the unchanged input checksum." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-08", + "metadata": {}, + "outputs": [], + "source": [ + "eht_settings = eht_dir / \"analysis-settings.txt\"\n", + "eht_settings.write_text(\"qa=positive-weights\\n\", encoding=\"utf-8\")\n", + "eht.add(\"analysis-settings.txt\")\n", + "eht.commit(\"Record EHT QA settings\")\n", + "eht_base = eht.branches()[\"current\"]\n", + "eht_hash = Repo.checksum(eht_dir / EHT_FILE)\n", + "eht.checkout(\"eht-provenance\")\n", + "eht_settings.write_text(\"qa=positive-weights-and-amplitude-review\\n\", encoding=\"utf-8\")\n", + "eht.add(\".\")\n", + "eht.commit(\"Record amplitude-review choice\")\n", + "eht.checkout(eht_base)\n", + "assert eht_settings.read_text(encoding=\"utf-8\") == \"qa=positive-weights\\n\"\n", + "assert Repo.checksum(eht_dir / EHT_FILE) == eht_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-09", + "metadata": {}, + "source": [ + "A worktree lets you keep an alternative branch open alongside the baseline. Create its sibling directory, open it with `Repo(path)`, and commit another settings value there. The original worktree retains its baseline settings. The new worktree restores the tracked settings file; the remote input is cataloged there but not downloaded." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-10", + "metadata": {}, + "outputs": [], + "source": [ + "eht.add_worktree(\"eht-alternate-qa\")\n", + "eht_other = Repo(ROOT / \"eht-alternate-qa\")\n", + "other_settings = eht_other.worktree / \"analysis-settings.txt\"\n", + "other_settings.write_text(\"qa=band-consistency-review\\n\", encoding=\"utf-8\")\n", + "eht_other.add(\".\")\n", + "eht_other.commit(\"Record alternative QA settings in a worktree\")\n", + "assert eht_settings.read_text(encoding=\"utf-8\") == \"qa=positive-weights\\n\"" + ] + }, + { + "cell_type": "markdown", + "id": "science-11", + "metadata": {}, + "source": [ + "## 2. DESI: spectrum inspection and redshift-quality choices\n", + "\n", + "Select only `redrock-main-dark-23040.fits` from [DR1 Iron HEALPixel 23040](https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/23040/). A scientific analysis could compare redshift-quality thresholds after checking the redshift and fibermap records by TARGETID. Inspecting spectra would also require the corresponding coadd, which this example does not download.\n", + "\n", + "The settings below illustrate a threshold comparison, not official DESI LSS selection cuts. See the [redrock data model](https://desidatamodel.readthedocs.io/en/latest/DESI_SPECTRO_REDUX/SPECPROD/healpix/SURVEY/PROGRAM/PIXGROUP/PIXNUM/redrock-SURVEY-PROGRAM-PIXNUM.html) and [DR1 known issues](https://data.desi.lbl.gov/doc/releases/dr1/known-issues/) for scientific use.\n", + "\n", + "Catalog the redrock file by its HTTPS URL, review the plan, and approve its download." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-12", + "metadata": {}, + "outputs": [], + "source": [ + "DESI_FILE = \"redrock-main-dark-23040.fits\"\n", + "desi_dir = ROOT / \"desi\"\n", + "desi = Repo.init(desi_dir)\n", + "desi.add(DESI_URL + DESI_FILE)\n", + "desi.commit(\"Catalog DESI input\")\n", + "download_after_review(desi, desi.plan_download(all_files=True))" + ] + }, + { + "cell_type": "markdown", + "id": "science-13", + "metadata": {}, + "source": [ + "Commit an illustrative DELTACHI2 threshold beside the cataloged input. Change only that threshold on `desi-quality`, commit it, then restore the baseline and check the unchanged input. No galaxy selection or FITS rewrite occurs here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-14", + "metadata": {}, + "outputs": [], + "source": [ + "desi_settings = desi_dir / \"analysis-settings.txt\"\n", + "desi_settings.write_text(\"min_deltachi2=25\\n\", encoding=\"utf-8\")\n", + "desi.add(\"analysis-settings.txt\")\n", + "desi.commit(\"Record DESI quality settings\")\n", + "desi_base = desi.branches()[\"current\"]\n", + "desi_hash = Repo.checksum(desi_dir / DESI_FILE)\n", + "desi.checkout(\"desi-quality\")\n", + "desi_settings.write_text(\"min_deltachi2=40\\n\", encoding=\"utf-8\")\n", + "desi.add(\".\")\n", + "desi.commit(\"Record stricter redshift-quality setting\")\n", + "desi.checkout(desi_base)\n", + "assert desi_settings.read_text(encoding=\"utf-8\") == \"min_deltachi2=25\\n\"\n", + "assert Repo.checksum(desi_dir / DESI_FILE) == desi_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-15", + "metadata": {}, + "source": [ + "## 3. SSH/SFTP: Roman–Rubin products\n", + "\n", + "Use the single `pointsource_10307.parquet` input from an existing export of the [OpenUniverse2024 shared-sky catalogs](https://irsa.ipac.caltech.edu/data/theory/openuniverse2024/roman/preview/roman_rubin_cats_v1.1.2_faint/). A scientific analysis could join source IDs with the separate flux table and compare Rubin color or magnitude selections. That additional table and the scientific processing are outside this example.\n", + "\n", + "The [CLI guide](scientific_workflows_cli.md#3-sshsftp-romanrubin-products-over-sftp) shows an illustrative `lab-data` SSH configuration. Replace its host, user, and key with your own settings and verify the server's host key. Set `VM_URL` to an export you already have permission to read; the notebook does not provision a VM or prepare its data.\n", + "\n", + "Catalog the selected file by its URL in the export's `stars/` directory, then review and approve its SFTP download. Other workflows do not use this SSH connection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-16", + "metadata": {}, + "outputs": [], + "source": [ + "VM_FILE = \"pointsource_10307.parquet\"\n", + "vm_dir = ROOT / \"roman-rubin\"\n", + "vm = Repo.init(vm_dir)\n", + "vm.add(VM_URL + VM_FILE)\n", + "vm.commit(\"Catalog Roman-Rubin input\")\n", + "download_after_review(vm, vm.plan_download(all_files=True))" + ] + }, + { + "cell_type": "markdown", + "id": "science-17", + "metadata": {}, + "source": [ + "Keep the simulation input unchanged and record an intended magnitude limit in the settings file. Commit a different limit on `roman-rubin-selection`, then return to the baseline and verify both the settings and input checksum. No simulated-source selection is applied." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-18", + "metadata": {}, + "outputs": [], + "source": [ + "vm_settings = vm_dir / \"analysis-settings.txt\"\n", + "vm_settings.write_text(\"magnitude_limit=24\\n\", encoding=\"utf-8\")\n", + "vm.add(\"analysis-settings.txt\")\n", + "vm.commit(\"Record Roman-Rubin selection settings\")\n", + "vm_base = vm.branches()[\"current\"]\n", + "vm_hash = Repo.checksum(vm_dir / VM_FILE)\n", + "vm.checkout(\"roman-rubin-selection\")\n", + "vm_settings.write_text(\"magnitude_limit=23\\n\", encoding=\"utf-8\")\n", + "vm.add(\".\")\n", + "vm.commit(\"Record brighter magnitude-limit setting\")\n", + "vm.checkout(vm_base)\n", + "assert vm_settings.read_text(encoding=\"utf-8\") == \"magnitude_limit=24\\n\"\n", + "assert Repo.checksum(vm_dir / VM_FILE) == vm_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-19", + "metadata": {}, + "source": [ + "## +1. Local init: DES Y3 covariance-aware analysis versions\n", + "\n", + "Start with your existing local `2pt_NG_final_2ptunblind_02_24_21_wnz_redmagic_covupdate.fits` file. This workflow makes no server connection. See the [DES Y3 papers](https://www.darkenergysurvey.org/des-year-3-cosmology-results-papers/) for the published analysis.\n", + "\n", + "A scientific angular selection must apply the same retained rows to the data vector and both covariance axes, retaining the corresponding redshift-distribution information. The thresholds below only illustrate versioned settings; they are not the DES team's bin-dependent scale cuts and do not create a reduced likelihood.\n", + "\n", + "Set `DES_LOCAL` to your input. Copy it into a new experiment directory and initialize a local repository there, leaving the original file untouched." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-20", + "metadata": {}, + "outputs": [], + "source": [ + "assert DES_LOCAL.is_file(), \"Set DES_LOCAL to your existing DES input\"\n", + "des_dir = ROOT / \"des\"\n", + "des_dir.mkdir()\n", + "des_file = Path(shutil.copy2(DES_LOCAL, des_dir / DES_LOCAL.name))\n", + "des = Repo.init(des_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "science-21", + "metadata": {}, + "source": [ + "Commit the copied input and angular settings. Record a second threshold on `des-angular-cut`, then restore the baseline and verify that both input copies still have the original checksum. This changes the text settings only; no covariance or data-vector processing occurs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "science-22", + "metadata": {}, + "outputs": [], + "source": [ + "des_settings = des_dir / \"analysis-settings.txt\"\n", + "des_settings.write_text(\"theta_min_arcmin=2.5\\n\", encoding=\"utf-8\")\n", + "des.add(\"{name}\")\n", + "des.commit(\"Preserve DES input and angular settings\")\n", + "des_base = des.branches()[\"current\"]\n", + "des_hash = Repo.checksum(DES_LOCAL)\n", + "des.checkout(\"des-angular-cut\")\n", + "des_settings.write_text(\"theta_min_arcmin=5\\n\", encoding=\"utf-8\")\n", + "des.add(\".\")\n", + "des.commit(\"Record alternative angular threshold\")\n", + "des.checkout(des_base)\n", + "assert des_settings.read_text(encoding=\"utf-8\") == \"theta_min_arcmin=2.5\\n\"\n", + "assert Repo.checksum(des_file) == Repo.checksum(DES_LOCAL) == des_hash" + ] + }, + { + "cell_type": "markdown", + "id": "science-23", + "metadata": {}, + "source": [ + "## Inspect disk use and keep or remove your experiment\n", + "\n", + "The plan reports available transfer sizes before approval. Allow room for each downloaded input; remote inputs are cataloged by URL and checksum rather than stored again in `.hm/objects`. The DES workflow copies your existing input once and stores a version in `.hm/objects`. Changing these small settings files does not make new versions of the unchanged scientific inputs.\n", + "\n", + "Keep `ROOT` to retain the experiments and their history. To discard them, remove only the new experiment directories you created, including `eht-alternate-qa` if you ran that cell. Do not delete a shared dataset directory or remove only a worktree's `.hm` link." + ] + }, + { + "cell_type": "markdown", + "id": "science-24", + "metadata": {}, + "source": [ + "## Validation record\n", + "\n", + "For this revision, all 11 code cells were run in order on Python 3.13 against tiny local HTTP fixtures that mirror the EHT, DESI and export paths, with the SFTP export's URL served over HTTP and each approval prompt answered `y`. The checks covered cataloging, published checksums, approval, settings commits, worktree isolation, input checksums, and branch recovery. SFTP cataloging and transfer are covered by the OpenSSH integration tests with a loopback server.\n", + "\n", + "The source notebook passes nbformat validation, keeps its outputs cleared, and requires explicit download approval. The matching [CLI guide](scientific_workflows_cli.md#disk-inspection-cleanup-and-validation) records the CLI and documentation checks. These fixtures do not validate scientific processing, public-server availability, or public download sizes; no research servers were contacted and no public datasets were downloaded." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/api.rst b/doc/api.rst index 6c7d878..9765913 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -23,18 +23,6 @@ Core Repository :members: :show-inheritance: -.. automodule:: hallmark.state - :members: - :show-inheritance: - -.. automodule:: hallmark.downloader - :members: - :show-inheritance: - -.. automodule:: hallmark.paraframe - :members: - :show-inheritance: - Repository Worktrees -------------------- @@ -64,10 +52,6 @@ Data Handling :members: :show-inheritance: -.. automodule:: hallmark.eht_datatree - :members: - :show-inheritance: - Downloading ----------- @@ -104,46 +88,126 @@ and interacting with Hallmark repositories. :members: :show-inheritance: -Building a repository ---------------------- +Preparing a repository +---------------------- -Build a repository and choose filename formats interactively:: +Create a local repository:: - hallmark build ./repositories EHTC_2018L1_Dec2024 + hallmark init ./local-project -Load formats and remotes from an existing configuration:: +Catalog remote files with a URL template, without downloading dataset files:: - hallmark build ./repositories EHTC_2018L1_Dec2024 \ - --config-file ./existing/config.yml + hallmark add 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/{pixel}/redrock-main-dark-{pixel}.fits' + hallmark add 'ssh://lab-data/srv/data/run{run:d}.h5' + hallmark commit -m 'Catalog remote data' -Provide filename formats directly:: +The URL up to the first path segment containing a ``{field}`` is recorded as +the template's source; the rest is matched against the listed files, one path +segment at a time, and its fields become catalog columns. Each template is +tracked beside the others; adding one again syncs it with the server, and +``hallmark rm --cached TEMPLATE`` stops tracking it. ``hallmark add -n`` previews +the matches, and ``hallmark ls-remote URL`` lists a directory and suggests +templates. Python uses ``repo.add(url_template)``. - hallmark build ./repositories EHTC_EXAMPLE \ - --fmt "images/{source}_{date}.fits=data" \ - --fmt "README.{format}=readme" +Use ``hallmark clone SOURCE PATH`` for an existing Hallmark Git repository +or published HTTP/SFTP snapshot. Git clones retain the full catalog and its +history; snapshots start new local history. Use ``--source-type git`` or +``--source-type catalog`` to override automatic detection. The older ``build`` +command remains available but is deprecated in favor of ``add`` with a URL +template. -Provide a custom remote or remotes:: +Downloading remote data +----------------------- - hallmark build ./repositories EHTC_EXAMPLE \ - --remote "origin=https://data.example.org/EHTC_EXAMPLE/" \ - --remote "backup=https://backup.example.org/EHTC_EXAMPLE/" +Preview the selected files using the local catalog, then confirm a download:: -The generated repository is named ``DATASET_NAME.hm``. The ``--fmt`` option -may be repeated, but it cannot be combined with ``--config-file``. + hallmark download --all --dry-run + hallmark download --include 'runs/**' -Downloading remote data ------------------------ +Explicit cataloged paths and ``--tsv data.tsv`` also select files. Every nonempty +transfer requires interactive confirmation. ``init`` and ``add`` only create the +repository or catalog files; they never offer downloads. Each remote template's +files download from its source, so one plan can span several servers; +``--remote NAME`` selects a configured data remote for every file instead. CLI +cloning copies the complete catalog, then reviews a download plan by default. +Use ``clone --include`` to narrow that plan, ``clone --interactive`` for a +chooser, or ``clone --no-download`` to skip review. ``clone --output`` specifies +the download directory; bare catalogs prompt for one if omitted. These download +options cannot be combined with ``--no-download``; ``--interactive`` also +conflicts with ``--include``. ``download --include`` narrows the saved catalog +without changing it. +The old download ``--yes`` option does not bypass approval. -Download specific remote-relative paths:: +Use ``hallmark download --interactive`` to choose patterns or all cataloged files, +review recorded sizes, then download, revise, or skip. Add ``--dry-run`` to stop at +the preview. Without a terminal, clone keeps the catalog and prints commands for +later; standalone ``download --interactive`` reports an error. Python ``add`` +and cloning remain metadata-only and never open this CLI chooser. - hallmark download README.md data/example.fits +Python uses the same plan. First inspect the selected files:: -Download everything represented by a configured TSV:: + from hallmark import Repo - hallmark download --tsv data.tsv + repo = Repo.init('lab') + repo.add('ssh://lab-data/srv/data/{group}/run{run:d}.h5', progress=True) + plan = repo.plan_download(include='runs/**') + print(plan.summary()) -Preview a complete repository download:: +After reviewing the plan, approve the download and check for failures:: - hallmark download --all --dry-run + result = repo.download(plan, approved=True, progress=True) + if result['failed']: + raise RuntimeError('\n'.join(result['errors'])) + +Plans preserve their selected sources, backend settings, destination and +checksums even when repository configuration later changes. Size estimates +require recorded file sizes; duration estimates also require a supplied +transfer rate. + +.. automodule:: hallmark.download_plan + :members: + +Data backends +------------- + +See :doc:`backends` for registration, installed plugins and the transfer +contract. Backend classes are also exported from ``hallmark``. Existing +``hallmark.transport`` imports remain compatibility aliases. + +.. automodule:: hallmark.backends + :members: + :show-inheritance: + +Dataset builders and data remotes +--------------------------------- + +.. autofunction:: hallmark.repo_builder.build_repo + +.. autofunction:: hallmark.repo_builder.list_remote_files + +SSH/SFTP uses standard ``~/.ssh/config`` and the SSH agent. Use URLs containing +the configured host alias; no Hallmark authentication profile is used. + +The legacy ``download_remote_data`` function also requires ``approved=True``. +Both download APIs return a dictionary containing ``succeeded``, +``failed``, ``total_bytes``, and ``errors``. Check ``failed`` and ``errors`` +for individual transfer failures. Configuration errors and failed SSH +connection checks raise ``DownloadError`` before downloads begin. +HTTP, SSH, and SFTP downloads use the same rules for destination paths +and checksum verification. + +See :doc:`private_data` for CLI and Python examples, and +:ref:`private-transport-reference` for authentication settings and +supported server configurations. + +Named data sources +------------------ + +List sources with ``hallmark sources`` and release/collection URLs with +``hallmark sources desi``, then catalog files beneath them with ``ls-remote`` and +``add``. See :doc:`private_data` for templates and migration. + +.. autofunction:: hallmark.repo_remote.suggest_templates -Large selections require confirmation unless ``--yes`` is supplied. \ No newline at end of file +.. automodule:: hallmark.sources + :members: DataSource, SourceRelease, register_source, get_source, list_sources diff --git a/doc/backends.rst b/doc/backends.rst new file mode 100644 index 0000000..d31a409 --- /dev/null +++ b/doc/backends.rst @@ -0,0 +1,231 @@ +Data backends +============= + +A data backend discovers files and transfers them from a data source. It does +not determine where the Hallmark catalog is hosted. The same catalog can be +shared through GitHub, an HTTP server or SFTP while its backend accesses a +separate data service or several servers. + +Built-in backends and configuration +----------------------------------- + +``hallmark.backends`` exposes the common ``DataBackend`` base and these +implementations: + +* ``HttpBackend`` transfers HTTP(S) files and discovers supported HTML indexes. +* ``SshBackend`` uses SFTP for both SSH and SFTP data URLs, including accounts + without a login shell. +* ``CyVerseBackend`` extends HTTP support with CyVerse directory and metadata + parsing. + +Users catalog files with ``hallmark add 'URL/TEMPLATE'``, and Hallmark selects +the transport from the URL automatically. Source authors can specify an adapter +and its nonsecret configuration in a release definition; templates added +beneath the release URL use it, so users need no backend flags or options file. + +Define a source and register it for the current Python process: + +.. code-block:: python + + from hallmark import DataSource, SourceRelease, Repo, register_source + + survey = DataSource("survey", "Example survey", { + "dr1": SourceRelease( + "https://survey.example.org/release/", + {"redshifts": ("catalogs/redshifts",)}, + ), + }) + register_source(survey) + repo = Repo.init("survey") + repo.add("https://survey.example.org/release/catalogs/redshifts/{name}.fits") + +Collection roots are relative directories below the release URL. To expose +this descriptor to other Python processes and the CLI, publish the ``survey`` +instance in the plugin's ``pyproject.toml``: + +.. code-block:: toml + + [project.entry-points."hallmark.sources"] + survey = "my_survey.sources:survey" + +Users can then run ``hallmark sources survey`` to see the URLs of available +releases and collections, and ``hallmark ls-remote`` and ``hallmark add`` with +those URLs. Source registration and listing must not contact the service. +Source names are unique across registered and installed descriptors. + +A template's data entry records its URL and, when the URL alone would select a +different transport, its ``backend`` and ``backend_options``. Python +``repo.add(url_template, backend=..., backend_options=...)`` selects them +explicitly. Data remotes configured with ``set-config`` keep their own +transport settings; the developer API ``repo.set_config(remote_backend=..., +remote_backend_options=...)`` updates them and has no CLI counterpart. +Existing catalog transport metadata remains supported. Options contain +shareable configuration, never credentials. SSH uses ``~/.ssh/config`` and its +agent; HTTP retains Requests authentication. + +``repo.plan_download()`` reads the local catalog without opening a connection. +The plan captures each source's backend and immutable options, URL, the +destination and file metadata. Later configuration changes cannot redirect an +approved plan. Install the selected transport plugins before executing a +transfer. + +Writing a backend +----------------- + +Implement ``DataBackend`` or extend an existing backend. All classes receive +an ``OperationContext`` through ``__init__(context)``. The context exposes the +validated ``RemoteSpec`` as ``context.remote`` and its frozen options as +``context.remote.backend_options``. ``context.session()`` supplies a Requests +session per worker thread; ``context.check_cancelled()`` raises when the +operation has been cancelled. + +.. list-table:: Backend contract + :header-rows: 1 + :widths: 35 65 + + * - Method + - Responsibility + * - ``prepare()`` + - Check access and prepare operation resources before discovery or transfer. + Repeated calls must be safe. + * - ``iter_entries(on_directory=None)`` + - Yield ``RemoteEntry`` records with literal logical paths relative to the + dataset root. Report completed directories through the optional callback. + Missing sizes, timestamps and published checksums remain ``None``. + When ``context.descend`` is set, skip each subdirectory for which it + returns False; it receives the relative path ending in ``/``. ``add`` + uses it so templates list only directories they can match. Ignoring it + is correct but slower. + * - ``read_text(relative_path, limit)`` + - Read metadata within the supplied byte limit and return decoded text. + Reject oversized responses and propagate access or connection errors. + * - ``fetch(relative_path, destination, *, chunk_size=8192)`` + - Write a payload to the caller-managed temporary path. Check cancellation + during transfer and call ``context.on_bytes(count)`` when set. + * - ``cancel()`` + - Stop resources started by this backend, including active child operations. + * - ``close()`` + - Release connections and processes when the operation ends. + +``list_entries()`` is a convenience wrapper over ``iter_entries()``. Raise +``CapabilityError`` when a backend cannot list or read metadata, and +``RemoteObjectMissing`` for an absent metadata object. These exceptions are +available from ``hallmark.backends``. Do not disguise authentication failures +as missing objects. + +Discovery reads listings and published metadata only. It must not download +payloads to infer checksums. The shared downloader owns approval, destination +path validation, checksum verification, atomic replacement and worker +concurrency. Backends map logical paths to server paths or API requests and +write only to the temporary destination they are given. + +An instance can serve concurrent ``fetch`` calls. Protect shared mutable +state, use thread-local sessions, and check cancellation in listing, metadata +and transfer loops. Document any server concurrency limit. ``cancel`` may run +while workers are active; make it and ``close`` safe to call during cleanup. +Keep resource acquisition in ``prepare`` or managed contexts so partial setup +can be cleaned up after failures. + +For example, an HTTP service can publish a small ``manifest.json`` array with +``path``, optional ``size`` and optional ``sha256`` fields. Save this adapter as +``my_survey/backend.py`` in an installable Python package: + +.. code-block:: python + + import json + + from hallmark.backends import HttpBackend, RemoteEntry + + + class SurveyBackend(HttpBackend): + """Read a published manifest; inherit bounded HTTP reads and transfers.""" + + def iter_entries(self, on_directory=None): + records = json.loads(self.read_text("manifest.json", self.context.text_limit)) + for record in records: + self.context.check_cancelled() + digest = record.get("sha256") + yield RemoteEntry( + path=record["path"], + size=record.get("size"), + checksum_algorithm="sha256" if digest else None, + checksum=digest, + ) + if on_directory is not None: + on_directory("") + +Register the class for the current Python process before cataloging or +transfer: + +.. code-block:: python + + from hallmark import DataSource, SourceRelease, Repo, register_backend, register_source + from my_survey.backend import SurveyBackend + + register_backend("survey", SurveyBackend) + register_source(DataSource("survey", "Manifest survey", { + "dr1": SourceRelease("https://survey.example.org/release/", {}, + backend="survey"), + })) + repo = Repo.init("survey") + repo.add("https://survey.example.org/release/{field}/{name}.fits") + +For CLI use and other collaborators, publish the class as an entry point in +the plugin package's ``pyproject.toml``: + +.. code-block:: toml + + [project.entry-points."hallmark.backends"] + survey = "my_survey.backend:SurveyBackend" + +Install that package in the same Python environment as Hallmark. Entry points +and ``register_backend`` use the same registry; duplicate names and classes +that do not subclass ``DataBackend`` are rejected. Register through one route +per process. Repository configuration stores registered names, never arbitrary +Python import paths. Old ``hallmark.transport`` imports remain supported as +compatibility aliases. + +Mapping one catalog across servers +---------------------------------- + +The :download:`multiple-server backend example <../demo/multi_server_backend.py>` +combines HTTP roots beneath logical prefixes. Its source definition supplies +the following adapter options: + +.. code-block:: yaml + + routes: + north: https://north.example.org/export/ + south: https://south.example.org/export/ + +The catalog records ``north/run_001.fits`` and ``south/run_002.fits``. The +backend maps those paths to their configured server roots and retains each +server's available metadata. Download plans capture the complete routing map. +These prefixes organize catalog paths; they do not require moving the data or +placing ``.hm`` on either server. + +From the source checkout, register the demonstration backend explicitly: + +.. code-block:: python + + from hallmark import DataSource, SourceRelease, Repo, register_source + from demo.multi_server_backend import register + + register() + register_source(DataSource("combined", "Two data servers", { + "v1": SourceRelease("https://north.example.org/export/", {}, + backend="multi-server", backend_options={"routes": { + "north": "https://north.example.org/export/", + "south": "https://south.example.org/export/", + }}), + })) + repo = Repo.init("combined") + repo.add("https://north.example.org/export/{site}/run_{run}.fits") + plan = repo.plan_download(include="north/**/*.fits") + print(plan.summary()) + +The example is tested with two disposable local HTTP servers. The URLs above +are illustrative. Collaboration-specific adapters for LSST, Roman, JWST or +Euclid can implement their own manifests, directory conventions or API +routing through the same contract; those service-specific plugins are not +included with Hallmark. diff --git a/doc/conf.py b/doc/conf.py index 2c15f9f..be01646 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -39,4 +39,4 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = 'alabaster' -html_static_path = ['_static'] +html_static_path = [] diff --git a/doc/design.rst b/doc/design.rst index 5e3e82b..f302dfc 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -77,8 +77,9 @@ Currently, |hallmark|_ has two built-in APIs: 1. Python API: the native API that all |hallmark|_ features are implemented in. ``State`` is the active object during the process lifetime. - ``Dothm`` and ``Worktree`` are optional depending on workflow (for - example, in-memory workflows may omit both). + ``Repo`` opens an on-disk repository. For an in-memory file index, + ``ParaFrame`` can discover and select files without ``Dothm`` or + ``Worktree``. 2. CLI: python features wrapped by ``click``. @@ -108,7 +109,7 @@ Repository ``Repo`` 2. bare repository::: - "repo.hm/" <--- Dothm, a standard or bare git repo + "repo.hm/" <--- Dothm, a standard git repo without a data worktree 3. shared repository (multiple worktrees)::: diff --git a/doc/index.rst b/doc/index.rst index d1b0f53..86da7d8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -4,6 +4,18 @@ contain the root `toctree` directive. .. include:: ../README.rst + :end-before: Private data remotes + +For private datasets, see :doc:`private_data` for SSH setup, CLI workflows, +Python examples and troubleshooting. + +Download the :download:`private data demo <../demo/private_data_demo.md>` +for a sequential CLI walkthrough. See :doc:`backends` to add +a data service or combine several servers under one catalog. + +The :download:`scientific CLI workflows <../demo/scientific_workflows_cli.md>` +and :download:`scientific Python notebook <../demo/scientific_workflows_python.ipynb>` +follow four examples using EHT, DESI, Roman–Rubin and local DES data. .. toctree:: :maxdepth: 2 @@ -11,6 +23,8 @@ design usecase + private_data + backends api diff --git a/doc/private_data.rst b/doc/private_data.rst new file mode 100644 index 0000000..5337e7a --- /dev/null +++ b/doc/private_data.rst @@ -0,0 +1,345 @@ +Data sources and private data +============================= + +``hallmark add`` catalogs remote files that match a URL template. It reads +directory listings and published checksum metadata without downloading dataset +files or offering transfers. ``hallmark ls-remote`` lists a remote directory +and suggests templates. ``hallmark clone`` copies a complete existing catalog, +then reviews a download plan by default. Use ``clone --no-download`` to skip +review. ``hallmark download`` selects and approves transfers from a catalog. + +1. Choose a source +------------------ + +List supported sources, then inspect DESI releases and collection roots. These +commands read source definitions without listing a dataset: + +.. code-block:: bash + + hallmark sources + hallmark sources desi + +DESI supports EDR and DR1. Its collections are ``redshifts`` (merged redshift +catalogs), ``spectra`` (HEALPixel and tile product directories), and ``vac`` +(value-added products). DR1 also provides ``lss`` (large-scale-structure +catalogs). The mappings follow the `DESI data organization +`_. + +``hallmark sources desi`` prints the URL of each release and collection. List +a collection to see its files and suggested templates, then add one of the +suggested commands to a repository: + +.. code-block:: bash + + hallmark init ./desi + cd desi + hallmark ls-remote https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/zcatalog/ + hallmark add 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/{pixel}/redrock-main-dark-{pixel}.fits' + hallmark commit -m 'Catalog DR1 redshifts' + +The source name only helps you find URLs; the catalog records the URL itself. +A release that declares a transport backend supplies it to templates beneath +its URL. Any browsable URL works the same way: + +.. code-block:: bash + + hallmark add 'https://data.example.org/export/{site}/{night}.fits' + +HTTP sources need a supported browsable index; HTTP file access alone does not +provide directory listings. Public HTTP sources need no credentials; private +HTTP retains Requests' local ``.netrc`` and environment behavior. URLs with +credentials, queries or fragments are rejected, because the URL is committed +with the catalog. See :doc:`backends` for source and transport plugins. + +2. Configure SSH or SFTP access +------------------------------- + +Use standard OpenSSH configuration. Add your server alias to ``~/.ssh/config``: + +.. code-block:: text + + Host lab-data + HostName data.example.org + User researcher + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes + StrictHostKeyChecking yes + +Use your actual host, user and key. Provision the server's verified host key in +``known_hosts`` and load an encrypted key into your SSH agent. Check that SFTP +works without an interactive password or MFA prompt: + +.. code-block:: bash + + ssh -V + sftp -o BatchMode=yes -b /dev/null lab-data + +Hallmark requires OpenSSH 9.6 or newer with connection multiplexing. Linux and +macOS are supported; Windows SSH transport is unsupported. Both ``ssh://`` and +``sftp://`` use structured SFTP listings and transfers. The server needs no +Hallmark, Git, Python or login shell. SSH configuration supplies identities, +users, ports and proxies, including ``ProxyJump``; explicit URL user/port values +take precedence. Hallmark enforces noninteractive access and strict host-key +checking. There is no Hallmark authentication file or profile-saving prompt. + +3. Write a template +------------------- + +Suppose the server contains ``runs/run_001.h5``, ``runs/run_002.h5`` and +``README.md``. Preview the run files, then catalog them and commit: + +.. code-block:: bash + + hallmark init ./lab + cd lab + hallmark add -n 'sftp://lab-data/srv/exports/lab/runs/run_{run}.h5' + hallmark add 'sftp://lab-data/srv/exports/lab/runs/run_{run}.h5' + hallmark commit -m 'Catalog lab runs' + +The URL is split at the first path segment containing a ``{field}``: +``sftp://lab-data/srv/exports/lab/runs/`` becomes the template's source and +``run_{run}.h5`` its filename template. Files are cataloged, and later +downloaded, at paths relative to the source, here ``run_001.h5``. Put a +directory in the template to keep it in the local layout: +``sftp://lab-data/srv/exports/lab/{group}/run_{run}.h5`` catalogs +``runs/run_001.h5``. + +Matching follows local ``hallmark add``: + +- Each field matches text within one path segment and never spans ``/``. +- Matching is case-sensitive. Hidden files and directories match only a + template segment that starts with ``.``. +- A file is cataloged only if the template re-creates its name exactly, so + ``run_{run:03d}.h5`` matches ``run_001.h5`` but not ``run_1.h5``. ``add`` + reports how many listed files fit the pattern but not its formatting. +- Field names cannot be reserved catalog columns such as ``path``, + ``checksum`` or ``size_bytes``. +- Literal parts of the URL are percent-decoded once: ``%20`` names a space and + ``%7B`` a literal brace. + +Only directories the template can match are listed, so a template deep in a +large tree reads few listings. A template that matches nothing is an error and +changes nothing. The catalog records each file's field values, size, +modification time and published checksum; unknown values remain unknown. +Hallmark never downloads payloads to compute hashes. Publish SHA-256 manifests +for reproducible verification where possible. Remote files are committed as +catalog entries, not as local objects. + +Run the same ``add`` command again to sync the template with the server. New +files are added, changed sizes or checksums are updated, and files that are no +longer listed are removed from the catalog. ``hallmark status`` summarizes +these changes per template before you commit them. + +When you do not know the layout, ``hallmark ls-remote URL`` lists the directory +and prints suggested templates, with the number of files each matches. +Suggestions come from Hallmark's filename-format detector and are heuristic: +``run_001.h5`` and ``run_002.h5`` yield ``run_{scan}.h5``. Rename fields before +adding a suggestion. ``ls-remote`` lists the whole tree below the URL and never +changes a repository. + +A branch can track several templates. Each has its own source and catalog +table, and local templates, such as ``reduced/{run}.dat``, can sit beside remote +ones. A file belongs to only one template. ``hallmark rm --cached TEMPLATE`` +stops tracking a template and leaves its files in place. ``hallmark init PATH`` +creates an empty repository; a PATH ending in ``.hm`` creates a bare catalog. + +4. Preview and approve downloads +-------------------------------- + +From the worktree, preview one run using only local catalog metadata, then +request the same transfer: + +.. code-block:: bash + + hallmark download --include 'run_001.h5' --dry-run + hallmark download --include 'run_001.h5' + +The download command displays the file count, known bytes, unknown-size count, +sources and destination, then asks for confirmation. Declining leaves the +catalog available. ``--include`` selects only cataloged files; explicit paths +cannot add files that no template catalogs. Downloads do not change catalog +membership. Use ``--all`` for the whole catalog or ``--tsv data.tsv`` for one +template's table. A bare catalog requires ``--output DIRECTORY`` with explicit +selectors; the interactive chooser can prompt for it. ``--max-workers`` +defaults to 4 and must be positive. Internal SSH session limits may further cap +concurrency. + +Files of each remote template download from its source; files of local +templates download from the default data remote, if one is configured. A plan +can therefore span several servers, and its preview lists each source with its +file count. ``--remote NAME`` downloads every selected file from a data remote +configured with ``set-config`` instead. Selected files without any source are +skipped and counted. + +Dataset transfers require approval even for one file. The legacy download +``--yes`` option does not bypass confirmation. To choose files interactively +instead of supplying selectors on the command line, run: + +.. code-block:: bash + + hallmark download --interactive + +The chooser warns about bandwidth and disk use, then offers ``patterns``, +``all``, or ``skip`` (the default). Enter one relative-path glob per line without +shell quotes; a blank line finishes the list. Spaces and commas remain part of +the pattern. Patterns are ORed, case-sensitive, and support recursive ``**``. +Finishing an empty list or matching no files returns to selection. + +The preview shows the sources, destination, file count, known bytes, +unknown-size count, and up to 20 paths with recorded sizes. It uses local +metadata only; unknown sizes remain unknown. Choose ``download`` to approve +that exact plan, ``change`` to select again and review a new plan, or ``skip`` +(the default). ``--interactive --dry-run`` previews the selection and exits +without offering execution. ``--output``, ``--remote`` and ``--max-workers`` are +supported with ``--interactive``; paths, ``--include``, ``--tsv`` and ``--all`` +cannot be combined with it. Bare ``hallmark download`` still requires +selectors or ``--interactive``. + +CLI cloning first copies the complete catalog, then shows a download plan and +asks for confirmation. All cataloged files are selected unless ``--include`` +narrows the planned transfers. These examples use separate destinations: + +.. code-block:: bash + + hallmark clone ./lab/.hm ./lab-copy-selected --include 'run_001.h5' + hallmark clone ./lab/.hm ./lab-copy-interactive --interactive + +Templates control catalog membership. Clone ``--include`` globs and the chooser +select only downloads, retaining the complete catalog and its history. +``clone --interactive`` opens the chooser instead of the default plan and cannot +be combined with ``--include``. ``clone --no-download`` skips review entirely and +cannot be combined with ``--interactive``, ``--include``, or ``--output``. +``clone --output DIRECTORY`` changes the download destination; a bare catalog +prompts for one if omitted. Declining or EOF during clone's review exits +successfully and leaves the catalog available. Ctrl+C or a transfer failure exits +unsuccessfully while preserving it. Empty catalogs and catalogs without a data +source skip review with an explanation. + +Without an interactive terminal, clone still copies the catalog, warns that +approval is unavailable, and prints repository-specific preview and download +commands reflecting its ``--include`` globs and output directory. Later +transfers still require explicit approval. Standalone ``download --interactive`` +requires a terminal. Download worker limits belong on ``download``; ``init``, +``add`` and ``clone`` do not accept them. Python ``add`` and cloning remain +metadata-only. + +5. Use the Python API +--------------------- + +Catalog remote files and inspect a plan before transferring selected data: + +.. code-block:: python + + from hallmark import Repo + + repo = Repo.init("lab-python") + repo.add("sftp://lab-data/srv/exports/lab/runs/run_{run:03d}.h5", + progress=True) + repo.commit("Catalog lab runs") + plan = repo.plan_download(include="run_001.h5") + print(plan.summary()) + +After reviewing the plan, approve it explicitly and check the results: + +.. code-block:: python + + result = repo.download(plan, approved=True, max_workers=4, progress=True) + if result["failed"]: + raise RuntimeError("; ".join(result["errors"])) + +``repo.add(url_template, dry_run=True)`` returns the matching files without +staging them. ``repo.add(url_template, backend=..., backend_options=...)`` +records a transport plugin for the template. Plans freeze paths, checksums, +sources, transport options and destination; subsequent catalog configuration +changes do not redirect them. ``plan.sources`` lists the servers a plan +contacts. ``estimated_bytes_per_second`` can provide a duration estimate when +every selected file has a known size. + +6. Share a catalog +------------------ + +Clone an existing catalog without download review, then select downloads separately: + +.. code-block:: bash + + hallmark clone ./lab/.hm ./lab-copy --no-download + cd lab-copy + hallmark download --include 'run_001.h5' --dry-run + hallmark download --include 'run_001.h5' + +Git clones preserve the full catalog and Git history. Published HTTP/SFTP +snapshots start new local history. Use ``--source-type git`` or +``--source-type catalog`` to override source detection. The catalog can be +hosted independently from the dataset, for example GitHub metadata with SFTP +data sources. Metadata cloning does not connect to those sources. Each client +configures its own SSH alias and credentials for later transfers. + +Downloads do not populate the local object store. To version downloaded files +locally, run ``hallmark rm --cached`` on the remote template and add the same +template as a local one. + +7. Migrate older commands +------------------------- + +This interface change intentionally provides no argument aliases. Existing +catalogs retain their local formats and transport metadata. Stored authentication +profile references produce an actionable migration error. + +.. list-table:: Interface migration + :header-rows: 1 + :widths: 35 65 + + * - Previous interface + - Replacement + * - ``init --from URL`` / ``--release`` / ``--collection`` + - ``hallmark init PATH``, then ``hallmark add 'URL/TEMPLATE'``; find URLs + with ``hallmark sources`` and ``hallmark ls-remote``. + * - ``init --filter`` / ``--include`` + - The template selects files; ``hallmark add -n`` previews the matches. + * - ``init --format`` / ``--extract`` / ``--fmt`` + - The template's fields; ``hallmark ls-remote`` suggests templates. + * - ``init --backend`` / ``--backend-options`` + - ``Repo.add(url_template, backend=..., backend_options=...)``; source + authors can declare a backend on a release. + * - ``download --filter`` / ``--fmt`` + - ``download --include`` with path globs. + * - ``clone --filter`` + - ``clone --include`` selects payloads only; the catalog remains complete. + * - ``init --with-download`` + - Add a template, then run ``download`` separately. + * - ``clone --with-download`` / ``--download`` + - Plain ``clone`` reviews downloads by default; ``--interactive`` opens the chooser. + * - ``clone --no-fetch-data`` or relying on catalog-only CLI defaults + - ``clone --no-download`` copies the complete catalog without download review. + * - Adding a different template to replace a branch's catalog + - The template is added beside the others; ``hallmark rm --cached`` removes one. + * - ``--auth``, ``--remote-auth``, ``--dataset-auth`` + - Configure ``~/.ssh/config`` and use its host alias in the URL. + * - ``Repo.init(source=...)`` / ``Repo.init(from_url=...)`` + - ``Repo.init(path)`` followed by ``repo.add(url_template)``. + * - Python ``filter=`` + - ``include=``. + * - Python clone/init download arguments and approval callbacks + - ``repo.plan_download()`` followed by ``repo.download(plan, approved=True)``. + +Remove obsolete ``auth`` fields from catalog ``config.yml`` after configuring +the host in SSH configuration. Hallmark neither reads nor deletes +``~/.config/hallmark/auth.yml``; ``HALLMARK_AUTH_FILE`` is ignored. Python profile +arguments and CLI backend-setting options have been removed. Existing +``backend`` and ``backend_options`` catalog metadata remain supported for +transport plugins. The legacy remote builder remains deprecated. + +8. Transfer failures and limits +------------------------------- + +Files are downloaded to temporary paths and published atomically after checksum +verification. Successful files remain if another transfer fails. Cancelling +stops owned SSH processes and waits for workers; HTTP checks cancellation +between chunks and remains subject to request timeouts. Missing checksums cannot +verify content. Setup and authentication failures stop the operation. + +If SSH fails, check the alias with the SFTP command above, trusted host keys, +agent access and server permissions. If listing fails, check that the URL +publishes a supported directory index. ``ls-remote`` lists the whole tree below +its URL, which can take many requests for a large root; ``add`` lists only the +directories its template can match. diff --git a/doc/usecase.rst b/doc/usecase.rst index 2996054..a644733 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -38,46 +38,52 @@ Dataset files are represented through staged ``sha1`` column in associated with each file in different rows. -2. CLI: Bare Repository with Worktree Ingest --------------------------------------------- +2. CLI: Bare Repository with Explicit Downloads +----------------------------------------------- -Bob prefers managing a bare hallmark repository for storage and -staging data from a linked worktree. -He initialized the bare repository and verify its mode:: +Bob prefers keeping the data index separate from downloaded files. +He initializes a bare |hallmark|_ catalog, adds a simulation export from a +campus server, and verifies its location:: - hallmark init --bare sim.hm + hallmark init sim.hm cd sim.hm + hallmark add 'ssh://campus/srv/export/run{run:d}/frame{frame:d}.h5' + hallmark commit -m "Catalog simulation export" hallmark info -He then attaches a worktree, stages discovered files, and commits:: +The ``.hm`` suffix selects a bare catalog without a data worktree. +``add`` records ``ssh://campus/srv/export/`` as the source of the template +``run{run:d}/frame{frame:d}.h5`` and catalogs the matching files without +downloading them. +Bob reviews the selection and approves downloading to an explicit directory:: - hallmark worktree add /data/outputs - cd /data/outputs - hallmark add "run{run:d}/frame{frame:d}.h5" - hallmark status - hallmark diff - hallmark commit -m "Simulation snapshots" + hallmark download --all --output ../outputs --dry-run + hallmark download --all --output ../outputs -The commit updates the same bare repository that owns the linked -worktree. +The catalog stays in ``sim.hm``; the downloaded files go to ``outputs``. +For local ingest and versioning, Bob can initialize ``outputs`` as a +standard repository and follow Alice's workflow. -3. CLI: Branch-Isolated Analysis with Multiple Worktrees --------------------------------------------------------- +3. Python: Branch-Isolated Analysis with Multiple Worktrees +----------------------------------------------------------- Carol wants to manage data from multiple simultaneous observations without mixing data. -She creates a new branch and attach a second worktree to it:: +Starting from a committed standard repository, she creates a new branch +and attaches a second worktree through the Python API:: - hallmark branch obs2 - hallmark worktree add remote:/data/obs obs2 + from hallmark import Repo -She lists linked worktrees and continue on the new branch:: + repo = Repo("obs") + repo.add_worktree("obs2") + second = Repo(repo.worktree.parent / "obs2") - hallmark worktree list - hallmark status - hallmark add "{site}/{year:d}/{day:d}.fits" - hallmark commit -m "Observation ingest on branch obs2" +After adding observation files beneath ``obs2``, she stages and commits +them in that worktree:: + + second.add("{site}/{year:d}/{day:d}.fits") + second.commit("Observation ingest on branch obs2") Each worktree stays isolated by branch, so staged state and commits do not interfere. @@ -91,12 +97,10 @@ repository:: from hallmark import Repo - repo = Repo.open("obs") - info = repo.info() - print(info.local_path, info.worktree_path) + repo = Repo("obs") + print(repo.dothm.path, repo.worktree) - for y in range(2000,2026,5): - repo.add(f"{{site}}/{y}/{{day:d}}.fits") # escape {{ and }} + repo.add("{site}/{year:d}/{day:d}.fits") repo.commit("Nightly ingest") This workflow is suitable for finer control of data ingest. @@ -105,18 +109,223 @@ This workflow is suitable for finer control of data ingest. 5. Python: In-Memory State Workflows ------------------------------------ -Emma can use an memory-backed facade for data transformations that do -not require git operations:: +Emma can use ``ParaFrame`` to discover and select files without +creating a Git repository:: - from hallmark import Repo + from hallmark import ParaFrame - repo = Repo() - repo.add("data/{site}/{year:d}/{day:d}.fits") - repo.worktree("data_transformed/{year:d}/{day:d}/{site}.fits") + data = ParaFrame.parse("{site}/{year:d}/{day:d}.fits", base_path="data") + selected = data.filter(year=2025) + print(selected) -This is especially useful for data (re-)organization. +This keeps the file index in memory. Emma can pass the selected paths to +her analysis tools; filtering the index does not modify the data files. .. |hallmark| replace:: ``hallmark`` .. _hallmark: https://github.com/l6a/hallmark + + +.. _private-transport-reference: + +.. _private-lab-data-over-ssh: + +6. CLI: Private Lab Data over SSH +---------------------------------- + +For a step-by-step CLI workflow and Python examples, see :doc:`private_data`. + +Frank works with simulation data stored on a private lab server. +He has an existing |hallmark|_ repository that indexes the files and +wants to download them to his local worktree. +He first configures the SSH alias ``campus``, verifies the server's host +key, and loads his key into an SSH agent or configures an identity file. + +From the local repository, he sets the data location, previews the +selection, and downloads the files:: + + hallmark set-config --remote-name campus --remote-url ssh://campus/srv/export/ + hallmark download --remote campus --all --dry-run + hallmark download --remote campus --all + +The data remote ``campus`` specifies where the dataset files are stored. +It is configured separately from the Git remote used to share the +history in ``.hm``, even when both remotes have the same name. +The ``--remote`` option selects one data server; |hallmark|_ does not +automatically try another server if the download fails. + +Downloads are written to temporary files and checked against any +checksums supplied with the selection before atomically replacing +their destinations. +The ``--tsv`` and ``--all`` options include catalog checksums. Explicit +paths also use their recorded checksums when available. +If a transfer or checksum check fails, the existing destination is +preserved and the temporary file is removed. +Files that have already downloaded successfully remain available. + +SSH access and file paths +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The client requires OpenSSH ``ssh`` and ``sftp`` version 9.6 or newer +with connection multiplexing enabled. +Linux and macOS have integration-test jobs. Windows SSH transport is +not supported. +HTTP downloads continue to use Requests, including its environment +settings and ``.netrc`` authentication. + +|hallmark|_ uses ``BatchMode=yes`` and defaults to +``StrictHostKeyChecking=yes``. Authentication must succeed without +password prompts, interactive two-factor authentication, or an initial +hardware-key prompt. Complete any interactive setup before downloading. +Each invocation opens and closes its own shared SSH connection, leaving +existing SSH connections alone. Settings such as ``ProxyJump`` belong +in the local SSH configuration. + +The URL must specify an absolute dataset directory. +Both ``ssh://`` and ``sftp://`` use SFTP for file transfers. +Usernames and ports can be included, as in +``ssh://researcher@campus:2222/srv/export/``. +IPv6 addresses require brackets, as in +``ssh://[2001:db8::1]/srv/export/``. +SCP shorthand and paths relative to a home directory are not supported. + +Reserved characters in URL paths must be percent-encoded, e.g., +``has%20space%23tag``. Paths in catalogs and checksum manifests are +literal: ``literal%20.h5`` names a file containing a percent sign. +Spaces, quotes, glob characters, and leading dashes are supported; +control characters and backslashes are rejected. +Existing destination symlinks and detected changes to symlinks are +rejected, although these checks cannot prevent every race with another +process modifying the destination. + +Cancelling a download stops the SSH processes started by |hallmark|_, +waits for the download workers, and closes its shared connection. +HTTP workers check for cancellation between chunks. A pending Requests +call may wait for its 10-second connection timeout or 30-second read +timeout; a response that keeps sending small amounts of data can delay +cancellation further. + +SSH configuration +~~~~~~~~~~~~~~~~~ + +Frank keeps connection settings in ``~/.ssh/config``. Hallmark passes the URL's +host alias to OpenSSH, which resolves the user, port, identity and any proxy. +Explicit URL user and port values take precedence. Hallmark does not read a +separate authentication file or prompt to create one. + +Catalog files on the server +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Frank can catalog the run files directly on the server:: + + hallmark init lab + cd lab + hallmark ls-remote ssh://campus/srv/export/runs/ + hallmark add 'ssh://campus/srv/export/runs/run_{i:d}.h5' + hallmark commit -m "Catalog lab runs" + hallmark download --all --dry-run + hallmark download --all + +``ls-remote`` lists the directory and suggests templates without changing the +repository. ``add`` records ``ssh://campus/srv/export/runs/``, the URL up to +the first path segment with a field, as the template's source, and catalogs +only the files matching ``run_{i:d}.h5``. They download to ``lab/run_1.h5`` +and so on; to keep the ``runs`` directory in the local layout, make it part of +the template, as in ``ssh://campus/srv/export/{group}/run_{i:d}.h5``. +Fields never span a ``/``, and only directories the template can match are +listed. A template that matches nothing is an error, and a file is cataloged +only if the template re-creates its name exactly; ``hallmark add -n`` previews +the matches. No dataset contents are downloaded. +The final command displays the transfer plan and asks Frank for confirmation. + +Both SSH and SFTP sources use structured SFTP directory enumeration. The server +needs no login shell, Python or Hallmark, and SFTP-only accounts work for +both listing and downloads. Symlinks and special files are skipped. +Authentication and permission failures stop listing rather than producing +an apparently complete catalog. + +When the server changes, Frank runs the same ``add`` command again. It syncs +the template's catalog: new files are added, changed sizes or checksums are +updated, and files no longer on the server are dropped. ``hallmark status`` +summarizes these catalog changes per template before he commits them. + +Frank keeps his reduced products in the worktree beside the server data. A +branch can track several templates, local and remote:: + + hallmark add 'reduced/run_{i:d}.dat' + hallmark commit -m "Add reduced runs" + +Local templates are committed as objects, so ``checkout`` restores their +files. Remote templates are committed as catalog entries only: checkout never +removes or restores downloaded copies, and ``status`` does not report missing +or downloaded copies as changes. After switching branches, downloading again +replaces a stale copy. ``hallmark rm --cached TEMPLATE`` stops tracking a +template and leaves its files in place. + +The data entry records the template's source URL. Data remotes configured +with ``set-config`` act as mirrors: ``download --remote NAME`` downloads every +selected file from that remote instead. Local templates download from the +default data remote, when one is configured. +Existing Git-hosted catalogs keep their complete catalog, history and +recorded sources when cloned. Select and approve transfers afterward with +``download --include``; downloads leave the catalog and history unchanged. +The Git catalog can be hosted separately from its data servers. + +Published manifest checksums are recorded in the catalog. Missing checksums +remain unknown; |hallmark|_ does not read dataset files locally or on the +server to compute them during discovery. Downloads verify any available +publisher checksum. Discovery covers the requested tree without an overall +time or entry limit. Individual requests have limits, and discovery can +be cancelled. + +CyVerse and common HTTPS directory indexes are detected automatically:: + + hallmark add \ + 'https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/healpix/main/dark/230/{pixel}/redrock-main-dark-{pixel}.fits' + +A server with no usable listing needs a published Hallmark catalog or a +backend plugin that understands its API. See :doc:`backends` for the shared +interface and registration. Discovery cannot infer hidden file URLs. +HTTP/SFTP catalog snapshots start local history; Git endpoints supply catalog +history. Use ``--source-type git`` for ambiguous Git URLs. + +Frank can also inspect and approve a transfer in Python:: + + from hallmark import Repo + + repo = Repo('lab') + plan = repo.plan_download(include='run_1.h5') + print(plan.summary()) + +After reviewing the plan, he approves the download and checks for failures:: + + result = repo.download(plan, approved=True, progress=True) + if result['failed']: + raise RuntimeError('\n'.join(result['errors'])) + +Plans are built from local metadata without contacting the server. Missing +sizes and duration estimates are reported as unknown. During transfer, +progress reports bytes and estimates remaining time when possible. +``build`` and ``build_repo`` remain available but are deprecated in favor of +``add`` with a URL template. +Cataloging remote files does not populate the local object store. To version +downloaded files locally, stop tracking the remote template with +``hallmark rm --cached`` and add the same template as a local one. + +Transport validation +~~~~~~~~~~~~~~~~~~~~ + +The regular test suite runs with ``pytest -q``. +SFTP parser tests use a local ``sftp-server`` process without networking. +To run the complete OpenSSH integration tests:: + + HALLMARK_RUN_SSH_TESTS=1 pytest test/test_ssh_integration.py -q + +These tests use a loopback server, temporary keys and data, and separate +SSH configuration and known-hosts files. +They do not connect to a lab server or change the user's SSH trust files. +Enabling the tests requires the OpenSSH tools to be installed; missing +tools cause a test failure. +CI installs the server and runs these tests separately from the +Python-version matrix. diff --git a/mod/hallmark/__init__.py b/mod/hallmark/__init__.py index efbd637..86bb763 100644 --- a/mod/hallmark/__init__.py +++ b/mod/hallmark/__init__.py @@ -15,7 +15,17 @@ """Hallmark python public API""" +from .sources import DataSource, SourceRelease, register_source from .repo import Repo from .paraframe import ParaFrame +from .download_plan import DownloadItem, DownloadPlan, DownloadSource +from .backends import ( + DataBackend, HttpBackend, SshBackend, CyVerseBackend, RemoteEntry, + register_backend, +) -__all__ = ["Repo", "ParaFrame"] +__all__ = [ + "DataSource", "SourceRelease", "register_source", "Repo", "ParaFrame", + "DownloadItem", "DownloadPlan", "DownloadSource", "DataBackend", + "HttpBackend", "SshBackend", "CyVerseBackend", "RemoteEntry", "register_backend", +] diff --git a/mod/hallmark/backends.py b/mod/hallmark/backends.py new file mode 100644 index 0000000..e31d11d --- /dev/null +++ b/mod/hallmark/backends.py @@ -0,0 +1,97 @@ +"""Public data backends and registration for collaboration-specific adapters. + +Installed adapters declare the ``hallmark.backends`` entry-point group. Only +the selected adapter is imported when opening an operation; parsing remote +configuration and creating download plans never load adapter code. +""" + +from importlib import metadata +from threading import RLock + +from .transport import OperationContext +from .transport.base import ( + CapabilityError, DataBackend, DownloadError, RemoteConfigurationError, + RemoteEntry, RemoteObjectMissing, RemoteSpec, TransferCancelled, backend_name, +) +from .transport.cyverse import CyVerseBackend +from .transport.http import HttpBackend +from .transport.ssh import SshBackend + + +_registered = {"http": HttpBackend, "ssh": SshBackend, "cyverse": CyVerseBackend} +_loaded = {} +_lock = RLock() + + +def _entry_points(name): + """Find matching plugin metadata on Python 3.9 and newer versions.""" + entries = metadata.entry_points() + if hasattr(entries, "select"): + entries = entries.select(group="hallmark.backends") + else: + entries = entries.get("hallmark.backends", ()) + return [entry for entry in entries if entry.name == name] + + +def _validate_class(backend_class): + if (not isinstance(backend_class, type) + or not issubclass(backend_class, DataBackend) + or backend_class is DataBackend): + raise RemoteConfigurationError("Backend must be a DataBackend subclass") + return backend_class + + +def register_backend(name, backend_class): + """ + Register a backend class under a persistent configuration name. + + Args: + name (str): Unique lowercase name using letters, digits, underscores, + and hyphens, beginning with a letter. + backend_class (type[DataBackend]): Class constructed with an + OperationContext when selected by a data remote. + + Raises: + RemoteConfigurationError: If the name is invalid, already registered + or installed, or the class does not implement the base interface. + + Register before opening a repository operation. Distribution entry points + provide automatic registration in fresh Python processes and the CLI. + """ + name = backend_name(name) + _validate_class(backend_class) + with _lock: + if name in _registered or name in _loaded or _entry_points(name): + raise RemoteConfigurationError(f"Backend {name!r} is already registered") + _registered[name] = backend_class + + +def get_backend(name): + """Resolve a registered class, lazily importing a selected installed plugin.""" + name = backend_name(name) + with _lock: + entries = _entry_points(name) + if len(entries) > 1 or (entries and name in _registered): + raise RemoteConfigurationError(f"Duplicate backend name {name!r}") + if name in _registered: + return _registered[name] + if name in _loaded: + return _loaded[name] + if not entries: + raise RemoteConfigurationError( + f"Backend {name!r} is not installed or registered") + try: + backend_class = entries[0].load() + except Exception: + raise RemoteConfigurationError( + f"Unable to load backend {name!r}; check its installation") from None + _loaded[name] = _validate_class(backend_class) + return _loaded[name] + + +__all__ = [ + "DataBackend", "HttpBackend", "SshBackend", "CyVerseBackend", + "register_backend", "RemoteEntry", "RemoteSpec", "OperationContext", + "CapabilityError", "DownloadError", "RemoteConfigurationError", + "RemoteObjectMissing", "TransferCancelled", +] diff --git a/mod/hallmark/catalog.py b/mod/hallmark/catalog.py new file mode 100644 index 0000000..1f758a5 --- /dev/null +++ b/mod/hallmark/catalog.py @@ -0,0 +1,205 @@ +"""Clone existing Hallmark catalogs from Git or published snapshots.""" + +from __future__ import annotations + +import re +from io import StringIO +from pathlib import Path +from shutil import rmtree +from urllib.parse import urlsplit + +import pandas as pd +import yaml + +from .dothm import Dothm +from .error import CloneError, DestinationExistsError +from .helper_functions import as_list_of_dicts +from .repo_config import fmt_fields, normalize_remotes, normalize_tsv_name +from .repo_manifest import row_relative_path +from .transport import OperationContext, RemoteSpec +from .transport.base import RemoteObjectMissing, literal_path +from .worktree import Worktree + + +def _catalog_names(config): + """Validate catalog filenames before reading any remote metadata.""" + entries = as_list_of_dicts(config.get("data", [])) + if entries is None or any(not isinstance(entry, dict) for entry in entries): + raise CloneError("Catalog data must be a mapping or list of mappings") + names = {"data.tsv"} + for entry in entries: + if entry.get("db"): + names.add(normalize_tsv_name(entry["db"])) + if entry.get("file"): + literal_path(entry["file"]) + normalize_remotes(config.get("remote")) + return sorted(names) + + +def _catalog_formats(config, name): + """Return the filename formats associated with a catalog TSV.""" + entries = as_list_of_dicts(config.get("data", [])) or [] + return [entry["fmt"] for entry in entries if entry.get("fmt") + and normalize_tsv_name(entry.get("db", "data.tsv")) == name] + + +def _row_path(row, formats): + """Resolve a catalog row to one literal relative path.""" + try: + return row_relative_path(row, formats).as_posix() + except ValueError as exc: + raise CloneError(str(exc)) from exc + + +def _validate_snapshot(files, config): + """Validate catalog tables and paths before writing the snapshot.""" + for name in _catalog_names(config): + try: + frame = pd.read_csv(StringIO(files[name]), sep="\t", dtype=str, + keep_default_na=False) + except (pd.errors.EmptyDataError, pd.errors.ParserError) as exc: + raise CloneError(f"Invalid published catalog table: {name}") from exc + formats = _catalog_formats(config, name) + has_fields = any(fmt_fields(fmt) and set(fmt_fields(fmt)) <= set(frame.columns) + for fmt in formats) + if not ({"path", "sha1"} & set(frame.columns) or has_fields): + raise CloneError(f"Unrecognized published catalog columns: {name}") + for _, row in frame.iterrows(): + _row_path(row, formats) + + +def _snapshot(context): + """ + Read and validate a published catalog without fetching dataset files. + + Return None when ``config.yml`` is absent. Once present, incomplete or + malformed catalog metadata raises CloneError rather than triggering fallback. + """ + try: + config_text = context.read_text("config.yml") + except RemoteObjectMissing: + return None + try: + config = yaml.safe_load(config_text) + except yaml.YAMLError as exc: + raise CloneError("Invalid published catalog config.yml") from exc + if not isinstance(config, dict) or "data" not in config: + raise CloneError("Published catalog config.yml must contain a data mapping") + try: + meta_text = context.read_text("meta.yml") + data_text = context.read_text("data.tsv") + except RemoteObjectMissing as exc: + raise CloneError( + "Published Hallmark catalog is missing required metadata") from exc + try: + meta = yaml.safe_load(meta_text) + except yaml.YAMLError as exc: + raise CloneError("Invalid published catalog metadata") from exc + if not isinstance(meta, dict): + raise CloneError("Published catalog meta.yml must be a mapping") + files = {"config.yml": config_text, "meta.yml": meta_text, + "data.tsv": data_text} + for name in _catalog_names(config): + if name not in files: + files[name] = context.read_text(name) + _validate_snapshot(files, config) + return files + + +def _metadata_commit(repo, files, message): + """Commit catalog metadata and reload the repository state.""" + repo.dothm.index.add(list(files)) + if repo.dothm.index.diff("HEAD"): + repo.dothm.index.commit(message) + repo.state = repo.dothm.load() + + +def _git_source(url, source_type): + """Determine whether the selected source should be cloned with Git.""" + if source_type == "git": + return True + if source_type != "auto": + return False + parsed = urlsplit(url) + scp_style = "://" not in url and re.match(r"^(?:[^/@:]+@)?[^/:]+:.+", url) + return (not parsed.scheme or parsed.scheme in {"git", "file", "ssh"} + or bool(scp_style) + or (parsed.scheme != "sftp" and parsed.path.rstrip("/").endswith(".git"))) + + +def _clone_git(cls, url, destination, display_path): + """Clone a catalog's Git history without modifying its tracked metadata.""" + dothm_path, worktree_path = cls.lwpaths(destination) + local = Path(url).expanduser() + git_url = str(local / ".hm") if (local / ".hm").is_dir() else url + try: + Dothm.clone(git_url, dothm_path, display_path=display_path) + except CloneError as exc: + raise CloneError( + f"{exc}\nFor a raw dataset, use hallmark init PATH, then " + "hallmark add 'URL/{field}...'.") from exc + if worktree_path: + Worktree.init(worktree_path) + return cls(destination) + + +def clone_catalog(cls, url, path, *, source_type="auto"): + """ + Clone an existing Git catalog or published HTTP/SFTP catalog snapshot. + + Existing destinations are rejected before source access. An incomplete + destination created by this call is removed on failure. Raw datasets are + cataloged with ``Repo.add(url_template)`` in a new repository. + + Args: + cls: Repository class used to initialize or open the result. + url (str): Existing Git catalog or published snapshot location. + path (Path | str): New repository destination. + source_type (str): ``auto``, ``git``, or ``catalog``. + + Returns: + Repo: Repository containing complete catalog metadata without payloads. + + Raises: + DestinationExistsError: If the destination already exists. + CloneError: If a requested catalog is missing or invalid. + ValueError: If source options are invalid. + DownloadError: If remote metadata cannot be read or validated. + """ + if source_type not in {"auto", "git", "catalog"}: + raise ValueError("source_type must be auto, git, or catalog") + url = str(url) + destination = Path(path).expanduser().absolute() + if destination.exists() or destination.is_symlink(): + raise DestinationExistsError( + f"fatal: destination path '{path}' already exists and is not empty.") + is_git = _git_source(url, source_type) + destination.parent.mkdir(parents=True, exist_ok=True) + try: + destination.mkdir() + except FileExistsError as exc: + raise DestinationExistsError(f"Destination already exists: {path}") from exc + try: + if is_git: + return _clone_git(cls, url, destination, path) + source = RemoteSpec.parse(url) + with OperationContext(source) as context: + snapshot = _snapshot(context) + if snapshot is None and not source.root.rstrip("/").endswith(".hm"): + nested = RemoteSpec.parse(url.rstrip("/") + "/.hm/") + with OperationContext(nested) as context: + snapshot = _snapshot(context) + if snapshot is None: + if source_type == "auto" and source.scheme in {"http", "https"}: + return _clone_git(cls, url, destination, path) + raise CloneError("No published Hallmark catalog at this URL; for a " + "raw dataset, use hallmark init PATH, then " + "hallmark add 'URL/{field}...'") + repo = cls.init(destination) + for name, contents in snapshot.items(): + (repo.dothm.path / name).write_text(contents, encoding="utf-8") + _metadata_commit(repo, snapshot, "Import published catalog snapshot") + return repo + except BaseException: + rmtree(destination, ignore_errors=True) + raise diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 1413e09..ce1b0a0 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -16,6 +16,8 @@ from contextlib import contextmanager from pathlib import Path +import shlex +import sys import click import requests @@ -26,13 +28,11 @@ from . import Repo from .helper_functions import validate_path_component from .repo_builder import build_repo -from .downloader import ( - BULK_DOWNLOAD_WARNING_FILE_COUNT, - DownloadError, - download_remote_data, - select_download_files) +from .downloader import DownloadError, _select_download_items, _select_remote_config +from .sources import get_source, list_sources from .error import CheckoutError, CloneError -from .repo_config import normalize_tsv_name +from .discovery import is_remote_url +from .repo_config import catalog_entries, normalize_tsv_name # use a context manager to translate application errors into clean Click errors @@ -72,6 +72,7 @@ def _translate_cli_errors(*error_types, prefix=None): # exception types translated to a clean CLI error by the hallmark build command _BUILD_DATASET_ERRORS = ( + DownloadError, RuntimeError, ValueError, FileNotFoundError, @@ -80,30 +81,6 @@ def _translate_cli_errors(*error_types, prefix=None): yaml.YAMLError) -def _confirm_bulk_download( - selected_files, - assume_yes: bool, - ) -> None: - """ - Used by _run_download. - Prompt the user for confirmation if the number of selected files exceeds - the warning threshold and the user has not opted to assume yes. - Args: - selected_files (list): List of files selected for download. - assume_yes (bool): Flag indicating whether to assume yes for prompts. - Raises: - ClickException: If the user does not confirm the download. - """ - file_count = len(selected_files) - if ( - file_count >= BULK_DOWNLOAD_WARNING_FILE_COUNT - and not assume_yes): - click.echo( - f"Selected {file_count} files for download.\n" - "The total size is not recorded and may be very large.") - click.confirm("Continue?", abort=True) - - def _report_download_results(results: dict) -> None: """ Used by _run_download. @@ -142,32 +119,203 @@ def _report_download_results(results: dict) -> None: raise ClickException(f"Failed to download {failed} file(s)") -def _run_download( - repo, - selected_files, - output_path, - *, - max_workers, - assume_yes, - remote_name=None, - ) -> None: - """ - Used by download and clone. - Confirm, execute, and report one selected download.""" - # confirm with user if the number of selected files exceeds warning threshold - _confirm_bulk_download(selected_files, assume_yes) - # download the selected files from the remote repository - results = download_remote_data( - repo, - output_path, - max_workers=max_workers, - show_progress=True, - selected_files=selected_files, - remote_name=remote_name) - # report the results of the download operation to the user +def _show_download_plan(plan, *, show_paths=False): + """Display recorded sizes without querying the data server.""" + click.echo(plan.summary()) + if show_paths: + click.echo(f"Files with unknown size: {plan.unknown_size_count}") + for item in plan.items[:20]: + size = (f"{item.size_bytes:,} bytes" if item.size_bytes is not None + else "size unknown") + click.echo(f" {item.relative_path.as_posix()} ({size})") + if plan.file_count > 20: + click.echo(f" ... {plan.file_count - 20} more file(s)") + + +def _execute_download(repo, plan, *, max_workers): + """Execute the exact plan the user approved and report transfer failures.""" + results = repo.download(plan, approved=True, max_workers=max_workers, + progress=True) _report_download_results(results) +def _confirm_download_plan(plan, *, decline_is_skip=False, show_paths=False): + """Display a plan and approve it explicitly, optionally treating refusal as skip.""" + _show_download_plan(plan, show_paths=show_paths) + if not plan.file_count: + click.echo("No files selected for download.") + return False + return click.confirm("Download these files?", default=False, + abort=not decline_is_skip) + + +def _run_download(repo, plan, *, max_workers): + """ + Display a download plan, request approval, and report the results. + + Args: + repo: The hallmark repository object. + plan (DownloadPlan): Files, source, and destination to display. + max_workers (int): Maximum concurrent download workers. + + Raises: + Abort: If a nonempty download is declined. + ClickException: If any file transfers fail. + DownloadError: If the download cannot be started. + """ + if _confirm_download_plan(plan): + _execute_download(repo, plan, max_workers=max_workers) + + +def _download_output(repo, output): + """Prompt for the missing destination of a bare catalog without creating it.""" + if repo.worktree is None and output is None: + return click.prompt( + "Output directory (relative to the current directory)", + type=click.Path(file_okay=False)) + return output + + +def _choose_download_plan(repo, *, output, remote_name, dry_run): + """Return an approved plan, or None after skipping or an offline preview.""" + while True: + choice = click.prompt( + "Select files", type=click.Choice(["patterns", "all", "skip"], + case_sensitive=False), default="skip") + if choice == "skip": + return None + patterns = [] + if choice == "patterns": + click.echo("Enter one relative-path glob per line, without shell quotes. " + "Blank finishes; patterns are ORed. ** matches recursively.") + while True: + pattern = click.prompt("Pattern", default="", show_default=False) + if not pattern: + break + patterns.append(pattern) + if not patterns: + continue + output = _download_output(repo, output) + plan = repo.plan_download(output, all_files=choice == "all", + include=patterns or None, remote_name=remote_name) + _show_download_plan(plan, show_paths=True) + if not plan.file_count: + click.echo("No cataloged files match. Choose another selection or skip.") + continue + if dry_run: + return None + action = click.prompt( + "Next action", type=click.Choice(["download", "change", "skip"], + case_sensitive=False), default="skip") + if action == "download": + return plan + if action == "skip": + return None + + +def _download_available(repo, remote_name=None): + """Explain an empty catalog or absent remote without contacting the server.""" + remote = _select_remote_config(repo, remote_name) + template_urls = remote_name is None and any( + entry.is_remote for entry in catalog_entries(repo.state.config)) + if not template_urls and (not remote or not remote.get("url")): + click.echo("No data remote is configured; skipping download. " + "Configure one with hallmark set-config --remote-url URL.") + return False + if not _select_download_items(repo, all_files=True, catalog_only=True): + click.echo("The catalog is empty; nothing to download.") + return False + return True + + +def _interactive_download(repo, *, output=None, remote_name=None, max_workers=4, + dry_run=False): + """Select and approve cataloged files in a terminal, without network preflight.""" + if not sys.stdin.isatty(): + raise ClickException( + "--interactive requires a terminal. Use --include 'PATTERN' --dry-run " + "to preview a selection; run in a terminal to approve a download.") + if not _download_available(repo, remote_name): + return + click.echo("Catalog ready. Downloading dataset files can consume substantial " + "bandwidth and disk space. Nothing transfers until you approve " + "a plan. Size estimates use recorded catalog metadata.") + try: + plan = _choose_download_plan(repo, output=output, remote_name=remote_name, + dry_run=dry_run) + except click.Abort as exc: + # Click wraps both EOF and Ctrl+C in Abort. EOF skips this optional + # step; retain Click's interrupt behavior for Ctrl+C. + if not isinstance(exc.__context__, EOFError): + raise + plan = None + if plan is None: + click.echo("No download started; the catalog is unchanged.") + return + # Outside the prompt handler: transfer errors must never become a skip. + _execute_download(repo, plan, max_workers=max_workers) + + +def _offer_download(repo, *, includes=(), output=None, interactive=False): + """Review downloads after cloning, or print equivalent commands for later.""" + if not sys.stdin.isatty(): + click.echo("Warning: download approval is unavailable without " + "a terminal. The catalog is ready; download was skipped.", err=True) + try: + _download_available(repo) + except (DownloadError, ValueError) as exc: + # A skipped handoff cannot fail successful catalog creation, even + # when later downloading will require extra configuration. + click.echo(f"Before downloading: {exc}", err=True) + directory = repo.worktree if repo.worktree is not None else repo.dothm.path + arguments = ["hallmark", "download"] + if includes: + for pattern in includes: + arguments.extend(["--include", pattern]) + else: + arguments.append("--all") + if output is not None: + # The example changes directory. Preserve --output's meaning relative + # to the original invocation, as plan_download does. + arguments.extend(["--output", str(Path(output).expanduser().resolve())]) + command = shlex.join(arguments) + needs_output = repo.worktree is None and output is None + if needs_output: + command += " --output '/path/to/downloads'" + click.echo("Preview this selection locally; use --include 'PATTERN' " + "instead of --all to narrow it:") + click.echo(f" cd -- {shlex.quote(str(directory))}") + click.echo(f" {command} --dry-run") + click.echo("Run the download in a terminal; explicit approval is required:") + click.echo(f" {command}") + if needs_output: + click.echo("Replace /path/to/downloads with your output directory.") + return + with _translate_cli_errors(DownloadError, ValueError, OSError, + prefix="Catalog created; download failed"): + if interactive: + _interactive_download(repo, output=output) + return + if not _download_available(repo): + return + click.echo("Catalog ready. Review the recorded sizes before downloading; " + "transfers can consume substantial bandwidth and disk space.") + try: + output = _download_output(repo, output) + plan = repo.plan_download(output, all_files=True, include=includes or None) + approved = _confirm_download_plan( + plan, decline_is_skip=True, show_paths=True) + except click.Abort as exc: + if not isinstance(exc.__context__, EOFError): + raise + approved = False + if not approved: + click.echo("No download started; the catalog is unchanged.") + return + # Keep transfer errors and interrupts outside the prompt's EOF handler. + _execute_download(repo, plan, max_workers=4) + + @click.group() @click.version_option() @click.pass_context @@ -178,29 +326,80 @@ def hallmark(ctx): manage data products in a complex workflow. """ # if the invoked subcommand is one of the commands that does not require a repo - if ctx.invoked_subcommand in [None, "init", "clone", "build"]: + if ctx.invoked_subcommand in [ + None, "init", "clone", "build", "sources", "ls-remote"]: # return early without attempting to open a repository return # attempt to open the hallmark repository in the current directory - with _translate_cli_errors(GitError, prefix="Failed to open hallmark repository"): + with _translate_cli_errors( + GitError, ValueError, prefix="Failed to open hallmark repository"): ctx.obj = Repo(".") -@hallmark.command(short_help="Initialize a hallmark repository.") +@hallmark.command(short_help="List named data sources and available collections.") +@click.argument("name", required=False) +def sources(name): + """Show supported sources, releases and collection roots without crawling.""" + with _translate_cli_errors(ValueError): + descriptors = [get_source(name)] if name else list_sources() + for source in descriptors: + click.echo(f"{source.name}: {source.description}") + for release, definition in source.releases.items(): + click.echo(f" {release}: {definition.description or release}") + click.echo(f" {definition.url}") + if name: + for collection, roots in definition.collections.items(): + description = definition.collection_descriptions.get( + collection, "") + click.echo(f" {collection}: {description}") + for root in roots: + click.echo(f" {definition.url.rstrip('/')}/{root}/") + if name: + click.echo(" Run hallmark ls-remote URL to find filename templates, " + "then hallmark add 'URL/TEMPLATE'.") + + +@hallmark.command(short_help="Initialize an empty local repository.") @click.argument("path") def init(path): - """Initialize a hallmark repository at PATH. - - If PATH ends with `.hm`, a bare repository is created. - Otherwise, a `.hm` directory is created inside PATH. + """Initialize an empty hallmark repository at PATH. + + Plain init creates an empty repository without scanning existing files. + Catalog files afterwards with hallmark add: a filename template such as + `a{a}_i{i}.h5` tracks local files, and a URL template such as + `https://host/ER2/{src}_{day}.h5` catalogs remote files without + downloading them. + A PATH ending in .hm creates a bare catalog; otherwise metadata lives in PATH/.hm. """ - # attempt to initialize the hallmark repository at the specified path with _translate_cli_errors( - GitError, - prefix=("Failed to initialize hallmark repository " f'at "{path}"')): + GitError, ValueError, OSError, yaml.YAMLError, + prefix=f'Failed to initialize hallmark repository at "{path}"'): Repo.init(path) +@hallmark.command("ls-remote", + short_help="List a remote directory and suggest templates.") +@click.argument("url") +def ls_remote(url): + """List the files at a remote directory URL and suggest URL templates. + + Nothing is downloaded and no repository is changed or required. Each + suggestion is a command that catalogs the matching files. + """ + from .repo_remote import suggest_templates + + with _translate_cli_errors(DownloadError, ValueError): + base, suggestions, total = suggest_templates(url, progress=True) + click.echo(f"{total} file(s) at {base}") + if not suggestions: + click.echo("No filename templates detected. Write one with {field} " + "placeholders and preview it with hallmark add -n 'URL/TEMPLATE'.") + return + click.echo("Suggested templates (files matched):") + for template, count in suggestions: + click.echo(f" hallmark add {shlex.quote(template)} ({count})") + + @hallmark.command(short_help="Show information of the current directory.") @click.pass_obj def info(repo): @@ -229,7 +428,7 @@ def status(repo): untracked = snapshot["untracked"] def emit_section(title, entries, fg): - if not entries: + if not any(paths for _, paths in entries): return click.echo("") click.secho(title, fg=fg) @@ -237,6 +436,13 @@ def emit_section(title, entries, fg): for path in paths: click.echo(" " + click.style(f"{label}: {path}", fg=fg)) + catalog_lines = [] + for change in staged.get("catalog", []): + source = f' from {change["url"]}' if change["url"] else "" + catalog_lines.append( + f'{", ".join(change["templates"])}{source}: ' + f'{change["added"]} new, {change["modified"]} modified, ' + f'{change["deleted"]} removed') emit_section( "Changes to be committed:", [ @@ -244,6 +450,7 @@ def emit_section(title, entries, fg): ("new file", staged["added"]), ("modified", staged["modified"]), ("deleted", staged["deleted"]), + ("catalog", catalog_lines), ], "green", ) @@ -275,33 +482,87 @@ def emit_section(title, entries, fg): default=False, show_default=True, help="Enable regex-based encoding rules from config.yml.") +@click.option("-n", "--dry-run", is_flag=True, + help="List the matching files without staging them.") @click.argument("inputs", nargs=-1, required=True) @click.pass_obj -def add(repo, encoding, inputs): +def add(repo, encoding, dry_run, inputs): """Add files to the hallmark index. - `hallmark add [--regex] FORMAT` uses the branch format string workflow. - `hallmark add "."` rebuilds the manifest from current files that match - the branch `fmt` in `config.yml`. - Explicit path inputs such as shell-expanded `*` are not supported yet - with the parameter-based manifest format. + `hallmark add [--regex] FORMAT` stages the files matching a filename + template such as `a{a}_i{i}.h5`. Each new template is tracked alongside + the existing ones; `hallmark rm --cached FORMAT` stops tracking one. + A new template must match at least one file. + `hallmark add "."` rescans every local template within the current + directory. Explicit path inputs such as shell-expanded `*` are not + supported yet with the parameter-based manifest format. + + `hallmark add 'URL/{field}...'` catalogs remote files matching a URL + template, such as `https://host/ER2/{src}_{day}.h5`, without downloading + them. The URL up to the first segment with a field is recorded as the + template's source. Adding it again syncs the catalog with the remote + directory. Use `hallmark ls-remote URL` to find templates. """ + remote = len(inputs) == 1 and is_remote_url(inputs[0]) # attempt to add the specified files to the hallmark index, handling any errors - with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError): + with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError, + DownloadError): # if there is only one input, use the add method for a single input if len(inputs) == 1: - pf = repo.add(inputs[0], encoding) + pf = repo.add(inputs[0], encoding, dry_run=dry_run, + **({"progress": True} if remote else {})) # oterhwise, use the add_paths method for multiple inputs else: pf = repo.add_paths(list(inputs)) + if remote: + _report_remote_add(pf, dry_run=dry_run) + return if pf.empty: click.echo("No files matched the format string.") else: - click.echo("Changes to be committed") + click.echo("Would add" if dry_run else "Changes to be committed") click.echo(pf.path.to_string(index=False, header=False)) +def _report_remote_add(pf, *, dry_run): + """Summarize a remote catalog update without listing every file.""" + paths = pf["path"].tolist() + click.echo("Would add" if dry_run else "Changes to be committed") + for path in paths[:20]: + click.echo(path) + if len(paths) > 20: + click.echo(f"... {len(paths) - 20} more file(s)") + mismatched = pf.attrs.get("mismatched", 0) + if mismatched: + click.echo(f"Skipped {mismatched} listed file(s) whose names the template " + "cannot re-create exactly.") + if not dry_run: + click.echo(f"Cataloged {len(paths)} remote file(s); nothing was downloaded. " + "Run hallmark commit -m MESSAGE, then hallmark download to " + "fetch files.") + + +@hallmark.command(short_help="Stop tracking a template.") +@click.option("--cached", is_flag=True, + help="Remove the template from the catalog and keep its files.") +@click.argument("template") +@click.pass_obj +def rm(repo, cached, template): + """Stop tracking TEMPLATE, like `git rm --cached`. + + The template's catalog is removed from the index; files are left in + place. TEMPLATE is the tracked filename template, or the full URL + template of a remote one. + """ + if not cached: + raise ClickException( + "hallmark rm requires --cached; files are never deleted") + with _translate_cli_errors(RuntimeError, ValueError, GitError): + entry = repo.rm_cached(template) + click.echo(f"Stopped tracking {entry.fmt or entry.db}; files were left in place.") + + @hallmark.command("set-config", short_help="Update hallmark branch config.") @click.option("--fmt") @click.option("--remote-name") @@ -312,7 +573,8 @@ def set_config(repo, fmt, remote_name, remote_url, encodings): """Update the current branch config.yml.""" # if no config changes are requested, raise a ClickException to inform the user if ( - fmt is None and remote_name is None and remote_url is None and not encodings): + fmt is None and remote_name is None and remote_url is None + and not encodings): raise ClickException("No config changes requested.") encoding_updates = {} for item in encodings: @@ -324,7 +586,7 @@ def set_config(repo, fmt, remote_name, remote_url, encodings): encoding_updates[field.strip()] = regex # use the _translate_cli_errors context manager to handle specific exceptions - with _translate_cli_errors(RuntimeError, ValueError, FileNotFoundError): + with _translate_cli_errors(RuntimeError, ValueError, OSError, yaml.YAMLError): repo.set_config( fmt=fmt, remote_name=remote_name, @@ -400,190 +662,111 @@ def checkout(repo, target_branch): click.echo(f'Switched to branch "{target_branch}".') -@hallmark.command( - short_help="Download files from the configured data remote.") +@hallmark.command(short_help="Download files from the configured data remote.") @click.argument("files", nargs=-1) -@click.option( - "--tsv", - "tsv_names", - multiple=True, - help="Download every file represented by a configured TSV. " - "May be repeated.") -@click.option( - "--all", - "download_all", - is_flag=True, - help="Download every configured TSV, static file, and metadata file.") -@click.option( - "--remote", - "remote_name", - help="Name of the configured remote to use.") -@click.option( - "--output", - type=click.Path(file_okay=False), - help="Output directory. Defaults to the repository worktree.") -@click.option( - "--max-workers", - type=click.IntRange(min=1), - default=4, - show_default=True, - help="Number of concurrent downloads.") -@click.option( - "--dry-run", - is_flag=True, - help="Show the selected files without downloading them.") -@click.option( - "-y", - "--yes", - is_flag=True, - help="Skip the bulk-download confirmation.") +@click.option("--tsv", "tsv_names", multiple=True, + help="Select a catalog TSV. May be repeated.") +@click.option("--all", "download_all", is_flag=True, + help="Select all cataloged files.") +@click.option("--include", "includes", multiple=True, + help="Select paths matching a glob. ** matches recursively. " + "May be repeated.") +@click.option("--remote", "remote_name", + help="Name of the configured data remote to use.") +@click.option("--output", type=click.Path(file_okay=False), + help="Output directory. Defaults to the repository worktree.") +@click.option("--max-workers", type=click.IntRange(min=1), default=4, + show_default=True) +@click.option("--dry-run", is_flag=True, + help="Show the download plan using only local catalog metadata.") +@click.option("--interactive", is_flag=True, + help="Choose cataloged files and review sizes before approval.") +@click.option("-y", "--yes", is_flag=True, hidden=True) @click.pass_obj -def download(repo, files, tsv_names, download_all, remote_name, output, max_workers, - dry_run, yes): +def download(repo, files, tsv_names, download_all, includes, remote_name, + output, max_workers, dry_run, interactive, yes): """ - Download files from the configured data remote. Options allow for selecting - specific files, TSVs, or downloading all files. Supports concurrent downloads - and dry-run mode for previewing selected files. - - Options: - --tsv: Download every file represented by a configured TSV. May be repeated. - --all: Download every configured TSV, static file, and metadata file. - --remote: Name of the configured remote to use. - --output: Output directory. Defaults to the repository worktree. - --max-workers: Number of concurrent downloads. Default is 4. - --dry-run: Show the selected files without downloading them. - -y, --yes: Skip the bulk-download confirmation. + Download selected files from a configured data remote. - Raises: - ClickException: If there are any issues with the provided arguments or - during the download process. + Preview the selection with --dry-run. Every nonempty download displays + its plan and asks for confirmation before transferring files. + Use --interactive to choose patterns or all files in a terminal. """ - # do not allow --all to be combined with file paths or --tsv + if interactive: + if files or tsv_names or download_all or includes: + raise ClickException( + "--interactive cannot be combined with paths, --include, " + "--tsv, or --all") + if yes: + click.echo("--yes is deprecated; downloads still require confirmation.", + err=True) + with _translate_cli_errors(DownloadError, ValueError, OSError): + _interactive_download(repo, output=output, remote_name=remote_name, + max_workers=max_workers, dry_run=dry_run) + return if download_all and (files or tsv_names): raise ClickException("--all cannot be combined with file paths or --tsv") - # require at least one of file paths, --tsv, or --all to be provided - if not files and not tsv_names and not download_all: - raise ClickException("Provide one or more file paths, --tsv, or --all") - - # if an output directory is specified, use it; otherwise, use the repo worktree - if output: - output_path = Path(output).expanduser() - elif repo.worktree is not None: - output_path = Path(repo.worktree) - # output is required when downloading from a bare .hm repository - else: - raise ClickException( - "--output is required when downloading from a bare .hm repository") - - # use the _translate_cli_errors context manager to handle DownloadError exceptions - with _translate_cli_errors(DownloadError): - selected_files = select_download_files( - repo, - file_paths=files, - tsv_names=tsv_names, - all_files=download_all) - click.echo(f"Selected {len(selected_files)} file(s) "f"for {output_path}") - - # don't actually download the files if --dry-run is specified + if not files and not tsv_names and not download_all and not includes: + raise ClickException("Provide file paths, --tsv, --all, or --include") + if repo.worktree is None and output is None: + raise ClickException("--output is required when downloading from a bare repo") + if yes: + click.echo("--yes is deprecated; downloads still require confirmation.", + err=True) + with _translate_cli_errors(DownloadError, ValueError): + plan = repo.plan_download( + output, file_paths=files, tsv_names=tsv_names, all_files=download_all, + include=includes or None, remote_name=remote_name) if dry_run: - # limit the number of files to preview to avoid overwhelming the user - preview_limit = 20 - # for each selected file, print its relative path to the user - for rel_path, _sha1 in selected_files[:preview_limit]: - click.echo(f" {rel_path.as_posix()}") - - remaining = len(selected_files) - preview_limit - # if there are more files than the preview limit - if remaining > 0: - # indicate how many more files are selected - click.echo(f" ... {remaining} more file(s)") - # bail out of the function early since this is a dry run + _show_download_plan(plan, show_paths=True) return - - if not selected_files: - click.echo("No files selected for download.") - # bail out of the function early since there are no files to download - return - - # attempt to download the selected files, handling any errors - _run_download( - repo, - selected_files, - output_path, - max_workers=max_workers, - assume_yes=yes, - remote_name=remote_name) + _run_download(repo, plan, max_workers=max_workers) -@hallmark.command(short_help="Clone a hallmark repository from a remote URL.") +@hallmark.command(short_help="Clone an existing Hallmark catalog.") @click.argument("url") @click.argument("path") -@click.option( - "--no-fetch-data", - is_flag=True, - help="Skip downloading remote data files after clone.") -@click.option( - "--max-workers", - type=click.IntRange(min=1), - default=4, - show_default=True, - help="Number of concurrent downloads.") -@click.option( - "-y", - "--yes", - is_flag=True, - help="Skip the bulk-download confirmation.") -def clone(url, path, no_fetch_data, max_workers, yes): - """ - Clone a hallmark repository from a remote URL to the specified path. - - Options: - --no-fetch-data: Skip downloading remote data files after clone. - --max-workers: Number of concurrent downloads. Default is 4. - -y, --yes: Skip the bulk-download confirmation. - - Raises: - ClickException: If there are any issues with the provided arguments or - during the clone process, such as a destination already existing, - a clone error, or a download error. +@click.option("--source-type", default="auto", show_default=True, + type=click.Choice(["auto", "git", "catalog"]), + help="Override automatic source detection.") +@click.option("--no-download", is_flag=True, + help="Copy the complete catalog without reviewing or downloading files.") +@click.option("--include", "includes", multiple=True, + help="Select downloads by relative-path glob; keep the full catalog. " + "May be repeated.") +@click.option("--interactive", is_flag=True, + help="Choose files interactively instead of reviewing the whole catalog.") +@click.option("--output", type=click.Path(file_okay=False), + help="Download directory. Defaults to the worktree; " + "prompted for bare repos.") +def clone(url, path, source_type, no_download, includes, interactive, output): + """Clone a complete Git catalog or published catalog snapshot at PATH. + + Copy the complete catalog first, then review a download plan and confirm + transfers. --include selects downloads without trimming the catalog. + --no-download skips review; --interactive opens the selection chooser. + Without a terminal, keep the catalog and print download commands for later. + For raw datasets, use init, then add a URL template. """ - # use context manager to handle DownloadError and GitError exceptions - with _translate_cli_errors(DownloadError, GitError): + if no_download and (interactive or includes or output is not None): + raise ClickException( + "--no-download cannot be combined with --interactive, " + "--include, or --output") + if interactive and includes: + raise ClickException("--interactive cannot be combined with --include") + with _translate_cli_errors(DownloadError, GitError, ValueError): try: - repo = Repo.clone(url, path, fetch_data=False) - # handle CloneError exceptions and provide a user-friendly error message + repo = Repo.clone(url, path, source_type=source_type, progress=True) except CloneError as exc: click.echo(str(exc), err=True) raise SystemExit(1) from exc click.echo(f'Successfully cloned to "{path}"') - # if the user has not opted to skip data fetching - if not no_fetch_data: - # get the worktree path of the cloned repository - _, worktree_path = Repo.lwpaths(path) - if worktree_path is None: - click.echo( - "Bare repository clone; skipping data download.") - # bail out of the function early since there is no worktree - return - - # select the files to download from the cloned repository - selected_files = select_download_files(repo, all_files=True) - # if there are no selected files, inform the user and exit - if not selected_files: - click.echo("No remote data files are configured.") - return - - # confirm with user if the num of selected files exceeds warning threshold - click.echo("Downloading remote data files...") - _run_download( - repo, - selected_files, - worktree_path, - max_workers=max_workers, - assume_yes=yes) - - -@hallmark.command(short_help="Build a hallmark repository from a remote dataset.") + if not no_download: + _offer_download(repo, includes=includes, output=output, + interactive=interactive) + + +@hallmark.command(short_help="Deprecated: use init, then add a URL template.") @click.argument("directory") @click.argument("dataset_name") @click.option( @@ -594,43 +777,41 @@ def clone(url, path, no_fetch_data, max_workers, yes): "--config-file", "config_file", type=click.Path(exists=True, dir_okay=True, file_okay=True), help="Path to config.yml or a repository directory containing config.yml. " - "build_repo loads fmts (and remotes unless --remote is provided).") + "Load formats and, unless --remote is given, data remotes.") @click.option( "--fmt", "fmts", multiple=True, - help="A fmt entry to use directly, as FMT=DB (e.g. " - "'a{a}_i{i}.h5=data.tsv'). May be repeated for multiple fmts; " - "skips the prompt entirely.") + help="Filename format and TSV name, as FMT=DB (e.g. " + "'a{a}_i{i}.h5=data.tsv'). May be repeated for multiple formats.") @click.option( "--overwrite", is_flag=True, help="Replace the destination repository if it already exists.") -def build(directory, dataset_name, remotes, config_file, fmts, overwrite): +@click.option("--dataset-url", + help="Exact dataset URL to search recursively. Recorded remotes " + "do not change this root.") +@click.option("--index-format", type=click.Choice(["cyverse-html"]), hidden=True) +@click.option("--allow-remote-commands", is_flag=True, + hidden=True, help="Deprecated; discovery uses SFTP.") +@click.option("--remote-hash", is_flag=True, + hidden=True, help="Unsupported; discovery does not hash dataset files.") +def build(directory, dataset_name, remotes, config_file, fmts, overwrite, + dataset_url, index_format, allow_remote_commands, remote_hash): """ - Build a hallmark repository at DIRECTORY for the remote dataset DATASET_NAME. - - The dataset is fetched from the remote index and stored in a new hallmark - repository at DIRECTORY. The remotes can be specified with - --remote NAME=URL or --remote NAME. - if no remotes are specified, the default remote from the dataset index will be used. - --config-file: Optional path to an existing config.yml to load fmts and remotes - --fmt: Optional fmt entries to use directly, specified as FMT=DB. May be repeated - for multiple fmts. - --overwrite: Optional flag to replace the destination repo if it already exists. - - Arguments: - - DIRECTORY: The file system path where the hallmark repository will be created. - DATASET_NAME: The name of the remote dataset to fetch. - --remote: Optional remote(s) to record, specified as NAME=URL or just NAME. - May be repeated for multiple remotes. - --config-file: Optional path to an existing config.yml to load fmts and remotes. - --fmt: Optional fmt entries to use directly, specified as FMT=DB. + Build a catalog at DIRECTORY/DATASET_NAME.hm. - Raises: - ClickException: If there is an error during the build process, such as - a network error, Git error, or invalid dataset name. + This command is deprecated. Use hallmark init PATH, then + hallmark add 'URL/{field}...' to catalog remote datasets. + + --dataset-url selects the exact discovery root; otherwise DATASET_NAME + identifies a dataset beneath the CyVerse curated-data directory. + --remote records data locations without changing the discovery root. + Supply filename formats with --fmt or --config-file. If neither is + provided, existing formats are reused or a generic path catalog is + created. Dataset files are not downloaded during catalog creation. """ + click.echo("build is deprecated; use hallmark init PATH, then " + "hallmark add 'URL/{field}...'.", err=True) if config_file and fmts: raise ClickException("Use only one of --config-file or --fmt, not both.") # validate the dataset name to ensure it is a valid path component @@ -673,6 +854,10 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite): # if the checks pass, append the fmt and db to the fmt_entries list fmt_entries.append({"fmt": fmt, "db": db}) + source_options = {key: value for key, value in { + "dataset_url": dataset_url, + "index_format": index_format, "allow_remote_commands": allow_remote_commands, + "remote_hash": remote_hash}.items() if value is not None and value is not False} # build the hallmark repository with the specified parameters try: with _translate_cli_errors(*_BUILD_DATASET_ERRORS): @@ -682,9 +867,9 @@ def build(directory, dataset_name, remotes, config_file, fmts, overwrite): fmt_entries=fmt_entries, config_file=config_file, remotes=parsed_remotes or None, - overwrite=overwrite) + overwrite=overwrite, **source_options) except requests.exceptions.RequestException as exc: raise ClickException( f"Failed to reach dataset {dataset_name!r}: {exc}") from exc - click.echo(f'Successfully built hallmark repository at "{repo_path}".') \ No newline at end of file + click.echo(f'Successfully built hallmark repository at "{repo_path}".') diff --git a/mod/hallmark/discovery.py b/mod/hallmark/discovery.py new file mode 100644 index 0000000..e0648f4 --- /dev/null +++ b/mod/hallmark/discovery.py @@ -0,0 +1,345 @@ +"""Discover remote file metadata without downloading dataset contents.""" + +from __future__ import annotations + +import fnmatch +import re +from dataclasses import replace +from functools import lru_cache +from pathlib import PurePosixPath +from string import Formatter +from typing import Optional +from urllib.parse import quote, unquote, urlsplit, urlunsplit + +import parse +from tqdm import tqdm + +from .helper_functions import CHECKSUM_ALGORITHMS_BY_STRENGTH, valid_checksum +from .repo_config import RESERVED_FIELDS +from .transport.base import ( + DownloadError, RemoteEntry, RemoteObjectMissing, literal_path, reject_controls, +) + + +def _glob_matches(path, pattern): + """Match path segments, including zero-directory matches for ``**/``.""" + parts, patterns = path.split("/"), pattern.split("/") + + @lru_cache(maxsize=None) + def match(path_index, pattern_index): + if pattern_index == len(patterns): + return path_index == len(parts) + if patterns[pattern_index] == "**": + return (match(path_index, pattern_index + 1) + or path_index < len(parts) and match(path_index + 1, pattern_index)) + return (path_index < len(parts) + and fnmatch.fnmatchcase(parts[path_index], patterns[pattern_index]) + and match(path_index + 1, pattern_index + 1)) + + return match(0, 0) + + +def _template_parts(segment: str): + """Split one template segment into literal text and field definitions.""" + try: + return list(Formatter().parse(segment)) + except ValueError as exc: + raise ValueError(f"Invalid filename template segment {segment!r}: {exc}") \ + from None + + +def _escape_braces(text: str) -> str: + return text.replace("{", "{{").replace("}", "}}") + + +def is_remote_url(value) -> bool: + """Return True for a string that names a remote location, such as a URL.""" + return isinstance(value, str) and "://" in value + + +def split_url_template(url: str) -> tuple[str, Optional[str]]: + """ + Split a URL template at the first path segment containing a field. + + The part before that segment is the remote directory; the rest is a + filename template relative to it. Literal parts of the template are + percent-decoded once, so ``%20`` names a space and ``%7B`` a literal brace. + A URL without fields names one file, unless it ends in ``/``. + + Args: + url (str): URL such as ``https://host/ER2/{src}_{day}.h5``. + + Returns: + tuple[str, str | None]: The directory URL, ending in ``/``, and the + template; the template is None for a directory URL. + + Raises: + ValueError: If the URL has credentials, a query or fragment, fields + outside its path, or an invalid template. + """ + if not is_remote_url(url): + raise ValueError("Expected a URL such as https://host/path/{field}.ext") + reject_controls(url, "URL") + parsed = urlsplit(url) + if any(brace in parsed.scheme + parsed.netloc for brace in "{}"): + raise ValueError("Template fields are allowed only in the URL path") + if "?" in url or "#" in url: + raise ValueError("Remote URLs must not contain a query or fragment") + if parsed.password is not None or ( + parsed.username is not None and parsed.scheme in {"http", "https"}): + raise ValueError( + "Remote URLs must not contain credentials; configure them locally, " + "for example in ~/.netrc or ~/.ssh/config") + segments = parsed.path.split("/") + index = next((position for position, segment in enumerate(segments) + if any(name is not None + for _, name, _, _ in _template_parts(segment))), None) + if index is None: + if not parsed.path.strip("/") or parsed.path.endswith("/"): + return url, None + index = len(segments) - 1 + base = urlunsplit((parsed.scheme, parsed.netloc, + "/".join(segments[:index]) + "/", "", "")) + decoded = [] + for segment in segments[index:]: + text = [] + for literal, name, spec, conversion in _template_parts(segment): + text.append(_escape_braces(unquote(literal))) + if name is not None: + if conversion is not None: + raise ValueError("Template fields cannot use conversions") + text.append("{" + name + (f":{spec}" if spec else "") + "}") + decoded.append("".join(text)) + return base, "/".join(decoded) + + +def template_url(base: str, template: str) -> str: + """Join a directory URL and a template, percent-encoding literal text.""" + segments = [] + for segment in template.split("/"): + text = [] + for literal, name, spec, _ in _template_parts(segment): + text.append(quote(literal, safe="")) + if name is not None: + text.append("{" + name + (f":{spec}" if spec else "") + "}") + segments.append("".join(text)) + return base.rstrip("/") + "/" + "/".join(segments) + + +class TemplateMatcher: + """ + Match relative paths against a filename template, one segment at a time. + + Fields never span ``/``, and a path segment starting with ``.`` matches + only a template segment starting with ``.``, as with local glob patterns. + + Args: + template (str): Filename template such as ``{src}/{day}.h5``. + + Raises: + ValueError: If the template is invalid or uses reserved field names. + """ + + def __init__(self, template: str): + if not isinstance(template, str) or not template.strip(): + raise ValueError("format must be a nonempty filename template") + segments = template.split("/") + if any(segment in {"", ".", ".."} for segment in segments): + raise ValueError( + f"Template {template!r} must be a relative path naming files") + self.template = template + self.fields = [] + self._parsers, self._hidden, globs = [], [], [] + for segment in segments: + stripped, pattern = [], [] + for literal, name, _, conversion in _template_parts(segment): + stripped.append(_escape_braces(literal)) + pattern.append(re.sub(r"([\[\]*?])", r"[\1]", literal)) + if name is None: + continue + if not name.isidentifier() or conversion is not None: + raise ValueError( + "Template fields must be simple names without conversions") + if name in RESERVED_FIELDS: + raise ValueError( + f"Template field {name!r} is reserved catalog metadata") + if name not in self.fields: + self.fields.append(name) + stripped.append("{" + name + "}") + pattern.append("*") + self._parsers.append(parse.compile("".join(stripped), case_sensitive=True)) + self._hidden.append(segment.startswith(".")) + globs.append(re.sub(r"\*+", "*", "".join(pattern))) + self.glob = "/".join(globs) + + def _match_segment(self, index: int, part: str) -> Optional[dict]: + if part.startswith(".") and not self._hidden[index]: + return None + result = self._parsers[index].parse(part) + return None if result is None else dict(result.named) + + def match(self, path: str) -> Optional[dict]: + """Return the field values of a matching path, or None.""" + parts = path.split("/") + if len(parts) != len(self._parsers): + return None + values = {} + for index, part in enumerate(parts): + named = self._match_segment(index, part) + if named is None: + return None + for name, value in named.items(): + if values.setdefault(name, value) != value: + return None + return values + + def may_contain(self, directory: str) -> bool: + """Return True if files under a relative directory could match.""" + parts = [part for part in directory.split("/") if part] + return len(parts) < len(self._parsers) and all( + self._match_segment(index, part) is not None + for index, part in enumerate(parts)) + + +def path_matches(path: str, include=None) -> bool: + """Match a relative path against any supplied case-sensitive inclusion glob. + + ``**`` spans zero or more directories. With no patterns every path matches. + """ + if include is None: + return True + if not isinstance(include, (str, list, tuple)): + raise ValueError("include must be a glob string or a list of glob strings") + patterns = [include] if isinstance(include, str) else list(include) + if not all(isinstance(pattern, str) and pattern for pattern in patterns): + raise ValueError("Inclusion patterns must be nonempty glob strings") + return any(_glob_matches(str(path), pattern) for pattern in patterns) + + +def _manifest_algorithm(path): + """Identify checksum manifests by their conventional filenames.""" + name = PurePosixPath(path).name.lower() + if re.fullmatch(r"checksums?(?:\.txt)?", name): + return "unknown" + for algorithm in CHECKSUM_ALGORITHMS_BY_STRENGTH: + if (re.fullmatch(rf"{algorithm}sums?(?:\.txt)?", name) + or re.fullmatch(rf".+\.{algorithm}sums?", name)): + return algorithm + return None + + +def _manifest_checksums(context, entries): + """Read published checksum metadata and attach it to discovered paths.""" + strength = {name: index for index, name in + enumerate(CHECKSUM_ALGORITHMS_BY_STRENGTH)} + for path in sorted(entries): + algorithm = _manifest_algorithm(path) + if algorithm is None: + continue + try: + text = context.read_text(path) + except RemoteObjectMissing: + continue + parent = PurePosixPath(path).parent + for line in text.splitlines(): + if line.startswith("\\"): + raise DownloadError("GNU escaped manifest filenames are unsupported") + match = re.fullmatch(r"([0-9a-fA-F]{8,}) [ *](.+)", line) + if not match: + continue + digest, filename = match.groups() + if not valid_checksum(algorithm, digest, allow_unknown_algorithm=True): + continue + while filename.startswith("./"): + filename = filename[2:] + filename = literal_path(filename).as_posix() + candidates = [(parent / filename).as_posix(), filename] + if parent.name == PurePosixPath(filename).parts[0]: + candidates.append((parent.parent / filename).as_posix()) + target = next((candidate for candidate in candidates + if candidate in entries), None) + if target is None: + continue + previous = entries[target] + if (previous.checksum_algorithm == algorithm and previous.checksum + and previous.checksum.lower() != digest.lower()): + raise DownloadError(f"Conflicting {algorithm} manifests for {target!r}") + if (previous.checksum is None + or strength.get(algorithm, 99) + < strength.get(previous.checksum_algorithm, 100)): + entries[target] = replace( + previous, checksum_algorithm=algorithm, checksum=digest.lower()) + + +def discover(context, *, include=None, progress=False, + descend=None) -> list[RemoteEntry]: + """ + Discover remote files and their published metadata recursively. + + Directory listings and checksum manifests are read before filtering. + Dataset files are not downloaded to compute missing checksums. + + Args: + context (OperationContext): Source connection and cancellation state. + include (str | list[str], optional): Relative path glob or globs. + descend (callable, optional): Predicate receiving a relative directory + path ending in ``/``; listings skip directories for which it + returns False. Checksum manifests in skipped directories are + not read. + progress (bool | callable): True displays a progress bar with an + unknown total. A callback receives dictionaries containing + ``directories``, ``files``, ``matched``, and ``current``. + Defaults to False. + + Returns: + list[RemoteEntry]: Matching files sorted by relative path. + Unavailable sizes, modification times, and checksums remain None. + + Raises: + ValueError: If an inclusion pattern is invalid. + CapabilityError: If the source provides no supported listing. + DownloadError: If discovery fails, paths are unsafe, or manifests conflict. + """ + # Validate selectors before contacting the source, even for an empty index. + path_matches("", include=include) + def matches(path): + return path_matches(path, include=include) + + entries = {} + counts = {"directories": 0, "files": 0, "matched": 0, "current": ""} + bar = tqdm(total=None, unit="dir", desc="Discovering", disable=not bool(progress)) \ + if not callable(progress) else None + + def report(directory=None): + if directory is not None: + counts["directories"] += 1 + counts["current"] = directory or "/" + if bar is not None: + bar.update() + if callable(progress): + progress(dict(counts)) + elif bar is not None: + bar.set_postfix(files=counts["files"], matched=counts["matched"]) + + def add(entry): + path = literal_path(entry.path).as_posix() + if path not in entries: + entries[path] = entry + counts["files"] += 1 + counts["matched"] += int(matches(path)) + report() + + previous_descend = getattr(context, "descend", None) + context.descend = descend + try: + context.transport.prepare() + for entry in context.transport.iter_entries(on_directory=report): + context.check_cancelled() + add(entry) + _manifest_checksums(context, entries) + return [entries[path] for path in sorted(entries) + if matches(path)] + finally: + context.descend = previous_descend + if bar is not None: + bar.close() diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index fbbedf5..fba45bd 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -27,7 +27,8 @@ from .error import CloneError, DothmError from .helper_functions import ( atomic_output_path, load_yaml_file, validate_path_component) -from .state import State +from .repo_config import catalog_table_names +from .state import DEFAULT_DB, State class _HallmarkYamlDumper(yaml.Dumper): @@ -85,7 +86,8 @@ class Dothm(Repo): """Local ``.hm`` storage backend. The backend version controls the hallmark ``State`` database files - (``config.yml``, ``meta.yml``, ``data.tsv``) on-disk. + (``config.yml``, ``meta.yml``, ``data.tsv`` and the catalog TSVs of + additional data entries) on-disk. It is itself a git worktree. """ def _storage_path(self, stem: Union[Path, str], suffix: str) -> Path: @@ -153,6 +155,7 @@ def config_template() -> str: remote: # name: origin # url: https://example.com/path/to/data/ + # SSH URL example: ssh://campus/srv/export/ (requires a trusted host key) """ @classmethod @@ -194,16 +197,52 @@ def link(self, path: Union[Path, str], branch: Optional[str] = None): return Dothm(path) def load(self) -> State: - return State( - config = self.load_yml("config"), + config = self.load_yml("config") + state = State( + config = config, meta = self.load_yml("meta"), data = self.load_tsv("data")) + # additional data entries keep their catalogs in their own TSVs + for name in catalog_table_names(config): + if name != DEFAULT_DB: + state.tables[name] = self._load_optional_tsv(name) + return state def dump(self, state: State) -> None: + # write catalogs before the config that references them + names = [DEFAULT_DB] + self.dump_tsv(state.data, "data") + for name, frame in state.tables.items(): + if (name in state.changed_tables + or not self._storage_path(name, ".tsv").exists()): + self.dump_tsv(frame, name) + names.append(name) + for name in sorted(state.removed_tables - set(state.tables) - {DEFAULT_DB}): + self.index.remove([name], ignore_unmatch=True) + self._storage_path(name, ".tsv").unlink(missing_ok=True) self.dump_yml(state.config, "config") self.dump_yml(state.meta, "meta") - self.dump_tsv(state.data, "data") - self.index.add(["config.yml", "meta.yml", "data.tsv"]) + self.index.add(["config.yml", "meta.yml", *names]) + state.changed_tables.clear() + state.removed_tables.clear() + + def _load_optional_tsv(self, stem: Union[Path, str]) -> pd.DataFrame: + """ + Used by load. + Load a catalog TSV, returning an empty table when it has not been written. + + Args: + stem (Union[Path, str]): The TSV file name. + + Returns: + pd.DataFrame: The table, or an empty table with a ``sha1`` column. + """ + if not self._storage_path(stem, ".tsv").is_file(): + return State().data.copy() + try: + return self.load_tsv(stem) + except pd.errors.EmptyDataError: + return State().data.copy() def load_yml(self, stem: Union[Path, str]) -> dict: """ diff --git a/mod/hallmark/download_plan.py b/mod/hallmark/download_plan.py new file mode 100644 index 0000000..3008ace --- /dev/null +++ b/mod/hallmark/download_plan.py @@ -0,0 +1,246 @@ +"""Immutable descriptions of dataset transfers, separate from their approval.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from math import isfinite +from pathlib import Path +from typing import Optional, Union +from urllib.parse import urlsplit, urlunsplit + +from .transport.base import RemoteSpec, backend_name, freeze_backend_options + + +def _redacted_url(url: str) -> str: + """Return a URL without credentials, query parameters, or fragments.""" + parsed = urlsplit(url) + return urlunsplit(parsed._replace( + netloc=parsed.netloc.rsplit("@", 1)[-1], query="", fragment="")) + + +@dataclass(frozen=True) +class DownloadSource: + """ + The data location files are downloaded from. + + Attributes: + url (str): Directory URL that cataloged paths are relative to. + name (str, optional): Name of the configured data remote, if any. + backend (str, optional): Registered backend selected for transfer. + backend_options (mapping): Immutable backend configuration snapshot. + """ + + url: str = field(repr=False) + name: Optional[str] = None + backend: Optional[str] = None + backend_options: Mapping = field(default_factory=dict, repr=False, hash=False) + + def __post_init__(self) -> None: + """Validate the source fields and freeze the backend options.""" + if not isinstance(self.url, str) or not self.url: + raise TypeError("source url must be a nonempty string") + if any(value is not None and not isinstance(value, str) + for value in (self.name, self.backend)): + raise TypeError("source name and backend must be strings or None") + if self.backend is not None: + backend_name(self.backend) + object.__setattr__(self, "backend_options", + freeze_backend_options(self.backend_options)) + + @property + def display(self) -> str: + """The URL without credentials, query parameters, or fragments.""" + return _redacted_url(self.url) + + def remote(self) -> RemoteSpec: + """Parse the source into a remote specification for transfer.""" + return RemoteSpec.parse(self.url, backend=self.backend, + backend_options=self.backend_options) + + +@dataclass(frozen=True) +class DownloadItem: + """ + A file and its catalog metadata at the time a download is planned. + + Attributes: + relative_path (Path): File path relative to the data source. + checksum (str | tuple, optional): Digest or (algorithm, digest) pair. + size_bytes (int, optional): Recorded file size; None if unknown. + mtime (str, optional): Recorded modification time; None if unknown. + source (DownloadSource, optional): Where the file is downloaded from; + None uses the plan's source. + """ + + relative_path: Path + checksum: Optional[Union[str, tuple[str, str]]] = None + size_bytes: Optional[int] = None + mtime: Optional[str] = None + source: Optional[DownloadSource] = None + + def __post_init__(self) -> None: + """Normalize the path and validate optional file metadata.""" + object.__setattr__(self, "relative_path", Path(self.relative_path)) + if isinstance(self.checksum, list): + object.__setattr__(self, "checksum", tuple(self.checksum)) + if self.checksum is not None and not ( + isinstance(self.checksum, str) + or (isinstance(self.checksum, tuple) and len(self.checksum) == 2 + and all(isinstance(part, str) for part in self.checksum)) + ): + raise ValueError("checksum must be a string or algorithm/value pair") + if self.size_bytes is not None and ( + isinstance(self.size_bytes, bool) + or not isinstance(self.size_bytes, int) + or self.size_bytes < 0 + ): + raise ValueError("size_bytes must be a nonnegative integer or None") + if self.mtime is not None and not isinstance(self.mtime, str): + raise ValueError("mtime must be a string or None") + if self.source is not None and not isinstance(self.source, DownloadSource): + raise TypeError("source must be a DownloadSource or None") + + +@dataclass(frozen=True) +class DownloadPlan: + """ + An immutable selection of files, sources, and destination for a download. + + Creating a plan does not approve a transfer. Sizes come from the catalog; + a duration estimate requires every file size and a supplied transfer rate. + Files may come from several sources, such as templates added from + different URLs. The plan-level source fields describe the source of items + without their own source; plans with one source fill them in. + + Attributes: + items (tuple[DownloadItem]): Selected files and their metadata. + remote_url (str, optional): Data source URL; None for an empty plan or + one whose items come from several sources. + output_path (Path): Absolute destination directory. + remote_name (str, optional): Name of the selected data remote. + estimated_bytes_per_second (float, optional): Positive rate supplied + by the caller for duration estimates. + remote_backend (str, optional): Registered backend selected for transfer. + backend_options (mapping): Immutable backend configuration snapshot. + unsourced_count (int): Cataloged files skipped because no data source + is configured for them. + """ + + items: tuple[DownloadItem, ...] + remote_url: Optional[str] = field(repr=False) + output_path: Path + remote_name: Optional[str] = None + estimated_bytes_per_second: Optional[float] = None + remote_backend: Optional[str] = None + backend_options: Mapping = field(default_factory=dict, repr=False, hash=False) + unsourced_count: int = 0 + + def __post_init__(self) -> None: + """Copy the selection and validate the source fields and rate.""" + object.__setattr__(self, "items", tuple(self.items)) + object.__setattr__( + self, "output_path", Path(self.output_path).expanduser().absolute()) + if any(not isinstance(item, DownloadItem) for item in self.items): + raise TypeError("items must contain DownloadItem values") + if any(value is not None and not isinstance(value, str) for value in + (self.remote_url, self.remote_name, + self.remote_backend)): + raise TypeError("remote fields must be strings or None") + if self.remote_backend is not None: + backend_name(self.remote_backend) + object.__setattr__(self, "backend_options", + freeze_backend_options(self.backend_options)) + if (isinstance(self.unsourced_count, bool) + or not isinstance(self.unsourced_count, int) + or self.unsourced_count < 0): + raise ValueError("unsourced_count must be a nonnegative integer") + rate = self.estimated_bytes_per_second + if rate is not None and ( + isinstance(rate, bool) or not isinstance(rate, (int, float)) + or not isfinite(rate) or rate <= 0 + ): + raise ValueError("estimated_bytes_per_second must be a positive rate") + + @property + def default_source(self) -> Optional[DownloadSource]: + """The plan-level source, or None when the plan has none.""" + if not self.remote_url: + return None + return DownloadSource(self.remote_url, self.remote_name, + self.remote_backend, self.backend_options) + + def source_for(self, item: DownloadItem) -> Optional[DownloadSource]: + """Return the source an item is downloaded from.""" + return item.source or self.default_source + + @property + def sources(self) -> tuple[DownloadSource, ...]: + """Distinct sources of the selected files, in plan order.""" + sources = [] + for item in self.items: + source = self.source_for(item) + if source is not None and source not in sources: + sources.append(source) + return tuple(sources) + + @property + def file_count(self) -> int: + """Number of files selected for download.""" + return len(self.items) + + @property + def known_bytes(self) -> int: + """Sum of recorded sizes, excluding files whose sizes are unknown.""" + return sum(item.size_bytes or 0 for item in self.items) + + @property + def unknown_size_count(self) -> int: + """Number of files without a recorded size.""" + return sum(item.size_bytes is None for item in self.items) + + @property + def total_bytes(self) -> Optional[int]: + """Estimated transfer size, or None if any file size is unknown.""" + return None if self.unknown_size_count else self.known_bytes + + @property + def estimated_seconds(self) -> Optional[float]: + """Estimated duration, or None when sizes or the transfer rate are unknown.""" + if self.total_bytes is None or self.estimated_bytes_per_second is None: + return None + return self.total_bytes / self.estimated_bytes_per_second + + def summary(self) -> str: + """ + Describe the planned transfer, including unavailable estimates. + + Returns: + str: File count, size and duration estimates, source, and destination. + URL credentials, query parameters, and fragments are omitted. + """ + size = f"{self.known_bytes:,} bytes" + if self.unknown_size_count: + size += f" known; {self.unknown_size_count} file(s) with unknown size" + duration = ("unknown" if self.estimated_seconds is None + else f"about {self.estimated_seconds:.1f} seconds") + sources = self.sources + if len(sources) > 1: + counts = {source: 0 for source in sources} + for item in self.items: + counts[self.source_for(item)] += 1 + lines = "".join( + f"\n {source.display} ({count} file(s)" + + (f"; transport {source.backend}" if source.backend else "") + ")" + for source, count in counts.items()) + origin = f"\nSources:{lines}" + else: + source = (sources[0] if sources else self.default_source) + origin = f"\nSource: {source.display if source else ''}" + if source is not None and source.backend: + origin += f"\nTransport: {source.backend}" + skipped = (f"\nSkipped {self.unsourced_count} cataloged file(s) without a " + "data source" if self.unsourced_count else "") + return (f"{self.file_count} file(s); {size}; estimated duration: {duration}" + f"{origin}" + f"\nDestination: {self.output_path}{skipped}") diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index a7992d0..98e7995 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -3,28 +3,30 @@ from __future__ import annotations from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from decimal import Decimal, InvalidOperation from pathlib import Path -from threading import local +from threading import Lock from typing import Mapping, Optional, Sequence, Union -from urllib.parse import quote, urljoin +from urllib.parse import unquote, urlsplit, urlunsplit from tqdm import tqdm import requests import pandas as pd -from .error import HallmarkError +from .transport import OperationContext, RemoteSpec +from .transport.base import DownloadError, literal_path +from .download_plan import DownloadItem, DownloadPlan, DownloadSource from .helper_functions import ( CHECKSUM_ALGORITHMS_BY_STRENGTH, - REMOTE_REQUEST_TIMEOUT, SUPPORTED_CHECKSUM_ALGORITHMS, as_list_of_dicts, atomic_output_path, file_checksum, normalize_nonempty_string, resolve_contained_path, - valid_checksum, - validate_relative_path) + valid_checksum) from .repo_config import ( + catalog_entries, normalize_remotes, normalize_tsv_name, row_to_path) @@ -33,21 +35,14 @@ BULK_DOWNLOAD_WARNING_FILE_COUNT = 100 # Maximum number of rows to read from a file at once when computing checksums. TSV_READ_CHUNK_SIZE = 10_000 -# Thread-local storage for download worker state, -# allowing each thread to maintain its own state. -_DOWNLOAD_WORKER_STATE = local() # union type for checksum specifications, string or a tuple of (algorithm, checksum). ChecksumSpec = Union[str, tuple[str, str]] # The size of chunks to read from a file when downloading or computing checksums. DOWNLOAD_CHUNK_SIZE = 8192 -class DownloadError(HallmarkError): - """Raised when remote data download fails.""" - - def _repository_config(repo) -> dict: """ - Used by _select_remote_config and select_download_files. + Used by _select_remote_config and _select_download_items. Return repository configuration as a dictionary. Args: @@ -72,7 +67,7 @@ def _repository_config(repo) -> dict: def _config_section_entries(config: dict, section_name: str) -> list[dict]: """ - Used by select_download_files. + Used by _select_download_items. Return the dict entries of a config section (e.g. "data" or "meta"), silently dropping any entries that aren't mappings. @@ -90,15 +85,6 @@ def _config_section_entries(config: dict, section_name: str) -> list[dict]: return [entry for entry in entries if isinstance(entry, dict)] -def _initialize_download_worker() -> None: - """ - Used by download_remote_data. - Create one reusable HTTP session for each downloader thread. - This function is called once per thread in the ThreadPoolExecutor. - """ - _DOWNLOAD_WORKER_STATE.session = requests.Session() - - def _require_positive_integer(value, *, label: str) -> int: """ Used by download_remote_data and _download_file. @@ -200,12 +186,13 @@ def _row_checksum(row: Union[pd.Series, Mapping[str, object]] if legacy_sha1 is not None: return legacy_sha1 # If not found, check the "checksum" and "checksum_algorithm" columns - return _checksum_spec(row.get("checksum"), row.get("checksum_algorithm")) + return (_checksum_spec(row.get("checksum"), row.get("checksum_algorithm")) + or _entry_checksum(row)) def _entry_checksum(entry: dict) -> Optional[ChecksumSpec]: """ - Used by select_download_files. + Used by _select_download_items. Extract either legacy or builder checksum metadata from a config entry. Args: @@ -265,7 +252,7 @@ def _checksum_identity(expected_checksum: Optional[ChecksumSpec] def _validate_checksum_spec(expected_checksum: Optional[ChecksumSpec] ) -> Optional[tuple[str, str]]: """ - Used by _download_file. + Used by _download_file and _fetch_file. Validate and normalize a checksum specification. Args: @@ -301,7 +288,7 @@ def _merge_selected_file( expected_checksum: Optional[ChecksumSpec] ) -> None: """ - Used by add_file and download_remote_data. + Used by add_file and _download_selected. Merge a selected file into the dictionary of selected files, ensuring no conflicting checksums exist. @@ -349,7 +336,7 @@ def _merge_selected_file( def _safe_remote_path(value: Union[str, Path]) -> Path: """ - Used by select_download_files and download_remote_data. + Used by _select_download_items and _download_selected. Validate and return a safe relative Path object for a remote file path. Args: value: The input path to validate. @@ -360,38 +347,26 @@ def _safe_remote_path(value: Union[str, Path]) -> Path: Raises: DownloadError: If the path is unsafe. """ - # Convert the input value to a string and strip whitespace. - raw_path = str(value).strip() - # Validate the path using the validate_relative_path function, which checks for - # unsafe characters, absolute paths, and other potential issues. - try: - return validate_relative_path(raw_path, label="remote path") - # Handle exceptions raised by validate_relative_path and raise a DownloadError - except ValueError as exc: - raise DownloadError(str(exc)) from exc + return literal_path(value) def _remote_file_url(remote_url: str, relative_path: Path) -> str: """ - Used by download_remote_data. - Construct the full URL for a remote file based on the remote URL and relative path. + Construct a file URL from a data remote and a literal catalog path. Args: - remote_url: The base URL of the remote repository. - relative_path: The relative path of the file to download. + remote_url (str): URL of the dataset root. + relative_path (Path): File path relative to the dataset root. Returns: - The full URL for the remote file as a string. + str: URL with the relative path encoded for the transport. """ - # quote the relative path to ensure it is safe for use in a URL, preserving slashes - encoded_path = quote(relative_path.as_posix(), safe="/") - # join the remote URL and the encoded path to create the full URL for the file - return urljoin(remote_url.rstrip("/") + "/", encoded_path) + return RemoteSpec.parse(remote_url).file_url(relative_path.as_posix()) def _download_tsv_name(value) -> str: """ - Used by select_download_files. + Used by _select_download_items. Normalize a TSV name and expose validation failures as DownloadError. Args: @@ -411,7 +386,7 @@ def _download_tsv_name(value) -> str: def _select_remote_config(repo, remote_name: Optional[str] = None) -> Optional[dict]: """ - Used by download_remote_data. + Used by download_remote_data and plan_download. Select a remote configuration from the repository. Args: @@ -537,7 +512,7 @@ def _verify_validated_checksum( chunk_size: int, ) -> None: """ - Used by _download_file. + Used by _download_file and _fetch_file. Verify a checksum that has already been normalized. Args: @@ -560,109 +535,168 @@ def _verify_validated_checksum( raise DownloadError(f"Checksum mismatch for {path.name} " f"({algorithm})") -def _download_file( - url: str, - destination: Path, - expected_checksum: Optional[ChecksumSpec] = None, - chunk_size: int = DOWNLOAD_CHUNK_SIZE, - ) -> int: +def _fetch_file(context, relative_path, destination, expected_checksum, chunk_size): """ - Used by download_remote_data. - Download one file and return the number of bytes written. + Download and verify a file before replacing its destination. Args: - url: The URL of the file to download. - destination: The path where the downloaded file will be saved. - expected_checksum: The expected checksum of the file. Can be a tuple of - (algorithm, checksum) or a legacy SHA-1 checksum. - chunk_size: The size of chunks to read the file in. + context (OperationContext): Transport and destination for this download. + relative_path (Path): Literal path relative to the dataset root. + destination (Path): Destination used for error reporting before + resolving the path against the context's output root. + expected_checksum: Optional digest or (algorithm, digest) pair. + chunk_size (int): Bytes per read when downloading or verifying. Returns: - The number of bytes written to the destination file. + int: Number of downloaded bytes. Raises: - DownloadError: If the download fails or the checksum verification fails. + DownloadError: If the transfer, checksum, or destination is invalid. """ - # Ensure that the chunk size is a positive integer for reading the file in chunks. - chunk_size = _require_positive_integer(chunk_size, label="chunk_size") - # Validate and normalize the expected checksum specification validated_checksum = _validate_checksum_spec(expected_checksum) - - # try to download the file from the specified URL and write it to the temporary file try: - # Ensure the destination directory exists, creating it if necessary. + context.check_cancelled() + destination = resolve_contained_path( + context.output_root, relative_path, label="download destination") destination.parent.mkdir(parents=True, exist_ok=True) - # use atomic_output_path to create a temporary file for the download - with atomic_output_path(destination, suffix=".part", ) as temp_path: - # get the request client from the thread-local state - request_client = getattr(_DOWNLOAD_WORKER_STATE, "session", requests) - # use the request client to perform a GET request to the URL - with request_client.get( - url, - stream=True, - timeout=REMOTE_REQUEST_TIMEOUT) as response: - response.raise_for_status() - # handle the response content in chunks - with temp_path.open("wb") as handle: - for chunk in response.iter_content(chunk_size=chunk_size): - if chunk: - # write the chunk to the temporary file - handle.write(chunk) - # verify the checksum of the downloaded file against the expected value + with atomic_output_path(destination, suffix=".part") as temp_path: + context.transport.fetch(relative_path.as_posix(), temp_path, + chunk_size=chunk_size) _verify_validated_checksum(temp_path, validated_checksum, chunk_size) - # size is the number of bytes written to the temporary file size = temp_path.stat().st_size - + # Recheck for symlink changes before replacing the destination. + # Another process can still change the path after this check. + resolve_contained_path(context.output_root, relative_path, + label="download destination") + context.check_cancelled() return size + except (OSError, ValueError) as exc: + raise DownloadError(f"Failed to write {destination.name}: {exc}") from None - # raise a DownloadError if the download request fails or issue writing the file - except requests.RequestException as exc: - raise DownloadError(f"Failed to download {url}: {exc}") from exc - except OSError as exc: - raise DownloadError(f"Failed to write {destination}: {exc}") from exc +def _download_file( + url: str, + destination: Path, + expected_checksum: Optional[ChecksumSpec] = None, + chunk_size: int = DOWNLOAD_CHUNK_SIZE, + ) -> int: + """ + Download one URL and verify its checksum before replacing the destination. + + This compatibility helper creates its own transport context. -def select_download_files( + Args: + url (str): Full URL of the remote file. + destination (Path): Local path, which may use a different filename. + expected_checksum: Optional digest or (algorithm, digest) pair. + chunk_size (int): Bytes per read. Defaults to DOWNLOAD_CHUNK_SIZE. + + Returns: + int: Number of downloaded bytes. + + Raises: + DownloadError: If the URL, transfer, checksum, or destination is invalid. + """ + chunk_size = _require_positive_integer(chunk_size, label="chunk_size") + _validate_checksum_spec(expected_checksum) + RemoteSpec.parse(url) + parsed = urlsplit(url) + base = urlunsplit(parsed._replace(path=parsed.path.rsplit("/", 1)[0] + "/")) + remote = RemoteSpec.parse(base) + relative = literal_path(unquote(parsed.path.rsplit("/", 1)[-1])) + destination = Path(destination).absolute() + # Direct callers may intentionally choose a different local filename. + with OperationContext(remote, destination.parent) as context: + if remote.scheme in {"http", "https"}: + context._local.session = requests + context.transport.direct_url = url + try: + destination.parent.mkdir(parents=True, exist_ok=True) + resolve_contained_path(destination.parent, destination.name) + with atomic_output_path(destination, suffix=".part") as temporary: + context.transport.fetch(relative.as_posix(), temporary, + chunk_size=chunk_size) + _verify_validated_checksum( + temporary, _validate_checksum_spec(expected_checksum), chunk_size) + size = temporary.stat().st_size + resolve_contained_path(destination.parent, destination.name) + context.check_cancelled() + return size + except OSError as exc: + raise DownloadError(f"Failed to write {destination}: {exc}") from None + + +def _select_download_items( repo, file_paths: Sequence[str] = (), tsv_names: Sequence[str] = (), all_files: bool = False, - ) -> list[tuple[Path, Optional[ChecksumSpec]]]: + catalog_only: bool = False, + resolve_source=None, + ) -> list[DownloadItem]: """ Select files to download from a hallmark repository based on the provided file paths, TSV names, and the repository's configuration. + With no paths or TSVs, every cataloged file is selected. + Args: repo: The hallmark repository object. file_paths: A sequence of specific file paths to download. tsv_names: A sequence of TSV names to download files from. all_files: If True, include all files from the repository's configuration. + catalog_only: If True, explicit paths must be cataloged. + resolve_source: Optional callable receiving the data entries of a + catalog table (empty for static files) and returning the + ``DownloadSource`` of its files, or None if none is configured. Returns: - A list of tuples containing the relative path and optional checksum - specification for each selected file. + list[DownloadItem]: Relative paths, checksums, available file metadata, + and sources when ``resolve_source`` is given. """ # Get the repository configuration, defaulting to an empty dictionary if not found. config = _repository_config(repo) - # get the data configuration entries from the "data" section of the config - data_config = _config_section_entries(config, "data") # dictionary to store selected files with relative paths and optional checksums. selected: dict[str, tuple[Path, Optional[ChecksumSpec]]] = {} + metadata = {} + sources: dict[str, Optional[DownloadSource]] = {} + explicit_paths = {_safe_remote_path(path).as_posix() for path in file_paths} + all_files = all_files or (not file_paths and not tsv_names) def add_file( value: Union[str, Path], - expected_checksum: Optional[ChecksumSpec] = None + expected_checksum: Optional[ChecksumSpec] = None, + row=None, + source: Optional[DownloadSource] = None, ) -> None: - """Add file to the selected files dictionary, ensuring it is safe and valid.""" + """Add a file and its catalog metadata, checking for conflicting records.""" # Resolve the input value to a safe relative path relative_path = _safe_remote_path(value) # Merge the selected file into the dictionary of selected files, ensuring no # conflicting checksums exist. _merge_selected_file(selected, relative_path, expected_checksum) - - def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: - """ Add every file represented by a manifest DataFrame.""" + key = relative_path.as_posix() + if source is not None: + if sources.get(key) not in (None, source): + raise DownloadError(f"Conflicting sources for {key!r}") + sources[key] = source + if row is not None: + size = _catalog_size(row.get("size_bytes")) + modified = row.get("mtime") + modified = (None if modified is None or pd.isna(modified) + or str(modified).strip() == "" else str(modified)) + previous_size, previous_modified = metadata.get(key, (None, None)) + if size is not None and previous_size not in (None, size): + raise DownloadError( + f"Conflicting sizes for {relative_path.as_posix()!r}") + metadata[key] = ( + size if size is not None else previous_size, + modified if modified is not None else previous_modified) + + def add_frame(frame: pd.DataFrame, fmt_entries: list[dict], + *, explicit_only=False, source=None) -> None: + """Add matching catalog rows, retaining their checksums and file metadata.""" # if the DataFrame is None or empty, return early without adding any files if frame is None or frame.empty: return @@ -673,23 +707,30 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: # create a dictionary mapping column names to their corresponding values row = dict(zip(columns, values)) # add the resolved remote path and its checksum to the selected files - add_file( - _resolve_remote_path(row, fmt_entries), _row_checksum(row)) + relative_path = _safe_remote_path(_resolve_remote_path(row, fmt_entries)) + if not explicit_only or relative_path.as_posix() in explicit_paths: + add_file(relative_path, _row_checksum(row), row, source) # Add explicitly requested file paths to the selected files. - for file_path in file_paths: - add_file(file_path) - # Organize data configuration entries by their TSV names for easier access. - entries_by_tsv: dict[str, list[dict]] = {} - for entry in data_config: - # Skip entries that do not have both "fmt" and "db" keys - if not entry.get("fmt") or not entry.get("db"): - continue - # download_tsv_name will normalize and validate the TSV name - tsv_name = _download_tsv_name(entry["db"]) - # Add the entry to the list of entries for the corresponding TSV name - entries_by_tsv.setdefault(tsv_name, []).append(entry) - + if not catalog_only: + for file_path in file_paths: + add_file(file_path) + # Group data entries by the TSV holding their rows. Entries without an + # explicit "db" share data.tsv, whose rows are loaded in the repository state. + try: + catalog = catalog_entries(config) + except ValueError as exc: + raise DownloadError(str(exc)) from exc + raw_entries = as_list_of_dicts(config.get("data")) or [] + tables: dict[str, list] = {} + for entry in catalog: + tables.setdefault(entry.db, []).append(entry) + in_state = {name for name, entries in tables.items() + if not any(raw_entries[entry.index].get("db") for entry in entries)} + # rows of an unreferenced data.tsv are cataloged by their path column + if "data.tsv" not in tables and not repo.state.data.empty: + tables["data.tsv"] = [] + in_state.add("data.tsv") requested_tsvs = [] seen_tsvs = set() @@ -703,20 +744,31 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: # if the all_files flag is set, include all TSVs in the requested list and seen set if all_files: - for tsv_name in entries_by_tsv: + for tsv_name in tables: if tsv_name not in seen_tsvs: requested_tsvs.append(tsv_name) seen_tsvs.add(tsv_name) - for tsv_name in requested_tsvs: - fmt_entries = entries_by_tsv.get(tsv_name) + lookup_tsvs = list(requested_tsvs) + if explicit_paths: + lookup_tsvs.extend(name for name in tables + if name not in requested_tsvs) + for tsv_name in lookup_tsvs: + entries = tables.get(tsv_name) # if there are no format entries for the requested TSV - if fmt_entries is None: - configured = ", ".join(entries_by_tsv) or "" + if entries is None: + configured = ", ".join(tables) or "" # raise an error indicating that the TSV is not configured in the repository raise DownloadError( f"TSV {tsv_name!r} is not configured. " f"Configured TSVs: {configured}") + fmt_entries = [raw_entries[entry.index] for entry in entries] + source = resolve_source(entries) if resolve_source is not None else None + explicit_only = tsv_name not in requested_tsvs + if tsv_name in in_state: + add_frame(repo.state.data, fmt_entries, explicit_only=explicit_only, + source=source) + continue # Construct the path to the TSV file in the repository's dothm directory. tsv_path = repo.dothm.path / tsv_name @@ -732,7 +784,8 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: keep_default_na=False, chunksize=TSV_READ_CHUNK_SIZE) for frame in frames: - add_frame(frame, fmt_entries) + add_frame(frame, fmt_entries, explicit_only=explicit_only, + source=source) # skip empty TSV files without raising an error except pd.errors.EmptyDataError: continue @@ -740,7 +793,8 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: except (OSError, UnicodeError, pd.errors.ParserError) as exc: raise DownloadError(f"Unable to read TSV {tsv_path}: {exc}") from exc - if all_files: + if all_files or explicit_paths: + static_source = resolve_source(()) if resolve_source is not None else None # for each section ("data" and "meta") in the repository configuration for section_name in ("data", "meta"): # iterate through each entry in the section @@ -748,20 +802,258 @@ def add_frame(frame: pd.DataFrame, fmt_entries: list[dict]) -> None: # get the file path from the entry, if it exists file_path = entry.get("file") # if a file path is specified in the entry, add it to the selected files - if file_path: - add_file(file_path, _entry_checksum(entry)) + if file_path and (all_files or _safe_remote_path(file_path).as_posix() + in explicit_paths): + add_file(file_path, _entry_checksum(entry), entry, static_source) + + missing = explicit_paths - set(selected) + if missing: + raise DownloadError( + "Paths are not in the catalog: " + ", ".join(sorted(missing))) + + return [DownloadItem(path, checksum, *metadata.get(path.as_posix(), (None, None)), + source=sources.get(path.as_posix())) + for path, checksum in selected.values()] - # Determine whether to use legacy data formats based on the presence of file paths, - # TSV names, and all_files flag. - use_legacy_data = (not file_paths and not tsv_names and not all_files - ) or (all_files and not entries_by_tsv) - if use_legacy_data: - # legacy_formats are entries in the data configuration that have a "fmt" key - legacy_formats = [entry for entry in data_config if entry.get("fmt")] - # add files from legacy formats to the selected files - add_frame(repo.state.data, legacy_formats) - return list(selected.values()) +def _download_source(remote: Optional[dict]) -> Optional[DownloadSource]: + """ + Used by _source_resolver. + Validate a configured data remote and describe it as a download source. + """ + if not remote or not remote.get("url"): + return None + spec = RemoteSpec.parse(remote["url"], backend=remote.get("backend"), + backend_options=remote.get("backend_options")) + return DownloadSource(remote["url"], remote.get("name"), spec.backend, + remote.get("backend_options") or {}) + + +def _source_resolver(repo, remote_name: Optional[str] = None): + """ + Used by plan_download. + Return a function choosing the download source of a catalog table. + + A named remote overrides every source. Otherwise files come from their + data entry's URL, or from the default configured remote. + """ + override = (_download_source(_select_remote_config(repo, remote_name)) + if remote_name is not None else None) + default = [] + + def resolve(entries) -> Optional[DownloadSource]: + if override is not None: + return override + entry = next((entry for entry in entries if entry.url), None) + if entry is not None: + spec = RemoteSpec.parse(entry.url, backend=entry.backend, + backend_options=entry.backend_options) + return DownloadSource(entry.url, None, spec.backend, + entry.backend_options or {}) + # the default remote is resolved only when a table needs it, since + # several unnamed choices are an error only then + if not default: + default.append(_download_source(_select_remote_config(repo))) + return default[0] + + return resolve + + +def _catalog_size(value) -> Optional[int]: + """Read a recorded size without treating missing or malformed sizes as zero.""" + if value is None or isinstance(value, bool) or pd.isna(value): + return None + try: + size = Decimal(str(value).strip()) + except InvalidOperation: + return None + if not size.is_finite() or size < 0 or size != size.to_integral_value(): + return None + return int(size) + + +def select_download_files( + repo, + file_paths: Sequence[str] = (), + tsv_names: Sequence[str] = (), + all_files: bool = False, +) -> list[tuple[Path, Optional[ChecksumSpec]]]: + """ + Select remote paths and their available catalog checksums. + + Args: + repo: The hallmark repository object. + file_paths (sequence[str]): Explicit remote-relative paths. + tsv_names (sequence[str]): Catalog TSVs to select. + all_files (bool): Include all configured files. Defaults to False. + + Returns: + list[tuple]: Pairs of relative paths and optional checksums. + Explicit paths use catalog checksums when available. + + Raises: + DownloadError: If catalog paths or checksum records are invalid. + """ + return [(item.relative_path, item.checksum) for item in _select_download_items( + repo, file_paths=file_paths, tsv_names=tsv_names, all_files=all_files)] + + +def plan_download( + repo, + output_path: Optional[Union[Path, str]] = None, + *, + file_paths: Optional[Sequence[str]] = None, + tsv_names: Optional[Sequence[str]] = None, + all_files: bool = False, + include: Optional[Union[str, Sequence[str]]] = None, + remote_name: Optional[str] = None, + estimated_bytes_per_second: Optional[float] = None, +) -> DownloadPlan: + """ + Plan a download using local catalog metadata. + + No network requests are made. With no explicit paths or TSVs, select + the complete catalog before applying inclusion globs. Missing sizes remain + unknown. + + Args: + repo: The hallmark repository object. + output_path (Path | str, optional): Destination directory. Defaults + to the worktree; required for a bare repository. + file_paths (sequence[str], optional): Remote-relative file paths. + tsv_names (sequence[str], optional): Catalog TSVs to select. + all_files (bool): Select all configured files. Cannot be combined + with explicit paths or TSVs. Defaults to False. + include (str | list[str], optional): Relative path glob or globs. + remote_name (str, optional): Configured data remote to use. + estimated_bytes_per_second (float, optional): Positive rate used + to estimate duration when every file size is known. + + Returns: + DownloadPlan: Immutable files, source, destination, and metadata + for review before approval. + + Raises: + DownloadError: If the catalog, remote, selection, or destination + is invalid. + ValueError: If an inclusion pattern or supplied rate is invalid. + """ + if output_path is None: + output_path = getattr(repo, "worktree", None) + if output_path is None: + raise DownloadError("output_path is required for a bare repository") + output_root = Path(output_path).expanduser().resolve() + if output_root.exists() and not output_root.is_dir(): + raise DownloadError(f"Download output is not a directory: {output_root}") + if isinstance(file_paths, (str, Path)): + file_paths = (str(file_paths),) + if isinstance(tsv_names, str): + tsv_names = (tsv_names,) + file_paths, tsv_names = tuple(file_paths or ()), tuple(tsv_names or ()) + if all_files and (file_paths or tsv_names): + raise DownloadError("all_files cannot be combined with paths or TSVs") + items = _select_download_items( + repo, file_paths=file_paths, tsv_names=tsv_names, + all_files=all_files or (not file_paths and not tsv_names), catalog_only=True, + resolve_source=_source_resolver(repo, remote_name)) + if include is not None: + from .discovery import path_matches + path_matches("validation", include=include) + items = [item for item in items if path_matches( + item.relative_path.as_posix(), include=include)] + unsourced = [item for item in items if item.source is None] + if unsourced and (file_paths or tsv_names or len(unsourced) == len(items)): + # explicitly requested files, or a selection with nothing downloadable + if len(unsourced) == len(items): + raise DownloadError("No remote URL is configured in config.yml") + shown = ", ".join(item.relative_path.as_posix() for item in unsourced[:5]) + raise DownloadError(f"No data source is configured for: {shown}") + items = [item for item in items if item.source is not None] + for item in items: + try: + resolve_contained_path(output_root, item.relative_path, + label="download destination") + except ValueError as exc: + raise DownloadError(str(exc)) from exc + sources = list(dict.fromkeys(item.source for item in items)) + if not sources and remote_name is None: + # describe the default source even when nothing is selected + try: + sources = [source for source in ( + _download_source(_select_remote_config(repo)),) if source] + except DownloadError: + sources = [] + elif not sources: + sources = [_download_source(_select_remote_config(repo, remote_name))] + source = sources[0] if len(sources) == 1 else None + return DownloadPlan( + tuple(items), source.url if source else None, output_root, + remote_name=source.name if source else None, + estimated_bytes_per_second=estimated_bytes_per_second, + remote_backend=source.backend if source else None, + backend_options=source.backend_options if source else {}, + unsourced_count=len(unsourced)) + + +def execute_download_plan( + repo, + plan: DownloadPlan, + *, + approved: bool = False, + max_workers: int = 4, + show_progress: bool = False, +) -> dict: + """ + Execute a download plan using its recorded source and destination. + + Args: + repo: Repository associated with the plan. Current configuration + is not used to change the planned transfer. + plan (DownloadPlan): Files and destination presented for approval. + approved (bool): Must be True for a nonempty plan. Defaults to False. + max_workers (int): Maximum concurrent download workers. Defaults to 4. + show_progress (bool): Show byte progress. Defaults to False. + + Returns: + dict: Results with ``succeeded``, ``failed``, ``total_bytes``, and + ``errors`` keys. Individual transfer failures are recorded here. + + Raises: + TypeError: If ``plan`` is not a DownloadPlan. + DownloadError: If approval is missing, setup fails, or the + destination is invalid. + """ + if not isinstance(plan, DownloadPlan): + raise TypeError("plan must be a DownloadPlan") + if plan.items and approved is not True: + raise DownloadError("Dataset downloads require explicit approval") + max_workers = _require_positive_integer(max_workers, label="max_workers") + if not plan.items: + return {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + if plan.output_path.resolve() != plan.output_path: + raise DownloadError("Download destination changed since planning") + groups: dict = {} + for item in plan.items: + source = plan.source_for(item) + if source is None: + raise DownloadError( + f"No data source for {item.relative_path.as_posix()!r}") + groups.setdefault(source, []).append(item) + # validate every source before transferring any file + remotes = {source: source.remote() for source in groups} + results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + for source, items in groups.items(): + sizes = [item.size_bytes for item in items] + group = _download_selected( + remotes[source], plan.output_path, + [(item.relative_path, item.checksum) for item in items], + max_workers=max_workers, show_progress=show_progress, + byte_progress=True, + total_bytes=None if None in sizes else sum(sizes)) + for key in ("succeeded", "failed", "total_bytes"): + results[key] += group[key] + results["errors"].extend(group["errors"]) + return results def download_remote_data( @@ -771,24 +1063,30 @@ def download_remote_data( show_progress: bool = False, selected_files: Optional[Sequence[tuple[Path, Optional[ChecksumSpec]]]] = None, remote_name: Optional[str] = None, + *, + approved: bool = False, ) -> dict: """ Download remote data files for a hallmark repository. Args: repo: The hallmark repository object. - worktree_path: The path to the working tree where files will be downloaded. - max_workers: The maximum number of concurrent download threads. - show_progress: Whether to display a progress bar. - selected_files: A sequence of tuples containing the relative path and optional - checksum of files to download. - remote_name: The name of the remote configuration to use. + worktree_path (Path): Directory where files will be downloaded. + max_workers (int): Maximum concurrent download workers. Defaults to 4. + show_progress (bool): Show progress by file count. Defaults to False. + selected_files (sequence[tuple], optional): Relative paths paired + with optional checksums. If omitted, use ``select_download_files``. + remote_name (str, optional): Configured data remote to use. + approved (bool): Must be True for a nonempty transfer. Defaults to False. Returns: - A dict with the download results (succeeded, failed, total_bytes, and errors) + dict: Results with ``succeeded``, ``failed``, ``total_bytes``, and + ``errors`` keys. Individual transfer failures are recorded here; + successful downloads remain available. Raises: - DownloadError: If the remote URL is not configured or if any download fails. + DownloadError: If approval is missing, configuration is invalid, + or connection setup fails before transfers begin. """ # Ensure that max_workers is a positive integer for concurrent downloads. max_workers = _require_positive_integer(max_workers, label="max_workers") @@ -814,18 +1112,41 @@ def download_remote_data( remote_url = normalize_nonempty_string( remote_config.get("url"), label="Remote URL", - exception_type=DownloadError).rstrip("/") + exception_type=DownloadError) # raise a DownloadError if the remote URL is not configured or invalid except DownloadError as exc: raise DownloadError("Remote URL not configured in config.yml") from exc # if the remote URL is not configured, raise a DownloadError to indicate the issue if not remote_url: raise DownloadError("Remote URL not configured in config.yml") + remote_spec = RemoteSpec.parse( + remote_url, backend=remote_config.get("backend"), + backend_options=remote_config.get("backend_options")) # If there are still no files selected for download if not selected_files: # return the results without attempting any downloads return results + if approved is not True: + raise DownloadError("Dataset downloads require explicit approval") + return _download_selected( + remote_spec, worktree_path, selected_files, + max_workers=max_workers, show_progress=show_progress) + + +def _download_selected( + remote_spec, + worktree_path, + selected_files, + *, + max_workers, + show_progress, + byte_progress=False, + total_bytes=None, +): + """Download selected files with shared resources and atomic file replacement.""" + results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + # output_root is the resolved absolute path where files will be downloaded output_root = Path(worktree_path).expanduser().resolve() # if the output root exists and is not a directory, raise a DownloadError @@ -861,71 +1182,96 @@ def download_remote_data( except ValueError as exc: raise DownloadError(str(exc)) from exc - # Append the download info (URL, destination, expected checksum) to the list - files_to_download.append(( - _remote_file_url(remote_url, relative_path), - destination, - expected_checksum)) + files_to_download.append((relative_path, destination, expected_checksum)) - # track the download progress using tqdm, with the number of files to download + # track byte totals for plans and file counts for the legacy API progress = tqdm( - total=len(files_to_download), - unit="file", + total=total_bytes if byte_progress else len(files_to_download), + unit="B" if byte_progress else "file", + unit_scale=byte_progress, disable=not show_progress,) + progress_lock = Lock() + + def advance_bytes(count): + # Transport callbacks run concurrently, one for each active worker. + with progress_lock: + progress.update(count) # try to download the files using a thread pool executor for concurrent downloads try: - with ThreadPoolExecutor(max_workers=max_workers, - initializer=_initialize_download_worker) as executor: - # create an iterator over the files to download - file_iterator = iter(files_to_download) - # initialize a set to keep track of pending download futures - pending = set() - # determine the window size for concurrent downloads, allow for buffering - window_size = max_workers * 2 - - def submit_next() -> bool: - """ - Submit the next file for download if available. - Returns: - bool: True if a file was submitted, - False if no more files are available. - """ - # try to get the next file from the iterator - try: - url, destination, checksum = next(file_iterator) - # Handle the exception when there are no more files to download - except StopIteration: - return False - # add the download task to the pending set using the executor - pending.add(executor.submit(_download_file, url, destination, checksum)) - # if the download was successfully submitted, return True - return True - - # for each file in the initial window size, submit it for download - for _ in range(min(window_size, len(files_to_download))): - submit_next() - while pending: - # wait for at least one of the pending download tasks to complete - completed, _ = wait(pending, return_when=FIRST_COMPLETED) - # for each completed download, process the result and update statistics - for future in completed: - pending.remove(future) - # try to get the result of the completed download task + with OperationContext(remote_spec, output_root) as context: + if byte_progress: + context.on_bytes = advance_bytes + context.transport.prepare() + executor = ThreadPoolExecutor( + max_workers=min(max_workers, context.settings.max_sessions) + if remote_spec.scheme in {"ssh", "sftp"} else max_workers) + try: + # create an iterator over the files to download + file_iterator = iter(files_to_download) + # initialize a set to keep track of pending download futures + pending = set() + # Bound queued work as well as running workers. + window_size = max_workers * 2 + + def submit_next() -> bool: + """ + Submit the next file for download if available. + Returns: + bool: True if a file was submitted, + False if no more files are available. + """ + # try to get the next file from the iterator try: - results["total_bytes"] += future.result() - results["succeeded"] += 1 - # Handle exceptions raised during the download - except DownloadError as exc: - results["failed"] += 1 - results["errors"].append(str(exc)) - # always update the progress bar, regardless of success or failure - finally: - progress.update(1) - # submit the next file for download if available + relative_path, destination, checksum = next(file_iterator) + # Handle the exception when there are no more files to download + except StopIteration: + return False + # add the download task to the pending set using the executor + pending.add(executor.submit( + _fetch_file, context, relative_path, destination, checksum, + DOWNLOAD_CHUNK_SIZE)) + # if the download was successfully submitted, return True + return True + + # for each file in the initial window size, submit it for download + for _ in range(min(window_size, len(files_to_download))): submit_next() + while pending: + # wait for at least one of the pending download tasks to complete + completed, _ = wait(pending, return_when=FIRST_COMPLETED) + # Aggregate per-file failures and keep successful downloads. + for future in completed: + pending.remove(future) + # try to get the result of the completed download task + try: + results["total_bytes"] += future.result() + results["succeeded"] += 1 + # Handle exceptions raised during the download + except DownloadError as exc: + results["failed"] += 1 + results["errors"].append(str(exc)) + # report completed files even when a transfer fails + finally: + if byte_progress: + with progress_lock: + finished = results["succeeded"] + results["failed"] + progress.set_postfix( + files=f"{finished}/{len(files_to_download)}", + failed=results["failed"]) + else: + progress.update(1) + # submit the next file for download if available + submit_next() + except BaseException: + context.cancel() + for future in pending: + future.cancel() + raise + finally: + executor.shutdown(wait=True, cancel_futures=True) # close after all downloads are complete, regardless of success or failure finally: progress.close() - return results \ No newline at end of file + return results diff --git a/mod/hallmark/helper_functions.py b/mod/hallmark/helper_functions.py index 38b4217..1680984 100644 --- a/mod/hallmark/helper_functions.py +++ b/mod/hallmark/helper_functions.py @@ -270,8 +270,8 @@ def normalize_nonempty_string( ) -> str: """ Used in downloader by _select_download_files and download_remote_data, repo by - _validate_branch_name, add, and commit, repo_config by normalize_remotes, - branch_fmt, and set_config. + _validate_branch_name, add, and commit, repo_config by normalize_remotes + and set_config. Normalize a string by stripping whitespace and ensuring it is non-empty. Args: diff --git a/mod/hallmark/objects.py b/mod/hallmark/objects.py index fc54798..a54af2d 100644 --- a/mod/hallmark/objects.py +++ b/mod/hallmark/objects.py @@ -195,16 +195,19 @@ def missing(self, checksums: Iterable[str]) -> list[str]: def store(self, src: Path, sha1: str, *, actual_sha1: Optional[str] = None) -> Path: """ Store a file in the object store using its SHA-1 checksum. + Args: src (Path): Path to the source file to store. sha1 (str): SHA-1 checksum of the source file. actual_sha1 (Optional[str]): Optional actual SHA-1 of the source file. If provided, it will be used to verify the integrity of the source file. + Returns: Path: Path to the stored file. + Raises: ValueError: If the actual SHA-1 checksum of the source file does not match - the expected SHA-1. + the expected SHA-1. """ src = Path(src) # split the SHA-1 checksum into its storage path diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index d17947b..bff0316 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -23,33 +23,121 @@ from typing import Dict, List, Optional, Tuple, Union from git.exc import GitCommandError +import pandas as pd + from .dothm import Dothm from .state import State from .worktree import Worktree from .objects import Objects +from .discovery import is_remote_url from .paraframe import ParaFrame -from .repo_manifest import manifest_frame_from_pf, manifest_map, iter_manifest_entries +from .repo_manifest import ( + catalog_map, is_local_table, iter_catalog_rows, manifest_frame_from_pf, + row_fingerprint, table_groups) from .repo_state import ( load_branch_data, load_head_state, find_remote_branch, fetch_missing_objects_from_remote) from .error import CheckoutError, DestinationExistsError, DothmError from .helper_functions import ( FILE_IO_CHUNK_SIZE, + as_list_of_dicts, chdir, iter_repository_files, normalize_nonempty_string, resolve_contained_path) from .repo_worktree import ( + effective_cwd, ensure_clean_tracked_files, - filtered_paraframe, tracked_paths, worktree_changes) from .repo_config import ( - branch_encodings, - branch_fmt, + CatalogEntry, + DEFAULT_DB, + add_entry, + catalog_entries, + check_template_fields, + entry_encodings, + find_entry, + is_placeholder, + next_db_name, + normalize_remotes, + remove_entry, row_to_path, - set_config, - single_data_fmt) + set_config) + +def _encoding_spec(config: dict) -> Optional[dict]: + """ + Used by add. + Return the entry holding encoding rules for a template not tracked yet: + the placeholder, or the only catalog entry. + """ + entries = as_list_of_dicts(config.get("data")) or [] + placeholder = next((entry for entry in entries if is_placeholder(entry)), None) + if placeholder is not None: + return placeholder + catalog = catalog_entries(config) + return entries[catalog[0].index] if len(catalog) == 1 else None + + +def _row_within(record: dict, fmt: str, within_root) -> bool: + """ + Used by add. + Return True if a row's file lies within the rescanned subtree, or if the + row no longer renders with its template and should be replaced. + """ + try: + return within_root(row_to_path(record, fmt).as_posix()) + except (KeyError, ValueError): + return True + + +def _entry_names(entry: CatalogEntry) -> set[str]: + """ + Used by rm_cached. + Return the spellings that identify an entry: its template, and for a remote + entry its full URL template. + """ + names = {entry.fmt} if entry.fmt else {entry.db} + if entry.url and entry.fmt: + names.add(entry.url.rstrip("/") + "/" + entry.fmt) + return names + + +def _catalog_changes(head_state: State, state: State) -> list[dict]: + """ + Used by status. + Summarize staged changes to catalog-only files for each catalog table. + + Args: + head_state (State): The committed state. + state (State): The staged state. + + Returns: + list[dict]: For each changed table, its ``templates``, ``url`` and + counts of ``added``, ``modified`` and ``deleted`` rows. + """ + def rows_by_table(source): + tables: dict[str, dict[str, str]] = {} + for row in iter_catalog_rows(source, local=False): + tables.setdefault(row.db, {})[row.path] = row_fingerprint(row.record) + return tables + + head_tables, staged_tables = rows_by_table(head_state), rows_by_table(state) + groups = {**table_groups(head_state.config), **table_groups(state.config)} + changes = [] + for name, entries in groups.items(): + head, staged = head_tables.get(name, {}), staged_tables.get(name, {}) + counts = { + "added": sum(path not in head for path in staged), + "modified": sum(path in head and head[path] != fingerprint + for path, fingerprint in staged.items()), + "deleted": sum(path not in staged for path in head)} + if any(counts.values()): + changes.append({ + "templates": [entry.fmt or name for entry in entries], + "url": entries[0].url, **counts}) + return changes + @dataclass(init=False) class Repo: @@ -97,6 +185,7 @@ def __init__(self, path: Union[Path, str]) -> None: self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) self.state = self.dothm.load() + normalize_remotes(self.state.config.get("remote")) self.paraframe_cls = ParaFrame self.download_result = None @@ -177,15 +266,18 @@ def _populate_checksums(self, pf: ParaFrame) -> None: @classmethod def init(cls, path: Union[Path, str]) -> "Repo": - ''' - Initialize a new hallmark repository. + """Initialize an empty local repository. + + Catalog local or remote files afterwards with ``add``; a URL template + such as ``https://host/ER2/{src}_{day}.h5`` catalogs remote files + without downloading them. Args: - paht(paht|string): path to initialize a worktree or ``.hm`` repository. + path (Path | str): Worktree or bare ``.hm`` repository destination. Returns: - Repo: newly created repository instance - ''' + Repo: The initialized repository. + """ dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path / "config.yml").write_text(Dothm.config_template(), @@ -198,77 +290,102 @@ def init(cls, path: Union[Path, str]) -> "Repo": return cls(path) @classmethod - def clone( - cls, - url: str, - path: Union[Path, str], - *, - fetch_data: bool = True, - max_workers: int = 4, - show_progress: bool = False, - ) -> "Repo": - ''' - Clone a remote hallmark repository. Raises DestinationExistsError - if the destination path already exists. Raises DownloadError if - data download is enabled and files fail to download. + def clone(cls, url: str, path: Union[Path, str], *, + source_type: str = "auto", progress: bool = False) -> "Repo": + """Clone a complete Hallmark Git catalog or published snapshot. + + Git sources retain their history. Published snapshots start new history. + Dataset files are never downloaded; plan and execute transfers separately. Args: - url(string): remote repository URL. - path(path|string): destination path for clone. - fetch_data (boolean): if true, downloads associated data files. - max_workers (integer): Number of parallel workers for downloading data. - show_progress (boolean): wether to display download progress. + url (str): Existing Git catalog or published snapshot location. + path (Path | str): New worktree or bare ``.hm`` repository path. + source_type (str): ``auto``, ``git``, or ``catalog``. + progress (bool): Reserved for metadata progress reporting. + Returns: - Repo: Cloned Repository instance - ''' - clone_path = Path(path) - if clone_path.exists(): - raise DestinationExistsError( - f"fatal: destination path '{clone_path}' already exists " - "and is not an empty directory." - ) + Repo: Complete cloned catalog without downloaded data files. - dothm_path, worktree_path = cls.lwpaths(path) - # try to clone the repository, and if it fails, clean up the destination path - try: - Dothm.clone(url, dothm_path, display_path=path) - except Exception: - # remove the partially created directory to avoid leaving a broken state - rmtree(clone_path, ignore_errors=True) - # re-raise the exception to propagate the error to the caller - raise - - # Initialize worktree if non-bare - if worktree_path: - Worktree.init(worktree_path) + Raises: + DestinationExistsError: If the destination already exists. + CloneError: If a requested catalog is missing or invalid. + ValueError: If source options are invalid. + DownloadError: If metadata access fails. + """ + from .catalog import clone_catalog + + return clone_catalog(cls, url, path, source_type=source_type) + + def plan_download(self, output_path=None, *, file_paths=None, tsv_names=None, + all_files=False, include=None, remote_name=None, + estimated_bytes_per_second=None): + """ + Plan a download using the local catalog without contacting a server. + + With no explicit paths or TSVs, select the complete catalog before + applying inclusion globs. Planning does not approve + the transfer. + + Args: + output_path (Path | str, optional): Destination directory. Defaults + to the worktree; required for a bare repository. + file_paths (sequence[str], optional): Remote-relative file paths. + tsv_names (sequence[str], optional): Catalog TSVs to select. + all_files (bool): Select all configured files. Cannot be combined + with explicit paths or TSVs. Defaults to False. + include (str | list[str], optional): Relative path glob or globs. + remote_name (str, optional): Configured data remote to use. + estimated_bytes_per_second (float, optional): Positive transfer + rate for duration estimates. No rate is measured while planning. + + Returns: + DownloadPlan: Immutable selection with its source, destination, + checksums, and available file metadata. + + Raises: + DownloadError: If the catalog, remote, selection, or destination + is invalid. + ValueError: If an inclusion pattern or supplied rate is invalid. + """ + from .downloader import plan_download + + return plan_download( + self, output_path, file_paths=file_paths, tsv_names=tsv_names, + all_files=all_files, include=include, remote_name=remote_name, + estimated_bytes_per_second=estimated_bytes_per_second) - repo = cls(path) - # If fetch_data is True and a worktree exists, download remote data files - if fetch_data and worktree_path: - from .downloader import (DownloadError, download_remote_data, - select_download_files) - # Select files to download from the remote repository - selected_files = select_download_files(repo, all_files=True) - result = download_remote_data( - repo, - worktree_path, - max_workers=max_workers, - show_progress=show_progress, - selected_files=selected_files,) - - repo.download_result = result - if result["failed"]: - errors = result.get("errors", []) - details = "\n".join(f" - {error}" for error in errors[:5]) - remaining = result["failed"] - len(errors[:5]) - if remaining > 0: - details += f"\n - ... {remaining} more error(s)" - raise DownloadError( - f"Failed to download {result['failed']} file(s):\n" - f"{details}" - ) - - return repo + def download(self, plan, *, approved=False, max_workers=4, progress=False): + """ + Download the files in an approved plan. + + The plan fixes the source and destination even if repository settings + change. Successful files remain available when another transfer fails. + + Args: + plan (DownloadPlan): Plan returned by ``plan_download``. + approved (bool): Must be True for a nonempty transfer. Defaults + to False; an empty plan requires no approval. + max_workers (int): Maximum concurrent download workers. Defaults + to 4; the local SSH session limit may reduce concurrency. + progress (bool): Show byte progress. Defaults to False. + + Returns: + dict: Results with ``succeeded``, ``failed``, ``total_bytes``, and + ``errors`` keys. Also stored in ``download_result``. Individual + transfer failures are recorded in ``failed`` and ``errors``. + + Raises: + TypeError: If ``plan`` is not a DownloadPlan. + DownloadError: If approval is missing, setup fails, or the + destination is invalid. + """ + from .downloader import execute_download_plan + + result = execute_download_plan( + self, plan, approved=approved, max_workers=max_workers, + show_progress=progress) + self.download_result = result + return result @staticmethod def checksum(path: Path, chunk_size: int = FILE_IO_CHUNK_SIZE) -> str: @@ -320,6 +437,8 @@ def set_config( remote_name: Optional[str] = None, remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, + remote_backend: Optional[str] = None, + remote_backend_options: Optional[dict] = None, ) -> dict: """ Update repository configuration values. @@ -328,6 +447,8 @@ def set_config( fmt (str, optional): Data format specification. remote_name (str, optional): Name of the remote repository. remote_url (str, optional): URL of the remote repository. + remote_backend (str, optional): Registered data backend name. + remote_backend_options (dict, optional): Backend-specific configuration. encoding_updates (dict[str, str], optional): Updates to encoding rules. Returns: @@ -338,7 +459,11 @@ def set_config( fmt=fmt, remote_name=remote_name, remote_url=remote_url, - encoding_updates=encoding_updates) + encoding_updates=encoding_updates, + **({"remote_backend": remote_backend} + if remote_backend is not None else {}), + **({"remote_backend_options": remote_backend_options} + if remote_backend_options is not None else {})) self.dothm.dump(self.state) return self.state.config @@ -347,19 +472,28 @@ def status(self) -> dict[str, object]: Return repository status information. Includes staged changes, worktree modifications, deletions, and untracked files. + Local files are listed individually. Changes to catalog-only files, + such as those of a remote template, are summarized per template. + Their absent or downloaded copies are never reported as deleted, + modified, or untracked. + Args: self: Repository instance. Returns: dict[str, object]: Status summary including: - branch (str) - - staged changes (dict) + - staged changes (dict), with per-template ``catalog`` summaries - worktree changes (dict) - untracked files (list[str]) """ head_state = load_head_state(self) - head_map = manifest_map(head_state) - staged_map = manifest_map(self.state) + head_rows = catalog_map(head_state) + staged_rows = catalog_map(self.state) + head_map = {path: row_fingerprint(row.record) + for path, row in head_rows.items() if row.local} + staged_map = {path: row_fingerprint(row.record) + for path, row in staged_rows.items() if row.local} state_changes = sorted({ diff.a_path or diff.b_path for diff in self.dothm.index.diff("HEAD") @@ -375,18 +509,20 @@ def status(self) -> dict[str, object]: worktree_modified: list[str] = [] worktree_deleted: list[str] = [] - staged_paths = set(staged_map) # If the repository has a worktree, check for modified and missing tracked files if self.worktree is not None: - worktree_modified, worktree_deleted = worktree_changes(self, staged_map) + local_checksums = {path: str(row.record["sha1"]) + for path, row in staged_rows.items() if row.local} + worktree_modified, worktree_deleted = worktree_changes( + self, local_checksums) worktree_root = Path(self.worktree) # generator that yields relative paths of all files in the worktree worktree_files = (full_path.relative_to(worktree_root).as_posix() for full_path in iter_repository_files(worktree_root)) - # filter out staged paths from the worktree files + # cataloged files, including downloaded copies, are not untracked untracked = sorted(path for path in worktree_files - if path not in staged_paths) + if path not in staged_rows) else: untracked = [] @@ -397,6 +533,7 @@ def status(self) -> dict[str, object]: "added": staged_added, "modified": staged_modified, "deleted": staged_deleted, + "catalog": _catalog_changes(head_state, self.state), }, "worktree": { "modified": sorted(worktree_modified), @@ -405,61 +542,247 @@ def status(self) -> dict[str, object]: "untracked": untracked, } - def add(self, fmt: str, encoding: bool = False) -> ParaFrame: + def add(self, fmt: str, encoding: bool = False, *, + dry_run: bool = False, progress=False, backend: Optional[str] = None, + backend_options: Optional[dict] = None) -> ParaFrame: ''' - Stage files or updated repository indecing from the worktree. + Stage the files matching a template, or rescan the tracked templates. + + A new template becomes another data entry with its own catalog table; + existing templates are kept. Adding a tracked template again merges + its current files. ``"."`` rescans every local template within the + current directory, dropping rows of files that no longer exist there. + + A URL template such as ``https://host/ER2/{src}_{day}.h5`` catalogs + the matching remote files without downloading them. The URL up to the + first segment with a field is recorded on the data entry. Adding the + URL template again syncs it with the remote directory. Args: - fmt (string): Format string or "." for full directory scan. + fmt (string): Format string, URL template, or "." for full + directory scan. encoding (boolean): Whether to apply encoding rules. + dry_run (boolean): List matching files without hashing or staging. + progress (bool | callable): Display remote discovery progress or + receive updates. + backend (str, optional): Registered backend for a URL template. + backend_options (dict, optional): Backend configuration for a URL + template. Returns: paraframe Parsed and filtered file index (without checksums). + Raises: + ValueError: If the template is invalid, matches no files when new, + or matches files tracked by another template. + RuntimeError: If a local template is added to a repository without + a worktree. + DownloadError: If remote discovery fails. ''' + if is_remote_url(fmt): + if encoding: + raise ValueError("--regex encoding rules apply to local files only") + from .repo_remote import add_remote_template + + return add_remote_template( + self, fmt, dry_run=dry_run, progress=progress, backend=backend, + backend_options=backend_options) + if backend is not None or backend_options is not None: + raise ValueError("backend options apply to URL templates only") if self.worktree is None: raise RuntimeError( "cannot add files in a bare repository without a worktree") # Normalize the format string to ensure it is a non-empty string fmt = normalize_nonempty_string(fmt, label="format") - # "." means rescan the whole worktree using the already-configured format - rescanning = fmt == "." - # use the current branch format; otherwise, use the provided format - if rescanning: - resolved_fmt = branch_fmt(self) - previous_fmt = resolved_fmt - else: - resolved_fmt = fmt - try: - previous_fmt = branch_fmt(self) - except RuntimeError: - previous_fmt = None + # "." means rescan the worktree using the already-configured templates + if fmt == ".": + return self._rescan_local(encoding=encoding, dry_run=dry_run) + + check_template_fields(fmt) + config = self.state.config + entry = find_entry(config, fmt) + if entry is not None and entry.is_remote: + raise ValueError( + f"template {fmt!r} is tracked from a remote URL; run " + f"hallmark rm --cached {fmt!r} before adding local files") + encodings = None + if encoding: + spec = (config["data"][entry.index] if entry is not None + else _encoding_spec(config)) + encodings = entry_encodings(spec, fmt) # with the working directory set to the worktree, parse files into a ParaFrame with chdir(self.worktree): pf = ParaFrame.parse( - resolved_fmt, + fmt, base_path=self.worktree, - encodings=branch_encodings(self) if encoding else None, + encodings=encodings, encoding=encoding) - # if rescanning, filter to include only files that match the configured format - if rescanning: - pf = filtered_paraframe(self, pf) + if dry_run: + return pf + if entry is None: + # like git add, a new pathspec must match something + if pf.empty: + raise ValueError(f"template {fmt!r} did not match any files") + self._check_ownership(fmt, pf["path"]) # Compute checksums for all files in the ParaFrame in parallel self._populate_checksums(pf) - - manifest = manifest_frame_from_pf(pf, resolved_fmt) - # if not rescanning, update the repository configuration with the new format - if not rescanning: - set_config(self, fmt=resolved_fmt) - # an explicit fmt replaces only if the format actually changed - if rescanning or previous_fmt != resolved_fmt: - self.state.replace(manifest) - # if the format is unchanged, update the existing state with new entries + manifest = manifest_frame_from_pf(pf, fmt) + if entry is None: + db = next_db_name(config, used=self._used_table_names()) + add_entry(config, {"fmt": fmt, **({"db": db} if db != DEFAULT_DB else {})}) + self.state.replace(manifest, db=db) + # adding a tracked template again merges its current files else: - self.state.update(manifest) + self.state.update(manifest, db=entry.db) self.dothm.dump(self.state) # return a ParaFrame without the "sha1" column for display purposes return pf.drop(columns=["sha1"], errors="ignore") + def _rescan_local(self, *, encoding: bool, dry_run: bool) -> ParaFrame: + """ + Used by add. + Rescan every local template within the current directory's subtree. + + Rows of files outside the subtree are kept. Files cataloged by a remote + template, such as downloaded copies, are skipped. + + Args: + encoding (bool): Apply the encoding rules of templates defining them. + dry_run (bool): List matching files without hashing or staging. + + Returns: + ParaFrame: The matching files of every local template. + + Raises: + RuntimeError: If no local template is tracked. + ValueError: If a file matches more than one local template. + """ + config = self.state.config + local = [entries[0] for name, entries in table_groups(config).items() + if is_local_table(entries, self.state.table(name))] + if not local: + raise RuntimeError( + "no local templates to rescan; run hallmark add TEMPLATE first") + with_encodings = [entry for entry in local + if entry_encodings(config["data"][entry.index], entry.fmt)] + if encoding and not with_encodings: + raise ValueError("no encoding rules are configured; use hallmark " + "set-config --encoding FIELD=REGEX") + worktree = Path(self.worktree).resolve() + relative_root = effective_cwd(self).relative_to(worktree) + catalog_only = {row.path for row in iter_catalog_rows(self.state, local=False)} + + def within_root(path) -> bool: + return relative_root == Path(".") or relative_root in Path(path).parents + + scans, claimed = [], {} + for entry in local: + use_encoding = encoding and entry in with_encodings + with chdir(self.worktree): + pf = ParaFrame.parse( + entry.fmt, base_path=self.worktree, + encodings=(entry_encodings(config["data"][entry.index], entry.fmt) + if use_encoding else None), + encoding=use_encoding) + if not pf.empty: + pf = pf[pf["path"].map(within_root) & ~pf["path"].isin(catalog_only)] + for path in pf["path"] if not pf.empty else (): + if path in claimed: + raise ValueError( + f'"{path}" matches templates {claimed[path]!r} and ' + f"{entry.fmt!r}; use more specific templates") + claimed[path] = entry.fmt + scans.append((entry, pf)) + frames = [pf for _, pf in scans if not pf.empty] + combined = (ParaFrame(pd.concat(frames, ignore_index=True)) if frames + else ParaFrame(columns=["path"])) + if dry_run: + return combined + for entry, pf in scans: + self._populate_checksums(pf) + manifest = manifest_frame_from_pf(pf, entry.fmt) + if relative_root != Path("."): + # keep rows of files outside the rescanned subtree + kept = [record for record in + self.state.table(entry.db).to_dict(orient="records") + if not _row_within(record, entry.fmt, within_root)] + manifest = pd.concat([pd.DataFrame(kept, columns=manifest.columns), + manifest], ignore_index=True) + self.state.replace(manifest, db=entry.db) + self.dothm.dump(self.state) + return combined.drop(columns=["sha1"], errors="ignore") + + def _check_ownership(self, fmt: str, paths) -> None: + """ + Used by add. + Reject a new template that matches files tracked by another template. + + Args: + fmt (str): The new template. + paths: Relative POSIX paths the template matches. + + Raises: + ValueError: If any path is already cataloged. + """ + owned = catalog_map(self.state) + overlap = sorted(path for path in set(paths) if path in owned) + if overlap: + shown = ", ".join(overlap[:5]) + more = f" and {len(overlap) - 5} more" if len(overlap) > 5 else "" + raise ValueError( + f"template {fmt!r} matches files tracked by another template " + f"({shown}{more}); use a more specific template or run " + "hallmark rm --cached TEMPLATE first") + + def _used_table_names(self) -> set[str]: + """ + Used by add. + Return catalog TSV names present now or at any point in history, so a + new template never reuses the history of a removed one. + """ + names = {path.name for path in self.dothm.path.glob("*.tsv")} + try: + history = self.dothm.git.log( + "--all", "--format=", "--name-only", "--", "*.tsv") + # a repository without commits has no history to consult + except GitCommandError: + history = "" + names.update(line.strip() for line in history.splitlines() if line.strip()) + return names + + def rm_cached(self, template: str) -> CatalogEntry: + ''' + Stop tracking a template without deleting any files, like + ``git rm --cached``. + + Args: + template (str): The tracked template, or a remote template's full + URL template. + + Returns: + CatalogEntry: The removed entry. + + Raises: + ValueError: If the template is not tracked or shares its catalog + table with another template. + ''' + template = normalize_nonempty_string(template, label="template") + entries = catalog_entries(self.state.config) + entry = next((candidate for candidate in entries + if template in _entry_names(candidate)), None) + if entry is None: + tracked = ", ".join(repr(candidate.fmt) for candidate in entries + if candidate.fmt) or "none" + raise ValueError( + f"template {template!r} is not tracked; tracked templates: {tracked}") + if any(other.db == entry.db for other in entries if other is not entry): + raise ValueError( + f"template {entry.fmt!r} shares its catalog table {entry.db} with " + "another template; edit config.yml to remove it") + remove_entry(self.state.config, entry.index) + self.state.drop_table(entry.db) + self.dothm.dump(self.state) + return entry + def commit(self, msg: str, allow_empty: bool = False) -> bool: ''' Commit staged changes to the repository. Raises ValueError if commit @@ -477,27 +800,19 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: if (not allow_empty and not self.dothm.index.diff("HEAD")): # return early since there are no changes to commit return False - # get the current format string and the HEAD state of the repository - current_fmt = branch_fmt(self) head_state = load_head_state(self) - # get the format string of the HEAD state for comparison - head_fmt = single_data_fmt(head_state.config) - # head entries are the set of (path, sha1) tuples from the HEAD state - head_entries: set[tuple[Path, str]] = set() - - if head_fmt == current_fmt: - # populate head_entries with the paths and checksums from the HEAD state - head_entries = {(relative_path, checksum.lower()) - for relative_path, checksum - in iter_manifest_entries(head_state, fmt=head_fmt)} + # head entries are the set of (path, sha1) tuples of HEAD's local files + head_entries = {(row.path, str(row.record["sha1"]).lower()) + for row in iter_catalog_rows(head_state, local=True)} # list of tuples containing (full path, expected sha1) for stored files files_to_store: list[tuple[Path, str]] = [] - # for each entry in the current manifest - for relative_path, checksum in iter_manifest_entries( - self.state, fmt=current_fmt): + # for each local file in the current catalog; catalog-only files, such + # as remote files, are committed as metadata without objects + for row in iter_catalog_rows(self.state, local=True): + relative_path = row.path # get the expected SHA1 checksum for the file - expected_sha1 = checksum.lower() + expected_sha1 = str(row.record["sha1"]).lower() # current_entry is a tuple of (relative path, expected sha1) for this file current_entry = (relative_path, expected_sha1) # if the manifest entry is not in the HEAD entries @@ -577,18 +892,24 @@ def checkout(self, target_branch: str) -> bool: create_new_branch = not has_local and not has_remote current_tracked = tracked_paths(self) target_state = load_branch_data(self, target_branch) - # Get the data format string for the target branch configuration - target_fmt = single_data_fmt(target_state.config) - if target_fmt is None: - # Raise an error if the target branch does not meet the expected criteria - raise CheckoutError( - "checkout currently supports only repositories with " - "one data format and data.tsv") - # list of (relative path, sha1) tuples for all entries in the target branch - target_entries_raw = list(iter_manifest_entries(target_state, fmt=target_fmt)) + # checkout restores and removes local files only; catalog-only files, + # such as downloaded copies of remote files, are left in place + try: + target_rows = list(iter_catalog_rows(target_state)) + current_rows = list(iter_catalog_rows(self.state)) + except ValueError as exc: + raise CheckoutError(str(exc)) from exc + # list of (relative path, sha1) tuples for the target branch's local files + target_entries_raw = [(Path(row.path), str(row.record["sha1"])) + for row in target_rows if row.local] # get the set of relative paths for all tracked files in the target branch target_tracked = {path for path, _ in target_entries_raw} + # the target's catalog-only files and the current branch's downloaded copies + target_catalog_sha = {Path(row.path): str(row.record.get("sha1", "")).lower() + for row in target_rows if not row.local} + current_catalog_only = {Path(row.path) for row in current_rows + if not row.local} # try to identify any missing objects in the target branch that are not present # in the object store @@ -645,18 +966,26 @@ def checkout(self, target_branch: str) -> bool: for (rel_path, full_path, expected_sha1) in conflict_candidates: # if the computed checksum does not match the expected checksum if (conflict_checksums[full_path] != expected_sha1): + if rel_path in current_catalog_only: + raise CheckoutError( + f'target tracked path "{rel_path}" conflicts with ' + "a downloaded copy cataloged by the current branch") raise CheckoutError( f'target tracked path "{rel_path}" ' "already exists as an untracked file") # Store the name of the currently active branch before switching original_branch = self.dothm.active_branch.name - # Get the current format string from the branch configuration - current_fmt = branch_fmt(self) - # Create a mapping of current tracked paths to their SHA1 checksums + # Create a mapping of current local tracked paths to their SHA1 checksums current_sha_by_path = { - row_to_path(row, current_fmt): str(row["sha1"]).lower() - for _, row in self.state.data.iterrows()} + Path(row.path): str(row.record["sha1"]).lower() + for row in current_rows if row.local} + # a local file that the target catalogs remotely with the same content + # stays in place instead of being removed and downloaded again + kept_paths = { + path for path in current_tracked - target_tracked + if target_catalog_sha.get(path) + and target_catalog_sha[path] == current_sha_by_path.get(path)} # tuples of (relative path, sha1) for all files in the target branch target_entries = [(path, sha1.lower()) for path, sha1 in target_entries_raw] # changed_target_entries are the files in the target branch that have different @@ -727,7 +1056,7 @@ def remove_empty_parents(path: Path) -> None: # paths that are either currently tracked but not in the target branch # or paths that are tracked but have changed from the current branch affected_paths = sorted((current_tracked - target_tracked - ) | changed_target_paths, + - kept_paths) | changed_target_paths, key=lambda path: (len(path.parts), path.as_posix()), reverse=True) for relative_path in affected_paths: # absolute path in the worktree for the affected relative path @@ -858,13 +1187,10 @@ def add_worktree(self, target_branch: str) -> bool: else: target_state = load_head_state(self) - # get the target fmt from the target branch configuration - target_fmt = single_data_fmt(target_state.config) - # if target branch does not have exactly one data format, raise a RuntimeError - if target_fmt is None: - raise RuntimeError( - "add_worktree requires exactly one data entry " - "with a non-empty fmt") + # the new worktree receives the target's local files; catalog-only + # files, such as remote files, are downloaded separately + target_local = [(row.path, str(row.record["sha1"])) + for row in iter_catalog_rows(target_state, local=True)] # if the target worktree does not exist if not target_dothm.exists(): @@ -872,7 +1198,7 @@ def add_worktree(self, target_branch: str) -> bool: # the object store try: missing_objects = self.objects.missing( - row["sha1"] for _, row in target_state.data.iterrows()) + sha1 for _, sha1 in target_local) # if a ValueError occurs during object existence check, raise cleanly except ValueError as exc: raise FileNotFoundError(str(exc)) from exc @@ -901,8 +1227,7 @@ def add_worktree(self, target_branch: str) -> bool: # iterate over the target state data and restore files from the object store try: - for _, row in target_state.data.iterrows(): - rel_path = row_to_path(row, target_fmt) + for rel_path, sha1 in target_local: # resolve relative path to an absolute path in the target worktree destination = resolve_contained_path( target, @@ -911,7 +1236,7 @@ def add_worktree(self, target_branch: str) -> bool: # ensure the parent directory exists before restoring the file destination.parent.mkdir(parents=True, exist_ok=True) # restore the file from the object store using its SHA-1 checksum - self.objects.restore(row["sha1"], destination) + self.objects.restore(sha1, destination) # if any exception occurs during file restoration, attempt a clean up except Exception as exc: @@ -944,4 +1269,4 @@ def add_worktree(self, target_branch: str) -> bool: f'failed to populate worktree for branch ' f'"{target_branch}": {exc}') from exc - return True \ No newline at end of file + return True diff --git a/mod/hallmark/repo_builder.py b/mod/hallmark/repo_builder.py index c2c0998..1fbc598 100755 --- a/mod/hallmark/repo_builder.py +++ b/mod/hallmark/repo_builder.py @@ -1,38 +1,35 @@ from __future__ import annotations -import hashlib import re import shutil import string import time -from concurrent.futures import ThreadPoolExecutor +import warnings from functools import lru_cache from pathlib import Path from string import Formatter -from threading import local -from urllib.parse import quote, unquote, urljoin, urlsplit +from urllib.parse import unquote, urlsplit import pandas as pd import parse -import requests from .repo import Repo +from .transport import OperationContext, RemoteSpec +from .transport.base import ( + CapabilityError, DownloadError, reject_controls, thaw_backend_options) from .fmt_detection import ( - detect_fmt, KNOWN_PROCESSING_STAGES, KNOWN_STATIC_FILE_STEMS) from .dothm import dump_yaml from .error import DothmError from .helper_functions import ( CHECKSUM_ALGORITHMS, - CHECKSUM_ALGORITHM_PATTERN, + CHECKSUM_ALGORITHMS_BY_STRENGTH, SUPPORTED_CHECKSUM_ALGORITHMS, - REMOTE_REQUEST_TIMEOUT, valid_checksum, atomic_output_path, load_yaml_file, normalize_nonempty_string, - prompt_choice, validate_path_component, validate_relative_path) from .repo_config import ( @@ -44,31 +41,10 @@ # base URL for the CyVerse curated data repository _CYVERSE_CURATED_BASE = \ "https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/" -# regular expression to match rows in an HTML index page for a directory listing -_INDEX_ROW_RE = re.compile( - r'' - r'.+)\."rf"(?P{CHECKSUM_ALGORITHM_PATTERN})sums$") -# keywords to identify checksum files based on their names -_CHECKSUM_NAME_KEYWORDS = ("sum", "checksum", "hash", "manifest", *CHECKSUM_ALGORITHMS) -# maximum number of worker threads for static checksum computation -_STATIC_CHECKSUM_MAX_WORKERS = 8 -# thread-local storage for network worker state, including requests sessions -_NETWORK_WORKER_STATE = local() -# maximum number of worker threads for remote crawling and checksum computation -_REMOTE_CRAWL_MAX_WORKERS = 8 -# minimum ratio of matching lines in a checksum file to consider it valid -_MANIFEST_LINE_MATCH_RATIO = 0.5 -# regular expression to match checksum algorithms in file names, case-insensitive -_ALGORITHM_IN_NAME_RE = re.compile(rf"({CHECKSUM_ALGORITHM_PATTERN})", re.IGNORECASE) +_SUMS_LINE_RE = re.compile(r"^([0-9a-fA-F]{8,}) [ *](.+)$", re.MULTILINE) # default checksum algorithm to use when the algorithm cannot be determined _UNKNOWN_CHECKSUM_ALGORITHM = "unknown" -# maximum file size (in bytes) for which we will compute a checksum -_MAX_DOWNLOAD_SIZE_FOR_CHECKSUM = 10 * 1024 * 1024 # 10 MB # maximum length of a literal segment in a format string that can be cleared _MAX_CLEARABLE_LITERAL_LENGTH = 2 # known field values for specific format fields, used for parsing and matching @@ -77,137 +53,12 @@ "algorithm": CHECKSUM_ALGORITHMS} -def _network_worker_session(): - """ - Used by _checksum_small_remote_url and _fetch_remote_text. - Get or create a requests session for the current thread. - - Returns: - A requests.Session object for the current thread. - """ - # get the requests session from thread-local storage - session = getattr(_NETWORK_WORKER_STATE, "session", None) - if session is None: - # create a new requests session for this thread if it doesn't exist - session = requests.Session() - # network worker state is thread-local, so each thread will have its own session - _NETWORK_WORKER_STATE.session = session - return session - - def _remote_url(base_url: str, relative_path: str) -> str: - """ - Used by list_remote_files and build_repo. - Construct a full remote URL by joining a base URL and a relative path. - - Args: - base_url: The base URL of the remote repository. - relative_path: The relative path to the file or directory. - - Returns: - The full remote URL as a string. - """ - # strip trailing slashes from the base URL and ensure it ends with a single slash - directory_url = f"{str(base_url).rstrip('/')}/" - # use urljoin to combine the base URL and the quoted relative path - return urljoin(directory_url, quote(str(relative_path), safe="/")) - - -def _fetch_remote_text(url: str) -> str: - """ - Used by _fetch_optional_remote_text and list_remote_files. - Fetch the text content of a remote URL using a thread-local requests session. - Args: - url: The URL to fetch. - Returns: - The text content of the response. - """ - # Use a requests session to fetch the HTML index page for the given URL. - response = _network_worker_session().get(url, timeout=REMOTE_REQUEST_TIMEOUT) - # Raise an exception if the request failed - response.raise_for_status() - - # Return the text content of the response, which is the HTML index page. - return response.text - - -def _fetch_optional_remote_text(url: str) -> str | None: - """ - Used by list_remote_files. - Fetch the text content of a remote URL, returning None if the request fails. - - Args: - url: The URL to fetch. - - Returns: - The text content of the response, or None if the request fails. - """ - try: - return _fetch_remote_text(url) - # if the request fails (e.g., network error, timeout, 404), return None - except requests.RequestException: - return None - - -def _checksum_small_remote_file(session, file_url: str) -> tuple[str, str]: - """ - Used by _checksum_small_remote_url. - Compute the MD5 checksum of a small remote file. - - Args: - session: The requests session to use for HTTP requests. - file_url: The URL of the remote file. - - Returns: - A tuple containing the checksum algorithm and the checksum value. - If the file is too large or an error occurs, returns ("unknown", "unknown"). - """ - unknown = (_UNKNOWN_CHECKSUM_ALGORITHM, _UNKNOWN_CHECKSUM_ALGORITHM) - # try to get the file size using a HEAD request to avoid downloading large files - try: - head_response = session.head(file_url, timeout=REMOTE_REQUEST_TIMEOUT) - head_response.raise_for_status() - # if the HEAD request fails, return unknown checksum values - except requests.RequestException: - return unknown - - # get the Content-Length header to determine the file size - content_length = head_response.headers.get("Content-Length") - # try to convert the Content-Length to an integer, return unknown if it fails - try: - file_size = int(content_length) - except (TypeError, ValueError): - return unknown - # if the file size is negative or exceeds the maximum allowed size - if (file_size < 0 or file_size > _MAX_DOWNLOAD_SIZE_FOR_CHECKSUM): - return unknown - - # try to download the file and compute its MD5 checksum - try: - file_response = session.get(file_url, timeout=REMOTE_REQUEST_TIMEOUT) - file_response.raise_for_status() - # if the GET request fails, return unknown checksum values - except requests.RequestException: - return unknown - - # if all requests succeed, compute and return the MD5 checksum of the file content - return ("md5", hashlib.md5(file_response.content).hexdigest()) - - -def _checksum_small_remote_url(file_url: str) -> tuple[str, str]: - """ - Used by build_repo. - Compute the checksum of a small remote file using a thread-local requests session. - - Args: - file_url: The URL of the remote file. - - Returns: - _checksum_small_remote_file(session, file_url), where session is a thread-local - requests session. Tuple contains the checksum algorithm and the checksum value. - If the file is too large or an error occurs, returns ("unknown", "unknown"). - """ - return _checksum_small_remote_file(_network_worker_session(), file_url) + """Construct a remote URL, preserving literal paths and directory slashes.""" + url = RemoteSpec.parse(base_url).file_url(relative_path) + if relative_path.endswith("/") and not url.endswith("/"): + url += "/" + return url # cache for performance, since the same fmt may be used for many files in a dataset @@ -527,7 +378,7 @@ def solve(segment_idx, path_idx): def _match_static_named_file_plain_only(rel_path: str, fmt_entries: list[dict] ) -> tuple[int, dict] | tuple[None, None]: """ - Used by build_repo. + Used by _build_repo. Match a static named file against a list of fmt entries using only the plain parse. Args: @@ -552,7 +403,7 @@ def _match_static_named_file_plain_only(rel_path: str, fmt_entries: list[dict] def _match_file_against_fmts(rel_path: str, fmt_entries: list[dict]): """ - Used by build_repo. + Used by _build_repo. Find the best fmt entry for one file, across all of them at once. Args: @@ -631,9 +482,21 @@ def _match_file_against_fmts(rel_path: str, fmt_entries: list[dict]): valid_fmts, key=lambda m: (m[2], -m[3])) return best_index, best_parse +def _record_checksum(checksums, path, algorithm, checksum): + """Keep the strongest checksum and reject conflicting digests.""" + previous_algorithm, previous = checksums.get(path, (None, None)) + if previous is not None and previous_algorithm == algorithm: + if previous.lower() != checksum.lower(): + raise DownloadError(f"Conflicting {algorithm} manifests for {path!r}") + return + strength = {name: i for i, name in enumerate(CHECKSUM_ALGORITHMS_BY_STRENGTH)} + if previous is None or strength.get(algorithm, 99) < strength.get( + previous_algorithm, 100): + checksums[path] = (algorithm, checksum) + + def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: """ - Used by list_remote_files. Parse a manifest text and return a list of (checksum, filename) tuples for each line in the manifest that has a valid checksum for the given algorithm. @@ -645,6 +508,8 @@ def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: A list of (checksum, filename) tuples for each line in the manifest text that has a valid checksum for the given algorithm, allowing unknown algorithms. """ + if any(line.startswith("\\") for line in text.splitlines()): + raise ValueError("GNU escaped manifest filenames are unsupported") # return a list of (checksum, filename) tuples for each line in the manifest text # if the checksum is valid for the given algorithm, allowing unknown algorithms return [ @@ -655,7 +520,6 @@ def _manifest_matches(text: str, algorithm: str) -> list[tuple[str, str]]: def _resolve_manifest_path(filename: str, rel_dir: str) -> str: """ - Used by list_remote_files. Resolve a manifest path relative to a given directory, ensuring it is valid. Args: @@ -670,7 +534,8 @@ def _resolve_manifest_path(filename: str, rel_dir: str) -> str: or if the resolved path is not a valid relative path. """ # strip whitespace from the filename to avoid issues with leading/trailing spaces - filename = str(filename).strip() + filename = str(filename) + reject_controls(filename, "Manifest path") # remove leading "./" from the filename to normalize the path while filename.startswith("./"): @@ -713,7 +578,6 @@ def _resolve_manifest_path(filename: str, rel_dir: str) -> str: def _normalize_index_href(href: str, *, is_directory: bool) -> str: """ - Used by list_remote_files. Normalize an index href to ensure it is a valid relative path. Args: @@ -727,7 +591,8 @@ def _normalize_index_href(href: str, *, is_directory: bool) -> str: ValueError: If the href is not a valid relative path. """ # use urlsplit to check for scheme, netloc, query, and fragment - parsed = urlsplit(str(href).strip()) + reject_controls(str(href), "Index href") + parsed = urlsplit(str(href)) # if any components are present, raise a ValueError as the href must be relative if ( @@ -740,6 +605,7 @@ def _normalize_index_href(href: str, *, is_directory: bool) -> str: # use unquote to decode any percent-encoded characters in the path decoded_path = unquote(parsed.path) + reject_controls(decoded_path, "Index path") # if the path is a directory, remove any trailing slashes; otherwise, keep it as-is candidate = (decoded_path.rstrip("/") if is_directory else decoded_path) # validate the candidate path to ensure it is a valid relative path @@ -754,7 +620,7 @@ def _normalize_index_href(href: str, *, is_directory: bool) -> str: def _normalize_fmt_entries(fmt_entries: list[dict]) -> list[dict]: """ - Used by build_repo. + Used by _build_repo. Normalize the fmt entries to ensure they are valid and consistent. Args: @@ -812,166 +678,33 @@ def _normalize_fmt_entries(fmt_entries: list[dict]) -> list[dict]: return normalized -def list_remote_files(base_url: str) \ - -> dict[str, tuple[str | None, str | None]]: +def list_remote_files(base_url: str, *, _context=None): """ - Recursively list every file under a CyVerse WebDAV directory, - collecting checksums along the way. - - Sibling checksum manifests enrich matching files with checksums, but - directories are still traversed because a manifest may be incomplete. + List remote files and their published checksums. Args: - base_url: A CyVerse WebDAV directory URL. + base_url (str): Exact URL root to search recursively. + _context (OperationContext, optional): Existing internal transport + context. Its remote is used when supplied. Returns: - A dictionary mapping relative paths to a tuple of (algorithm, checksum), - where algorithm is the checksum algorithm used (e.g., "md5", "sha256"), - and checksum is the corresponding checksum value. If no checksum is found, - both values will be None. + dict: Relative paths mapped to (algorithm, checksum) pairs. + Missing checksum values remain None. + + Raises: + DownloadError: If discovery fails or checksum manifests conflict. + CapabilityError: If the source provides no supported listing. """ - # normalize the base URL to ensure it is a valid WebDAV directory URL - base_url = _remote_url(base_url, "") - - # stores all files with their checksums, keyed by relative path, for all algorithms - file_checksums: dict[str, tuple[str | None, str | None]] = {} - - directories_to_open = [""] - visited_directories: set[str] = set() - # raw index text for directories that have been fetched but not yet processed - prefetched_indexes: dict[str, str] = {} - # Use a requests session and ThreadPoolExecutor to fetch remote indexes concurrently - with ThreadPoolExecutor(max_workers=_REMOTE_CRAWL_MAX_WORKERS) as crawl_executor: - # while there are directories to open or prefetched indexes to process - while directories_to_open or prefetched_indexes: - # if there are no prefetched indexes, fetch a batch of directories to open - if not prefetched_indexes: - batch = [] - # while there are directories to open and we haven't reached max workers - while (directories_to_open and len(batch) < _REMOTE_CRAWL_MAX_WORKERS): - # pop a directory from the stack to process - candidate = directories_to_open.pop() - # skip if this directory has already been visited to avoid cycles - if candidate in visited_directories: - continue - # mark directory as visited and add it to the batch for fetching - visited_directories.add(candidate) - batch.append(candidate) - # if the batch is empty, continue to the next iteration of the loop - if not batch: - continue - - # construct the full URLs for the batch of directories to fetch - directory_urls = [_remote_url(base_url, rel_dir) for rel_dir in batch] - # fetch the index texts for the batch of directories concurrently - index_texts = crawl_executor.map(_fetch_remote_text, directory_urls) - # update the prefetched indexes with the fetched index texts - prefetched_indexes.update(zip(batch, index_texts)) - - # get the next directory to process from the prefetched indexes - rel_dir = next(iter(prefetched_indexes)) - # pop the index text for this directory from the prefetched indexes - index_text = prefetched_indexes.pop(rel_dir) - # parse the index text to extract entries using regex - raw_entries = _INDEX_ROW_RE.findall(index_text) - entries = [] - for entry_type, href in raw_entries: - # try to normalize the href to ensure it is a valid relative path - try: - normalized_href = _normalize_index_href( - href, - is_directory=(entry_type == "collection")) - # if the href is invalid, skip this entry and continue to the next one - except ValueError: - continue - # add the normalized entry to the list of entries for this directory - entries.append((entry_type, normalized_href)) - - # find manifest files that are not directories and match expected patterns - manifest_hrefs = [href for entry_type, href in entries - if (entry_type != "collection" - and (_SUMS_FILENAME_RE.match(href) - or any(keyword in href.lower() - for keyword in _CHECKSUM_NAME_KEYWORDS)))] - # call _remote_url to construct the full URLs for the manifest files - manifest_urls = [ - _remote_url(base_url, rel_dir + href) for href in manifest_hrefs] - # crawl the manifest URLs concurrently to fetch their text contents - manifest_texts = crawl_executor.map(_fetch_optional_remote_text, - manifest_urls) - # create a dictionary mapping manifest hrefs to their fetched text contents - manifest_text_by_href = { - href: text for href, text in zip(manifest_hrefs, manifest_texts) - if text is not None} - # track only files explicitly covered by parsed manifest lines - files_covered_by_manifest: set[str] = set() - - def record_manifest_matches( - matches: list[tuple[str, str]], algorithm: str) -> None: - """ - Record matches from a manifest file into the file_checksums dictionary. - Args: - matches: List of (checksum, filename) extracted from the manifest. - algorithm: The checksum algorithm used in the manifest. - """ - for checksum, filename in matches: - try: - full_path = _resolve_manifest_path(filename, rel_dir) - except ValueError: - continue + from .discovery import discover - file_checksums[full_path] = (algorithm, checksum) - files_covered_by_manifest.add(full_path) - - # Iterate over each entry in the current directory - for entry_type, href in entries: - # if the entry is a collection (directory), skip it for now - if entry_type == "collection": - continue - # get the manifest text for this href from the prefetched manifest texts - text = manifest_text_by_href.get(href) - # skip if manifest text is None, meaning the file could not be fetched - if text is None: - continue - # Match against expected pattern to extract covered directory and algo - sibling_match = _SUMS_FILENAME_RE.match(href) - if sibling_match: - algorithm = sibling_match.group("algorithm") - # match if the checksum is valid for the given algorithm - matches = _manifest_matches(text, algorithm) - # add each path and its checksum to the file_checksums dictionary - record_manifest_matches(matches, algorithm) - # go to the next entry since this one has been processed - continue - - # lines in manifest text that are not empty after stripping whitespace - lines = [line for line in text.splitlines() if line.strip()] - - # search for the checksum algorithm in the filename using regex - name_match = _ALGORITHM_IN_NAME_RE.search(href) - # if a match is found, use the matched algorithm; otherwise, use unknown - algorithm = (name_match.group(1).lower() if name_match - else _UNKNOWN_CHECKSUM_ALGORITHM) - # only matches that are valid for the given algorithm are considered - matches = _manifest_matches(text, algorithm) - # if the file is empty or the number of valid matches is too low, skip - if (not lines or len(matches) < _MANIFEST_LINE_MATCH_RATIO * len(lines) - ): - continue - # record the matches from the manifest into file_checksums dictionary - record_manifest_matches(matches, algorithm) - - # After processing all entries, add any uncovered directories to the stack - for entry_type, href in entries: - rel_path = rel_dir + href - if entry_type == "collection": - directories_to_open.append(rel_path) - else: - # add files not explicitly covered by any manifest line with None - if rel_path not in files_covered_by_manifest: - file_checksums.setdefault(rel_path, (None, None)) - # return the complete mapping of relative paths to their checksums organized by algo - return file_checksums + def inventory(context): + return {entry.path: (entry.checksum_algorithm, entry.checksum) + for entry in discover(context)} + + if _context is None: + with OperationContext(RemoteSpec.parse(base_url)) as context: + return inventory(context) + return inventory(_context) def build_repo( @@ -981,27 +714,111 @@ def build_repo( config_file: Path | str | None = None, remotes: list[dict | str] | dict | str | None = None, overwrite: bool = False, + *, + dataset_url: str | None = None, + index_format: str | None = None, + allow_remote_commands: bool = False, + remote_hash: bool = False, + backend: str | None = None, + backend_options=None, +) -> "Repo": + """ + Build a remote catalog using the deprecated builder interface. + + Prefer ``Repo.init(path)`` followed by ``Repo.add(url_template)`` for + remote datasets. + Discovery reads listings and published checksum manifests without + downloading dataset files. + + Args: + repo_path (Path): Destination repository path. + dataset_name (str): Dataset name recorded in metadata and used for + the default CyVerse URL. + fmt_entries (list[dict], optional): Filename formats and TSV names. + If omitted, reuse existing formats or create a generic path catalog. + config_file (Path | str, optional): Configuration file or repository + directory supplying formats and, unless overridden, data remotes. + Cannot be combined with ``fmt_entries``. + remotes (list | dict | str, optional): Data remotes to record. + These do not change the discovery root. + overwrite (bool): Allow replacement of an existing destination. + Defaults to False; existing formats may be reused without it. + dataset_url (str, optional): Exact discovery root. Defaults to the + CyVerse curated-data directory for ``dataset_name``. + index_format (str, optional): Obsolete listing option. None, + ``auto``, and ``cyverse-html`` are accepted; detection is automatic. + allow_remote_commands (bool): Obsolete option, ignored when supplied. + remote_hash (bool): Unsupported when True. Defaults to False; + discovery does not compute checksums from dataset contents. + backend (str, optional): Registered discovery backend name. + backend_options (mapping, optional): Non-secret backend configuration. + + Returns: + Repo: Repository containing the prepared catalog. + + Raises: + ValueError: If the dataset name or catalog configuration is invalid. + FileExistsError: If the destination exists and cannot be reused. + FileNotFoundError: If the supplied configuration file is missing. + CapabilityError: If remote hashing or an unsupported index is requested. + DownloadError: If remote discovery fails. + """ + warnings.warn( + "build_repo is deprecated; use Repo.init(path) and " + "Repo.add(url_template)", + DeprecationWarning, stacklevel=2) + if remote_hash: + raise CapabilityError( + "remote_hash is no longer supported: catalog discovery does not read " + "dataset payloads to compute checksums") + if index_format not in {None, "auto", "cyverse-html"}: + raise CapabilityError("Directory indexes are detected automatically") + if index_format is not None or allow_remote_commands: + warnings.warn( + "index_format and allow_remote_commands are obsolete and ignored", + DeprecationWarning, stacklevel=2) + dataset_name = validate_path_component(dataset_name, label="dataset name") + base_url = (dataset_url if dataset_url is not None else + _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/")) + source = RemoteSpec.parse( + base_url, backend=backend, backend_options=backend_options) + with OperationContext(source) as context: + return _build_repo(repo_path, dataset_name, fmt_entries, config_file, + remotes, overwrite, context) + + +def _build_repo( + repo_path: Path, + dataset_name: str, + fmt_entries: list[dict] | None = None, + config_file: Path | str | None = None, + remotes: list[dict | str] | dict | str | None = None, + overwrite: bool = False, + source=None, ) -> "Repo": """ - Build a hallmark repository from the given dataset and format entries. - Creates separate TSV files for each fmt, and one config.yml file with the manifest. + Build a catalog using a resolved data source. Args: - repo_path: Path where the hallmark repo will be created. - fmt_entries: dict entries with "fmt", "db", and optional "name" keys. - If None, the function will attempt to detect the format entries automatically. - config_file: Path to an existing config.yml file to load fmt entries from. - remotes: Optional list of remote repos to add to the repo. Each remote can be a - dict with "name" and "url" keys, or a str representing the name of the remote. + repo_path (Path): Destination repository path. + dataset_name (str): Dataset name recorded in metadata. + fmt_entries (list[dict], optional): Entries with ``fmt``, ``db``, + and optional ``name`` keys. Reuse existing formats or a generic + path format when omitted. + config_file (Path | str, optional): Configuration file or directory + supplying formats and data remotes. + remotes (list | dict | str, optional): Data remotes to record. + overwrite (bool): Allow replacement of the destination. Defaults + to False, except when reusing its existing catalog formats. + source (OperationContext): Resolved source supplied by ``build_repo``. Returns: - The initialized Repo object, sitting on the main branch. + Repo: Repository containing the prepared catalog. """ # validate the dataset name to ensure it is a valid path component dataset_name = validate_path_component(dataset_name, label="dataset name") - # CyVerse's curated Data Commons datasets live at a predictable URL, - # built directly from dataset_name. - base_url = _remote_url(_CYVERSE_CURATED_BASE, f"{dataset_name}/") + # build_repo resolves the source URL before calling this helper + base_url = source.remote.url # Determine if remotes were provided by the user remotes_provided = remotes is not None @@ -1054,10 +871,9 @@ def build_repo( remote_files = None def _ensure_remote_files_listed(): """ - Used by build_repo. - Ensure that the remote files have been listed and checksums collected. - This function is called lazily to avoid unnecessary network requests if the - user provides their own fmt entries or chooses to reuse an existing config. + List remote files after validating the destination and configuration. + + Reuse the inventory on subsequent calls within this build. """ # create nonlocal references to the outer variables so they can be modified nonlocal file_checksums, remote_files @@ -1068,111 +884,28 @@ def _ensure_remote_files_listed(): # use time.perf_counter() to measure the time taken to list remote files # perf_counter() is preferred for measuring elapsed time with high resolution list_start = time.perf_counter() - file_checksums = list_remote_files(base_url) + file_checksums = list_remote_files(base_url, _context=source) print(f"Found {len(file_checksums)} files in " f"{time.perf_counter() - list_start:.1f}s") # sort the remote files by their relative paths for consistent ordering remote_files = sorted(file_checksums) - # if the repo path already exists and has a config.yml, we will reuse it - reused_from_existing_repo = False - # records the interactive include_drives choice when fmts are auto-detected, so it - # can be persisted to meta.yml and isn't lost once the build finishes - detect_include_drives: bool | None = None - - if fmt_entries is None and existing_fmt_entries: - print(f"Found existing config.yml with {len(existing_fmt_entries)} fmt(s):") - for entry in existing_fmt_entries: - print(f" {entry['fmt']!r} -> {entry['db']}") - # prompt the user to either use the existing fmt entries or update them - keep_choice = prompt_choice("Use these fmts as-is, or update the list? " - "[use/update]: ", {"use", "update"}) - # if user chooses to use existing fmts, set them as the current fmt_entries - if keep_choice == "use": - fmt_entries = existing_fmt_entries - reused_from_existing_repo = True - - # if repo path exists and we are not reusing it, check if overwrite is allowed - if repo_path.exists() and not reused_from_existing_repo and not overwrite: - # raise an error to prevent accidental overwriting of an existing repo + # preserve existing formats when refreshing through the legacy builder + reused_from_existing_repo = fmt_entries is None and bool(existing_fmt_entries) + if reused_from_existing_repo: + fmt_entries = existing_fmt_entries + elif repo_path.exists() and not overwrite: raise FileExistsError( f'Destination "{repo_path}" already exists. ' "Use overwrite=True to replace it.") - if fmt_entries is None: - # prompt the user to choose how to supply fmt entries - choice = prompt_choice( - "Load fmts from an existing config file, detect them " - "automatically, or input them yourself? " - "[config/detect/input]: ", {"config", "detect", "input"}) - if choice == "config": - # prompt the user for the path to an existing config.yml or repo directory - config_path = Path( - input( - "Path to the existing config.yml or repository " - "directory to load fmts from: ").strip()).expanduser() - # if the provided path is a directory, assume it contains a config.yml - if config_path.is_dir(): - config_path = config_path / "config.yml" - # if the config file does not exist, raise an error - if not config_path.is_file(): - raise FileNotFoundError(f"Config file does not exist: {config_path}") - - # load the config.yml file and extract the fmt entries - loaded_config = load_yaml_file(config_path) - fmt_entries = fmt_entries_from_config(loaded_config) - if not fmt_entries: - raise ValueError( - f"No fmt entries found in config file {config_path}.") - # if remotes were not provided and the loaded config has a "remote" key - if not remotes_provided and loaded_config.get("remote"): - # normalize the remotes from the loaded config and use them - remotes = normalize_remotes(loaded_config["remote"]) - - elif choice == "detect": - # ask the user if they want to include drive files during fmt detection - include_drives = prompt_choice("Include drive/archive files during fmt " \ - "detection? [yes/no]: ", {"yes", "no"}, ) == "yes" - # remember the choice so it can be recorded in meta.yml below - detect_include_drives = include_drives - - # check if the remote files have already been listed; if not, list them now - _ensure_remote_files_listed() - detected_fmts = detect_fmt(remote_files, include_drives=include_drives) - if not detected_fmts: - raise ValueError( - "No fmts could be automatically detected from this dataset.") - fmt_entries = [] - # create a fmt entry for each detected format - if len(detected_fmts) == 1: - # if there is only one detected format, assume it is "data.tsv" - print(f"Detected fmt: {detected_fmts[0]!r}\n" - f" Only one fmt detected; using db name 'data.tsv'.") - fmt_entries.append({"fmt": detected_fmts[0], "db": "data.tsv"}) - else: - for fmt in detected_fmts: - # prompt the user to enter a database name for each detected format - db_name = normalize_tsv_name(input( - f"Detected fmt: {fmt!r}\n" - f" Enter a db name for this fmt (e.g. 'data.tsv'): ")) - fmt_entries.append({"fmt": fmt, "db": db_name}) - elif choice == "input": - fmt_entries = [] - print("Enter fmt entries one at a time. Leave the fmt blank to finish.") - # allow the user to input multiple fmt entries manually - while True: - fmt = input("fmt (blank to finish): ").strip() - # if the user inputs a blank fmt, exit the loop - if not fmt: - break - # prompt the user to enter a database name for the provided format - db_name = normalize_tsv_name(input(" db name for this fmt: ")) - # append the user-provided fmt and db name to the fmt_entries list - fmt_entries.append({"fmt": fmt, "db": db_name}) + fmt_entries = [{"fmt": "{path}", "db": "data.tsv"}] # normalize the fmt entries to ensure they are valid and consistent fmt_entries = _normalize_fmt_entries(fmt_entries) + _ensure_remote_files_listed() + # if the repo path already exists and has a config.yml, we will reuse it if reused_from_existing_repo: repo = existing_repo @@ -1256,23 +989,6 @@ def _ensure_remote_files_listed(): static_file_entries = [] meta_file_entries: list[dict] = [] - # identify which unmatched paths need checksums computed (those without a checksum) - paths_needing_checksums = [ - rel_path - for rel_path in unmatched_paths - if file_checksums.get(rel_path, (None, None))[1] is None] - computed_checksums = {} - if paths_needing_checksums: - # construct the full URLs for the unmatched paths that need checksums - urls = [_remote_url(base_url, rel_path) for rel_path in paths_needing_checksums] - # use ThreadPoolExecutor to compute checksums for small remote files in parallel - with ThreadPoolExecutor(max_workers=min(_STATIC_CHECKSUM_MAX_WORKERS, len(urls)) - ) as executor: - # map the _checksum_small_remote_url function over the list of URLs - results = executor.map(_checksum_small_remote_url, urls) - # zip the paths needing checksums with their computed results into a dict - computed_checksums = dict(zip(paths_needing_checksums, results)) - for rel_path in unmatched_paths: path = Path(rel_path) stem_name = path.stem.split(".")[-1] @@ -1281,9 +997,6 @@ def _ensure_remote_files_listed(): is_meta_file = stem_name.lower() == "meta" # for each unmatched path, look up its checksum algo, checksum = file_checksums.get(rel_path, (None, None)) - # if not available, use the computed checksum from the parallel computation - if checksum is None: - algo, checksum = computed_checksums[rel_path] # create an entry for the static file with its relative path entry = {"file": rel_path} @@ -1308,9 +1021,6 @@ def _ensure_remote_files_listed(): # hallmark's bookkeeping: create and commit the meta.yml file meta_dict: dict = {"dataset": dataset_name} - # record the include_drives choice so a detect-built repo's fmts stay consistent - if detect_include_drives is not None: - meta_dict["detect_include_drives"] = detect_include_drives repo.dothm.dump_yml(meta_dict, "meta") repo.dothm.index.add(["meta.yml"]) repo.dothm.index.commit(f"Initialize dataset: {dataset_name}") @@ -1347,8 +1057,19 @@ def _ensure_remote_files_listed(): if not remotes: # use a default remote named "origin" pointing to the base_url if none provided remotes = [{"name": "origin"}] + if source.remote.backend_options: + remotes[0]["backend_options"] = thaw_backend_options( + source.remote.backend_options) # create the final remotes list by adding the base_url to each remote entry final_remotes = [{"url": base_url, **remote} for remote in remotes] + default_backend = RemoteSpec.parse(base_url).backend + for remote in final_remotes: + if remote["url"] == base_url: + if source.remote.backend != default_backend: + remote.setdefault("backend", source.remote.backend) + if source.remote.backend_options: + remote.setdefault("backend_options", thaw_backend_options( + source.remote.backend_options)) repo.state.config["remote"] = final_remotes # if a meta file entry exists, add it to the config @@ -1391,4 +1112,4 @@ def _ensure_remote_files_listed(): # reload the repo state from the .dothm directory to reflect the latest changes repo.state = repo.dothm.load() - return repo \ No newline at end of file + return repo diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index bde03eb..79cc293 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -7,27 +7,41 @@ from __future__ import annotations +import re +from copy import deepcopy +from dataclasses import dataclass, field from pathlib import Path from string import Formatter -from typing import Dict, Optional +from typing import Dict, Iterable, Optional from .helper_functions import ( as_list_of_dicts, coerce_fmt_value, normalize_nonempty_string, validate_path_component, validate_relative_path) +from .state import DEFAULT_DB, METADATA_COLUMNS + +from .transport.base import ( + RemoteSpec, backend_name, reject_controls, thaw_backend_options) + def _update_remote_config( config: dict, remote_name: Optional[str], remote_url: Optional[str], + remote_backend: Optional[str] = None, + remote_backend_options=None, ) -> None: """ Used by set_config. - Update the repository configuration with a new remote name and/or URL. + Update the repository data remote and its transport configuration. Args: config (dict): The repository configuration dictionary. remote_name (str, optional): New remote repository name. remote_url (str, optional): New remote repository URL. + remote_backend (str, optional): Registered backend name. An empty string + restores automatic selection; None leaves it unchanged. + remote_backend_options (mapping, optional): Replace backend options. + An empty mapping clears options; None leaves them unchanged. Raises: ValueError: If the remote configuration is invalid or if the specified @@ -81,6 +95,13 @@ def _update_remote_config( selected["name"] = remote_name if remote_url is not None: selected["url"] = remote_url + if remote_backend == "": + selected.pop("backend", None) + elif remote_backend is not None: + selected["backend"] = backend_name(remote_backend) + if remote_backend_options is not None: + selected["backend_options"] = thaw_backend_options( + remote_backend_options) # normalize the remotes configuration to ensure it is a list of dictionaries normalized = normalize_remotes(remotes) # if preserve_list is True, store the normalized list; @@ -156,6 +177,26 @@ def normalize_remotes(remotes) -> list[dict]: else: raise ValueError(f"remote {index} must be a string or dictionary") + if "auth" in entry: + raise ValueError( + "Hallmark auth profiles have been removed. Configure the host in " + "~/.ssh/config, use its alias in the remote URL, and remove the " + "obsolete auth field from config.yml.") + if set(entry) - {"name", "url", "backend", "backend_options"}: + raise ValueError( + "Remote entries support only name, url, backend and " + "backend_options fields") + if "backend" in entry: + entry["backend"] = backend_name(entry["backend"]) + if "backend_options" in entry: + entry["backend_options"] = thaw_backend_options( + entry["backend_options"]) + if isinstance(entry.get("url"), str): + reject_controls(entry["url"], "Remote URL") + # allow names without URLs while configuring a data remote + RemoteSpec.parse( + entry["url"], backend=entry.get("backend"), + backend_options=entry.get("backend_options")) # for each required key ("name" and "url"), validate that it exists for key in ("name", "url"): # if the key is not present in the entry, skip to the next key @@ -254,64 +295,311 @@ def single_data_fmt(config: dict) -> Optional[str]: return fmt.strip() -def get_or_create_branch_data_spec(config: dict) -> dict: +# Template fields that would collide with catalog columns. The checksum +# algorithm names are read as digests when a download plan is built. +RESERVED_FIELDS = frozenset( + {"path", "sha1", "md5", "sha256", "sha512", *METADATA_COLUMNS}) + + +def check_template_fields(fmt: str) -> list[str]: + """ + Validate the field names of a filename template. + + Args: + fmt (str): Filename template. + + Returns: + list[str]: Unique field names in the order they appear. + + Raises: + ValueError: If a field name is reserved catalog metadata. """ - Ensure the configuration contains a valid data specification. + fields = fmt_fields(fmt) + reserved = [name for name in fields if name in RESERVED_FIELDS] + if reserved: + raise ValueError( + f"Template field {reserved[0]!r} is reserved catalog metadata") + return fields + + +@dataclass(frozen=True) +class CatalogEntry: + """ + A data entry that owns a catalog table. + + Attributes: + index (int): Position of the entry in the config ``data`` list. + fmt (str, optional): Filename template relative to the worktree, or to + ``url`` for a remote entry. Builder path catalogs may omit it. + db (str): TSV name of the entry's catalog table. + url (str, optional): Remote directory the template is relative to. + Entries without a URL describe files in the worktree. + backend (str, optional): Registered transport backend for ``url``. + backend_options (dict, optional): Backend-specific configuration. + """ + + index: int + fmt: Optional[str] + db: str + url: Optional[str] = None + backend: Optional[str] = None + backend_options: Optional[dict] = field(default=None, hash=False, compare=False) + + @property + def is_remote(self) -> bool: + """True when the entry catalogs files at a remote URL.""" + return bool(self.url) + + +def _data_entry_list(config) -> list: + """Return the config ``data`` section as a list, or an empty list.""" + if not isinstance(config, dict): + return [] + return as_list_of_dicts(config.get("data")) or [] + + +def is_placeholder(entry) -> bool: + """ + Return True for a data entry that does not catalog files yet, such as the + ``encoding`` placeholder written by ``hallmark init``. + """ + return isinstance(entry, dict) and not {"fmt", "url", "file", "db"} & set(entry) + - If the ``data`` entry is missing or malformed, it is initialized with - a single empty dictionary. +def _entry_fmt(entry: dict) -> Optional[str]: + """Return an entry's stripped template, or None if it has none.""" + fmt = entry.get("fmt") + return fmt.strip() if isinstance(fmt, str) and fmt.strip() else None + + +def catalog_entries(config) -> list[CatalogEntry]: + """ + Return the data entries that own catalog tables, in config order. + + Placeholders, static ``file`` entries and entries that are not mappings + are skipped. An entry without an explicit ``db`` uses ``data.tsv``. Args: - config (dict): Repository configuration. + config (dict): The repository configuration dictionary. Returns: - dict: The branch data specification. + list[CatalogEntry]: The catalog entries. + + Raises: + ValueError: If the data section is malformed or an entry names an + invalid TSV. + """ + data = config.get("data") if isinstance(config, dict) else None + if data is not None and as_list_of_dicts(data) is None: + raise ValueError('config "data" must be a mapping or list of mappings') + entries = [] + for index, entry in enumerate(_data_entry_list(config)): + if not isinstance(entry, dict): + continue + fmt = _entry_fmt(entry) + if fmt is None and not entry.get("db"): + continue + url = entry.get("url") + entries.append(CatalogEntry( + index=index, + fmt=fmt, + db=normalize_tsv_name(entry["db"]) if entry.get("db") else DEFAULT_DB, + url=url if isinstance(url, str) and url else None, + backend=entry.get("backend"), + backend_options=entry.get("backend_options"))) + return entries + + +def catalog_table_names(config) -> list[str]: + """ + Return the TSV names referenced by catalog entries, skipping invalid names. + + Used when loading a repository, before any operation validates the + configuration. + + Args: + config (dict): The repository configuration dictionary. + + Returns: + list[str]: Unique TSV names in config order. + """ + names = [] + for entry in _data_entry_list(config): + if not isinstance(entry, dict): + continue + if entry.get("db"): + try: + name = normalize_tsv_name(entry["db"]) + except ValueError: + continue + elif _entry_fmt(entry) is not None: + name = DEFAULT_DB + else: + continue + if name not in names: + names.append(name) + return names + + +def find_entry(config, fmt: str) -> Optional[CatalogEntry]: + """ + Return the catalog entry whose template is ``fmt``. + + Args: + config (dict): The repository configuration dictionary. + fmt (str): Filename template. + + Returns: + CatalogEntry | None: The matching entry, or None. + """ + return next((entry for entry in catalog_entries(config) if entry.fmt == fmt), + None) + + +def next_db_name(config, used: Iterable[str] = ()) -> str: + """ + Choose the TSV name for a new data entry. + + The first entry uses ``data.tsv``. Later entries use ``data-N.tsv`` with a + number larger than any in the config or in ``used``, so a removed table's + history is not reused by a different template. + + Args: + config (dict): The repository configuration dictionary. + used (Iterable[str]): TSV names used previously, such as in history. + + Returns: + str: The TSV name for the new entry. + """ + entries = catalog_entries(config) + if all(entry.db != DEFAULT_DB for entry in entries): + return DEFAULT_DB + numbers = [1] + for name in [*(entry.db for entry in entries), *used]: + match = re.fullmatch(r"data-(\d+)\.tsv", str(name), flags=re.IGNORECASE) + if match: + numbers.append(int(match.group(1))) + return f"data-{max(numbers) + 1}.tsv" + + +def add_entry(config: dict, entry: dict) -> int: + """ + Add a data entry to the configuration. + + The first entry fills the placeholder written by ``hallmark init`` so that + single-template repositories keep their existing layout. Later entries are + appended. + + Args: + config (dict): The repository configuration dictionary, updated in place. + entry (dict): The new data entry. + + Returns: + int: The entry's position in the config ``data`` list. + + Raises: + ValueError: If the config ``data`` section is malformed. """ - # get the "data" section from the configuration data = config.get("data") - # if "data" already defines exactly one dictionary entry, use it as-is - if isinstance(data, list) and len(data) == 1 and isinstance(data[0], dict): - return data[0] - # otherwise (missing, empty, or malformed), initialize it with a single empty dict - config["data"] = [{}] - return config["data"][0] + entries = [] if data is None else as_list_of_dicts(data) + if entries is None: + raise ValueError('config "data" must be a mapping or list of mappings') + entries = list(entries) + config["data"] = entries + if not catalog_entries(config): + for index, existing in enumerate(entries): + if is_placeholder(existing): + merged = dict(entry) + for key, value in existing.items(): + # a remote entry has no use for an empty encoding placeholder + if key in merged or (entry.get("url") and value is None): + continue + merged[key] = value + entries[index] = merged + return index + entries.append(entry) + return len(entries) - 1 + + +def remove_entry(config: dict, index: int) -> dict: + """ + Remove a data entry from the configuration. + Args: + config (dict): The repository configuration dictionary, updated in place. + index (int): The entry's position in the config ``data`` list. -def require_branch_data_spec(repo) -> dict: + Returns: + dict: The removed entry. + """ + entries = list(_data_entry_list(config)) + removed = entries.pop(index) + config["data"] = entries + return removed + + +def sole_data_spec(config: dict, *, create: bool = False) -> Optional[dict]: """ - Return the branch data specification. Raises RuntimeError if - the configuration does not define exactly one entry under ``data``. + Return the data entry that single-template settings apply to. + + This is the only catalog entry, or the placeholder when there is none. Args: - repo: Repository object. + config (dict): The repository configuration dictionary. + create (bool): Add an empty entry when none exists. Defaults to False. Returns: - dict: The branch data specification. + dict | None: The data entry, or None when none exists and ``create`` + is False. + + Raises: + RuntimeError: If the branch has more than one catalog entry. """ - # call the helper function to get the single data specification from the config - spec = _data_spec_or_none(repo.state.config) - # raise a RuntimeError if there is no valid single data specification - if spec is None: + entries = catalog_entries(config) + if len(entries) > 1: raise RuntimeError( - 'branch config must define exactly one entry under "data" in config.yml') - return spec + "this branch tracks several templates; use hallmark add TEMPLATE or " + "hallmark rm --cached TEMPLATE instead of changing config.yml") + data = _data_entry_list(config) + if entries: + return data[entries[0].index] + placeholder = next((entry for entry in data if is_placeholder(entry)), None) + if placeholder is not None or not create: + return placeholder + config["data"] = [*data, {}] + return config["data"][-1] + + +def get_or_create_branch_data_spec(config: dict) -> dict: + """ + Ensure the configuration contains a data specification for single-template + settings and return it. + + Args: + config (dict): Repository configuration. + + Returns: + dict: The branch data specification. + """ + return sole_data_spec(config, create=True) -def branch_fmt(repo) -> str: +def require_branch_data_spec(repo) -> dict: """ - Return the configured filename format. Raises RuntimeError if - no valid format string is defined. + Return the branch data specification. Raises RuntimeError if the + configuration defines no data entry or several catalog entries. Args: repo: Repository object. Returns: - str: The format string stored in ``data[0].fmt``. + dict: The branch data specification. """ - return normalize_nonempty_string( - require_branch_data_spec(repo).get("fmt"), - label="branch data[0].fmt", - exception_type=RuntimeError) + spec = sole_data_spec(repo.state.config) + if spec is None: + raise RuntimeError( + 'branch config must define an entry under "data" in config.yml') + return spec def set_config( @@ -321,6 +609,8 @@ def set_config( remote_name: Optional[str] = None, remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, + remote_backend: Optional[str] = None, + remote_backend_options=None, ) -> dict: """ Update the repository configuration. @@ -333,13 +623,17 @@ def set_config( fmt (str, optional): Filename format. remote_name (str, optional): Remote repository name. remote_url (str, optional): Remote repository URL. + remote_backend (str, optional): Registered backend name. An empty string + restores automatic selection; None leaves it unchanged. + remote_backend_options (mapping, optional): Replace backend options. + An empty mapping clears options; None leaves them unchanged. encoding_updates (dict, optional): Encoding values to merge into the existing configuration. Returns: dict: The updated configuration. """ - config = repo.state.config + config = deepcopy(repo.state.config) # raise a ValueError if the provided config is not a dictionary if not isinstance(config, dict): raise ValueError("repository config must be a mapping") @@ -374,12 +668,18 @@ def set_config( remote_name = normalize_nonempty_string(remote_name, label="remote_name") # if a new remote URL is provided, validate that it is a non-empty string if remote_url is not None: + if isinstance(remote_url, str): + reject_controls(remote_url, "Remote URL") remote_url = normalize_nonempty_string(remote_url, label="remote_url") # if a new format string or encoding updates are provided if fmt is not None or encoding_updates is not None: - # ensure that the "data" section has exactly one entry and retrieve it - spec = get_or_create_branch_data_spec(config) + # retrieve the only data entry; several templates cannot share settings + spec = sole_data_spec(config, create=True) + if fmt is not None and spec.get("url"): + raise ValueError( + "cannot change the template of a remote entry; run " + "hallmark add URL/TEMPLATE instead") updated_spec = {} # if a new format string is provided, update the "fmt" key in the spec if fmt is not None: @@ -407,29 +707,38 @@ def set_config( # exclude "fmt" and "encoding" keys since they are already handled if key not in {"fmt", "encoding"}: updated_spec[key] = value - # update the first entry in the "data" list of the configuration - config["data"][0] = updated_spec - - # if a new remote name or URL is provided, update the remote configuration - if remote_name is not None or remote_url is not None: - _update_remote_config(config, remote_name, remote_url) - + # replace the entry at its position in the "data" list + index = next(position for position, entry in enumerate(config["data"]) + if entry is spec) + config["data"][index] = updated_spec + + # update the remote when its name, URL, or transport configuration changes + if any(value is not None for value in ( + remote_name, remote_url, + remote_backend, remote_backend_options)): + _update_remote_config(config, remote_name, remote_url, + remote_backend, remote_backend_options) + + repo.state.config = config return config -def branch_encodings(repo) -> list[dict]: +def entry_encodings(spec: Optional[dict], fmt: str) -> list[dict]: """ - Return the configured filename encodings. + Return the filename encodings that apply to a template. Args: - repo: Repository object. + spec (dict, optional): The data entry or placeholder holding the + encoding rules. + fmt (str): The template being added. Returns: - list[dict]: A list containing the encoding specification, or an - empty list if no encodings are defined. + list[dict]: A list containing the encoding specification for ``fmt``, + or an empty list if no encodings are defined. """ - spec = require_branch_data_spec(repo) - return [spec] if isinstance(spec.get("encoding"), dict) else [] + if not isinstance(spec, dict) or not isinstance(spec.get("encoding"), dict): + return [] + return [{**spec, "fmt": fmt}] def fmt_fields(fmt: str) -> list[str]: @@ -506,4 +815,4 @@ def row_to_path(row, fmt: str) -> Path: # render the path using the format string and coerced values rendered_path = fmt.format(**values) # validate the rendered path to ensure it is a safe relative path - return validate_relative_path(rendered_path, label="formatted data path") \ No newline at end of file + return validate_relative_path(rendered_path, label="formatted data path") diff --git a/mod/hallmark/repo_manifest.py b/mod/hallmark/repo_manifest.py index 0c27019..66a4e7b 100644 --- a/mod/hallmark/repo_manifest.py +++ b/mod/hallmark/repo_manifest.py @@ -1,12 +1,14 @@ from __future__ import annotations +from dataclasses import dataclass from pathlib import Path from collections.abc import Iterator import pandas as pd from .helper_functions import safe_str -from .repo_config import fmt_fields, row_to_path, single_data_fmt +from .repo_config import CatalogEntry, catalog_entries, fmt_fields, row_to_path +from .transport.base import literal_path def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: @@ -48,19 +50,184 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: return pd.DataFrame(rows, columns=columns) +@dataclass(frozen=True) +class CatalogRow: + """ + One cataloged file. + + Attributes: + path (str): POSIX path relative to the worktree and the data source. + record (dict): The catalog row. + db (str): TSV name of the catalog table. + entries (tuple[CatalogEntry, ...]): Data entries sharing the table. + local (bool): True when the file's content is tracked in the worktree + and object store; False for catalog-only rows such as remote files. + """ + + path: str + record: dict + db: str + entries: tuple + local: bool + + +def _cell_text(value) -> str: + """Return a catalog cell as text, treating missing markers as empty.""" + if value is None or (not isinstance(value, str) and pd.isna(value)): + return "" + text = str(value).strip() + return "" if text.lower() in {"none", "nan", ""} else text + + +def row_relative_path(row, formats) -> Path: + """ + Resolve a catalog row to one literal relative path. + + A ``path`` column is authoritative. Otherwise exactly one template must + render the row. + + Args: + row: Catalog row mapping. + formats (list[str]): Templates of the data entries sharing the table. + + Returns: + Path: The relative file path. + + Raises: + ValueError: If the row cannot be resolved to a unique safe path. + """ + value = row.get("path") + if _cell_text(value): + return Path(literal_path(str(value)).as_posix()) + paths = set() + for template in formats: + try: + paths.add(row_to_path(row, template).as_posix()) + except (ValueError, KeyError): + continue + if len(paths) != 1: + raise ValueError("Cannot resolve a catalog row to a unique file path") + return Path(paths.pop()) + + +def table_groups(config) -> dict[str, tuple[CatalogEntry, ...]]: + """ + Group catalog entries by the TSV that holds their rows. + + Args: + config (dict): The repository configuration dictionary. + + Returns: + dict[str, tuple[CatalogEntry, ...]]: Entries keyed by TSV name. + """ + groups: dict[str, list[CatalogEntry]] = {} + for entry in catalog_entries(config): + groups.setdefault(entry.db, []).append(entry) + return {name: tuple(entries) for name, entries in groups.items()} + + +def is_local_table(entries, frame: pd.DataFrame) -> bool: + """ + Return True when a table's rows describe worktree files stored as objects. + + Remote entries, path catalogs, and builder tables shared by several + templates are catalog-only: their files are not committed as objects, + restored by checkout, or checked in the worktree. + + Args: + entries (tuple[CatalogEntry, ...]): Data entries sharing the table. + frame (pd.DataFrame): The catalog table. + + Returns: + bool: True for a local table. + """ + return (len(entries) == 1 and entries[0].fmt is not None + and not entries[0].is_remote + and "sha1" in frame.columns and "path" not in frame.columns) + + +def iter_catalog_rows(state, *, local: bool | None = None) -> Iterator[CatalogRow]: + """ + Iterate over the files cataloged by every data entry. + + Args: + state: Repository state. + local (bool, optional): True for local rows only, False for + catalog-only rows only, or None for all rows. + + Yields: + CatalogRow: Each cataloged file. + + Raises: + ValueError: If a row cannot be resolved to a safe relative path. + """ + for name, entries in table_groups(state.config).items(): + frame = state.table(name) + if frame.empty: + continue + local_table = is_local_table(entries, frame) + if local is not None and local_table != local: + continue + formats = [entry.fmt for entry in entries if entry.fmt] + for record in frame.to_dict(orient="records"): + if local_table: + path = row_to_path(record, formats[0]) + else: + path = row_relative_path(record, formats) + yield CatalogRow(path.as_posix(), record, name, entries, local_table) + + +def row_fingerprint(record) -> str: + """ + Summarize the recorded content of a catalog row. + + The SHA-1 is preferred, then a published checksum, then size and + modification time. + + Args: + record: Catalog row mapping. + + Returns: + str: A comparable fingerprint. + """ + sha1 = _cell_text(record.get("sha1")) + if sha1: + return sha1.lower() + checksum = _cell_text(record.get("checksum")) + if checksum: + algorithm = _cell_text(record.get("checksum_algorithm")) or "unknown" + return f"{algorithm.lower()}:{checksum.lower()}" + size = _cell_text(record.get("size_bytes")) + mtime = _cell_text(record.get("mtime")) + return f"size:{size};mtime:{mtime}" + + +def catalog_map(state) -> dict[str, CatalogRow]: + """ + Map every cataloged path to its catalog row. + + Args: + state: Repository state. + + Returns: + dict[str, CatalogRow]: Rows keyed by relative POSIX path. + """ + return {row.path: row for row in iter_catalog_rows(state)} + + def iter_manifest_entries( state, *, fmt: str | None = None, ) -> Iterator[tuple[Path, str]]: """ - Iterate over the manifest entries in the repository state. + Iterate over the local manifest entries in the repository state. Args: state: Repository state. - fmt (str | None): Optional filename format to use for parsing paths. - If not provided, the format will be determined from the repository - configuration. + fmt (str | None): Optional filename format applied to ``data.tsv``. + If not provided, the local catalog entries of the repository + configuration are used. Returns: Iterator[tuple[Path, str]]: An iterator over tuples containing the @@ -69,16 +236,13 @@ def iter_manifest_entries( Yields: tuple[Path, str]: Tuples of the relative file path and its SHA-1 checksum. """ + if fmt is None: + for row in iter_catalog_rows(state, local=True): + yield Path(row.path), str(row.record["sha1"]) + return # If the repository state has no data, return immediately if state.data.empty: return - # If no format is provided, determine it from the repository configuration. - if fmt is None: - # get the single data format from the repository configuration - fmt = single_data_fmt(state.config) - # if the format is still None, return immediately - if fmt is None: - return # convert each record in the repository state to a dictionary for record in state.data.to_dict(orient="records"): # yield a tuple containing the relative file path and its SHA-1 checksum @@ -87,13 +251,13 @@ def iter_manifest_entries( def manifest_map(state, *, fmt: str | None = None) -> dict[str, str]: """ - Create a mapping from file paths to SHA-1 checksums. + Create a mapping from local file paths to SHA-1 checksums. Args: state: Repository state. - fmt (str | None): Optional filename format to use for parsing paths. - If not provided, the format will be determined from the repository - configuration. + fmt (str | None): Optional filename format applied to ``data.tsv``. + If not provided, the local catalog entries of the repository + configuration are used. Returns: dict[str, str]: Dictionary mapping relative file paths to their @@ -101,4 +265,4 @@ def manifest_map(state, *, fmt: str | None = None) -> dict[str, str]: """ # call iter_manifest_entries to get an iterator of (path, checksum) tuples return {path.as_posix(): checksum - for path, checksum in iter_manifest_entries(state, fmt=fmt)} \ No newline at end of file + for path, checksum in iter_manifest_entries(state, fmt=fmt)} diff --git a/mod/hallmark/repo_remote.py b/mod/hallmark/repo_remote.py new file mode 100644 index 0000000..a0ad7a5 --- /dev/null +++ b/mod/hallmark/repo_remote.py @@ -0,0 +1,233 @@ +"""Catalog remote files that match a URL template, without downloading them.""" + +from __future__ import annotations + +from typing import Optional + +import pandas as pd + +from .discovery import ( + TemplateMatcher, discover, split_url_template, template_url) +from .fmt_detection import detect_fmt +from .paraframe import ParaFrame +from .repo_config import ( + DEFAULT_DB, add_entry, find_entry, next_db_name, row_to_path) +from .repo_manifest import catalog_map +from .transport import OperationContext, RemoteSpec +from .transport.base import thaw_backend_options + +# Columns of a remote catalog table, before the template fields. +REMOTE_COLUMNS = ("sha1", "checksum_algorithm", "checksum", "size_bytes", "mtime") + + +def _remote(base: str, backend=None, backend_options=None) -> RemoteSpec: + """Parse a directory URL, inheriting a registered source's backend.""" + if backend is None: + from .sources import backend_for_url + + inherited = backend_for_url(base) + if inherited is not None: + backend, options = inherited + backend_options = options if backend_options is None else backend_options + return RemoteSpec.parse(base, backend=backend, backend_options=backend_options) + + +def _require_template(url: str) -> tuple[str, str]: + """Split a URL template, pointing directory URLs to ls-remote.""" + base, template = split_url_template(url) + if template is None: + raise ValueError( + f"{base} is a directory; run hallmark ls-remote {base} to see " + "suggested templates, then add URL/TEMPLATE") + return base, template + + +def match_remote_template(url: str, *, progress=False, backend=None, + backend_options=None): + """ + List the remote files matching a URL template without downloading them. + + Only directories the template can match are listed. Published checksum + manifests in those directories provide checksums. + + Args: + url (str): URL template such as ``https://host/ER2/{src}_{day}.h5``. + progress (bool | callable): Display discovery progress or receive updates. + backend (str, optional): Registered backend for the URL. + backend_options (dict, optional): Backend-specific configuration. + + Returns: + tuple[RemoteSpec, str, pd.DataFrame, int]: The remote directory, the + template, catalog rows sorted by path (with a ``path`` column), and + the number of listed files that matched the pattern but not the + template's exact formatting. + + Raises: + ValueError: If the URL or template is invalid. + DownloadError: If discovery fails. + """ + base, template = _require_template(url) + matcher = TemplateMatcher(template) + remote = _remote(base, backend, backend_options) + with OperationContext(remote) as context: + entries = discover(context, include=matcher.glob, progress=progress, + descend=matcher.may_contain) + rows, mismatched = [], 0 + for entry in entries: + values = matcher.match(entry.path) + if values is None: + continue + # a file must be re-creatable from its fields, as local rows are + try: + rendered = row_to_path(values, template).as_posix() + except (KeyError, TypeError, ValueError): + rendered = None + if rendered != entry.path: + mismatched += 1 + continue + published_sha1 = (entry.checksum if entry.checksum_algorithm == "sha1" + else "") + rows.append({ + "path": entry.path, "sha1": published_sha1 or "", + "checksum_algorithm": entry.checksum_algorithm or "", + "checksum": entry.checksum or "", + "size_bytes": "" if entry.size is None else str(entry.size), + "mtime": "" if entry.mtime is None else str(entry.mtime), + **values}) + frame = pd.DataFrame( + rows, columns=["path", *REMOTE_COLUMNS, *matcher.fields], dtype=object) + return remote, template, frame, mismatched + + +def add_remote_template(repo, url: str, *, dry_run=False, progress=False, + backend=None, backend_options=None) -> ParaFrame: + """ + Catalog the remote files matching a URL template as a data entry. + + A new template becomes another data entry recording the directory URL. + Adding a tracked URL template again syncs it: new files are added, + changed files updated, and files no longer listed are dropped. + + Args: + repo: Repository object. + url (str): URL template such as ``https://host/ER2/{src}_{day}.h5``. + dry_run (bool): List the matching files without staging them. + progress (bool | callable): Display discovery progress or receive updates. + backend (str, optional): Registered backend recorded for the URL. + backend_options (dict, optional): Backend-specific configuration. + + Returns: + ParaFrame: The matching files with their ``path`` and field values. + ``attrs["mismatched"]`` counts listed files skipped because the + template cannot re-create their names exactly. + + Raises: + ValueError: If the URL is invalid, matches nothing, conflicts with a + tracked template, or matches files tracked by another template. + DownloadError: If discovery fails. + """ + base, template = _require_template(url) + TemplateMatcher(template) + config = repo.state.config + entry = find_entry(config, template) + if entry is not None and (entry.url or "").rstrip("/") != base.rstrip("/"): + where = "the worktree" if not entry.url else entry.url + raise ValueError( + f"template {template!r} is already tracked from {where}; run " + f"hallmark rm --cached {template!r} first") + if entry is not None and backend is None: + backend, backend_options = entry.backend, entry.backend_options + remote, template, frame, mismatched = match_remote_template( + url, progress=progress, backend=backend, backend_options=backend_options) + if frame.empty: + detail = (f"; {mismatched} listed file(s) fit the pattern but not the " + "template's formatting" if mismatched else "") + raise ValueError( + f"template {template!r} did not match any files at {base}{detail}") + result = ParaFrame(frame[["path", *frame.columns[len(REMOTE_COLUMNS) + 1:]]]) + result.attrs["mismatched"] = mismatched + if dry_run: + return result + owned = catalog_map(repo.state) + overlap = sorted(path for path in frame["path"] + if path in owned and owned[path].db != getattr(entry, "db", None)) + if overlap: + shown = ", ".join(overlap[:5]) + more = f" and {len(overlap) - 5} more" if len(overlap) > 5 else "" + raise ValueError( + f"template {template!r} matches files tracked by another template " + f"({shown}{more}); use a more specific template or run " + "hallmark rm --cached TEMPLATE first") + if entry is None: + db = next_db_name(config, used=repo._used_table_names()) + spec = {"fmt": template, "url": base} + # record a backend chosen explicitly or inherited from a source release + if remote.backend != RemoteSpec.parse(base).backend: + spec["backend"] = remote.backend + if remote.backend_options: + spec["backend_options"] = thaw_backend_options(remote.backend_options) + if db != DEFAULT_DB: + spec["db"] = db + add_entry(config, spec) + else: + db = entry.db + # a remote template is synced as a whole, like git add + repo.state.replace(frame.drop(columns=["path"]), db=db) + repo.dothm.dump(repo.state) + return result + + +def suggest_templates(url: str, *, progress=False, backend=None, + backend_options=None) -> tuple[str, list[tuple[str, int]], int]: + """ + List a remote directory and suggest URL templates for its files. + + Args: + url (str): Directory URL. + progress (bool | callable): Display discovery progress or receive updates. + backend (str, optional): Registered backend for the URL. + backend_options (dict, optional): Backend-specific configuration. + + Returns: + tuple[str, list[tuple[str, int]], int]: The directory URL, suggested + URL templates with the number of files each matches (most first), and + the number of files listed. + + Raises: + ValueError: If the URL has template fields or is otherwise invalid. + DownloadError: If discovery fails. + """ + base, template = split_url_template(url.rstrip("/") + "/") + if template is not None: + raise ValueError("ls-remote takes a directory URL without template fields") + remote = _remote(base, backend, backend_options) + with OperationContext(remote) as context: + paths = [entry.path for entry in discover(context, progress=progress)] + suggestions = [] + for candidate in detect_fmt(paths): + try: + matcher = TemplateMatcher(candidate) + # inference can mistake literal braces for fields or pick reserved names + except ValueError: + continue + count = 0 + for path in paths: + values = matcher.match(path) + if values is None: + continue + try: + if row_to_path(values, candidate).as_posix() == path: + count += 1 + except (KeyError, TypeError, ValueError): + continue + if count: + suggestions.append((template_url(base, candidate), count)) + suggestions.sort(key=lambda item: (-item[1], item[0])) + return base, suggestions, len(paths) + + +def remote_url_template(entry) -> Optional[str]: + """Return a remote entry's full URL template, or None for a local entry.""" + if not entry.url or not entry.fmt: + return None + return template_url(entry.url, entry.fmt) diff --git a/mod/hallmark/repo_state.py b/mod/hallmark/repo_state.py index 340a021..c24ef4a 100644 --- a/mod/hallmark/repo_state.py +++ b/mod/hallmark/repo_state.py @@ -10,7 +10,8 @@ from .helper_functions import load_yaml from .objects import Objects -from .state import State +from .repo_config import catalog_table_names +from .state import DEFAULT_DB, State def _load_revision_yaml( repo, @@ -75,10 +76,13 @@ def _copy_current_state(repo, *, include_data: bool) -> State: """ # if include_data is True, copy current data; otherwise, create an empty DataFrame data = (repo.state.data.copy() if include_data else State().data.copy()) + tables = ({name: frame.copy() for name, frame in repo.state.tables.items()} + if include_data else {}) # new State object with deep copies of current config and meta, and the copied data return State( - config=deepcopy(repo.state.config), meta=deepcopy(repo.state.meta), data=data) + config=deepcopy(repo.state.config), meta=deepcopy(repo.state.meta), data=data, + tables=tables) def _load_revision_state(repo, revision: str) -> State: @@ -97,10 +101,21 @@ def _load_revision_state(repo, revision: str) -> State: data_text = repo.dothm.git.show(f"{revision}:data.tsv") # Load the config.yml and meta.yml content from the specified revision, # falling back to current state if not found - return State( + state = State( config=_load_revision_yaml(repo, revision, "config.yml", repo.state.config), meta=_load_revision_yaml(repo, revision, "meta.yml", repo.state.meta), data=_parse_data_tsv(data_text)) + # load the catalogs of additional data entries named by that revision's config + for name in catalog_table_names(state.config): + if name == DEFAULT_DB: + continue + try: + state.tables[name] = _parse_data_tsv( + repo.dothm.git.show(f"{revision}:{name}")) + # a catalog staged but never committed is empty at this revision + except GitCommandError: + state.tables[name] = State().data.copy() + return state def load_branch_data(repo, branch: str) -> State: diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index 254d2c1..fedf7d7 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -5,7 +5,6 @@ from .error import CheckoutError from .helper_functions import ( SymlinkPathError, resolve_contained_path, validate_relative_path) -from .repo_config import branch_fmt from .repo_manifest import manifest_map, iter_manifest_entries @@ -17,7 +16,7 @@ def effective_cwd(repo) -> Path: Args: repo (repo): repository object Returns: - path: The current working directory if it is inside the repository + Path: The current working directory if it is inside the repository worktree; otherwise, the worktree root. ''' if repo.worktree is None: @@ -85,13 +84,12 @@ def tracked_paths(repo) -> set[Path]: repo (Repo): Repository object. Returns: - set[Path]: Paths of all files tracked in the current - repository state. + set[Path]: Paths of all local files tracked in the current + repository state. Catalog-only files, such as remote files, are + not managed in the worktree and are excluded. ''' - # call branch_fmt to get the filename format from the repository configuration - fmt = branch_fmt(repo) - # use iter_manifest_entries to iterate over the manifest entries and collect paths - return {path for path, _ in iter_manifest_entries(repo.state, fmt=fmt)} + # use iter_manifest_entries to iterate over the local entries and collect paths + return {path for path, _ in iter_manifest_entries(repo.state)} def worktree_changes(repo, expected_checksums: dict[str, str] @@ -168,10 +166,8 @@ def ensure_clean_tracked_files(repo) -> None: if repo.worktree is None: raise CheckoutError("cannot checkout without a worktree") - # determine the filename format from the repository configuration - fmt = branch_fmt(repo) - # get the expected checksums for all tracked files in the repository - expected_checksums = manifest_map(repo.state, fmt=fmt) + # get the expected checksums for all local tracked files in the repository + expected_checksums = manifest_map(repo.state) # try to get the modified and missing tracked files using worktree_changes try: modified, missing = worktree_changes(repo, expected_checksums) diff --git a/mod/hallmark/sources.py b/mod/hallmark/sources.py new file mode 100644 index 0000000..ef247a1 --- /dev/null +++ b/mod/hallmark/sources.py @@ -0,0 +1,168 @@ +"""Named data sources with explicit releases and collection directory roots.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib import metadata +from threading import RLock +from types import MappingProxyType + +from .transport.base import RemoteSpec, backend_name, literal_path + + +@dataclass(frozen=True) +class SourceRelease: + """A release URL and collection names mapped to relative directory roots. + + ``backend`` and ``backend_options`` are optional adapter configuration for + source authors. Collection paths and catalog paths use the release root. + """ + + url: str + collections: Mapping + description: str = "" + backend: str | None = None + backend_options: Mapping = field(default_factory=dict) + collection_descriptions: Mapping = field(default_factory=dict) + + def __post_init__(self): + remote = RemoteSpec.parse(self.url, backend=self.backend, + backend_options=self.backend_options) + collections = {} + for name, paths in self.collections.items(): + backend_name(name) + if isinstance(paths, str): + paths = (paths,) + paths = tuple(literal_path(path).as_posix() for path in paths) + if not paths: + raise ValueError("Collections must have at least one directory root") + collections[name] = paths + object.__setattr__(self, "collections", MappingProxyType(collections)) + object.__setattr__(self, "backend_options", remote.backend_options) + object.__setattr__(self, "collection_descriptions", + MappingProxyType(dict(self.collection_descriptions))) + + def remote(self): + """Return the validated transport specification without connecting.""" + return RemoteSpec.parse(self.url, backend=self.backend, + backend_options=self.backend_options) + + +@dataclass(frozen=True) +class DataSource: + """A named service with a description and explicit release definitions.""" + + name: str + description: str + releases: Mapping + + def __post_init__(self): + backend_name(self.name) + releases = dict(self.releases) + if not releases or any(not isinstance(value, SourceRelease) + for value in releases.values()): + raise ValueError("Sources require named SourceRelease definitions") + for name in releases: + backend_name(name) + object.__setattr__(self, "releases", MappingProxyType(releases)) + + +def _desi_release(release, production): + collections = { + "redshifts": (f"spectro/redux/{production}/zcatalog",), + "spectra": (f"spectro/redux/{production}/healpix", + f"spectro/redux/{production}/tiles"), + "vac": ("vac",), + } + if release == "dr1": + collections["lss"] = ("survey/catalogs/dr1/LSS",) + return SourceRelease( + f"https://data.desi.lbl.gov/public/{release}/", collections, + description="Early Data Release" if release == "edr" else "Data Release 1", + collection_descriptions={ + "redshifts": "Merged redshift catalogs (all published versions)", + "spectra": "HEALPixel and tile product directories", + "vac": "Value-added products", + **({"lss": "Large-scale-structure catalogs"} if release == "dr1" else {}), + }) + + +_registered = {"desi": DataSource( + "desi", "Dark Energy Spectroscopic Instrument public data", { + "edr": _desi_release("edr", "fuji"), + "dr1": _desi_release("dr1", "iron"), + })} +_loaded = {} +_lock = RLock() + + +def _entry_points(): + entries = metadata.entry_points() + if hasattr(entries, "select"): + return list(entries.select(group="hallmark.sources")) + return list(entries.get("hallmark.sources", ())) + + +def register_source(source: DataSource): + """Register a source for this process; installed plugins use hallmark.sources. + + Entry points expose a ``DataSource`` instance. Duplicate names are rejected. + Registration and source descriptions never contact a data service. + """ + if not isinstance(source, DataSource): + raise TypeError("Expected a DataSource") + with _lock: + if (source.name in _registered or source.name in _loaded + or any(item.name == source.name for item in _entry_points())): + raise ValueError(f"Duplicate data source {source.name!r}") + _registered[source.name] = source + + +def get_source(name: str) -> DataSource: + """Resolve a registered source or load its installed descriptor.""" + backend_name(name) + with _lock: + entries = [item for item in _entry_points() if item.name == name] + if len(entries) > 1 or (entries and name in _registered): + raise ValueError(f"Duplicate data source {name!r}") + if name in _registered: + return _registered[name] + if name in _loaded: + return _loaded[name] + if not entries: + raise ValueError(f"Unknown data source {name!r}; use hallmark sources") + try: + source = entries[0].load() + except Exception: + raise ValueError(f"Unable to load data source {name!r}") from None + if not isinstance(source, DataSource) or source.name != name: + raise ValueError( + f"Source plugin {name!r} must expose a matching DataSource") + _loaded[name] = source + return source + + +def list_sources(): + """Return source descriptors in name order without discovering files.""" + names = set(_registered) | {item.name for item in _entry_points()} + return [get_source(name) for name in sorted(names)] + + +def backend_for_url(url: str): + """ + Return the backend a registered source release declares for a URL. + + Args: + url (str): Remote directory URL. + + Returns: + tuple[str, Mapping] | None: The backend name and options of the release + whose URL contains ``url``, or None if no such release declares one. + """ + for source in list_sources(): + for release in source.releases.values(): + root = release.url.rstrip("/") + "/" + if release.backend and (url.rstrip("/") + "/").startswith(root): + return release.backend, release.backend_options + return None diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 78c72d2..9b55cb5 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -19,6 +19,10 @@ # Define the default columns for the state DataFrame COLUMNS = ["sha1"] +# The catalog table of the first data entry; other entries name their own TSV. +DEFAULT_DB = "data.tsv" +# Remote file metadata columns. They describe a row but do not identify it. +METADATA_COLUMNS = ("checksum_algorithm", "checksum", "size_bytes", "mtime") def _normalized_state_data(frame: pd.DataFrame) -> pd.DataFrame: @@ -50,6 +54,9 @@ def _normalized_state_data(frame: pd.DataFrame) -> pd.DataFrame: # normalize the data by retaining only the relevant columns normalized = frame.loc[:, columns].copy() + # remote rows without a published SHA-1 store an empty value, not NaN + normalized["sha1"] = normalized["sha1"].map( + lambda value: "" if pd.isna(value) else value) for column in parameter_columns: # normalize non-checksum columns to string type, # replacing NaN values with empty strings @@ -67,8 +74,12 @@ class State: Attributes: config: Repository configuration values. meta: Repository metadata. - data: Tabular file index containing indexed object checksums - (``sha1``) and associated metadata. + data: Tabular file index of the default catalog (``data.tsv``), + containing indexed object checksums (``sha1``) and associated + metadata. + tables: Catalog tables of additional data entries, keyed by TSV name. + changed_tables: Additional tables modified since they were last written. + removed_tables: Additional tables to delete when the state is written. """ config: dict = field(default_factory=dict) @@ -76,31 +87,83 @@ class State: data: pd.DataFrame = field( default_factory=lambda: pd.DataFrame(columns=COLUMNS) ) + tables: dict = field(default_factory=dict) + changed_tables: set = field(default_factory=set, repr=False, compare=False) + removed_tables: set = field(default_factory=set, repr=False, compare=False) - def update(self, pf) -> None: + def table(self, db: str = DEFAULT_DB) -> pd.DataFrame: """ - Merge ``ParaFrame`` rows into the state database. + Return a catalog table, or an empty table if it has no rows yet. + + Args: + db (str): TSV name of the table. Defaults to ``data.tsv``. + + Returns: + pd.DataFrame: The catalog table. + """ + if db == DEFAULT_DB: + return self.data + frame = self.tables.get(db) + return frame if frame is not None else pd.DataFrame(columns=COLUMNS) + + def set_table(self, db: str, frame: pd.DataFrame) -> None: + """ + Replace a catalog table and mark it for writing. + + Args: + db (str): TSV name of the table. + frame (pd.DataFrame): The new table contents. + """ + if db == DEFAULT_DB: + self.data = frame + return + self.tables[db] = frame + self.changed_tables.add(db) + self.removed_tables.discard(db) + + def drop_table(self, db: str) -> None: + """ + Remove a catalog table. ``data.tsv`` is emptied rather than deleted, + because every Hallmark repository contains it. + + Args: + db (str): TSV name of the table. + """ + if db == DEFAULT_DB: + self.data = pd.DataFrame(columns=COLUMNS) + return + self.tables.pop(db, None) + self.changed_tables.discard(db) + self.removed_tables.add(db) + + def update(self, pf, db: str = DEFAULT_DB) -> None: + """ + Merge ``ParaFrame`` rows into a catalog table. Existing rows with matching keys are updated, while new rows are - appended. + appended. Checksums and remote file metadata are not part of the key. Args: pf (ParaFrame): ``ParaFrame`` containing rows to add or update. + db (str): TSV name of the table. Defaults to ``data.tsv``. Returns: None. """ + current = self.table(db) if pf.empty: - # create empty DataFrame with the same columns as the existing state data. - columns = (self.data.columns if len(self.data.columns) else COLUMNS) + # create empty DataFrame with the same columns as the existing table. + columns = (current.columns if len(current.columns) else COLUMNS) incoming = pd.DataFrame(columns=columns) # if the provided ParaFrame is not empty, normalize its data else: incoming = _normalized_state_data(pf) # Merge the incoming rows with the existing state. - merged = pd.concat([self.data, incoming], ignore_index=True, sort=False) + merged = pd.concat([current, incoming], ignore_index=True, sort=False) - key_columns = [column for column in merged.columns if column != "sha1"] + value_columns = [column for column in merged.columns if column != "sha1"] + key_columns = [column for column in value_columns + if column not in METADATA_COLUMNS] if key_columns: # Remove duplicate entries while keeping the most recent row. deduped = merged.drop_duplicates(subset=key_columns, keep="last") @@ -109,22 +172,23 @@ def update(self, pf) -> None: deduped = merged.tail(1) # reset the index of the deduplicated DataFrame - # retain only the "sha1" and key columns. - self.data = ( - deduped.loc[:, ["sha1", *key_columns]].reset_index(drop=True)) + # retain the "sha1" column followed by the other columns. + self.set_table( + db, deduped.loc[:, ["sha1", *value_columns]].reset_index(drop=True)) - def replace(self, pf) -> None: + def replace(self, pf, db: str = DEFAULT_DB) -> None: """ - Replace the contents of the state database. + Replace the contents of a catalog table. Existing rows are discarded and replaced with the rows from the provided ``ParaFrame``. Args: pf (ParaFrame): ``ParaFrame`` containing the replacement rows. + db (str): TSV name of the table. Defaults to ``data.tsv``. Returns: None. """ # call the normalization function to ensure consistent data types and structure - self.data = _normalized_state_data(pf) + self.set_table(db, _normalized_state_data(pf)) diff --git a/mod/hallmark/transport/__init__.py b/mod/hallmark/transport/__init__.py new file mode 100644 index 0000000..22ca24b --- /dev/null +++ b/mod/hallmark/transport/__init__.py @@ -0,0 +1,118 @@ +"""Manage transport selection and resources for each remote operation.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path +from threading import Event, Lock, local + +import requests + +from .settings import SSHSettings +from .base import DataBackend, RemoteEntry, RemoteSpec, TransferCancelled + + +class OperationContext: + """ + Manage transport resources for one discovery or download operation. + + Leaving the context closes its transport and HTTP sessions. An exception + also cancels active work. + + Args: + remote (RemoteSpec): Data source and transport configuration. + output_root (Path | str, optional): Root used to validate download paths. + """ + + def __init__( + self, + remote, + output_root=None, + ): + self.remote = remote + self.settings = SSHSettings() + self.output_root = Path(output_root) if output_root is not None else None + self.cancelled = Event() + self.on_bytes = None + # Optional predicate receiving a relative directory path ending in "/". + # Listings skip directories for which it returns False. + self.descend = None + self.text_limit = 16 * 1024 * 1024 + self.listing_timeout = 60 + self._local = local() + self._lock = Lock() + self._sessions = [] + from ..backends import get_backend + + try: + self.transport = get_backend(remote.backend)(self) + except BaseException: + self.cancelled.set() + for session in self._sessions: + session.close() + raise + self.backend = self.transport + + @contextmanager + def executor(self, max_workers): + """ + Create a worker pool whose tasks share this operation's resources. + + Args: + max_workers (int): Maximum concurrent workers. + + Yields: + ThreadPoolExecutor: Worker pool. Interrupted work is cancelled + before waiting for workers to finish. + """ + executor = ThreadPoolExecutor(max_workers=max_workers) + try: + yield executor + except BaseException: + self.cancel() + raise + finally: + executor.shutdown(wait=True, cancel_futures=True) + + def session(self): + """Return the reusable HTTP session for the current thread.""" + if not hasattr(self._local, "session"): + session = requests.Session() + with self._lock: + self._sessions.append(session) + self._local.session = session + return self._local.session + + def read_text(self, path): + """Read remote metadata within the configured size limit.""" + return self.transport.read_text(path, self.text_limit) + + def check_cancelled(self): + """Raise TransferCancelled if this operation was cancelled.""" + if self.cancelled.is_set(): + raise TransferCancelled("Download cancelled") + + def cancel(self): + """Cancel this operation and stop its active transport work.""" + self.cancelled.set() + cancel = getattr(self.transport, "cancel", None) + if cancel is not None: + cancel() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + try: + if exc_type is not None: + self.cancel() + finally: + try: + self.transport.close() + finally: + for session in self._sessions: + session.close() + + +__all__ = ["DataBackend", "OperationContext", "RemoteEntry", "RemoteSpec"] diff --git a/mod/hallmark/transport/base.py b/mod/hallmark/transport/base.py new file mode 100644 index 0000000..e29cbc5 --- /dev/null +++ b/mod/hallmark/transport/base.py @@ -0,0 +1,371 @@ +"""Shared transport interfaces and data-remote URL validation.""" + +from __future__ import annotations + +import ipaddress +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from urllib.parse import quote, unquote, urljoin, urlsplit, urlunsplit + +from ..error import HallmarkError +from ..helper_functions import validate_relative_path + + +class DownloadError(HallmarkError): + """Raised when download setup, transfer, or verification fails.""" + + +class RemoteConfigurationError(DownloadError, ValueError): + """Raised when a data remote or its local settings are invalid.""" + + +class RemoteObjectMissing(DownloadError): + """Raised when the server reports that a remote file is missing.""" + + +class CapabilityError(DownloadError): + """Raised when the source or client does not support an operation.""" + + +class TransferCancelled(DownloadError): + """Raised when a transfer is cancelled.""" + + +def backend_name(value): + """Validate a registered backend name, never an import path.""" + if not isinstance(value, str) or not re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", value): + raise RemoteConfigurationError( + "Backend name must match [a-z][a-z0-9_-]{0,63}") + return value + + +def freeze_backend_options(options=None): + """Copy YAML-compatible options into a deeply immutable mapping.""" + active = set() + + def freeze(value): + if isinstance(value, (Mapping, list, tuple)): + if id(value) in active: + raise RemoteConfigurationError("Backend options cannot contain cycles") + active.add(id(value)) + try: + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise RemoteConfigurationError( + "Backend option keys must be strings") + return MappingProxyType({key: freeze(item) + for key, item in value.items()}) + return tuple(freeze(item) for item in value) + finally: + active.remove(id(value)) + if value is None or isinstance(value, (str, bool, int, float)): + return value + raise RemoteConfigurationError( + "Backend options must contain YAML scalar, list, or mapping values") + + if options is None: + options = {} + if not isinstance(options, Mapping): + raise RemoteConfigurationError("Backend options must be a mapping") + return freeze(options) + + +def thaw_backend_options(options=None): + """Return an independent YAML-compatible copy of backend options.""" + def thaw(value): + if isinstance(value, Mapping): + return {key: thaw(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [thaw(item) for item in value] + return value + + return thaw(freeze_backend_options(options)) + + +def reject_controls(value: str, label: str) -> None: + """Reject control characters and text that cannot be encoded as UTF-8.""" + try: + value.encode("utf-8") + except UnicodeError: + raise RemoteConfigurationError(f"{label} must be valid UTF-8") from None + if any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in value): + raise RemoteConfigurationError(f"{label} contains control characters") + + +def literal_path(value) -> Path: + """Validate a literal catalog path without decoding or trimming it.""" + raw = str(value) + reject_controls(raw, "Remote path") + if not raw.strip(): + raise RemoteConfigurationError("Remote path cannot be empty") + try: + return validate_relative_path(raw, label="remote path") + except ValueError as exc: + raise RemoteConfigurationError(str(exc)) from None + + +def ssh_host(value: str) -> str: + # Aliases are not necessarily DNS names. Limit expansion tokens to characters + # that cannot become shell syntax in a user's ProxyCommand/Match configuration. + """Validate an SSH hostname, configuration alias, or IPv6 address.""" + if ":" in value: + if "%" in value: + raise RemoteConfigurationError("Scoped IPv6 SSH hosts are unsupported") + try: + ipaddress.IPv6Address(value) + except ValueError: + raise RemoteConfigurationError("Invalid IPv6 SSH host") from None + elif not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", value): + raise RemoteConfigurationError("Invalid SSH host or config alias") + return value + + +def ssh_user(value: str) -> str: + """Validate and return an SSH username.""" + if not isinstance(value, str) or not re.fullmatch( + r"[A-Za-z0-9_][A-Za-z0-9_.-]*", value + ): + raise RemoteConfigurationError("Invalid SSH user") + return value + + +@dataclass(frozen=True) +class RemoteEntry: + """ + A discovered file and its available metadata. + + Attributes: + path (str): Literal path relative to the dataset root. + size (int, optional): File size in bytes. + mtime (float, optional): Modification time reported by the server. + checksum_algorithm (str, optional): Published checksum algorithm. + checksum (str, optional): Published digest. + + Unavailable metadata remains None; file contents are not read to fill it. + """ + + path: str + size: int | None = None + mtime: float | None = None + checksum_algorithm: str | None = None + checksum: str | None = None + + +@dataclass(frozen=True) +class RemoteSpec: + """ + A parsed data-remote URL and transport configuration. + + Use ``parse`` to validate a URL. SSH roots are decoded once; catalog + paths appended to the root are treated as literal filenames. + + Attributes: + url (str): Original source URL after trimming surrounding whitespace. + scheme (str): HTTP, HTTPS, SSH, or SFTP scheme in lowercase. + host (str): Hostname or SSH configuration alias. + root (str): Dataset root, decoded for SSH and SFTP. + user (str, optional): Explicit SSH username. + port (int, optional): Explicit port number. + backend (str): Registered backend name, resolved from the URL by default. + backend_options (Mapping): Deeply immutable backend configuration. + """ + + url: str = field(repr=False) + scheme: str + host: str + root: str = field(repr=False) + user: str | None = field(default=None, repr=False) + port: int | None = None + backend: str | None = None + backend_options: Mapping = field(default_factory=lambda: MappingProxyType({}), + repr=False, hash=False) + + def __post_init__(self): + selected = self.backend + if selected is None: + selected = "ssh" if self.scheme in {"ssh", "sftp"} else "http" + if selected == "http" and (self.host == "cyverse.org" + or self.host.endswith(".cyverse.org")): + selected = "cyverse" + object.__setattr__(self, "backend", backend_name(selected)) + object.__setattr__(self, "backend_options", + freeze_backend_options(self.backend_options)) + + @classmethod + def parse(cls, url: str, *, + backend=None, backend_options=None) -> RemoteSpec: + """ + Parse and validate a data-remote URL. + + Args: + url (str): HTTP(S) URL or SSH/SFTP URL with an absolute dataset root. + backend (str, optional): Registered backend name. Defaults to URL + detection. Parsing does not load plugins. + backend_options (Mapping, optional): Backend-specific configuration. + + Returns: + RemoteSpec: Validated source description. + + Raises: + RemoteConfigurationError: If the URL is invalid. + """ + if not isinstance(url, str) or not url.strip(): + raise RemoteConfigurationError("Remote URL must be a non-empty string") + reject_controls(url, "Remote URL") + url = url.strip() + try: + parsed = urlsplit(url) + host, port = parsed.hostname, parsed.port + except ValueError: + raise RemoteConfigurationError("Invalid remote URL host or port") from None + scheme = parsed.scheme.lower() + if scheme not in {"http", "https", "ssh", "sftp"}: + raise RemoteConfigurationError( + "Use an http(s):// or ssh://HOST/absolute/root/ URL; " + "scp shorthand and this scheme are unsupported" + ) + if not host or port == 0 or parsed.netloc.endswith(":"): + raise RemoteConfigurationError("Invalid remote URL host or port") + if scheme in {"ssh", "sftp"}: + ssh_host(host) + if parsed.password is not None: + raise RemoteConfigurationError("SSH URLs cannot contain passwords") + if "?" in url or "#" in url: + raise RemoteConfigurationError( + "SSH URLs cannot contain query or fragment fields; " + "percent-encode filename characters" + ) + if re.search(r"%(?![0-9A-Fa-f]{2})", parsed.path): + raise RemoteConfigurationError("Invalid URL percent escape") + try: + root = unquote(parsed.path, errors="strict") + user = ( + ssh_user(unquote(parsed.username, errors="strict")) + if parsed.username is not None + else None + ) + except UnicodeError: + raise RemoteConfigurationError("SSH URL must encode UTF-8") from None + reject_controls(root, "SSH root") + if ( + not root.startswith("/") + or root.startswith("//") + or ".." in root.split("/") + or "\\" in root + ): + raise RemoteConfigurationError( + "SSH root must be absolute, without traversal or backslashes" + ) + else: + root, user = parsed.path, None + return cls(url, scheme, host, root, user, port, + backend, backend_options) + + def pathname(self, relative_path: str) -> str: + """Append a literal relative path to the dataset root.""" + return self.root.rstrip("/") + "/" + literal_path(relative_path).as_posix() + + def file_url(self, relative_path: str) -> str: + """Encode a literal catalog path as a URL beneath the dataset root.""" + path = literal_path(relative_path).as_posix() if relative_path else "" + if relative_path.endswith("/") and path: + path += "/" + if self.scheme in {"http", "https"}: + return urljoin(self.url.rstrip("/") + "/", quote(path, safe="/")) + parsed = urlsplit(self.url) + root = self.root.rstrip("/") + "/" + path + return urlunsplit((self.scheme, parsed.netloc, quote(root, safe="/"), "", "")) + + @property + def display(self) -> str: + """Source scheme and host, with credentials and paths omitted.""" + host = self.host if re.fullmatch(r"[A-Za-z0-9_.:-]+", self.host) else "remote" + return f"{self.scheme}://{host}" + + +class DataBackend: + """ + Common interface for dataset discovery and data transfer. + + Implementations receive shared cancellation state and thread-local HTTP + sessions through ``context``. Fetch may run concurrently: protect mutable + backend state, check ``context.check_cancelled()`` during long operations, + and release owned resources in ``close``. Authentication secrets belong in + local configuration, not persisted backend options. + + Recursive listings should skip a subdirectory when ``context.descend`` is + set and returns False for its relative path (ending in ``/``). Discovery + sets it to avoid listing directories a filename template cannot match; + backends that ignore it remain correct, only slower. + + Args: + context (OperationContext): Resources and settings for this operation. + """ + + def __init__(self, context): + self.context = context + + def prepare(self): + """Prepare for discovery or transfer; repeated calls must be safe.""" + pass + + def fetch(self, relative_path, destination, *, chunk_size=8192): + """ + Transfer a file to a caller-managed temporary destination. + + Args: + relative_path (str): Literal path relative to the dataset root. + destination (Path): Local file to write. + chunk_size (int): Read size for transports that support it. + + Checksum verification and atomic replacement are handled by the downloader. + """ + raise NotImplementedError + + def list_entries(self): + """Return the relative paths from a recursive directory listing.""" + return [entry.path for entry in self.iter_entries()] + + def iter_entries(self, on_directory=None): + """ + Yield file metadata from a recursive directory listing. + + Args: + on_directory (callable, optional): Callback receiving directory paths + relative to the dataset root. + + Raises: + CapabilityError: If the transport does not provide directory listings. + """ + raise CapabilityError("Recursive listing is unsupported by this transport") + + def read_text(self, relative_path, limit): + """ + Read a remote metadata file within a byte limit. + + Args: + relative_path (str): Metadata path relative to the dataset root. + limit (int): Maximum number of bytes to read. + + Returns: + str: Decoded metadata text. + + Raises: + CapabilityError: If the transport does not support metadata reads. + """ + raise CapabilityError("Reading metadata is unsupported by this transport") + + def close(self): + """Release connections and processes used by this transport.""" + pass + + def cancel(self): + """Stop active backend work after the context's cancellation is set.""" + pass + + +# Existing transport imports remain valid for downstream integrations. +Transport = DataBackend diff --git a/mod/hallmark/transport/cyverse.py b/mod/hallmark/transport/cyverse.py new file mode 100644 index 0000000..f2940b0 --- /dev/null +++ b/mod/hallmark/transport/cyverse.py @@ -0,0 +1,40 @@ +"""CyVerse WebDAV HTML listing support, sharing the HTTP transfer backend.""" + +from html.parser import HTMLParser + +from .http import HttpBackend +from .index import _IndexParser, _parse_index + + +class CyVerseIndexParser(_IndexParser): + """Read CyVerse object and collection rows, including exact byte sizes.""" + + def handle_starttag(self, tag, attrs): + super().handle_starttag(tag, attrs) + if tag == "tr": + classes = dict(attrs).get("class", "").split() + if "object" in classes: + self.listing = True + self.row_kind = "directory" if "collection" in classes else "file" + + +def is_cyverse_index(text): + """Recognize CyVerse markup independently of its hosting domain.""" + class Probe(HTMLParser): + found = False + + def handle_starttag(self, tag, attrs): + if tag == "tr" and "object" in dict(attrs).get("class", "").split(): + self.found = True + + parser = Probe() + parser.feed(text) + return parser.found + + +class CyVerseBackend(HttpBackend): + """Discover CyVerse collections and transfer their files over HTTP(S).""" + + def _parse_index(self, text, directory): + return _parse_index(text, self.context.remote.url, directory, + parser_class=CyVerseIndexParser) diff --git a/mod/hallmark/transport/http.py b/mod/hallmark/transport/http.py new file mode 100644 index 0000000..ad21cad --- /dev/null +++ b/mod/hallmark/transport/http.py @@ -0,0 +1,146 @@ +"""HTTP transport using Requests with its normal .netrc and environment settings.""" + +from __future__ import annotations + +from collections import deque +from urllib.parse import unquote, urljoin, urlsplit + +import requests + +from ..helper_functions import REMOTE_REQUEST_TIMEOUT +from .base import (DownloadError, RemoteConfigurationError, RemoteEntry, + RemoteObjectMissing, DataBackend, reject_controls) +from .index import _parse_index, _response_directory + + +class HttpBackend(DataBackend): + """Transfer files and metadata using reusable Requests sessions.""" + def __init__(self, context): + super().__init__(context) + if context.remote.scheme not in {"http", "https"}: + raise RemoteConfigurationError("HTTP backends require an HTTP(S) URL") + self.text_urls = {} + + def _parse_index(self, text, directory): + """Read generic indexes and recognize the CyVerse dialect automatically.""" + from .cyverse import CyVerseIndexParser, is_cyverse_index + + if is_cyverse_index(text): + return _parse_index(text, self.context.remote.url, directory, + parser_class=CyVerseIndexParser) + return _parse_index(text, self.context.remote.url, directory) + + def iter_entries(self, on_directory=None): + """Yield metadata from recursive listings without reading payloads.""" + descend = getattr(self.context, "descend", None) + queue, visited = deque([""]), set() + while queue: + directory = queue.popleft() + if directory in visited: + continue + visited.add(directory) + self.context.check_cancelled() + text = self.context.read_text(directory) + canonical = _response_directory(self.context, directory) + if canonical != directory and canonical in visited: + continue + visited.add(canonical) + for path, is_directory, size, mtime in self._parse_index(text, canonical): + if is_directory: + if path not in visited and (descend is None or descend(path)): + queue.append(path) + else: + yield RemoteEntry(path=path, size=size, mtime=mtime) + if on_directory is not None: + on_directory(canonical) + + def _get(self, path): + """Open a streaming response with the configured request timeout.""" + return self.context.session().get( + getattr(self, "direct_url", None) or self.context.remote.file_url(path), + stream=True, + timeout=REMOTE_REQUEST_TIMEOUT, + ) + + def _error(self, exc): + # Requests exceptions can embed credentials in their URL or response body. + """Describe an HTTP failure without exposing response contents.""" + status = getattr(getattr(exc, "response", None), "status_code", None) + cls = RemoteObjectMissing if status == 404 else DownloadError + detail = f"HTTP {status}" if isinstance(status, int) else type(exc).__name__ + return cls(f"Failed to download from {self.context.remote.display}: {detail}") + + def _metadata_get(self, relative_path): + """Follow bounded metadata redirects only within the declared source root.""" + url = self.context.remote.file_url(relative_path) + root = urlsplit(self.context.remote.url) + root_path = unquote(root.path, errors="strict").rstrip("/") or "/" + + def endpoint(parsed): + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return parsed.scheme, parsed.hostname, port + + for _ in range(11): + self.context.check_cancelled() + try: + reject_controls(url, "Metadata URL") + target = urlsplit(url) + path = unquote(target.path, errors="strict") + reject_controls(path, "Metadata path") + within_root = (root_path == "/" or path == root_path + or path.startswith(root_path + "/")) + if (endpoint(target) != endpoint(root) or not within_root + or ".." in path.split("/") or "\\" in path + or target.username != root.username + or target.password != root.password): + raise DownloadError("Metadata redirect leaves the source root") + except (ValueError, UnicodeError): + raise DownloadError("Invalid metadata redirect URL") from None + response = self.context.session().get( + url, stream=True, timeout=REMOTE_REQUEST_TIMEOUT, + allow_redirects=False, + ) + if response.status_code not in {301, 302, 303, 307, 308}: + return response, url + location = response.headers.get("Location") + response.close() + if not isinstance(location, str) or not location: + raise DownloadError("Metadata redirect has no location") + url = urljoin(url, location) + raise DownloadError("Metadata redirect limit exceeded") + + def fetch(self, relative_path, destination, *, chunk_size=8192): + """Stream a file to the destination, checking for cancellation.""" + try: + with self._get(relative_path) as response: + response.raise_for_status() + with destination.open("wb") as handle: + for chunk in response.iter_content(chunk_size=chunk_size): + self.context.check_cancelled() + if chunk: + handle.write(chunk) + if self.context.on_bytes is not None: + self.context.on_bytes(len(chunk)) + except requests.RequestException as exc: + raise self._error(exc) from None + def read_text(self, relative_path, limit): + """Read metadata within the source root and byte limit.""" + content = bytearray() + try: + response, final_url = self._metadata_get(relative_path) + with response: + response.raise_for_status() + for chunk in response.iter_content(chunk_size=8192): + self.context.check_cancelled() + content.extend(chunk) + if len(content) > limit: + raise DownloadError("Remote text exceeds the configured limit") + encoding = response.encoding or "utf-8" + text = content.decode(encoding, errors="replace") + self.text_urls[relative_path] = final_url + return text + except requests.RequestException as exc: + raise self._error(exc) from None + + +HttpTransport = HttpBackend diff --git a/mod/hallmark/transport/index.py b/mod/hallmark/transport/index.py new file mode 100644 index 0000000..5208f07 --- /dev/null +++ b/mod/hallmark/transport/index.py @@ -0,0 +1,149 @@ +"""Parse scoped HTML directory indexes for HTTP data backends.""" + +from __future__ import annotations + +import re +from html.parser import HTMLParser +from urllib.parse import quote, unquote, urljoin, urlsplit + +from .base import CapabilityError, DownloadError, literal_path, reject_controls + + +class _IndexParser(HTMLParser): + """Read links and recognizable index markers without depending on layout.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.links = [] + self.text = [] + self.row_kind = None + self.listing = False + self.has_table = False + self.has_form = False + self.current = None + self.in_anchor = False + self.cell_role = None + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "table": + self.has_table = True + elif tag == "form": + self.has_form = True + elif tag == "tr": + self.current = None + self.row_kind = None + elif tag == "td": + self.cell_role = attrs.get("class", "") + elif tag == "a" and "href" in attrs: + self.current = {"href": attrs["href"], "kind": self.row_kind, + "tail": "", "size": None, "mtime": None} + for field in ("size", "mtime"): + value = attrs.get(f"data-{field}") + if value is not None and value.isdigit(): + self.current[field] = int(value) + self.links.append(self.current) + self.in_anchor = True + + def handle_endtag(self, tag): + if tag == "tr": + self.row_kind = None + self.current = None + elif tag == "a": + self.in_anchor = False + elif tag == "td": + if self.current is not None: + self.current["tail"] += " " + self.cell_role = None + + def handle_data(self, data): + self.text.append(data) + if self.current is not None and not self.in_anchor: + self.current["tail"] += data + if self.cell_role in {"size", "bytes"}: + raw = data.strip().replace(",", "") + if raw.isdigit(): + self.current["size"] = int(raw) + + +def _origin(url): + """Return connection fields used to compare directory-listing URLs.""" + return (url.scheme, url.hostname, + url.port or (443 if url.scheme == "https" else 80), + url.username, url.password) + + +def _index_link(root_url, directory, href, kind): + """Resolve a link only when it stays beneath the supplied URL root.""" + reject_controls(href, "Index href") + root = urlsplit(root_url.rstrip("/") + "/") + current = urljoin(root.geturl(), quote(directory, safe="/")) + resolved = urlsplit(urljoin(current, href)) + if _origin(resolved) != _origin(root): + return None + # Sorting and navigation links do not identify dataset files. + if resolved.query or resolved.fragment: + return None + root_path = unquote(root.path, errors="strict") + path = unquote(resolved.path, errors="strict") + reject_controls(path, "Index path") + if "\\" in path or ".." in path.split("/"): + raise DownloadError("Remote index contains an unsafe path") + if not path.startswith(root_path): + return None + relative = path[len(root_path):] + if not relative: + return None + if any(part.lower() in {".hm", ".git"} for part in relative.split("/")): + return None + is_directory = kind == "directory" or resolved.path.endswith("/") + relative = literal_path(relative.rstrip("/")).as_posix() + return relative + ("/" if is_directory else ""), is_directory + + +def _parse_index(text, root_url, directory, parser_class=_IndexParser): + """Extract file and directory metadata from a supported HTML index.""" + parser = parser_class() + parser.feed(text) + links = [] + for entry in parser.links: + link = _index_link(root_url, directory, entry["href"], entry["kind"]) + if link is not None: + path, is_directory = link + size = entry["size"] + if size is None: + tail = re.search( + r"(?:\d{4}-\d\d-\d\d|\d\d-[A-Za-z]{3}-\d{4})" + r"\s+\d\d:\d\d(?::\d\d)?\s+(\d+)(?:\s|$)", entry["tail"]) + if tail: + size = int(tail.group(1)) + links.append((path, is_directory, size, entry["mtime"])) + markers = " ".join(parser.text).lower() + recognized = (parser.listing or "index of" in markers + or "directory listing" in markers or "parent directory" in markers + or any(is_dir for _, is_dir, _, _ in links)) + # CyVerse can emit a completely empty table without row-level markers. + empty_table = (parser.has_table and not parser.links + and not parser.has_form and not markers.strip()) + if not recognized and not empty_table: + raise CapabilityError( + "This URL exposes no usable directory listing. Use a browsable " + "directory URL or a published Hallmark catalog.") + return links + + +def _response_directory(context, requested): + """Use a redirected listing's canonical location when resolving child links.""" + transport = getattr(context, "transport", None) + final_url = getattr(transport, "text_urls", {}).get(requested) + if final_url is None: + return requested + root = urlsplit(context.remote.url.rstrip("/") + "/") + final = urlsplit(final_url) + if (_origin(root) == _origin(final) + and unquote(root.path).rstrip("/") == unquote(final.path).rstrip("/")): + return "" + link = _index_link(context.remote.url, "", final_url, "directory") + if link is None: + raise DownloadError("Redirected directory is outside the dataset root") + return link[0] diff --git a/mod/hallmark/transport/settings.py b/mod/hallmark/transport/settings.py new file mode 100644 index 0000000..237d256 --- /dev/null +++ b/mod/hallmark/transport/settings.py @@ -0,0 +1,17 @@ +"""Internal resource limits for SSH transfers.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SSHSettings: + """ + Local SSH settings shared by the connections in one operation. + + Timeouts are in seconds. ``transfer_timeout`` limits the total time + for each file, and ``max_sessions`` limits concurrent SFTP sessions. + """ + connect_timeout: int = 10 + transfer_timeout: int = 3600 + shutdown_timeout: int = 2 + max_sessions: int = 4 diff --git a/mod/hallmark/transport/sftp.py b/mod/hallmark/transport/sftp.py new file mode 100644 index 0000000..fae1ae6 --- /dev/null +++ b/mod/hallmark/transport/sftp.py @@ -0,0 +1,316 @@ +"""Read SFTP v3 metadata through an OpenSSH subsystem. + +OpenSSH handles authentication and encryption. Names and attributes are read +from binary protocol fields without parsing shell commands or ``ls`` output. +""" + +from __future__ import annotations + +import os +import selectors +import struct +import subprocess +import time + +from .base import CapabilityError, DownloadError, RemoteObjectMissing + + +_MAX_PACKET = 1024 * 1024 +_KNOWN_ATTRIBUTES = 0x8000000F + + +def _string(value): + """Encode a byte string or UTF-8 text as an SFTP string.""" + if isinstance(value, str): + value = value.encode("utf-8") + return struct.pack(">I", len(value)) + value + + +class _Packet: + """Read and validate fields in an SFTP response packet.""" + def __init__(self, data): + self.data = data + self.offset = 0 + + def take(self, size): + """Read the requested bytes, rejecting truncated packets.""" + end = self.offset + size + if end > len(self.data): + raise DownloadError("Truncated SFTP metadata packet") + result = self.data[self.offset:end] + self.offset = end + return result + + def uint32(self): + """Read an unsigned 32-bit integer in network byte order.""" + return struct.unpack(">I", self.take(4))[0] + + def string(self): + """Read a length-prefixed byte string.""" + return self.take(self.uint32()) + + def text(self): + """Read a UTF-8 filename from a length-prefixed string.""" + try: + return self.string().decode("utf-8") + except UnicodeError: + raise DownloadError("SFTP filename must contain valid UTF-8") from None + + def finish(self): + """Reject unconsumed bytes after the expected fields.""" + if self.offset != len(self.data): + raise DownloadError("Unexpected trailing SFTP metadata") + + def attributes(self): + """Read size, mode, and modification time; retain unknowns as None.""" + flags = self.uint32() + if flags & ~_KNOWN_ATTRIBUTES: + raise DownloadError("Unsupported SFTP attribute flags") + size = struct.unpack(">Q", self.take(8))[0] if flags & 1 else None + if flags & 2: + self.take(8) # uid and gid + mode = self.uint32() if flags & 4 else None + mtime = None + if flags & 8: + self.uint32() # atime + mtime = self.uint32() + if flags & 0x80000000: + count = self.uint32() + if count > (len(self.data) - self.offset) // 8: + raise DownloadError("Invalid SFTP attribute count") + for _ in range(count): + self.string() + self.string() + return {"size": size, "mode": mode, "mtime": mtime} + + +class SftpMetadata: + """ + Read SFTP metadata through a dedicated OpenSSH subsystem process. + + Requests run sequentially with size limits, timeouts, and cancellation + checks. Use this object as a context manager to close the process and pipes. + + Args: + transport (SshTransport): Shared SSH connection and session limits. + """ + + def __init__(self, transport): + self.transport = transport + self.context = transport.context + self.process = None + self.selector = None + self.buffer = bytearray() + self.stderr_size = 0 + self.request_id = 0 + self.deadline = None + self._slot = False + + def __enter__(self): + self.transport.prepare() + while not self.transport._slots.acquire(timeout=0.05): + self.context.check_cancelled() + self._slot = True + try: + self.process = self.transport._spawn( + self.transport.ssh + self.transport._options() + + ["-T", "-s", "--", self.context.remote.host, "sftp"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0, + ) + self.selector = selectors.DefaultSelector() + for pipe in (self.process.stdin, self.process.stdout, self.process.stderr): + os.set_blocking(pipe.fileno(), False) + self.selector.register(self.process.stdout, selectors.EVENT_READ) + self.selector.register(self.process.stderr, selectors.EVENT_READ) + self._new_deadline() + self._send(b"\x01" + struct.pack(">I", 3)) + kind, packet = self._receive() + if kind != 2 or packet.uint32() != 3: + raise CapabilityError("Server must support SFTP version 3") + while packet.offset < len(packet.data): + packet.string() + packet.string() # Ignore unneeded negotiated extensions. + return self + except BaseException: + self.__exit__(None, None, None) + raise + + def __exit__(self, exc_type, exc, tb): + try: + if self.process is not None: + self.transport._stop(self.process) + finally: + if self.selector is not None: + self.selector.close() + if self.process is not None: + for pipe in ( + self.process.stdin, self.process.stdout, self.process.stderr, + ): + pipe.close() + if self._slot: + self.transport._slots.release() + self._slot = False + + def _new_deadline(self): + """Start the timeout for one metadata request.""" + self.deadline = time.monotonic() + self.context.listing_timeout + + def _check(self): + """Check cancellation and the current request deadline.""" + self.context.check_cancelled() + if time.monotonic() >= self.deadline: + raise DownloadError("SFTP metadata request exceeded its time limit") + + def _pump(self): + """Buffer available output and report whether stdin is writable.""" + self._check() + writable = False + for key, events in self.selector.select(timeout=0.05): + if events & selectors.EVENT_WRITE: + writable = True + continue + try: + chunk = os.read(key.fileobj.fileno(), 65536) + except BlockingIOError: + continue + if not chunk: + self.selector.unregister(key.fileobj) + elif key.fileobj is self.process.stdout: + self.buffer.extend(chunk) + if len(self.buffer) > _MAX_PACKET + 4: + raise DownloadError("SFTP metadata packet exceeds its size limit") + else: + self.stderr_size += len(chunk) + if self.stderr_size > 65536: + raise DownloadError("SSH diagnostics exceeded their size limit") + return writable + + def _send(self, payload): + """Write one size-limited packet while checking cancellation.""" + if len(payload) > _MAX_PACKET: + raise DownloadError("SFTP metadata request exceeds its size limit") + data = memoryview(struct.pack(">I", len(payload)) + payload) + self.selector.register(self.process.stdin, selectors.EVENT_WRITE) + try: + while data: + if self._pump(): + try: + written = os.write(self.process.stdin.fileno(), data) + except BlockingIOError: + continue + except BrokenPipeError: + self.context.check_cancelled() + raise DownloadError("SFTP subsystem disconnected") from None + data = data[written:] + finally: + self.selector.unregister(self.process.stdin) + + def _read(self, size): + """Read the requested bytes within the current deadline.""" + while len(self.buffer) < size: + self._pump() + self.context.check_cancelled() + if (len(self.buffer) < size + and self.process.stdout.fileno() not in self.selector.get_map()): + raise DownloadError("SFTP subsystem returned truncated metadata") + self._check() + result = bytes(self.buffer[:size]) + del self.buffer[:size] + return result + + def _receive(self): + """Read a response type and its validated packet body.""" + length = struct.unpack(">I", self._read(4))[0] + if not 1 <= length <= _MAX_PACKET: + raise DownloadError("Invalid or oversized SFTP metadata packet") + data = self._read(length) + return data[0], _Packet(data[1:]) + + def _request(self, kind, data, expected, *, eof=False): + """Send a request and validate its response identifier and status.""" + self._new_deadline() + self.request_id = (self.request_id + 1) % (2 ** 32) + self._send(bytes([kind]) + struct.pack(">I", self.request_id) + data) + response, packet = self._receive() + if packet.uint32() != self.request_id: + raise DownloadError("Mismatched SFTP response identifier") + if response == 101: + status = packet.uint32() + packet.string() # Untrusted server diagnostics are never echoed. + packet.string() + packet.finish() + if status == 1 and eof: + return None + if status == 0 and expected == 101: + return packet + if status == 2: + raise RemoteObjectMissing("Remote metadata object does not exist") + if status == 8: + raise CapabilityError("Server does not support SFTP metadata access") + raise DownloadError(f"SFTP metadata operation failed (status {status})") + if response != expected: + raise DownloadError("Unexpected SFTP metadata response") + return packet + + def realpath(self, path): + """Return the canonical path reported by the SFTP server.""" + packet = self._request(16, _string(path), 104) + if packet.uint32() != 1: + raise DownloadError("Invalid SFTP canonical path response") + result = packet.text() + packet.string() + packet.attributes() + packet.finish() + return result + + def lstat(self, path): + """Read file attributes without following a final symlink.""" + packet = self._request(7, _string(path), 105) + result = packet.attributes() + packet.finish() + return result + + def iterdir(self, path): + """ + Yield names and attributes from one remote directory. + + Args: + path (str): Absolute directory path on the server. + + Yields: + tuple: Filename and attribute dictionary. Missing sizes, modes, + and modification times are None. + + Raises: + DownloadError: If the directory cannot be read or responses are invalid. + """ + packet = self._request(11, _string(path), 102) + handle = packet.string() + packet.finish() + if not handle or len(handle) > 256: + raise DownloadError("Invalid SFTP directory handle") + completed = False + try: + while True: + packet = self._request(12, _string(handle), 104, eof=True) + if packet is None: + completed = True + return + count = packet.uint32() + if count == 0 or count > (len(packet.data) - packet.offset) // 12: + raise DownloadError("Invalid SFTP directory entry count") + entries = [] + for _ in range(count): + name = packet.text() + packet.string() # Human-readable longname is not parsed. + entries.append((name, packet.attributes())) + packet.finish() + yield from entries + finally: + if not self.context.cancelled.is_set(): + try: + self._request(4, _string(handle), 101) + except DownloadError: + if completed: + raise diff --git a/mod/hallmark/transport/ssh.py b/mod/hallmark/transport/ssh.py new file mode 100644 index 0000000..7cc10ef --- /dev/null +++ b/mod/hallmark/transport/ssh.py @@ -0,0 +1,422 @@ +"""Share an OpenSSH connection for SFTP transfers and metadata reads.""" + +from __future__ import annotations + +import os +import re +import selectors +import stat +import shutil +import signal +import subprocess +import tempfile +import time +from pathlib import Path +from threading import BoundedSemaphore, Lock + +from .base import ( + CapabilityError, + DownloadError, + RemoteConfigurationError, + RemoteEntry, + DataBackend, + literal_path, + reject_controls, +) + + +def batch_argument(path): + """ + Escape an absolute path for a literal SFTP batch argument. + + Args: + path (Path | str): Absolute local or remote path. + + Returns: + str: Argument with command and glob characters escaped. + + Raises: + RemoteConfigurationError: If the path is relative or contains controls. + """ + text = str(path) + reject_controls(text, "SFTP path") + if not text.startswith("/"): + raise RemoteConfigurationError("SFTP operands must be absolute paths") + # Use unquoted escapes: OpenSSH adds glob escapes itself inside quotes, + # so adding backslashes there would instead request literal backslashes. + special = " \\\"'#?[*" # ] has no special meaning outside a bracket pattern. + return "".join("\\" + char if char in special else char for char in text) + + +class SshBackend(DataBackend): + """ + Share one OpenSSH connection for SFTP transfers and metadata requests. + + Args: + context (OperationContext): Local settings and cancellation state. + ssh (list[str], optional): Internal command override for offline tests. + sftp (list[str], optional): Internal command override for offline tests. + """ + def __init__(self, context, *, ssh=None, sftp=None): + super().__init__(context) + if context.remote.scheme not in {"ssh", "sftp"}: + raise RemoteConfigurationError("SSH backends require an SSH/SFTP URL") + # Command overrides support offline tests and cannot come from repo YAML. + self.ssh = ssh or ["ssh"] + self.sftp = sftp or ["sftp"] + self._injected = ssh is not None or sftp is not None + self._prepare_lock = Lock() + self._process_lock = Lock() + self._processes = set() + self._master = None + self._socket_dir = None + self._socket = None + self._slots = BoundedSemaphore(context.settings.max_sessions) + + def _options(self, *, master=False): + """Build noninteractive SSH options for the shared connection.""" + settings = self.context.settings + options = [ + "BatchMode=yes", + "StrictHostKeyChecking=yes", + "ForwardAgent=no", + "ForwardX11=no", + "PermitLocalCommand=no", + "ClearAllForwardings=yes", + "RequestTTY=no", + "RemoteCommand=none", + "ForkAfterAuthentication=no", + "ControlPersist=no", + "ServerAliveInterval=15", + "ServerAliveCountMax=2", + "ConnectionAttempts=1", + f"ConnectTimeout={settings.connect_timeout}", + f"ControlPath={self._socket}", + "ControlMaster=yes" if master else "ControlMaster=no", + ] + if not master: + # A failed shared connection must not reauthenticate for each file. + options.append("ProxyCommand=false") + if self.context.remote.user is not None: + options.append(f"User={self.context.remote.user}") + if self.context.remote.port is not None: + options.append(f"Port={self.context.remote.port}") + args = [part for option in options for part in ("-o", option)] + return args + + def _spawn(self, argv, **kwargs): + """Start a client in a separate process group and track it for cleanup.""" + with self._process_lock: + self.context.check_cancelled() + try: + process = subprocess.Popen( + argv, shell=False, start_new_session=True, **kwargs + ) + except OSError: + raise DownloadError("Unable to start the OpenSSH client") from None + self._processes.add(process) + return process + + def _stop(self, process): + # Every owned client starts a new process group, including proxy children. + """Stop a client process group and wait for the child to exit.""" + def signal_group(signum): + try: + os.killpg(process.pid, signum) + except ProcessLookupError: + pass + except PermissionError as exc: + # macOS can report EPERM for a group containing only zombies. + # poll() may return None while another thread holds Popen's + # wait lock, so allow a bounded wait for concurrent reaping. + try: + process.wait(timeout=self.context.settings.shutdown_timeout) + except subprocess.TimeoutExpired: + raise exc from None + + signal_group(signal.SIGTERM) + try: + process.wait(timeout=self.context.settings.shutdown_timeout) + except subprocess.TimeoutExpired: + pass + signal_group(signal.SIGKILL) + process.wait() + with self._process_lock: + self._processes.discard(process) + + def _run( + self, + argv, + *, + timeout, + data=b"", + limit=65536, + destination=None, + file_limit=None, + capture_stderr=False, + report_progress=False, + ): + """ + Run an OpenSSH command with output limits and a total time limit. + + Optionally bound a downloaded metadata file's size or report received + dataset bytes. Always stop and reap the client before returning. + """ + output = bytearray() + stderr_size = 0 + diagnostics = bytearray() + reported_bytes = 0 + + def report_bytes(): + nonlocal reported_bytes + if report_progress and destination is not None and destination.exists(): + current = destination.stat().st_size + if current > reported_bytes: + if self.context.on_bytes is not None: + self.context.on_bytes(current - reported_bytes) + reported_bytes = current + + with tempfile.TemporaryFile() as batch: + batch.write(data) + batch.seek(0) + process = self._spawn( + argv, stdin=batch, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + deadline = time.monotonic() + timeout + try: + with selectors.DefaultSelector() as selector: + for pipe in (process.stdout, process.stderr): + os.set_blocking(pipe.fileno(), False) + selector.register(pipe, selectors.EVENT_READ) + while selector.get_map() or process.poll() is None: + self.context.check_cancelled() + report_bytes() + if time.monotonic() >= deadline: + raise DownloadError("SSH operation exceeded its time limit") + if ( + file_limit is not None + and destination.exists() + and destination.stat().st_size > file_limit + ): + raise DownloadError("Remote text exceeds its size limit") + for key, _ in selector.select(timeout=0.05): + chunk = os.read(key.fileobj.fileno(), 65536) + if not chunk: + selector.unregister(key.fileobj) + elif key.fileobj is process.stdout: + output.extend(chunk) + else: + stderr_size += len(chunk) + if capture_stderr: + diagnostics.extend(chunk) + if len(output) > limit or stderr_size > 65536: + raise DownloadError( + "SSH output exceeded its size limit" + ) + self.context.check_cancelled() + report_bytes() + if process.wait() != 0: + # stderr is untrusted and may contain echoed credentials/paths. + # SFTP exits do not reliably distinguish absence from auth errors. + raise DownloadError( + f"SSH operation failed for {self.context.remote.display}; " + "check access, file existence, and server capabilities" + ) + return bytes(diagnostics if capture_stderr else output) + finally: + self._stop(process) + process.stdout.close() + process.stderr.close() + + def prepare(self): + """Check client support and establish the shared SSH connection.""" + with self._prepare_lock: + if self._master is not None: + if self._master.poll() is not None: + raise DownloadError("The owned SSH master disconnected") + return + if os.name != "posix": + raise CapabilityError("SSH transport requires a Linux/macOS client") + if not self._injected: + if not shutil.which("ssh") or not shutil.which("sftp"): + raise CapabilityError("Install OpenSSH ssh and sftp clients (9.6+)") + version = self._run(self.ssh + ["-V"], timeout=5, capture_stderr=True) + match = re.search(rb"OpenSSH_(\d+)\.(\d+)", version) + if not match or tuple(map(int, match.groups())) < (9, 6): + raise CapabilityError("SSH transport requires OpenSSH 9.6 or newer") + self._socket_dir = tempfile.TemporaryDirectory(prefix="hm-", dir="/tmp") + os.chmod(self._socket_dir.name, 0o700) + self._socket = str(Path(self._socket_dir.name) / "s") + if len(os.fsencode(self._socket)) > 100: + raise CapabilityError("SSH control socket path is too long") + try: + self._master = self._spawn( + self.ssh + + self._options(master=True) + + ["-N", "--", self.context.remote.host], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.monotonic() + self.context.settings.connect_timeout + while not Path(self._socket).exists(): + self.context.check_cancelled() + if self._master.poll() is not None: + raise DownloadError( + f"Cannot establish SSH connection to " + f"{self.context.remote.display}; check trusted host key, " + "loaded key/agent, network and OpenSSH settings" + ) + if time.monotonic() > deadline: + raise DownloadError("SSH connection exceeded its time limit") + self.context.cancelled.wait(0.05) + self._run( + self.ssh + + self._options() + + ["-O", "check", "--", self.context.remote.host], + timeout=self.context.settings.connect_timeout, + ) + except BaseException: + self.close() + raise + + def _fetch(self, relative_path, destination, file_limit=None): + """Fetch one literal path within the session and transfer limits.""" + remote = batch_argument(self.context.remote.pathname(relative_path)) + local = batch_argument(destination) + batch = f"get {remote} {local}\n".encode("utf-8") + if len(batch) > 8000: + raise RemoteConfigurationError("SFTP command exceeds the client path limit") + self.prepare() + host = self.context.remote.host + if ":" in host: + host = f"[{host}]" + with self._slots: + self._run( + self.sftp + self._options() + ["-b", "-", "--", host], + timeout=self.context.settings.transfer_timeout, + data=batch, + destination=destination, + file_limit=file_limit, + report_progress=file_limit is None, + ) + + def fetch(self, relative_path, destination, *, chunk_size=8192): + """Download one file over SFTP; OpenSSH controls the read size.""" + self._fetch(relative_path, destination) + + def read_text(self, relative_path, limit): + """Read UTF-8 metadata with size checks before and after transfer.""" + entry = self.stat(relative_path) + if entry.size is not None and entry.size > limit: + raise DownloadError("Remote text exceeds its size limit") + with tempfile.TemporaryDirectory(prefix="hm-text-") as directory: + destination = Path(directory) / "content" + self._fetch(relative_path, destination, file_limit=limit) + if destination.stat().st_size > limit: + raise DownloadError("Remote text exceeds its size limit") + try: + return destination.read_text(encoding="utf-8") + except UnicodeError: + raise DownloadError("Remote manifest must contain UTF-8 text") from None + + def _metadata(self): + """Create an SFTP metadata session on the shared connection.""" + from .sftp import SftpMetadata + + return SftpMetadata(self) + + def stat(self, relative_path): + """Read regular-file attributes without fetching the file contents.""" + path = literal_path(relative_path).as_posix() + with self._metadata() as session: + attrs = session.lstat(self.context.remote.pathname(path)) + mode = attrs["mode"] + if mode is None or not stat.S_ISREG(mode): + raise DownloadError("Remote metadata must be a regular file") + return RemoteEntry(path, attrs["size"], attrs["mtime"]) + + def iter_entries(self, on_directory=None): + """ + Yield regular files recursively using the SFTP subsystem. + + Args: + on_directory (callable, optional): Callback receiving each directory + path relative to the dataset root. + + Yields: + RemoteEntry: File path, size, and modification time when available. + Symlinks, special files, and repository metadata directories are skipped. + + Raises: + DownloadError: If discovery fails or a path escapes the dataset root. + CapabilityError: If the server omits required file types. + """ + descend = getattr(self.context, "descend", None) + with self._metadata() as session: + root = session.realpath(self.context.remote.root).rstrip("/") or "/" + reject_controls(root, "SFTP root") + if not root.startswith("/") or ".." in root.split("/"): + raise DownloadError("Invalid SFTP canonical root") + pending = [("", root)] + visited = set() + while pending: + self.context.check_cancelled() + relative, directory = pending.pop() + canonical = session.realpath(directory).rstrip("/") or "/" + if (root != "/" and canonical != root + and not canonical.startswith(root + "/")): + raise DownloadError("SFTP directory escapes the source root") + if canonical in visited: + continue + visited.add(canonical) + attrs = session.lstat(directory) + if attrs["mode"] is None: + raise CapabilityError("SFTP server omitted directory type") + if not stat.S_ISDIR(attrs["mode"]): + raise DownloadError("SFTP discovery root must be a directory") + if on_directory is not None: + on_directory(relative) + names = set() + for name, attrs in session.iterdir(directory): + self.context.check_cancelled() + if name in {".", ".."}: + continue + if name.lower() in {".hm", ".git"}: + continue + if not name or "/" in name or name in names: + raise DownloadError("Invalid or duplicate SFTP directory name") + names.add(name) + path = literal_path(relative + name).as_posix() + absolute = directory.rstrip("/") + "/" + name + mode = attrs["mode"] + if mode is None: + attrs = session.lstat(absolute) + mode = attrs["mode"] + if mode is None: + raise CapabilityError("SFTP server omitted file type") + if stat.S_ISDIR(mode): + if descend is None or descend(path + "/"): + pending.append((path + "/", absolute)) + elif stat.S_ISREG(mode): + entry = RemoteEntry(path, attrs["size"], attrs["mtime"]) + yield entry + # Symlinks and special files are deliberately not traversed. + + def cancel(self): + """Stop the client process groups started by this transport.""" + with self._process_lock: + processes = list(self._processes) + for process in processes: + self._stop(process) + + def close(self): + """Stop clients and remove the shared connection socket directory.""" + self.cancel() + self._master = None + if self._socket_dir is not None: + self._socket_dir.cleanup() + self._socket_dir = None + + +SshTransport = SshBackend diff --git a/pyproject.toml b/pyproject.toml index 20a137d..50c77bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,3 +55,8 @@ exclude = ["doc", "demo"] [tool.pytest.ini_options] testpaths = ["test"] +addopts = "--strict-markers" +markers = [ + "ssh_client: exercise the actual local OpenSSH SFTP command parser", + "ssh_integration: disposable loopback sshd (HALLMARK_RUN_SSH_TESTS=1 required)", +] diff --git a/test/mock_server.py b/test/mock_server.py index 7de10d6..487442b 100644 --- a/test/mock_server.py +++ b/test/mock_server.py @@ -33,7 +33,12 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): return False - def get(self, url, timeout=None): + def close(self): + """Provide the Requests session cleanup interface for tests.""" + pass + + def get(self, url, timeout=None, stream=False, allow_redirects=True): + """Return a registered response through the Requests session interface.""" return self.fake_get(url, timeout=timeout) def head(self, url, timeout=None): @@ -94,11 +99,15 @@ def fake_get(self, url, timeout=None): resp.raise_for_status = lambda: None if url in self._html_by_url: resp.text = self._html_by_url[url] + resp.content = resp.text.encode() elif url in self._text_by_url: resp.text = self._text_by_url[url] resp.content = self._content_by_url[url] else: raise AssertionError(f"MockServer: no registered response for GET {url}") + resp.encoding = "utf-8" + resp.iter_content = lambda chunk_size: iter([resp.content]) + resp.__enter__.return_value = resp return resp def fake_head(self, url, timeout=None): @@ -118,4 +127,5 @@ def fake_head(self, url, timeout=None): resp.headers = {"Content-Length": str(len(self._content_by_url[url]))} else: raise AssertionError(f"MockServer: no registered response for HEAD {url}") + resp.__enter__.return_value = resp return resp diff --git a/test/test_backend_config.py b/test/test_backend_config.py new file mode 100644 index 0000000..29a8873 --- /dev/null +++ b/test/test_backend_config.py @@ -0,0 +1,100 @@ +"""Persist backend settings without importing plugins or sharing mutable options.""" + +import pytest + +from hallmark import Repo +from hallmark import backends +from hallmark.backends import DataBackend, RemoteEntry, register_backend +from hallmark.repo_builder import build_repo +from hallmark.repo_config import normalize_remotes + + +def test_backend_config_roundtrip_and_selective_updates(tmp_path): + repo = Repo.init(tmp_path / "repo") + options = {"servers": [{"url": "https://data.test/", "prefix": "north"}]} + repo.set_config(remote_url="https://api.test/", remote_backend="survey", + remote_backend_options=options) + options["servers"][0]["prefix"] = "changed" + reopened = Repo(repo.worktree) + remote = reopened.state.config["remote"] + assert remote["backend"] == "survey" + assert remote["backend_options"]["servers"][0]["prefix"] == "north" + reopened.set_config(remote_name="archive") + assert Repo(repo.worktree).state.config["remote"]["backend"] == "survey" + reopened.set_config(remote_backend="", remote_backend_options={}) + remote = Repo(repo.worktree).state.config["remote"] + assert "backend" not in remote + assert remote["backend_options"] == {} + + +@pytest.mark.parametrize("update", [ + {"remote_backend": "arbitrary.module:Class"}, + {"remote_backend_options": ["not", "a", "mapping"]}, + {"remote_backend_options": {"nested": object()}}, +]) +def test_invalid_backend_update_preserves_existing_config(tmp_path, update): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="https://data.test/") + before = (repo.dothm.path / "config.yml").read_bytes() + with pytest.raises(ValueError): + repo.set_config(**update) + assert (repo.dothm.path / "config.yml").read_bytes() == before + assert "backend" not in repo.state.config["remote"] + + +def test_normalization_copies_nested_backend_options(): + source = {"name": "archive", "backend": "survey", + "backend_options": {"releases": [1, 2]}} + normalized = normalize_remotes(source) + source["backend_options"]["releases"].clear() + assert normalized[0]["backend_options"]["releases"] == [1, 2] + + +def test_builder_passes_backend_settings_to_discovery(monkeypatch, tmp_path): + captured = [] + + def build(*args): + captured.append(args[-1].remote) + + # No backend installation or network is required to inspect this hand-off. + class Context: + def __init__(self, remote): + self.remote = remote + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + monkeypatch.setattr("hallmark.repo_builder.OperationContext", Context) + monkeypatch.setattr("hallmark.repo_builder._build_repo", build) + with pytest.warns(DeprecationWarning, match="Repo.init"): + build_repo(tmp_path / "data.hm", "dataset", dataset_url="https://api.test/", + backend="survey", backend_options={"release": 3}) + assert captured[0].backend == "survey" + assert captured[0].backend_options["release"] == 3 + + +@pytest.mark.parametrize("mirror", [None, "https://mirror.test/data/"]) +def test_builder_persists_backend_only_for_its_data_source( + monkeypatch, tmp_path, mirror): + class SurveyBackend(DataBackend): + def iter_entries(self, on_directory=None): + assert self.context.remote.backend_options["release"] == 3 + yield RemoteEntry("tile.fits", size=12) + + monkeypatch.setattr(backends, "_registered", dict(backends._registered)) + monkeypatch.setattr(backends.metadata, "entry_points", lambda: {}) + register_backend("builder-survey", SurveyBackend) + with pytest.warns(DeprecationWarning): + repo = build_repo( + tmp_path / "survey.hm", "survey", dataset_url="https://api.test/", + backend="builder-survey", backend_options={"release": 3}, + remotes=[{"name": "mirror", "url": mirror}] if mirror else None) + remote = Repo(repo.dothm.path).state.config["remote"][0] + if mirror: + assert remote == {"name": "mirror", "url": mirror} + else: + assert remote["backend"] == "builder-survey" + assert remote["backend_options"] == {"release": 3} diff --git a/test/test_backends.py b/test/test_backends.py new file mode 100644 index 0000000..939b961 --- /dev/null +++ b/test/test_backends.py @@ -0,0 +1,339 @@ +"""Public backend registration and multi-server routing without public services.""" + +from contextlib import contextmanager +import hashlib +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +from pathlib import Path +from threading import Thread +from types import SimpleNamespace + +import pytest + +from hallmark import Repo, DataBackend, HttpBackend, SshBackend, CyVerseBackend +from hallmark import backends +from hallmark.discovery import discover +from hallmark.transport import OperationContext +from hallmark.transport.base import ( + CapabilityError, RemoteConfigurationError, RemoteEntry, RemoteSpec, Transport, + TransferCancelled, freeze_backend_options, thaw_backend_options, +) + + +@pytest.fixture(autouse=True) +def isolated_registry(monkeypatch): + monkeypatch.setattr(backends, "_registered", { + "http": HttpBackend, "ssh": SshBackend, "cyverse": CyVerseBackend}) + monkeypatch.setattr(backends, "_loaded", {}) + monkeypatch.setattr(backends.metadata, "entry_points", lambda: {}) + + +@pytest.mark.parametrize("url, expected", [ + ("https://lab.test/data/", HttpBackend), + ("http://lab.test/data/", HttpBackend), + ("ssh://lab/srv/data/", SshBackend), + ("sftp://lab/srv/data/", SshBackend), + ("https://data.cyverse.org/dav-anon/data/", CyVerseBackend), +]) +def test_builtin_detection(url, expected, monkeypatch, tmp_path): + monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + with OperationContext(RemoteSpec.parse(url)) as context: + assert type(context.backend) is expected + assert context.backend is context.transport + + +def test_explicit_selection_and_compatibility_aliases(): + from hallmark.transport.http import HttpTransport + from hallmark.transport.ssh import SshTransport + + assert Transport is DataBackend + assert HttpTransport is HttpBackend + assert SshTransport is SshBackend + with OperationContext(RemoteSpec.parse( + "https://data.cyverse.org/data/", backend="http")) as context: + assert type(context.backend) is HttpBackend + with OperationContext(RemoteSpec.parse( + "https://lab.test/data/", backend="cyverse")) as context: + assert type(context.backend) is CyVerseBackend + + +def test_backend_lifecycle_and_generic_discovery(tmp_path): + events = [] + + class SurveyBackend(DataBackend): + def iter_entries(self, on_directory=None): + on_directory("") + yield RemoteEntry("tile.fits", size=8) + + def prepare(self): + events.append("prepare") + + def fetch(self, relative_path, destination, *, chunk_size=8192): + self.context.check_cancelled() + destination.write_bytes(b"contents") + + def read_text(self, relative_path, limit): + assert limit == 20 + return "metadata" + + def cancel(self): + events.append("cancel") + + def close(self): + events.append("close") + + backends.register_backend("survey", SurveyBackend) + remote = RemoteSpec.parse("https://lab.test/", backend="survey") + with OperationContext(remote) as context: + context.text_limit = 20 + context.backend.prepare() + assert context.read_text("info") == "metadata" + assert discover(context) == [RemoteEntry("tile.fits", size=8)] + context.backend.fetch("tile.fits", tmp_path / "payload") + assert (tmp_path / "payload").read_bytes() == b"contents" + assert events == ["prepare", "prepare", "close"] + with pytest.raises(RuntimeError): + with OperationContext(remote): + raise RuntimeError("interrupted") + assert events[-2:] == ["cancel", "close"] + + +@pytest.mark.parametrize("name", ["", "Bad", "module.Backend", "bad/name", 12]) +def test_invalid_registry_names(name): + with pytest.raises(RemoteConfigurationError, match="Backend name"): + backends.register_backend(name, HttpBackend) + + +@pytest.mark.parametrize("backend_class", [ + object, object(), DataBackend, "module.Class", +]) +def test_invalid_registry_classes(backend_class): + with pytest.raises(RemoteConfigurationError, match="DataBackend subclass"): + backends.register_backend("invalid", backend_class) + + +def test_duplicate_registration_rejected(): + with pytest.raises(RemoteConfigurationError, match="already registered"): + backends.register_backend("http", HttpBackend) + backends.register_backend("survey", HttpBackend) + with pytest.raises(RemoteConfigurationError, match="already registered"): + backends.register_backend("survey", HttpBackend) + + +def test_plugin_loading_is_lazy_and_cached(monkeypatch): + loaded = [] + + class Entry: + name = "survey" + + def load(self): + loaded.append(self.name) + return HttpBackend + + monkeypatch.setattr(backends.metadata, "entry_points", + lambda: {"hallmark.backends": [Entry()]}) + remote = RemoteSpec.parse("https://lab.test/", backend="survey") + assert not loaded + for _ in range(2): + with OperationContext(remote) as context: + assert isinstance(context.backend, HttpBackend) + assert loaded == ["survey"] + with pytest.raises(RemoteConfigurationError, match="already registered"): + backends.register_backend("survey", HttpBackend) + + +def test_modern_entry_point_selection(monkeypatch): + entry = SimpleNamespace(name="survey", load=lambda: HttpBackend) + + class Entries: + def select(self, **kwargs): + assert kwargs == {"group": "hallmark.backends"} + return [entry] + + monkeypatch.setattr(backends.metadata, "entry_points", Entries) + assert backends.get_backend("survey") is HttpBackend + + +def test_duplicate_and_invalid_installed_backends(monkeypatch): + entry = SimpleNamespace(name="survey", load=lambda: object) + entries = [entry, entry] + monkeypatch.setattr(backends.metadata, "entry_points", + lambda: {"hallmark.backends": entries}) + with pytest.raises(RemoteConfigurationError, match="Duplicate backend"): + backends.get_backend("survey") + entries.pop() + with pytest.raises(RemoteConfigurationError, match="DataBackend subclass"): + backends.get_backend("survey") + entries[:] = [SimpleNamespace(name="http", load=lambda: HttpBackend)] + with pytest.raises(RemoteConfigurationError, match="Duplicate backend"): + backends.get_backend("http") + + +def test_missing_plugin_can_be_parsed_without_loading(monkeypatch): + def fail(): + raise AssertionError("Parsing must not inspect installed plugins") + + monkeypatch.setattr(backends.metadata, "entry_points", fail) + remote = RemoteSpec.parse("https://lab.test/", backend="uninstalled") + monkeypatch.setattr(backends.metadata, "entry_points", lambda: {}) + with pytest.raises(RemoteConfigurationError, match="not installed or registered"): + OperationContext(remote) + + +def test_plugin_load_failure_is_actionable_and_sanitized(monkeypatch): + def fail(): + raise ImportError("sensitive internal path") + + monkeypatch.setattr(backends.metadata, "entry_points", lambda: { + "hallmark.backends": [SimpleNamespace(name="survey", load=fail)]}) + with pytest.raises(RemoteConfigurationError, match="Unable to load backend") as exc: + backends.get_backend("survey") + assert "sensitive" not in str(exc.value) + + +def test_options_are_copied_and_deeply_immutable(): + options = {"routes": [{"names": ["first"]}], "enabled": True} + remote = RemoteSpec.parse("https://lab.test/", backend_options=options) + options["routes"][0]["names"].append("later") + assert remote.backend_options["routes"][0]["names"] == ("first",) + with pytest.raises(TypeError): + remote.backend_options["routes"][0]["names"] = () + assert thaw_backend_options(remote.backend_options) == { + "routes": [{"names": ["first"]}], "enabled": True} + assert freeze_backend_options() == {} + assert thaw_backend_options() == {} + + +@pytest.mark.parametrize("options", [[], {1: "bad"}, {"object": object()}]) +def test_invalid_options_are_rejected(options): + with pytest.raises(RemoteConfigurationError, match="Backend option"): + RemoteSpec.parse("https://lab.test/", backend_options=options) + + +def test_recursive_options_are_rejected(): + options = {} + options["cycle"] = options + with pytest.raises(RemoteConfigurationError, match="cycles"): + freeze_backend_options(options) + + +def test_base_capabilities(): + backend = DataBackend(SimpleNamespace()) + with pytest.raises(CapabilityError): + backend.iter_entries() + with pytest.raises(CapabilityError): + backend.read_text("index", 10) + with pytest.raises(NotImplementedError): + backend.fetch("file", Path("unused")) + + +def test_constructor_failure_closes_context_sessions(monkeypatch): + closed = [] + monkeypatch.setattr("hallmark.transport.requests.Session", lambda: SimpleNamespace( + close=lambda: closed.append(True))) + + class BrokenBackend(DataBackend): + def __init__(self, context): + super().__init__(context) + context.session() + raise RuntimeError("setup failed") + + backends.register_backend("broken", BrokenBackend) + with pytest.raises(RuntimeError, match="setup failed"): + OperationContext(RemoteSpec.parse("https://lab.test/", backend="broken")) + assert closed == [True] + + +def test_cancel_failure_still_closes_resources(): + closed = [] + + class BrokenBackend(DataBackend): + def cancel(self): + raise RuntimeError("cancel failed") + + def close(self): + closed.append(True) + + backends.register_backend("broken", BrokenBackend) + with pytest.raises(RuntimeError, match="cancel failed"): + with OperationContext(RemoteSpec.parse("https://lab.test/", backend="broken")): + raise ValueError("operation failed") + assert closed == [True] + + +@contextmanager +def local_server(root, requests): + class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(root), **kwargs) + + def log_message(self, *args): + pass + + def do_GET(self): + requests.append(self.path) + super().do_GET() + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/" + finally: + server.shutdown() + thread.join() + server.server_close() + + +def test_multi_server_example_uses_shared_discovery_and_verified_downloads(tmp_path): + path = Path(__file__).parents[1] / "demo" / "multi_server_backend.py" + module_spec = importlib.util.spec_from_file_location("example_backend", path) + example = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(example) + example.register() + requests = {"north": [], "south": []} + for name in requests: + root = tmp_path / name + root.mkdir() + payload = (name + " payload").encode() + (root / "file.fits").write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + (root / "sha256sums").write_text(digest + " file.fits\n") + with local_server(tmp_path / "north", requests["north"]) as north: + with local_server(tmp_path / "south", requests["south"]) as south: + options = {"routes": {"north": north, "south": south}} + from hallmark import DataSource, SourceRelease, register_source + register_source(DataSource("multi-example", "Multiple servers", { + "v1": SourceRelease("https://logical.test/dataset/", {}, + backend="multi-server", backend_options=options)})) + repo = Repo.init(tmp_path / "repo") + # the template inherits the release's backend and routes + repo.add("https://logical.test/dataset/{site}/file.fits") + assert repo.state.data["site"].tolist() == ["north", "south"] + assert repo.state.data["checksum_algorithm"].tolist() == [ + "sha256", "sha256"] + assert requests == {"north": ["/", "/sha256sums"], + "south": ["/", "/sha256sums"]} + repo = Repo(repo.worktree) + before = {name: list(paths) for name, paths in requests.items()} + plan = repo.plan_download() + assert requests == before + assert plan.remote_backend == "multi-server" + assert plan.backend_options["routes"] == options["routes"] + options["routes"]["north"] = south + repo.set_config(remote_backend_options={"routes": {"north": south}}) + result = repo.download(plan=plan, approved=True, + max_workers=2, progress=False) + assert result["succeeded"] == 2 and result["failed"] == 0 + for name in requests: + assert (repo.worktree / name / "file.fits").read_bytes() == ( + name + " payload").encode() + assert requests[name][-1] == "/file.fits" + remote = RemoteSpec.parse(plan.remote_url, + backend=plan.remote_backend, + backend_options=plan.backend_options) + with OperationContext(remote) as context: + context.cancel() + with pytest.raises(TransferCancelled): + next(context.backend.iter_entries()) diff --git a/test/test_catalog.py b/test/test_catalog.py new file mode 100644 index 0000000..8d7f795 --- /dev/null +++ b/test/test_catalog.py @@ -0,0 +1,328 @@ +"""Test cloning catalogs and cataloging remote files without downloading payloads.""" + +from pathlib import Path + +import pytest +import yaml + +from hallmark import Repo +from hallmark.error import CloneError, DestinationExistsError +from hallmark.transport import OperationContext +from hallmark.transport.base import DownloadError, RemoteObjectMissing + + +@pytest.fixture +def metadata_server(monkeypatch): + """Serve registered metadata and record each requested URL.""" + pages, reads = {}, [] + + def read_text(context, path): + url = context.remote.url.rstrip("/") + "/" + path + reads.append(url) + if url not in pages: + raise RemoteObjectMissing("No such metadata object") + return pages[url] + + monkeypatch.setattr(OperationContext, "read_text", read_text) + return pages, reads + + +def test_remote_add_decodes_a_literal_file_url(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/odd/" + pages[root] = '

Index of odd

odd name' + repo = Repo.init(tmp_path / "clone") + repo.add(root + "odd%20name") + assert repo.state.config["data"][0] == {"fmt": "odd name", "url": root} + assert repo.plan_download().items[0].relative_path == Path("odd name") + + +@pytest.mark.parametrize("failure", [KeyboardInterrupt, DownloadError]) +def test_failed_remote_discovery_leaves_the_catalog_unchanged( + tmp_path, metadata_server, monkeypatch, failure): + repo = Repo.init(tmp_path / "existing") + (repo.worktree / "keep.h5").write_bytes(b"existing scientific data") + repo.add("{name}.h5") + stored = {path.name: path.read_bytes() + for path in repo.dothm.path.iterdir() if path.is_file()} + staged = {key: entry.binsha for key, entry in repo.dothm.index.entries.items()} + + def fail(*args, **kwargs): + raise failure("interrupted") + + monkeypatch.setattr("hallmark.repo_remote.discover", fail) + with pytest.raises(failure): + repo.add("https://example.test/data/{run}.fits") + assert {path.name: path.read_bytes() + for path in repo.dothm.path.iterdir() if path.is_file()} == stored + assert {key: entry.binsha + for key, entry in repo.dothm.index.entries.items()} == staged + assert (repo.worktree / "keep.h5").read_bytes() == b"existing scientific data" + + +@pytest.mark.parametrize("nested", [False, True]) +def test_published_snapshot_preserves_data_remote(tmp_path, metadata_server, nested): + pages, reads = metadata_server + root = "https://example.test/published/" + prefix = root + (".hm/" if nested else "") + pages[prefix + "config.yml"] = yaml.safe_dump({ + "data": [{"db": "data.tsv"}], + "remote": [{"name": "origin", "url": "https://mirror.test/data/"}]}) + pages[prefix + "meta.yml"] = "dataset: example\n" + pages[prefix + "data.tsv"] = "path\tsize_bytes\na.h5\t12\nb.txt\t20\n" + plans = [] + repo = Repo.clone(root, tmp_path / "clone") + plans.append(repo.plan_download(include="**/*.h5")) + assert repo.state.data["path"].tolist() == ["a.h5", "b.txt"] + assert plans[0].total_bytes == 12 + assert repo.plan_download().remote_url == "https://mirror.test/data/" + assert not any(url.endswith("a.h5") for url in reads) + assert repo.dothm.head.commit.message.strip() == "Import published catalog snapshot" + + +def test_filtered_git_clone_preserves_history_without_objects(tmp_path, monkeypatch): + source = Repo.init(tmp_path / "source") + (source.worktree / "run1.h5").write_text("one") + (source.worktree / "run2.h5").write_text("two") + source.add("run{run:d}.h5") + source.set_config(remote_url="https://example.test/data/") + source.commit("Original scientific data") + head = source.dothm.head.commit.hexsha + assert list(source.objects.root.rglob("*")) + + def fail(*args, **kwargs): + raise AssertionError("Filtered metadata clone must not transfer data") + + monkeypatch.setattr("hallmark.downloader._fetch_file", fail) + plans = [] + repo = Repo.clone(str(source.dothm.path), tmp_path / "clone") + plans.append(repo.plan_download(include="run1.h5")) + assert repo.dothm.head.commit.hexsha == head + assert repo.dothm.index.diff("HEAD") == [] + assert len(repo.state.data) == 2 + assert plans[0].items[0].relative_path == Path("run1.h5") + assert plans[0].file_count == 1 + assert not repo.objects.root.exists() + assert not (repo.worktree / "run1.h5").exists() + assert repo.dothm.remotes.origin.url == str(source.dothm.path) + + +def test_clone_requires_callback_before_download_intent(tmp_path, metadata_server): + _, reads = metadata_server + with pytest.raises(TypeError, match="download"): + Repo.clone("https://example.test/data/", tmp_path / "clone", download=True) + assert reads == [] + assert not (tmp_path / "clone").exists() + + +def test_existing_destination_is_never_changed(tmp_path, metadata_server): + destination = tmp_path / "clone" + destination.mkdir() + (destination / "keep").write_text("keep") + with pytest.raises(DestinationExistsError): + Repo.clone("https://example.test/data/", destination) + assert (destination / "keep").read_text() == "keep" + assert metadata_server[1] == [] + + +def test_missing_snapshot_does_not_publish_empty_success(tmp_path, metadata_server): + with pytest.raises(CloneError, match="No published"): + Repo.clone("https://example.test/data/", tmp_path / "clone", + source_type="catalog") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_rejects_external_catalog_paths(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root + "config.yml"] = "data:\n- db: ../escape.tsv\n" + pages[root + "meta.yml"] = "{}\n" + pages[root + "data.tsv"] = "path\n" + with pytest.raises(ValueError): + Repo.clone(root, tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_filter_normalizes_legacy_db_names(tmp_path, metadata_server): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root + "config.yml"] = ( + "data:\n- db: data\n fmt: 'item_{i}.dat'\n" + "remote:\n- name: origin\n url: https://data.test/files/\n") + pages[root + "meta.yml"] = "{}\n" + pages[root + "data.tsv"] = "i\n1\n2\n" + plans = [] + repo = Repo.clone(root, tmp_path / "clone") + plans.append(repo.plan_download(include="item_1.dat")) + assert repo.state.data["i"].astype(str).tolist() == ["1", "2"] + assert [item.relative_path for item in plans[0].items] == [Path("item_1.dat")] + + +@pytest.mark.parametrize("table", [ + "Login required\n", "", "path\n../escape\n", + "path\n.hm/config.yml\n", "unrelated\nrecord\n", +]) +def test_snapshot_validates_tables_without_filter(tmp_path, metadata_server, table): + pages, _ = metadata_server + root = "https://example.test/data/" + pages[root + "config.yml"] = "data:\n- db: data.tsv\n" + pages[root + "meta.yml"] = "{}\n" + pages[root + "data.tsv"] = table + with pytest.raises((CloneError, ValueError)): + Repo.clone(root, tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +@pytest.mark.parametrize("kwargs", [ + {"filter": "*.fits"}, {"fmt": "{name}.fits"}, {"source_type": "directory"}, + {"download": True, "approve": lambda plan: True, "fmt": "{name:invalid}"}, +]) +def test_clone_invalid_selection_fails_before_source_access( + tmp_path, metadata_server, kwargs): + with pytest.raises((TypeError, ValueError)): + Repo.clone("https://example.test/data/", tmp_path / "clone", **kwargs) + assert metadata_server[1] == [] + assert not (tmp_path / "clone").exists() + + +def test_raw_directory_clone_fails_with_init_guidance( + tmp_path, metadata_server, monkeypatch): + pages, reads = metadata_server + root = "https://example.test/data/" + pages[root] = '

Index of data

a.h5' + + def fail(*args, **kwargs): + raise CloneError("not a Git repository") + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(CloneError, match="hallmark init PATH, then hallmark add"): + Repo.clone(root, tmp_path / "clone") + assert root not in reads + assert not (tmp_path / "clone").exists() + + +def test_suffixless_http_git_source_falls_back_after_absent_snapshots( + tmp_path, metadata_server, monkeypatch): + source = Repo.init(tmp_path / "source") + source.dothm.index.commit("Existing complete catalog") + original = type(source.dothm).clone + calls = [] + + def clone(url, destination, **kwargs): + calls.append(url) + return original(str(source.dothm.path), destination, **kwargs) + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", clone) + url = "https://git.example.test/group/dataset" + repo = Repo.clone(url, tmp_path / "clone") + assert calls == [url] + assert metadata_server[1] == [url + "/config.yml", url + "/.hm/config.yml"] + assert repo.dothm.head.commit.hexsha == source.dothm.head.commit.hexsha + + +@pytest.mark.parametrize("url", [ + "ssh://git.example.test/dataset", "git@example.test:group/dataset", + "example.test:group/dataset", "https://example.test/dataset.git", +]) +def test_git_sources_do_not_probe_snapshot_metadata(tmp_path, metadata_server, + monkeypatch, url): + calls = [] + + def fail(source, *args, **kwargs): + calls.append(source) + raise CloneError("test Git failure") + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(CloneError, match="test Git failure"): + Repo.clone(url, tmp_path / "clone") + assert calls == [url] + assert metadata_server[1] == [] + + +@pytest.mark.parametrize("config,meta,table", [ + ("data: [", "{}", "path\n"), + ("[]", "{}", "path\n"), + ("unrelated: config", "{}", "path\n"), + ("data: []", None, "path\n"), + ("data: []", "{}", None), +]) +def test_malformed_snapshot_never_falls_back_to_git( + tmp_path, metadata_server, monkeypatch, config, meta, table): + root = "https://example.test/catalog/" + for name, content in [("config.yml", config), ("meta.yml", meta), + ("data.tsv", table)]: + if content is not None: + metadata_server[0][root + name] = content + + def fail(*args, **kwargs): + raise AssertionError("Malformed metadata must not trigger Git fallback") + + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(CloneError): + Repo.clone(root, tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_clone_does_not_require_payload_authentication( + tmp_path, metadata_server, monkeypatch): + root = "https://catalog.example.test/published/" + remote = {"name": "origin", "url": "ssh://private.example.test/science/", + "backend": "ssh", + "backend_options": {"collection": ["release-1"]}} + metadata_server[0].update({ + root + "config.yml": yaml.safe_dump({"data": [{"db": "data.tsv"}], + "remote": [remote]}), + root + "meta.yml": "{}\n", root + "data.tsv": "path\na.fits\n"}) + repo = Repo.clone(root, tmp_path / "clone") + assert repo.state.config["remote"] == [remote] + assert all(url.startswith(root) for url in metadata_server[1]) + assert repo.plan_download().remote_backend == "ssh" + assert not repo.dothm.remotes + + +def test_snapshot_access_failure_never_falls_back_to_git(tmp_path, monkeypatch): + def inaccessible(context, path): + raise DownloadError("Metadata server requires authentication") + + def fail(*args, **kwargs): + raise AssertionError("Access failure must not trigger Git fallback") + + monkeypatch.setattr(OperationContext, "read_text", inaccessible) + monkeypatch.setattr("hallmark.catalog.Dothm.clone", fail) + with pytest.raises(DownloadError, match="requires authentication"): + Repo.clone("https://catalog.example.test/data/", tmp_path / "clone") + assert not (tmp_path / "clone").exists() + + +def test_snapshot_download_uses_independent_payload_server( + tmp_path, metadata_server, monkeypatch): + import requests + from mock_server import MockServer + + catalog_url = "https://catalog.example.test/published/" + payload_url = "https://data.example.test/science/" + metadata_server[0].update({ + catalog_url + "config.yml": yaml.safe_dump({ + "data": [{"db": "data.tsv"}], + "remote": [{"name": "origin", "url": payload_url}]}), + catalog_url + "meta.yml": "{}\n", + catalog_url + "data.tsv": "path\tsize_bytes\na.fits\t4\n"}) + server = MockServer(payload_url) + server.add_file("a.fits", b"fits") + requests_made = [] + original_get = server.get + + def get(url, **kwargs): + requests_made.append(url) + return original_get(url, **kwargs) + + server.get = get + monkeypatch.setattr(requests, "Session", lambda: server) + repo = Repo.clone(catalog_url, tmp_path / "clone") + plan = repo.plan_download() + assert plan.remote_url == payload_url + repo.download(plan, approved=True) + assert requests_made == [payload_url + "a.fits"] + assert all(url.startswith(catalog_url) for url in metadata_server[1]) + assert (repo.worktree / "a.fits").read_bytes() == b"fits" + assert repo.download_result["succeeded"] == 1 diff --git a/test/test_catalog_entries.py b/test/test_catalog_entries.py new file mode 100644 index 0000000..c3dfb57 --- /dev/null +++ b/test/test_catalog_entries.py @@ -0,0 +1,622 @@ +"""Test catalogs with several data entries, each stored in its own TSV.""" + +from pathlib import Path + +import pandas as pd +import pytest +import yaml + +from hallmark import Repo +from hallmark.dothm import Dothm +from hallmark.repo_config import ( + add_entry, catalog_entries, catalog_table_names, next_db_name, remove_entry, + set_config, sole_data_spec) +from hallmark.repo_manifest import ( + catalog_map, iter_catalog_rows, manifest_map, row_fingerprint) +from hallmark.repo_state import load_head_state +from hallmark.state import State + + +def _template_config(): + return yaml.safe_load(Dothm.config_template()) + + +def test_first_entry_fills_the_init_placeholder(): + config = _template_config() + assert catalog_entries(config) == [] + assert add_entry(config, {"fmt": "a{a}.h5"}) == 0 + assert config["data"] == [{"fmt": "a{a}.h5", "encoding": None}] + assert add_entry(config, {"fmt": "{run}.dat", "db": "data-2.tsv"}) == 1 + assert [entry.db for entry in catalog_entries(config)] == [ + "data.tsv", "data-2.tsv"] + + +def test_remote_entry_drops_the_empty_encoding_placeholder(): + config = _template_config() + add_entry(config, {"fmt": "{src}.h5", "url": "https://example.test/er2/"}) + assert config["data"] == [ + {"fmt": "{src}.h5", "url": "https://example.test/er2/"}] + assert catalog_entries(config)[0].is_remote + + +def test_catalog_entries_skip_placeholders_and_static_files(): + config = {"data": [{"encoding": None}, {"file": "README.md", "md5": "0" * 32}, + {"fmt": "a{a}.h5", "db": "a"}, {"db": "paths"}]} + entries = catalog_entries(config) + assert [(entry.index, entry.fmt, entry.db) for entry in entries] == [ + (2, "a{a}.h5", "a.tsv"), (3, None, "paths.tsv")] + assert catalog_table_names(config) == ["a.tsv", "paths.tsv"] + + +def test_catalog_table_names_skip_invalid_names(): + assert catalog_table_names({"data": [{"fmt": "x{a}", "db": "../x"}]}) == [] + + +def test_next_db_name_never_reuses_a_number(): + config = {"data": [{"fmt": "a{a}.h5"}]} + assert next_db_name({"data": []}) == "data.tsv" + assert next_db_name(config) == "data-2.tsv" + assert next_db_name(config, used=["data-2.tsv", "DATA-7.TSV"]) == "data-8.tsv" + add_entry(config, {"fmt": "{b}.dat", "db": "data-3.tsv"}) + remove_entry(config, 0) + # data.tsv always exists, so a later first entry takes it back + assert next_db_name(config) == "data.tsv" + + +def test_single_template_settings_refuse_several_entries(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "a{a}.h5"}, + {"fmt": "{b}.dat", "db": "data-2.tsv"}] + with pytest.raises(RuntimeError, match="several templates"): + set_config(repo, fmt="c{c}.h5") + with pytest.raises(RuntimeError, match="several templates"): + sole_data_spec(repo.state.config) + + +def test_set_config_refuses_to_change_a_remote_template(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "{a}.h5", "url": "https://example.test/"}] + with pytest.raises(ValueError, match="remote entry"): + set_config(repo, fmt="{b}.h5") + + +def test_set_config_updates_the_sole_entry_in_place(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"file": "README.md"}, {"fmt": "a{a}.h5"}] + set_config(repo, fmt="b{b}.h5", encoding_updates={"b": "x(.*)"}) + assert repo.state.config["data"] == [ + {"file": "README.md"}, {"fmt": "b{b}.h5", "encoding": {"b": "x(.*)"}}] + + +def test_tables_round_trip_through_dump_load_and_history(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [ + {"fmt": "a{a}.h5", "encoding": None}, + {"fmt": "{src}.h5", "db": "data-2.tsv", "url": "https://example.test/"}] + repo.state.replace(pd.DataFrame({"sha1": ["1" * 40], "a": ["0"]})) + remote = pd.DataFrame({"sha1": [None], "checksum_algorithm": ["md5"], + "checksum": ["f" * 32], "size_bytes": ["7"], + "mtime": [""], "src": ["M87"]}) + repo.state.replace(remote, db="data-2.tsv") + repo.dothm.dump(repo.state) + repo.dothm.index.commit("Two templates") + + reopened = Repo(tmp_path / "repo") + assert reopened.state.data["a"].tolist() == ["0"] + table = reopened.state.table("data-2.tsv") + assert table.to_dict("records") == [{ + "sha1": "", "checksum_algorithm": "md5", "checksum": "f" * 32, + "size_bytes": "7", "mtime": "", "src": "M87"}] + head = load_head_state(reopened) + assert head.table("data-2.tsv")["src"].tolist() == ["M87"] + + +def test_a_configured_table_that_was_never_written_loads_empty(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"db": "later.tsv"}] + repo.dothm.dump_yml(repo.state.config, "config") + assert Repo(tmp_path / "repo").state.table("later.tsv").empty + + +def test_dropped_tables_are_deleted_and_unstaged(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "a{a}.h5"}, + {"fmt": "{b}.dat", "db": "data-2.tsv"}] + repo.state.replace(pd.DataFrame({"sha1": ["2" * 40], "b": ["1"]}), + db="data-2.tsv") + repo.dothm.dump(repo.state) + assert (repo.dothm.path / "data-2.tsv").is_file() + remove_entry(repo.state.config, 1) + repo.state.drop_table("data-2.tsv") + repo.dothm.dump(repo.state) + assert not (repo.dothm.path / "data-2.tsv").exists() + assert ("data-2.tsv", 0) not in repo.dothm.index.entries + + +def test_update_ignores_metadata_columns_when_matching_rows(): + state = State() + old = pd.DataFrame({"sha1": [""], "checksum": ["a" * 32], "size_bytes": ["1"], + "name": ["x"]}) + new = old.assign(checksum="b" * 32, size_bytes="2") + state.update(old) + state.update(new) + assert state.data.to_dict("records") == [ + {"sha1": "", "checksum": "b" * 32, "size_bytes": "2", "name": "x"}] + + +def test_rows_of_remote_path_and_shared_tables_are_catalog_only(): + state = State(config={"data": [ + {"fmt": "a{a}.h5"}, + {"fmt": "{src}.h5", "db": "remote.tsv", "url": "https://example.test/"}, + {"fmt": "x{x}.fits", "db": "built.tsv"}, + {"fmt": "y{y}.fits", "db": "built.tsv"}]}, + data=pd.DataFrame({"sha1": ["1" * 40], "a": ["0"]})) + state.tables["remote.tsv"] = pd.DataFrame({"sha1": [""], "src": ["M87"]}) + state.tables["built.tsv"] = pd.DataFrame({ + "path": ["x1.fits", "y2.fits"], "checksum": ["None", "c" * 32], + "x": ["1", "None"], "y": ["None", "2"]}) + rows = {row.path: row.local for row in iter_catalog_rows(state)} + assert rows == {"a0.h5": True, "M87.h5": False, "x1.fits": False, + "y2.fits": False} + assert manifest_map(state) == {"a0.h5": "1" * 40} + assert set(catalog_map(state)) == set(rows) + + +def test_row_fingerprint_prefers_sha1_then_checksum_then_size(): + assert row_fingerprint({"sha1": "ABC", "checksum": "d"}) == "abc" + assert row_fingerprint({"sha1": "", "checksum_algorithm": "MD5", + "checksum": "D"}) == "md5:d" + assert row_fingerprint({"checksum": "None", "size_bytes": "3", + "mtime": None}) == "size:3;mtime:" + + +def test_single_template_repository_layout_is_unchanged(tmp_path): + repo = Repo.init(tmp_path / "repo") + (repo.worktree / "a1.h5").write_text("one\n", encoding="utf-8") + repo.add("a{a}.h5") + assert sorted(path.name for path in repo.dothm.path.glob("*.tsv")) == [ + "data.tsv"] + assert Path(repo.dothm.path / "config.yml").read_text(encoding="utf-8") == ( + "data:\n- fmt: a{a}.h5\n encoding: null\nremote: null\n") + + +REMOTE = "https://example.test/er2/" + + +def _write(repo, name, text): + path = repo.worktree / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return repo.checksum(path) + + +def _commit_local(repo, files, message="local data"): + """Track files as the branch's only template and commit their contents.""" + repo.state.config["data"] = [{"fmt": "{name}.h5", "encoding": None}] + repo.state.replace(pd.DataFrame({ + "sha1": [_write(repo, f"{name}.h5", text) for name, text in files.items()], + "name": list(files)})) + repo.dothm.dump(repo.state) + repo.commit(message) + + +def _catalog_remote(repo, rows, *, keep_local=False): + """Catalog remote files by name, with their published SHA-1 or ''.""" + entries = [{"fmt": "{name}.h5", "db": "data-2.tsv", "url": REMOTE}] + if keep_local: + entries.insert(0, {"fmt": "{run}.dat", "encoding": None}) + else: + repo.state.drop_table("data.tsv") + repo.state.config["data"] = entries + repo.state.replace(pd.DataFrame({ + "sha1": list(rows.values()), "checksum_algorithm": ["md5"] * len(rows), + "checksum": [str(index) * 32 for index, _ in enumerate(rows)], + "size_bytes": ["4"] * len(rows), "mtime": [""] * len(rows), + "name": list(rows)}), db="data-2.tsv") + repo.dothm.dump(repo.state) + + +def test_commit_stores_objects_only_for_local_entries(tmp_path): + repo = Repo.init(tmp_path / "repo") + run_sha = _write(repo, "1.dat", "run\n") + _catalog_remote(repo, {"M87": ""}, keep_local=True) + repo.state.replace(pd.DataFrame({"sha1": [run_sha], "run": ["1"]})) + repo.dothm.dump(repo.state) + assert repo.commit("mixed templates") + assert repo.objects.contains(run_sha) + assert len([path for path in repo.objects.root.rglob("*") if path.is_file()]) == 1 + assert repo.status()["staged"]["catalog"] == [] + + +def test_status_summarizes_remote_changes_and_ignores_downloads(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": "", "SGRA": "", "OJ287": ""}) + repo.commit("remote catalog") + _write(repo, "M87.h5", "downloaded\n") + snapshot = repo.status() + assert snapshot["untracked"] == [] + assert snapshot["worktree"] == {"modified": [], "deleted": []} + + table = repo.state.table("data-2.tsv") + table = table[table["name"] != "OJ287"].copy() + table.loc[table["name"] == "SGRA", "checksum"] = "e" * 32 + table.loc[len(table)] = ["", "md5", "f" * 32, "4", "", "3C279"] + repo.state.set_table("data-2.tsv", table) + repo.dothm.dump(repo.state) + assert repo.status()["staged"]["catalog"] == [{ + "templates": ["{name}.h5"], "url": REMOTE, + "added": 1, "modified": 1, "deleted": 1}] + + +def test_cli_status_prints_remote_catalog_summaries(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": "", "SGRA": ""}) + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["status"]) + assert result.exit_code == 0, result.output + assert f"catalog: {{name}}.h5 from {REMOTE}: 2 new, 0 modified, 0 removed" \ + in result.output + + +def test_checkout_removes_local_files_a_remote_branch_catalogs_differently( + tmp_path): + repo = Repo.init(tmp_path / "repo") + _commit_local(repo, {"M87": "local\n"}) + repo.checkout("remote") + _catalog_remote(repo, {"M87": ""}) + repo.commit("catalog the archive copy") + repo.checkout("main") + repo.checkout("remote") + assert not (repo.worktree / "M87.h5").exists() + repo.checkout("main") + assert (repo.worktree / "M87.h5").read_text(encoding="utf-8") == "local\n" + + +def test_checkout_keeps_local_files_a_remote_branch_catalogs_identically( + tmp_path): + repo = Repo.init(tmp_path / "repo") + _commit_local(repo, {"M87": "same\n"}) + sha1 = repo.checksum(repo.worktree / "M87.h5") + repo.checkout("remote") + _catalog_remote(repo, {"M87": sha1}) + repo.commit("catalog the archive copy") + repo.checkout("main") + repo.checkout("remote") + assert (repo.worktree / "M87.h5").read_text(encoding="utf-8") == "same\n" + + +def test_checkout_rejects_a_different_downloaded_copy_of_a_local_file(tmp_path): + repo = Repo.init(tmp_path / "repo") + _commit_local(repo, {"M87": "local\n"}) + repo.checkout("remote") + _catalog_remote(repo, {"M87": ""}) + repo.commit("catalog the archive copy") + _write(repo, "M87.h5", "downloaded\n") + with pytest.raises(Exception, match="downloaded copy"): + repo.checkout("main") + assert repo.dothm.active_branch.name == "remote" + _write(repo, "M87.h5", "local\n") + assert repo.checkout("main") + + +def test_checkout_between_remote_branches_leaves_downloads_alone(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + repo.commit("first archive") + _write(repo, "M87.h5", "downloaded\n") + repo.checkout("other") + _catalog_remote(repo, {"SGRA": ""}) + repo.commit("second archive") + repo.checkout("main") + assert (repo.worktree / "M87.h5").read_text(encoding="utf-8") == "downloaded\n" + assert repo.status()["untracked"] == [] + + +def test_add_worktree_restores_only_local_files(tmp_path): + repo = Repo.init(tmp_path / "main") + run_sha = _write(repo, "1.dat", "run\n") + _catalog_remote(repo, {"M87": ""}, keep_local=True) + repo.state.replace(pd.DataFrame({"sha1": [run_sha], "run": ["1"]})) + repo.dothm.dump(repo.state) + repo.commit("mixed templates") + assert repo.add_worktree("experiment") + experiment = tmp_path / "experiment" + assert (experiment / "1.dat").read_text(encoding="utf-8") == "run\n" + assert not (experiment / "M87.h5").exists() + assert Repo(experiment).state.table("data-2.tsv")["name"].tolist() == ["M87"] + + +def test_templates_accumulate_and_round_trip_through_checkout(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + _write(repo, "runs/run_1.dat", "dat\n") + repo.add("{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + assert [entry.db for entry in catalog_entries(repo.state.config)] == [ + "data.tsv", "data-2.tsv"] + repo.commit("two templates") + repo.checkout("empty") + repo.rm_cached("{src}_{day}.h5") + repo.rm_cached("runs/run_{run}.dat") + repo.commit("nothing tracked") + repo.checkout("main") + assert (repo.worktree / "M87_095.h5").read_text(encoding="utf-8") == "h5\n" + assert (repo.worktree / "runs/run_1.dat").read_text(encoding="utf-8") == "dat\n" + assert repo.status()["untracked"] == [] + + +def test_new_template_must_match_files_and_leaves_config_unchanged(tmp_path): + repo = Repo.init(tmp_path / "repo") + before = (repo.dothm.path / "config.yml").read_text(encoding="utf-8") + with pytest.raises(ValueError, match="did not match any files"): + repo.add("{src}.fits") + assert (repo.dothm.path / "config.yml").read_text(encoding="utf-8") == before + + +def test_new_template_cannot_claim_files_of_another_template(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + repo.add("{src}_{day}.h5") + with pytest.raises(ValueError, match=r"tracked by another template \(M87_095.h5\)"): + repo.add("{name}.h5") + assert len(catalog_entries(repo.state.config)) == 1 + + +def test_reserved_fields_are_rejected(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "a.h5", "a\n") + for fmt in ("{path}.h5", "{sha256}.h5", "{size_bytes}.h5"): + with pytest.raises(ValueError, match="reserved catalog metadata"): + repo.add(fmt) + + +def test_local_add_refuses_a_template_tracked_remotely(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + _write(repo, "M87.h5", "downloaded\n") + with pytest.raises(ValueError, match="rm --cached"): + repo.add("{name}.h5") + + +def test_dry_run_lists_matches_without_hashing_or_staging(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + staged = {key: entry.binsha for key, entry in repo.dothm.index.entries.items()} + monkeypatch.setattr(Repo, "checksum_many", + lambda paths: pytest.fail("dry run must not hash")) + result = repo.add("{src}_{day}.h5", dry_run=True) + assert result["path"].tolist() == ["M87_095.h5"] + assert catalog_entries(repo.state.config) == [] + assert {key: entry.binsha + for key, entry in repo.dothm.index.entries.items()} == staged + + +def test_dot_rescans_every_local_template_and_skips_downloads(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + _write(repo, "runs/run_1.dat", "dat\n") + repo.add("{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + _write(repo, "SGRA_096.h5", "new\n") + (repo.worktree / "runs/run_1.dat").unlink() + result = repo.add(".") + assert sorted(result["path"]) == ["M87_095.h5", "SGRA_096.h5"] + assert repo.state.data["src"].tolist() == ["M87", "SGRA"] + assert repo.state.table("data-2.tsv").empty + + +def test_dot_from_a_subdirectory_keeps_rows_outside_it(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "repo") + _write(repo, "a/1.dat", "a1\n") + _write(repo, "b/1.dat", "b1\n") + repo.add("{group}/{run}.dat") + (repo.worktree / "a/1.dat").unlink() + _write(repo, "a/2.dat", "a2\n") + monkeypatch.chdir(repo.worktree / "a") + assert repo.add(".")["path"].tolist() == ["a/2.dat"] + rows = repo.state.data.sort_values("group")[["group", "run"]] + assert rows.values.tolist() == [["a", "2"], ["b", "1"]] + + +def test_dot_rejects_a_file_matching_two_local_templates(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "x.h5", "x\n") + _write(repo, "y_1.dat", "y\n") + repo.add("{name}.h5") + repo.add("{name}_{run}.dat") + repo.state.config["data"][1]["fmt"] = "{name}_{run}.{ext}" + _write(repo, "z_2.h5", "z\n") + with pytest.raises(ValueError, match="matches templates"): + repo.add(".") + + +def test_rm_cached_untracks_a_template_and_keeps_its_files(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + _write(repo, "runs/run_1.dat", "dat\n") + repo.add("{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + repo.commit("two templates") + removed = repo.rm_cached("runs/run_{run}.dat") + assert removed.db == "data-2.tsv" + assert not (repo.dothm.path / "data-2.tsv").exists() + assert (repo.worktree / "runs/run_1.dat").exists() + assert repo.status()["staged"]["deleted"] == ["runs/run_1.dat"] + repo.commit("drop runs") + repo.add("runs/run_{run}.dat") + # the removed table's history is never reused by a new template + assert find_db(repo, "runs/run_{run}.dat") == "data-3.tsv" + + +def find_db(repo, fmt): + return next(entry.db for entry in catalog_entries(repo.state.config) + if entry.fmt == fmt) + + +def test_rm_cached_accepts_a_remote_url_template_and_reports_unknown_ones(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + with pytest.raises(ValueError, + match=r"not tracked; tracked templates: '\{name\}.h5'"): + repo.rm_cached("{src}.h5") + repo.rm_cached(REMOTE + "{name}.h5") + assert catalog_entries(repo.state.config) == [] + + +def test_rm_cached_refuses_tables_shared_by_builder_templates(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config["data"] = [{"fmt": "x{x}.fits", "db": "built.tsv"}, + {"fmt": "y{y}.fits", "db": "built.tsv"}] + with pytest.raises(ValueError, match="shares its catalog table"): + repo.rm_cached("x{x}.fits") + + +def test_cli_add_dry_run_and_rm_cached(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _write(repo, "M87_095.h5", "h5\n") + monkeypatch.chdir(repo.worktree) + runner = CliRunner() + preview = runner.invoke(hallmark, ["add", "-n", "{src}_{day}.h5"]) + assert preview.exit_code == 0, preview.output + assert preview.output.splitlines() == ["Would add", "M87_095.h5"] + assert runner.invoke(hallmark, ["add", "{src}_{day}.h5"]).exit_code == 0 + refused = runner.invoke(hallmark, ["rm", "{src}_{day}.h5"]) + assert refused.exit_code != 0 + assert "requires --cached" in refused.output + removed = runner.invoke(hallmark, ["rm", "--cached", "{src}_{day}.h5"]) + assert removed.exit_code == 0, removed.output + assert "files were left in place" in removed.output + missing = runner.invoke(hallmark, ["add", "{src}.fits"]) + assert missing.exit_code != 0 + assert "did not match any files" in missing.output + + +MIRROR = "https://mirror.test/export/" +SIMS = "https://sims.test/runs/" + + +def _two_archives(repo, *, local=True): + """Catalog a local run, archive images, and simulation files.""" + entries = [{"fmt": "{name}.h5", "db": "data-2.tsv", "url": REMOTE}, + {"fmt": "{run}.dat", "db": "data-3.tsv", "url": SIMS}] + if local: + entries.insert(0, {"fmt": "notes_{n}.txt", "encoding": None}) + repo.state.replace(pd.DataFrame({ + "sha1": [_write(repo, "notes_1.txt", "note\n")], "n": ["1"]})) + repo.state.config["data"] = entries + remote_columns = {"sha1": "", "checksum_algorithm": "", "checksum": "", + "mtime": ""} + repo.state.replace(pd.DataFrame([ + {**remote_columns, "size_bytes": "5", "name": "M87"}, + {**remote_columns, "size_bytes": "6", "name": "SGRA"}]), db="data-2.tsv") + repo.state.replace(pd.DataFrame([ + {**remote_columns, "size_bytes": "7", "run": "001"}]), db="data-3.tsv") + repo.dothm.dump(repo.state) + + +def test_plan_downloads_each_template_from_its_own_url(tmp_path): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo, local=False) + plan = repo.plan_download(all_files=True) + assert {item.relative_path.as_posix(): item.source.url + for item in plan.items} == { + "M87.h5": REMOTE, "SGRA.h5": REMOTE, "001.dat": SIMS} + assert [source.url for source in plan.sources] == [REMOTE, SIMS] + assert plan.remote_url is None + assert plan.total_bytes == 18 + summary = plan.summary() + assert f"Sources:\n {REMOTE} (2 file(s); transport http)" in summary + assert f"\n {SIMS} (1 file(s); transport http)" in summary + + +def test_plan_with_one_source_keeps_the_plan_level_fields(tmp_path): + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + plan = repo.plan_download() + assert plan.remote_url == REMOTE + assert plan.remote_backend == "http" + assert f"Source: {REMOTE}\nTransport: http" in plan.summary() + + +def test_local_templates_use_the_default_remote_and_skip_without_one(tmp_path): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo) + plan = repo.plan_download(all_files=True) + assert plan.file_count == 3 + assert plan.unsourced_count == 1 + assert "Skipped 1 cataloged file(s) without a data source" in plan.summary() + with pytest.raises(Exception, match="No data source is configured for: notes"): + repo.plan_download(file_paths=["notes_1.txt", "M87.h5"]) + + repo.set_config(remote_url=MIRROR) + plan = repo.plan_download(all_files=True) + assert {item.relative_path.as_posix(): item.source.url + for item in plan.items}["notes_1.txt"] == MIRROR + assert plan.unsourced_count == 0 + + +def test_named_remote_overrides_every_source(tmp_path): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo) + repo.set_config(remote_name="mirror", remote_url=MIRROR) + plan = repo.plan_download(all_files=True, remote_name="mirror") + assert {item.source.url for item in plan.items} == {MIRROR} + assert plan.remote_name == "mirror" + + +def test_nothing_downloadable_reports_the_missing_remote(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write(repo, "a1.h5", "one\n") + repo.add("a{a}.h5") + with pytest.raises(Exception, match="No remote URL is configured"): + repo.plan_download(all_files=True) + + +def test_execute_downloads_each_source_in_turn(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "repo") + _two_archives(repo, local=False) + fetched = [] + + def fake_fetch(context, relative_path, destination, checksum, chunk_size): + fetched.append(context.remote.file_url(relative_path.as_posix())) + return 1 + + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_fetch) + result = repo.download(repo.plan_download(all_files=True), approved=True) + assert result == {"succeeded": 3, "failed": 0, "total_bytes": 3, "errors": []} + assert sorted(fetched) == [ + REMOTE + "M87.h5", REMOTE + "SGRA.h5", SIMS + "001.dat"] + + +def test_cli_download_dry_run_lists_every_source(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _two_archives(repo, local=False) + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["download", "--all", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "3 file(s); 18 bytes" in result.output + assert "Sources:" in result.output and SIMS in result.output + + +def test_cli_status_omits_headers_of_empty_sections(tmp_path, monkeypatch): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + _catalog_remote(repo, {"M87": ""}) + monkeypatch.chdir(repo.worktree) + staged = CliRunner().invoke(hallmark, ["status"]) + assert "Changes to be committed:" in staged.output + assert "Changes not staged for commit:" not in staged.output + repo.commit("remote catalog") + clean = CliRunner().invoke(hallmark, ["status"]) + assert clean.output == "On branch main\n\nnothing to commit, working tree clean\n" diff --git a/test/test_cli.py b/test/test_cli.py index 86e5ad7..29e56ed 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -18,15 +18,17 @@ import yaml import importlib import pytest +import pandas as pd from pathlib import Path from click.testing import CliRunner from git import Repo as GitRepo from git.exc import GitError from types import SimpleNamespace -from hallmark import ParaFrame +from hallmark import ParaFrame, Repo from hallmark.cli import hallmark -from hallmark.downloader import DownloadError, BULK_DOWNLOAD_WARNING_FILE_COUNT +from hallmark.downloader import DownloadError +from hallmark.download_plan import DownloadItem, DownloadPlan from hallmark.helper_functions import chdir cli_module = importlib.import_module("hallmark.cli") @@ -50,21 +52,34 @@ def _install_repo(monkeypatch, worktree=Path("worktree")): A SimpleNamespace object representing the fake repository, with a 'worktree' attribute set to the given worktree path. """ - repo = SimpleNamespace(worktree=worktree) + repo = SimpleNamespace( + worktree=worktree, + plan_download=lambda output=None, **kwargs: _download_plan( + 0, output or worktree or "downloads")) monkeypatch.setattr(cli_module, "Repo", lambda path: repo) return repo -def _selection(count): - """ - Return a list of (Path, None) tuples for testing download selection, - with the given count of files. - Args: - count: The number of files to generate for the selection. - Returns: - A list of tuples, each containing a Path object for a file and None. - """ - return [(Path(f"file-{index:03d}.dat"), None) for index in range(count)] +def _download_plan(count, output=Path("worktree")): + """Create a download plan for CLI display and approval tests.""" + return DownloadPlan(tuple(DownloadItem(Path(f"file-{index:03d}.dat"), + size_bytes=8 if index == 0 else None) + for index in range(count)), + "https://example.test/data/", Path(output)) + + +def _local_cli_catalog(path): + """Create a local catalog for a small, simulated HTTP dataset.""" + repo = Repo.init(path) + repo.state.config = { + "data": [{"db": "data.tsv"}], + "remote": {"name": "origin", "url": "https://example.test/data/"}} + repo.state.data = pd.DataFrame([ + {"path": "tiny.fits", "size_bytes": 4}, + {"path": "other.txt", "size_bytes": 5}]) + repo.dothm.dump(repo.state) + repo.dothm.index.commit("Catalog two remote files") + return repo def parse(result): @@ -262,18 +277,20 @@ def test_cli_add_regex_flag(monkeypatch): with chdir("repo"): called = {} - def fake_add(self, fmt, encoding=False): + def fake_add(self, fmt, encoding=False, *, dry_run=False): """Fake add method to capture arguments passed to Repo.add.""" called["fmt"] = fmt called["encoding"] = encoding + called["dry_run"] = dry_run return ParaFrame([{"path": "am0.5_i30.h5"}]) monkeypatch.setattr("hallmark.cli.Repo.add", fake_add) result = runner.invoke(hallmark, ["add", "--regex", "."]) assert result.exit_code == 0, f"Expected exit code 0 for add with \ '--regex', got {result.exit_code}" - assert called == {"fmt": ".", "encoding": True}, f"Expected add to be \ - called with fmt='.' and encoding=True, got {called}" + assert called == {"fmt": ".", "encoding": True, "dry_run": False}, \ + f"Expected add to be called with fmt='.' and encoding=True, \ + got {called}" def test_cli_status(): @@ -544,7 +561,7 @@ def test_clone_existing_destination_fails_with_plain_git_stderr(): (target / "placeholder.txt").write_text("test\n", encoding="utf-8") result = runner.invoke( hallmark, - ["clone", "--no-fetch-data", str(source / ".hm"), str(target)]) + ["clone", str(source / ".hm"), str(target)]) assert result.exit_code != 0, \ f"Expected non-zero exit code for clone, got {result.exit_code}" @@ -557,7 +574,7 @@ def test_clone_existing_destination_fails_with_plain_git_stderr(): assert ( result.output.strip() == "fatal: destination path 'repo3' already exists and " - "is not an empty directory."), \ + "is not empty."), \ f"Expected git error message, got: {result.output.strip()}" @@ -602,7 +619,7 @@ def test_clone_copies_committed_hallmark_state(): GitRepo(str(source / ".hm")).index.commit("commit initial hallmark state") result = runner.invoke( hallmark, - ["clone", "--no-fetch-data", str(source / ".hm"), "target"]) + ["clone", str(source / ".hm"), "target"]) assert result.exit_code == 0, \ f"Expected exit code 0 for clone, got {result.exit_code}" @@ -618,102 +635,39 @@ def test_clone_copies_committed_hallmark_state(): "Expected data.tsv to exist in target after clone" -def test_clone_reports_download_error_cleanly(monkeypatch): - """ - Test that the hallmark CLI 'clone' command reports a download error cleanly - when the remote URL is not configured in config.yml. - This test initializes a hallmark repository, monkeypatches the download function - to raise a DownloadError, and verifies that the clone command fails with the - expected error message. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - runner = CliRunner() - with runner.isolated_filesystem(): - source = Path("source") - result = runner.invoke(hallmark, ["init", str(source)]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for init, got {result.exit_code}" - - GitRepo(str(source / ".hm")).index.commit("commit initial hallmark state") - def boom(*args, **kwargs): - """Simulate a download error by raising a DownloadError.""" - raise DownloadError("Remote URL not configured in config.yml") - monkeypatch.setattr("hallmark.cli.download_remote_data", boom) - monkeypatch.setattr( - "hallmark.cli.select_download_files", - lambda *args, **kwargs: [(Path("data.bin"), None)]) - result = runner.invoke( - hallmark, - ["clone", str(source / ".hm"), "target"]) +def test_clone_rejects_removed_download_option(tmp_path): + result = CliRunner().invoke(hallmark, [ + "clone", "source", str(tmp_path / "target"), "--download"]) + assert result.exit_code == 2 + assert "No such option" in result.output + assert not (tmp_path / "target").exists() - assert result.exit_code != 0, f"Expected non-zero exit code for clone with \ - download error, got {result.exit_code}" - assert "Remote URL not configured in config.yml" in result.output, \ - f"Expected download error message in output, got: {result.output}" +def test_clone_cli_skips_download_when_no_remote_files(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "empty") -def test_clone_cli_skips_download_when_no_remote_files(monkeypatch): - """ - Test that the hallmark CLI 'clone' command skips the download step when there are - no remote files configured in the source repository. - This test monkeypatches the Repo class to simulate a source repository with no - remote files and verifies that the clone command completes successfully without - attempting to download any files. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - repo = SimpleNamespace() class FakeRepo: - """Fake Repo class to simulate a hallmark repository with no remote files.""" + lwpaths = Repo.lwpaths + @staticmethod - def clone(url, path, fetch_data=False): - """Simulate cloning a repository by returning a SimpleNamespace.""" + def clone(url, path, **kwargs): return repo - @staticmethod - def lwpaths(path): - """Return the paths to the .hm directory and the worktree.""" - return Path(path) / ".hm", Path(path) monkeypatch.setattr(cli_module, "Repo", FakeRepo) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: []) - result = CliRunner().invoke(hallmark, ["clone", "source", "target"]) - - assert result.exit_code == 0, f"Expected exit code 0 for clone with no remote \ - files, got {result.exit_code}" - assert 'Successfully cloned to "target"' in result.output, \ - f"Expected success message in output, got: {result.output}" - assert "No remote data files are configured." in result.output, \ - f"Expected message about no remote files, got: {result.output}" - - -@pytest.mark.parametrize("max_workers", [0, -1]) -def test_clone_rejects_nonpositive_max_workers(max_workers): - """ - Test that the hallmark CLI 'clone' command rejects non-positive values for the - --max-workers option. This test verifies that the clone command fails with an - appropriate error message when a non-positive value is provided. - Args: - max_workers: The non-positive value to test for the --max-workers option. - """ - result = CliRunner().invoke( - hallmark,[ - "clone", - "--max-workers", - str(max_workers), - "source", - "target"]) + result = CliRunner().invoke(hallmark, [ + "clone", "source", "target"]) + assert result.exit_code == 0, result.output + assert 'Successfully cloned to "target"' in result.output + assert "Download these files?" not in result.output - assert result.exit_code != 0, f"Expected non-zero exit code for clone with \ - non-positive max workers, got {result.exit_code}" - assert "Invalid value for '--max-workers'" in result.output, \ - f"Expected error message about non-positive max workers, got: {result.output}" - assert "x>=1" in result.output, \ - f"Expected error message about non-positive max workers, got: {result.output}" +@pytest.mark.parametrize("max_workers", [0, -1, 4]) +def test_clone_rejects_removed_max_workers(max_workers): + result = CliRunner().invoke(hallmark, [ + "clone", "source", "target", "--max-workers", str(max_workers)]) + assert result.exit_code == 2 + assert "No such option" in result.output -### build tests ### def test_build_cli_parses_fmts_remotes_and_db_suffix(monkeypatch): """ @@ -1174,7 +1128,7 @@ def test_build_cli_reports_missing_config_yml_in_directory(tmp_path): @pytest.mark.parametrize( "arguments, message", [ - (["download"], "Provide one or more file paths, --tsv, or --all"), + (["download"], "Provide file paths, --tsv, --all, or --include"), (["download", "file.dat", "--all"], "--all cannot be combined"), (["download", "--tsv", "data", "--all"], "--all cannot be combined")]) def test_download_cli_rejects_invalid_selection_combinations( @@ -1236,272 +1190,251 @@ def test_download_cli_rejects_nonpositive_worker_count(monkeypatch): def test_download_cli_dry_run_limits_preview(monkeypatch): - """ - Test that the hallmark CLI 'download' command in dry-run mode limits the preview - of selected files to the first 20 files. This test monkeypatches the repository - installation and the selection function to simulate a selection of 23 files, and - verifies that the download command in dry-run mode outputs only the first 20 files - and indicates that there are more files selected. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - Raises: - AssertionError: If the download function is called during the dry-run, which - should not happen. - """ - _install_repo(monkeypatch) - selected = _selection(23) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: selected) - def should_not_download(*args, **kwargs): - """Simulate a dry-run by raising an AssertionError if download is attempted.""" + repo = _install_repo(monkeypatch) + plan = _download_plan(23) + repo.plan_download = lambda *args, **kwargs: plan + + def reject_download(*args, **kwargs): raise AssertionError("dry-run must not download") - monkeypatch.setattr(cli_module, "download_remote_data", should_not_download) - result = CliRunner().invoke(hallmark, ["download", "--all", "--dry-run"]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for dry-run download, got {result.exit_code}" - assert "Selected 23 file(s)" in result.output, \ - f"Expected message about 23 selected files, got: {result.output}" - assert "file-000.dat" in result.output, \ - f"Expected first selected file in output, got: {result.output}" - assert "file-019.dat" in result.output, \ - f"Expected 20th selected file in output, got: {result.output}" - assert "file-020.dat" not in result.output, \ - f"Expected 21st selected file not in output, got: {result.output}" - assert "... 3 more file(s)" in result.output, \ - f"Expected message about 3 more files, got: {result.output}" + repo.download = reject_download + result = CliRunner().invoke(hallmark, ["download", "--all", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "23 file(s)" in result.output + assert "8 bytes known; 22 file(s) with unknown size" in result.output + assert "estimated duration: unknown" in result.output + assert "Source: https://example.test/data/" in result.output + assert f"Destination: {plan.output_path}" in result.output + assert "file-000.dat" in result.output + assert "file-019.dat" in result.output + assert "file-020.dat" not in result.output + assert "... 3 more file(s)" in result.output + assert "Download these files?" not in result.output def test_download_cli_reports_empty_selection(monkeypatch): - """ - Test that the hallmark CLI 'download' command reports an empty selection when no - files are selected for download. This test monkeypatches the repository installation - and the selection function to simulate an empty selection, and verifies that the - download command outputs the expected message indicating that no files were selected - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ _install_repo(monkeypatch) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: []) result = CliRunner().invoke(hallmark, ["download", "--all"]) - - assert result.exit_code == 0, f"Expected exit code 0 for download with empty \ - selection, got {result.exit_code}" - assert "No files selected for download." in result.output, \ - f"Expected message about no files selected, got: {result.output}" + assert result.exit_code == 0, result.output + assert "Download these files?" not in result.output def test_download_cli_passes_selection_and_options_to_downloader(monkeypatch): - """ - Test that the hallmark CLI 'download' command passes the selected files and options - to the downloader function. This test monkeypatches the repository installation, - the selection function, and the downloader function to capture the arguments passed - to them, and verifies that the download command outputs the expected success message - and that the captured arguments match the expected values. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ repo = _install_repo(monkeypatch) - selected = [(Path("nested/file.dat"), "abc")] + plan = _download_plan(1) captured = {} - def fake_select(actual_repo, **kwargs): - """Fake selection function to capture arguments passed to it.""" - captured["select"] = (actual_repo, kwargs) - return selected - - def fake_download(actual_repo, output_path, **kwargs): - """Fake downloader function to capture arguments passed to it.""" - captured["download"] = (actual_repo, output_path, kwargs) - return { - "succeeded": 1, - "failed": 0, - "total_bytes": 1024 * 1024, - "errors": []} - monkeypatch.setattr(cli_module, "select_download_files", fake_select) - monkeypatch.setattr(cli_module, "download_remote_data", fake_download) - result = CliRunner().invoke( - hallmark, - [ - "download", - "nested/file.dat", - "--remote", - "mirror", - "--max-workers", - "2", - "--yes"]) - assert result.exit_code == 0, \ - f"Expected exit code 0 for download, got {result.exit_code}" - assert "Successfully downloaded 1 files (1.0 MB)" in result.output, \ - f"Expected success message in output, got: {result.output}" - assert captured["select"] == ( - repo, - {"file_paths": ("nested/file.dat",), "tsv_names": (), "all_files": False}), \ - f"Expected captured selection arguments to match expected values, \ - got: {captured['select']}" - assert captured["download"] == ( - repo, - Path("worktree"), - { - "max_workers": 2, - "show_progress": True, - "selected_files": selected, - "remote_name": "mirror"}), f"Expected captured download arguments to match \ - expected values, got: {captured['download']}" + def fake_plan(output, **kwargs): + captured["plan"] = (output, kwargs) + return plan + + def fake_download(actual_plan, **kwargs): + captured["download"] = (actual_plan, kwargs) + return {"succeeded": 1, "failed": 0, "total_bytes": 1024 * 1024, + "errors": []} + + repo.plan_download = fake_plan + repo.download = fake_download + result = CliRunner().invoke(hallmark, [ + "download", "nested/file.dat", "--remote", "mirror", "--max-workers", "2", + "--include", "**/*.dat"], input="y\n") + assert result.exit_code == 0, result.output + assert "Successfully downloaded 1 files (1.0 MB)" in result.output + assert captured["plan"] == (None, { + "file_paths": ("nested/file.dat",), "tsv_names": (), "all_files": False, + "include": ("**/*.dat",), "remote_name": "mirror"}) + assert captured["download"][0] is plan + assert captured["download"][1] == { + "max_workers": 2, "progress": True, "approved": True} def test_download_cli_allows_explicit_output_for_bare_repository(monkeypatch): - """ - Test that the hallmark CLI 'download' command allows an explicit output path to be - specified when the repository is bare (i.e., has no worktree). This test - monkeypatches the repository installation to simulate a bare repository and verifies - the download command completes successfully when an explicit output path is provided - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ _install_repo(monkeypatch, worktree=None) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: []) - - result = CliRunner().invoke( - hallmark, ["download", "--all", "--output", "downloads", "--dry-run"]) - - assert result.exit_code == 0, f"Expected exit code 0 for download with explicit \ - output, got {result.exit_code}" - assert "for downloads" in result.output, \ - f"Expected message about output path, got: {result.output}" + result = CliRunner().invoke(hallmark, [ + "download", "--all", "--output", "downloads", "--dry-run"]) + assert result.exit_code == 0, result.output + assert f"Destination: {Path('downloads').absolute()}" in result.output def test_download_cli_converts_selection_errors_to_click_errors(monkeypatch): - """ - Test that the CLI 'download' command converts selection errors to click errors. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - Raises: - AssertionError: If the download function is called during the selection error, - which should not happen. - """ - _install_repo(monkeypatch) - def fail_selection(*args, **kwargs): - """Simulate a selection error by raising a DownloadError.""" + repo = _install_repo(monkeypatch) + + def fail_plan(*args, **kwargs): raise DownloadError("bad selection") - monkeypatch.setattr(cli_module, "select_download_files", fail_selection) - result = CliRunner().invoke(hallmark, ["download", "--tsv", "missing"]) - assert result.exit_code != 0, f"Expected non-zero exit code for download with \ - selection error, got {result.exit_code}" - assert "Error: bad selection" in result.output, \ - f"Expected error message about selection error, got: {result.output}" + repo.plan_download = fail_plan + result = CliRunner().invoke(hallmark, ["download", "--tsv", "missing"]) + assert result.exit_code != 0 + assert "Error: bad selection" in result.output def test_download_cli_reports_only_first_ten_errors(monkeypatch): - """ - Test that the hallmark CLI 'download' command reports only the first ten errors - when there are more than ten failed downloads. This test monkeypatches the repo - installation, the selection function, and the downloader function to simulate a - scenario with 12 failed downloads, and verifies that the download command outputs - only the first ten errors and indicates that there are more errors. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - _install_repo(monkeypatch) - monkeypatch.setattr( - cli_module, - "select_download_files", - lambda *args, **kwargs: [(Path("file.dat"), None)]) - errors = [f"failure-{index}" for index in range(12)] - monkeypatch.setattr( - cli_module, - "download_remote_data", - lambda *args, **kwargs: { - "succeeded": 2, - "failed": 12, - "total_bytes": 0, - "errors": errors}) - result = CliRunner().invoke(hallmark, ["download", "file.dat", "--yes"]) - - assert result.exit_code != 0, f"Expected non-zero exit code for download with \ - multiple errors, got {result.exit_code}" - assert "2 succeeded, 12 failed" in result.output, \ - f"Expected summary of succeeded and failed downloads, got: {result.output}" - assert "failure-0" in result.output, \ - f"Expected first error in output, got: {result.output}" - assert "failure-9" in result.output, \ - f"Expected tenth error in output, got: {result.output}" - assert "failure-10" not in result.output, \ - f"Expected eleventh error not in output, got: {result.output}" - assert "... 2 more error(s)" in result.output, \ - f"Expected message about additional errors, got: {result.output}" - assert "Failed to download 12 file(s)" in result.output, \ - f"Expected summary of failed downloads, got: {result.output}" - - -def test_download_cli_prompts_for_bulk_selection(monkeypatch): - """ - Test that the hallmark CLI 'download' command prompts for confirmation when a bulk - selection of files is made. This test monkeypatches the repository installation, - the selection function, and the downloader function to simulate a bulk selection, - and verifies that the download command prompts for confirmation and aborts when - the user responds with 'n'. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - Raises: - AssertionError: If the download function is called during the aborted selection, - which should not happen. - """ - _install_repo(monkeypatch) - selected = _selection(BULK_DOWNLOAD_WARNING_FILE_COUNT) - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: selected) - - def should_not_download(*args, **kwargs): - """Simulate an aborted selection by raising an AssertionError.""" - raise AssertionError("aborted selection must not download") - monkeypatch.setattr(cli_module, "download_remote_data", should_not_download) + repo = _install_repo(monkeypatch) + repo.plan_download = lambda *args, **kwargs: _download_plan(14) + repo.download = lambda *args, **kwargs: { + "succeeded": 2, "failed": 12, "total_bytes": 0, + "errors": [f"failure-{index}" for index in range(12)]} + result = CliRunner().invoke(hallmark, ["download", "--all"], input="y\n") + assert result.exit_code != 0 + assert "2 succeeded, 12 failed" in result.output + assert "failure-0" in result.output + assert "failure-9" in result.output + assert "failure-10" not in result.output + assert "... 2 more error(s)" in result.output + assert "Failed to download 12 file(s)" in result.output + + +@pytest.mark.parametrize("count", [1, 100]) +@pytest.mark.parametrize("answer", ["n\n", "", "\n"]) +@pytest.mark.parametrize("legacy_yes", [False, True]) +def test_download_cli_requires_affirmative_approval( + monkeypatch, count, answer, legacy_yes): + repo = _install_repo(monkeypatch) + repo.plan_download = lambda *args, **kwargs: _download_plan(count) + + def reject_download(*args, **kwargs): + raise AssertionError("rejected or absent approval must not download") + + repo.download = reject_download + args = ["download", "--all"] + (["--yes"] if legacy_yes else []) + result = CliRunner().invoke(hallmark, args, input=answer) + assert result.exit_code != 0 + assert f"{count} file(s)" in result.output + assert "Download these files? [y/N]" in result.output + assert "Aborted!" in result.output + if legacy_yes: + assert "--yes is deprecated" in result.output + + +@pytest.mark.parametrize("arguments", [[], ["--no-download"]]) +def test_clone_cli_skips_download_without_terminal_or_when_disabled( + monkeypatch, tmp_path, arguments): + source = _local_cli_catalog(tmp_path / "source") + + def reject_download(*args, **kwargs): + raise AssertionError("noninteractive or disabled download must not run") + + monkeypatch.setattr(Repo, "plan_download", reject_download) + monkeypatch.setattr(Repo, "download", reject_download) + target = tmp_path / "target" + result = CliRunner().invoke(hallmark, [ + "clone", str(source.dothm.path), str(target), *arguments]) + assert result.exit_code == 0, result.output + assert (target / ".hm/data.tsv").is_file() + assert not (target / "tiny.fits").exists() + assert "Download these files?" not in result.output + + +def test_init_cli_creates_only_a_local_repository(monkeypatch, tmp_path): + captured = [] + monkeypatch.setattr(Repo, "init", lambda *args, **kwargs: captured.append( + (args, kwargs))) + result = CliRunner().invoke(hallmark, ["init", str(tmp_path / "target")]) + assert result.exit_code == 0, result.output + assert captured == [((str(tmp_path / "target"),), {})] + + +def test_clone_cli_rejects_conflicting_download_aliases(): + result = CliRunner().invoke(hallmark, [ + "clone", "source", "target", "--no-fetch-data"]) + assert result.exit_code != 0 + assert "No such option" in result.output + + +@pytest.mark.parametrize("during_clone", [False, True]) +@pytest.mark.parametrize("answer", ["y\n", "n\n", ""]) +def test_cli_downloads_only_approved_selected_payload( + monkeypatch, tmp_path, during_clone, answer): + from mock_server import MockServer + + source = _local_cli_catalog(tmp_path / "source") + server = MockServer("https://example.test/data/") + server.add_file("tiny.fits", b"fits") + server.add_file("other.txt", b"notes") + requests_made = [] + original_get = server.get + + def capture_get(url, **kwargs): + requests_made.append(url) + return original_get(url, **kwargs) + + server.get = capture_get + monkeypatch.setattr(requests, "Session", lambda: server) + if during_clone: + target = tmp_path / "target" + cloned = CliRunner().invoke(hallmark, [ + "clone", str(source.dothm.path), str(target)]) + assert cloned.exit_code == 0, cloned.output + monkeypatch.chdir(target) + arguments = ["download", "--include", "*.fits"] + else: + target = source.worktree + monkeypatch.chdir(target) + arguments = ["download", "--include", "*.fits"] + result = CliRunner().invoke(hallmark, arguments, input=answer) + assert "1 file(s); 4 bytes" in result.output + assert "Download these files? [y/N]" in result.output + assert "Source: https://example.test/data/" in result.output + if answer == "y\n": + assert result.exit_code == 0, result.output + assert requests_made == ["https://example.test/data/tiny.fits"] + assert (target / "tiny.fits").read_bytes() == b"fits" + else: + assert result.exit_code != 0 + assert requests_made == [] + assert not (target / "tiny.fits").exists() + assert not (target / "other.txt").exists() + + +@pytest.mark.parametrize("arguments", [ + ["--interactive", "--include", "*.fits"], ["--fmt", "{name}.fits"], + ["--with-download", "--fmt", "{name:invalid}"], +]) +def test_clone_cli_rejects_invalid_selection_before_source_access( + monkeypatch, arguments): + def fail(*args, **kwargs): + raise AssertionError("Invalid selection must fail before cloning") + + monkeypatch.setattr(Repo, "clone", fail) + result = CliRunner().invoke(hallmark, ["clone", "source", "target", *arguments]) + assert result.exit_code != 0 + assert "Error:" in result.output + + +@pytest.mark.parametrize("option", ["--remote-backend", "--remote-backend-options"]) +def test_set_config_cli_rejects_backend_flags(monkeypatch, tmp_path, option): + repo = Repo.init(tmp_path / "repo") + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["set-config", option, "unused"]) + assert result.exit_code == 2 + assert "No such option" in result.output + + +@pytest.mark.parametrize("answer", ["y\n", "n\n", ""]) +def test_remote_add_downloads_only_after_confirmation(monkeypatch, tmp_path, answer): + from hallmark.transport import OperationContext + from hallmark.transport.base import RemoteObjectMissing + from mock_server import MockServer + + def metadata(context, path): + if path == "": + return '

Index of data

a.fits' + raise RemoteObjectMissing("Missing metadata") + + server = MockServer("https://example.test/data/") + server.add_file("a.fits", b"fits") + monkeypatch.setattr(OperationContext, "read_text", metadata) + monkeypatch.setattr(requests, "Session", lambda: server) + destination = tmp_path / "target" + result = CliRunner().invoke(hallmark, ["init", str(destination)]) + assert result.exit_code == 0, result.output + monkeypatch.chdir(destination) result = CliRunner().invoke( - hallmark, ["download", "--all"], input="n\n") - - assert result.exit_code != 0, f"Expected non-zero exit code for download with \ - aborted selection, got {result.exit_code}" - assert ( - f"Selected {BULK_DOWNLOAD_WARNING_FILE_COUNT} files for download" in - result.output), f"Expected message about bulk selection, got: {result.output}" - assert "Continue? [y/N]" in result.output, \ - f"Expected prompt for confirmation, got: {result.output}" - assert "Aborted!" in result.output, \ - f"Expected message about aborted download, got: {result.output}" - - -def test_download_cli_yes_skips_bulk_prompt(monkeypatch): - """ - Test that the hallmark CLI 'download' command skips the bulk selection prompt when - the --yes option is provided. This test monkeypatches the repository installation, - the selection function, and the downloader function to simulate a bulk selection, - and verifies that the download command completes successfully without prompting for - confirmation when the --yes option is used. - Args: - monkeypatch: pytest fixture for monkeypatching functions and attributes. - """ - _install_repo(monkeypatch) - selected = _selection(BULK_DOWNLOAD_WARNING_FILE_COUNT) - called = {"download": False} - monkeypatch.setattr( - cli_module, "select_download_files", lambda *args, **kwargs: selected) - def fake_download(*args, **kwargs): - """Fake downloader function to simulate a successful download.""" - called["download"] = True - return { - "succeeded": len(selected), - "failed": 0, - "total_bytes": 0, - "errors": []} - monkeypatch.setattr(cli_module, "download_remote_data", fake_download) - result = CliRunner().invoke(hallmark, ["download", "--all", "--yes"]) - - assert result.exit_code == 0, \ - f"Expected exit code 0 for download with --yes, got {result.exit_code}" - assert called["download"], \ - "Expected download function to be called with --yes, but it was not" - assert "Continue?" not in result.output, \ - f"Expected no prompt for confirmation with --yes, got: {result.output}" + hallmark, ["add", "https://example.test/data/{name}.fits"]) + assert result.exit_code == 0, result.output + assert "Download these files?" not in result.output + result = CliRunner().invoke(hallmark, ["download", "--all"], input=answer) + assert "Download these files? [y/N]" in result.output + assert Repo(destination).state.data["name"].tolist() == ["a"] + assert (destination / "a.fits").exists() == (answer == "y\n") + assert (result.exit_code == 0) == (answer == "y\n") diff --git a/test/test_discovery.py b/test/test_discovery.py new file mode 100644 index 0000000..f524ff5 --- /dev/null +++ b/test/test_discovery.py @@ -0,0 +1,270 @@ +"""Metadata-only discovery against local fixtures; no public network is used.""" + +from types import SimpleNamespace + +import pytest + +from hallmark.discovery import discover, path_matches +from hallmark.backends import HttpBackend +from hallmark.transport.base import ( + CapabilityError, DownloadError, RemoteEntry, RemoteSpec, TransferCancelled, +) + + +class Source: + """Serve directory listings and record requests for discovery tests.""" + def __init__(self, pages, url="https://data.test/public/"): + self.remote = RemoteSpec.parse(url) + self.pages = pages + self.reads = [] + if self.remote.scheme in {"http", "https"}: + self.transport = HttpBackend(self) + + def read_text(self, path): + self.reads.append(path) + assert path in self.pages, f"Unexpected payload request: {path}" + return self.pages[path] + + def check_cancelled(self): + pass + + +def index(*links): + """Render a small HTML directory index from the supplied links.""" + return 'Index of /' + ''.join( + f'{path}' for path in links) + + +@pytest.mark.parametrize("url", ["https://data.test/public/", "http://data.test/public"]) +def test_auto_index_walks_all_directories_without_payloads(url): + source = Source({ + "": index("root.fits", "nested/", "nested/", "../"), + "nested/": index("item.bin", "deeper/", "../"), + "nested/deeper/": index("file.txt"), + }, url) + entries = discover(source) + assert [entry.path for entry in entries] == [ + "nested/deeper/file.txt", "nested/item.bin", "root.fits"] + assert source.reads == ["", "nested/", "nested/deeper/"] + assert all(entry.checksum is None for entry in entries) + + +def test_cyverse_markup_is_detected_automatically(): + source = Source({"": '' + '
file' + '
'}) + assert [entry.path for entry in discover(source)] == ["file.fits"] + + +@pytest.mark.parametrize("page, size, mtime", [ + ('Index of /file' + ' 17-Sep-2026 02:31 12345\n', 12345, None), + ('Index of /
file' + '2026-09-17 02:3112345
', 12345, None), + ('
file' + '12,345
', 12345, None), + ('Index of /file', 12345, 1750000000), + ('Index of /file' + ' 17-Sep-2026 02:31 1.2M\n', None, None), +]) +def test_exact_sizes_are_preserved_and_rounded_sizes_remain_unknown(page, size, mtime): + assert discover(Source({"": page})) == [ + RemoteEntry(path="file.fits", size=size, mtime=mtime)] + + +def test_hallmark_and_git_directories_are_not_dataset_files(): + source = Source({"": index(".hm/", ".git/", ".HM/", "file.fits")}) + assert [entry.path for entry in discover(source)] == ["file.fits"] + assert source.reads == [""] + + +def test_desi_style_landing_page_links_to_scoped_indexes(): + source = Source({ + "": '

DESI public data

', + "dr1/": index("spectro/"), + "dr1/spectro/": index("redshift.fits"), + "edr/": index("old.fits"), + }) + assert [entry.path for entry in discover(source)] == [ + "dr1/spectro/redshift.fits", "edr/old.fits"] + + +@pytest.mark.parametrize("page", [ + 'Index of /empty/', '
', +]) +def test_recognized_empty_index(page): + assert discover(Source({"": page})) == [] + + +@pytest.mark.parametrize("page", [ + '

Welcome

Log in', + '
', + '
Access denied
', + 'not a directory index', +]) +def test_unsupported_page_is_not_an_empty_catalog(page): + with pytest.raises(CapabilityError, match="no usable directory listing"): + discover(Source({"": page})) + + +def test_link_resolution_preserves_literal_names_and_stays_under_root(): + source = Source({ + "": index( + "https://other.test/x/", "/elsewhere/", "../", "?C=N;O=D", + "#top", "/public/sub%20dir/", "/public/%2520/", "root%25.fits"), + "sub dir/": index("file%20name.fits", "../"), + "%20/": index("literal%2520.fits"), + }) + assert [entry.path for entry in discover(source)] == [ + "%20/literal%20.fits", "root%.fits", "sub dir/file name.fits"] + assert source.reads == ["", "sub dir/", "%20/"] + + +@pytest.mark.parametrize("href", ["%2e%2e/secret/", "sub%5cfile", "bad%00file"]) +def test_encoded_unsafe_paths_are_rejected(href): + with pytest.raises((DownloadError, ValueError)): + discover(Source({"": index(href)})) + + +@pytest.mark.parametrize("path, pattern, matched", [ + ("root.fits", "**/*.fits", True), + ("a/b/root.fits", "**/*.fits", True), + ("a/b/root.txt", "**/*.fits", False), + ("a/b/root.fits", "a/*.fits", False), + ("a/root.fits", "a/?.fits", False), + ("a/a.fits", "a/[ab].fits", True), + ("root.txt", ["**/*.fits", "**/*.txt"], True), +]) +def test_recursive_globs(path, pattern, matched): + assert path_matches(path, include=pattern) is matched + + +def test_globs_are_case_sensitive(): + assert path_matches("sample_001.fits", include="sample_*.fits") + assert not path_matches("Sample_001.fits", include="sample_*.fits") + + +@pytest.mark.parametrize("invalid", [12, {"*.fits": True}, iter(["*.fits"])]) +def test_invalid_filter_types_fail_before_source_access(invalid): + source = Source({"": index("file.fits")}) + with pytest.raises(ValueError, match="glob string"): + discover(source, include=invalid) + assert source.reads == [] + + +@pytest.mark.parametrize("final_url", [ + "https://data.test/public/release%20one/", + "https://DATA.TEST:443/public/release%20one/", +]) +def test_redirected_listing_uses_final_directory_and_deduplicates_crawl(final_url): + source = Source({ + "": index("alias/", "release%20one/"), + "alias/": index("file.fits", "nested/"), + "release one/nested/": index("other.fits"), + }) + source.transport.text_urls = {"alias/": final_url} + entries = discover(source) + assert [entry.path for entry in entries] == [ + "release one/file.fits", "release one/nested/other.fits"] + assert source.reads == ["", "alias/", "release one/nested/"] + + +def test_filter_traverses_unmatched_directories_and_excludes_other_payloads(): + source = Source({"": index("root.fits", "nested/", "notes.txt"), + "nested/": index("other.fits", "other.bin")}) + assert [entry.path for entry in discover(source, include="**/*.fits")] == [ + "nested/other.fits", "root.fits"] + assert source.reads == ["", "nested/"] + + +def test_only_conventional_checksum_metadata_is_read(): + digest = "a" * 64 + source = Source({ + "": index("data.fits", "sha256sums.txt", "custom_checksum_payload.bin"), + "sha256sums.txt": f"{digest} data.fits\n", + }) + entries = discover(source, include="*.fits") + assert entries == [RemoteEntry(path="data.fits", checksum_algorithm="sha256", + checksum=digest)] + assert source.reads == ["", "sha256sums.txt"] + + +def test_checksum_manifest_cannot_add_unlisted_files(): + source = Source({"": index("sha256sums.txt"), + "sha256sums.txt": "a" * 64 + " phantom.fits\n"}) + assert [entry.path for entry in discover(source)] == ["sha256sums.txt"] + + +def test_nested_manifest_accepts_published_directory_prefixed_paths(): + source = Source({ + "": index("release/"), + "release/": index("project/"), + "release/project/": index("sha256sums", "file.fits"), + "release/project/sha256sums": "a" * 64 + " project/file.fits\n", + }) + assert discover(source, include="**/*.fits")[0].checksum == "a" * 64 + + +def test_conflicting_manifests_fail(): + source = Source({ + "": index("file.fits", "sha256sums", "project.sha256sums"), + "sha256sums": "a" * 64 + " file.fits\n", + "project.sha256sums": "b" * 64 + " file.fits\n", + }) + with pytest.raises(DownloadError, match="Conflicting"): + discover(source) + + +def test_sftp_metadata_and_progress_callback(): + source = Source({}, url="sftp://lab/srv/data/") + + def entries(on_directory): + on_directory("") + yield RemoteEntry(path="root.fits", size=12, mtime=123) + on_directory("nested") + yield RemoteEntry(path="nested/file.bin", size=20) + + source.transport = SimpleNamespace(iter_entries=entries, prepare=lambda: None) + snapshots = [] + selected = discover(source, include="**/*.fits", progress=snapshots.append) + assert selected == [RemoteEntry(path="root.fits", size=12, mtime=123)] + assert snapshots[-1] == {"directories": 2, "files": 2, + "matched": 1, "current": "nested"} + assert source.reads == [] + + +def test_cancellation_does_not_return_a_partial_inventory(): + source = Source({"": index("a/"), "a/": index("file")}) + + def cancel(): + if source.reads: + raise TransferCancelled("cancelled") + + source.check_cancelled = cancel + with pytest.raises(TransferCancelled): + discover(source) + + +def test_discovery_bar_has_unknown_total(monkeypatch): + bars = [] + + class Bar: + def __init__(self, **kwargs): + bars.append(kwargs) + + def update(self): + pass + + def set_postfix(self, **kwargs): + pass + + def close(self): + pass + + monkeypatch.setattr("hallmark.discovery.tqdm", Bar) + discover(Source({"": index("a.fits")}), progress=True) + assert bars[0]["total"] is None + assert not bars[0]["disable"] diff --git a/test/test_download_plan.py b/test/test_download_plan.py new file mode 100644 index 0000000..5a6462f --- /dev/null +++ b/test/test_download_plan.py @@ -0,0 +1,281 @@ +"""Test download planning and approval using local catalog metadata.""" + +from dataclasses import FrozenInstanceError, replace +from hashlib import sha256 +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from hallmark.download_plan import DownloadItem, DownloadPlan +from hallmark.downloader import ( + DownloadError, + download_remote_data, + execute_download_plan, + plan_download, + select_download_files, +) + + +@pytest.fixture +def catalog(tmp_path): + """Create a local catalog with known and unknown file metadata.""" + metadata = tmp_path / ".hm" + metadata.mkdir() + frame = pd.DataFrame([ + {"path": "nested/a.fits", "size_bytes": "6", "mtime": "2026-09-18", + "sha256": sha256(b"abcdef").hexdigest()}, + {"path": "b.txt", "size_bytes": "", "mtime": "", "sha256": ""}, + {"path": "empty.fits", "size_bytes": "0", "mtime": "", "sha256": ""}, + ]) + frame.to_csv(metadata / "data.tsv", sep="\t", index=False) + return SimpleNamespace( + dothm=SimpleNamespace(path=metadata), worktree=tmp_path, + state=SimpleNamespace(data=frame, config={ + "data": [{"db": "data.tsv"}], + "remote": {"name": "origin", "url": "https://source.test/data/"}, + })) + + +def test_plan_reads_path_catalog_offline_and_preserves_unknown_sizes( + catalog, monkeypatch): + def reject_network(*args, **kwargs): + raise AssertionError("planning must not contact a server") + monkeypatch.setattr("hallmark.downloader.OperationContext", reject_network) + monkeypatch.setattr("requests.sessions.Session.request", reject_network) + plan = plan_download(catalog) + assert plan.file_count == 3 + assert plan.known_bytes == 6 + assert plan.unknown_size_count == 1 + assert plan.total_bytes is None + assert plan.estimated_seconds is None + assert "1 file(s) with unknown size" in plan.summary() + assert "estimated duration: unknown" in plan.summary() + assert plan.items[0].mtime == "2026-09-18" + assert plan.items[2].size_bytes == 0 + + +def test_explicit_paths_retain_catalog_checksums_and_metadata(catalog): + plan = plan_download(catalog, file_paths=["nested/a.fits"]) + assert plan.file_count == 1 + assert plan.items[0].checksum == ("sha256", sha256(b"abcdef").hexdigest()) + assert plan.items[0].size_bytes == 6 + with pytest.raises(DownloadError, match="not in the catalog"): + plan_download(catalog, file_paths=["unlisted.bin"]) + assert select_download_files(catalog, file_paths=["nested/a.fits"]) == [ + (Path("nested/a.fits"), plan.items[0].checksum)] + + +def test_sizes_survive_nullable_numeric_tsv_serialization(catalog): + frame = catalog.state.data.copy() + frame["size_bytes"] = [6, None, 0] + frame.to_csv(catalog.dothm.path / "data.tsv", sep="\t", index=False) + plan = plan_download(catalog) + assert [item.size_bytes for item in plan.items] == [6, None, 0] + + +def test_plan_filters_without_authorizing_download(catalog): + plan = plan_download(catalog, include="**/*.fits") + assert [item.relative_path.as_posix() for item in plan.items] == [ + "nested/a.fits", "empty.fits"] + assert plan.total_bytes == 6 + with pytest.raises(DownloadError, match="approval"): + execute_download_plan(catalog, plan) + + +def test_duration_requires_supplied_rate_and_complete_sizes(catalog): + known = plan_download(catalog, file_paths="nested/a.fits", + estimated_bytes_per_second=3) + assert known.estimated_seconds == 2 + unknown = plan_download(catalog, estimated_bytes_per_second=3) + assert unknown.estimated_seconds is None + with pytest.raises(ValueError, match="positive rate"): + replace(known, estimated_bytes_per_second=float("nan")) + + +def test_plan_is_immutable_and_copies_input_sequences(tmp_path): + checksum = ["sha256", "a" * 64] + item = DownloadItem(Path("a.bin"), checksum) + items = [item] + plan = DownloadPlan(items, "https://source.test/", tmp_path) + checksum[1] = "b" * 64 + items.clear() + assert plan.file_count == 1 + assert plan.items[0].checksum == ("sha256", "a" * 64) + with pytest.raises(FrozenInstanceError): + plan.remote_url = "https://different.test/" + with pytest.raises(FrozenInstanceError): + plan.items[0].size_bytes = 4 + + +def test_plan_display_redacts_url_credentials_without_changing_source(tmp_path): + url = "https://user:secret@example.test:8443/data/?token=secret#secret" + plan = DownloadPlan((DownloadItem(Path("a.bin")),), url, tmp_path) + assert plan.remote_url == url + assert "Source: https://example.test:8443/data/" in plan.summary() + assert "secret" not in plan.summary() + assert "user" not in plan.summary() + assert "secret" not in repr(plan) + + +@pytest.mark.parametrize("approval", [False, None, 1, "yes"]) +def test_unapproved_execution_never_opens_transport(catalog, monkeypatch, approval): + plan = plan_download(catalog) + def reject_open(*args, **kwargs): + raise AssertionError("unapproved transfer opened its transport") + monkeypatch.setattr("hallmark.downloader.OperationContext", reject_open) + with pytest.raises(DownloadError, match="approval"): + execute_download_plan(catalog, plan, approved=approval) + with pytest.raises(DownloadError, match="approval"): + download_remote_data(catalog, catalog.worktree, approved=approval) + + +@pytest.mark.parametrize("known_size", [True, False]) +@pytest.mark.parametrize("transfer_fails", [False, True]) +def test_approved_execution_uses_pinned_source_and_byte_progress( + catalog, monkeypatch, known_size, transfer_fails): + plan = plan_download(catalog, file_paths="nested/a.fits") + if not known_size: + plan = replace(plan, items=(replace(plan.items[0], size_bytes=None),)) + catalog.state.config["remote"]["url"] = "https://changed.test/" + (catalog.dothm.path / "data.tsv").write_text("path\nother.bin\n") + opened = [] + ticks = [] + progress_options = [] + completion_counts = [] + + class Progress: + def __init__(self, **kwargs): + progress_options.append(kwargs) + + def update(self, count): + ticks.append(count) + + def set_postfix(self, **kwargs): + completion_counts.append(kwargs) + + def close(self): + pass + + class Context: + def __init__(self, remote, output_root): + opened.append(remote) + self.output_root = output_root + self.transport = self + self.on_bytes = None + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def prepare(self): + pass + + def check_cancelled(self): + pass + + def fetch(self, relative_path, destination, **kwargs): + assert relative_path == "nested/a.fits" + with destination.open("wb") as handle: + for chunk in (b"abc", b"def"): + handle.write(chunk) + self.on_bytes(len(chunk)) + if transfer_fails: + raise DownloadError("interrupted transfer") + + monkeypatch.setattr("hallmark.downloader.OperationContext", Context) + monkeypatch.setattr("hallmark.downloader.tqdm", Progress) + result = execute_download_plan(catalog, plan, approved=True, show_progress=True) + assert opened[0].url == "https://source.test/data/" + assert ticks == [3, 3] + assert progress_options[0]["total"] == (6 if known_size else None) + assert progress_options[0]["unit"] == "B" + assert completion_counts[-1] == {"files": "1/1", "failed": int(transfer_fails)} + if transfer_fails: + assert result["failed"] == 1 + assert not (catalog.worktree / "nested/a.fits").exists() + else: + assert result["total_bytes"] == 6 + assert (catalog.worktree / "nested/a.fits").read_bytes() == b"abcdef" + + +def test_execution_rechecks_destination_after_approval(catalog, tmp_path): + output = tmp_path / "output" + output.mkdir() + plan = plan_download(catalog, output, file_paths="nested/a.fits") + outside = tmp_path / "outside" + outside.mkdir() + (output / "nested").symlink_to(outside, target_is_directory=True) + with pytest.raises(DownloadError, match="outside|symbolic link"): + execute_download_plan(catalog, plan, approved=True) + assert not (outside / "a.fits").exists() + + +def test_execution_rejects_changed_output_root(catalog, tmp_path): + output = tmp_path / "output" + output.mkdir() + plan = plan_download(catalog, output, file_paths="nested/a.fits") + output.rmdir() + output.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(DownloadError, match="destination changed"): + execute_download_plan(catalog, plan, approved=True) + + +def test_empty_plan_needs_no_approval(catalog): + plan = plan_download(catalog, include="**/*.absent") + assert execute_download_plan(catalog, plan) == { + "succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} + + +def test_bare_catalog_requires_explicit_output(catalog): + catalog.worktree = None + with pytest.raises(DownloadError, match="output_path"): + plan_download(catalog) + + +def test_plan_pins_backend_and_nested_options_without_loading_plugin( + catalog, monkeypatch): + options = {"servers": [{"url": "https://original.test/"}]} + catalog.state.config["remote"].update( + backend="external-survey", backend_options=options) + + def reject_network(*args, **kwargs): + raise AssertionError("planning must not load or contact a backend") + + monkeypatch.setattr("hallmark.downloader.OperationContext", reject_network) + plan = plan_download(catalog, file_paths="nested/a.fits") + options["servers"][0]["url"] = "https://changed.test/" + catalog.state.config["remote"]["backend"] = "another-survey" + assert plan.remote_backend == "external-survey" + assert plan.backend_options["servers"][0]["url"] == "https://original.test/" + with pytest.raises(TypeError): + plan.backend_options["servers"][0]["url"] = "https://changed.test/" + with pytest.raises(TypeError): + plan.backend_options["extra"] = True + assert hash(plan) == hash(replace(plan)) + captured = [] + + def download(source, *args, **kwargs): + captured.append(source) + return {"succeeded": 1, "failed": 0, "total_bytes": 6, "errors": []} + + monkeypatch.setattr("hallmark.downloader._download_selected", download) + execute_download_plan(catalog, plan, approved=True) + assert captured[0].backend == "external-survey" + assert captured[0].backend_options["servers"][0]["url"] == \ + "https://original.test/" + + +def test_legacy_remote_pins_default_backend(catalog): + assert plan_download(catalog).remote_backend == "http" + + +def test_missing_plugin_can_be_planned_but_not_executed(catalog): + catalog.state.config["remote"]["backend"] = "missing-survey-plugin" + plan = plan_download(catalog, file_paths="nested/a.fits") + with pytest.raises(DownloadError, match="missing-survey-plugin"): + execute_download_plan(catalog, plan, approved=True) + assert not (catalog.worktree / "nested/a.fits").exists() diff --git a/test/test_downloader.py b/test/test_downloader.py index a6f9fb5..c60c12c 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -307,14 +307,9 @@ def test_safe_remote_path_rejects_unsafe_and_reserved_paths(unsafe_path): _safe_remote_path(unsafe_path) -def test_safe_remote_path_normalizes_valid_relative_path(): - """ - Test that _safe_remote_path correctly normalizes a valid relative path by stripping - leading/trailing whitespace and converting it to a Path object. - """ - assert _safe_remote_path(" nested/file.txt ") == Path("nested/file.txt"), \ - f"Expected normalized path 'nested/file.txt', \ - got {_safe_remote_path(' nested/file.txt ')}" +def test_safe_remote_path_preserves_literal_whitespace(): + """Test that catalog paths retain significant leading and trailing spaces.""" + assert _safe_remote_path(" nested/file.txt ") == Path(" nested/file.txt ") @pytest.mark.parametrize( @@ -456,11 +451,12 @@ def test_download_remote_data_builds_urls_and_destinations_from_fmt( tmp_path: A pytest fixture providing a temporary directory. """ captured = {} - def fake_download_file(url, destination, sha1, chunk_size=8192): + def fake_download_file(context, relative_path, destination, sha1, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """A fake _download_file function that records its arguments""" captured.setdefault("files", []).append((url, destination, sha1)) return 123 - monkeypatch.setattr("hallmark.downloader._download_file", fake_download_file) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download_file) repo = SimpleNamespace( state=SimpleNamespace( config={ @@ -479,7 +475,7 @@ def fake_download_file(url, destination, sha1, chunk_size=8192): "pipeline": "hops", "step": "netcal", "type": "StokesI"}]))) - result = download_remote_data(repo, tmp_path) + result = download_remote_data(repo, tmp_path, approved=True) assert result == {"succeeded": 1, "failed": 0, "total_bytes": 123, "errors": []},\ f"Expected result {{'succeeded': 1, 'failed': 0, 'total_bytes': 123, \ @@ -499,12 +495,12 @@ def test_download_remote_data_returns_empty_result_without_remote(tmp_path): tmp_path: A pytest fixture providing a temporary directory.""" repo = _repo(tmp_path) - assert download_remote_data(repo, tmp_path) == { + assert download_remote_data(repo, tmp_path, approved=True) == { "succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []}, f"Expected empty result for repo without remote, but got \ - {download_remote_data(repo, tmp_path)}" + {download_remote_data(repo, tmp_path, approved=True)}" def test_download_remote_data_requires_remote_url(tmp_path): @@ -518,7 +514,7 @@ def test_download_remote_data_requires_remote_url(tmp_path): repo = _repo(tmp_path, {"remote": {"name": "origin"}}) with pytest.raises(DownloadError, match="Remote URL not configured"): - download_remote_data(repo, tmp_path, selected_files=[]) + download_remote_data(repo, tmp_path, selected_files=[], approved=True) def test_download_remote_data_returns_empty_result_for_empty_selection(tmp_path): @@ -531,7 +527,7 @@ def test_download_remote_data_returns_empty_result_for_empty_selection(tmp_path) repo = _repo( tmp_path, {"remote": {"name": "origin", "url": "https://example.test/data"}}) - result = download_remote_data(repo, tmp_path, selected_files=[]) + result = download_remote_data(repo, tmp_path, selected_files=[], approved=True) assert result["succeeded"] == result["failed"] == result["total_bytes"] == 0, \ f"Expected all counts to be 0, but got {result}" @@ -556,7 +552,8 @@ def test_download_remote_data_aggregates_successes_and_failures(monkeypatch, tmp {"name": "origin", "url": "https://origin.test/data"}, {"name": "mirror", "url": "https://mirror.test/base/"}]}) calls = [] - def fake_download(url, destination, sha1): + def fake_download(context, relative_path, destination, sha1, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """A fake _download_file function that records its arguments and simulates a failure for a specific file""" calls.append((url, destination, sha1)) @@ -575,7 +572,7 @@ def update(self, amount): def close(self): self.closed = True progress = Progress() - monkeypatch.setattr("hallmark.downloader._download_file", fake_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download) monkeypatch.setattr("hallmark.downloader.tqdm", lambda **kwargs: progress) result = download_remote_data( repo, @@ -585,7 +582,7 @@ def close(self): selected_files=[ (Path("nested/good.bin"), "good-sha"), (Path("bad.bin"), None)], - remote_name="mirror") + remote_name="mirror", approved=True) assert result == { "succeeded": 1, @@ -624,7 +621,7 @@ def test_download_remote_data_revalidates_selected_paths(tmp_path): download_remote_data( repo, tmp_path, - selected_files=[(Path("../escape.bin"), None)]) + selected_files=[(Path("../escape.bin"), None)], approved=True) def test_download_remote_data_rejects_symlink_parent_escape(tmp_path): """ @@ -657,7 +654,7 @@ def test_download_remote_data_rejects_symlink_parent_escape(tmp_path): download_remote_data( repo, output_root, - selected_files=[(Path("linked/file.bin"), None)]) + selected_files=[(Path("linked/file.bin"), None)], approved=True) assert not (outside_root / "file.bin").exists(), \ "Expected no file to be created outside the download root, but it exists" @@ -694,7 +691,7 @@ def test_download_remote_data_rejects_symlink_destination(tmp_path): download_remote_data( repo, output_root, - selected_files=[(Path("file.bin"), None)]) + selected_files=[(Path("file.bin"), None)], approved=True) assert outside_file.read_bytes() == b"do not overwrite", \ "Expected the outside file to remain unchanged, but it was modified" @@ -714,19 +711,20 @@ def test_download_remote_data_deduplicates_selected_paths(monkeypatch, tmp_path) "name": "origin", "url": "https://example.test/data"}}) calls = [] - def fake_download(url, destination, checksum): + def fake_download(context, relative_path, destination, checksum, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """A fake _download_file function that records its arguments and simulates a download.""" calls.append((url, destination, checksum)) return 4 - monkeypatch.setattr("hallmark.downloader._download_file", fake_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download) result = download_remote_data( repo, tmp_path, selected_files=[ (Path("file.bin"), None), (Path("file.bin"), "abc123"), - (Path("file.bin"), "abc123")]) + (Path("file.bin"), "abc123")], approved=True) assert result["succeeded"] == 1, \ f"Expected succeeded to be 1, but got {result['succeeded']}" @@ -753,7 +751,8 @@ def test_download_remote_data_rejects_conflicting_checksums(tmp_path): download_remote_data( repo, tmp_path, - selected_files=[(Path("file.bin"), "first"), (Path("file.bin"), "second")]) + selected_files=[(Path("file.bin"), "first"), (Path("file.bin"), "second")], + approved=True) @pytest.mark.parametrize("max_workers", [0, -1, 1.5, True, None]) @@ -771,7 +770,7 @@ def test_download_remote_data_rejects_invalid_worker_count(max_workers, tmp_path with pytest.raises( DownloadError, match="max_workers must be a positive integer"): - download_remote_data(repo, tmp_path, max_workers=max_workers) + download_remote_data(repo, tmp_path, max_workers=max_workers, approved=True) def test_download_remote_data_rejects_selected_files_without_remote(tmp_path): @@ -787,7 +786,8 @@ def test_download_remote_data_rejects_selected_files_without_remote(tmp_path): repo = _repo(tmp_path) with pytest.raises(DownloadError, match="No remote is configured"): - download_remote_data(repo, tmp_path, selected_files=[(Path("data.bin"), None)]) + download_remote_data(repo, tmp_path, + selected_files=[(Path("data.bin"), None)], approved=True) def test_download_remote_data_reuses_session_per_worker(monkeypatch, tmp_path): @@ -806,6 +806,8 @@ class FakeSession: def __init__(self): self.urls = [] sessions.append(self) + def close(self): + self.closed = True def get(self, url, **kwargs): self.urls.append(url) return _Response([b"contents"]) @@ -818,10 +820,12 @@ def get(self, url, **kwargs): repo, tmp_path, selected_files=[ - (Path("first.dat"), None), (Path("second.dat"), None)], max_workers=1) + (Path("first.dat"), None), (Path("second.dat"), None)], + max_workers=1, approved=True) assert result["succeeded"] == 2, \ f"Expected 2 successful downloads, but got {result['succeeded']}" + assert sessions[0].closed assert len(sessions) == 1, f"Expected a single session to be reused for both \ downloads, but got {len(sessions)}" assert sessions[0].urls == [ @@ -846,7 +850,7 @@ def test_download_remote_data_rejects_malformed_selection(tmp_path): with pytest.raises(DownloadError, match="path, checksum"): download_remote_data( - repo, tmp_path, selected_files=[("file.dat", None, "extra")]) + repo, tmp_path, selected_files=[("file.dat", None, "extra")], approved=True) def test_download_remote_data_rejects_malformed_checksum_tuple(tmp_path): @@ -866,7 +870,8 @@ def test_download_remote_data_rejects_malformed_checksum_tuple(tmp_path): download_remote_data( repo, tmp_path, - selected_files=[(Path("file.dat"), ("sha1", "abc123", "extra"))]) + selected_files=[(Path("file.dat"), ("sha1", "abc123", "extra"))], + approved=True) def test_download_remote_data_rejects_output_path_that_is_file(tmp_path): @@ -884,7 +889,7 @@ def test_download_remote_data_rejects_output_path_that_is_file(tmp_path): download_remote_data( repo, output_path, - selected_files=[(Path("file.bin"), None)]) + selected_files=[(Path("file.bin"), None)], approved=True) def test_download_remote_data_deduplicates_equivalent_checksum_tuples( @@ -902,11 +907,12 @@ def test_download_remote_data_deduplicates_equivalent_checksum_tuples( "name": "origin", "url": "https://example.test/data"}}) calls = [] - def fake_download(url, destination, checksum): + def fake_download(context, relative_path, destination, checksum, chunk_size): + url = context.remote.file_url(relative_path.as_posix()) """Record download requests and return a fixed byte count.""" calls.append((url, destination, checksum)) return 11 - monkeypatch.setattr("hallmark.downloader._download_file", fake_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download) checksum_upper = ("SHA256", "A" * 64) checksum_lower = ("sha256", "a" * 64) result = download_remote_data( @@ -914,7 +920,7 @@ def fake_download(url, destination, checksum): tmp_path, selected_files=[ (Path("file.bin"), checksum_upper), - (Path("file.bin"), checksum_lower)]) + (Path("file.bin"), checksum_lower)], approved=True) assert result["succeeded"] == 1, \ f"Expected succeeded to be 1, but got {result['succeeded']}" @@ -1491,4 +1497,4 @@ def test_entry_checksum_prefers_strongest_named_checksum(): entry = {"md5": "a" * 32, "sha1": "b" * 40, "sha256": "c" * 64, "sha512": "d" * 128} assert _entry_checksum(entry) == ("sha512", "d" * 128), \ - f"Expected strongest checksum to be sha512, but got {_entry_checksum(entry)}" \ No newline at end of file + f"Expected strongest checksum to be sha512, but got {_entry_checksum(entry)}" diff --git a/test/test_hallmark.py b/test/test_hallmark.py index 524d2aa..bad452c 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -353,7 +353,7 @@ def test_repo_add_pattern_keeps_deleted_manifest_rows(tmp_path): ] -def test_repo_add_pattern_replaces_manifest_when_fmt_changes(tmp_path): +def test_repo_add_new_pattern_appends_a_template(tmp_path): repo = Repo.init(tmp_path / "repo") _write_files(repo.worktree, ["a0.4_i30_w3.h5", "b0.4_i30_w3.h5"]) @@ -362,8 +362,18 @@ def test_repo_add_pattern_replaces_manifest_when_fmt_changes(tmp_path): repo.checkout("experiment") repo.add("b{a}_i{i}_w{w}.h5") - assert repo.state.config["data"] == [{"fmt": "b{a}_i{i}_w{w}.h5", "encoding": None}] + assert repo.state.config["data"] == [ + {"fmt": "a{a}_i{i}_w{w}.h5", "encoding": None}, + {"fmt": "b{a}_i{i}_w{w}.h5", "db": "data-2.tsv"}] assert repo.state.data.to_dict(orient="records") == [ + { + "sha1": Repo.checksum(repo.worktree / "a0.4_i30_w3.h5"), + "a": "0.4", + "i": "30", + "w": "3", + } + ] + assert repo.state.table("data-2.tsv").to_dict(orient="records") == [ { "sha1": Repo.checksum(repo.worktree / "b0.4_i30_w3.h5"), "a": "0.4", @@ -755,6 +765,7 @@ def test_repo_status_reports_staged_worktree_and_untracked_changes(tmp_path): "added": [], "modified": [], "deleted": [], + "catalog": [], } assert snapshot["worktree"]["modified"] == ["a0_i0.h5"] assert snapshot["worktree"]["deleted"] == ["a0_i30.h5"] @@ -939,6 +950,7 @@ def test_checkout_rebuilds_worktree_for_branch_specific_nested_fmt(tmp_path): repo.commit("main data") repo.checkout("experiment") + repo.rm_cached("main/a{a}_i{i}.h5") (repo.worktree / "exp" / "run1").mkdir(parents=True) _write_files(repo.worktree, ["exp/run1/b0_i0.h5"]) repo.add("exp/run{run}/b{a}_i{i}.h5") @@ -1028,6 +1040,7 @@ def test_checkout_rejects_symlink_destination_escape(tmp_path): repo.add("main/a{a}_i{i}.h5") repo.commit("main data") repo.checkout("experiment") + repo.rm_cached("main/a{a}_i{i}.h5") (repo.worktree / "main/a0_i0.h5").unlink() (repo.worktree / "exp").mkdir() _write_files(repo.worktree, ["exp/a1_i45.h5"]) @@ -1163,6 +1176,7 @@ def test_checkout_rejects_directory_at_target_file_path(tmp_path): repo.add("data_{number}.txt") repo.commit("main data") repo.checkout("experiment") + repo.rm_cached("data_{number}.txt") data_path.unlink() experiment_path = (repo.worktree / "experiment_1.txt") experiment_path.write_text("experiment\n", encoding="utf-8") @@ -1277,7 +1291,7 @@ def record_restore(sha1, destination): ### Repo.clone() tests ### -def test_repo_clone_downloads_remote_data_by_default(monkeypatch, tmp_path): +def test_repo_clone_downloads_after_plan_approval(monkeypatch, tmp_path): source = Repo.init(tmp_path / "source") _write_files(source.worktree, ["a0_i0.h5"]) source.add("a{a}_i{i}.h5") @@ -1287,16 +1301,19 @@ def test_repo_clone_downloads_remote_data_by_default(monkeypatch, tmp_path): captured = {} - def fake_download_file(url, destination, sha1, chunk_size=8192): - captured["url"] = url + def fake_download_file(context, relative_path, destination, sha1, chunk_size): + captured["url"] = context.remote.file_url(str(relative_path)) captured["sha1"] = sha1 destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text("downloaded\n", encoding="utf-8") return destination.stat().st_size - monkeypatch.setattr("hallmark.downloader._download_file", fake_download_file) + monkeypatch.setattr("hallmark.downloader._fetch_file", fake_download_file) clone = Repo.clone(str(source.dothm.path), tmp_path / "clone") + plan = clone.plan_download() + assert plan.file_count == 1 + clone.download(plan, approved=True) assert captured == { "url": "https://example.com/data/a0_i0.h5", @@ -1318,9 +1335,9 @@ def test_repo_clone_can_skip_remote_data_download(monkeypatch, tmp_path): def fail_download(*args, **kwargs): raise AssertionError("download should not be attempted") - monkeypatch.setattr("hallmark.downloader._download_file", fail_download) + monkeypatch.setattr("hallmark.downloader._fetch_file", fail_download) - clone = Repo.clone(str(source.dothm.path), tmp_path / "clone", fetch_data=False) + clone = Repo.clone(str(source.dothm.path), tmp_path / "clone") assert not (clone.worktree / "a0_i0.h5").exists() assert clone.download_result is None @@ -1346,7 +1363,7 @@ def test_repo_clone_removes_incomplete_destination(tmp_path): destination = tmp_path / "clone" with pytest.raises(CloneError, match="missing required file"): - Repo.clone(str(source.dothm.path), destination, fetch_data=False) + Repo.clone(str(source.dothm.path), destination) assert not destination.exists(), f"Expected incomplete clone destination \ {destination} to be removed after failed clone attempt" @@ -2231,16 +2248,15 @@ def test_add_worktree_rejects_invalid_data_config_before_creation(tmp_path): Args: tmp_path: pytest fixture that provides a temporary directory for the test. Raises: - RuntimeError: If the repository's data configuration is invalid - (not exactly one entry). + ValueError: If the repository's data configuration is malformed. """ repo = Repo.init(tmp_path / "repo") - repo.state.config["data"] = [] + repo.state.config["data"] = "not a list of entries" repo.dothm.dump(repo.state) repo.dothm.index.commit("invalid data configuration") destination = tmp_path / "experiment" - with pytest.raises(RuntimeError, match="requires exactly one data entry"): + with pytest.raises(ValueError, match="must be a mapping or list of mappings"): repo.add_worktree("experiment") assert not destination.exists(), \ f"Expected destination {destination} to not exist, but it does." @@ -2769,7 +2785,6 @@ def test_checkout_remote_branch(tmp_path): clone = Repo.clone( str(source.dothm.path), tmp_path / "clone", - fetch_data=False, ) # The cloned repository should have the tracked file in its worktree. @@ -2846,4 +2861,4 @@ def test_checkout_remote_branch(tmp_path): # The worktree should now contain the version from the remote branch. assert (clone.worktree / "data.txt").read_text( encoding="utf-8" - ) == "experiment contents\n" \ No newline at end of file + ) == "experiment contents\n" diff --git a/test/test_interactive_download.py b/test/test_interactive_download.py new file mode 100644 index 0000000..d8a1879 --- /dev/null +++ b/test/test_interactive_download.py @@ -0,0 +1,600 @@ +"""Interactive downloads and default clone review after metadata creation.""" + +from contextlib import contextmanager +import importlib +from pathlib import Path +import shlex +import sys +from unittest.mock import patch +from urllib.parse import quote + +from click import termui +from click.testing import CliRunner +import pandas as pd +import pytest + +from hallmark import Repo +from hallmark.cli import hallmark +from hallmark.transport import OperationContext +from mock_server import MockServer + +cli = importlib.import_module("hallmark.cli") +ROOT = "https://example.test/data/" +README = "README with space,comma.md" + + +class TerminalRunner(CliRunner): + """Model a real terminal, including EOF on Click 8.1 / Python 3.9.""" + + @contextmanager + def isolation(self, *args, **kwargs): + with super().isolation(*args, **kwargs) as streams: + with patch.object(sys.stdin, "isatty", return_value=True), patch.object( + termui, "visible_prompt_func", input): + yield streams + + +@pytest.fixture +def catalog(tmp_path, monkeypatch): + repo = Repo.init(tmp_path / "source catalog") + repo.state.config = { + "data": [{"db": "data.tsv", "extraction": { + "automatic": False, "formats": ["runs/run_{run:03d}.h5"]}}], + "remote": [{"name": "origin", "url": ROOT}]} + repo.state.data = pd.DataFrame([ + {"path": "runs/run_001.h5", "size_bytes": 4, "run": "001"}, + {"path": "runs/run_002.h5", "size_bytes": None, "run": "002"}, + {"path": README, "size_bytes": 5, "run": ""}, + ]) + repo.dothm.dump(repo.state) + repo.dothm.index.add(["config.yml", "data.tsv"]) + repo.dothm.index.commit("Catalog fixture") + server = MockServer(ROOT) + for name in ("runs/run_001.h5", "runs/run_002.h5", README, "omitted.bin"): + server.add_file(quote(name, safe="/"), b"notes" if name == README else b"data") + transfers = [] + original = server.get + + def get(url, **kwargs): + transfers.append(url) + return original(url, **kwargs) + + server.get = get + monkeypatch.setattr("requests.Session", lambda: server) + monkeypatch.chdir(repo.worktree) + return repo, transfers + + +def metadata(repo): + return {name: (repo.dothm.path / name).read_bytes() + for name in ("data.tsv", "config.yml", "meta.yml")} + + +@pytest.mark.parametrize("answer", ["\n", "skip\n", "", "patterns\n", "all\n", + "all\n\n", "all\nskip\n"]) +def test_skip_and_eof_never_transfer(catalog, answer): + repo, transfers = catalog + before = metadata(repo) + result = TerminalRunner().invoke(hallmark, ["download", "--interactive"], + input=answer) + assert result.exit_code == 0, result.output + assert "No download started" in result.output + assert transfers == [] + assert metadata(Repo(repo.worktree)) == before + + +def test_patterns_are_raw_ored_and_approval_executes_displayed_plan( + catalog, monkeypatch): + repo, transfers = catalog + before = metadata(repo) + shown, executed = [], [] + display, execute = cli._show_download_plan, Repo.download + + def show(plan, **kwargs): + assert transfers == [] + shown.append(plan) + display(plan, **kwargs) + + def download(self, plan, **kwargs): + assert kwargs == {"approved": True, "max_workers": 2, "progress": True} + assert plan is shown[-1] + executed.append(plan) + return execute(self, plan, **kwargs) + + monkeypatch.setattr(cli, "_show_download_plan", show) + monkeypatch.setattr(Repo, "download", download) + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--max-workers", "2"], + input=f"patterns\nruns/*001.h5\n{README}\n\ndownload\n") + assert result.exit_code == 0, result.output + assert "2 file(s); 9 bytes" in result.output + assert f"{README} (5 bytes)" in result.output + assert "Files with unknown size: 0" in result.output + assert len(transfers) == 2 + assert len(executed) == 1 + assert (repo.worktree / README).read_bytes() == b"notes" + assert not (repo.worktree / "runs/run_002.h5").exists() + assert metadata(Repo(repo.worktree)) == before + + +def test_revision_does_not_approve_previous_selection(catalog): + repo, transfers = catalog + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive"], + input="all\nchange\npatterns\nruns/*001.h5\n\ndownload\n") + assert result.exit_code == 0, result.output + assert "3 file(s)" in result.output and "1 file(s); 4 bytes" in result.output + assert "1 file(s) with unknown size" in result.output + assert "runs/run_002.h5 (size unknown)" in result.output + assert transfers == [ROOT + "runs/run_001.h5"] + assert not (repo.worktree / README).exists() + + +def test_empty_patterns_and_no_matches_return_to_selection(catalog): + _, transfers = catalog + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive"], + input="patterns\n\npatterns\nomitted*\n\nall\nskip\n") + assert result.exit_code == 0, result.output + assert "No cataloged files match" in result.output + assert "3 file(s)" in result.output + assert transfers == [] + + +def test_interactive_dry_run_and_remote_override(catalog, monkeypatch, tmp_path): + repo, transfers = catalog + repo.set_config(remote_name="mirror", remote_url="https://mirror.test/") + destination = tmp_path / "uncreated downloads" + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--dry-run", "--remote", "mirror", + "--output", str(destination)], input="all\n") + assert result.exit_code == 0, result.output + assert "https://mirror.test/" in result.output + assert f"Destination: {destination}" in result.output + assert "Next action" not in result.output + assert not destination.exists() + assert transfers == [] + + +def test_preview_truncates_paths_but_not_totals(catalog): + repo, transfers = catalog + repo.state.data = pd.DataFrame([ + {"path": f"file-{number:03d}.dat", "size_bytes": number} + for number in range(25)]) + repo.dothm.dump(repo.state) + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--dry-run"], input="all\n") + assert result.exit_code == 0, result.output + assert "25 file(s); 300 bytes" in result.output + assert "file-019.dat (19 bytes)" in result.output + assert "file-020.dat" not in result.output + assert "... 5 more file(s)" in result.output + assert transfers == [] + + +@pytest.mark.parametrize("selectors", [["runs/run_001.h5"], ["--all"], + ["--include", "*.h5"], ["--tsv", "data.tsv"]]) +def test_interactive_rejects_explicit_selectors(catalog, selectors): + _, transfers = catalog + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", *selectors]) + assert result.exit_code != 0 + assert "--interactive cannot be combined" in result.output + assert transfers == [] + + +def test_standalone_interactive_requires_terminal(catalog): + _, transfers = catalog + result = CliRunner().invoke(hallmark, ["download", "--interactive"]) + assert result.exit_code != 0 + assert "requires a terminal" in result.output + assert "--dry-run" in result.output + assert "Select files" not in result.output + assert transfers == [] + + +@pytest.mark.parametrize("kind", ["empty", "no-remote"]) +def test_unavailable_download_skips_without_prompt(catalog, kind): + repo, transfers = catalog + if kind == "empty": + repo.state.data = repo.state.data.iloc[:0] + else: + repo.state.config.pop("remote") + repo.dothm.dump(repo.state) + result = TerminalRunner().invoke(hallmark, ["download", "--interactive"]) + assert result.exit_code == 0, result.output + message = "catalog is empty" if kind == "empty" else "No data remote" + assert message in result.output + assert "Select files" not in result.output + assert transfers == [] + + +def init_pages(monkeypatch): + pages = { + ROOT: '

Index of data

runs/' + 'README.mdomitted.bin', + ROOT + "runs/": '

Index of runs

' + 'run_001.h5' + 'run_002.h5', + } + monkeypatch.setattr(OperationContext, "read_text", + lambda context, path: pages[context.remote.url + path]) + + +def test_bare_catalog_prompts_validates_output_without_creating_it( + catalog, tmp_path, monkeypatch): + source, transfers = catalog + repo = Repo.clone(str(source.dothm.path), tmp_path / "bare.hm") + monkeypatch.chdir(repo.dothm.path) + not_directory = tmp_path / "file" + not_directory.write_text("keep") + destination = tmp_path / "download output" + result = TerminalRunner().invoke( + hallmark, ["download", "--interactive", "--dry-run"], + input=f"all\n{not_directory}\n{destination}\n") + assert result.exit_code == 0, result.output + assert "is a file" in result.output + assert f"Destination: {destination}" in result.output + assert not destination.exists() + assert not_directory.read_text() == "keep" + assert transfers == [] + + +@pytest.mark.parametrize("stage", ["selection", "patterns", "output", "approval"]) +def test_interrupt_is_not_a_successful_skip(catalog, tmp_path, monkeypatch, stage): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if stage == "output" else "copy") + responses = iter({"selection": [], "patterns": ["patterns"], + "output": ["all"], "approval": ["all"]}[stage]) + choose = cli._choose_download_plan + + def interrupted(prompt): + try: + return next(responses) + except StopIteration: + raise KeyboardInterrupt() from None + + def interruptible_choice(repo, **kwargs): + with patch.object(termui, "visible_prompt_func", interrupted): + return choose(repo, **kwargs) + + monkeypatch.setattr(cli, "_choose_download_plan", interruptible_choice) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), + "--interactive"]) + assert result.exit_code != 0 + assert "Aborted" in result.output + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +def test_all_download_requires_approval_even_with_legacy_yes(catalog): + repo, transfers = catalog + result = TerminalRunner().invoke(hallmark, ["download", "--interactive", "--yes"], + input="all\ndownload\n") + assert result.exit_code == 0, result.output + assert "still require confirmation" in result.output + assert "Next action" in result.output + assert len(transfers) == 3 + assert all((repo.worktree / name).exists() + for name in ("runs/run_001.h5", "runs/run_002.h5", README)) + + +def test_transfer_interrupt_is_not_caught_as_prompt_eof(catalog, monkeypatch): + repo, transfers = catalog + before = metadata(repo) + + def interrupt(self, plan, **kwargs): + raise KeyboardInterrupt() + + monkeypatch.setattr(Repo, "download", interrupt) + result = TerminalRunner().invoke(hallmark, ["download", "--interactive"], + input="all\ndownload\n") + assert result.exit_code != 0 + assert "Aborted" in result.output + assert "No download started" not in result.output + assert metadata(Repo(repo.worktree)) == before + assert transfers == [] + + +@pytest.mark.parametrize("bare", [False, True]) +@pytest.mark.parametrize("options", [[], ["--interactive"], + ["--include", "runs/*001.h5", "--include", README], + ["--output", "../output 'files'"]]) +def test_noninteractive_clone_keeps_catalog_and_prints_equivalent_commands( + catalog, tmp_path, bare, options): + source, transfers = catalog + destination = tmp_path / ("new 'catalog'.hm" if bare else "new 'catalog'") + cwd = Path.cwd() + result = CliRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), *options], + input="y\n") + assert result.exit_code == 0, result.output + target = Repo(destination) + assert "download was skipped" in result.output + assert "Download these files?" not in result.output + location = target.dothm.path if bare else target.worktree + assert "cd -- " + shlex.quote(str(location)) in result.output + commands = [line.strip() for line in result.output.splitlines() + if line.startswith(" hallmark download")] + assert len(commands) == 2 + preview, execute = map(shlex.split, commands) + assert preview == execute + ["--dry-run"] + if "--include" in options: + assert execute[2:6] == options + else: + assert execute[2] == "--all" + if "--output" in options: + assert execute[-2:] == ["--output", str((cwd / options[1]).resolve())] + elif bare: + assert execute[-2:] == ["--output", "/path/to/downloads"] + else: + assert "--output" not in execute + assert "explicit approval is required" in result.output + assert Path.cwd() == cwd + assert transfers == [] + assert metadata(target) == metadata(source) + assert target.dothm.head.commit.hexsha == source.dothm.head.commit.hexsha + + +@pytest.mark.parametrize("options,input_text,expected", [ + ([], "y\n", ["runs/run_001.h5", "runs/run_002.h5", README]), + ([], "n\n", []), + ([], "\n", []), + ([], "", []), + (["--include", "runs/*001.h5", "--include", README], "y\n", + ["runs/run_001.h5", README]), + (["--include", "omitted*"], "", []), + (["--interactive"], "patterns\nruns/*001.h5\n\ndownload\n", ["runs/run_001.h5"]), + (["--interactive"], "skip\n", []), +]) +def test_clone_reviews_only_download_selection_after_complete_copy( + catalog, tmp_path, monkeypatch, options, input_text, expected): + source, transfers = catalog + destination = tmp_path / "new catalog" + before = metadata(source) + shown, executed = [], [] + show, download = cli._show_download_plan, Repo.download + + def show_after_creation(plan, **kwargs): + assert transfers == [] + assert metadata(Repo(destination)) == before + assert (Repo(destination).dothm.head.commit.hexsha + == source.dothm.head.commit.hexsha) + assert plan.output_path == destination + shown.append(plan) + show(plan, **kwargs) + + def execute(self, plan, **kwargs): + assert plan is shown[-1] + assert kwargs == {"approved": True, "max_workers": 4, "progress": True} + executed.append(plan) + return download(self, plan, **kwargs) + + monkeypatch.setattr(cli, "_show_download_plan", show_after_creation) + monkeypatch.setattr(Repo, "download", execute) + initial_cwd = Path.cwd() + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), *options], + input=input_text) + assert result.exit_code == 0, result.output + assert result.output.index("Successfully") < result.output.index("Catalog ready") + if "--interactive" not in options: + assert "Select files" not in result.output + assert metadata(Repo(destination)) == before + assert Path.cwd() == initial_cwd + assert set(transfers) == {ROOT + quote(name, safe="/") for name in expected} + assert bool(executed) == bool(expected) + assert Repo(destination).plan_download().file_count == 3 + assert not (destination / "omitted.bin").exists() + if not options: + assert "3 file(s); 9 bytes known; 1 file(s) with unknown size" in result.output + assert "runs/run_002.h5 (size unknown)" in result.output + + +@pytest.mark.parametrize("bare", [False, True]) +def test_no_download_disables_review_and_planning(catalog, tmp_path, monkeypatch, bare): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if bare else "copy") + + def unexpected(*args, **kwargs): + raise AssertionError("No download planning or review was requested") + + monkeypatch.setattr(cli, "_offer_download", unexpected) + monkeypatch.setattr(Repo, "plan_download", unexpected) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination), "--no-download"]) + assert result.exit_code == 0, result.output + assert "Download these files?" not in result.output + assert "download was skipped" not in result.output + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +@pytest.mark.parametrize("options", [ + ["--no-download", "--interactive"], ["--no-download", "--include", "*.fits"], + ["--no-download", "--output", "output"], ["--interactive", "--include", "*.fits"], + ["--with-download"], +]) +def test_clone_invalid_options_fail_before_source_access( + tmp_path, monkeypatch, options): + def unexpected(*args, **kwargs): + raise AssertionError("Must reject options before cloning") + + monkeypatch.setattr(Repo, "clone", unexpected) + destination = tmp_path / "new" + result = TerminalRunner().invoke(hallmark, ["clone", "source", str(destination), + *options]) + assert result.exit_code != 0 + assert "Error:" in result.output + assert not isinstance(result.exception, AssertionError) + assert not destination.exists() + + +@pytest.mark.parametrize("failure", ["setup", "partial", "interrupt"]) +def test_clone_download_failure_preserves_catalog( + catalog, tmp_path, monkeypatch, failure): + source, _ = catalog + destination = tmp_path / "created" + + def fail(self, plan, **kwargs): + if failure == "setup": + raise cli.DownloadError("fixture access denied") + if failure == "interrupt": + raise KeyboardInterrupt() + return {"succeeded": 1, "failed": 1, "total_bytes": 4, + "errors": ["fixture error"]} + + monkeypatch.setattr(Repo, "download", fail) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination)], input="y\n") + assert result.exit_code != 0 + assert "Successfully cloned" in result.output + assert "Failed to clone" not in result.output + assert "No download started" not in result.output + assert metadata(Repo(destination)) == metadata(source) + assert Repo(destination).dothm.head.commit.hexsha == source.dothm.head.commit.hexsha + + +@pytest.mark.parametrize("provided", [False, True]) +@pytest.mark.parametrize("bare", [False, True]) +def test_clone_output_and_bare_prompt(catalog, tmp_path, bare, provided): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if bare else "copy") + output = tmp_path / "downloads" + arguments = ["clone", str(source.dothm.path), str(destination), + "--include", "runs/*001.h5"] + if provided: + arguments += ["--output", str(output)] + answer = (str(output) + "\n" if bare and not provided else "") + "y\n" + result = TerminalRunner().invoke(hallmark, arguments, input=answer) + assert result.exit_code == 0, result.output + assert ("Output directory" in result.output) == (bare and not provided) + target = output if bare or provided else destination + assert (target / "runs/run_001.h5").read_bytes() == b"data" + assert transfers == [ROOT + "runs/run_001.h5"] + assert metadata(Repo(destination)) == metadata(source) + + +@pytest.mark.parametrize("interrupt", [False, True]) +@pytest.mark.parametrize("bare", [False, True]) +def test_clone_eof_or_interrupt_before_transfer(catalog, tmp_path, monkeypatch, + bare, interrupt): + source, transfers = catalog + destination = tmp_path / ("copy.hm" if bare else "copy") + if interrupt: + def stop(*args, **kwargs): + raise KeyboardInterrupt() + # Default review uses click.confirm for approval and click.prompt for output. + monkeypatch.setattr(cli, "_confirm_download_plan", stop) + if bare: + monkeypatch.setattr(cli, "_download_output", stop) + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination)], input="") + assert (result.exit_code != 0) == interrupt + assert ("No download started" in result.output) != interrupt + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +@pytest.mark.parametrize("kind", ["empty", "no-remote"]) +def test_clone_without_downloadable_data_completes(catalog, tmp_path, kind): + source, transfers = catalog + if kind == "empty": + source.state.data = source.state.data.iloc[:0] + else: + source.state.config.pop("remote") + source.dothm.dump(source.state) + source.dothm.index.add(["config.yml", "data.tsv"]) + source.dothm.index.commit("Unavailable download") + destination = tmp_path / "copy" + result = TerminalRunner().invoke( + hallmark, ["clone", str(source.dothm.path), str(destination)]) + assert result.exit_code == 0, result.output + message = "catalog is empty" if kind == "empty" else "No data remote" + assert message in result.output + assert "Download these files?" not in result.output + assert metadata(Repo(destination)) == metadata(source) + assert transfers == [] + + +@pytest.mark.parametrize("remote", [False, True]) +def test_init_and_remote_add_never_scan_local_files_or_offer_downloads( + catalog, tmp_path, monkeypatch, remote): + _, transfers = catalog + destination = tmp_path / "initialized" + destination.mkdir() + local = destination / "existing.h5" + local.write_bytes(b"untouched") + if not remote: + def no_remote_access(*args, **kwargs): + raise AssertionError("Local init must not contact a remote") + monkeypatch.setattr(OperationContext, "__enter__", no_remote_access) + + def no_download(*args, **kwargs): + raise AssertionError("Init and add must not offer, plan, or execute downloads") + + monkeypatch.setattr(cli, "_offer_download", no_download) + monkeypatch.setattr(Repo, "plan_download", no_download) + monkeypatch.setattr(Repo, "download", no_download) + result = TerminalRunner().invoke(hallmark, ["init", str(destination)]) + assert result.exit_code == 0, result.output + if remote: + init_pages(monkeypatch) + monkeypatch.chdir(destination) + result = TerminalRunner().invoke( + hallmark, ["add", ROOT + "{group}/run_{run:03d}.h5"]) + assert result.exit_code == 0, result.output + repo = Repo(destination) + assert local.read_bytes() == b"untouched" + assert transfers == [] + assert "Download these files?" not in result.output + assert "Select files" not in result.output + if remote: + assert "then hallmark download" in result.output + assert repo.state.data[["group", "run"]].values.tolist() == [ + ["runs", "001"], ["runs", "002"]] + else: + assert repo.state.data.empty + assert not list((repo.dothm.path / "objects").rglob("*")) + + +def test_failed_clone_never_reviews_download(catalog, monkeypatch): + repo, transfers = catalog + + def unexpected(*args, **kwargs): + raise AssertionError("No handoff before successful creation") + + monkeypatch.setattr(cli, "_offer_download", unexpected) + result = TerminalRunner().invoke(hallmark, ["clone", str(repo.dothm.path), + str(repo.worktree)]) + assert result.exit_code != 0 + assert not isinstance(result.exception, AssertionError) + assert transfers == [] + + +def test_snapshot_clone_reviews_downloads_from_saved_remote( + catalog, tmp_path, monkeypatch): + source, transfers = catalog + snapshot = metadata(source) + snapshot_url = "https://example.test/published/" + reads = [] + + def read(context, path): + assert context.remote.url == snapshot_url + reads.append(path) + return snapshot[path].decode() + + monkeypatch.setattr(OperationContext, "read_text", read) + destination = tmp_path / "snapshot" + result = TerminalRunner().invoke( + hallmark, ["clone", snapshot_url, str(destination), "--source-type", "catalog", + "--include", "runs/*001.h5"], input="y\n") + assert result.exit_code == 0, result.output + assert set(reads) == {"config.yml", "meta.yml", "data.tsv"} + assert transfers == [ROOT + "runs/run_001.h5"] + target = Repo(destination) + assert metadata(target) == snapshot + assert target.plan_download().file_count == 3 + message = target.dothm.head.commit.message.strip() + assert message == "Import published catalog snapshot" diff --git a/test/test_remote_add.py b/test/test_remote_add.py new file mode 100644 index 0000000..0ca185f --- /dev/null +++ b/test/test_remote_add.py @@ -0,0 +1,246 @@ +"""Test cataloging remote files with URL templates in hallmark add.""" + +import hashlib + +import pytest +import requests +from click.testing import CliRunner + +from hallmark import Repo +from hallmark.cli import hallmark +from hallmark.discovery import TemplateMatcher, split_url_template +from hallmark.repo_config import catalog_entries +from hallmark.transport import OperationContext +from hallmark.transport.base import RemoteObjectMissing + +ROOT = "https://archive.test/2026MOVIE/ER2/" +FILES = {"M87_095.h5": b"m87-095", "M87_096.h5": b"m87-096", + "SGRA_095.h5": b"sgra-095"} + + +def _index(entries): + """Render an Apache-style index with sizes for files and directories.""" + links = "".join( + f'{name}' if name.endswith("/") + else f'{name}' + for name, size in entries) + return f"

Index of listing

{links}" + + +@pytest.fixture +def archive(monkeypatch): + """Serve an archive listing with a checksum manifest; record listing reads.""" + pages, reads = {}, [] + + def read_text(context, path): + url = context.remote.url.rstrip("/") + "/" + path + reads.append(url) + if url not in pages: + raise RemoteObjectMissing("No such metadata object") + return pages[url] + + monkeypatch.setattr(OperationContext, "read_text", read_text) + sums = "".join(f"{hashlib.md5(data).hexdigest()} {name}\n" + for name, data in FILES.items()) + pages[ROOT] = _index([*((name, len(data)) for name, data in FILES.items()), + ("md5sums.txt", len(sums)), ("README.md", 5), + ("calibration/", None), (".hidden_1.h5", 3)]) + pages[ROOT + "md5sums.txt"] = sums + pages[ROOT + "calibration/"] = _index([("M87_cal.h5", 4)]) + return pages, reads + + +@pytest.fixture +def repo(tmp_path): + return Repo.init(tmp_path / "repo") + + +def test_split_url_template_separates_directory_and_template(): + assert split_url_template(ROOT + "{src}_{day}.h5") == (ROOT, "{src}_{day}.h5") + assert split_url_template("https://h/a%20b/{run}/x%7B%25.dat") == ( + "https://h/a%20b/", "{run}/x{{%.dat") + assert split_url_template("ssh://campus/srv/export/{run}.h5") == ( + "ssh://campus/srv/export/", "{run}.h5") + assert split_url_template(ROOT) == (ROOT, None) + assert split_url_template(ROOT + "README.md") == (ROOT, "README.md") + + +@pytest.mark.parametrize("url, message", [ + ("https://user:secret@h/d/{a}.h5", "credentials"), + ("https://token@h/d/{a}.h5", "credentials"), + ("https://h/d/{a}.h5?sig=1", "query or fragment"), + ("https://h/d/{a}.h5#part", "query or fragment"), + ("https://{host}/d/{a}.h5", "only in the URL path"), + ("https://h/d/{a!r}.h5", "conversions"), + ("https://h/d/{a.h5", "Invalid filename template"), +]) +def test_split_url_template_rejects_unsafe_urls(url, message): + with pytest.raises(ValueError, match=message): + split_url_template(url) + + +def test_template_matcher_keeps_fields_within_segments(): + matcher = TemplateMatcher("{src}/{day}_[x].h5") + assert matcher.glob == "*/*_[[]x[]].h5" + assert matcher.match("M87/095_[x].h5") == {"src": "M87", "day": "095"} + assert matcher.match("M87/sub/095_[x].h5") is None + assert matcher.match(".cache/095_[x].h5") is None + assert matcher.may_contain("M87/") + assert not matcher.may_contain("M87/095/") + assert TemplateMatcher("{a}{b}.h5").glob == "*.h5" + with pytest.raises(ValueError, match="reserved"): + TemplateMatcher("{size_bytes}.h5") + with pytest.raises(ValueError, match="naming files"): + TemplateMatcher("{src}/") + + +def test_add_catalogs_matching_remote_files_without_payloads(repo, archive): + pages, reads = archive + result = repo.add(ROOT + "{src}_{day}.h5") + assert result["path"].tolist() == sorted(FILES) + assert repo.state.config["data"] == [{"fmt": "{src}_{day}.h5", "url": ROOT}] + rows = repo.state.data.to_dict("records") + assert rows[0] == { + "sha1": "", "checksum_algorithm": "md5", + "checksum": hashlib.md5(FILES["M87_095.h5"]).hexdigest(), + "size_bytes": "7", "mtime": "", "src": "M87", "day": "095"} + # only the root and the manifest were read; subdirectories were pruned + assert reads == [ROOT, ROOT + "md5sums.txt"] + assert not any((repo.worktree / name).exists() for name in FILES) + reopened = Repo(repo.worktree) + assert reopened.state.data.to_dict("records") == rows + + +def test_remote_catalog_commits_without_objects_and_status_stays_clean( + repo, archive): + repo.add(ROOT + "{src}_{day}.h5") + snapshot = repo.status() + assert snapshot["staged"]["catalog"] == [{ + "templates": ["{src}_{day}.h5"], "url": ROOT, + "added": 3, "modified": 0, "deleted": 0}] + assert repo.commit("Add remote EHT data") + assert not any(path.is_file() for path in repo.objects.root.rglob("*")) + snapshot = repo.status() + assert snapshot["staged"]["catalog"] == [] + assert snapshot["worktree"] == {"modified": [], "deleted": []} + assert snapshot["untracked"] == [] + + +def test_download_verifies_published_checksums(repo, archive, monkeypatch): + from mock_server import MockServer + + repo.add(ROOT + "{src}_{day}.h5") + repo.commit("Add remote EHT data") + server = MockServer(ROOT) + for name, data in FILES.items(): + server.add_file(name, data) + monkeypatch.setattr(requests, "Session", lambda: server) + plan = repo.plan_download() + assert plan.total_bytes == sum(map(len, FILES.values())) + result = repo.download(plan, approved=True) + assert result["succeeded"] == 3 and result["failed"] == 0 + assert (repo.worktree / "SGRA_095.h5").read_bytes() == b"sgra-095" + snapshot = repo.status() + assert snapshot["untracked"] == [] + assert snapshot["worktree"] == {"modified": [], "deleted": []} + + +def test_readding_a_remote_template_syncs_removals(repo, archive): + pages, _ = archive + repo.add(ROOT + "{src}_{day}.h5") + repo.commit("Add remote EHT data") + pages[ROOT] = _index([("M87_095.h5", 7), ("M87_097.h5", 9)]) + del pages[ROOT + "md5sums.txt"] + repo.add(ROOT + "{src}_{day}.h5") + assert repo.state.data["day"].tolist() == ["095", "097"] + assert repo.status()["staged"]["catalog"] == [{ + "templates": ["{src}_{day}.h5"], "url": ROOT, + "added": 1, "modified": 1, "deleted": 2}] + + +def test_no_match_is_an_error_and_changes_nothing(repo, archive): + before = (repo.dothm.path / "config.yml").read_text(encoding="utf-8") + with pytest.raises(ValueError, match="did not match any files"): + repo.add(ROOT + "{src}.fits") + assert (repo.dothm.path / "config.yml").read_text(encoding="utf-8") == before + + +def test_files_the_template_cannot_recreate_are_counted(repo, archive): + with pytest.raises(ValueError, match="3 listed file.s. fit the pattern"): + repo.add(ROOT + "{src}_{day:02d}.h5") + pages, _ = archive + pages[ROOT] = _index([("M87_7.h5", 1), ("M87_07.h5", 1)]) + result = repo.add(ROOT + "{src}_{run:02d}.h5") + assert result["path"].tolist() == ["M87_07.h5"] + assert result.attrs["mismatched"] == 1 + + +def test_template_segments_and_dry_run(repo, archive): + pages, reads = archive + pages[ROOT + "calibration/"] = _index([("M87_cal.h5", 4)]) + preview = repo.add(ROOT + "calibration/{src}_cal.h5", dry_run=True) + assert preview["path"].tolist() == ["M87_cal.h5"] + assert catalog_entries(repo.state.config) == [] + repo.add(ROOT + "{kind}/{src}_cal.h5") + assert repo.state.data.to_dict("records")[0]["kind"] == "calibration" + + +def test_conflicting_templates_are_rejected(repo, archive): + (repo.worktree / "local_1.h5").write_text("x\n", encoding="utf-8") + repo.add(ROOT + "{src}_{day}.h5") + with pytest.raises(ValueError, match="tracked by another template"): + repo.add(ROOT + "{name}.h5") + with pytest.raises(ValueError, match="already tracked from"): + repo.add("https://mirror.test/ER2/{src}_{day}.h5") + with pytest.raises(ValueError, match="local files only"): + repo.add(ROOT + "{src}_{day}.h5", encoding=True) + repo.add("local_{n}.h5") + with pytest.raises(ValueError, match="already tracked from the worktree"): + repo.add(ROOT + "local_{n}.h5") + + +def test_remote_and_local_templates_share_a_branch(repo, archive): + (repo.worktree / "runs").mkdir() + (repo.worktree / "runs/run_1.dat").write_text("run\n", encoding="utf-8") + repo.add(ROOT + "{src}_{day}.h5") + repo.add("runs/run_{run}.dat") + assert [(entry.db, entry.url) for entry in catalog_entries(repo.state.config)] \ + == [("data.tsv", ROOT), ("data-2.tsv", None)] + assert repo.commit("remote images and local runs") + assert len([path for path in repo.objects.root.rglob("*") + if path.is_file()]) == 1 + assert repo.add(".")["path"].tolist() == ["runs/run_1.dat"] + + +def test_directory_url_points_to_ls_remote(repo, archive): + with pytest.raises(ValueError, match="hallmark ls-remote"): + repo.add(ROOT) + + +def test_remote_add_works_in_a_bare_repository(tmp_path, archive): + bare = Repo.init(tmp_path / "catalog.hm") + bare.add(ROOT + "{src}_{day}.h5") + assert bare.commit("bare catalog") + assert bare.plan_download(tmp_path / "out").file_count == 3 + + +def test_explicit_backend_is_recorded_on_the_entry(repo, archive): + repo.add(ROOT + "{src}_{day}.h5", backend="cyverse") + assert repo.state.config["data"][0]["backend"] == "cyverse" + assert repo.plan_download().remote_backend == "cyverse" + + +def test_cli_add_url_summarizes_and_hides_credentials(repo, archive, monkeypatch): + monkeypatch.chdir(repo.worktree) + runner = CliRunner() + result = runner.invoke(hallmark, ["add", ROOT + "{src}_{day}.h5"]) + assert result.exit_code == 0, result.output + assert "Cataloged 3 remote file(s); nothing was downloaded." in result.output + assert "SGRA_095.h5" in result.output + secret = runner.invoke( + hallmark, ["add", "https://user:hunter2@archive.test/ER2/{a}.h5"]) + assert secret.exit_code != 0 + assert "hunter2" not in secret.output + directory = runner.invoke(hallmark, ["add", ROOT]) + assert directory.exit_code != 0 + assert "ls-remote" in directory.output diff --git a/test/test_repo_builder.py b/test/test_repo_builder.py index f72852f..a2f7f05 100755 --- a/test/test_repo_builder.py +++ b/test/test_repo_builder.py @@ -10,6 +10,7 @@ import requests from mock_server import MockServer +from hallmark.transport.base import CapabilityError from hallmark.repo_builder import ( KNOWN_FIELD_VALUES, _match_file_against_fmts, @@ -43,7 +44,7 @@ def _served(server: MockServer): Returns: A context manager that patches requests.get and requests.head. """ - return patch("hallmark.repo_builder.requests.Session", return_value=server) + return patch("hallmark.transport.requests.Session", return_value=server) # helper function to monkeypatch list_remote_files for testing build_repo @@ -58,8 +59,8 @@ def _inventory(monkeypatch, files): A list of base_url values passed to the fake list_remote_files function. """ calls = [] - def fake_list(base_url): - """create a fake list_remote_files list""" + def fake_list(base_url, **kwargs): + """Return the configured inventory and record the requested URL.""" calls.append(base_url) return dict(files) monkeypatch.setattr("hallmark.repo_builder.list_remote_files", fake_list) @@ -182,24 +183,16 @@ def test_list_remote_files_partial_sibling_manifest_does_not_hide_files(): expected (None, None) because the file was not listed in the manifest" -def test_list_remote_files_generic_manifest_detected_by_content(): - """ - A manifest file whose name is generic (like "md5sum.txt") should - still be detected as a manifest if its content looks like one. - """ +def test_list_remote_files_does_not_probe_custom_checksum_payloads(): + """Test that discovery reads only recognized checksum manifests.""" server = MockServer(BASE_URL) server.add_directory("", [ ("data-object", "data.tar"), - ("data-object", "custom_checksum_manifest.log"),]) - server.add_file("custom_checksum_manifest.log", - "aa11bb22cc33dd44ee55ff66aa11bb22 data.tar\n") + ("data-object", "custom_checksum_manifest.log")]) + # No file contents are registered, so fetching either file would fail the test. with _served(server): - file_checksums = list_remote_files(server.base_url) - - assert file_checksums["data.tar"] == \ - ("unknown", "aa11bb22cc33dd44ee55ff66aa11bb22"), \ - f"unexpected checksum for data.tar: {file_checksums}, \ - expected unknown aa11bb22cc33dd44ee55ff66aa11bb22" + files = list_remote_files(server.base_url) + assert files["data.tar"] == (None, None) def test_list_remote_files_generic_manifest_algorithm_from_own_filename(): @@ -333,8 +326,8 @@ def test_list_remote_files_fetches_each_directory_once(): server.add_directory("nested/", [("data-object", "file.dat")]) requested_urls = [] original_get = server.get - def recording_get(url, timeout=None): - """A wrapper around the original server.get that records requested URLs.""" + def recording_get(url, timeout=None, **kwargs): + """Record requested URLs before returning their registered responses.""" requested_urls.append(url) return original_get(url, timeout=timeout) server.get = recording_get @@ -356,11 +349,12 @@ def test_list_remote_files_unavailable_manifest_does_not_hide_files(): server.add_directory( "", [("data-object", "data.tar"), ("data-object", "checksums.txt")]) original_get = server.get - def unavailable_manifest_get(url, timeout=None): - """A wrapper around the original server.get that simulates - an unavailable manifest.""" + def unavailable_manifest_get(url, timeout=None, **kwargs): + """Simulate a missing checksum manifest while serving other registered files.""" if url.endswith("/checksums.txt"): - raise requests.HTTPError("manifest unavailable") + response = requests.Response() + response.status_code = 404 + raise requests.HTTPError("manifest unavailable", response=response) return original_get(url, timeout=timeout) server.get = unavailable_manifest_get with _served(server): @@ -820,6 +814,7 @@ def huge_head(url, timeout=None): resp = MagicMock() resp.raise_for_status = lambda: None resp.headers = {"Content-Length": str(5 * 1024 * 1024 * 1024)} + resp.__enter__.return_value = resp return resp server.fake_head = huge_head def refuse_get_body(url, timeout=None): @@ -839,114 +834,39 @@ def refuse_get_body(url, timeout=None): e for e in repo.state.config["data"] if e.get("file") == "huge_unmatched.dat") - assert entry == {"file": "huge_unmatched.dat", "checksum": "unknown"}, \ + assert entry == {"file": "huge_unmatched.dat"}, \ f"unexpected entry for huge_unmatched.dat: {entry}, expected checksum 'unknown'" -def test_build_repo_small_unmatched_file_still_downloaded_and_hashed(tmp_path, - simple_dataset_server): - """ - Test that build_repo still downloads and hashes a small unmatched file, - using a simple mock dataset server to avoid hitting the real network. - Args: - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - """ - with _served(simple_dataset_server): - repo = build_repo( - repo_path=tmp_path / "repo4.hm", - dataset_name="EHTC_TEST", - fmt_entries=[],) - entry = next(e for e in repo.state.config["data"] if e.get("file") == "README.md") - - has_legacy_md5 = ("md5" in entry and entry["md5"] != "unknown") - has_normalized_md5 = ( - entry.get("checksum_algorithm") == "md5" - and entry.get("checksum") not in (None, "unknown")) - - assert has_legacy_md5 or has_normalized_md5, f"unexpected entry for README.md: \ - {entry}, expected either legacy md5 or normalized checksum schema" - - -def test_build_repo_interactive_detect_branch(monkeypatch, tmp_path, - simple_dataset_server): - """ - fmt_entries=None should enter interactive detect flow and accept strict yes/no. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - """ - answers = iter(["detect", "no", "data.tsv"]) - monkeypatch.setattr("builtins.input", lambda _: next(answers)) - called = {"include_drives": None} - def _fake_detect_fmt(rel_paths, include_drives=False): - called["include_drives"] = include_drives - return ["proj/data.{format}"] - monkeypatch.setattr("hallmark.repo_builder.detect_fmt", _fake_detect_fmt) - with _served(simple_dataset_server): - repo = build_repo( - repo_path=tmp_path / "interactive_detect.hm", - dataset_name="EHTC_TEST", - fmt_entries=None,) +def test_build_repo_small_unmatched_file_is_not_downloaded(tmp_path): + """Test that even small unmatched files remain remote during catalog creation.""" + server = MockServer(BASE_URL) + server.add_directory("", [("data-object", "README.md")]) + with _served(server): + repo = build_repo(tmp_path / "repo.hm", "EHTC_TEST", fmt_entries=[]) + assert repo.state.config["data"] == [{"name": "readme", "file": "README.md"}] - assert called["include_drives"] is False, \ - f"expected include_drives=False, got {called['include_drives']}" - data_tsv = repo.dothm.path / "data.tsv" - assert data_tsv.exists(), f"expected {data_tsv} to be created" - df = pd.read_csv(data_tsv, sep="\t", dtype=str) - assert "proj/data.tar" in set(df["path"]), \ - f"expected proj/data.tar in data.tsv, got {df.to_dict(orient='records')}" +def test_build_repo_defaults_to_generic_path_catalog_without_input( + monkeypatch, tmp_path): + """Test that omitted formats produce a path catalog without prompting.""" + _inventory(monkeypatch, { + "arbitrary.bin": (None, None), "nested/a.fits": (None, None)}) + monkeypatch.setattr("builtins.input", lambda *_: pytest.fail("Unexpected prompt")) + repo = build_repo(tmp_path / "catalog", "label") + assert repo.state.config["data"] == [{"fmt": "{path}", "db": "data.tsv"}] + frame = pd.read_csv(repo.dothm.path / "data.tsv", sep="\t") + assert set(frame["path"]) == {"arbitrary.bin", "nested/a.fits"} -def test_build_repo_interactive_input_branch(monkeypatch, - tmp_path, simple_dataset_server): - """ - fmt_entries=None should allow manual interactive input flow. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - """ - answers = iter(["input", "proj/data.{format}", "data.tsv", ""]) - monkeypatch.setattr("builtins.input", lambda _: next(answers)) - with _served(simple_dataset_server): - repo = build_repo( - repo_path=tmp_path / "interactive_input.hm", - dataset_name="EHTC_TEST", - fmt_entries=None,) - data_tsv = repo.dothm.path / "data.tsv" - assert data_tsv.exists(), f"expected {data_tsv} to be created" - df = pd.read_csv(data_tsv, sep="\t", dtype=str) - assert "proj/data.tar" in set(df["path"]), \ - f"expected proj/data.tar in data.tsv, got {df.to_dict(orient='records')}" -def test_build_repo_interactive_detect_invalid_yes_no_raises( - monkeypatch, tmp_path, simple_dataset_server -): - """ - Detect flow should raise on any include_drives answer besides strict yes/no. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - simple_dataset_server: A fixture providing a MockServer instance - with a simple dataset structure and manifest. - Raises: - ValueError: If the include_drives answer is not "yes" or "no". - """ - answers = iter(["detect", "y"]) - monkeypatch.setattr("builtins.input", lambda _: next(answers)) - with _served(simple_dataset_server): - with pytest.raises(ValueError): - build_repo( - repo_path=tmp_path / "interactive_invalid.hm", - dataset_name="EHTC_TEST", - fmt_entries=None,) +def test_build_repo_rejects_payload_hashing_before_network(monkeypatch, tmp_path): + """Test that remote hashing is rejected before opening a connection.""" + monkeypatch.setattr("hallmark.repo_builder.list_remote_files", + lambda *_: pytest.fail("Unexpected network access")) + with pytest.raises(CapabilityError, match="remote_hash"): + build_repo(tmp_path / "catalog", "label", remote_hash=True) def test_build_repo_multiple_remotes(tmp_path, simple_dataset_server): @@ -1156,11 +1076,10 @@ def test_build_repo_empty_dataset_writes_valid_empty_data_config(monkeypatch, tm expected 'data: []' at the start" -def test_build_repo_loads_formats_and_remotes_interactively_from_config( +def test_build_repo_loads_formats_and_remotes_from_explicit_config( monkeypatch, tmp_path): """ - If the user selects a config file that has formats and remotes, build_repo should - load those formats and remotes into the new repo's config + Test that build_repo loads formats and remotes from an explicit config file. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1178,7 +1097,8 @@ def test_build_repo_loads_formats_and_remotes_interactively_from_config( answers = iter(["config", str(source_config)]) monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - repo = build_repo(tmp_path / "from-config.hm", "EHTC_TEST") + repo = build_repo( + tmp_path / "from-config.hm", "EHTC_TEST", config_file=source_config) assert {entry.get("fmt") for entry in repo.state.config["data"]} == { "item_{number}.dat"}, f"unexpected fmt entries: {repo.state.config['data']}, \ @@ -1188,10 +1108,9 @@ def test_build_repo_loads_formats_and_remotes_interactively_from_config( {repo.state.config['remote']}, expected to load from config" -def test_build_repo_explicit_remotes_override_interactive_config(monkeypatch, tmp_path): +def test_build_repo_explicit_remotes_override_loaded_config(monkeypatch, tmp_path): """ - If the user selects a config file that has remotes, but also provides explicit - remotes to build_repo, the explicit remotes should override the config remotes. + Test that explicit remotes override those loaded from a config file. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1209,17 +1128,17 @@ def test_build_repo_explicit_remotes_override_interactive_config(monkeypatch, tm repo = build_repo( tmp_path / "override.hm", "EHTC_TEST", - remotes=[{"name": "new", "url": "https://new.test"}]) + remotes=[{"name": "new", "url": "https://new.test"}], + config_file=source_config) assert repo.state.config["remote"] == [{"name": "new", "url": "https://new.test"}],\ f"unexpected remotes: {repo.state.config['remote']}, \ expected to override with new remote" -def test_build_repo_interactive_config_requires_a_format(monkeypatch, tmp_path): +def test_build_repo_explicit_config_requires_a_format(monkeypatch, tmp_path): """ - If the user selects a config file that has no fmt entries, build_repo should raise - a ValueError indicating that no formats were found. + Test that a supplied config file must contain at least one filename format. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1232,13 +1151,12 @@ def test_build_repo_interactive_config_requires_a_format(monkeypatch, tmp_path): monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) with pytest.raises(ValueError, match="No fmt entries found"): - build_repo(tmp_path / "missing-fmt.hm", "EHTC_TEST") + build_repo(tmp_path / "missing-fmt.hm", "EHTC_TEST", config_file=source_config) def test_build_repo_reuses_existing_formats_and_remotes(monkeypatch, tmp_path): """ - If a repo already exists, build_repo should prompt to reuse its formats and remotes. - If user agrees, the existing formats and remotes should be preserved in the new repo + Test that an existing repository retains its formats and data remotes. Args: monkeypatch: A pytest fixture for safely patching builtins and other objects. tmp_path: A temporary directory provided by pytest for the test. @@ -1266,88 +1184,19 @@ def test_build_repo_reuses_existing_formats_and_remotes(monkeypatch, tmp_path): {calls}, expected two calls to {BASE_URL}" -def test_build_repo_rejects_invalid_existing_repo_choice_before_network( - monkeypatch, tmp_path): - """ - If the user chooses an invalid option when prompted to reuse an existing repo, - build_repo should raise a ValueError before attempting any network operations. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - AssertionError: If build_repo attempts to perform network operations after an - invalid choice is made. - ValueError: If the user provides an unrecognized choice when prompted to reuse - an existing repo. - """ - repo_path = tmp_path / "existing.hm" - _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - build_repo( - repo_path, - "EHTC_TEST", - fmt_entries=[{"fmt": "item_{number}.dat", "db": "items.tsv"}]) - monkeypatch.setattr("builtins.input", lambda prompt: "invalid") - def network_must_not_run(base_url): - """check the invalid input is caught before attempting network operations""" - raise AssertionError("remote listing should remain lazy") - monkeypatch.setattr( - "hallmark.repo_builder.list_remote_files", network_must_not_run) +def test_build_repo_reuses_existing_catalog_without_input(monkeypatch, tmp_path): + """Test that an existing catalog is refreshed without interactive input.""" + _inventory(monkeypatch, {"item_1.dat": (None, None)}) + path = tmp_path / "catalog" + build_repo(path, "label", [{"fmt": "item_{number}.dat", "db": "data.tsv"}]) + monkeypatch.setattr("builtins.input", lambda *_: pytest.fail("Unexpected prompt")) + repo = build_repo(path, "label") + assert repo.state.config["data"] == [{"fmt": "item_{number}.dat", "db": "data.tsv"}] - with pytest.raises(ValueError, match="Unrecognized choice"): - build_repo(repo_path, "EHTC_TEST") -def test_build_repo_detects_single_format_and_uses_data_tsv(monkeypatch, tmp_path): - """ - If detect_fmt returns a single format, build_repo should use it and default to - "data.tsv" as the db name. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - """ - _inventory(monkeypatch, - {"item_1.dat": ("md5", "a" * 32), "item_2.dat": ("md5", "b" * 32)}) - answers = iter(["detect", "yes"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - detected = {} - def fake_detect(paths, include_drives=False): - """return a single format for testing purposes, and record the arguments""" - detected["args"] = (paths, include_drives) - return ["item_{number}.dat"] - monkeypatch.setattr("hallmark.repo_builder.detect_fmt", fake_detect) - repo = build_repo(tmp_path / "detected.hm", "EHTC_TEST") - - assert detected["args"] == (["item_1.dat", "item_2.dat"], True), \ - f"unexpected arguments to detect_fmt: {detected['args']}, expected \ - (['item_1.dat', 'item_2.dat'], True)" - assert (repo.dothm.path / "data.tsv").is_file(), f"expected data.tsv to be created,\ - but it does not exist at {repo.dothm.path / 'data.tsv'}" - assert [entry for entry in repo.state.config["data"] if "fmt" in entry] == [ - {"fmt": "item_{number}.dat", "db": "data.tsv"}], f"unexpected data config: \ - {repo.state.config['data']}, expected a single fmt entry with db 'data.tsv'" -def test_build_repo_records_detect_include_drives_in_meta(monkeypatch, tmp_path): - """ - When fmts are auto-detected, build_repo should record the include_drives choice - in meta.yml so it can be recovered later without re-running detection interactively. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - """ - _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - answers = iter(["detect", "yes"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - monkeypatch.setattr( - "hallmark.repo_builder.detect_fmt", - lambda paths, include_drives=False: ["item_{number}.dat"]) - repo = build_repo(tmp_path / "detected.hm", "EHTC_TEST") - - assert repo.state.meta.get("detect_include_drives") is True, f"expected meta.yml to\ - record detect_include_drives=True, got: {repo.state.meta}" - assert repo.dothm.load_yml("meta").get("detect_include_drives") is True, \ - "expected meta.yml on disk to record detect_include_drives=True" - def test_build_repo_omits_detect_include_drives_when_fmt_entries_given( monkeypatch, tmp_path): @@ -1367,63 +1216,10 @@ def test_build_repo_omits_detect_include_drives_when_fmt_entries_given( f"did not expect detect_include_drives in meta.yml, got: {repo.state.meta}" -def test_build_repo_detects_multiple_formats_and_normalizes_db_names( - monkeypatch, tmp_path): - """ - If detect_fmt returns multiple formats, build_repo should prompt for a db name - for each format and normalize the db names to be unique. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - """ - _inventory(monkeypatch, - {"a_1.dat": ("md5", "a" * 32), "b_2.dat": ("md5", "b" * 32)}) - answers = iter(["detect", "no", "a", "b.tsv"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - monkeypatch.setattr( - "hallmark.repo_builder.detect_fmt", - lambda paths, include_drives=False: ["a_{n}.dat", "b_{n}.dat"]) - repo = build_repo(tmp_path / "multi-detect.hm", "EHTC_TEST") - assert [entry for entry in repo.state.config["data"] if "fmt" in entry] == [ - {"fmt": "a_{n}.dat", "db": "a.tsv"}, - {"fmt": "b_{n}.dat", "db": "b.tsv"}], f"unexpected data config: \ - {repo.state.config['data']}, expected two fmt entries with normalized db names" - - -def test_build_repo_detect_rejects_no_detected_formats(monkeypatch, tmp_path): - """ - If detect_fmt returns no formats, build_repo should raise a ValueError. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - ValueError: If no formats could be automatically detected. - """ - _inventory(monkeypatch, {"item.dat": ("md5", "a" * 32)}) - answers = iter(["detect", "no"]) - monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) - monkeypatch.setattr( - "hallmark.repo_builder.detect_fmt", - lambda paths, include_drives=False: [],) - with pytest.raises(ValueError, match="No fmts could be automatically detected"): - build_repo(tmp_path / "nothing-detected.hm", "EHTC_TEST") -def test_build_repo_rejects_unknown_format_input_mode(monkeypatch, tmp_path): - monkeypatch.setattr("builtins.input", lambda prompt: "unknown") - """ - Test that build_repo raises a ValueError when an unrecognized choice is provided - in interactive mode. - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - ValueError: If the provided choice is not recognized. - """ - with pytest.raises(ValueError, match="Unrecognized choice"): - build_repo(tmp_path / "unknown.hm", "EHTC_TEST") def test_build_repo_preserves_existing_destination_by_default( @@ -1473,7 +1269,7 @@ def test_build_repo_replaces_destination_when_overwrite_is_explicit( sentinel.write_text("old data\n", encoding="utf-8") monkeypatch.setattr( "hallmark.repo_builder.list_remote_files", - lambda _base_url: {}) + lambda _base_url, **kwargs: {}) repo = build_repo( repo_path=destination, dataset_name="EHTC_TEST", @@ -1565,7 +1361,7 @@ def test_build_repo_accepts_config_repository_directory(monkeypatch, tmp_path): answers = iter(["config", str(source_repo)]) monkeypatch.setattr("builtins.input", lambda prompt: next(answers)) _inventory(monkeypatch, {"item_1.dat": ("md5", "a" * 32)}) - repo = build_repo(tmp_path / "result.hm", "EHTC_TEST") + repo = build_repo(tmp_path / "result.hm", "EHTC_TEST", config_file=source_repo) actual = [ entry for entry in repo.state.config["data"] if "fmt" in entry] expected = [{"fmt": "item_{number}.dat", "db": "items.tsv"}] @@ -1590,41 +1386,13 @@ def test_build_repo_preserves_meta_file_checksum(monkeypatch, tmp_path): expected meta.yml with checksum {checksum}" -def test_build_repo_continues_when_static_checksum_request_fails(monkeypatch, tmp_path): - """ - Test that build_repo continues to create the repo even if a request to retrieve - the checksum for a static file fails, and that it records the checksum as "unknown". - Args: - monkeypatch: A pytest fixture for safely patching builtins and other objects. - tmp_path: A temporary directory provided by pytest for the test. - Raises: - HTTPError: If the request to retrieve the checksum for a static file fails. - """ +def test_build_repo_never_requests_static_checksum(monkeypatch, tmp_path): + """Test that missing static-file checksums do not trigger dataset downloads.""" _inventory(monkeypatch, {"README.md": (None, None)}) - class FailingChecksumSession: - """ - A mock requests.Session that simulates a failure to retrieve the checksum for - a static file, by raising an HTTPError when the head() method is called. - """ - def __enter__(self): - """enter the context manager""" - return self - - def __exit__(self, exc_type, exc_value, traceback): - """exit the context manager""" - return False - - def head(self, *args, **kwargs): - """Simulate a failure to retrieve the checksum for a static file.""" - raise requests.HTTPError("HEAD unavailable") - monkeypatch.setattr("hallmark.repo_builder.requests.Session", - FailingChecksumSession) + monkeypatch.setattr("hallmark.transport.requests.Session", + lambda: pytest.fail("Unexpected checksum request")) repo = build_repo(tmp_path / "repo.hm", "EHTC_TEST", fmt_entries=[]) - - assert repo.state.config["data"] == [ - {"name": "readme", "file": "README.md", "checksum": "unknown"}], \ - f"unexpected data config: {repo.state.config['data']}, \ - expected README.md with checksum 'unknown'" + assert repo.state.config["data"] == [{"name": "readme", "file": "README.md"}] def test_build_repo_preserves_multiple_meta_files(monkeypatch, tmp_path): @@ -1945,4 +1713,4 @@ def test_build_repo_explicit_remotes_override_config_file_remotes( assert repo.state.config["remote"] == [{"name": "new", "url": "https://new.test"}],\ f"unexpected remotes: {repo.state.config['remote']}, \ - expected explicit remotes to override config_file remotes" \ No newline at end of file + expected explicit remotes to override config_file remotes" diff --git a/test/test_sftp_metadata.py b/test/test_sftp_metadata.py new file mode 100644 index 0000000..80fae34 --- /dev/null +++ b/test/test_sftp_metadata.py @@ -0,0 +1,259 @@ +"""Structured SFTP discovery, using a local subsystem without any network.""" + +from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext +from pathlib import Path +import stat +import struct +import sys +import time + +import pytest + +from hallmark.transport import OperationContext, RemoteSpec +from hallmark.transport.base import ( + CapabilityError, DownloadError, RemoteConfigurationError, + RemoteObjectMissing, TransferCancelled, +) +from hallmark.transport.sftp import _Packet + + +@pytest.fixture +def local_sftp(tmp_path, monkeypatch): + """Connect the transport to a local SFTP subsystem without networking.""" + server = next((path for path in ( + "/usr/lib/openssh/sftp-server", "/usr/libexec/sftp-server", + ) if Path(path).exists()), None) + if server is None: + pytest.skip("Local OpenSSH SFTP subsystem binary is unavailable") + root = tmp_path / "source" + root.mkdir() + with OperationContext(RemoteSpec.parse(f"sftp://unused{root}/")) as context: + transport = context.transport + monkeypatch.setattr(transport, "prepare", lambda: None) + spawn = transport._spawn + + def spawn_subsystem(argv, **kwargs): + assert argv[-5:] == ["-T", "-s", "--", "unused", "sftp"] + assert "BatchMode=yes" in argv + assert "StrictHostKeyChecking=yes" in argv + return spawn([server], **kwargs) + + monkeypatch.setattr(transport, "_spawn", spawn_subsystem) + yield context, root + + +def test_structured_paths_sizes_progress_and_missing_metadata(local_sftp): + context, root = local_sftp + names = ["plain", "a b#?%+ü'\"[]*.fits", "-leading", " leading ", "literal%20"] + for name in names: + (root / name).write_text(name) + (root / "nested").mkdir() + (root / "nested" / "child").write_bytes(b"child") + (root / "link").symlink_to(root / "nested", target_is_directory=True) + directories = [] + iterator = context.transport.iter_entries(on_directory=directories.append) + first = next(iterator) + assert directories == [""] + entries = [first, *iterator] + expected = {name: len(name.encode()) for name in names} + expected["nested/child"] = 5 + assert {entry.path: entry.size for entry in entries} == expected + assert directories == ["", "nested/"] + assert all(entry.mtime is not None for entry in entries) + assert len(context.transport._processes) == 0 + assert context.transport.stat("plain").size == 5 + with pytest.raises(RemoteObjectMissing): + context.transport.stat("missing") + assert not context.transport._processes + + +def test_multiple_readdir_packets_and_cancellation_cleanup(local_sftp): + context, root = local_sftp + for index in range(1500): + (root / f"file-{index:04d}").touch() + assert len(list(context.transport.iter_entries())) == 1500 + iterator = context.transport.iter_entries() + next(iterator) + context.cancel() + with pytest.raises(TransferCancelled): + next(iterator) + assert not context.transport._processes + + +def test_reject_controls_without_silently_omitting_file(local_sftp): + context, root = local_sftp + (root / "bad\nname").touch() + with pytest.raises(RemoteConfigurationError, match="control"): + list(context.transport.iter_entries()) + assert not context.transport._processes + + +def test_discovery_excludes_root_repository_metadata_and_objects(local_sftp): + context, root = local_sftp + for metadata in (".hm", ".git", ".HM", ".GIT"): + (root / metadata / "objects").mkdir(parents=True) + (root / metadata / "objects" / "payload").write_bytes(b"private") + (root / "nested" / ".hm" / "objects").mkdir(parents=True) + (root / "nested" / ".hm" / "objects" / "payload").write_bytes(b"private") + (root / "science.fits").write_bytes(b"science") + directories = [] + entries = list(context.transport.iter_entries(on_directory=directories.append)) + assert [entry.path for entry in entries] == ["science.fits"] + assert directories == ["", "nested/"] + + +def test_metadata_file_size_limit_before_transfer(local_sftp, monkeypatch): + context, root = local_sftp + (root / "metadata").write_bytes(b"oversized") + + def unexpected(*args, **kwargs): + raise AssertionError("Known oversized metadata must not be fetched") + + monkeypatch.setattr(context.transport, "_fetch", unexpected) + with pytest.raises(DownloadError, match="size limit"): + context.transport.read_text("metadata", 2) + + +def _fake_subsystem(monkeypatch, context, script): + """Use a supplied script when starting an SFTP subsystem process.""" + transport = context.transport + monkeypatch.setattr(transport, "prepare", lambda: None) + spawn = transport._spawn + monkeypatch.setattr( + transport, "_spawn", + lambda argv, **kw: spawn([sys.executable, "-c", script], **kw), + ) + + +@pytest.mark.parametrize("payload, message", [ + (struct.pack(">I", 2 ** 30), "oversized"), + (struct.pack(">I", 5) + b"\x02\x00", "truncated"), + (struct.pack(">IBI", 5, 2, 4), "version 3"), +]) +def test_bad_handshake_is_bounded_and_cleans_process(monkeypatch, payload, message): + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + _fake_subsystem( + monkeypatch, context, + "import os; os.read(0, 9); os.write(1, " + repr(payload) + ")", + ) + with pytest.raises((DownloadError, CapabilityError), match=message): + with context.transport._metadata(): + pytest.fail("Invalid handshake was accepted") + assert not context.transport._processes + + +def test_metadata_timeout_is_per_request_and_cancellable(monkeypatch): + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + _fake_subsystem(monkeypatch, context, "import time; time.sleep(30)") + context.listing_timeout = 0.1 + with pytest.raises(DownloadError, match="time limit"): + with context.transport._metadata(): + pass + assert not context.transport._processes + context.listing_timeout = 30 + with ThreadPoolExecutor(1) as pool: + future = pool.submit(context.transport._metadata().__enter__) + deadline = time.monotonic() + 2 + while not context.transport._processes and time.monotonic() < deadline: + time.sleep(0.01) + context.cancel() + with pytest.raises(TransferCancelled): + future.result(timeout=2) + assert not context.transport._processes + + +def test_attributes_preserve_unknowns_and_reject_truncation(): + assert _Packet(struct.pack(">I", 0)).attributes() == { + "size": None, "mode": None, "mtime": None, + } + data = struct.pack(">IQIII", 13, 2 ** 40, stat.S_IFREG | 0o600, 123, 456) + assert _Packet(data).attributes() == { + "size": 2 ** 40, "mode": stat.S_IFREG | 0o600, "mtime": 456, + } + with pytest.raises(DownloadError, match="Truncated"): + _Packet(data[:-1]).attributes() + with pytest.raises(DownloadError, match="flags"): + _Packet(struct.pack(">I", 0x10)).attributes() + + +def test_missing_type_uses_lstat_and_cannot_escape_root(monkeypatch): + class Session: + escape = False + + def realpath(self, path): + return "/outside" if self.escape and path.endswith("/nested") else path + + def lstat(self, path): + mode = stat.S_IFREG if path.endswith("/file") else stat.S_IFDIR + return {"size": 4, "mode": mode, "mtime": 123} + + def iterdir(self, path): + if path == "/data": + yield "nested", {"size": None, "mode": None, "mtime": None} + else: + yield "file", {"size": None, "mode": None, "mtime": None} + + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + session = Session() + monkeypatch.setattr( + context.transport, "_metadata", lambda: nullcontext(session), + ) + entries = list(context.transport.iter_entries()) + assert [(entry.path, entry.size) for entry in entries] == [("nested/file", 4)] + session.escape = True + with pytest.raises(DownloadError, match="escapes"): + list(context.transport.iter_entries()) + + +@pytest.mark.parametrize("kind, response_id, message", [ + (105, 999, "identifier"), + (104, 1, "Unexpected"), +]) +def test_wrong_response_cannot_be_treated_as_file_attributes( + monkeypatch, kind, response_id, message, +): + response = struct.pack(">BII", kind, response_id, 0) + script = """import struct, sys +source, output = sys.stdin.buffer, sys.stdout.buffer +source.read(9) +output.write(struct.pack('>IBI', 5, 2, 3)); output.flush() +length = struct.unpack('>I', source.read(4))[0] +source.read(length) +payload = """ + repr(response) + """ +output.write(struct.pack('>I', len(payload)) + payload); output.flush() +""" + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + _fake_subsystem(monkeypatch, context, script) + with pytest.raises(DownloadError, match=message): + with context.transport._metadata() as session: + session.lstat("/data") + assert not context.transport._processes + + +def test_sftp_listing_skips_directories_rejected_by_descend(monkeypatch): + listed = [] + + class Session: + def realpath(self, path): + return path + + def lstat(self, path): + return {"size": 4, "mode": stat.S_IFDIR, "mtime": 1} + + def iterdir(self, path): + listed.append(path) + if path == "/data": + yield "keep", {"size": None, "mode": stat.S_IFDIR, "mtime": None} + yield "skip", {"size": None, "mode": stat.S_IFDIR, "mtime": None} + else: + yield "file", {"size": 4, "mode": stat.S_IFREG, "mtime": 1} + + with OperationContext(RemoteSpec.parse("sftp://unused/data")) as context: + monkeypatch.setattr( + context.transport, "_metadata", lambda: nullcontext(Session()), + ) + context.descend = lambda directory: directory == "keep/" + entries = list(context.transport.iter_entries()) + assert [entry.path for entry in entries] == ["keep/file"] + assert listed == ["/data", "/data/keep"] diff --git a/test/test_sources.py b/test/test_sources.py new file mode 100644 index 0000000..33e77fc --- /dev/null +++ b/test/test_sources.py @@ -0,0 +1,260 @@ +"""Named sources, template suggestions and URL templates with in-memory listings.""" + +from types import SimpleNamespace + +import pytest +import yaml +from click.testing import CliRunner + +from hallmark import DataSource, Repo, SourceRelease, register_source +from hallmark import sources +from hallmark.cli import hallmark +from hallmark.discovery import TemplateMatcher +from hallmark.repo_remote import suggest_templates +from hallmark.sources import backend_for_url, get_source, list_sources +from hallmark.transport import OperationContext +from hallmark.transport.base import RemoteObjectMissing + + +def index(*paths): + return '

Index of data

' + ''.join( + f'{path}' for path in paths) + + +@pytest.fixture +def pages(monkeypatch): + content, reads = {}, [] + + def read(context, path): + url = context.remote.url.rstrip("/") + "/" + path + reads.append(url) + if url not in content: + raise RemoteObjectMissing(f"Unexpected request: {url}") + return content[url] + + monkeypatch.setattr(OperationContext, "read_text", read) + return content, reads + + +@pytest.fixture +def registry(monkeypatch): + monkeypatch.setattr(sources, "_registered", dict(sources._registered)) + monkeypatch.setattr(sources, "_loaded", {}) + monkeypatch.setattr(sources, "_entry_points", lambda: []) + + +def test_desi_definitions_are_explicit_and_offline(pages): + descriptor = get_source("desi") + assert list(descriptor.releases) == ["edr", "dr1"] + assert descriptor.releases["edr"].collections["redshifts"] == ( + "spectro/redux/fuji/zcatalog",) + assert "lss" not in descriptor.releases["edr"].collections + assert descriptor.releases["dr1"].collections["lss"] == ( + "survey/catalogs/dr1/LSS",) + assert "desi" in [item.name for item in list_sources()] + result = CliRunner().invoke(hallmark, ["sources", "desi"]) + assert result.exit_code == 0, result.output + assert "spectro/redux/iron/zcatalog/" in result.output + assert "hallmark ls-remote URL" in result.output + assert pages[1] == [] + + +def test_collection_urls_from_sources_feed_add(tmp_path, pages): + content, reads = pages + zcatalog = "https://data.desi.lbl.gov/public/dr1/spectro/redux/iron/zcatalog/" + content[zcatalog] = index("zall-pix-iron.fits", "zall-tilecumulative-iron.fits", + "notes.txt", "v1/") + repo = Repo.init(tmp_path / "desi") + repo.add(zcatalog + "zall-{kind}-iron.fits") + assert repo.state.data["kind"].tolist() == ["pix", "tilecumulative"] + # the template has one segment, so subdirectories are never listed + assert reads == [zcatalog] + plan = repo.plan_download(include="*pix*") + assert plan.remote_url == zcatalog + assert [item.relative_path.as_posix() for item in plan.items] == [ + "zall-pix-iron.fits"] + + +@pytest.mark.parametrize("template", [ + "{", "{path}", "{checksum}", "{sha1}", "{md5}", "{size_bytes}", "{mtime}", + "{checksum_algorithm}", "{}", "{0}", "{a.b}", "{a[0]}", "{run!r}", "", + "{run}/", +]) +def test_invalid_url_templates_are_rejected_before_access(tmp_path, pages, template): + repo = Repo.init(tmp_path / "repo") + with pytest.raises(ValueError): + repo.add("https://test/data/" + template) + assert pages[1] == [] + + +def test_remote_template_survives_reload_rename_and_clone( + tmp_path, pages, monkeypatch): + from mock_server import MockServer + + root = "https://test/data/" + pages[0][root] = index("run_001.h5", "RUN_002.h5", "README.md", "omit.bin") + repo = Repo.init(tmp_path / "repo") + repo.add(root + "run_{run:03d}.h5") + repo.commit("Catalog runs") + before = (repo.dothm.path / "data.tsv").read_bytes() + repo = Repo(repo.worktree) + # templates are case-sensitive, and other files are not cataloged + assert repo.state.data["run"].tolist() == ["001"] + server = MockServer(root) + server.add_file("run_001.h5", b"run one") + monkeypatch.setattr("requests.Session", lambda: server) + plan = repo.plan_download() + assert repo.download(plan, approved=True)["succeeded"] == 1 + assert (repo.worktree / "run_001.h5").read_bytes() == b"run one" + assert (repo.dothm.path / "data.tsv").read_bytes() == before + # Configuring a mirror and cloning the catalog preserve the template source. + repo.set_config(remote_name="archive", remote_url="https://mirror.test/data/") + repo.commit("Add a mirror") + clone = Repo.clone(str(repo.dothm.path), tmp_path / "copy") + assert clone.state.config["data"] == repo.state.config["data"] + assert clone.plan_download().remote_url == root + assert clone.plan_download(remote_name="archive").remote_url == ( + "https://mirror.test/data/") + assert not (clone.worktree / "run_001.h5").exists() + + +def test_source_registration_and_backend_lookup(registry): + release = SourceRelease("https://test/", {"both": ["a", "a/b"], "one": "a"}, + backend="cyverse") + descriptor = DataSource("example", "Example source", {"v1": release}) + register_source(descriptor) + assert backend_for_url("https://test/a/b/")[0] == "cyverse" + assert backend_for_url("https://other.test/") is None + with pytest.raises(ValueError, match="Duplicate"): + register_source(descriptor) + with pytest.raises(TypeError): + descriptor.releases["v2"] = release + + +def test_add_inherits_the_backend_of_a_registered_release(tmp_path, pages, registry): + root = "https://test/release/" + register_source(DataSource("example", "Example source", {"v1": SourceRelease( + root, {}, backend="cyverse")})) + pages[0][root + "runs/"] = index("run_1.h5") + repo = Repo.init(tmp_path / "repo") + repo.add(root + "runs/run_{run}.h5") + assert repo.state.config["data"][0]["backend"] == "cyverse" + + +def test_source_plugin_validation(registry, monkeypatch): + descriptor = DataSource("plugin", "A plugin", {"v1": SourceRelease( + "https://test/", {})}) + entry = SimpleNamespace(name="plugin", load=lambda: descriptor) + monkeypatch.setattr(sources, "_entry_points", lambda: [entry]) + assert get_source("plugin") is descriptor + assert get_source("plugin") is descriptor + monkeypatch.setattr(sources, "_entry_points", lambda: [entry, entry]) + with pytest.raises(ValueError, match="Duplicate"): + get_source("plugin") + + +@pytest.mark.parametrize("operation", ["open", "git-clone", "snapshot-clone"]) +def test_old_auth_references_require_explicit_migration(tmp_path, pages, operation): + repo = Repo.init(tmp_path / "legacy") + config = {"data": [{"db": "data.tsv"}], "remote": [{ + "name": "origin", "url": "sftp://lab-data/export/", "auth": "old"}]} + repo.dothm.dump_yml(config, "config") + repo.dothm.index.add(["config.yml"]) + repo.dothm.index.commit("Legacy profile reference") + destination = tmp_path / "copy" + if operation == "snapshot-clone": + root = "https://test/catalog/" + pages[0].update({root + "config.yml": yaml.safe_dump(config), + root + "meta.yml": "{}\n", root + "data.tsv": "path\n"}) + with pytest.raises(ValueError, match="~/.ssh/config"): + if operation == "open": + Repo(repo.worktree) + elif operation == "git-clone": + Repo.clone(str(repo.dothm.path), destination) + else: + Repo.clone(root, destination, source_type="catalog") + assert not destination.exists() + assert yaml.safe_load((repo.dothm.path / "config.yml").read_text()) == config + + +@pytest.mark.parametrize("option", [ + "--auth", "--backend", "--backend-options", "--include", "--extract", "--fmt", + "--max-workers", "--with-download", "--from", "--release", "--collection", + "--filter", "--format", +]) +def test_init_removed_options_have_no_alias(tmp_path, pages, option): + destination = tmp_path / "new" + result = CliRunner().invoke(hallmark, ["init", str(destination), option, "x"]) + assert result.exit_code == 2 + assert "No such option" in result.output + assert not destination.exists() + assert pages[1] == [] + + +@pytest.mark.parametrize("kwargs", [ + {"from_url": "https://test/"}, {"auth": "old"}, {"backend": "http"}, + {"include": "*"}, {"extract": "{name}"}, {"fmt": "{name}"}, {"download": True}, + {"source": "https://test/"}, {"release": "dr1"}, {"collections": ["a"]}, + {"filter": "*"}, {"format": "{name}"}, +]) +def test_init_removed_python_arguments_have_no_shim(tmp_path, kwargs): + with pytest.raises(TypeError): + Repo.init(tmp_path / "new", **kwargs) + assert not (tmp_path / "new").exists() + + +def test_ls_remote_suggests_templates_from_filenames(tmp_path, pages): + root = "https://test/data/" + paths = ["run_001.h5", "run_002.h5", "sgra_20170406_nustar.fits", + "sgra_20170411_chandra.fits", "README.md", "archive.tar.gz"] + pages[0][root] = index(*paths) + base, suggestions, total = suggest_templates(root) + assert pages[1] == [root] # Detection uses filenames, never payloads. + assert (base, total) == (root, len(paths)) + assert suggestions == [(root + "run_{scan}.h5", 2), + (root + "sgra_{date}_{p0}.fits", 2)] + repo = Repo.init(tmp_path / "repo") + repo.add(suggestions[1][0]) + assert repo.state.data[["date", "p0"]].values.tolist() == [ + ["20170406", "nustar"], ["20170411", "chandra"]] + + +def test_ls_remote_skips_suggestions_with_reserved_or_literal_braces(pages): + root = "https://test/data/" + pages[0][root] = index("{path}_001.h5", "{path}_002.h5", "README.md") + _, suggestions, total = suggest_templates(root) + assert total == 3 + for template, _ in suggestions: + TemplateMatcher(template[len(root):]) + + +def test_cli_ls_remote_then_add_workflow(tmp_path, pages, monkeypatch): + root = "https://test/data/" + pages[0][root] = index("run_001.h5", "run_002.h5", "README.md", "omit.bin") + runner = CliRunner() + listing = runner.invoke(hallmark, ["ls-remote", root]) + assert listing.exit_code == 0, listing.output + assert "4 file(s) at https://test/data/" in listing.output + assert f"hallmark add '{root}run_{{scan}}.h5' (2)" in listing.output + assert runner.invoke(hallmark, ["init", str(tmp_path / "cli")]).exit_code == 0 + monkeypatch.chdir(tmp_path / "cli") + preview = runner.invoke(hallmark, ["add", "-n", root + "run_{run:03d}.h5"]) + assert preview.exit_code == 0, preview.output + # Click 8.1, used on Python 3.9, mixes the stderr progress bar into stdout + lines = preview.stdout.splitlines() + start = lines.index("Would add") + assert lines[start:start + 3] == ["Would add", "run_001.h5", "run_002.h5"] + assert not Repo(tmp_path / "cli").state.config["data"][0].get("fmt") + added = runner.invoke(hallmark, ["add", root + "run_{run:03d}.h5"]) + assert added.exit_code == 0, added.output + assert set(Repo(tmp_path / "cli").state.data["run"]) == {"001", "002"} + + +@pytest.mark.parametrize("option", ["--extract", "--format", "--fmt", "--filter"]) +def test_download_rejects_removed_and_initialization_only_flags( + tmp_path, monkeypatch, option): + repo = Repo.init(tmp_path / "repo") + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["download", option, "value"]) + assert result.exit_code == 2 + assert "No such option" in result.output diff --git a/test/test_ssh_integration.py b/test/test_ssh_integration.py new file mode 100644 index 0000000..d48d8ac --- /dev/null +++ b/test/test_ssh_integration.py @@ -0,0 +1,547 @@ +"""Disposable, loopback-only OpenSSH tests. No external hosts or persistent keys.""" + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import getpass +import hashlib +import os +from pathlib import Path +import shutil +import socket +import subprocess +import time +from urllib.parse import quote + +from click.testing import CliRunner +import pytest + +from hallmark import Repo +from hallmark.cli import hallmark +from hallmark.downloader import download_remote_data, select_download_files +from hallmark.repo_builder import build_repo +from hallmark.transport import OperationContext, RemoteSpec +from hallmark.transport.base import ( + DownloadError, RemoteObjectMissing, TransferCancelled, +) +from hallmark.transport.ssh import SshTransport + +pytestmark = pytest.mark.ssh_integration + + +@pytest.fixture +def ssh_server(tmp_path, monkeypatch, request): + """Start a loopback SSH server with temporary keys and configuration.""" + if os.environ.get("HALLMARK_RUN_SSH_TESTS") != "1": + pytest.skip("Set HALLMARK_RUN_SSH_TESTS=1 for disposable loopback SSH tests") + sshd = shutil.which("sshd") or "/usr/sbin/sshd" + if not Path(sshd).exists(): + pytest.fail("Opted-in SSH tests require openssh-server") + for name in ("host", "client", "second", "untrusted"): + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(tmp_path / name)], + check=True, + ) + authorized = tmp_path / "authorized" + authorized.write_text( + (tmp_path / "client.pub").read_text() + (tmp_path / "second.pub").read_text() + ) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + config = tmp_path / "sshd_config" + config.write_text(f"""ListenAddress 127.0.0.1 +Port {port} +HostKey {tmp_path}/host +PidFile {tmp_path}/sshd.pid +AuthorizedKeysFile {authorized} +StrictModes no +UsePAM no +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin prohibit-password +MaxSessions 4 +Subsystem sftp internal-sftp +""") + mode = getattr(request, "param", None) + if mode == "sftp-only": + config.write_text(config.read_text() + "ForceCommand internal-sftp\n") + elif mode == "no-sftp": + config.write_text( + config.read_text().replace("Subsystem sftp internal-sftp\n", "") + ) + elif mode == "max-one": + config.write_text(config.read_text().replace("MaxSessions 4", "MaxSessions 1")) + log = (tmp_path / "sshd.log").open("wb") + daemon = subprocess.Popen( + [sshd, "-D", "-e", "-f", str(config)], stderr=log, start_new_session=True + ) + known = tmp_path / "known_hosts" + known.write_text(f"[127.0.0.1]:{port} " + (tmp_path / "host.pub").read_text()) + client_config = tmp_path / "ssh_config" + client_config.write_text(f"""Host hm-test + HostName 127.0.0.1 +Host * + User {getpass.getuser()} + Port {port} + IdentityFile {tmp_path}/client + IdentityAgent none + IdentitiesOnly yes + UserKnownHostsFile {known} + GlobalKnownHostsFile /dev/null +""") + original = SshTransport._options + monkeypatch.setattr( + SshTransport, + "_options", + lambda self, **kw: ["-F", str(client_config)] + original(self, **kw), + ) + monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + root = tmp_path / "export" + root.mkdir() + try: + deadline = time.monotonic() + 5 + while True: + if daemon.poll() is not None: + log.flush() + pytest.fail((tmp_path / "sshd.log").read_text()) + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.1): + break + except OSError: + if time.monotonic() >= deadline: + pytest.fail("Local SSH test server did not start") + time.sleep(0.05) + yield { + "root": root, + "port": port, + "known": known, + "base": tmp_path, + "config": client_config, + "daemon": daemon, + "url": "ssh://hm-test" + quote(str(root), safe="/") + "/", + } + finally: + daemon.terminate() + daemon.wait(timeout=5) + log.close() + + +@pytest.mark.parametrize("scheme", ["ssh", "sftp"]) +def test_parallel_exact_files_and_master_cleanup(ssh_server, tmp_path, scheme): + root = ssh_server["root"] + names = [ + "a b#?%+ü.h5", + "quotes'\"[x]*?.h5", + "-leading.h5", + " tail ", + "literal%20.h5", + "nested/data.bin", + ] + for name in names: + path = root / name + path.parent.mkdir(exist_ok=True) + path.write_text(name) + url = ssh_server["url"].replace("ssh:", scheme + ":") + with OperationContext(RemoteSpec.parse(url)) as context: + transport = context.transport + transport.prepare() + master = transport._master + control = transport._socket + assert Path(control).stat().st_mode & 0o077 == 0 + output = tmp_path / "output" + output.mkdir() + with ThreadPoolExecutor(4) as pool: + tasks = [ + pool.submit(transport.fetch, name, output / str(i)) + for i, name in enumerate(names) + ] + for task in tasks: + task.result() + assert transport._master is master + assert len(transport._processes) == 1 + for i, name in enumerate(names): + assert (output / str(i)).read_text() == name + assert master.poll() is not None + assert not Path(control).exists() + + +def test_explicit_endpoint_and_missing_file(ssh_server, tmp_path): + server = ssh_server + url = f"sftp://{getpass.getuser()}@127.0.0.1:{server['port']}{server['root']}/" + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=url) + output = repo.worktree / "missing" + output.write_bytes(b"keep") + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("missing"), None)] + , approved=True) + assert result["failed"] == 1 + assert output.read_bytes() == b"keep" + assert not list(Path(repo.worktree).glob("*.part")) + + +@pytest.mark.parametrize("trust", ["unknown", "changed", "no-key"]) +def test_host_trust_and_auth_fail_preflight(ssh_server, tmp_path, trust): + server = ssh_server + if trust == "unknown": + server["known"].write_text("") + elif trust == "changed": + server["known"].write_text( + f"[127.0.0.1]:{server['port']} " + + (server["base"] / "untrusted.pub").read_text() + ) + else: + # Both keys are ephemeral; the replacement is not authorized on the server. + config = server["config"].read_text().replace("/client\n", "/untrusted\n") + server["config"].write_text(config) + with OperationContext(RemoteSpec.parse(server["url"])) as context: + with pytest.raises(DownloadError, match="Cannot establish"): + context.transport.prepare() + assert not context.transport._processes + assert not (tmp_path / "download").exists() + + +def test_simultaneous_ssh_aliases_have_separate_masters(ssh_server): + server = ssh_server + (server["root"] / "item").write_bytes(b"data") + original = server["config"].read_text() + aliases = "".join( + f"Host {alias}\n HostName 127.0.0.1\n" + f" IdentityFile {server['base'] / key}\n" + for alias, key in [("one", "client"), ("two", "second")]) + server["config"].write_text(aliases + original.replace( + f" IdentityFile {server['base'] / 'client'}\n", "")) + first_url = server["url"].replace("hm-test", "one") + second_url = server["url"].replace("hm-test", "two") + with OperationContext(RemoteSpec.parse(first_url)) as one: + with OperationContext(RemoteSpec.parse(second_url)) as two: + with ThreadPoolExecutor(2) as pool: + list(pool.map(lambda context: context.transport.prepare(), [one, two])) + assert one.transport._socket != two.transport._socket + assert one.transport._master.pid != two.transport._master.pid + first = one.transport._master + second = two.transport._master + assert second.poll() is not None + assert first.poll() is None + one.transport.fetch("item", server["base"] / "copy") + assert first.poll() is not None + + +def test_build_manifest_download_clone_workflow(ssh_server, tmp_path, monkeypatch): + root = ssh_server["root"] + (root / "nested").mkdir() + (root / "nested/item_1.dat").write_bytes(b"science") + (root / "README.md").write_bytes(b"notes") + (root / "bad_2.dat").write_bytes(b"changed") + strong = hashlib.sha256(b"science").hexdigest() + (root / "export.sha256sums").write_text( + strong + " nested/item_1.dat\n" + "a" * 64 + " bad_2.dat\n" + ) + repo = build_repo( + tmp_path / "catalog", + "lab", + [ + {"fmt": "nested/item_{i}.dat", "db": "data.tsv"}, + {"fmt": "bad_{i}.dat", "db": "bad.tsv"}, + ], + dataset_url=ssh_server["url"], + ) + assert repo.state.config["remote"] == [{"name": "origin", "url": ssh_server["url"]}] + static = next( + entry for entry in repo.state.config["data"] if entry.get("file") == "README.md" + ) + assert not any(key in static for key in ("md5", "sha1", "sha256")) + assert static.get("checksum") in (None, "unknown") + destination = tmp_path / "downloads" + files = select_download_files(repo, all_files=True) + result = download_remote_data( + repo, destination, selected_files=files, approved=True, + ) + assert result["succeeded"] == 3 + assert result["failed"] == 1 + assert (destination / "nested/item_1.dat").read_bytes() == b"science" + assert not (destination / "bad_2.dat").exists() + assert not list(destination.rglob("*.part")) + + # Python and CLI clones both use the recorded SSH data remote. + single = build_repo( + tmp_path / "single", + "lab", + [{"fmt": "nested/item_{i}.dat", "db": "data.tsv"}], + dataset_url=ssh_server["url"], + ) + # Remove the intentionally incorrect static entry before cloning. + single.state.config["data"] = [ + entry + for entry in single.state.config["data"] + if entry.get("file") != "bad_2.dat" + ] + single.dothm.dump(single.state) + single.dothm.index.add(["config.yml"]) + single.dothm.index.commit("Remove deliberately invalid test entry") + clone = Repo.clone( + str(single.dothm.path), tmp_path / "python-clone", + ) + clone.download(clone.plan_download(), approved=True) + assert (clone.worktree / "nested/item_1.dat").read_bytes() == b"science" + result = CliRunner().invoke( + hallmark, + ["clone", str(single.dothm.path), str(tmp_path / "cli-clone")], + input="y\n", + ) + assert result.exit_code == 0, result.output + monkeypatch.chdir(tmp_path / "cli-clone") + result = CliRunner().invoke(hallmark, ["download", "--all"], input="y\n") + assert result.exit_code == 0, result.output + assert (tmp_path / "cli-clone/nested/item_1.dat").read_bytes() == b"science" + + +def _wait_for_partial_file(directory, pattern, total_size, task): + """Wait for actual SFTP bytes before interrupting a transfer.""" + directory = Path(directory) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if task.done(): + task.result() # Surface a transfer error instead of a polling timeout. + pytest.fail("Transfer completed before it could be interrupted") + for path in directory.glob(pattern): + try: + size = path.stat().st_size + except FileNotFoundError: + continue + if 0 < size < total_size: + return + time.sleep(0.01) + pytest.fail("SFTP did not write a partial file before the deadline") + + +def test_real_cancel_transfer_and_preserve_other_master(ssh_server, tmp_path): + server = ssh_server + payload = b"x" * 128 * 1024 + (server["root"] / "large").write_bytes(payload) + with OperationContext(RemoteSpec.parse(server["url"])) as survivor: + survivor.transport.prepare() + with OperationContext(RemoteSpec.parse(server["url"])) as context: + context.settings = replace( + context.settings, shutdown_timeout=1, transfer_timeout=15 + ) + context.transport.prepare() + # Bound requests as well as bandwidth: a small file can otherwise + # finish inside the client's initial buffering window on macOS. + context.transport.sftp = ["sftp", "-B", "1024", "-R", "1", "-l", "8"] + target = tmp_path / "partial" + progress = [] + context.on_bytes = progress.append + with ThreadPoolExecutor(1) as pool: + task = pool.submit(context.transport.fetch, "large", target) + try: + _wait_for_partial_file(tmp_path, "partial", len(payload), task) + deadline = time.monotonic() + 2 + while not progress and time.monotonic() < deadline: + time.sleep(0.01) + assert 0 < sum(progress) < len(payload) + assert all(delta > 0 for delta in progress) + context.cancel() + with pytest.raises(TransferCancelled): + task.result(timeout=5) + finally: + context.cancel() + assert not context.transport._processes + assert survivor.transport._master.poll() is None + survivor.transport.fetch("large", tmp_path / "survivor-copy") + assert (tmp_path / "survivor-copy").read_bytes() == payload + + +@pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) +def test_sftp_only_discovers_metadata_and_fetches(ssh_server, tmp_path): + root = ssh_server["root"] + (root / "item").write_bytes(b"data") + (root / "nested").mkdir() + name = "nested/a b#?%+ü'\"[*].fits" + (root / name).write_bytes(b"science") + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + directories = [] + entries = list(context.transport.iter_entries(on_directory=directories.append)) + assert {entry.path: entry.size for entry in entries} == {"item": 4, name: 7} + assert directories == ["", "nested/"] + assert all(entry.mtime is not None for entry in entries) + assert context.read_text("item") == "data" + with pytest.raises(RemoteObjectMissing): + context.read_text("absent-config.yml") + transferred = [] + context.on_bytes = transferred.append + context.transport.fetch("item", tmp_path / "copy") + assert (tmp_path / "copy").read_bytes() == b"data" + assert sum(transferred) == 4 + + +@pytest.mark.parametrize("ssh_server", ["sftp-only"], indirect=True) +@pytest.mark.parametrize("scheme", ["ssh", "sftp"]) +def test_sftp_only_add_plans_then_requires_payload_approval( + ssh_server, tmp_path, monkeypatch, scheme, +): + root = ssh_server["root"] + (root / "nested" / "deeper").mkdir(parents=True) + (root / "nested" / "science.fits").write_bytes(b"science") + (root / "nested" / "deeper" / "skipped.fits").write_bytes(b"deep") + (root / "notes.txt").write_bytes(b"notes") + fetched = [] + fetch = SshTransport._fetch + + def record_fetch(self, path, destination, file_limit=None): + fetched.append(path) + return fetch(self, path, destination, file_limit) + + monkeypatch.setattr(SshTransport, "_fetch", record_fetch) + plans = [] + + def decline(plan): + plans.append(plan) + return False + + repo = Repo.init(tmp_path / "initialized") + url = ssh_server["url"].replace("ssh:", scheme + ":").rstrip("/") + repo.add(url + "/{group}/{name}.fits") + assert repo.commit("Catalog remote science files") + assert not any(path.is_file() for path in repo.objects.root.rglob("*")) + decline(repo.plan_download()) + assert fetched == [] + assert repo.state.data[["group", "name"]].values.tolist() == [ + ["nested", "science"]] + assert len(plans) == 1 + assert plans[0].total_bytes == 7 + assert not (repo.worktree / "nested").exists() + with pytest.raises(DownloadError, match="approval"): + repo.download(plans[0]) + assert fetched == [] + result = repo.download(plans[0], approved=True) + assert result["succeeded"] == 1 + assert fetched == ["nested/science.fits"] + assert (repo.worktree / "nested/science.fits").read_bytes() == b"science" + assert repo.status()["untracked"] == [] + + +@pytest.mark.parametrize("ssh_server", ["no-sftp"], indirect=True) +def test_disabled_subsystem_fails(ssh_server, tmp_path): + (ssh_server["root"] / "item").write_bytes(b"data") + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + with pytest.raises(DownloadError): + context.transport.fetch("item", tmp_path / "copy") + assert not (tmp_path / "copy").exists() + + +@pytest.mark.parametrize("ssh_server", ["max-one"], indirect=True) +def test_local_session_limit(ssh_server, tmp_path, monkeypatch): + for i in range(4): + (ssh_server["root"] / str(i)).write_bytes(b"data") + from hallmark.transport.settings import SSHSettings + monkeypatch.setattr("hallmark.transport.SSHSettings", + lambda: SSHSettings(max_sessions=1)) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=ssh_server["url"]) + result = download_remote_data( + repo, + repo.worktree, + max_workers=4, + selected_files=[(Path(str(i)), None) for i in range(4)], + approved=True) + assert result["succeeded"] == 4 + assert result["failed"] == 0 + + +def test_ssh_config_source_recording_and_explicit_output( + ssh_server, tmp_path, monkeypatch +): + auth = tmp_path / "auth.yml" + auth.write_text("version: 1\nprofiles:\n lab:\n hosts: [hm-test]\n") + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(auth)) + repo = build_repo( + tmp_path / "catalog", + "label", + [], + dataset_url=ssh_server["url"], + ) + assert "auth" not in repo.state.config["remote"][0] + explicit = build_repo( + tmp_path / "explicit", + "label", + [], + remotes=[{"name": "mirror", "url": "https://example.test/data"}], + dataset_url=ssh_server["url"], + ) + assert explicit.state.config["remote"] == [ + {"name": "mirror", "url": "https://example.test/data"} + ] + + +def test_ssh_listing_omits_symlinks_and_rejects_controls(ssh_server, tmp_path): + root = ssh_server["root"] + (root / "item").write_bytes(b"data") + (root / "link").symlink_to(root / "item") + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + assert context.transport.list_entries() == ["item"] + (root / "bad\nname").write_bytes(b"data") + with OperationContext(RemoteSpec.parse(ssh_server["url"])) as context: + with pytest.raises(DownloadError, match="control"): + context.transport.list_entries() + + +def test_permission_denied_preserves_existing_file(ssh_server, tmp_path): + source = ssh_server["root"] / "private" + source.write_bytes(b"unreadable") + source.chmod(0) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=ssh_server["url"]) + destination = repo.worktree / "private" + destination.write_bytes(b"original") + try: + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("private"), None)] + , approved=True) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + finally: + source.chmod(0o600) + + +def test_disconnected_master_cleans_partial_transfer(ssh_server, tmp_path, monkeypatch): + payload = b"x" * 128 * 1024 + (ssh_server["root"] / "large").write_bytes(payload) + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url=ssh_server["url"]) + destination = repo.worktree / "large" + destination.write_bytes(b"original") + transports = [] + prepare = SshTransport.prepare + + def slow_prepare(self): + prepare(self) + self.context.settings = replace( + self.context.settings, shutdown_timeout=1, transfer_timeout=15 + ) + self.sftp = ["sftp", "-B", "1024", "-R", "1", "-l", "8"] + if self not in transports: + transports.append(self) + + monkeypatch.setattr(SshTransport, "prepare", slow_prepare) + with ThreadPoolExecutor(1) as pool: + task = pool.submit( + download_remote_data, + repo, + repo.worktree, + selected_files=[(Path("large"), None)], + approved=True) + try: + _wait_for_partial_file(repo.worktree, "*.part", len(payload), task) + assert len(transports) == 1 + transports[0]._master.terminate() + result = task.result(timeout=5) + finally: + for transport in transports: + transport.context.cancel() + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + assert not transports[0]._processes diff --git a/test/test_transport.py b/test/test_transport.py new file mode 100644 index 0000000..5093c15 --- /dev/null +++ b/test/test_transport.py @@ -0,0 +1,671 @@ +"""Transport boundaries, local policy, HTTP compatibility and process ownership.""" + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import errno +import hashlib +from pathlib import Path +import signal +import subprocess +import sys +import time +import traceback +from urllib.parse import quote + +import pandas as pd +import pytest +import requests + +from hallmark import Repo +from hallmark.downloader import _download_file, download_remote_data +from hallmark.repo_builder import _manifest_matches, build_repo +from hallmark.repo_config import normalize_remotes +from hallmark.transport import OperationContext, RemoteSpec +from hallmark.transport.base import ( + CapabilityError, + DownloadError, + RemoteConfigurationError, + TransferCancelled, + literal_path, +) +from hallmark.transport.ssh import SshTransport, batch_argument + + +@pytest.fixture(autouse=True) +def isolated_auth(monkeypatch, tmp_path): + """Isolate local authentication settings from the user environment.""" + monkeypatch.delenv("HALLMARK_AUTH_FILE", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config")) + + +@pytest.mark.parametrize("scheme", ["ssh", "sftp"]) +@pytest.mark.parametrize( + "name", + [ + "has space#tag.h5", + "a?b%+ü.h5", + "quotes'\"[]*.h5", + "-weird.fits", + "literal%20.h5", + " leading ", + ], +) +def test_literal_url_round_trip(scheme, name): + remote = RemoteSpec.parse(f"{scheme}://user@campus:2222/srv/a%20b/") + url = remote.file_url(name) + assert RemoteSpec.parse(url).root == "/srv/a b/" + name + assert remote.pathname(name) == "/srv/a b/" + name + assert quote(name, safe="/") in url + + +@pytest.mark.parametrize( + "url", + [ + "campus:/data", + "file:///etc/passwd", + "ssh:///data", + "ssh://-o/data", + "ssh://u:secret@campus/data", + "ssh://campus/data?secret", + "ssh://campus/data#x", + "ssh://campus/../data", + "ssh://campus/%2e%2e/data", + "ssh://campus", + "ssh://campus:0/data", + "ssh://campus:65536/data", + "ssh://campus:/data", + "ssh://campus/%0adata", + "ssh://campus/d\nata", + "\tssh://campus/data", + "ssh://u%24%28id%29@campus/data", + "ssh://campus$(id)/data", + "ssh://[::1/data", + "ssh://campus/%xx", + "ssh://campus/%ff", + "ssh://campus//data", + "ssh://campus/data\\file", + ], +) +def test_bad_remote_fails_without_echoing_secrets(url): + with pytest.raises(RemoteConfigurationError) as error: + RemoteSpec.parse(url) + assert "secret" not in str(error.value) + + +@pytest.mark.parametrize("host", ["campus_alias", "a.b-c", "[::1]", "[2001:db8::1]"]) +def test_hosts_and_aliases(host): + assert RemoteSpec.parse(f"ssh://{host}/").host == host.strip("[]") + + +@pytest.mark.parametrize("path", ["bad\nget /secret /tmp/leak", "a\tb", "a\rb", "a\0b"]) +def test_controls_rejected_at_all_path_boundaries(path, tmp_path): + with pytest.raises(RemoteConfigurationError): + literal_path(path) + with pytest.raises(RemoteConfigurationError): + batch_argument("/" + path) + with pytest.raises(RemoteConfigurationError): + _download_file("ssh://campus/" + path, tmp_path / "out") + + +def test_obsolete_auth_files_and_environment_are_ignored(monkeypatch, tmp_path): + folder = tmp_path / "config" / "hallmark" + folder.mkdir(parents=True) + obsolete = folder / "auth.yml" + obsolete.write_text("invalid: [yaml") + monkeypatch.setenv("HALLMARK_AUTH_FILE", str(obsolete)) + with OperationContext(RemoteSpec.parse("ssh://carol@campus:2222/data")) as ctx: + options = ctx.transport._options() + assert "User=carol" in options + assert "Port=2222" in options + assert "-i" not in options + with OperationContext(RemoteSpec.parse("sftp://campus/data")) as ctx: + options = ctx.transport._options() + assert not any(option.startswith(("User=", "Port=")) for option in options) + assert "-i" not in options + assert obsolete.read_text() == "invalid: [yaml" + + +@pytest.mark.parametrize("auth", ["lab", None, 3]) +def test_stored_auth_profiles_report_migration(auth): + with pytest.raises(ValueError, match="~/.ssh/config"): + normalize_remotes({"url": "ssh://campus/data", "auth": auth}) + + +@pytest.mark.parametrize( + "entry", + [ + {"url": "ssh://campus/data", "token": "secret"}, + {"url": "ssh://campus/data", "auth": 3}, + ], +) +def test_remote_schema(entry): + with pytest.raises(ValueError): + normalize_remotes(entry) + + +def test_netrc_is_preserved(monkeypatch, tmp_path): + netrc = tmp_path / "netrc" + netrc.write_text("machine example.test login scientist password test-value\n") + monkeypatch.setenv("NETRC", str(netrc)) + with OperationContext(RemoteSpec.parse("https://example.test/data")) as context: + prepared = context.session().prepare_request( + requests.Request("GET", "https://example.test/data/file") + ) + assert prepared.headers["Authorization"].startswith("Basic ") + assert context.session() is context.session() + + +def test_http_error_redacts_exception_and_cause(monkeypatch, tmp_path): + def fail(*args, **kwargs): + raise requests.ConnectionError("https://user:secret@example.test/?token=secret") + + monkeypatch.setattr(requests, "get", fail) + url = "https://user:secret@example.test/data?token=secret" + with pytest.raises(DownloadError) as error: + _download_file(url, tmp_path / "out") + formatted = traceback.format_exception( + type(error.value), error.value, error.value.__traceback__ + ) + assert "secret" not in "".join(formatted) + assert "secret" not in repr(error.value) + + +@pytest.mark.parametrize("location", [ + "https://other.test/data/", "https://example.test/outside/", + "http://example.test/data/", "/data/%2e%2e/outside/", + "https://new:secret@example.test/data/", "https://example.test:444/data/", +]) +def test_metadata_redirect_never_requests_outside_root(monkeypatch, location): + calls = [] + closed = [] + with OperationContext(RemoteSpec.parse("https://example.test/data/")) as context: + response = requests.Response() + response.status_code = 302 + response.headers["Location"] = location + response.close = lambda: closed.append(True) + + def get(url, **kwargs): + calls.append(url) + assert kwargs["allow_redirects"] is False + return response + + monkeypatch.setattr(context.session(), "get", get) + with pytest.raises(DownloadError, match="source root"): + context.read_text("") + assert calls == ["https://example.test/data/"] + assert closed == [True] + + +def test_metadata_redirect_within_root_and_default_port_is_supported(monkeypatch): + calls = [] + with OperationContext(RemoteSpec.parse("https://example.test/data/")) as context: + first = requests.Response() + first.status_code = 301 + first.headers["Location"] = "https://example.test:443/data/listing/" + first.close = lambda: None + second = requests.Response() + second.status_code = 200 + second.encoding = "utf-8" + second._content = b"

Index of data

" + second._content_consumed = True + + def get(url, **kwargs): + calls.append(url) + assert kwargs["allow_redirects"] is False + return first if len(calls) == 1 else second + + monkeypatch.setattr(context.session(), "get", get) + assert context.read_text("") == "

Index of data

" + assert context.transport.text_urls[""] == ( + "https://example.test:443/data/listing/") + assert calls == ["https://example.test/data/", + "https://example.test:443/data/listing/"] + + +def test_metadata_redirect_loop_is_bounded(monkeypatch): + calls = [] + with OperationContext(RemoteSpec.parse("https://example.test/data/")) as context: + response = requests.Response() + response.status_code = 307 + response.headers["Location"] = "/data/" + response.close = lambda: None + + def get(url, **kwargs): + calls.append(url) + return response + + monkeypatch.setattr(context.session(), "get", get) + with pytest.raises(DownloadError, match="redirect limit"): + context.read_text("") + assert len(calls) == 11 + + +def test_symlink_swap_after_planning(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="https://example.test/") + folder = repo.worktree / "sub" + folder.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + + def swap(self): + folder.rmdir() + folder.symlink_to(outside, target_is_directory=True) + + monkeypatch.setattr("hallmark.transport.http.HttpTransport.prepare", swap) + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("sub/data"), None)] + , approved=True) + assert result["failed"] == 1 + assert list(outside.iterdir()) == [] + + +@pytest.mark.parametrize("scheme", ["https", "ssh"]) +def test_builder_discovers_without_backend_flags(monkeypatch, tmp_path, scheme): + calls = [] + monkeypatch.setattr( + "hallmark.repo_builder._build_repo", lambda *a, **kw: calls.append((a, kw)) + ) + build_repo(tmp_path / "repo", "lab", [], dataset_url=f"{scheme}://unused/data") + assert len(calls) == 1 + assert not (tmp_path / "repo").exists() + + +def test_manifest_preserves_whitespace_and_percent(): + text = "a" * 64 + " leading%20 name \n" + assert _manifest_matches(text, "sha256") == [("a" * 64, " leading%20 name ")] + with pytest.raises(ValueError, match="escaped"): + _manifest_matches("\\" + "a" * 64 + " line\\nbreak\n", "sha256") + + +@pytest.mark.ssh_client +@pytest.mark.parametrize( + "name", + ["a b#?%+ü.h5", "quotes'\"[x]*?.h5", "-leading.h5", " tail ", "literal%20.h5"], +) +def test_real_sftp_parser_without_network(tmp_path, name): + import shutil + + server = next( + ( + p + for p in ["/usr/lib/openssh/sftp-server", "/usr/libexec/sftp-server"] + if Path(p).exists() + ), + None, + ) + if not server or not shutil.which("sftp"): + pytest.skip("OpenSSH client and local SFTP server binary required") + source = tmp_path / name + source.write_text(name) + output = tmp_path / ("output-" + name) + # A glob collision must never produce two matches or replace another file. + (tmp_path / "a b#X%+ü.h5").write_text("decoy") + batch = f"get {batch_argument(source)} {batch_argument(output)}\n" + result = subprocess.run( + ["sftp", "-D", server, "-b", "-"], + input=batch.encode(), + capture_output=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + assert output.read_bytes() == source.read_bytes() + + +@pytest.fixture +def fake_process(tmp_path): + """Create a local command for testing SSH process failures.""" + script = tmp_path / "fake.py" + script.write_text("""import os, signal, subprocess, sys, time +mode = sys.argv[1] +if mode == "fail": + os.write(2, b"secret-invalid-utf8-\\xff") + sys.exit(1) +elif mode == "flood": + os.write(2, b"x" * 100000) +elif mode == "partial": + open(sys.argv[2], "wb").write(b"partial") + sys.exit(1) +elif mode == "sleep": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + open(sys.argv[2], "w").write(str(os.getpid()) + " " + str(child.pid)) + time.sleep(60) +else: + raise RuntimeError("Unexpected fake request") +""") + return [sys.executable, str(script)] + + +@pytest.mark.parametrize( + "mode, message", [("fail", "SSH operation failed"), ("flood", "size limit")] +) +def test_bounded_process_errors(fake_process, mode, message): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + transport = context.transport + with pytest.raises(DownloadError, match=message) as error: + transport._run(fake_process + [mode], timeout=5) + assert "secret" not in str(error.value) + assert not transport._processes + + +def test_process_start_failure(): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + with pytest.raises(DownloadError, match="Unable to start"): + context.transport._run(["/nonexistent/hallmark-test"], timeout=1) + + +@pytest.mark.parametrize("reaping_in_progress", [False, True]) +def test_cleanup_exited_process_group_permission_error( + monkeypatch, reaping_in_progress +): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + transport = context.transport + process = transport._spawn([sys.executable, "-c", "pass"]) + process.wait(timeout=5) + signals = [] + + def zombie_group(pid, signum): + assert pid == process.pid + signals.append(signum) + raise PermissionError(errno.EPERM, "Operation not permitted") + + with monkeypatch.context() as patch: + patch.setattr("hallmark.transport.ssh.os.killpg", zombie_group) + if reaping_in_progress: + # Popen.poll() cannot acquire its wait lock while another + # thread reaps the child and can temporarily report None. + patch.setattr(process, "poll", lambda: None) + transport._stop(process) + # Exited leaders may still have proxy children: attempt both signals. + assert signals == [signal.SIGTERM, signal.SIGKILL] + assert process.returncode == 0 + assert not transport._processes + + +def test_cleanup_live_process_permission_error_is_not_suppressed(monkeypatch): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + transport = context.transport + process = transport._spawn( + [sys.executable, "-c", "import time; time.sleep(60)"] + ) + + def denied(pid, signum): + raise PermissionError(errno.EPERM, "Operation not permitted") + + with monkeypatch.context() as patch: + patch.setattr("hallmark.transport.ssh.os.killpg", denied) + with pytest.raises(PermissionError): + transport._stop(process) + assert process.poll() is None + assert process in transport._processes + # Restore real signals before the context closes and reaps this child. + assert process.poll() is not None + assert not transport._processes + + +def test_cancel_active_process_group(fake_process, tmp_path): + pids = tmp_path / "pids" + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + context.settings = replace(context.settings, shutdown_timeout=1) + with ThreadPoolExecutor(1) as pool: + future = pool.submit( + context.transport._run, fake_process + ["sleep", str(pids)], timeout=30 + ) + deadline = time.monotonic() + 5 + while not pids.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert pids.exists() + started = time.monotonic() + context.cancel() + with pytest.raises(TransferCancelled): + future.result(timeout=5) + assert time.monotonic() - started < 4 + assert not context.transport._processes + for pid in map(int, pids.read_text().split()): + proc = Path(f"/proc/{pid}/stat") + assert not proc.exists() or proc.read_text().split()[2] == "Z" + + +def test_partial_failure_preserves_destination(monkeypatch, fake_process, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + destination = repo.worktree / "data.bin" + destination.write_bytes(b"original") + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + + def fetch(self, path, destination, **kwargs): + self._run(fake_process + ["partial", str(destination)], timeout=5) + + monkeypatch.setattr(SshTransport, "fetch", fetch) + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("data.bin"), None)] + , approved=True) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + + +def test_checksum_failure_is_atomic(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + destination = repo.worktree / "data.bin" + destination.write_bytes(b"original") + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + monkeypatch.setattr( + SshTransport, "fetch", lambda self, path, dest, **kw: dest.write_bytes(b"wrong") + ) + result = download_remote_data( + repo, + repo.worktree, + selected_files=[ + (Path("data.bin"), ("sha256", hashlib.sha256(b"right").hexdigest())) + ], + approved=True) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + + +def test_cancel_before_publish(monkeypatch, tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + destination = repo.worktree / "item" + destination.write_bytes(b"original") + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + + def fetch(self, path, dest, **kwargs): + dest.write_bytes(b"complete") + self.context.cancel() + + monkeypatch.setattr(SshTransport, "fetch", fetch) + result = download_remote_data( + repo, repo.worktree, selected_files=[(Path("item"), None)] + , approved=True) + assert result["failed"] == 1 + assert destination.read_bytes() == b"original" + assert not list(Path(repo.worktree).glob("*.part")) + + +def test_keyboard_interrupt_stops_owned_workers(monkeypatch, tmp_path, fake_process): + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://unused/data") + transports = [] + ready = tmp_path / "pids" + monkeypatch.setattr(SshTransport, "prepare", lambda self: None) + + def fetch(self, path, destination, **kwargs): + transports.append(self) + destination.write_bytes(b"partial") + self._run(fake_process + ["sleep", str(ready)], timeout=30) + + def interrupt(*args, **kwargs): + deadline = time.monotonic() + 5 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert ready.exists() + raise KeyboardInterrupt + + monkeypatch.setattr(SshTransport, "fetch", fetch) + monkeypatch.setattr("hallmark.downloader.wait", interrupt) + with pytest.raises(KeyboardInterrupt): + download_remote_data( + repo, repo.worktree, max_workers=1, selected_files=[(Path("item"), None)] + , approved=True) + assert not (repo.worktree / "item").exists() + assert not list(Path(repo.worktree).glob("*.part")) + assert all(not transport._processes for transport in transports) + + +def test_cli_dry_run_does_not_resolve_auth_or_start_clients(monkeypatch, tmp_path): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + repo.set_config(remote_url="ssh://campus/data") + repo.state.data = pd.DataFrame({"path": ["item.dat"]}) + repo.dothm.dump(repo.state) + monkeypatch.chdir(repo.worktree) + + def fail(*args, **kwargs): + raise AssertionError("Dry-run must not create an operation context") + + monkeypatch.setattr("hallmark.downloader.OperationContext", fail) + result = CliRunner().invoke(hallmark, ["download", "item.dat", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "item.dat" in result.output + + +def test_cli_rejects_removed_auth_flag(monkeypatch, tmp_path): + from click.testing import CliRunner + from hallmark.cli import hallmark + + repo = Repo.init(tmp_path / "repo") + monkeypatch.chdir(repo.worktree) + result = CliRunner().invoke(hallmark, ["set-config", "--remote-auth", "lab"]) + assert result.exit_code == 2 + assert "No such option" in result.output + + +def test_http_manifest_auth_failure_is_not_optional(monkeypatch): + from hallmark.repo_builder import list_remote_files + from mock_server import MockServer + + server = MockServer("https://example.test/data/") + server.add_directory("", [("data-object", "export.sha256sums")]) + get = server.get + + def unauthorized(url, **kwargs): + if url.endswith("sha256sums"): + response = requests.Response() + response.status_code = 401 + raise requests.HTTPError("credentials-must-not-leak", response=response) + return get(url, **kwargs) + + server.get = unauthorized + monkeypatch.setattr(requests, "Session", lambda: server) + with pytest.raises(DownloadError, match="HTTP 401"): + list_remote_files(server.base_url) + + +@pytest.mark.parametrize( + "url", + [ + "ssh://[fe80::1%25eth0]/data", + "ssh://[fe80::1%25$(id)]/data", + "ssh://campus/bad\udcff", + ], +) +def test_reject_scoped_ipv6_and_invalid_unicode(url): + with pytest.raises(RemoteConfigurationError): + RemoteSpec.parse(url) + + +def test_minimum_openssh_is_checked_before_connection(monkeypatch): + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + monkeypatch.setattr("hallmark.transport.ssh.shutil.which", lambda name: name) + monkeypatch.setattr( + context.transport, "_run", lambda *a, **kw: b"OpenSSH_9.5p1" + ) + with pytest.raises(CapabilityError, match="9.6"): + context.transport.prepare() + assert context.transport._master is None + assert context.transport._socket_dir is None + + +def test_short_socket_ignores_long_tempdir(monkeypatch, tmp_path): + # Long macOS TMPDIR/worktree paths can exceed AF_UNIX limits. + monkeypatch.setenv("TMPDIR", str(tmp_path / ("long" * 35))) + with OperationContext(RemoteSpec.parse("ssh://unused/data")) as context: + monkeypatch.setattr( + context.transport, "_run", lambda *a, **kw: b"OpenSSH_9.6p1" + ) + monkeypatch.setattr("hallmark.transport.ssh.shutil.which", lambda name: name) + + def fail_start(*args, **kwargs): + assert len(context.transport._socket.encode()) <= 100 + assert context.transport._socket.startswith("/tmp/hm-") + raise DownloadError("deliberate startup failure") + + monkeypatch.setattr(context.transport, "_spawn", fail_start) + with pytest.raises(DownloadError, match="deliberate"): + context.transport.prepare() + assert context.transport._socket_dir is None + + +def test_manifest_strength_and_conflict(): + from hallmark.repo_builder import _record_checksum + + checksums = {} + _record_checksum(checksums, "file", "sha256", "a" * 64) + _record_checksum(checksums, "file", "md5", "b" * 32) + assert checksums["file"] == ("sha256", "a" * 64) + with pytest.raises(DownloadError, match="Conflicting"): + _record_checksum(checksums, "file", "sha256", "c" * 64) + + +def test_explicit_http_source_is_exact_and_keeps_output_remotes(monkeypatch, tmp_path): + from mock_server import MockServer + + server = MockServer("https://example.test/already-the-dataset/") + server.add_directory("", [("data-object", "README.md")]) + server.add_file("README.md", b"notes") + monkeypatch.setattr(requests, "Session", lambda: server) + repo = build_repo( + tmp_path / "catalog", + "label", + [], + dataset_url=server.base_url, + index_format="cyverse-html", + remotes=[{"name": "mirror", "url": "https://elsewhere.test/data"}], + ) + assert repo.state.config["remote"][0]["url"] == "https://elsewhere.test/data" + entry = repo.state.config["data"][0] + assert not any(key in entry for key in ("md5", "sha1", "sha256")) + assert entry.get("checksum") in (None, "unknown") + + +def test_build_cli_passes_source_controls(monkeypatch, tmp_path): + from click.testing import CliRunner + from hallmark.cli import hallmark + + calls = [] + monkeypatch.setattr("hallmark.cli.build_repo", lambda **kw: calls.append(kw)) + result = CliRunner().invoke( + hallmark, + [ + "build", + str(tmp_path), + "lab", + "--fmt", + "run_{i}.h5=data.tsv", + "--dataset-url", + "ssh://campus/data", + "--allow-remote-commands", + "--remote-hash", + ], + ) + assert result.exit_code == 0, result.output + assert calls[0]["dataset_url"] == "ssh://campus/data" + assert calls[0]["allow_remote_commands"] is True + assert calls[0]["remote_hash"] is True