ENH: Added utility functions to create frozen datasets

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
 
     steps:
     - name: Git checkout

CHANGELOG.rst

@@ -4,6 +4,39 @@ CHANGELOG
 This project uses `semantic versioning <http://semver.org/>`_.
 This change log uses principles from `keep a changelog <http://keepachangelog.com/>`_.
 
+[3.20.0] - 2025-12-08
+---------------------
+
+
+Added
+^^^^^
+
+- ``freeze_with_manifest`` method on ``ProtoDataSet`` class for converting a
+  proto-dataset to a frozen dataset using a pre-computed manifest with
+  client-provided hashes. Validates that README and all manifest items exist
+  in storage before freezing. Useful for server applications that trust
+  client-computed hashes.
+
+Changed
+^^^^^^^
+
+- changed build system to ``flit``
+
+Deprecated
+^^^^^^^^^^
+
+
+Removed
+^^^^^^^
+
+
+Fixed
+^^^^^
+
+
+Security
+^^^^^^^^
+
 
 [3.19.0] - 2024-12-16
 ---------------------

dtoolcore/__init__.py

@@ -60,16 +60,12 @@ def _generate_storage_broker_lookup():
     logger.debug("In _generate_storage_broker_lookup...")
     storage_broker_lookup = dict()
 
-    if sys.version_info >= (3, 8):
-        from importlib.metadata import entry_points
-        eps = entry_points()
-        if sys.version_info >= (3, 10):
-            entrypoints = eps.select(group="dtool.storage_brokers")
-        else:
-            entrypoints = eps.get("dtool.storage_brokers", [])
+    from importlib.metadata import entry_points
+    eps = entry_points()
+    if sys.version_info >= (3, 10):
+        entrypoints = eps.select(group="dtool.storage_brokers")
     else:
-        from pkg_resources import iter_entry_points
-        entrypoints = iter_entry_points("dtool.storage_brokers")
+        entrypoints = eps.get("dtool.storage_brokers", [])
 
     for entrypoint in entrypoints:
         StorageBroker = entrypoint.load()
@@ -164,8 +160,13 @@ def generate_proto_dataset(admin_metadata, base_uri, config_path=None):
     :param admin_metadata: dataset administrative metadata
     :param base_uri: base URI for proto dataset
     :param config_path: path to dtool configuration file
+    :raises: DtoolCoreInvalidNameError if the name in the administrative
+             metadata is missing or invalid
     """
     logger.debug("In generate_proto_dataset...")
+    name = admin_metadata.get("name")
+    if name is None or not dtoolcore.utils.name_is_valid(name):
+        raise(DtoolCoreInvalidNameError())
     uri = _generate_uri(admin_metadata, base_uri)
     return ProtoDataSet(uri, admin_metadata, config_path)
 
@@ -356,6 +357,11 @@ def copy_resume(src_uri, dest_base_uri, config_path=None, progressbar=None):
     # the "frozen_at" timestamp, thus this property must be reassigned when
     # resuming the transfer process.
     # to, 2021/09/27: Your understanding is correct and we don't need it.
+    if "frozen_at" not in dataset._admin_metadata:
+        raise DtoolCoreValueError(
+            "Source dataset is missing 'frozen_at' in its administrative "
+            "metadata; cannot resume copy from {}".format(src_uri)
+        )
     proto_dataset._admin_metadata["frozen_at"] = dataset._admin_metadata["frozen_at"]  # NOQA
     proto_dataset.freeze(progressbar=progressbar)
 
@@ -858,6 +864,110 @@ def freeze(self, progressbar=None):
         # Clean up using the storage broker's post freeze hook.
         self._storage_broker.post_freeze_hook()
 
+    def freeze_with_manifest(self, manifest, frozen_at=None):
+        """
+        Convert :class:`dtoolcore.ProtoDataSet` to :class:`dtoolcore.DataSet`
+        using a pre-computed manifest.
+
+        This method freezes the dataset without computing hashes server-side.
+        The caller provides a manifest with pre-computed item properties
+        (hash, size, timestamp). This is useful for server-side operations
+        where the client has already computed hashes during upload.
+
+        Before freezing, this method validates that:
+        - The README file exists in storage
+        - All items listed in the manifest exist in storage
+
+        Note: This method does NOT verify that the hashes match - it trusts
+        the client-provided hashes in the manifest.
+
+        :param manifest: dict with structure::
+
+            {
+                "dtoolcore_version": <version>,
+                "hash_function": <hash_function_name>,
+                "items": {
+                    <identifier>: {
+                        "relpath": <path>,
+                        "size_in_bytes": <int>,
+                        "hash": <hash_string>,
+                        "utc_timestamp": <float>
+                    }
+                }
+            }
+
+        :param frozen_at: optional timestamp for when the dataset was frozen.
+            If not provided, uses the current UTC time.
+        :raises: DtoolCoreValueError if README or any manifest item is missing
+        """
+        logger.debug("Freeze dataset with manifest {}".format(self))
+
+        # Validate that README exists
+        try:
+            self._storage_broker.get_readme_content()
+        except Exception as e:
+            raise DtoolCoreValueError(
+                f"README file is missing or cannot be read: {e}"
+            )
+
+        # Validate that all items in the manifest exist in storage
+        manifest_items = manifest.get("items", {})
+        if manifest_items:
+            # Get identifiers of items that actually exist in storage
+            existing_handles = set(self._storage_broker.iter_item_handles())
+            existing_identifiers = set(
+                dtoolcore.utils.generate_identifier(h) for h in existing_handles
+            )
+
+            # Check for missing items
+            expected_identifiers = set(manifest_items.keys())
+            missing_identifiers = expected_identifiers - existing_identifiers
+
+            if missing_identifiers:
+                # Get relpaths of missing items for better error message
+                missing_relpaths = [
+                    manifest_items[ident].get("relpath", ident)
+                    for ident in list(missing_identifiers)[:5]  # Limit to 5
+                ]
+                if len(missing_identifiers) > 5:
+                    missing_relpaths.append(
+                        f"... and {len(missing_identifiers) - 5} more"
+                    )
+                raise DtoolCoreValueError(
+                    f"Missing {len(missing_identifiers)} item(s) in storage: "
+                    f"{missing_relpaths}"
+                )
+
+        # Call the storage broker pre_freeze hook.
+        self._storage_broker.pre_freeze_hook()
+
+        # Use provided manifest instead of computing
+        self._storage_broker.put_manifest(manifest)
+
+        # Generate and persist overlays from any item metadata that has been
+        # added.
+        overlays = self._generate_overlays()
+        for overlay_name, overlay in overlays.items():
+            self._put_overlay(overlay_name, overlay)
+
+        # Change the type of the dataset from "protodataset" to "dataset"
+        # in the administrative metadata.
+        metadata_update = {"type": "dataset"}
+
+        # Use provided frozen_at or generate one
+        if frozen_at is not None:
+            metadata_update["frozen_at"] = frozen_at
+        elif "frozen_at" not in self._admin_metadata:
+            datetime_obj = datetime.datetime.utcnow()
+            metadata_update["frozen_at"] = dtoolcore.utils.timestamp(datetime_obj)
+
+        # Apply the change(s) to the administrative metadata.
+        self._admin_metadata.update(metadata_update)
+        self._storage_broker.put_admin_metadata(self._admin_metadata)
+
+        # Clean up using the storage broker's post freeze hook.
+        self._storage_broker.post_freeze_hook()
+
 
 class DataSetCreator(object):
     """Context manager for creating a dataset.

pyproject.toml

@@ -1,17 +1,18 @@
 [build-system]
-requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"]
-build-backend = "setuptools.build_meta"
+requires = ["flit_scm"]
+build-backend = "flit_scm:buildapi"
 
 [project]
 name = "dtoolcore"
 description = "Core API for managing (scientific) data"
 readme = "README.rst"
-license = {file = "LICENSE"}
+license = {text = "MIT"}
 authors = [
     {name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"}
 ]
 dynamic = ["version"]
-dependencies = ["setuptools"]
+requires-python = ">=3.9"
+dependencies = []
 
 [project.optional-dependencies]
 test = [
@@ -29,13 +30,20 @@ Documentation = "https://dtoolcore.readthedocs.io"
 Repository = "https://github.com/jic-dtool/dtoolcore"
 Changelog = "https://github.com/jic-dtool/dtoolcore/blob/master/CHANGELOG.rst"
 
+[project.entry-points."dtool.storage_brokers"]
+DiskStorageBroker = "dtoolcore.storagebroker:DiskStorageBroker"
+
+[tool.flit.module]
+name = "dtoolcore"
+
 [tool.setuptools_scm]
 version_scheme = "guess-next-dev"
 local_scheme = "no-local-version"
 write_to = "dtoolcore/version.py"
 
-[tool.setuptools]
-packages = ["dtoolcore"]
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=dtoolcore --cov-report=term-missing"
 
-[project.entry-points."dtool.storage_brokers"]
-"DiskStorageBroker" = "dtoolcore.storagebroker:DiskStorageBroker"
+[tool.flake8]
+exclude = ["env*", ".tox", ".git", "*.egg", "build", "docs", "venv"]

tests/test_audit_fixes.py

@@ -0,0 +1,118 @@
+"""Tests for issues identified in the 2026-06 stack audit.
+
+- generate_proto_dataset must validate the dataset name in custom
+  admin metadata.
+- copy_resume must give a helpful error when the source dataset is
+  missing the frozen_at admin metadata, instead of a raw KeyError.
+"""
+
+import json
+import os
+
+import pytest
+
+from . import tmp_dir_fixture  # NOQA
+
+from dtoolcore.utils import (
+    IS_WINDOWS,
+    generous_parse_uri,
+    windows_to_unix_path,
+)
+
+
+def _sanitise_base_uri(tmp_dir):
+    base_uri = tmp_dir
+    if IS_WINDOWS:
+        parsed_base_uri = generous_parse_uri(tmp_dir)
+        unix_path = windows_to_unix_path(parsed_base_uri.path)
+        base_uri = "file://{}".format(unix_path)
+    return base_uri
+
+
+def test_generate_proto_dataset_rejects_invalid_name(tmp_dir_fixture):  # NOQA
+    import dtoolcore
+
+    base_uri = _sanitise_base_uri(tmp_dir_fixture)
+
+    admin_metadata = dtoolcore.generate_admin_metadata(
+        "valid-name", creator_username="tester")
+    admin_metadata["name"] = "INVALID!!!NAME"
+
+    with pytest.raises(dtoolcore.DtoolCoreInvalidNameError):
+        dtoolcore.generate_proto_dataset(
+            admin_metadata=admin_metadata,
+            base_uri=base_uri,
+        )
+
+
+def test_generate_proto_dataset_rejects_missing_name(tmp_dir_fixture):  # NOQA
+    import dtoolcore
+
+    base_uri = _sanitise_base_uri(tmp_dir_fixture)
+
+    admin_metadata = dtoolcore.generate_admin_metadata(
+        "valid-name", creator_username="tester")
+    del admin_metadata["name"]
+
+    with pytest.raises(dtoolcore.DtoolCoreInvalidNameError):
+        dtoolcore.generate_proto_dataset(
+            admin_metadata=admin_metadata,
+            base_uri=base_uri,
+        )
+
+
+def test_generate_proto_dataset_accepts_valid_name(tmp_dir_fixture):  # NOQA
+    import dtoolcore
+
+    base_uri = _sanitise_base_uri(tmp_dir_fixture)
+
+    admin_metadata = dtoolcore.generate_admin_metadata(
+        "valid-name", creator_username="tester")
+
+    proto_dataset = dtoolcore.generate_proto_dataset(
+        admin_metadata=admin_metadata,
+        base_uri=base_uri,
+    )
+    assert proto_dataset.name == "valid-name"
+
+
+def test_copy_resume_missing_frozen_at_gives_helpful_error(tmp_dir_fixture):  # NOQA
+    import dtoolcore
+
+    base_uri = _sanitise_base_uri(tmp_dir_fixture)
+
+    src_dir = os.path.join(tmp_dir_fixture, "src")
+    dest_dir = os.path.join(tmp_dir_fixture, "dest")
+    os.mkdir(src_dir)
+    os.mkdir(dest_dir)
+    src_base_uri = _sanitise_base_uri(src_dir)
+    dest_base_uri = _sanitise_base_uri(dest_dir)
+
+    # Create and freeze a source dataset.
+    proto = dtoolcore.create_proto_dataset(
+        name="src-dataset",
+        base_uri=src_base_uri,
+        creator_username="tester",
+    )
+    proto.freeze()
+
+    # Simulate an interrupted copy: destination proto dataset exists.
+    src_dataset = dtoolcore.DataSet.from_uri(proto.uri)
+    dtoolcore._copy_create_proto_dataset(src_dataset, dest_base_uri)
+
+    # Corrupt the source admin metadata by removing frozen_at.
+    parsed = generous_parse_uri(proto.uri)
+    dataset_path = parsed.path
+    if IS_WINDOWS:
+        from dtoolcore.utils import unix_to_windows_path
+        dataset_path = unix_to_windows_path(dataset_path)
+    admin_metadata_fpath = os.path.join(dataset_path, ".dtool", "dtool")
+    with open(admin_metadata_fpath) as fh:
+        admin_metadata = json.load(fh)
+    del admin_metadata["frozen_at"]
+    with open(admin_metadata_fpath, "w") as fh:
+        json.dump(admin_metadata, fh)
+
+    # copy_resume should raise a descriptive error, not a raw KeyError.
+    with pytest.raises(dtoolcore.DtoolCoreValueError, match="frozen_at"):
+        dtoolcore.copy_resume(proto.uri, dest_base_uri)