From d6619dea642c51cb019716d2be726dcc65d2fcdc Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 09:13:17 -0700 Subject: [PATCH 1/6] Preliminary test framework creation --- CLAUDE.md | 116 +++ pyproject.toml | 36 +- .../pipelines/jsonlines/settings.py | 6 +- .../pipelines/ncbi_ftp_download.py | 9 +- .../pipelines/ncbi_ftp_promote.py | 8 +- src/cdm_data_loaders/pipelines/uniprot_kb.py | 4 +- tests/cdm_data_loaders/pipelines/conftest.py | 38 +- tests/cdm_data_loaders/pipelines/framework.py | 723 ++++++++++++++++ tests/cdm_data_loaders/pipelines/helpers.py | 9 +- .../jsonlines/integration/conftest.py | 23 +- .../integration/test_jsonlines_end_to_end.py | 15 +- .../test_jsonlines_reference_tests.py | 9 +- .../integration/test_jsonschema_end_to_end.py | 28 +- .../integration/test_reference_end_to_end.py | 42 +- .../jsonlines/test_extract_pipeline.py | 2 +- .../jsonlines/test_jsonschema_pipeline.py | 2 +- .../pipelines/test_framework.py | 412 +++++++++ .../pipelines/uniprot/test_uniprot.py | 8 +- .../uniprot/test_uniprot_end_to_end.py | 58 +- .../uniref/test_uniref_end_to_end.py | 63 +- .../integration/test_xml2db_end_to_end.py | 21 +- .../integration/test_xmltodict_end_to_end.py | 27 +- .../xmltodict/test_xmltodict_pipeline.py | 6 +- .../xsv/integration/test_xsv_end_to_end.py | 29 +- tests/conftest.py | 26 + uv.lock | 788 ++++++++++++------ 26 files changed, 2024 insertions(+), 484 deletions(-) create mode 100644 CLAUDE.md create mode 100644 tests/cdm_data_loaders/pipelines/framework.py create mode 100644 tests/cdm_data_loaders/pipelines/test_framework.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..d2f1c773 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,116 @@ +# CLAUDE.md + +Guidance for working in this repository. Rules with stricter, active requirements live in `.claude/rules/` (coding-style, testing, heresy) and apply to all changes. + +## Project + +`cdm-data-loaders`: data loaders and wranglers for the KBase CDM. Python >= 3.13, managed and executed with `uv`. Package source is under `src/cdm_data_loaders/` (~115 files, 8 top-level packages). Tests mirror the source layout under `tests/cdm_data_loaders/` (including pipeline end-to-end suites under `tests/cdm_data_loaders/pipelines/`), with multi-phase NCBI FTP integration tests in `tests/integration/ncbi_ftp/` and vcrpy cassettes in `tests/cassettes/`. + +Autouse fixtures in `tests/conftest.py` (`_isolated_cli_and_env` and `_isolated_dlt_config` from `tests/dlt_config_isolation.py`) reset `sys.argv = ["pytest"]`, strip `CDL_*` environment variables, and snapshot/restore `dlt.config` between tests. Pipeline end-to-end tests use `PipelineTestRunner` (`tests/cdm_data_loaders/pipelines/framework.py`) to run pipelines across `ExecutionMode.CORE`, `ExecutionMode.CLI`, and `ExecutionMode.DOCKER` (Docker tests are marked `requires_docker` and require `PIPELINE_DOCKER_IMAGE` to be set). + +Useful invocations: + +- Standard local test run (skips Spark, Ceph, and unset Docker image tests): `uv run pytest -m "not requires_spark and not requires_ceph"` +- Full pipelines suite, excluding the bulk downloader: `uv run pytest tests/cdm_data_loaders/pipelines --ignore=tests/cdm_data_loaders/pipelines/all_the_bacteria` +- Unit vs. container test script: `./scripts/run_tests.sh [unit|container|all]` +- Lint: `uv run ruff check` (line length 120, target py313, full rule selection in `pyproject.toml`). + +## Architecture + +Domain logic lives in the top-level packages and is orchestration-free; orchestration lives in `pipelines/`. Data flows source -> reader -> intermediate representation (IR) -> emitter -> target: + +- Converters: reader (source schema -> IR) + emitter (IR -> target format) + thin facade composing them. See `docs/converters_ir.md` for the full IR contract and package layout. +- IR is a frozen, slotted node tree (`converters/core/ir.py`) that preserves source facts without choosing target policy; emitters make all target-specific decisions. +- `converters/core/` imports no dlt, PyIceberg, or PySpark modules. The only cross-package import is the `x-xsv-config` contract reusing `X_XSV_CONFIG_SCHEMA` from `cdm_data_loaders.readers.jsonschema_xsv.xsv_validator.custom_metaschema`. + +Settings hierarchy: pipeline settings in `pipelines/` extend the hierarchy in `core/settings.py`: `CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`. Settings use `env_prefix="CDL_"`, `cli_kebab_case=True`, and `default_settings_with_shortcuts(...)`. `CdmDataLoadersBase.__pydantic_init_subclass__` calls `check_aliases()` at class creation time to verify that CLI argument names, aliases, and `cli_shortcuts` do not collide or claim reserved flags (`h`, `help`). CLI parsing is invoked explicitly via `pydantic_settings.CliApp.run` in `pipelines/core.py` (`PipelineCli` / `run_cli`), not implicitly on model instantiation. Every pipeline registers a `cli` entry point in `pyproject.toml` under `[project.scripts]` (and in `scripts/entrypoint.sh` for Docker execution): `all_the_bacteria`, `jsonl_extract`, `jsonl_jsonschema`, `jsonl_pydantic`, `ncbi_ftp_promote`, `ncbi_ftp_sync`, `ncbi_rest_api`, `pdb_manifest`, `uniprot`, `uniref`, `xmltodict`, `xml2db`, `xsv`. + +## Package Map + +All paths below are relative to `src/cdm_data_loaders/`. + +### `core/` + +Framework-level foundations: + +- `fields.py`: common annotated field types (`BufferSize`, `DatasetName`, `FileGlob`, `LoaderFileFormat`, `TableName`, `WriteDisposition`, etc.), `LoaderFileFormatEnum` (`parquet`, `jsonl`), and `DEFAULTS`. +- `path_fields.py`: `LocalInputDir` and `LocalOutputDir` path types (`file://` or plain local directory paths) and URL/path validation helpers (`normalize_local_path`, `validate_directory`, `validate_url_protocol`). +- `settings.py`: `pydantic-settings` base class hierarchy (`CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`), `CLI_SHORTCUTS`, `default_settings_with_shortcuts()`, and CLI flag/shortcut collision checking (`check_aliases()`). + +### `converters/` + +Schema-format translation. See `docs/converters_ir.md` for the full contract. + +- `core/`: shared, target-independent foundation + - `ir.py` (NodeType, Field, SchemaDocument, Reader/Emitter protocols), `ir_values.py` (`freeze_value` and friends), `errors.py` (ConversionError), `extensions.py` (ExtensionRegistry, DEFAULT_EXTENSIONS), `guards.py` (input guards), `inference.py` (type inference, decimal scale), `metadata.py` (draft-specific JSON Schema keyword selection), `io.py` (schema loading), `paths.py` (`set_nested`). +- `readers/`: source -> IR + - `json_schema.py` (JsonSchemaReader), `dlt.py` (DltReader over dlt's normalized table forest), `iceberg.py` (IcebergReader). +- `emitters/`: IR -> target + - `json_schema.py`, `dlt.py`, `pyspark.py`, `pyarrow.py`, `linkml.py`. +- Facades: `jsonschema_to_dlt.py`, `jsonschema_to_pyspark.py`, `jsonschema_to_pyarrow.py`, `dlt_to_jsonschema.py`, `pyiceberg_to_jsonschema.py`, `pyiceberg_to_linkml.py`, plus CLI dumps `pyiceberg_catalog_to_jsonschema.py` and `pyiceberg_catalog_to_linkml.py`. + +### `readers/` + +File-format readers feeding pipelines: + +- `jsonlines.py`: JSONL line parsing (`parse_jsonl_line`), paged streaming (`stream_jsonl_lines`), and table routing (`route_jsonl_lines`). +- `xml.py`: shared XML streaming and `xmltodict` helpers (`parse_head_matter`, `stream_xml_file`, `process_xml_file_to_dict`, `process_xml_file`, `process_xml_file_batches`, `process_xml_file_items`, and `build_xml_file_resource` over dlt's `filesystem` source). +- `xml2db_doc.py`: `xml2db` model building (`build_xml2db_model`), bounded-memory chunk fragment streaming (`iter_xml2db_chunk_fragments`), primary/foreign key rewriting (content-addressed keys for reused tables, file/chunk-namespaced keys otherwise), document flattening (`flatten_xml2db_document`), and `process_xml_file_with_xml2db`. +- `xsd.py`: XSD schema loading (`load_schema`) and parent/child element cardinality classification (`find_list_and_single_child_paths`). +- `jsonschema_xsv/`: `xsv_reader.py` (casts validated CSV/TSV rows to JSON Schema property types and resolves `x-xsv-config` parsing parameters) plus `xsv_validator/` (`qsv.py` wraps the external `qsv` binary for two-pass validation, normalization, and null replacement; `custom_metaschema.py` defines the `https://json-schema.kbase.us/draft/2020-12-xsv/schema` metaschema and `X_XSV_CONFIG_SCHEMA`, also imported by `converters`; `schema_utils.py` and `helpers.py`). + +### `parsers/` + +Parsers for third-party outputs and formats: + +- `uniprot/`: `uniprot_kb.py` (UniProt XML), `uniref.py` (UniRef XML), `metalink.py`, `relnotes.py`, `idmapping.schema.json`; shared XML helpers in `utils/xml_utils.py`. +- `ncbi/datasets_api/openapi_pydantic.py`: pydantic models for NCBI Datasets API responses (bundled `AnnotationReportResponse.schema.json` and `DatasetReportResponse.schema.json` alongside). +- `ncbi/ftp_mod_times/`: `ftp_mod_times.schema.json` for NCBI FTP modification-time files. +- `bbmap_stats.py` (bbmap `stats.sh` output), `checkm2.py` (quality reports), `pdb_holdings_file.py`. +- `all_the_bacteria/schema/` and `gtdb/schema/`: JSON Schemas for ATB and GTDB tabular files (no Python modules). + +### `ncbi_ftp/` + +NCBI FTP assembly domain logic, phased: `manifest.py` (phase 1: assembly summary diffing and manifest generation), download is the phase 2 pipeline, `promote.py` (phase 3: promote staged files to final Lakehouse paths in S3), plus `assembly.py` (assembly-specific logic), `metadata.py` (frictionless data package descriptors), `constants.py`. + +### `pdb/` + +PDB pipeline constants, enums, and globals (`constants.py`); the pipeline itself is `pipelines/pdb_manifest.py`. + +### `pipelines/` + +dlt-based orchestrators; each exposes a `cli = PipelineCli(...)` registered in `pyproject.toml`: + +- `core.py`: `PipelineCli`, `run_cli`, `run_pipeline`, `setup_dlt_Pipeline`, `execute_dlt_pipeline`, and `configure_pipeline_logging`. +- `all_the_bacteria.py`: All The Bacteria bulk data downloader (excluded from fast test runs). +- `jsonlines/`: JSONL ingest pipelines (`settings.py`, shared machinery in `base.py`) - `extract_pipeline.py` (`jsonl_extract`, plain load into `` / `_invalid`), `extract_jsonschema_validate_pipeline.py` (`jsonl_jsonschema`, validates against `ENTITY_SCHEMAS`), `extract_pydantic_validate_pipeline.py` (`jsonl_pydantic`, validates against `ENTITY_MODELS`). See `docs/jsonschema_ingest.md`. +- `ncbi_ftp_download.py` / `ncbi_ftp_promote.py`: phases 2 (`ncbi_ftp_sync`) and 3 (`ncbi_ftp_promote`) of the NCBI FTP flow. +- `ncbi_rest_api.py`: NCBI REST API import. +- `pdb_manifest.py`: PDB holdings transfer manifest. +- `uniprot_kb.py` / `uniref.py`: UniProt (`uniprot`) and UniRef (`uniref`) dlt pipelines. +- `xml2db/` / `xmltodict/` / `xsv/`: XML-to-relational, XML-to-dictionary, and XSV (CSV/TSV) ingestion, each with `pipeline.py` and `settings.py`; `xml2db/compaction.py` runs DuckDB-based post-load deduplication of "reused" tables on filesystem destinations. + +### `utils/` + +Cross-cutting helpers: `batcher.py` (`NumericFileSequenceBatcher`, `get_file_batches`), `buffer.py` (per-table row buffers for dlt: `DictBuffer`, `ListBuffer`, `ArrowDictBuffer`, `ArrowListBuffer`), `cdm_logger.py` (structured JSON logging), `checksums.py`, `gz.py` (`open_maybe_gzip`), `helpers.py`, `progress.py` (tqdm helpers), `spark.py`, `xml_utils.py`, `jsonschema/dereferencer.py` (`$ref` resolution), `dlt/dlt_normalization.py` (lossless parent-linked normalization of dlt table definitions, used by the converters' `DltReader`), and `file_transfer/` - `core.py` (tenacity-retried downloads), `sync_client.py` / `async_client.py` / `ftp_client.py`, and `s3/` (`client.py`, `object_utils.py`, HTTP-to-S3 streaming in `streamer.py`, `transfer_config.py`, `cli.py`). + +### Other Top-Level Directories + +- `scripts/`: `entrypoint.sh` (container entrypoint dispatching to registered pipeline CLIs), `run_tests.sh` (`unit`, `container`, or `all` test suites), `filter_ncbi_updates.py` (CLI filtering NCBI assembly IDs by modification date), `s3_local.py` (thin S3 CLI for local Ceph testing), `dtd2xsd.pl` (DTD-to-XSD converter), `run_tools.sh`, `parse_index.py`. +- `docs/`: architecture and pipeline guides (`converters_ir.md`, `jsonschema_ingest.md`, `ncbi_ftp_e2e_walkthrough.md`, `pdb_e2e_walkthrough.md`, `rclone_instructions.md`, `uniprot.md`). +- `cdl_skill/`: skill definition (`SKILL.md`) and generator (`scripts/skill_gen.sh`) for packaging pipeline documentation and schemas. + +## Conventions + +- Frozen, slotted dataclasses for models; constructors validate and recursively copy mappings. +- CLI argument parsing by pydantic-settings +- Fully type-annotated code (ruff `ANN`); datetime values carry an explicit timezone (`DTZ`); use `Decimal` for exact arithmetic, never constructed from floats. +- External HTTP calls in tests are recorded and replayed with vcrpy; AWS is mocked with moto; Spark and Ceph tests run in containers via `docker-compose.yml` and are skipped by default (`-m "not requires_spark and not requires_ceph"`). +- Tests are module-based `def test_*() -> None:` functions with pytest fixtures in `conftest.py`, named `test___`. + +## Gotchas + +- dlt 1.30 DuckDB destination fails with a Binder Error "Ambiguous reference to catalog or schema" when the db filename stem equals the dataset name; keep the db filename distinct from both pipeline and dataset names. +- dlt 1.30: `AnyDestination` is a quoted TypeAlias (a str at runtime), so `AnyDestination | None` in an evaluated annotation raises TypeError on import. Use `TDestinationReferenceArg` from `dlt.common.destination.reference`. +- pyiceberg SqlCatalog engines leak sqlite connections; use the context-manager protocol (`with load_catalog(...) as catalog`) so `close()` disposes the engine. +- pipelines are run on the KBase CTS (Task Service), which only saves the contents of the output directory (or dlt destination) if the container exits with exit code 0. For any other error code, the container STDERR and STDOUT are saved, but nothing else. As a result, even if pipelines experience failure (e.g. `load_info.has_failed_packages` is true), the container must exit 0 to ensure that logs, parsed data, and other artefacts are not deleted. Errors that occur before a pipeline is run (e.g. argument parsing errors) are not caught, so they should be picked up by the container STDERR/STDOUT. diff --git a/pyproject.toml b/pyproject.toml index 9b205d15..c68d127a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "pandas>=3.0.3", "pydantic>=2.13.4", "pydantic-settings>=2.14.1", - "pyiceberg[pandas,pyarrow,pyiceberg-core,snappy]>=0.11.1", + "pyiceberg[pandas,pyarrow,pyiceberg-core,snappy]>=0.12", "smart-open>=8.0.1", "tqdm>=4.70.1", "xml2db>=0.14.0", @@ -39,9 +39,9 @@ dependencies = [ [project.scripts] all_the_bacteria = "cdm_data_loaders.pipelines.all_the_bacteria:cli" -jsonl_extract = "cdm_data_loaders.pipelines.jsonlines.extract_pipeline:cli" -jsonl_jsonschema = "cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline:cli" -jsonl_pydantic = "cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline:cli" +jsonlines_extract = "cdm_data_loaders.pipelines.jsonlines.extract_pipeline:cli" +jsonlines_jsonschema = "cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline:cli" +jsonlines_pydantic = "cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline:cli" ncbi_ftp_promote = "cdm_data_loaders.pipelines.ncbi_ftp_promote:cli" ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" @@ -51,7 +51,7 @@ uniref = "cdm_data_loaders.pipelines.uniref:cli" xmltodict = "cdm_data_loaders.pipelines.xmltodict.pipeline:cli" xml2db = "cdm_data_loaders.pipelines.xml2db.pipeline:cli" xsv = "cdm_data_loaders.pipelines.xsv.pipeline:cli" - +hello_world = "cdm_data_loaders.utils.hello_world:cli" [dependency-groups] dev = [ @@ -75,6 +75,10 @@ models = [ xml = [ "xsdata[cli,lxml]>=26.2", ] +mutants = [ + "mutmut>=3.8.0", + "pytest-gremlins>=1.12.2", +] [tool.ruff] line-length = 120 @@ -213,6 +217,12 @@ max-complexity = 15 requires = ["uv_build>=0.12.0,<0.20.0"] build-backend = "uv_build" +[tool.mutmut] +source_paths = [ "src/" ] +pytest_add_cli_args_test_selection= [ "-m", "not requires_spark", "-m", "not slow_test", "-m", "not requires_ceph", "-m", "not external_request", "-m", "not docker", "--ignore", "tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py" ] + + + [tool.pytest] pythonpath = ["src"] log_cli = true @@ -225,6 +235,7 @@ markers = [ "slow_test: does what it says on the tin", "requires_ceph: end-to-end tests requiring a running CEPH instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)", + "docker: tests that build and run the pipeline inside a Docker container", "settings_cls(cls): the pydantic-settings class under test for alias parametrization", "cli_fields: used for restricting cli-related tests to a subset of fields" ] @@ -243,13 +254,18 @@ AWS_SECRET_ACCESS_KEY = "test_access_secret" # required for KBase Lakehouse env USER = "fake_user" KBASE_AUTH_TOKEN = "test-token-123" -CDM_TASK_SERVICE_URL = "http://localhost:8080" -BERDL_POD_IP = "192.168.1.100" -SPARK_MASTER_URL = "spark://localhost:7077" + BERDL_HIVE_METASTORE_URI = "thrift://localhost:9083" -SPARK_CLUSTER_MANAGER_API_URL = "http://localhost:8000" -GOVERNANCE_API_URL = "http://localhost:8000" +# BERDL_POD_IP = "192.168.1.100" +BERDL_POD_IP = "spark-notebook" +CDM_TASK_SERVICE_URL = "http://localhost:8080" DATALAKE_MCP_SERVER_URL = "http://localhost:8080" +GOVERNANCE_API_URL = "http://localhost:8000" +POLARIS_CATALOG_URI = "http://polaris:8181/api/catalog" +SPARK_CLUSTER_MANAGER_API_URL = "http://localhost:8000" +SPARK_MASTER_URL = "spark://localhost:7077" +# SPARK_MASTER_URL = "spark://spark-master:7077" +DLT_PROJECT_DIR = "/tmp/dlt" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } diff --git a/src/cdm_data_loaders/pipelines/jsonlines/settings.py b/src/cdm_data_loaders/pipelines/jsonlines/settings.py index 0b32c74b..6a82e588 100644 --- a/src/cdm_data_loaders/pipelines/jsonlines/settings.py +++ b/src/cdm_data_loaders/pipelines/jsonlines/settings.py @@ -22,9 +22,9 @@ default_settings_with_shortcuts, ) -EXTRACT_PIPELINE_NAME: Final[str] = "jsonlines" -PYDANTIC_PIPELINE_NAME: Final[str] = "jsonlines_pydantic_validator" -JSONSCHEMA_PIPELINE_NAME: Final[str] = "jsonlines_jsonschema_validator" +EXTRACT_PIPELINE_NAME: Final[str] = "jsonlines_extract" +PYDANTIC_PIPELINE_NAME: Final[str] = "jsonlines_pydantic" +JSONSCHEMA_PIPELINE_NAME: Final[str] = "jsonlines_jsonschema" ENTITY_MODELS_MODULE: Final[str] = "entity_models_module" SCHEMA_FILES_MODULE: Final[str] = "schema_files_module" diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index b083803e..c3589a7a 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -15,14 +15,15 @@ from ftplib import error_temp from logging import Logger, getLogger from pathlib import Path, PurePosixPath -from typing import Any +from typing import Any, Final import tqdm from pydantic import AliasChoices, Field +from pydantic_settings import SettingsConfigDict from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from cdm_data_loaders.core.fields import INPUT_MOUNT, OUTPUT_MOUNT -from cdm_data_loaders.core.settings import LoggerSettings +from cdm_data_loaders.core.settings import LoggerSettings, default_settings_with_shortcuts from cdm_data_loaders.ncbi_ftp.assembly import ( FTP_HOST, build_accession_path, @@ -36,13 +37,15 @@ logger: Logger = getLogger(__name__) - +PIPELINE_NAME: Final[str] = "ncbi_ftp_sync" DEFAULT_STAGING_KEY_PREFIX: PurePosixPath = PurePosixPath("staging") class DownloadSettings(LoggerSettings): """Configuration for the NCBI FTP assembly download pipeline.""" + model_config: SettingsConfigDict = default_settings_with_shortcuts(cli_prog_name=PIPELINE_NAME) + manifest: Path = Field( default=Path(INPUT_MOUNT) / "transfer_manifest.txt", description="Path to the transfer manifest file listing FTP paths to download", diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py index 288d317b..33f17387 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py @@ -6,14 +6,16 @@ import logging from pathlib import Path, PurePosixPath +from typing import Final from pydantic import AliasChoices, Field -from pydantic_settings import CliImplicitFlag +from pydantic_settings import CliImplicitFlag, SettingsConfigDict -from cdm_data_loaders.core.settings import LoggerSettings +from cdm_data_loaders.core.settings import LoggerSettings, default_settings_with_shortcuts from cdm_data_loaders.ncbi_ftp.promote import promote_from_s3 from cdm_data_loaders.pipelines.core import PipelineCli +PIPELINE_NAME: Final[str] = "ncbi_ftp_promote" DEFAULT_STAGING_BUCKET: PurePosixPath = PurePosixPath("cts") DEFAULT_DESTINATION_BUCKET: PurePosixPath = PurePosixPath("cdm-lake") DEFAULT_DESTINATION_PREFIX: PurePosixPath = PurePosixPath("tenant-general-warehouse/kbase/datasets/ncbi") @@ -26,6 +28,8 @@ class PromoteSettings(LoggerSettings): """Configuration for the NCBI FTP file promotion pipeline.""" + model_config: SettingsConfigDict = default_settings_with_shortcuts(cli_prog_name=PIPELINE_NAME) + staging_bucket: PurePosixPath = Field( default=DEFAULT_STAGING_BUCKET, description="Bucket where staged files are located after download", diff --git a/src/cdm_data_loaders/pipelines/uniprot_kb.py b/src/cdm_data_loaders/pipelines/uniprot_kb.py index 917e7c22..7ab79185 100644 --- a/src/cdm_data_loaders/pipelines/uniprot_kb.py +++ b/src/cdm_data_loaders/pipelines/uniprot_kb.py @@ -21,7 +21,7 @@ ) from cdm_data_loaders.readers.xml import build_xml_file_resource -PIPELINE_NAME: Final[str] = "uniprot_kb" +PIPELINE_NAME: Final[str] = "uniprot" UNIPROT_LOG_INTERVAL: Final[int] = 1000 @@ -29,7 +29,7 @@ class UniProtSettings(CtsSettings): """Configuration for running the UniProt KB import pipeline.""" model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name="uniprot", cli_shortcuts=CLI_SHORTCUTS + cli_prog_name=PIPELINE_NAME, cli_shortcuts=CLI_SHORTCUTS ) log_interval: Annotated[ diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index ad71aae4..20a58e88 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -22,6 +22,12 @@ LOAD_INFO_TABLE_NAME, ) from cdm_data_loaders.utils import cdm_logger +from tests.cdm_data_loaders.pipelines.framework import ( + EXECUTION_MODES, + ExecutionMode, + PipelineTestRunner, + ensure_docker_image, +) from tests.conftest import LOCAL_FS, S3 DEFAULT_DLT_TABLES = {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", LOAD_INFO_TABLE_NAME} @@ -120,11 +126,32 @@ def mock_dlt(monkeypatch: pytest.MonkeyPatch) -> MagicMock: return mock +@pytest.fixture(params=EXECUTION_MODES) +def execution_mode(request: pytest.FixtureRequest) -> ExecutionMode: + """Execution mode for pipeline end-to-end tests (core, cli, or docker).""" + return request.param + + +@pytest.fixture(scope="session") +def docker_image(request: pytest.FixtureRequest) -> str: + """Build or resolve the Docker image tag used for @pytest.mark.docker pipeline tests.""" + configured_tag = request.config.getoption("--docker-image") + return ensure_docker_image(configured_tag) + + +@pytest.fixture +def pipeline_runner(execution_mode: ExecutionMode, request: pytest.FixtureRequest) -> PipelineTestRunner: + """Provide a PipelineTestRunner configured for the active execution_mode.""" + image_tag = request.config.getoption("--docker-image") if execution_mode == ExecutionMode.DOCKER else None + return PipelineTestRunner(mode=execution_mode, docker_image=image_tag) + + def isolated_run( tmp_path: Path, dataset_prefix: str, run_fn: Callable[..., LoadInfo | None], base_kwargs: dict[str, Any], + pipeline_runner: PipelineTestRunner | None = None, ) -> Callable[..., tuple[LoadInfo | None, Path]]: """Return a factory that runs a pipeline with per-run isolated output state. @@ -139,6 +166,8 @@ def isolated_run( :type run_fn: Callable[..., LoadInfo | None] :param base_kwargs: base settings kwargs; keys open to override per call :type base_kwargs: dict[str, Any] + :param pipeline_runner: optional PipelineTestRunner to dispatch through + :type pipeline_runner: PipelineTestRunner | None :return: factory(chunk_dir=None, **overrides) -> (load_info, output_dir) :rtype: Callable[..., tuple[LoadInfo | None, Path]] """ @@ -160,7 +189,7 @@ def _factory(**overrides: Any) -> tuple[LoadInfo | None, Path]: settings_cls = kwargs.pop("settings_cls") settings = settings_cls(**kwargs) with dlt.config.values({"destination.local_fs.bucket_url": str(settings.output_dir)}): - load_info = run_fn(settings) + load_info = pipeline_runner.run(run_fn, settings) if pipeline_runner is not None else run_fn(settings) return load_info, settings.output_dir return _factory @@ -171,9 +200,12 @@ def isolated_run_factory(tmp_path: Path) -> Callable[..., Callable[..., tuple[Lo """Return a builder for isolated pipeline run factories bound to this test's tmp_path.""" def _build( - dataset_prefix: str, run_fn: Callable[..., LoadInfo | None], base_kwargs: dict[str, Any] + dataset_prefix: str, + run_fn: Callable[..., LoadInfo | None], + base_kwargs: dict[str, Any], + pipeline_runner: PipelineTestRunner | None = None, ) -> Callable[..., tuple[LoadInfo | None, Path]]: - return isolated_run(tmp_path, dataset_prefix, run_fn, base_kwargs) + return isolated_run(tmp_path, dataset_prefix, run_fn, base_kwargs, pipeline_runner=pipeline_runner) return _build diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py new file mode 100644 index 00000000..825abffe --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -0,0 +1,723 @@ +"""Unified test execution framework for running CDM data ingestion pipelines in core, cli, or docker mode.""" + +import gzip +import json +import logging +import os +import shutil +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime +from enum import StrEnum +from importlib import import_module +from pathlib import Path +from types import ModuleType +from typing import Any, Final, Self +from unittest.mock import patch +from uuid import uuid4 + +import dlt +import pytest +from dlt.common.pipeline import LoadInfo +from dlt.destinations import filesystem + +from cdm_data_loaders.core.settings import CtsSettings, LoggerSettings +from cdm_data_loaders.pipelines import core +from cdm_data_loaders.pipelines.core import LOAD_INFO_TABLE_NAME, PipelineCli +from cdm_data_loaders.utils import cdm_logger +from tests.cdm_data_loaders.pipelines.helpers import settings_to_argv +from tests.conftest import BASE_DIR, CONFIG_BUCKET, LOCAL_FS + +DEFAULT_DOCKER_IMAGE_TAG: Final[str] = "cdm-data-loaders-test:latest" +CONTAINER_INPUT_DIR: Final[str] = "/input_dir" +CONTAINER_OUTPUT_DIR: Final[str] = "/output_dir" +CONTAINER_AUX_DIR: Final[str] = "/aux_dir" +CONTAINER_PYMODULES_DIR: Final[str] = "/pymodules" + +# Flags whose values are local file paths that must be staged into the container in Docker mode +FILE_PATH_FLAGS: Final[frozenset[str]] = frozenset( + { + "--xsd-file", + "--xml2db-config-file", + "--schema-file", + "-s", + "--log-config-file", + } +) + +# Flags whose values are Python module paths that may be dynamically created in sys.modules +MODULE_PATH_FLAGS: Final[frozenset[str]] = frozenset( + { + "--entity-models-module", + "--schema-files-module", + "-m", + } +) + +INPUT_DIR_FLAGS: Final[frozenset[str]] = frozenset({"--input-dir", "-i"}) +OUTPUT_DIR_FLAGS: Final[frozenset[str]] = frozenset({"--output-dir", "-o"}) +DESTINATION_FLAGS: Final[frozenset[str]] = frozenset({"--use-destination", "-d"}) + +# Registry mapping console-script / pipeline names to (cli_command, module_path) +PIPELINE_TARGETS: Final[dict[str, tuple[str, str]]] = { + "all_the_bacteria": ("all_the_bacteria", "cdm_data_loaders.pipelines.all_the_bacteria"), + "jsonlines_extract": ("jsonlines_extract", "cdm_data_loaders.pipelines.jsonlines.extract_pipeline"), + "jsonlines_jsonschema": ( + "jsonlines_jsonschema", + "cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline", + ), + "jsonlines_pydantic": ( + "jsonlines_pydantic", + "cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline", + ), + "ncbi_ftp_promote": ("ncbi_ftp_promote", "cdm_data_loaders.pipelines.ncbi_ftp_promote"), + "ncbi_ftp_sync": ("ncbi_ftp_sync", "cdm_data_loaders.pipelines.ncbi_ftp_download"), + "ncbi_rest_api": ("ncbi_rest_api", "cdm_data_loaders.pipelines.ncbi_rest_api"), + "pdb_manifest": ("pdb_manifest", "cdm_data_loaders.pipelines.pdb_manifest"), + "uniprot": ("uniprot", "cdm_data_loaders.pipelines.uniprot_kb"), + "uniref": ("uniref", "cdm_data_loaders.pipelines.uniref"), + "xmltodict": ("xmltodict", "cdm_data_loaders.pipelines.xmltodict.pipeline"), + "xml2db": ("xml2db", "cdm_data_loaders.pipelines.xml2db.pipeline"), + "xsv": ("xsv", "cdm_data_loaders.pipelines.xsv.pipeline"), +} + +MODULE_TO_CLI_COMMAND: Final[dict[str, str]] = { + module_path: cli_command for cli_command, module_path in PIPELINE_TARGETS.values() +} + +_built_docker_images: set[str] = set() + + +class ExecutionMode(StrEnum): + """Supported pipeline test execution modes.""" + + CORE = "core" + CLI = "cli" + DOCKER = "docker" + + +EXECUTION_MODES: Final[tuple[Any, ...]] = ( + pytest.param(ExecutionMode.CORE, id="core"), + pytest.param(ExecutionMode.CLI, id="cli"), + pytest.param(ExecutionMode.DOCKER, id="docker", marks=pytest.mark.docker), +) + + +@dataclass(frozen=True, slots=True) +class PipelineSpec: + """Resolved metadata for executing a pipeline across core, cli, and docker modes.""" + + cli_command: str + module_path: str + settings_cls: type[LoggerSettings] | None = None + pipeline_fn: Callable[[Any], LoadInfo | None] | None = None + cli_callable: Callable[[], Any] | None = None + + +class _PreservedStrDatetime(datetime): + """Datetime subclass that preserves the exact string representation written to the report file.""" + + _raw_str: str + + def __new__(cls, raw_str: str) -> Self: + """Create a datetime from an ISO-formatted string while preserving its original string spelling.""" + dt = datetime.fromisoformat(raw_str) + instance = super().__new__( + cls, + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + tzinfo=dt.tzinfo, + ) + instance._raw_str = raw_str + return instance + + def __str__(self) -> str: + """Return the verbatim timestamp string from the pipeline report.""" + return self._raw_str + + +@dataclass +class ReconstructedLoadInfo: + """LoadInfo-compatible object reconstructed from pipeline output files on disk.""" + + pipeline: dlt.Pipeline + dataset_name: str + loads_ids: list[str] + started_at: datetime + has_failed_jobs: bool = False + finished_at: datetime | None = None + first_run: bool = False + destination_name: str = LOCAL_FS + destination_type: str = "filesystem" + load_packages: list[Any] = field(default_factory=list) + metrics: dict[str, Any] = field(default_factory=dict) + _raw_dict: dict[str, Any] = field(default_factory=dict, repr=False) + + def asdict(self) -> dict[str, Any]: + """Return a dictionary representation matching dlt's LoadInfo.asdict().""" + if self._raw_dict: + return dict(self._raw_dict) + return { + "dataset_name": self.dataset_name, + "destination_name": self.destination_name, + "destination_type": self.destination_type, + "finished_at": self.finished_at, + "first_run": self.first_run, + "has_failed_jobs": self.has_failed_jobs, + "loads_ids": list(self.loads_ids), + "pipeline": {"pipeline_name": self.pipeline.pipeline_name}, + "started_at": self.started_at, + } + + +type PipelineIdentifier = str | ModuleType | PipelineCli[Any] | PipelineSpec | Callable[..., LoadInfo | None] + + +def _spec_from_module(cli_command: str, module: ModuleType) -> PipelineSpec: + """Build a PipelineSpec by inspecting a pipeline module.""" + cli_attr = getattr(module, "cli", None) + if isinstance(cli_attr, PipelineCli): + return PipelineSpec( + cli_command=cli_command, + module_path=module.__name__, + settings_cls=cli_attr.settings_cls, + pipeline_fn=cli_attr.pipeline_fn, + cli_callable=cli_attr, + ) + return PipelineSpec( + cli_command=cli_command, + module_path=module.__name__, + settings_cls=getattr(module, "Settings", None), + pipeline_fn=getattr(module, "run_pipeline", None), + cli_callable=cli_attr if callable(cli_attr) else None, + ) + + +def resolve_pipeline(pipeline: PipelineIdentifier) -> PipelineSpec: + """Resolve a pipeline name, module, PipelineCli, or callable into a PipelineSpec. + + :param pipeline: pipeline identifier (name, module, PipelineCli, PipelineSpec, or run function) + :type pipeline: PipelineIdentifier + :raises ValueError: if the pipeline cannot be resolved + :return: resolved pipeline specification + :rtype: PipelineSpec + """ + if isinstance(pipeline, PipelineSpec): + return pipeline + + if isinstance(pipeline, PipelineCli): + module_path = pipeline.pipeline_fn.__module__ + cli_command = MODULE_TO_CLI_COMMAND.get(module_path, module_path.rsplit(".", maxsplit=1)[-1]) + return PipelineSpec( + cli_command=cli_command, + module_path=module_path, + settings_cls=pipeline.settings_cls, + pipeline_fn=pipeline.pipeline_fn, + cli_callable=pipeline, + ) + + if isinstance(pipeline, ModuleType): + module_path = pipeline.__name__ + short_key = module_path.removeprefix("cdm_data_loaders.pipelines.") + cli_command = MODULE_TO_CLI_COMMAND.get( + module_path, + PIPELINE_TARGETS.get(short_key, (short_key.rsplit(".", maxsplit=1)[-1], module_path))[0], + ) + return _spec_from_module(cli_command, pipeline) + + if callable(pipeline): + module_path = getattr(pipeline, "__module__", "") + if module_path in MODULE_TO_CLI_COMMAND: + module = import_module(module_path) + return _spec_from_module(MODULE_TO_CLI_COMMAND[module_path], module) + err_msg = f"Cannot resolve pipeline from callable {pipeline!r} (module {module_path!r})" + raise ValueError(err_msg) + + if isinstance(pipeline, str): + normalized = pipeline.strip().removeprefix("cdm_data_loaders.pipelines.") + if normalized in PIPELINE_TARGETS: + cli_command, module_path = PIPELINE_TARGETS[normalized] + module = import_module(module_path) + return _spec_from_module(cli_command, module) + full_module_path = f"cdm_data_loaders.pipelines.{normalized}" + if full_module_path in MODULE_TO_CLI_COMMAND: + cli_command = MODULE_TO_CLI_COMMAND[full_module_path] + module = import_module(full_module_path) + return _spec_from_module(cli_command, module) + + err_msg = f"Unknown pipeline {pipeline!r}. Known pipelines: {', '.join(sorted(PIPELINE_TARGETS))}" + raise ValueError(err_msg) + + +def _read_saved_load_info_row(dataset_dir: Path) -> dict[str, Any]: + """Read the serialized load_info row from ``/_load_info`` if present.""" + load_info_dir = dataset_dir / LOAD_INFO_TABLE_NAME + if not load_info_dir.is_dir(): + return {} + for file_path in sorted(load_info_dir.glob("*.jsonl*")): + if not file_path.is_file(): + continue + with file_path.open("rb") as fh: + is_gzipped = fh.read(2) == b"\x1f\x8b" + opener = gzip.open if is_gzipped else open + with opener(file_path, "rt", encoding="utf-8") as fh: + for line in fh: + stripped = line.strip() + if stripped: + return json.loads(stripped) + return {} + + +def reconstruct_load_info(output_dir: Path, bucket_dir: Path | None = None) -> LoadInfo | None: + """Reconstruct a LoadInfo-compatible object and attached dlt.Pipeline from disk artifacts. + + Reads the ``report_.txt`` file written by ``save_pipeline_metadata`` in + ``output_dir`` and attaches a ``dlt.Pipeline`` pointing at the filesystem destination + containing ````. + + :param output_dir: directory containing ``report_.txt`` + :type output_dir: Path + :param bucket_dir: optional separate filesystem destination bucket directory + :type bucket_dir: Path | None + :return: reconstructed LoadInfo, or None if the report indicates an error or no report exists + :rtype: LoadInfo | None + """ + output_dir = Path(output_dir) + report_files = sorted(output_dir.glob("report_*.txt"), key=lambda path: path.stat().st_mtime) + if not report_files: + return None + + report_file = report_files[-1] + lines = report_file.read_text(encoding="utf-8").splitlines() + if len(lines) < 3: # noqa: PLR2004 + return None + + # Line 0: "Pipeline dataset " + header_prefix = "Pipeline " + dataset_sep = " dataset " + if not lines[0].startswith(header_prefix) or dataset_sep not in lines[0]: + return None + pipeline_name, dataset_name = lines[0].removeprefix(header_prefix).rsplit(dataset_sep, maxsplit=1) + pipeline_name = pipeline_name.strip() + dataset_name = dataset_name.strip() + + # Line 1: "Started: " + started_str = lines[1].removeprefix("Started: ").strip() + started_at = _PreservedStrDatetime(started_str) + + # Line 2: "Status: completed" | "Status: completed with errors" | "Status: error" + status_str = lines[2].removeprefix("Status: ").strip() + if status_str == "error": + return None + has_failed_jobs = status_str != "completed" + + # Line 3+: "Completed loads: , , ..." + loads_ids: list[str] = [] + for line in lines[3:]: + if line.startswith("Completed loads:"): + loads_raw = line.removeprefix("Completed loads:").strip() + loads_ids = [item.strip() for item in loads_raw.split(",") if item.strip()] + break + + # Determine where the dataset directory actually lives on disk + if (output_dir / dataset_name).is_dir(): + actual_bucket_dir = output_dir + elif bucket_dir is not None and (Path(bucket_dir) / dataset_name).is_dir(): + actual_bucket_dir = Path(bucket_dir) + else: + actual_bucket_dir = Path(bucket_dir) if bucket_dir is not None else output_dir + + saved_row = _read_saved_load_info_row(actual_bucket_dir / dataset_name) + finished_at_raw = saved_row.get("finished_at") + finished_at = _PreservedStrDatetime(str(finished_at_raw)) if finished_at_raw else None + + # Use a fresh working directory outside output_dir so dlt syncs the schema from the destination + # without colliding with stale schemas in ~/.dlt/pipelines or adding files to output_dir. + pipeline_e2e_id = report_file.stem.removeprefix("report_") + reconstructed_pipelines_dir = output_dir.parent / f".reconstructed_dlt_{pipeline_e2e_id}" + pipeline = dlt.pipeline( + pipeline_name=pipeline_name, + destination=filesystem(bucket_url=str(actual_bucket_dir)), + dataset_name=dataset_name, + pipelines_dir=str(reconstructed_pipelines_dir), + ) + + return ReconstructedLoadInfo( # pyright: ignore[reportReturnType] + pipeline=pipeline, + dataset_name=dataset_name, + loads_ids=loads_ids, + started_at=started_at, + has_failed_jobs=has_failed_jobs, + finished_at=finished_at, + first_run=bool(saved_row.get("first_run", False)), + destination_name=str(saved_row.get("destination_name", LOCAL_FS)), + destination_type=str(saved_row.get("destination_type", "filesystem")), + _raw_dict=saved_row, + ) + + +def ensure_docker_image(image_tag: str | None = None, *, force_rebuild: bool = False) -> str: + """Build the Docker image from the repository Dockerfile if not already built in this session. + + :param image_tag: optional image tag; defaults to ``cdm-data-loaders-test:latest`` + :type image_tag: str | None + :param force_rebuild: force running ``docker build`` even if already built in this session + :type force_rebuild: bool + :raises RuntimeError: if ``docker build`` fails + :return: Docker image tag + :rtype: str + """ + tag = image_tag or os.environ.get("CDM_TEST_DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE_TAG + if not force_rebuild and tag in _built_docker_images: + return tag + + cmd = ["docker", "build", "-t", tag, str(BASE_DIR.resolve())] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) # noqa: S603 + if result.returncode != 0: + err_msg = ( + f"Docker image build failed (exit {result.returncode}):\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + raise RuntimeError(err_msg) + + _built_docker_images.add(tag) + return tag + + +def _extract_flag_value(cli_args: list[str], flags: frozenset[str]) -> str | None: + """Extract the value immediately following any of ``flags`` in ``cli_args``.""" + for idx, arg in enumerate(cli_args): + if arg in flags and idx + 1 < len(cli_args): + return cli_args[idx + 1] + for flag in flags: + if arg.startswith(f"{flag}="): + return arg.split("=", maxsplit=1)[1] + return None + + +def _normalize_cli_args(cli_args: list[str]) -> list[str]: + """Strip 0-byte ``.conf`` placeholder log files created by test fixtures that bypass ``init_logger``.""" + normalized: list[str] = [] + idx = 0 + while idx < len(cli_args): + arg = cli_args[idx] + if arg == "--log-config-file" and idx + 1 < len(cli_args): + log_path = Path(cli_args[idx + 1]) + if log_path.suffix.lower() == ".conf" and log_path.exists() and log_path.stat().st_size == 0: + idx += 2 + continue + normalized.append(arg) + idx += 1 + return normalized + + +def _materialize_dynamic_module(module_name: str, pymodules_dir: Path) -> bool: + """If ``module_name`` is an in-memory module in ``sys.modules``, write a ``.py`` file for Docker.""" + module = sys.modules.get(module_name) + if module is None or getattr(module, "__file__", None) is not None: + return False + + pymodules_dir.mkdir(parents=True, exist_ok=True) + target_file = pymodules_dir / f"{module_name}.py" + + if hasattr(module, "ENTITY_SCHEMAS"): + schemas = module.ENTITY_SCHEMAS + target_file.write_text(f"ENTITY_SCHEMAS = {schemas!r}\n", encoding="utf-8") + return True + + if hasattr(module, "ENTITY_MODELS"): + models: dict[str, Any] = module.ENTITY_MODELS + imports: list[str] = [] + entries: list[str] = [] + for idx, (table_name, model_cls) in enumerate(models.items()): + alias = f"_Model_{idx}" + imports.append(f"from {model_cls.__module__} import {model_cls.__name__} as {alias}") + entries.append(f" {table_name!r}: {alias},") + code = "\n".join([*imports, "", "ENTITY_MODELS = {", *entries, "}", ""]) + target_file.write_text(code, encoding="utf-8") + return True + + return False + + +def _write_docker_dlt_config(config_path: Path, use_destination: str) -> None: + """Write ``config.toml`` for the container with ``/input_dir`` and ``/output_dir`` buckets.""" + config_path.parent.mkdir(parents=True, exist_ok=True) + destinations = {"cts", LOCAL_FS, use_destination} + blocks: list[str] = [] + for dest in sorted(destinations): + blocks.append(f'[destination.{dest}]\nbucket_url = "{CONTAINER_OUTPUT_DIR}"\ndestination_type = "filesystem"\n') + blocks.append(f'[sources.{dest}]\nbucket_url = "{CONTAINER_INPUT_DIR}"\n') + config_path.write_text("\n".join(blocks), encoding="utf-8") + + +class PipelineTestRunner: + """Execute any CDM ingestion pipeline in ``core``, ``cli``, or ``docker`` mode.""" + + def __init__(self, mode: ExecutionMode | str = ExecutionMode.CORE, docker_image: str | None = None) -> None: + """Initialize the runner with the target execution mode and optional Docker image tag.""" + self.mode = ExecutionMode(mode) + self.docker_image = docker_image + + def __call__( + self, + pipeline: PipelineIdentifier, + settings: CtsSettings | LoggerSettings | None = None, + *, + cli_args: list[str] | None = None, + **overrides: Any, # noqa: ANN401 + ) -> LoadInfo | None: + """Alias for :meth:`run` so the runner can be passed as a ``run_fn`` callback.""" + return self.run(pipeline, settings, cli_args=cli_args, **overrides) + + def run( + self, + pipeline: PipelineIdentifier, + settings: CtsSettings | LoggerSettings | None = None, + *, + cli_args: list[str] | None = None, + **overrides: Any, # noqa: ANN401 + ) -> LoadInfo | None: + """Execute ``pipeline`` in the configured mode and return its ``LoadInfo``. + + :param pipeline: pipeline name, module, PipelineCli, PipelineSpec, or pipeline run function + :type pipeline: PipelineIdentifier + :param settings: optional pre-built settings instance + :type settings: CtsSettings | LoggerSettings | None + :param cli_args: optional explicit CLI argument list (excluding the program name) + :type cli_args: list[str] | None + :param overrides: optional field overrides applied if building or updating settings + :type overrides: Any + :raises ValueError: if neither settings nor cli_args are provided and settings cannot be built + :return: pipeline LoadInfo (or ReconstructedLoadInfo in cli/docker modes) + :rtype: LoadInfo | None + """ + spec = resolve_pipeline(pipeline) + effective_settings = settings + if effective_settings is not None and overrides: + effective_settings = effective_settings.model_copy(update=overrides) + elif effective_settings is None and cli_args is None: + if spec.settings_cls is None: + err_msg = f"Pipeline {spec.cli_command!r} has no settings_cls; supply cli_args explicitly." + raise ValueError(err_msg) + effective_settings = spec.settings_cls(**overrides) + + raw_args = list(cli_args) if cli_args is not None else settings_to_argv(effective_settings) + # Strip leading program name if the caller passed a full sys.argv list + if raw_args and not raw_args[0].startswith("-"): + raw_args = raw_args[1:] + args = _normalize_cli_args(raw_args) + + output_dir_str = _extract_flag_value(args, OUTPUT_DIR_FLAGS) or ( + str(getattr(effective_settings, "output_dir", "")) if effective_settings else None + ) + if not output_dir_str: + err_msg = "Could not determine output_dir from settings or cli_args" + raise ValueError(err_msg) + output_dir = Path(output_dir_str) + output_dir.mkdir(parents=True, exist_ok=True) + + use_destination = ( + _extract_flag_value(args, DESTINATION_FLAGS) + or (str(getattr(effective_settings, "use_destination", LOCAL_FS)) if effective_settings else None) + or LOCAL_FS + ) + + if self.mode == ExecutionMode.CORE: + return self._run_core(spec, args, output_dir, use_destination) + if self.mode == ExecutionMode.CLI: + return self._run_cli(spec, args, output_dir, use_destination) + return self._run_docker(spec, args, output_dir, use_destination) + + def _resolve_bucket_dir(self, output_dir: Path, use_destination: str) -> Path: + """Determine the active host bucket directory for ``use_destination``.""" + configured = dlt.config.get(f"destination.{use_destination}.bucket_url") + default_buckets = {CONFIG_BUCKET.get(LOCAL_FS), "/configured/out/", None} + if configured and configured not in default_buckets and Path(str(configured)).exists(): + return Path(str(configured)) + return output_dir + + def _run_core( + self, + spec: PipelineSpec, + args: list[str], + output_dir: Path, + use_destination: str, + ) -> LoadInfo | None: + """Execute the pipeline in-process via ``cdm_data_loaders.pipelines.core.run_cli``.""" + if spec.settings_cls is None or spec.pipeline_fn is None: + err_msg = f"Pipeline {spec.cli_command!r} does not expose settings_cls and pipeline_fn for core.run_cli" + raise ValueError(err_msg) + + bucket_dir = self._resolve_bucket_dir(output_dir, use_destination) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + applied_key = cdm_logger.applied_key + try: + with dlt.config.values( + { + f"destination.{use_destination}.destination_type": "filesystem", + f"destination.{use_destination}.bucket_url": str(bucket_dir), + } + ): + return core.run_cli(spec.settings_cls, spec.pipeline_fn, cli_args=args) + finally: + for handler in root.handlers[:]: + root.removeHandler(handler) + for handler in handlers: + root.addHandler(handler) + root.setLevel(level) + cdm_logger.applied_key = applied_key + + def _run_cli( + self, + spec: PipelineSpec, + args: list[str], + output_dir: Path, + use_destination: str, + ) -> LoadInfo | None: + """Execute the pipeline via ``cdm_data_loaders.pipelines..cli()`` and reconstruct LoadInfo.""" + if spec.cli_callable is None: + err_msg = f"Pipeline {spec.cli_command!r} does not expose a cli() entry point" + raise ValueError(err_msg) + + bucket_dir = self._resolve_bucket_dir(output_dir, use_destination) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + applied_key = cdm_logger.applied_key + try: + with ( + patch.object(sys, "argv", [spec.cli_command, *args]), + dlt.config.values( + { + f"destination.{use_destination}.destination_type": "filesystem", + f"destination.{use_destination}.bucket_url": str(bucket_dir), + } + ), + ): + spec.cli_callable() + finally: + for handler in root.handlers[:]: + root.removeHandler(handler) + for handler in handlers: + root.addHandler(handler) + root.setLevel(level) + cdm_logger.applied_key = applied_key + + return reconstruct_load_info(output_dir, bucket_dir=bucket_dir) + + def _run_docker( + self, + spec: PipelineSpec, + args: list[str], + output_dir: Path, + use_destination: str, + ) -> LoadInfo | None: + """Execute the pipeline inside the Docker container and reconstruct LoadInfo from mounted output.""" + image_tag = ensure_docker_image(self.docker_image) + input_dir_str = _extract_flag_value(args, INPUT_DIR_FLAGS) + if not input_dir_str: + err_msg = "Docker execution mode requires --input-dir / -i in settings or cli_args" + raise ValueError(err_msg) + host_input_dir = Path(input_dir_str).resolve() + host_output_dir = output_dir.resolve() + + stage_dir = host_output_dir.parent / f".docker_stage_{uuid4().hex[:8]}" + staged_input_dir = stage_dir / "input_dir" + aux_dir = stage_dir / "aux" + pymodules_dir = stage_dir / "pymodules" + staged_input_dir.mkdir(parents=True, exist_ok=True) + + if host_input_dir.is_dir(): + shutil.copytree(host_input_dir, staged_input_dir, dirs_exist_ok=True) + + _write_docker_dlt_config(staged_input_dir / "dlt_conf" / "config.toml", use_destination) + + container_args = self._translate_args_for_docker(args, aux_dir, pymodules_dir) + + docker_cmd: list[str] = [ + "docker", + "run", + "--rm", + "-v", + f"{staged_input_dir}:{CONTAINER_INPUT_DIR}", + "-v", + f"{host_output_dir}:{CONTAINER_OUTPUT_DIR}", + "-v", + f"{(BASE_DIR / 'tests').resolve()}:/app/tests:ro", + ] + if aux_dir.is_dir(): + docker_cmd.extend(["-v", f"{aux_dir.resolve()}:{CONTAINER_AUX_DIR}:ro"]) + if pymodules_dir.is_dir(): + docker_cmd.extend( + [ + "-v", + f"{pymodules_dir.resolve()}:{CONTAINER_PYMODULES_DIR}:ro", + "-e", + f"PYTHONPATH={CONTAINER_PYMODULES_DIR}:/app/src:/app", + ] + ) + else: + docker_cmd.extend(["-e", "PYTHONPATH=/app/src:/app"]) + + docker_cmd.extend([image_tag, "pipeline", spec.cli_command, *container_args]) + + result = subprocess.run(docker_cmd, capture_output=True, text=True, check=False) # noqa: S603 + if result.returncode != 0: + stderr_and_stdout = f"{result.stdout}\n{result.stderr}".strip() + if "Unknown table name" in stderr_and_stdout or "ValueError" in stderr_and_stdout: + raise ValueError(stderr_and_stdout) + err_msg = ( + f"Docker pipeline '{spec.cli_command}' failed with exit code {result.returncode}:\n" + f"Command: {' '.join(docker_cmd)}\n" + f"STDOUT:\n{result.stdout}\n" + f"STDERR:\n{result.stderr}" + ) + raise RuntimeError(err_msg) + + return reconstruct_load_info(host_output_dir, bucket_dir=host_output_dir) + + def _translate_args_for_docker( + self, + args: list[str], + aux_dir: Path, + pymodules_dir: Path, + ) -> list[str]: + """Rewrite host file paths and materialize dynamic modules for container execution.""" + translated: list[str] = [] + idx = 0 + aux_counter = 0 + while idx < len(args): + token = args[idx] + if token in INPUT_DIR_FLAGS and idx + 1 < len(args): + translated.extend([token, CONTAINER_INPUT_DIR]) + idx += 2 + continue + if token in OUTPUT_DIR_FLAGS and idx + 1 < len(args): + translated.extend([token, CONTAINER_OUTPUT_DIR]) + idx += 2 + continue + if token in FILE_PATH_FLAGS and idx + 1 < len(args): + host_file = Path(args[idx + 1]).resolve() + aux_dir.mkdir(parents=True, exist_ok=True) + staged_name = f"{aux_counter}_{host_file.name}" + aux_counter += 1 + shutil.copy2(host_file, aux_dir / staged_name) + translated.extend([token, f"{CONTAINER_AUX_DIR}/{staged_name}"]) + idx += 2 + continue + if token in MODULE_PATH_FLAGS and idx + 1 < len(args): + module_name = args[idx + 1] + _materialize_dynamic_module(module_name, pymodules_dir) + translated.extend([token, module_name]) + idx += 2 + continue + translated.append(token) + idx += 1 + return translated diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index 5b78da68..b8d827be 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -96,7 +96,8 @@ def settings_to_argv(settings: Any) -> list[str]: """Derive CLI arguments from a settings instance. Every serialized field becomes a ``--kebab-case`` flag, with bools rendered - as their ``true``/``false`` CLI spelling and None-valued fields skipped. + as their ``true``/``false`` CLI spelling, sequence fields expanded after the + flag, and None-valued or computed fields skipped. :param settings: a CtsSettings (or derivative) instance :type settings: Any @@ -104,12 +105,16 @@ def settings_to_argv(settings: Any) -> list[str]: :rtype: list[str] """ argv: list[str] = [] - for name, value in settings.model_dump().items(): + computed_fields: set[str] = set(getattr(type(settings), "model_computed_fields", {}).keys()) + for name, value in settings.model_dump(exclude=computed_fields).items(): flag = f"--{name.replace('_', '-')}" if isinstance(value, bool): argv.extend([flag, "true" if value else "false"]) elif value is None: continue + elif isinstance(value, list | tuple): + if value: + argv.extend([flag, *(str(item) for item in value)]) else: argv.extend([flag, str(value)]) return argv diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py index 36cb2fa2..4a021cf1 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py @@ -18,6 +18,9 @@ run_jsonlines_with_validation_pipeline, ) from cdm_data_loaders.pipelines.jsonlines.settings import ( + EXTRACT_PIPELINE_NAME, + JSONSCHEMA_PIPELINE_NAME, + PYDANTIC_PIPELINE_NAME, JsonlExtractSettings, JsonlJsonschemaIngestSettings, JsonlPydanticIngestSettings, @@ -33,7 +36,7 @@ type PrepareReference = Callable[[dict[str, Any]], dict[str, Any]] -@pytest.fixture(params=["extract", "extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.fixture(params=[EXTRACT_PIPELINE_NAME, PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def pipeline_kind(request: pytest.FixtureRequest) -> str: """Run the same reference contract against all three JSONL pipelines.""" return request.param @@ -48,14 +51,14 @@ def run_reference_pipeline( ) -> PipelineRunner: """Run the selected JSONL pipeline on a chunk dir with per-run isolated output directories.""" settings_cls, registry_kwarg, dataset_prefix, file_glob = { - "extract": (JsonlExtractSettings, {"table_name": "dataset"}, "jsonl_extract_run", "**/*.jsonl*"), - "extract_validate": ( + EXTRACT_PIPELINE_NAME: (JsonlExtractSettings, {"table_name": "dataset"}, "jsonl_extract_run", "**/*.jsonl*"), + PYDANTIC_PIPELINE_NAME: ( JsonlPydanticIngestSettings, {"entity_models_module": "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models"}, "jsonl_extract_validate_run", "*.jsonl*", ), - "extract_jsonschema_validate": ( + JSONSCHEMA_PIPELINE_NAME: ( JsonlJsonschemaIngestSettings, {"schema_files_module": "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas"}, "jsonschema_run", @@ -63,9 +66,9 @@ def run_reference_pipeline( ), }[pipeline_kind] run_fn = { - "extract": run_jsonlines_pipeline, - "extract_validate": run_jsonlines_with_validation_pipeline, - "extract_jsonschema_validate": run_jsonschema_pipeline, + EXTRACT_PIPELINE_NAME: run_jsonlines_pipeline, + PYDANTIC_PIPELINE_NAME: run_jsonlines_with_validation_pipeline, + JSONSCHEMA_PIPELINE_NAME: run_jsonschema_pipeline, }[pipeline_kind] base_kwargs: dict[str, Any] = { "buffer_size": 10, @@ -90,7 +93,7 @@ def prepare_reference(pipeline_kind: str) -> PrepareReference: """Apply Pydantic defaults only for model validation; other pipelines preserve input.""" def _prepare(entry: dict[str, Any]) -> dict[str, Any]: - if pipeline_kind == "extract_validate": + if pipeline_kind == PYDANTIC_PIPELINE_NAME: return ENTITY_MODELS["dataset"].model_validate(entry).model_dump(mode="json") return entry @@ -106,13 +109,13 @@ def reference_registry_factory( """Register the requested tables using the selected pipeline's assembly contract.""" def _factory(table_names: list[str]) -> dict[str, str]: - if pipeline_kind == "extract_validate": + if pipeline_kind == PYDANTIC_PIPELINE_NAME: return { "entity_models_module": entity_models_module_factory( dict.fromkeys(table_names, ENTITY_MODELS["dataset"]) ) } - if pipeline_kind == "extract_jsonschema_validate": + if pipeline_kind == JSONSCHEMA_PIPELINE_NAME: return {"schema_files_module": schema_module_factory(dict.fromkeys(table_names, ENTITY_SCHEMAS["dataset"]))} return {} diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py index 4421a137..ec0b0e3e 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py @@ -1,28 +1,27 @@ """End-to-end tests for the jsonlines pipeline.""" -import sys from collections.abc import Callable from pathlib import Path import pytest from cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline import ( - cli, load_entity_models, - run_jsonlines_with_validation_pipeline, ) from cdm_data_loaders.pipelines.jsonlines.settings import PYDANTIC_PIPELINE_NAME, JsonlPydanticIngestSettings +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written def test_run_jsonlines_pipeline_pass_writes_expected_parquet( scenario_input_dir: Callable[[str], str], settings_factory: Callable[..., JsonlPydanticIngestSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """Running the pipeline on a mixed batch puts the correct rows in both output tables.""" settings = settings_factory(input_dir=scenario_input_dir("mixed")) - load_info = run_jsonlines_with_validation_pipeline(settings) + load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings) assert load_info is not None dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "c"] @@ -35,12 +34,13 @@ def test_run_jsonlines_pipeline_pass_writes_expected_parquet( def test_run_jsonlines_pipeline_fail_unknown_table_name_raises( settings_factory: Callable[..., JsonlPydanticIngestSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """A table name in table_names that is not in the entity model registry raises ValueError.""" settings = settings_factory(table_names=["not_a_real_table"]) with pytest.raises(ValueError, match="Unknown table name"): - run_jsonlines_with_validation_pipeline(settings) + pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings) def test_run_jsonlines_pipeline_pass_table_names_none_processes_every_registered_table( @@ -60,7 +60,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( dlt_destination_config: str, entity_models_module: str, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" log_config_file = tmp_path / "logging.json" @@ -85,9 +85,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--use-output-dir-for-pipeline-metadata", "true", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, cli_args=argv[1:]) assert load_info is not None assert not load_info.has_failed_jobs diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py index 3a3d4598..fa40b0f9 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py @@ -11,6 +11,11 @@ from jsonschema import FormatChecker from jsonschema.validators import validator_for +from cdm_data_loaders.pipelines.jsonlines.settings import ( + EXTRACT_PIPELINE_NAME, + JSONSCHEMA_PIPELINE_NAME, + PYDANTIC_PIPELINE_NAME, +) from tests.cdm_data_loaders.pipelines.helpers import ( BUFFER_SIZES, LOADER_FILE_FORMATS, @@ -34,7 +39,7 @@ @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str) -@pytest.mark.parametrize("pipeline_kind", ["extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.mark.parametrize("pipeline_kind", [PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def test_jsonlines_validation_pass_reference_records_load( run_reference_pipeline: Callable[..., Any], loader_file_format: str ) -> None: @@ -297,7 +302,7 @@ def test_jsonlines_pass_nesting_matches_pipeline_contract( tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) data_tables = {name for name in tables if not name.startswith("_dlt")} columns = set().union(*(row.keys() for row in tables["dataset"])) - if pipeline_kind != "extract": + if pipeline_kind != EXTRACT_PIPELINE_NAME: assert data_tables == {"dataset"} assert {"organism", "assembly_info", "additional_submitters"} <= columns else: diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py index 664d32a3..dfa873de 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py @@ -2,18 +2,14 @@ import gzip import json -import sys from collections.abc import Callable from pathlib import Path from typing import Any import pytest -from cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline import ( - cli, - run_jsonlines_pipeline, -) from cdm_data_loaders.pipelines.jsonlines.settings import JSONSCHEMA_PIPELINE_NAME, JsonlJsonschemaIngestSettings +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import ( LOADER_FILE_FORMATS, assert_pipeline_report_written, @@ -61,13 +57,14 @@ def test_run_jsonlines_pipeline_pass_persists_valid_and_rejected_rows( schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], loader_file_format: str, buffer_size: int, + pipeline_runner: PipelineTestRunner, ) -> None: """Mixed files load every row exactly once with intact nested data and rejection provenance.""" settings = schema_settings_factory( input_dir=str(schema_input_dir), loader_file_format=loader_file_format, buffer_size=buffer_size ) - load_info = run_jsonlines_pipeline(settings) + load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) assert load_info is not None assert not load_info.has_failed_jobs @@ -104,7 +101,9 @@ def test_run_jsonlines_pipeline_pass_persists_valid_and_rejected_rows( def test_run_jsonlines_pipeline_pass_all_invalid( - schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], loader_file_format: str + schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], + loader_file_format: str, + pipeline_runner: PipelineTestRunner, ) -> None: """An entirely invalid entity still produces its complete rejection table.""" settings = schema_settings_factory(loader_file_format=loader_file_format, buffer_size=1) @@ -112,7 +111,7 @@ def test_run_jsonlines_pipeline_pass_all_invalid( table_dir.mkdir() (table_dir / "invalid.jsonl").write_text(f"{MALFORMED_RAW}\n{TYPE_ERROR_RAW}", encoding="utf-8") - load_info = run_jsonlines_pipeline(settings) + load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) assert load_info is not None assert not load_info.has_failed_jobs @@ -134,15 +133,14 @@ def test_cli_pass_loads_selected_entity( schema_input_dir: Path, schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], loader_file_format: str, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: """The real CTS CLI accepts a registry shortcut, table selection, and the requested file format.""" settings = schema_settings_factory(input_dir=str(schema_input_dir), loader_file_format=loader_file_format) - monkeypatch.setattr( - sys, - "argv", - [ - JSONSCHEMA_PIPELINE_NAME, + + load_info = pipeline_runner.run( + JSONSCHEMA_PIPELINE_NAME, + cli_args=[ "--input-dir", str(settings.input_dir), "--output-dir", @@ -164,8 +162,6 @@ def test_cli_pass_loads_selected_entity( ], ) - load_info = cli.run() - assert load_info is not None assert not load_info.has_failed_jobs assert load_info.dataset_name == "schema_cli" diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 13a6fb9b..31140384 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -1,16 +1,18 @@ """Error and boundary tests for all JSONL pipelines using assembly-report contracts.""" import json -import sys from collections.abc import Callable from pathlib import Path from typing import Any import pytest -from cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline import cli as jsonschema_cli -from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import cli as extract_cli -from cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline import cli as validate_cli +from cdm_data_loaders.pipelines.jsonlines.settings import ( + EXTRACT_PIPELINE_NAME, + JSONSCHEMA_PIPELINE_NAME, + PYDANTIC_PIPELINE_NAME, +) +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, read_pipeline_tables, run_and_read from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( canonicalize_pipeline_entry, @@ -56,9 +58,11 @@ def test_jsonlines_fail_malformed_rows_preserve_provenance( json.dumps(canonicalize_reference_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in entries ) - suffix, error_key = ("invalid", "parse_error") if pipeline_kind == "extract" else ("rejected", "error_detail") + suffix, error_key = ( + ("invalid", "parse_error") if pipeline_kind == EXTRACT_PIPELINE_NAME else ("rejected", "error_detail") + ) rejected = sorted(tables[f"dataset_{suffix}"], key=lambda row: row["source_file"]) - prefix = "dataset/" if pipeline_kind == "extract" else "" + prefix = "dataset/" if pipeline_kind == EXTRACT_PIPELINE_NAME else "" with pytest.raises(json.JSONDecodeError) as error_info: json.loads(malformed) expected_error = str(error_info.value) @@ -147,7 +151,7 @@ def test_jsonlines_pass_multiple_entities_and_file_glob( tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) expected_tables = ( {"dataset": sorted(expected.values())} - if pipeline_kind == "extract" + if pipeline_kind == EXTRACT_PIPELINE_NAME else {table_name: [accession] for table_name, accession in expected.items()} ) assert { @@ -178,7 +182,7 @@ def test_jsonlines_pass_multiple_entities_and_file_glob( pytest.param("text", "model_type", [], "'text' is not of type 'object'", id="json-string"), ], ) -@pytest.mark.parametrize("pipeline_kind", ["extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.mark.parametrize("pipeline_kind", [PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def test_jsonlines_validation_fail_schema_invalid_rows_are_rejected( *, run_reference_pipeline: Callable[..., Any], @@ -214,7 +218,7 @@ def test_jsonlines_validation_fail_schema_invalid_rows_are_rejected( assert (row["source_file"], row["line_no"]) == ("records.jsonl", 2) assert row["raw_record"] == raw_record errors = json.loads(row["error_detail"]) - if pipeline_kind == "extract_validate": + if pipeline_kind == PYDANTIC_PIPELINE_NAME: assert reconstructed_entries(tables, "dataset_rejected")[0].get("record") == record assert len(errors) == 1 assert errors[0]["type"] == error_type @@ -224,7 +228,7 @@ def test_jsonlines_validation_fail_schema_invalid_rows_are_rejected( assert errors == [error_message] -@pytest.mark.parametrize("pipeline_kind", ["extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.mark.parametrize("pipeline_kind", [PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def test_jsonlines_validation_pass_table_selection_skips_other_entities( *, run_reference_pipeline: Callable[..., Any], @@ -260,7 +264,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( tmp_path: Path, loader_file_format: str, dlt_destination_config: str, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, sorted_reference_entities: list[str], ) -> None: """All real CLI entry points load the reference dataset from command-line settings.""" @@ -268,18 +272,18 @@ def test_jsonlines_cli_pass_reference_dataset_loads( log_config.write_text('{"version": 1}', encoding="utf-8") output_dir = tmp_path / "bucket" registry_flag, registry_module = { - "extract": ("table-name", "dataset"), - "extract_validate": ( + EXTRACT_PIPELINE_NAME: ("table-name", "dataset"), + PYDANTIC_PIPELINE_NAME: ( "entity-models-module", "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models", ), - "extract_jsonschema_validate": ( + JSONSCHEMA_PIPELINE_NAME: ( "schema-files-module", "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas", ), }[pipeline_kind] argv = [ - "jsonlines", + pipeline_kind, "--input-dir", str(reference_data_dir / "chunk_52"), "--output-dir", @@ -299,13 +303,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( ] if registry_flag: argv.extend([f"--{registry_flag}", registry_module]) - monkeypatch.setattr(sys, "argv", argv) - cli = { - "extract": extract_cli, - "extract_validate": validate_cli, - "extract_jsonschema_validate": jsonschema_cli, - }[pipeline_kind] - load_info = cli.run() + load_info = pipeline_runner.run(pipeline_kind, cli_args=argv[1:]) assert load_info is not None assert not load_info.has_failed_jobs tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py index 5c1d5c00..3c7513a1 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py @@ -131,7 +131,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( log_config_file.write_text('{"version": 1}') argv = [ - "jsonlines", + EXTRACT_PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py index 7727c53f..769b2523 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py @@ -287,7 +287,7 @@ def test_settings_pass_table_selection( """JSON Schema settings retain the shared table selection contract.""" settings = schema_settings_factory(table_names=table_names) assert settings.table_names == expected - assert settings.model_config["cli_prog_name"] == "jsonlines_jsonschema_validator" + assert settings.model_config["cli_prog_name"] == "jsonlines_jsonschema" @pytest.mark.parametrize("buffer_size", [0, -1], ids=["zero", "negative"]) diff --git a/tests/cdm_data_loaders/pipelines/test_framework.py b/tests/cdm_data_loaders/pipelines/test_framework.py new file mode 100644 index 00000000..27116169 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/test_framework.py @@ -0,0 +1,412 @@ +"""Unit and integration tests for the unified pipeline test execution framework.""" + +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest +from pydantic import BaseModel + +from cdm_data_loaders.pipelines import uniprot_kb +from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import ( + cli as jsonl_extract_cli, +) +from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import ( + run_jsonlines_pipeline, +) +from cdm_data_loaders.pipelines.jsonlines.settings import JsonlExtractSettings, JsonlJsonschemaIngestSettings +from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings +from tests.cdm_data_loaders.pipelines import framework +from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import ( + CONTAINER_AUX_DIR, + CONTAINER_INPUT_DIR, + CONTAINER_OUTPUT_DIR, + CONTAINER_PYMODULES_DIR, + PIPELINE_TARGETS, + ExecutionMode, + PipelineSpec, + PipelineTestRunner, + ensure_docker_image, + reconstruct_load_info, + resolve_pipeline, +) +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + read_data_rows, + settings_to_argv, +) +from tests.conftest import LOCAL_FS + + +class _SampleEntityModel(BaseModel): + """Sample Pydantic model for dynamic module materialization tests.""" + + record_id: str + + +@pytest.mark.parametrize("target_key", sorted(PIPELINE_TARGETS)) +def test_resolve_pipeline_pass_all_registered_targets(target_key: str) -> None: + """Every registered pipeline resolves to a valid PipelineSpec with a synchronized cli_prog_name.""" + spec = resolve_pipeline(target_key) + expected_cli, expected_module = PIPELINE_TARGETS[target_key] + assert spec.cli_command == expected_cli + assert spec.module_path == expected_module + assert callable(spec.cli_callable) + assert spec.settings_cls is not None + assert spec.settings_cls.model_config.get("cli_prog_name") == expected_cli + + +def test_resolve_pipeline_pass_from_module_cli_spec_and_callable() -> None: + """resolve_pipeline accepts ModuleType, PipelineCli, PipelineSpec, and pipeline run functions.""" + from_module = resolve_pipeline(uniprot_kb) + assert from_module.cli_command == "uniprot" + assert from_module.settings_cls is uniprot_kb.UniProtSettings + assert from_module.pipeline_fn is uniprot_kb.run_uniprot_pipeline + + from_cli = resolve_pipeline(jsonl_extract_cli) + assert from_cli.cli_command == "jsonlines_extract" + assert from_cli.settings_cls is JsonlExtractSettings + assert from_cli.pipeline_fn is run_jsonlines_pipeline + + from_callable = resolve_pipeline(run_jsonlines_pipeline) + assert from_callable.cli_command == "jsonlines_extract" + assert from_callable.pipeline_fn is run_jsonlines_pipeline + + from_full_module_str = resolve_pipeline("cdm_data_loaders.pipelines.uniprot_kb") + assert from_full_module_str.cli_command == "uniprot" + + from_spec = resolve_pipeline(from_cli) + assert from_spec is from_cli + + +def test_resolve_pipeline_fail_unknown_identifier() -> None: + """resolve_pipeline raises ValueError for unknown pipeline names or unmapped callables.""" + with pytest.raises(ValueError, match="Unknown pipeline 'nonexistent_pipeline'"): + resolve_pipeline("nonexistent_pipeline") + + with pytest.raises(ValueError, match="Cannot resolve pipeline from callable"): + resolve_pipeline(lambda _settings: None) + + +def test_settings_to_argv_pass_excludes_computed_fields_and_expands_lists(tmp_path: Path) -> None: + """settings_to_argv omits computed fields and None values, formats bools, and expands list fields.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + settings = JsonlJsonschemaIngestSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="custom_ds", + loader_file_format="jsonl", + schema_files_module="some.module", + table_names=["alpha", "beta"], + ) + argv = settings_to_argv(settings) + assert "--pipeline-dir" not in argv + assert "--log-config-file" not in argv + assert "--use-output-dir-for-pipeline-metadata" in argv + idx_meta = argv.index("--use-output-dir-for-pipeline-metadata") + assert argv[idx_meta + 1] == "true" + idx_tables = argv.index("--table-names") + assert argv[idx_tables + 1 : idx_tables + 3] == ["alpha", "beta"] + + +def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: + """reconstruct_load_info returns None on missing or error reports and reconstructs completed reports.""" + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + assert reconstruct_load_info(empty_dir) is None + + error_dir = tmp_path / "error_out" + error_dir.mkdir() + (error_dir / "report_123_456.txt").write_text( + "Pipeline test_pipe dataset test_ds\nStarted: 2026-01-01 00:00:00+00:00\nStatus: error\nPipeline failed\n", + encoding="utf-8", + ) + assert reconstruct_load_info(error_dir) is None + + completed_dir = tmp_path / "completed_out" + completed_dir.mkdir() + started_str = "2026-01-01 12:34:56.789000+00:00" + (completed_dir / "report_111_222.txt").write_text( + f"Pipeline test_pipe dataset test_ds\n" + f"Started: {started_str}\n" + f"Status: completed with errors\n" + f"Completed loads: 111, 222\n" + f"Pipeline load info saved\n", + encoding="utf-8", + ) + reconstructed = reconstruct_load_info(completed_dir) + assert reconstructed is not None + assert reconstructed.pipeline.pipeline_name == "test_pipe" + assert reconstructed.dataset_name == "test_ds" + assert str(reconstructed.started_at) == started_str + assert reconstructed.has_failed_jobs is True + assert reconstructed.loads_ids == ["111", "222"] + as_dict = reconstructed.asdict() + assert as_dict["dataset_name"] == "test_ds" + assert as_dict["loads_ids"] == ["111", "222"] + + +def test_ensure_docker_image_pass_caches_and_fail_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """ensure_docker_image builds once per tag and raises RuntimeError when docker build fails.""" + monkeypatch.setattr(framework, "_built_docker_images", set()) + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(cmd) + return subprocess.CompletedProcess(cmd, returncode=0, stdout="built", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + tag = ensure_docker_image("custom-image:test") + assert tag == "custom-image:test" + assert len(calls) == 1 + assert calls[0][:4] == ["docker", "build", "-t", "custom-image:test"] + + # Second call uses cache without invoking subprocess.run + assert ensure_docker_image("custom-image:test") == "custom-image:test" + assert len(calls) == 1 + + # Failing build raises RuntimeError + def fake_fail(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(cmd, returncode=1, stdout="", stderr="build boom") + + monkeypatch.setattr(subprocess, "run", fake_fail) + with pytest.raises(RuntimeError, match="Docker image build failed"): + ensure_docker_image("failing-image:test") + + +@pytest.mark.parametrize("mode", [ExecutionMode.CORE, ExecutionMode.CLI], ids=str) +def test_pipeline_test_runner_pass_core_and_cli_modes_end_to_end( + tmp_path: Path, + mode: ExecutionMode, +) -> None: + """PipelineTestRunner executes a pipeline in core and cli modes and hooks into all result assertions.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + expected_records = [{"id": "rec_1", "value": 10}, {"id": "rec_2", "value": 20}] + (input_dir / "data.jsonl").write_text( + "\n".join(f'{{"id": "{r["id"]}", "value": {r["value"]}}}' for r in expected_records) + "\n", + encoding="utf-8", + ) + output_dir = tmp_path / "output" + output_dir.mkdir() + + settings = JsonlExtractSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="framework_test_ds", + table_name="items", + loader_file_format="jsonl", + ) + + runner = PipelineTestRunner(mode=mode) + load_info = assert_run_ok( + runner( + "jsonlines_extract", + settings=settings, + ) + ) + + assert_pipeline_report_written(output_dir, load_info, metadata_moved=True) + assert set(load_info.pipeline.dataset().tables) >= set(DEFAULT_DLT_TABLES) + assert read_data_rows(output_dir / settings.dataset_name) == {"items": expected_records} + + +def test_pipeline_test_runner_pass_docker_mode_command_and_staging( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Docker mode stages input, dlt config, auxiliary files, dynamic modules, and reconstructs LoadInfo.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + (input_dir / "sample.xml").write_text("1", encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + xsd_file = tmp_path / "schema.xsd" + xsd_file.write_text("", encoding="utf-8") + + # Create dynamic in-memory modules for both ENTITY_SCHEMAS and ENTITY_MODELS + dyn_schema_mod_name = "_test_dynamic_schema_mod" + dyn_schema_mod = ModuleType(dyn_schema_mod_name) + dyn_schema_mod.ENTITY_SCHEMAS = {"widget": {"type": "object"}} # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, dyn_schema_mod_name, dyn_schema_mod) + + dyn_model_mod_name = "_test_dynamic_model_mod" + dyn_model_mod = ModuleType(dyn_model_mod_name) + dyn_model_mod.ENTITY_MODELS = {"widget": _SampleEntityModel} # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, dyn_model_mod_name, dyn_model_mod) + + monkeypatch.setattr(framework, "_built_docker_images", {"test-image:latest"}) + captured_cmds: list[list[str]] = [] + + def fake_docker_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + captured_cmds.append(cmd) + # Simulate the container writing a completed report to the mounted output_dir + (output_dir / "report_999_1000.txt").write_text( + "Pipeline xmltodict dataset xml_docker_ds\n" + "Started: 2026-02-01 00:00:00+00:00\n" + "Status: completed\n" + "Completed loads: 999\n" + "Pipeline load info saved\n", + encoding="utf-8", + ) + return subprocess.CompletedProcess(cmd, returncode=0, stdout="ok", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_docker_run) + + settings = XmlToDictSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="xml_docker_ds", + table_name="entries", + xml_tag="entry", + loader_file_format="parquet", + ) + + runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image="test-image:latest") + cli_args = [ + *settings_to_argv(settings), + "--xsd-file", + str(xsd_file), + "--schema-files-module", + dyn_schema_mod_name, + "--entity-models-module", + dyn_model_mod_name, + ] + load_info = runner.run("xmltodict", cli_args=cli_args) + + assert load_info is not None + assert load_info.pipeline.pipeline_name == "xmltodict" + assert load_info.dataset_name == "xml_docker_ds" + assert len(captured_cmds) == 1 + + docker_cmd = captured_cmds[0] + assert docker_cmd[:3] == ["docker", "run", "--rm"] + idx_image = docker_cmd.index("test-image:latest") + assert docker_cmd[idx_image : idx_image + 3] == ["test-image:latest", "pipeline", "xmltodict"] + + container_args = docker_cmd[idx_image + 3 :] + idx_in = container_args.index("--input-dir") + assert container_args[idx_in + 1] == CONTAINER_INPUT_DIR + idx_out = container_args.index("--output-dir") + assert container_args[idx_out + 1] == CONTAINER_OUTPUT_DIR + idx_xsd = container_args.index("--xsd-file") + assert container_args[idx_xsd + 1].startswith(f"{CONTAINER_AUX_DIR}/") + assert f"PYTHONPATH={CONTAINER_PYMODULES_DIR}:/app/src:/app" in docker_cmd + + +def test_pipeline_test_runner_fail_docker_mode_nonzero_exit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Docker mode raises ValueError on validation errors and RuntimeError on general container failures.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + monkeypatch.setattr(framework, "_built_docker_images", {"test-image:latest"}) + + def fake_fail_runtime(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(cmd, returncode=2, stdout="", stderr="container crashed") + + monkeypatch.setattr(subprocess, "run", fake_fail_runtime) + runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image="test-image:latest") + with pytest.raises(RuntimeError, match="failed with exit code 2"): + runner.run( + "jsonlines_extract", + cli_args=["--input-dir", str(input_dir), "--output-dir", str(output_dir)], + ) + + def fake_fail_value_error(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(cmd, returncode=1, stdout="", stderr="ValueError: Unknown table name: foo") + + monkeypatch.setattr(subprocess, "run", fake_fail_value_error) + with pytest.raises(ValueError, match="Unknown table name"): + runner.run( + "jsonlines_extract", + cli_args=["--input-dir", str(input_dir), "--output-dir", str(output_dir)], + ) + + +def test_pipeline_test_runner_fail_missing_required_arguments(tmp_path: Path) -> None: + """PipelineTestRunner raises ValueError when output_dir or input_dir (in docker mode) is missing.""" + runner = PipelineTestRunner(mode=ExecutionMode.CORE) + with pytest.raises(ValueError, match="Could not determine output_dir"): + runner.run("jsonlines_extract", cli_args=["--input-dir", str(tmp_path)]) + + spec_no_settings = PipelineSpec(cli_command="custom", module_path="custom.mod") + with pytest.raises(ValueError, match="has no settings_cls"): + runner.run(spec_no_settings) + + with pytest.raises(ValueError, match="does not expose settings_cls and pipeline_fn"): + runner.run(spec_no_settings, cli_args=["--output-dir", str(tmp_path)]) + + cli_runner = PipelineTestRunner(mode=ExecutionMode.CLI) + with pytest.raises(ValueError, match="does not expose a cli\\(\\) entry point"): + cli_runner.run(spec_no_settings, cli_args=["--output-dir", str(tmp_path)]) + + docker_runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image="test-image:latest") + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(framework, "_built_docker_images", {"test-image:latest"}) + try: + with pytest.raises(ValueError, match="Docker execution mode requires --input-dir"): + docker_runner.run("jsonlines_extract", cli_args=["--output-dir", str(tmp_path)]) + finally: + monkeypatch.undo() + + +def test_resolve_pipeline_pass_legacy_module_without_pipeline_cli() -> None: + """resolve_pipeline handles modules whose cli attribute is a plain function rather than PipelineCli.""" + legacy_mod = ModuleType("cdm_data_loaders.pipelines.custom_legacy") + legacy_mod.Settings = JsonlExtractSettings # type: ignore[attr-defined] + legacy_mod.run_pipeline = run_jsonlines_pipeline # type: ignore[attr-defined] + legacy_mod.cli = lambda: None # type: ignore[attr-defined] + spec = resolve_pipeline(legacy_mod) + assert spec.cli_command == "custom_legacy" + assert spec.settings_cls is JsonlExtractSettings + assert spec.pipeline_fn is run_jsonlines_pipeline + assert spec.cli_callable is legacy_mod.cli + + +@pytest.mark.docker +def test_pipeline_test_runner_pass_real_docker_container(tmp_path: Path, docker_image: str) -> None: + """Execute a real Docker container run via PipelineTestRunner when --run-docker is enabled.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + expected_records = [{"id": "docker_1", "value": 42}] + (input_dir / "data.jsonl").write_text('{"id": "docker_1", "value": 42}\n', encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + + settings = JsonlExtractSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="docker_framework_ds", + table_name="items", + loader_file_format="jsonl", + ) + + runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image=docker_image) + load_info = assert_run_ok(runner.run("jsonlines_extract", settings=settings)) + + assert_pipeline_report_written(output_dir, load_info, metadata_moved=True) + assert set(load_info.pipeline.dataset().tables) >= set(DEFAULT_DLT_TABLES) + assert read_data_rows(output_dir / settings.dataset_name) == {"items": expected_records} diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py index 5c0c6b49..b52061b7 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py @@ -33,8 +33,8 @@ def test_run_uniprot_pipeline_args_set_correctly(test_settings: UniProtSettings) _, kwargs = mock_run_pipeline.call_args assert kwargs.keys() == {"settings", "resource", "pipeline_kwargs"} assert kwargs["pipeline_kwargs"] == { - "pipeline_name": "uniprot_kb", - "dataset_name": "uniprot_kb", + "pipeline_name": "uniprot", + "dataset_name": "uniprot", } assert kwargs["settings"] == test_settings assert isinstance(kwargs["resource"], Callable) @@ -57,8 +57,8 @@ def test_run_uniprot_pipeline_sets_core_run_pipeline_args_correctly( mock_dlt.destination.assert_called_once_with(test_settings.use_destination) mock_dlt.pipeline.assert_called_once_with( - pipeline_name="uniprot_kb", - dataset_name="uniprot_kb", + pipeline_name="uniprot", + dataset_name="uniprot", ) assert mock_dlt.pipeline.return_value.destination == mock_dlt.destination.return_value first_run, load_info_save_run = mock_dlt.pipeline.return_value.run.call_args_list diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py index a01d44db..49103b80 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py @@ -4,20 +4,19 @@ per-table counts below are derived from those fixtures. """ -from collections.abc import Callable from pathlib import Path from typing import Any import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines import uniprot_kb as uniprot_module from cdm_data_loaders.pipelines.uniprot_kb import ( UniProtSettings, - cli, parse_uniprot, ) -from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner +from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.conftest import LOCAL_FS, TEST_DATA_DIR UNIPROT_FIXTURE_DIR = TEST_DATA_DIR / "uniprot" / "uniprot_kb" / "chunk_4" @@ -48,7 +47,7 @@ def duckdb_uniprot_settings(tmp_path: Path) -> UniProtSettings: input_dir=str(UNIPROT_FIXTURE_DIR), output_dir=str(output_dir), use_destination=LOCAL_FS, - use_output_dir_for_pipeline_metadata=False, + use_output_dir_for_pipeline_metadata=True, ) # pyright: ignore[reportCallIssue] @@ -113,43 +112,18 @@ def test_integration_uniprot_pipeline_entity_and_source_file_content( def test_integration_cli_uniprot_pipeline_output_validated( duckdb_uniprot_settings: UniProtSettings, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: - """Exercise the real ``cli.run()`` wiring end-to-end against a DuckDB destination.""" - captured: dict[str, Any] = {} - - def fake_run_cli( - settings_cls: type[UniProtSettings], - pipeline_fn: Callable[..., Any], - cli_args: list[str] | None = None, - settings_kwargs: dict[str, Any] | None = None, - ) -> Any: - """Replacement for core.run_cli that dispatches to pipeline_fn with the fixture settings.""" - assert settings_cls is UniProtSettings - return pipeline_fn(duckdb_uniprot_settings) - - def fake_run_pipeline( - *, - settings: UniProtSettings, - resource: Any, - pipeline_kwargs: dict[str, Any], - ) -> None: - """Replacement for core.run_pipeline that runs the resource through DuckDB.""" - assert settings is duckdb_uniprot_settings - assert pipeline_kwargs == {"pipeline_name": "uniprot_kb", "dataset_name": "uniprot_kb"} - pipeline = duckdb_pipeline(tmp_path, "test_uniprot_cli_pipeline", "test_uniprot_cli") - captured["load_info"] = pipeline.run(resource) - captured["pipeline"] = pipeline - - monkeypatch.setattr("cdm_data_loaders.pipelines.core.run_cli", fake_run_cli) - monkeypatch.setattr(uniprot_module, "run_pipeline", fake_run_pipeline) - - cli.run() - - load_info = captured["load_info"] - pipeline = captured["pipeline"] + """Exercise the UniProt pipeline end-to-end across core, cli, and docker execution modes.""" + load_info = pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings) + + assert load_info is not None assert not load_info.has_failed_jobs + assert load_info.pipeline.pipeline_name == "uniprot" + assert load_info.dataset_name == "uniprot" + assert_pipeline_report_written(Path(duckdb_uniprot_settings.output_dir), load_info, metadata_moved=True) - ((entity_count,),) = _fetch_all(pipeline, "SELECT COUNT(*) FROM entity") - assert entity_count == UNIPROT_ENTRY_COUNT + dataset_dir = Path(duckdb_uniprot_settings.output_dir) / load_info.dataset_name + for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): + rows = read_destination_rows(dataset_dir, table_name) + assert len(rows) == expected_count, f"expected {expected_count} rows in '{table_name}', got {len(rows)}" diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py index 8dd9bef7..cf047063 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py @@ -4,21 +4,20 @@ derived from that fixture. Every test runs once per valid UniRef variant. """ -from collections.abc import Callable from pathlib import Path from typing import Any import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines import uniref as uniref_module from cdm_data_loaders.pipelines.uniref import ( UNIREF_VARIANTS, UnirefSettings, - cli, parse_uniref, ) -from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner +from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.conftest import LOCAL_FS, TEST_DATA_DIR TEST_DEFAULT_UNIREF_VARIANT = "50" @@ -49,7 +48,7 @@ def duckdb_uniref_settings(tmp_path: Path, request: pytest.FixtureRequest) -> Un input_dir=str(UNIREF_FIXTURE_DIR), output_dir=str(output_dir), use_destination=LOCAL_FS, - use_output_dir_for_pipeline_metadata=False, + use_output_dir_for_pipeline_metadata=True, ) # pyright: ignore[reportCallIssue] @@ -118,48 +117,18 @@ def test_integration_uniref_pipeline_entity_and_cluster_content( def test_integration_cli_uniref_pipeline_output_validated( duckdb_uniref_settings: UnirefSettings, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: - """Exercise the real ``cli.run()`` wiring end-to-end against a DuckDB destination.""" - captured: dict[str, Any] = {} - - def fake_run_cli( - settings_cls: type[UnirefSettings], - pipeline_fn: Callable[..., Any], - cli_args: list[str] | None = None, - settings_kwargs: dict[str, Any] | None = None, - ) -> Any: - """Replacement for core.run_cli that dispatches to pipeline_fn with the fixture settings.""" - assert settings_cls is UnirefSettings - return pipeline_fn(duckdb_uniref_settings) - - def fake_run_pipeline( - *, - settings: UnirefSettings, - resource: Any, - pipeline_kwargs: dict[str, Any], - ) -> None: - """Replacement for core.run_pipeline that runs the resource through DuckDB.""" - assert settings is duckdb_uniref_settings - assert pipeline_kwargs == { - "pipeline_name": "uniref", - "dataset_name": f"uniref_{duckdb_uniref_settings.variant}", - } - pipeline = duckdb_pipeline( - tmp_path, f"test_uniref_cli_pipeline_{duckdb_uniref_settings.variant}", "test_uniref_cli" - ) - captured["load_info"] = pipeline.run(resource) - captured["pipeline"] = pipeline - - monkeypatch.setattr("cdm_data_loaders.pipelines.core.run_cli", fake_run_cli) - monkeypatch.setattr(uniref_module, "run_pipeline", fake_run_pipeline) - - cli.run() - - load_info = captured["load_info"] - pipeline = captured["pipeline"] + """Exercise the UniRef pipeline end-to-end across core, cli, and docker execution modes.""" + load_info = pipeline_runner.run("uniref", settings=duckdb_uniref_settings) + + assert load_info is not None assert not load_info.has_failed_jobs + assert load_info.pipeline.pipeline_name == "uniref" + assert load_info.dataset_name == f"uniref_{duckdb_uniref_settings.variant}" + assert_pipeline_report_written(Path(duckdb_uniref_settings.output_dir), load_info, metadata_moved=True) - ((entity_count,),) = _fetch_all(pipeline, "SELECT COUNT(*) FROM entity") - assert entity_count == UNIREF_CLUSTER_COUNT + dataset_dir = Path(duckdb_uniref_settings.output_dir) / load_info.dataset_name + for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): + rows = read_destination_rows(dataset_dir, table_name) + assert len(rows) == expected_count, f"expected {expected_count} rows in '{table_name}', got {len(rows)}" diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py index eb18406a..7e963918 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py @@ -1,16 +1,14 @@ """End-to-end tests for the xml2db pipeline, using a small library.xsd schema.""" -import sys from collections.abc import Callable from pathlib import Path -import pytest from dlt.common.pipeline import LoadInfo from pandas import DataFrame -from cdm_data_loaders.pipelines.xml2db.pipeline import cli, run_xml2db_pipeline from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.xml_samples import SIMPLE_LIBRARY_XML @@ -32,6 +30,7 @@ def check_book_list_results(load_info: LoadInfo | None) -> tuple[DataFrame, Data def test_run_xml2db_pipeline_pass_writes_expected_output( settings_factory: Callable[..., Xml2DbSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """Running the pipeline loads one row per library root and one row per book.""" settings = settings_factory() @@ -39,7 +38,7 @@ def test_run_xml2db_pipeline_pass_writes_expected_output( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] @@ -50,6 +49,7 @@ def test_run_xml2db_pipeline_pass_writes_expected_output( def test_run_xml2db_pipeline_pass_gzip_files_are_loaded( settings_factory: Callable[..., Xml2DbSettings], write_gzip_file: Callable[[Path, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """Gzip-compressed xml files matching the glob are decompressed and loaded. @@ -64,7 +64,7 @@ def test_run_xml2db_pipeline_pass_gzip_files_are_loaded( (input_dir / "plain.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") write_gzip_file(input_dir, "library.xml.gz", SIMPLE_LIBRARY_XML) - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) library_df, book_df = check_book_list_results(load_info) # one root row from each of the two files: the root table is never compacted assert len(library_df) == EXPECTED_LIBRARY_ROOT_COUNT_TWO_FILES @@ -74,12 +74,13 @@ def test_run_xml2db_pipeline_pass_gzip_files_are_loaded( def test_run_xml2db_pipeline_pass_no_matching_files_yields_no_data_table( settings_factory: Callable[..., Xml2DbSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """An input dir with no matching xml files does not fail the run and creates no data table.""" settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() @@ -90,6 +91,7 @@ def test_run_xml2db_pipeline_pass_no_matching_files_yields_no_data_table( def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( settings_factory: Callable[..., Xml2DbSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """The `short_name` setting only affects the virtual root table name for multi-root schemas. @@ -102,7 +104,7 @@ def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] @@ -110,9 +112,9 @@ def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, dlt_destination_config: str, library_xsd: Path, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" input_dir = tmp_path / "cli_input" @@ -142,9 +144,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--dlt-dev-mode", "false", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py index 9182c7e1..b76d2803 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py @@ -1,6 +1,5 @@ """End-to-end tests for the xmltodict pipeline.""" -import sys from collections.abc import Callable from pathlib import Path @@ -8,12 +7,9 @@ from dlt.common.pipeline import LoadInfo from pandas import DataFrame -from cdm_data_loaders.pipelines.xmltodict.pipeline import ( - cli, - run_xmltodict_pipeline, -) -from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings +from cdm_data_loaders.pipelines.xmltodict.settings import PIPELINE_NAME, XmlToDictSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, assert_pipeline_report_written from tests.xml_samples import SIMPLE_LIBRARY_XML @@ -34,6 +30,7 @@ def check_book_list_results(load_info: LoadInfo | None, table_name: str | None = def test_run_xmltodict_pipeline_pass_writes_expected_output( settings_factory: Callable[..., XmlToDictSettings], loader_file_format: str, + pipeline_runner: PipelineTestRunner, ) -> None: """Running the pipeline on xml files loads one row per matching element into the configured table.""" settings = settings_factory(loader_file_format=loader_file_format) @@ -41,7 +38,7 @@ def test_run_xmltodict_pipeline_pass_writes_expected_output( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -50,6 +47,7 @@ def test_run_xmltodict_pipeline_pass_writes_expected_output( def test_run_xmltodict_pipeline_pass_gzip_files_are_loaded( settings_factory: Callable[..., XmlToDictSettings], write_gzip_file: Callable[[Path, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """Gzip-compressed xml files matching the glob are decompressed and loaded.""" settings = settings_factory() @@ -58,7 +56,7 @@ def test_run_xmltodict_pipeline_pass_gzip_files_are_loaded( (input_dir / "plain.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") write_gzip_file(input_dir, "library.xml.gz", SIMPLE_LIBRARY_XML) - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) book_df = check_book_list_results(load_info) # one copy of the three books from the plain file, one from the gzipped copy assert sorted(book_df["book___id"].tolist()) == ["1", "1", "2", "2", "3", "3"] @@ -66,12 +64,13 @@ def test_run_xmltodict_pipeline_pass_gzip_files_are_loaded( def test_run_xmltodict_pipeline_pass_no_matching_files_yields_no_data_table( settings_factory: Callable[..., XmlToDictSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """An input dir with no matching xml files does not fail the run and creates no data table.""" settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() @@ -82,6 +81,7 @@ def test_run_xmltodict_pipeline_pass_no_matching_files_yields_no_data_table( def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( settings_factory: Callable[..., XmlToDictSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """Rows land in the table named by table_name, not in a name derived from the xml tag.""" settings = settings_factory(table_name="library_entries") @@ -89,7 +89,7 @@ def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) book_df = check_book_list_results(load_info, "library_entries") # the record dict is keyed by the xml tag, so the flattened column prefix is `book` assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] @@ -97,8 +97,8 @@ def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, dlt_destination_config: str, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" input_dir = tmp_path / "cli_input" @@ -110,7 +110,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", @@ -130,9 +130,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--dlt-dev-mode", "false", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] assert_pipeline_report_written(output_dir, load_info) diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py index 11c1304f..74e788fb 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py @@ -93,7 +93,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", @@ -158,7 +158,7 @@ def test_cli_pass_writes_rows_for_matching_xml_files( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", @@ -223,7 +223,7 @@ def test_cli_fail_missing_required_args_raise( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(tmp_path / "input"), "--output-dir", diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py index 5424fa8a..425fe9fd 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py @@ -1,6 +1,5 @@ """End-to-end tests for the xsv pipeline, using a real qsv binary and local_fs destination.""" -import sys from collections.abc import Callable from pathlib import Path from uuid import uuid4 @@ -8,9 +7,9 @@ import pandas as pd import pytest -from cdm_data_loaders.pipelines.xsv.pipeline import cli, run_xsv_pipeline from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.cdm_data_loaders.pipelines.xsv.integration.conftest import ( ALL_RAGGED_ROWS, @@ -29,12 +28,13 @@ def test_run_xsv_pipeline_pass_writes_expected_output( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A clean, valid file is loaded, with every row cast to its declared schema types.""" settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -52,13 +52,14 @@ def test_run_xsv_pipeline_pass_writes_expected_output( def test_run_xsv_pipeline_pass_multiple_files_loaded_together( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """Every file matching file_glob is cleaned, validated, and loaded into the same table.""" settings = settings_factory() write_xsv_file(settings, "part_1.csv", rows_to_csv([VALID_ROWS[0]])) write_xsv_file(settings, "part_2.csv", rows_to_csv([VALID_ROWS[1]])) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -69,6 +70,7 @@ def test_run_xsv_pipeline_pass_multiple_files_loaded_together( def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A file with one ragged row loads its surviving valid rows AND records the row loss. @@ -79,7 +81,7 @@ def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(PARTIAL_RAGGED_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -98,12 +100,13 @@ def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( def test_run_xsv_pipeline_pass_fully_ragged_file_is_entirely_rejected( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A file where every row is ragged yields no valid rows; the failure is recorded as rejected.""" settings = settings_factory() write_xsv_file(settings, "bad.csv", rows_to_csv(ALL_RAGGED_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -118,11 +121,12 @@ def test_run_xsv_pipeline_pass_fully_ragged_file_is_entirely_rejected( def test_run_xsv_pipeline_pass_no_matching_files_yields_no_data_table( settings_factory: Callable[..., XsvIngestSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """An input dir with no matching data files does not fail the run and creates no data table.""" settings = settings_factory() - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -133,6 +137,7 @@ def test_run_xsv_pipeline_pass_no_matching_files_yields_no_data_table( def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A schema-configured null_regex placeholder is replaced with a null value before loading.""" schema = { @@ -148,7 +153,7 @@ def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( ), ) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -161,6 +166,7 @@ def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """The pipeline also loads successfully when loader_file_format is parquet. @@ -171,7 +177,7 @@ def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( settings = settings_factory(loader_file_format="parquet") write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -182,7 +188,7 @@ def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( def test_cli_pass_runs_end_to_end_from_command_line_arguments( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" settings = settings_factory(dataset_name=f"cli_dataset_{uuid4().hex}") @@ -207,9 +213,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--dlt-dev-mode", "false", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) assert load_info is not None assert load_info.has_failed_jobs is False diff --git a/tests/conftest.py b/tests/conftest.py index fc0f7c90..46182b59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,6 +62,32 @@ _notebook_utils_available: bool | None = None +def pytest_addoption(parser: pytest.Parser) -> None: + """Register custom command-line options for running Docker container pipeline tests.""" + parser.addoption( + "--run-docker", + action="store_true", + default=False, + help="Build and run Docker container tests marked with @pytest.mark.docker.", + ) + parser.addoption( + "--docker-image", + action="store", + default=None, + help="Pre-built Docker image tag to use for @pytest.mark.docker tests instead of building.", + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Skip tests marked with ``docker`` unless ``--run-docker`` was passed on the command line.""" + if config.getoption("--run-docker"): + return + skip_docker = pytest.mark.skip(reason="need --run-docker option to run Docker container tests") + for item in items: + if "docker" in item.keywords: + item.add_marker(skip_docker) + + def _find_notebook_utils() -> bool: """Check whether the current environment has the KBase Lakehouse notebook_utils lib available. diff --git a/uv.lock b/uv.lock index 31c28b51..26009a94 100644 --- a/uv.lock +++ b/uv.lock @@ -2,14 +2,15 @@ version = 1 revision = 5 requires-python = ">=3.13" resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", + "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -364,7 +365,7 @@ wheels = [ [[package]] name = "bioregistry" -version = "0.15.3" +version = "0.15.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -379,9 +380,9 @@ dependencies = [ { name = "tqdm" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/b5/a12e14e450da63ee293a5c3ca536461de9d6fbf2a79c716baa78f3212bfb/bioregistry-0.15.3.tar.gz", hash = "sha256:0a6e3f326c7935138434745a02d5184019e13aeaf8cf8520784d2d15fe3d1a7c", size = 7271806, upload-time = "2026-10-10T06:01:05.259Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/4ea7b1537369526482a77a635e1725c45ea3f314e29f2c1eb572bb0fa82e/bioregistry-0.15.5.tar.gz", hash = "sha256:63fa06045d906c71edf1c5a8a81afea18d083fb8613d3d81ca7485f79433764b", size = 7275690, upload-time = "2026-10-17T04:33:53.438Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/fb/549e0af1ba4597578d0d08ef21bbc823b4e760c02d6a1bf6a4814d399518/bioregistry-0.15.3-py3-none-any.whl", hash = "sha256:a714f088375adb46a78937d4906941011a4a7ebd58ed77ca6f82c76d5d89b34c", size = 7345811, upload-time = "2026-10-10T06:01:01.804Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b1/6f2e5efbbc62c2bf762b1de5433f51c004d793463fbdd2d3aafe3b515854/bioregistry-0.15.5-py3-none-any.whl", hash = "sha256:342a7a8a98aeccb70655cc4f7ba87e5c1deba16a3ccf0941a06943c662de8aaa", size = 7349749, upload-time = "2026-10-17T04:33:51.748Z" }, ] [[package]] @@ -451,20 +452,20 @@ crt = [ [[package]] name = "botocore-stubs" -version = "1.43.113" +version = "1.43.115" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/87/86c5d1ab625e8253547e2c1a967b2575c7b00df27d7158cf636d16935b39/botocore_stubs-1.43.113.tar.gz", hash = "sha256:49deda166eceee64243cca0b5dc9de5ce005949dee9ead477f8df22bafaee374", size = 42883, upload-time = "2026-10-13T22:31:59.108Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/a1/aaf4f30377a712b607a7b829e6257933a6857ef1aeafb84362d75e22314e/botocore_stubs-1.43.115.tar.gz", hash = "sha256:8dc2089a0eb8109faa6dc07b1916ddb3d8d33bd8e2c42825f13309dd4176879d", size = 42908, upload-time = "2026-10-19T20:58:21.643Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/34/047044056dfae6e5206fafdac4c359bc2055f43827252087caa4eb041efe/botocore_stubs-1.43.113-py3-none-any.whl", hash = "sha256:716fdf4650fbcca50ec5086882d749f9d0f68047370830d9203a2de297c5515f", size = 67249, upload-time = "2026-10-13T22:31:56.771Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8f/3f84f0f4af41074ce938c47c5ba8ea4c7afa7e74c07ac0d62b78fd32b076/botocore_stubs-1.43.115-py3-none-any.whl", hash = "sha256:0fa02aedd7228b4b777d61bb720ec14dff92b59413c3cc891fff05a78d66a08d", size = 67247, upload-time = "2026-10-19T20:58:19.875Z" }, ] [[package]] name = "cachetools" -version = "7.2.1" +version = "7.2.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/44/71476a5812da1ddf2c9a3efd31ae76d01480a1cf03ed13ac28aa8f2402e4/cachetools-7.2.1.tar.gz", hash = "sha256:b1a7537025c06abf96fcc1443e496af9a3fb95e774e70e1f0af226f73f7f2dcc", size = 41357, upload-time = "2026-10-05T18:40:06.361Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/5d/15ea0494ebcb9062c7ca79bf026cbba380a50538ae73753fb40bdffa79e2/cachetools-7.2.4.tar.gz", hash = "sha256:e0203b04b75fe13e62098eeb5b2018c4eb1a14c04e2b5b450e70db3ee7e92ec5", size = 44251, upload-time = "2026-10-19T19:09:44.931Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/c9/2a61d784caf0d869a3326728c57c7203f50cc53f3cca2ee76bf924769eb4/cachetools-7.2.1-py3-none-any.whl", hash = "sha256:63aa53dfe7473c10cccdd5a01dedf76ef2c4b73a58840d9396e7d0752cbdac3b", size = 17006, upload-time = "2026-10-05T18:40:04.827Z" }, + { url = "https://files.pythonhosted.org/packages/5a/4c/694f7d44f2399b3901232d6ded1c88442a02fc8141936396d9289d951848/cachetools-7.2.4-py3-none-any.whl", hash = "sha256:4cfbebf3eec4040d2073b19c460511ad47eef7b852dbf9a0d385099b4dafa31e", size = 18571, upload-time = "2026-10-19T19:09:43.296Z" }, ] [[package]] @@ -518,6 +519,10 @@ models = [ { name = "genson" }, { name = "json2python-models" }, ] +mutants = [ + { name = "mutmut" }, + { name = "pytest-gremlins" }, +] xml = [ { name = "xsdata", extra = ["cli", "lxml"] }, ] @@ -544,7 +549,7 @@ requires-dist = [ { name = "pandas", specifier = ">=3.0.3" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pydantic-settings", specifier = ">=2.14.1" }, - { name = "pyiceberg", extras = ["pandas", "pyarrow", "pyiceberg-core", "snappy"], specifier = ">=0.11.1" }, + { name = "pyiceberg", extras = ["pandas", "pyarrow", "pyiceberg-core", "snappy"], specifier = ">=0.12" }, { name = "smart-open", specifier = ">=8.0.1" }, { name = "tqdm", specifier = ">=4.70.1" }, { name = "xml2db", specifier = ">=0.14.0" }, @@ -571,15 +576,19 @@ models = [ { name = "genson", specifier = ">=1.3.0" }, { name = "json2python-models", specifier = ">=0.3.1" }, ] +mutants = [ + { name = "mutmut", specifier = ">=3.8.0" }, + { name = "pytest-gremlins", specifier = ">=1.12.2" }, +] xml = [{ name = "xsdata", extras = ["cli", "lxml"], specifier = ">=26.2" }] [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -666,37 +675,37 @@ sdist = { url = "https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d [[package]] name = "chardet" -version = "7.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/cd61c567092a6cec796144510a68aff158ebfc1df82950a45bae65f28413/chardet-7.6.0.tar.gz", hash = "sha256:93d9df6089ded42ed1fe9f57e272c0b74bd0464d45c0c7d50f09f26f31105c3c", size = 914462, upload-time = "2026-08-14T20:36:59.305Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/29/16a7419edfbd60e901e6a797cbc3e038cb2a81903bc16c029db755f0156f/chardet-7.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:57e6846cc13ce1ff59979f4ec9da770c57e12aa99046073f632de5a51d9a6f20", size = 1088449, upload-time = "2026-08-14T20:36:34.723Z" }, - { url = "https://files.pythonhosted.org/packages/1d/36/3a14b0f8ddeb302f157281ca656a3ce6874b78e2d6af03682f520b487245/chardet-7.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:089e3bb81a0a07e94f15461ded9f9ee66d349615b1a9fd557d4de1003e2fc12e", size = 1065126, upload-time = "2026-08-14T20:36:36.21Z" }, - { url = "https://files.pythonhosted.org/packages/d2/4c/f59a39c2bfe4ac99baba8da842e8d2ea0b84dff7ba53a96e7ad8c71602d7/chardet-7.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43ea433e43a23c55e8e17f3fad1e07f5cfe5450c73124b95b0d849c21ad379ee", size = 1482492, upload-time = "2026-08-14T20:36:37.649Z" }, - { url = "https://files.pythonhosted.org/packages/bd/eb/93e8036681157f2217a18769927a035984526e6dbd5e91f28a375ca41c14/chardet-7.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b5d31f9b7f793e15e81cca877e7ccd72bffffa2a3443a9d47be9dfee84fad69", size = 1514185, upload-time = "2026-08-14T20:36:39.119Z" }, - { url = "https://files.pythonhosted.org/packages/10/04/0066d7ab2c135e404a6fa166bb7fa49d1c7bf7af07b86ed95b1c48a348c7/chardet-7.6.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c54b6a8d3b219560fa5cf4c28df932c37471afe047afdc152067104e741f38c1", size = 1454420, upload-time = "2026-08-14T20:36:40.549Z" }, - { url = "https://files.pythonhosted.org/packages/6c/9f/e965f1d9eddfb86cf118f980d329cbee43c4ac4b562448b369a6b6ef36af/chardet-7.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:b3b4c96c4df93899b3c8b9e8159e06b1f55c66d7ca384d91481108e251a06eb0", size = 1159067, upload-time = "2026-08-14T20:36:41.816Z" }, - { url = "https://files.pythonhosted.org/packages/cf/78/e14991c9487277ed7d006fff58f3084ac792ed94cced081788abb95df70c/chardet-7.6.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c6061adf247ab5dda173b67010e13904c6071717660c7c8077fb50aca362b264", size = 1087678, upload-time = "2026-08-14T20:36:43.2Z" }, - { url = "https://files.pythonhosted.org/packages/7a/40/0f95e04cb1820e0a582cd6d86bbf26be8302a94ccf330f8ba5f69735389d/chardet-7.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fc1e1571321baf8927582fe34363ad7f02279f11c8c2839c14b4c76894148db6", size = 1065310, upload-time = "2026-08-14T20:36:44.489Z" }, - { url = "https://files.pythonhosted.org/packages/3b/a1/04404dcc9d6e1253b02583e562d2c7ddca50a7e91f460d62eff7e1d07c92/chardet-7.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8900f6c7cf6b015b17a51767cc6144689059ba1cdceaa383d29eb037ac28579e", size = 1485028, upload-time = "2026-08-14T20:36:45.797Z" }, - { url = "https://files.pythonhosted.org/packages/f4/88/360064c4c7d9d0664561dae03b74c871d2f5332b329f5c99f1c997fb869a/chardet-7.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cedbc584789eb2edfde20fd03669972a833ce6019e60014ae613f9bfc440e8e3", size = 1510242, upload-time = "2026-08-14T20:36:47.206Z" }, - { url = "https://files.pythonhosted.org/packages/43/4c/302869fa1c69a5a41e4e78782680b12552ffd4286c3ae3c839b7b8df53d4/chardet-7.6.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5dc835e40e0e09c2c3eab43731a8b5127834f42786dda09ba2f4b699ccd527a", size = 1456456, upload-time = "2026-08-14T20:36:48.668Z" }, - { url = "https://files.pythonhosted.org/packages/7c/a9/ff4fef15ed25fc3f945a3b981ae0f43c8559b3fbedb40267e59e583d105b/chardet-7.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:0f304de7041afaec0195ad6464937cd112392002e9d72ed15d55f20a9abd3a13", size = 1159103, upload-time = "2026-08-14T20:36:50.107Z" }, - { url = "https://files.pythonhosted.org/packages/31/44/94e6f89485ce6c630e8fec6d388e3fa747e58b7246b0cc8c5c532ba98650/chardet-7.6.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4f0a368ad04d5def08bdfaa17c7e15e71552f93923dc2aa9b2f7d9dee02fbb6", size = 1120302, upload-time = "2026-08-14T20:36:51.484Z" }, - { url = "https://files.pythonhosted.org/packages/e0/cd/8e68ba12f13aaf4ee82d54ef8a1f83d62942e7a6bc1e579dd2d530a3e26e/chardet-7.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75d6c3a4d2046d49e83d2d2206eb073a1f390743e856d90c1bbc19949b26acf4", size = 1093157, upload-time = "2026-08-14T20:36:52.987Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f6/6a35342b9efa69dcceab0ea8966571c6442a59c336bf460f0a2af95ed234/chardet-7.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:459e2b1c98f9a86a4698112aa42dffa802bbbff883c1ff144071f87224125862", size = 1521213, upload-time = "2026-08-14T20:36:54.812Z" }, - { url = "https://files.pythonhosted.org/packages/df/8c/8f09bfabbaedae45caa72b996e96d5e3f6436dfddc55c1311ffa2f6bd7d2/chardet-7.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bdb6f03107b7ace3f44e0edd91aa24456ee558787df265cc19daf45785b31c7", size = 1528427, upload-time = "2026-08-14T20:36:56.224Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3d/2540627b193112e08b8045f3cad9295570866208555aa6625b63cc69c6cd/chardet-7.6.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:0c44a32da32cc8b23d6b20d98ace15ec7600950e4955d1bf5ab1f849b0187fdb", size = 1087374, upload-time = "2026-08-14T21:04:14.994Z" }, - { url = "https://files.pythonhosted.org/packages/fe/de/dac4f550cde73c4732b4916e72ec489ae36933fbbd1697e4122d3ae2e9dd/chardet-7.6.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7bbc8a9652c7f859c593847f220c1d264f25749369abb1a267b404ee8cceb209", size = 1064688, upload-time = "2026-08-14T21:04:16.806Z" }, - { url = "https://files.pythonhosted.org/packages/06/45/f4f3f288496797d2cf1d1e9036f920ed2b4c79787b21c42a3314f4a6d532/chardet-7.6.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83512a475a2f3886166aa0bca1bbb39343a4eb3186dd5532127d6f2591d09118", size = 1486320, upload-time = "2026-08-14T21:04:18.535Z" }, - { url = "https://files.pythonhosted.org/packages/c0/81/3ca30c16e6c6015b737fca22d9342957cc617d8a65329d3e963ad754a31b/chardet-7.6.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:271ab71ec1be61dbbce0436de0848895c03eae051c379e937a39573d9ce403d9", size = 1515374, upload-time = "2026-08-14T21:04:20.724Z" }, - { url = "https://files.pythonhosted.org/packages/6e/98/163a75b8b3372a6b6503f5f5a4154a222ac135c7a706b5d176f8e4b237a2/chardet-7.6.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da86fc1b40ff5996fbb5e4c2d2dca770eac2c893cef157dacc050b8b4d929846", size = 1469504, upload-time = "2026-08-14T21:04:22.352Z" }, - { url = "https://files.pythonhosted.org/packages/d7/4f/3ad1b1bd27ae7f0d3d13cf711366525cd781a9e067950d8a09aa280916cb/chardet-7.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:b73f277c1ac09c4f8076c4214b816c7aa78a0a2f0cb7156742f4303f856bedc3", size = 1158551, upload-time = "2026-08-14T21:04:24.259Z" }, - { url = "https://files.pythonhosted.org/packages/8d/92/22a609c68c5123ebdccd8fe1a80e423609012ce20166de19a2ed3955b2f7/chardet-7.6.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:61238d5945b36af9a2ad13494f8969b7deb3c3b4abe223e54670c064e73f5328", size = 1119186, upload-time = "2026-08-14T21:04:25.761Z" }, - { url = "https://files.pythonhosted.org/packages/29/35/75a90143e4200e197f4f6cf5895d379e22bc785d0c7711120df18fa5347c/chardet-7.6.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f2ec3c78cc6b54bf8e091ec4ee885473078b5d7ef18ab1b01c86ae1e98bf88f7", size = 1092596, upload-time = "2026-08-14T21:04:27.884Z" }, - { url = "https://files.pythonhosted.org/packages/d1/2c/b6d5f47d878c46e04ae6fcb58e0969467925bc0819b333c682e0c41bbc5c/chardet-7.6.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:167d7ba3ee08b654e36d7b43ebd9a36606c9a12e2fabdb361757a095ca3b7e3d", size = 1516644, upload-time = "2026-08-14T21:04:29.624Z" }, - { url = "https://files.pythonhosted.org/packages/25/d2/2bc3f1066c6f27bc3fa3a98dfd8a2a9c1e969a9d6bdc1edcd35278791fc4/chardet-7.6.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f14f46ef1977e41ce1f4814ca6984cea7f8b6baf8cbc6626ef7bf3d13cf7ea13", size = 1533221, upload-time = "2026-08-14T21:04:31.39Z" }, - { url = "https://files.pythonhosted.org/packages/cf/6e/5a0b348fa4cd7847567a28c6e697ccf58391960bfd13a6e7473ee23ca2f2/chardet-7.6.0-py3-none-any.whl", hash = "sha256:4076d795897ce45239825956a1334e134322ecc4bfe84dbb12acd5390de0fbc1", size = 680279, upload-time = "2026-08-14T20:36:57.763Z" }, +version = "7.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/92/3d893ac5d338b9a9af34c5b517d77492dcea8a10254254eead4215a96f3f/chardet-7.7.0.tar.gz", hash = "sha256:0ba38a05201b0abf7570c12d8af45c30b2af835bd5cc52bfc7105082eda82c1e", size = 1008480, upload-time = "2026-10-18T00:06:25.352Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/73/0c434bc65adc411e49b1bbebee24ebb36a3511c80a058f1fde17f43fb537/chardet-7.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:11263887c082c3d8c32026aeddbdef37ac8feccd1faece5a22b2ff00a29b115a", size = 1202232, upload-time = "2026-10-18T00:05:45.657Z" }, + { url = "https://files.pythonhosted.org/packages/9d/be/222e5628af10d6b80750ccdda7f6d2610646dddb082c37947623d5e55bf7/chardet-7.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e8c718e6961981fe1685e1243742f326ac553f0fa0dfd0c6d3d17dd4f92cd3ff", size = 1173273, upload-time = "2026-10-18T00:05:46.984Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/889cf390db3748918789818d93ba9a7d712b326030199698432fc50bf8cd/chardet-7.7.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a76f4d1f769b4ae8888cefd46e604cf1f5769f1176b55e78b90eaeefa888ce92", size = 1616968, upload-time = "2026-10-18T00:05:48.457Z" }, + { url = "https://files.pythonhosted.org/packages/ca/04/0b5fd38375ae182cec6446dd3121aaff5ae18fd12b98b10161a58b720db9/chardet-7.7.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c519c10c315a675bcfd4b55c0e0de1859ca5a39881d8f69750500102c2fc5d2f", size = 1658830, upload-time = "2026-10-18T00:05:49.938Z" }, + { url = "https://files.pythonhosted.org/packages/99/c0/7e642d7bc6aca242ef41b4aa5f5be27dbb57912e8ce0f38043d8d1999cd1/chardet-7.7.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:46a044166fe0d99931884dc28405faac558d5eabf28316a2b3d2a1a6c7dc439a", size = 1583300, upload-time = "2026-10-18T00:05:51.567Z" }, + { url = "https://files.pythonhosted.org/packages/30/ee/53147716ced6d43837682a2acfb09dd45add8bc496107f21051c5aeae1f6/chardet-7.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:113bbd3ab85835e3e99499ae66829915ed3b4cf5c69525ab1742260f77638449", size = 1308030, upload-time = "2026-10-18T00:05:52.997Z" }, + { url = "https://files.pythonhosted.org/packages/1a/61/800a9442f352912c86f2a00de84fd6914dd9353d29091b1e223e7c08850e/chardet-7.7.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad286a0654ae7477bc454a1dee4f396cb28adffeacc6bc7d4412f607c73de0c3", size = 1200940, upload-time = "2026-10-18T00:05:54.59Z" }, + { url = "https://files.pythonhosted.org/packages/fb/92/3c6ce24dd68c626cc6aa0b5953c2aebb1d853c7b07d3535f0300b7428288/chardet-7.7.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4f67f20c9ad3f3024bf71d4aae1d25722b12da921ca9586f3cce7a4639695aa0", size = 1173087, upload-time = "2026-10-18T00:05:56.257Z" }, + { url = "https://files.pythonhosted.org/packages/4d/32/5a952b376c00dda01465b396958a8985916d9a9293a772d6bda2c37f2cc5/chardet-7.7.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a05667635c0496dab47eca8c845c044d1da6d74d34a9bf687199c9c53a38119d", size = 1624001, upload-time = "2026-10-18T00:05:57.832Z" }, + { url = "https://files.pythonhosted.org/packages/04/f7/09296ca1d4621c9c2766d074822035eee6aebb79eff80c0af3bd96283d05/chardet-7.7.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:79eec5af9d7715334695254234c8d7c29165841526c11083951d9b9f6458e372", size = 1653785, upload-time = "2026-10-18T00:05:59.379Z" }, + { url = "https://files.pythonhosted.org/packages/cb/08/80571baad57db2962d6931b3ee1828c82e3e14e9b3464f80532f536e23ac/chardet-7.7.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4167165812219e70f311083a8e06e3b627ed5b27aa8c15c3fdd56f350fd3213e", size = 1582280, upload-time = "2026-10-18T00:06:00.825Z" }, + { url = "https://files.pythonhosted.org/packages/18/82/3c07070fdcf3c4263e16680c90e9af7c2cd2e5e59cfb1d372efa60fde8cc/chardet-7.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:546180f6170da39778069d16130449a9cff49a8fe144535947d1ef26f5330b96", size = 1310108, upload-time = "2026-10-18T00:06:02.342Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b0/d703d408c7f8931b1564110078fc3dcff9825a159990fa69b80da52be160/chardet-7.7.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4c2e83b09a540a4d33279d2bee8ed3bfd57ac6645b359ae8b8afd5b4578c3c2", size = 1237782, upload-time = "2026-10-18T00:06:03.763Z" }, + { url = "https://files.pythonhosted.org/packages/d8/3e/0eafeebd53eb096400f75138ab18614ab5be12b8794b76dd4b35ef1a8e05/chardet-7.7.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:259dd55b1ba05a66868151a42663483763046a3e6579bde6b0aa65d834635b5a", size = 1207710, upload-time = "2026-10-18T00:06:05.221Z" }, + { url = "https://files.pythonhosted.org/packages/b8/8d/3c200098027def56be1b6443a10c72efe013a42eb76444d889c809890ca5/chardet-7.7.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56e5a5c782548b301b44371de327115d13a08de391ddaf9db4ac420d8511c5d7", size = 1660832, upload-time = "2026-10-18T00:06:06.619Z" }, + { url = "https://files.pythonhosted.org/packages/08/81/02f98389fa72345f40d23d69c2d857511d11a0911779a7cf8ed1cecb5037/chardet-7.7.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d227d2aa329fcf848f4640ed9a4d99e1753840c1cfa19f7ca5a48f30c03982ad", size = 1678291, upload-time = "2026-10-18T00:06:08.196Z" }, + { url = "https://files.pythonhosted.org/packages/5e/24/0ae866b759a148d0f6087ffb30920a2b956653d0d455b4be93026edbee95/chardet-7.7.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c4a591b2dbfc0a047f8d60331c2fbfb4c3589911000406666a3085fa7d5da69f", size = 1200696, upload-time = "2026-10-18T00:06:09.701Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a6/5e0a1f56aca6c44b562d05d9e360f31b5a8f4b311d940a324fa337d277ba/chardet-7.7.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:eb625c3ab2386ca7f3be9ac2b9b822f4c5b792595c8bb8100b3b6ea68120e92c", size = 1172821, upload-time = "2026-10-18T00:06:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/89/05/868f52672cde7e188a7333f3f65deaa62f5acbfcf0f81f3582013776e8ad/chardet-7.7.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddedfb252d05c86c2d6e9a3090f8bd574876bbd39679e7c588ed57d21595c097", size = 1625481, upload-time = "2026-10-18T00:06:12.499Z" }, + { url = "https://files.pythonhosted.org/packages/f9/12/2b3a006ae31a7ab4ccac11a00abb27af23b4a5d57ddb00e937375e747a2a/chardet-7.7.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c4d8a2dc8abcc067417adc13c2f64f48482e48ce48e3af769b5a44214e5a5c", size = 1660352, upload-time = "2026-10-18T00:06:13.977Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/f995a2873a78aa3ec8308f02e61f0717deb4b01c8f1a74c2da329f325925/chardet-7.7.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78db6186c1e73d38a2106cdcbbc8f43612eaed9ef8bc4adaf0fe67b6e8a2049d", size = 1597363, upload-time = "2026-10-18T00:06:15.492Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7b/22b916b52c7bc4eb076ba1736ea9b99cf1065ebb7b56d356409144887c73/chardet-7.7.0-cp315-cp315-win_amd64.whl", hash = "sha256:8f1d611f15530e1642856d91b7aa259851466d7e961f27b888921e7e29718edb", size = 1309528, upload-time = "2026-10-18T00:06:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/ccaf5ffa0c8adca280e2687d75524cc07113a0df70cbf542da3445f2b300/chardet-7.7.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:1db39957379a801cf1798a5e50cdb3d3fc32683e23c4166ecfa157b980d1931e", size = 1236698, upload-time = "2026-10-18T00:06:18.191Z" }, + { url = "https://files.pythonhosted.org/packages/0d/7e/32f45f7ddeb27d94feb529a1a23e5a38a6d3614d12e23f0c89a9eb89098b/chardet-7.7.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:7f3b2ca3bac2bdbe42c4dfe7a2888c4b98e2e86a83ddb631960b229d1980979e", size = 1207247, upload-time = "2026-10-18T00:06:19.751Z" }, + { url = "https://files.pythonhosted.org/packages/26/40/8c1ab360c39a0bb823786e8934984cbc3e83076dfce2a7f3760086ef778c/chardet-7.7.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80db1c109e6e3c102ac589897449437ea2edf26dda1042a3ff669368e21d90e5", size = 1653659, upload-time = "2026-10-18T00:06:21.189Z" }, + { url = "https://files.pythonhosted.org/packages/1b/3e/bd5c25f525ff2466e6365cbfb098f928761357331b884eebc92ac50f8878/chardet-7.7.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:091f50d3567b1a066f797835ee5367dbcf3cb77c5e56993b3f61b28264b8e6f8", size = 1677031, upload-time = "2026-10-18T00:06:22.564Z" }, + { url = "https://files.pythonhosted.org/packages/cc/75/88a3eb36264375ff951bd2f9397c409727ab6aa6a18e1bd933851571d421/chardet-7.7.0-py3-none-any.whl", hash = "sha256:11cf9b719ea3675a2a7d8a3071bbe4d7ea24b26bfe39a51588d515ddd05cacad", size = 706680, upload-time = "2026-10-18T00:06:24.018Z" }, ] [[package]] @@ -1097,21 +1106,20 @@ wheels = [ [[package]] name = "curies" -version = "0.15.3" +version = "0.15.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, - { name = "pystow" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/9a/10def0c00f9f75e21f696d33cae90a46dfb5a97f5bf2a8a812f869b43ad5/curies-0.15.3.tar.gz", hash = "sha256:6c613b63c3d3f485fcc42248309d011911cc1c51fc0c6a54a30311d4dc87a6aa", size = 74155, upload-time = "2026-10-08T17:50:22.744Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/2d/79253d3482e3d8dbeac15a973e458d3f9b261f5faf4a6f726843a6acb175/curies-0.15.4.tar.gz", hash = "sha256:0dc7d44943b9991c518942e824905c1a19bbe24f43d4a790e87d2e42472483df", size = 79920, upload-time = "2026-10-18T15:33:22.572Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/07/1e62b10fe0102b2702e8ee92e9169a29c340e32136e0ed0ba1c4ceb5ecfa/curies-0.15.3-py3-none-any.whl", hash = "sha256:7b83bc82798aac78711578fced0e99055371fbc3d43d3d4319a66e1e5598ce95", size = 83356, upload-time = "2026-10-08T17:50:21.225Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/e2f6eb642b4497a5a161df45f07dede60745b412e85f93048bd45fa0dff7/curies-0.15.4-py3-none-any.whl", hash = "sha256:25a7aaf4c58cd787887f11622ea38975bc4a847d20283bc24ede6d8af5e83687", size = 83473, upload-time = "2026-10-18T15:33:21.329Z" }, ] [[package]] name = "datamodel-code-generator" -version = "0.83.0" +version = "0.86.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "argcomplete" }, @@ -1124,9 +1132,9 @@ dependencies = [ { name = "pyyaml" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/ad/bd0c954c4c4671e586b340059139ead38570ba916a237086e245982ab14d/datamodel_code_generator-0.83.0.tar.gz", hash = "sha256:30d841e31b82095c4ef11f3dbfc99525f559a146250b9e685e2a2e4a196ca0a1", size = 3074761, upload-time = "2026-09-24T08:56:40.622Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/69/56726c1ffd4790f90dc47630f96baed1a29a8f25375982510fce3b3e61af/datamodel_code_generator-0.86.0.tar.gz", hash = "sha256:1834ba0e84307dfed625eeacfbb07d790ede01a3adf5414a1c9947c5b9139078", size = 6255349, upload-time = "2026-10-18T15:43:30.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/4e/4a0231a7d3d4043cd27a746a50be87bc7637463023d7db9991610cd5fe56/datamodel_code_generator-0.83.0-py3-none-any.whl", hash = "sha256:371d80be360b2a57bc48462169553af57fa30225c0264ad85ae5f0d59b9d143a", size = 751087, upload-time = "2026-09-24T08:56:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/f76dac052451e88fc0b622c3d996fa48637da9fa1e3c2b9ca2b84aead28a/datamodel_code_generator-0.86.0-py3-none-any.whl", hash = "sha256:1a3d6f0a14a33e334b4251d04e67b0b6ef05c96d4625fb99bd09d716e30ea084", size = 1605265, upload-time = "2026-10-18T15:43:27.808Z" }, ] [[package]] @@ -1187,7 +1195,7 @@ wheels = [ [[package]] name = "dlt" version = "1.31.0" -source = { git = "https://github.com/dlt-hub/dlt?branch=devel#7f09267b4430b5c2e95ddcdae36fc9d614dce514" } +source = { git = "https://github.com/dlt-hub/dlt?branch=devel#2c984279acff03b29b7dc2f7de496cc65898ea18" } dependencies = [ { name = "click" }, { name = "croniter" }, @@ -1326,7 +1334,7 @@ wheels = [ [[package]] name = "frictionless" -version = "5.20.0" +version = "5.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -1350,9 +1358,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "validators" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/e1/39a7624321497e037692d3675b45ad51445575e3d2ba935978701d247f1e/frictionless-5.20.0.tar.gz", hash = "sha256:d83aca7148fe146254ac00a8c9517407c8f4974afd12b09f06bdca0c407448bf", size = 74402411, upload-time = "2026-10-08T20:47:52.43Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/f2/a0978b650b0c8b19e087e545aeb992d3f0b3596c763244565f1ea9075056/frictionless-5.21.0.tar.gz", hash = "sha256:4d9848570bc682943375824a4be7633e8bbdeff228affee763cab346e48000b4", size = 74413561, upload-time = "2026-10-15T21:52:18.022Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/4c/db4275f56375169edf065cc3461091f0a0421f46ccde636de1cde49105df/frictionless-5.20.0-py3-none-any.whl", hash = "sha256:c7dbe8d27b8a21a561b622cddf764ce3d5a2ee1688aea096c5caee7b106c05c3", size = 563587, upload-time = "2026-10-08T20:47:49.597Z" }, + { url = "https://files.pythonhosted.org/packages/88/e1/8140a97773b48ceb2b8d04179a13aec003c3999be764813ad47830fab657/frictionless-5.21.0-py3-none-any.whl", hash = "sha256:c76db03abaee4862aa04116057dbc13e788a2fc18b9be37c2d35dc07a67e1b11", size = 575379, upload-time = "2026-10-15T21:52:14.863Z" }, ] [package.optional-dependencies] @@ -1790,20 +1798,20 @@ wheels = [ [[package]] name = "jsonpath-ng" -version = "1.10.1" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4c/dc/178bf7bb75d2df2532d0d1796805381f2599eb805c40eeda089538af9393/jsonpath_ng-1.10.1.tar.gz", hash = "sha256:1247d0983361ebe44f47741e759bbb76e74213c68f25abb4b65f6de21d1934d6", size = 87626, upload-time = "2026-10-12T12:57:12.048Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/2d/ebcea1c9a2338a08a5f6107bd274bd4b1e0c9a2ebc82ddbc815271ffee3b/jsonpath_ng-1.11.0.tar.gz", hash = "sha256:938b9c8d02f625c079c884f2b76241d0ac54ee7d1c8f10a502c42d8e31ff0c4b", size = 94460, upload-time = "2026-10-16T20:11:02.037Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/e6/d0f38911783aa7bc69afb0cdf5151e8cefeecd8ca3944c5453e13fc5afda/jsonpath_ng-1.10.1-py3-none-any.whl", hash = "sha256:9355047e5e6a8919f5ae0ccfd5b793bff69e4165f1248b1763e8962457b58ff5", size = 75386, upload-time = "2026-10-12T12:57:10.48Z" }, + { url = "https://files.pythonhosted.org/packages/c6/28/7bf49fd8ef55167239820e05e7bb501e0cc4cb20075c9e4da377e1616761/jsonpath_ng-1.11.0-py3-none-any.whl", hash = "sha256:84b325f791e43ae40810ff4059f50744b4c22fe53066b0884a659f0191c0b308", size = 76222, upload-time = "2026-10-16T20:11:00.574Z" }, ] [[package]] name = "jsonpointer" -version = "3.2.1" +version = "3.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/a2/c92f0a7ed439c490d2c8ad712fdb074c311afa4f77870987826a3b1483ba/jsonpointer-3.2.1.tar.gz", hash = "sha256:47c846513b3a4ec46eecef1105207fba075e2a3659048e362bd7daff0fc33342", size = 11309, upload-time = "2026-10-10T21:11:06.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/77/9be5cce647fa4d7afa9047c0c1029bc73292c27430e64fc9d2393ccf6727/jsonpointer-3.4.tar.gz", hash = "sha256:e57015ae65af3821c6a3976069860789cf326103f38b22b2178d41d021adad25", size = 13113, upload-time = "2026-10-19T10:33:24.436Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/29/accef8eea16670b88f3a23102c35dee62c6f159db30d9628908043b3e21b/jsonpointer-3.2.1-py3-none-any.whl", hash = "sha256:b19ee68644e9ffb51440448d8f7811af2b7406eea1db90603e93f5849323119a", size = 8332, upload-time = "2026-10-10T21:11:05.648Z" }, + { url = "https://files.pythonhosted.org/packages/10/d0/3d41544018b879e9c608c32bd5fb468fd87669d53c2dbf5ae67c1bad9882/jsonpointer-3.4-py3-none-any.whl", hash = "sha256:136e2e3273fd82a1502274f56ec172f331b6a1703d86a1824d503af4d9a96142", size = 9047, upload-time = "2026-10-19T10:33:22.963Z" }, ] [[package]] @@ -1892,28 +1900,111 @@ wheels = [ [[package]] name = "lazy-object-proxy" -version = "1.12.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/a2/69df9c6ba6d316cfd81fe2381e464db3e6de5db45f8c43c6a23504abf8cb/lazy_object_proxy-1.12.0.tar.gz", hash = "sha256:1f5a462d92fd0cfb82f1fab28b51bfb209fabbe6aabf7f0d51472c0c124c0c61", size = 43681, upload-time = "2025-08-22T13:50:06.783Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/26/b74c791008841f8ad896c7f293415136c66cc27e7c7577de4ee68040c110/lazy_object_proxy-1.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:86fd61cb2ba249b9f436d789d1356deae69ad3231dc3c0f17293ac535162672e", size = 26745, upload-time = "2025-08-22T13:42:44.982Z" }, - { url = "https://files.pythonhosted.org/packages/9b/52/641870d309e5d1fb1ea7d462a818ca727e43bfa431d8c34b173eb090348c/lazy_object_proxy-1.12.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:81d1852fb30fab81696f93db1b1e55a5d1ff7940838191062f5f56987d5fcc3e", size = 71537, upload-time = "2025-08-22T13:42:46.141Z" }, - { url = "https://files.pythonhosted.org/packages/47/b6/919118e99d51c5e76e8bf5a27df406884921c0acf2c7b8a3b38d847ab3e9/lazy_object_proxy-1.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be9045646d83f6c2664c1330904b245ae2371b5c57a3195e4028aedc9f999655", size = 71141, upload-time = "2025-08-22T13:42:47.375Z" }, - { url = "https://files.pythonhosted.org/packages/e5/47/1d20e626567b41de085cf4d4fb3661a56c159feaa73c825917b3b4d4f806/lazy_object_proxy-1.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:67f07ab742f1adfb3966c40f630baaa7902be4222a17941f3d85fd1dae5565ff", size = 69449, upload-time = "2025-08-22T13:42:48.49Z" }, - { url = "https://files.pythonhosted.org/packages/58/8d/25c20ff1a1a8426d9af2d0b6f29f6388005fc8cd10d6ee71f48bff86fdd0/lazy_object_proxy-1.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ba769017b944fcacbf6a80c18b2761a1795b03f8899acdad1f1c39db4409be", size = 70744, upload-time = "2025-08-22T13:42:49.608Z" }, - { url = "https://files.pythonhosted.org/packages/c0/67/8ec9abe15c4f8a4bcc6e65160a2c667240d025cbb6591b879bea55625263/lazy_object_proxy-1.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:7b22c2bbfb155706b928ac4d74c1a63ac8552a55ba7fff4445155523ea4067e1", size = 26568, upload-time = "2025-08-22T13:42:57.719Z" }, - { url = "https://files.pythonhosted.org/packages/23/12/cd2235463f3469fd6c62d41d92b7f120e8134f76e52421413a0ad16d493e/lazy_object_proxy-1.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4a79b909aa16bde8ae606f06e6bbc9d3219d2e57fb3e0076e17879072b742c65", size = 27391, upload-time = "2025-08-22T13:42:50.62Z" }, - { url = "https://files.pythonhosted.org/packages/60/9e/f1c53e39bbebad2e8609c67d0830cc275f694d0ea23d78e8f6db526c12d3/lazy_object_proxy-1.12.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:338ab2f132276203e404951205fe80c3fd59429b3a724e7b662b2eb539bb1be9", size = 80552, upload-time = "2025-08-22T13:42:51.731Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b6/6c513693448dcb317d9d8c91d91f47addc09553613379e504435b4cc8b3e/lazy_object_proxy-1.12.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c40b3c9faee2e32bfce0df4ae63f4e73529766893258eca78548bac801c8f66", size = 82857, upload-time = "2025-08-22T13:42:53.225Z" }, - { url = "https://files.pythonhosted.org/packages/12/1c/d9c4aaa4c75da11eb7c22c43d7c90a53b4fca0e27784a5ab207768debea7/lazy_object_proxy-1.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:717484c309df78cedf48396e420fa57fc8a2b1f06ea889df7248fdd156e58847", size = 80833, upload-time = "2025-08-22T13:42:54.391Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ae/29117275aac7d7d78ae4f5a4787f36ff33262499d486ac0bf3e0b97889f6/lazy_object_proxy-1.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a6b7ea5ea1ffe15059eb44bcbcb258f97bcb40e139b88152c40d07b1a1dfc9ac", size = 79516, upload-time = "2025-08-22T13:42:55.812Z" }, - { url = "https://files.pythonhosted.org/packages/19/40/b4e48b2c38c69392ae702ae7afa7b6551e0ca5d38263198b7c79de8b3bdf/lazy_object_proxy-1.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:08c465fb5cd23527512f9bd7b4c7ba6cec33e28aad36fbbe46bf7b858f9f3f7f", size = 27656, upload-time = "2025-08-22T13:42:56.793Z" }, - { url = "https://files.pythonhosted.org/packages/ef/3a/277857b51ae419a1574557c0b12e0d06bf327b758ba94cafc664cb1e2f66/lazy_object_proxy-1.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c9defba70ab943f1df98a656247966d7729da2fe9c2d5d85346464bf320820a3", size = 26582, upload-time = "2025-08-22T13:49:49.366Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b6/c5e0fa43535bb9c87880e0ba037cdb1c50e01850b0831e80eb4f4762f270/lazy_object_proxy-1.12.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6763941dbf97eea6b90f5b06eb4da9418cc088fce0e3883f5816090f9afcde4a", size = 71059, upload-time = "2025-08-22T13:49:50.488Z" }, - { url = "https://files.pythonhosted.org/packages/06/8a/7dcad19c685963c652624702f1a968ff10220b16bfcc442257038216bf55/lazy_object_proxy-1.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fdc70d81235fc586b9e3d1aeef7d1553259b62ecaae9db2167a5d2550dcc391a", size = 71034, upload-time = "2025-08-22T13:49:54.224Z" }, - { url = "https://files.pythonhosted.org/packages/12/ac/34cbfb433a10e28c7fd830f91c5a348462ba748413cbb950c7f259e67aa7/lazy_object_proxy-1.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0a83c6f7a6b2bfc11ef3ed67f8cbe99f8ff500b05655d8e7df9aab993a6abc95", size = 69529, upload-time = "2025-08-22T13:49:55.29Z" }, - { url = "https://files.pythonhosted.org/packages/6f/6a/11ad7e349307c3ca4c0175db7a77d60ce42a41c60bcb11800aabd6a8acb8/lazy_object_proxy-1.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:256262384ebd2a77b023ad02fbcc9326282bcfd16484d5531154b02bc304f4c5", size = 70391, upload-time = "2025-08-22T13:49:56.35Z" }, - { url = "https://files.pythonhosted.org/packages/59/97/9b410ed8fbc6e79c1ee8b13f8777a80137d4bc189caf2c6202358e66192c/lazy_object_proxy-1.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:7601ec171c7e8584f8ff3f4e440aa2eebf93e854f04639263875b8c2971f819f", size = 26988, upload-time = "2025-08-22T13:49:57.302Z" }, +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/8e/f26b09778fcb26261fdeb4cc2b011975292316529bff9b541a5a02acb976/lazy_object_proxy-1.13.0.tar.gz", hash = "sha256:ce34046cb021d69c72d05ddb83821ecfd4d2d0a5d43a93c5cd01c9e34779cf14", size = 45649, upload-time = "2026-10-17T16:38:01.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/20/b0258e552870cc67de8f4da00fe6608a1f1438db24252be9011d238d4fd8/lazy_object_proxy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7edc00d4792dba391667fcc5459b023b55190b9743735e7c1281855851c3d91b", size = 26810, upload-time = "2026-10-17T16:37:45.788Z" }, + { url = "https://files.pythonhosted.org/packages/6f/0d/853912d47395268e30f2f7ba9a8a9c2319e1626c4ffdfc866bae9a2e141e/lazy_object_proxy-1.13.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4046337019bd862440764930dffc47747a36b2e5d74346f0d404eadf1e3e6fa7", size = 71472, upload-time = "2026-10-20T01:05:41.44Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e0/90469d72e87eaedacb6ee7656ac99567f1880b87c2be9ce9eb72cac62ebe/lazy_object_proxy-1.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:903fc3eb8d87ff8fcb0b2f62e7d84e9808712aa085635612d2930b6ee2d8d4f5", size = 71090, upload-time = "2026-10-20T01:05:42.538Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/f5e6366b21de5860ec14c8155ee39cae25964117a8edb310dc023b562616/lazy_object_proxy-1.13.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:168e1eb2773a187c40544653346622917152c5f93163effd80dcce848888f01e", size = 65796, upload-time = "2026-10-20T01:05:43.827Z" }, + { url = "https://files.pythonhosted.org/packages/f4/69/fce5cb19ba6c1af5f3700c651ed397d306b759de4c2f7de7e4a88d0e051f/lazy_object_proxy-1.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:78a351da00aebe3ef8089f2c74bcaff8e851123419590fb0b86d8241a9ea3fc4", size = 69399, upload-time = "2026-10-20T01:05:44.991Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4a/edda1af23f22c69179591ec4b186bdd0549fa4ff5fd1b6c602dbcad4f30f/lazy_object_proxy-1.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ea9e832c3d9112992eb1f6f16543ef555d2b51ff288c5a16c0741b9a9ca88984", size = 65101, upload-time = "2026-10-20T01:05:46.231Z" }, + { url = "https://files.pythonhosted.org/packages/dd/0b/3effaa4ec12ef98c57cd39da8f36f8d93817e9b5e7875b0d57f4637ee645/lazy_object_proxy-1.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:65a369a41028bd2ba53dd31da0f6ee97682b70bcf2bf7413cb69ba11c63e4fd4", size = 70677, upload-time = "2026-10-20T01:05:47.399Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/6239ba7aa95a1a9f383c54050a37af4d6da449af2b711240154411623646/lazy_object_proxy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:54fdf11db2733dfd8dd8177f79d8d9c80dd99a912c0b1a8906412306ad18ade2", size = 25818, upload-time = "2026-10-17T16:37:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/1c/20/cd330c8425ad9e0c57b8cc3a72d4c64b60d7525d2cb946c588d3ee66f6f1/lazy_object_proxy-1.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:042284eea98fc08b9d347e43b7c258e6c862b4c03d5881d536bc61d48e6be88e", size = 24473, upload-time = "2026-10-17T16:37:47.919Z" }, + { url = "https://files.pythonhosted.org/packages/d0/0b/410ea9a56ad59421d8969535099560f0bad7162d483869e1c07ff5880f35/lazy_object_proxy-1.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:62612150e2b67d1cb26c2201d020773e9bab9c596d783d55ada42885a695f749", size = 26856, upload-time = "2026-10-17T16:37:48.864Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/79ab37585905cf6841134806e73ace576d61a87c4eee333be4622161b23f/lazy_object_proxy-1.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ad7630266b5775463f2a1806165c1d9dff07780cf1bba3c53f311e1d026e8921", size = 70985, upload-time = "2026-10-20T01:05:49.274Z" }, + { url = "https://files.pythonhosted.org/packages/18/2e/66bbb45f20aab9e81139e2bf9fea1a29850b5af769cbf7f181221ce062dc/lazy_object_proxy-1.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73f05b09e37b0eb0ca7d6e80e2b0689c49bedac8c5d69667c1ebb9f02a4ed5c7", size = 70984, upload-time = "2026-10-20T01:05:50.548Z" }, + { url = "https://files.pythonhosted.org/packages/74/e0/63dc39750c7b266e9e25022962eba32ba80e4f17fdcd674fc4ad4143fd90/lazy_object_proxy-1.13.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:91a725d3c64dfe13d365cf78c469c2a8f95884859379aeca89cb8d2ed291649a", size = 65689, upload-time = "2026-10-20T01:05:51.765Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6c/b94a8779eec1331477557ca3ea4742860d59dc9bcd4dcf21bcef3cd8c4f2/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e28a75d33378141927c08f3eb291cc98d0a22776670ae330f8347f898cd3b175", size = 69492, upload-time = "2026-10-20T01:05:53.116Z" }, + { url = "https://files.pythonhosted.org/packages/60/99/6c3ff4a32f15e807ffb868c47de136ebd4be31e7320af42b699dc43a7495/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a785fe44b61fcbffc0e264fe6b735a909e0b5ef922f0e976e281bbe8477fb47a", size = 65019, upload-time = "2026-10-20T01:05:54.462Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1a/447ff40629a11f879f194e839d59e5a026dd26f4d0eb8672de858644be85/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e321425f76451885c93083ecec0850b64de0f83bb94c113e930e8a0a630e45ab", size = 70334, upload-time = "2026-10-20T01:05:55.619Z" }, + { url = "https://files.pythonhosted.org/packages/70/52/2ca84fb914b17d4b02193cbf8b6ba9344b36459944a19711d3da1d14b03d/lazy_object_proxy-1.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:0e52094b7fff1f26f4e227b88be3f350817078e4bb79b4c80846fca0d15efae4", size = 26122, upload-time = "2026-10-17T16:37:53.02Z" }, + { url = "https://files.pythonhosted.org/packages/84/c7/ebbeb84639e84fbf6f07a48373f76fe048d23fbcfd4557509b48f35d284d/lazy_object_proxy-1.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:b2dd1c37d64cf9f3f4a2072971639408989b0803a3b7f05e75c1fe28d1a93c1f", size = 24746, upload-time = "2026-10-17T16:37:53.954Z" }, + { url = "https://files.pythonhosted.org/packages/38/ac/8ff35cc4d1316e056144edfb816dbff9b5db9ccd7e98c69f6b602c11bc29/lazy_object_proxy-1.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:348370676ea6d9ed08f9b74545c37e5be4cf009c45b9d50c6aff476fb7fcc7ec", size = 27524, upload-time = "2026-10-17T16:37:49.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/c5/8afe34ead41fdd48a207c4535192a4e931c56c791d5f0a57c0abd4558c1c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:dee2fb3326cd81384d2b752c3f6888d7ed9eba2cdbf0583097b5168b8bb4a703", size = 80523, upload-time = "2026-10-20T01:05:57.129Z" }, + { url = "https://files.pythonhosted.org/packages/38/fd/0e68a385daaf141b1b1d943b132b8a9887bd53abb88886dab286cca9b62e/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b4c446cbc8e411b116c9ee9b475cb664b8bc5c3d912bcbf1bab5c785fc69a02", size = 82823, upload-time = "2026-10-20T01:05:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/91/ce/648e09f90dc59cb6007d9093c2cc17d72e2d93c06553828037126205ea7c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1e671e0e9f2793759f382a69a3853c8742474bc011865ed3595187340ac5956d", size = 74655, upload-time = "2026-10-20T01:05:59.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/4d/811442390862dbf85e00520ecb0cb58c7dbdf824dc563ad933729f9e20f3/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a0155084a5dc8ce3e1a3d6e3f2fb4d84d5fc09e66ed84dc0e5f8a2697d421ea0", size = 80848, upload-time = "2026-10-20T01:06:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/24/4e/c30ea3678b50a7ed7774760fbde917106cbb860edb88cc31a17a8385b3d4/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2d07e0bde2738922a7117fccc1a82514aefbc9c07ee8ff0bb84a1c606dbb5ac8", size = 74127, upload-time = "2026-10-20T01:06:01.949Z" }, + { url = "https://files.pythonhosted.org/packages/19/d0/862f0f96b031dab7faa036679e2076bbd6a18ea06e34fb1400fca65b03f0/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da9f1b8780678af93c4ddba46826166c53a25d8d71133fc818542bf7b481e033", size = 79497, upload-time = "2026-10-20T01:06:03.122Z" }, + { url = "https://files.pythonhosted.org/packages/d5/50/20454483d8f95a11710b951518f86aaaaaae9195982fe27c12eeed130660/lazy_object_proxy-1.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f21b4711e6744d46a4672497ca2db390cbdb03241d9b1654bbf38dbbc4db2a5e", size = 26670, upload-time = "2026-10-17T16:37:50.977Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/c1ff843fd9c4978d9a50ef2bceda3897d71645d4127ec1413aa15204cb64/lazy_object_proxy-1.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:de295ae25e5c1fd712088547f251c4b2976d2f60df28057b6c416e771a99842b", size = 25297, upload-time = "2026-10-17T16:37:52.088Z" }, + { url = "https://files.pythonhosted.org/packages/8e/87/3ef9b53ab82ec40edbedb5487095eea5b86e8f6260f86246dbbb0021eb55/lazy_object_proxy-1.13.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:14fbc6debdf55fe6a801a6f562456c20056f1d3b0a191f64552ecc4f9bbdcc69", size = 26873, upload-time = "2026-10-17T16:37:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/b3/2f42624f4c46464bc7fd1c15d50e6efb4e013faeb18bd58b94a34b8bf26a/lazy_object_proxy-1.13.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e5090a94e1f148b1323031342920ac4416d4702db93317e238724ea5252a0e52", size = 71663, upload-time = "2026-10-20T01:06:05.465Z" }, + { url = "https://files.pythonhosted.org/packages/33/6f/018118a339833a38f3a46f249ad04d2e97fa24ef2825b0515d71522ba19b/lazy_object_proxy-1.13.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89331615366c05e30bf5c0821b4a0536b42117c9e83d2aef03d47c8beebcdf0c", size = 71249, upload-time = "2026-10-20T01:06:06.531Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/c47648b865407872079d218f248180aa21d7cdfd2eaf13b5e0080cdb8a80/lazy_object_proxy-1.13.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:90e841c41f8cdd380d38f4e64df824d92e8245ea8798bf6ee7533c941b3c2956", size = 67114, upload-time = "2026-10-20T01:06:07.877Z" }, + { url = "https://files.pythonhosted.org/packages/74/29/453db8ff486f77e21b54bd70f792416c6e2a0660e5fb7d60739976741e4a/lazy_object_proxy-1.13.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:956757a130126c1c281c1f3cdb2856cef8ab933edf61bc7931c7947c91866af9", size = 69832, upload-time = "2026-10-20T01:06:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/9e/42/cb3f9cc75dfd59754f345a610fd90b7b7955e3b00963175c1b441dc7ee16/lazy_object_proxy-1.13.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:28d3de97c2dba54e2d1da3b3f2aef5c2bfab4a37520bd5ee275f7497914593ff", size = 66318, upload-time = "2026-10-20T01:06:10.107Z" }, + { url = "https://files.pythonhosted.org/packages/7b/81/eb932ad6b975f39010da76e27099ab25ca16617317aae06f00b3b4fa4e05/lazy_object_proxy-1.13.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c81002e6284f724cc1cb616951f254bb312b8fecc185de8ee205cd6288d370ad", size = 71004, upload-time = "2026-10-20T01:06:11.299Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cc/8f1fc032a070b6bf9ebd10340bf7e081bb90f979bebda1b5e7517c2ff8e6/lazy_object_proxy-1.13.0-cp315-cp315-win_amd64.whl", hash = "sha256:ca684bb99c88693f12df2f71d3753bec784810351b9053959ea4c17a453b9269", size = 26123, upload-time = "2026-10-17T16:37:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ca/c00a8eaeaf2100c012fc211c400ae17ff6b0659479f65eca9eedac3dcd62/lazy_object_proxy-1.13.0-cp315-cp315-win_arm64.whl", hash = "sha256:32fcf9849e1a27a1e8a79a8e593860661cc93e81b090283f911d823c8c52b092", size = 24739, upload-time = "2026-10-17T16:38:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/1dacde4090f1b78500e02539d531211ecd44e776d1b019cc56f11f49c8c1/lazy_object_proxy-1.13.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:cedfa35e1d9f2b91599b5f635e36d04c9accb68e10c413cbdb7d02d4327d20fc", size = 27502, upload-time = "2026-10-17T16:37:56.398Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9e/97b53e02d0c6fd7aac6b0b69f17ad60db4b347e276df134039a221f508b5/lazy_object_proxy-1.13.0-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:666d040e5f94c0d8420d5ce3e4e8926c1a51fbb26432a60dc61440c452c9b30c", size = 81091, upload-time = "2026-10-20T01:06:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/c7/98/4abead513874ed7abf13da76d986760aaa88afba32fa115bcd873e870882/lazy_object_proxy-1.13.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c842c320aadf7524b7361df692eea9a1d76ce533e35658188c4d51bc0dd7e85e", size = 82827, upload-time = "2026-10-20T01:06:14.029Z" }, + { url = "https://files.pythonhosted.org/packages/0a/26/6568132acafc3a1902d09ce47ce2ad51e63bfbed8cb696334be7db8710d9/lazy_object_proxy-1.13.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7fd1cbcf0ac498aad0a3ff4142cc4daf70961017ac895c6b2ed92e8a128d5009", size = 75963, upload-time = "2026-10-20T01:06:15.357Z" }, + { url = "https://files.pythonhosted.org/packages/e8/73/39585234cc2cb6b97338ccff83e174f6d031b0366e454db3a8626e35fee7/lazy_object_proxy-1.13.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:1bdad8baf318ca048af58d70fa1274e31e20b0fa073fbe9fd4ad7b32a51a3e61", size = 80810, upload-time = "2026-10-20T01:06:16.567Z" }, + { url = "https://files.pythonhosted.org/packages/61/2c/8a69f4e18b056504ee6530b1c3c40d1379fe62eefdb44416db87b996a675/lazy_object_proxy-1.13.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:291263a1647524673146f10192caf27113a32c2e6d169e19c6a257f2876be343", size = 75485, upload-time = "2026-10-20T01:06:17.63Z" }, + { url = "https://files.pythonhosted.org/packages/b6/07/1acf68a04a60ab916bce01637f91f051bf1e7dd0e79a5a92b8e1afe8691d/lazy_object_proxy-1.13.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dd0c8ce17ec708cc81d541697d3746f2dcd8e45d039078aee980f865f2f5d41b", size = 79936, upload-time = "2026-10-20T01:06:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/34/72/da0ef19fd182290d4cc465a9944be676c89529569e2e1bdb158bf0312850/lazy_object_proxy-1.13.0-cp315-cp315t-win_amd64.whl", hash = "sha256:68d6e346300c136dd4714a4772e592521beb1c0744c3f18f797854affeca96b5", size = 26657, upload-time = "2026-10-17T16:37:57.699Z" }, + { url = "https://files.pythonhosted.org/packages/6d/46/e27854a795dc70349a15f30195d62d1ecf53b107d9b4fd80a3834df63b6c/lazy_object_proxy-1.13.0-cp315-cp315t-win_arm64.whl", hash = "sha256:e97aa65f98cc4521f2e366a49dfb6a3c2315dbbf5d84e97d5d1e21b649de72fa", size = 25291, upload-time = "2026-10-17T16:37:58.808Z" }, +] + +[[package]] +name = "libcst" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version >= '3.14'" }, + { name = "pyyaml-ft", marker = "python_full_version < '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/c0/098e5c91ff1537f00c85a6438b6cb1863d17144680cc91f47c87f104a200/libcst-1.9.0.tar.gz", hash = "sha256:087b58a9afe076bb08e2d726478e1f16cb928d67ffa9092817e033c335de522a", size = 914739, upload-time = "2026-07-29T21:28:43.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/b1/befc0544283bb3923a928accf79ed685e5a725524bdb3491826670affc07/libcst-1.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b9df30f524317b097dc53065b25dda33d6a4cc3c7c8bf4fc83ca7559c58cc0", size = 2043754, upload-time = "2026-07-29T19:24:59.885Z" }, + { url = "https://files.pythonhosted.org/packages/45/50/fef7c172a8457c95894edf5fb04805024899cdfea41fa01b0636587b79e1/libcst-1.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e465a7bc9c2b9533eb9e06d2391f8819f811b5112919d4064c9fb8565aaafa08", size = 2203548, upload-time = "2026-07-29T19:25:01.323Z" }, + { url = "https://files.pythonhosted.org/packages/18/ff/764cd2be1fd99d774fc44039c319dc0ed1d9d9afeaa02759a05121cccd4b/libcst-1.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8504b422c95676a8c27b517e1ac01413ece91bf356865c587ca9bdcd5708a2f7", size = 2255274, upload-time = "2026-07-29T19:25:02.764Z" }, + { url = "https://files.pythonhosted.org/packages/34/a7/474748a27a02fa83e3556b260d5f5236ca48164fa7beffecf3b2cad24dca/libcst-1.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bcb9f9d4fcfe2ec7a40d2c26e03a538d5d9dc189c38551eb3f94ab661afee7c0", size = 2269126, upload-time = "2026-07-29T19:25:04.158Z" }, + { url = "https://files.pythonhosted.org/packages/8a/b7/655e45363b8cf87b91e41e060b21c89e5316c7ef36eb14a1a498b27bb71e/libcst-1.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8d671c39a431c309476099b8ec811e412503ec0f4465f6fc907cb51c70e8e6b", size = 2378602, upload-time = "2026-07-29T19:25:06.424Z" }, + { url = "https://files.pythonhosted.org/packages/db/13/6da63f0902ece43bf9d737017251ad7ad06edd9d3c4e1856450403cb4473/libcst-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:4d382fba04077eb556a1ea4295a4482e192aa93639c5a07ba0885841965ea0c0", size = 2103486, upload-time = "2026-07-29T19:25:07.979Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ff/dccb1a55e38b4e47256a97242d61d2bef5366c744faf88fb087d4fa0f995/libcst-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:a621e261990148c1cfbe26c1798bd2375c131cf358a44a7a4659fc41c1a336e1", size = 1979907, upload-time = "2026-07-29T19:25:09.532Z" }, + { url = "https://files.pythonhosted.org/packages/65/2a/4943c71d90975bc59034a057dea346b365c276f308c1d31f5cf2bd85492d/libcst-1.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eccf4c57d273cdd3fe1c67b72cf9bb1bbd4547aa011824e96ccf5b7136057aa4", size = 2044529, upload-time = "2026-07-29T19:25:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1168b98c2a0f338be3a44753647baab051feaf6167cc887bf4447f8fd920/libcst-1.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32395244edfe6538e0ea2bf82051d60103d3f54861274805c4fb3745efb70a85", size = 2203519, upload-time = "2026-07-29T19:25:12.543Z" }, + { url = "https://files.pythonhosted.org/packages/25/7d/2eaa697a80f899bcf2245680bbfcc1e478eb3b3328c21d4b2880bdf00dac/libcst-1.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:444e84c76cd035cd2fe136838c1a524d34b08216521f6f5093df8a6f6cfa5799", size = 2255879, upload-time = "2026-07-29T19:25:14.137Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/793b9fd96dd52d202f5f2b88d7e54f05ebed767d1bb3b32395a1817bf6ab/libcst-1.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb5d0946f2b4c6711b5d69fe4f833b364f9e7a1a2b08f88b98619dc18975099a", size = 2269807, upload-time = "2026-07-29T19:25:15.647Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/1bc7aaea03971c45e8a885fed9fc1c73176dbbd00b0296a3cde9529bafd6/libcst-1.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45808c03528b3ad40b14095348a918e08d98c47b4a125d631cb78e817c0a5b16", size = 2378171, upload-time = "2026-07-29T19:25:17.289Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f7/bc49e367d2bc8817213594dd52cf6b2da2dbbda90b5382d60653999274ac/libcst-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:568288cdbfe3b4ca3ae4852cb0a439ff053dd54c841bc4995bf2b71238b5de40", size = 2177847, upload-time = "2026-07-29T19:25:19.35Z" }, + { url = "https://files.pythonhosted.org/packages/87/80/4d81577a22e6d535d1a3409f3a1c6903e09036f0e17fc47f92169dbc3501/libcst-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:107593af46945593e7825821793393262bc4fa1d3ea24c3ed487b61269bbbdf8", size = 2058134, upload-time = "2026-07-29T19:25:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7f/c3f3a0e7a1a2adaa76e815e82a7814d6bfe7d49432276bba652248c68d0b/libcst-1.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f6248cb07444ab9a6733a855737a9febed8b9adca51347019348b09a3ac7dfe9", size = 2036102, upload-time = "2026-07-29T19:25:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/4e/af/2f5543255b2c7d749b966adeccc6bfb1652cf8b425afb913d0f3f025a865/libcst-1.9.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:496c24e0d3240bc7da45dae543aff3f5b6509978c39262d6b84ed2fb999dded2", size = 2194806, upload-time = "2026-07-29T19:25:24.017Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5b/03f4cddce426d005e208b39ea7b2d456e667cdee0f1891360f0fc8430f20/libcst-1.9.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ea490fa8540503db5f321f0268becab46eb50f710e8cec8041e241fd66f6874f", size = 2246762, upload-time = "2026-07-29T19:25:25.365Z" }, + { url = "https://files.pythonhosted.org/packages/d8/31/9d5fe1e43dc3dbcc74f70f3e0e73fcdd8d84effc1059d8b45974f0d0d2eb/libcst-1.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:50ab94bb2524b419056d4003032b8c67102ac800f8d85b3c4260a01746d94dbb", size = 2259633, upload-time = "2026-07-29T19:25:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2b/6752b28d88c3a19b3bc0b9e1838443b6760d5c862b4f4b37955402659e24/libcst-1.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a2faaf92500d0226358125630f5aab4758e8aad3f2d70a10892ec3c700781a54", size = 2368043, upload-time = "2026-07-29T19:25:28.344Z" }, + { url = "https://files.pythonhosted.org/packages/e2/94/775825b2637f8ab05694b6a4b3802ae6783b4e799f9b58d2400c7e2d4369/libcst-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c7b548512db25af9c2997a95fa731bd6b6928ecbad6c0915d7482d8bb42d34f", size = 2176432, upload-time = "2026-07-29T19:25:29.921Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3d/88ad67427c6fd9db929e087912b0a540e5140e5cb77e7ca4170edaac8531/libcst-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:497d5329345f1f5df84e41b0bbd00204b64a2fd30dfe3cfaeaebca633a31e877", size = 2052931, upload-time = "2026-07-29T19:25:31.397Z" }, + { url = "https://files.pythonhosted.org/packages/39/e4/ad790b043a38b10cea13166c8dd716654ad8b227c20f12eebf6326191cd9/libcst-1.9.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a5068bf6114f6f4d79af7a6c80a28d1deb50441150ea1db13b5458ab34bec159", size = 2043987, upload-time = "2026-08-11T05:54:02.122Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/d3cd8275cc54ffc9817ade91442b3e6e9edd1a4518bd4e6dda13e3152dbe/libcst-1.9.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:7acfd18adcdd32dcf41ce676cf121002afcbe9ad2c72dc0c18d78465beedc228", size = 2204003, upload-time = "2026-08-11T05:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/da/62/87eddccb11d5d221d50ac4fff4b6e9b8d48c547028d01f7d0fbc5c8a1d75/libcst-1.9.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:86361e2b426bd1b403e52375703c8b764e226e571adcb30442510710681edbf0", size = 2256053, upload-time = "2026-08-11T05:54:06.228Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/a44126aeb9fca8b4e342e0cc803ea00ca8f6cd5712619a7897b3eba13797/libcst-1.9.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8c14abe844bec8b021111b3985474e49f5af799c020e8e40dedcac87c97a40c5", size = 2270576, upload-time = "2026-08-11T05:54:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/bb/51/3af3dd44b117d66486e0ed61e43a16b7fc8cc5c372e1dd4ca0e70b888d46/libcst-1.9.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:8930971d2299b7006bb5d10c10038f44efb6da89d5bca2823595e31a9cdb96af", size = 2378573, upload-time = "2026-08-11T05:54:10.093Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e8/e545087d298dbf6494e84a8092f0f5dbd62eedacabd4ecd6b364367b4804/libcst-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5891ce9cff815077614f509e3a23888c5ddb8081d6188e8ba1172c0ccc369022", size = 2177937, upload-time = "2026-08-11T05:54:12.168Z" }, + { url = "https://files.pythonhosted.org/packages/84/17/93a00a8e03494a84db102dd0e3d35b8876b1084ca2c194b331a168d86eb1/libcst-1.9.0-cp315-cp315-win_arm64.whl", hash = "sha256:1260b5d070a3324447a53a00387225a55472a62077ce01922d30a2a78cc4b138", size = 2058633, upload-time = "2026-08-11T05:54:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/d7/00/c3751b12eb81822ea0b6131d374a98bc6c024c4b9ec5f6ea8f53dc23e967/libcst-1.9.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:02ee2dbdb5c218116f16a021350fc267a14be205b50d81c33f5d73857ab6fbf7", size = 2036178, upload-time = "2026-08-11T05:54:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/2f/12/48ff486eb5adc593f4818569e8896b74b672414ab4722a74b17e4c4cf31e/libcst-1.9.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:3e5b684191a0462b2d40a73261ea7f4da6a49e7a030b7e0fceca46bebb51dfe5", size = 2195496, upload-time = "2026-08-11T05:54:17.625Z" }, + { url = "https://files.pythonhosted.org/packages/51/18/b13a4669864d41a4801fed7b86ede1049dc9a1e1dde250a7ee1f9c3b1285/libcst-1.9.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:2b150c4f298fe54eb0fe73abf71f57db74d070796140a8c206c9615b6861f01e", size = 2245769, upload-time = "2026-08-11T05:54:19.343Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c7/cf2d46744825e84afbd7228fdeeea8d23cf6c4b2074253c27efb7705c653/libcst-1.9.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:50b913e37187f00a6fb88962009364e1cf1b1284aa1762304f34b52b972f2218", size = 2260576, upload-time = "2026-08-11T05:54:21.444Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/5433d3d62250b2e4325938db101766bcab2a006819684713bcccb6259a88/libcst-1.9.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:9f92c75283030fd58fb7d6e560168a00381e0e3368ec8a026a3ec8a828a05f93", size = 2368709, upload-time = "2026-08-11T05:54:23.276Z" }, + { url = "https://files.pythonhosted.org/packages/87/39/9f0e1690727623f99489895db0e42048a3e1e8cea0627517c593e437fd83/libcst-1.9.0-cp315-cp315t-win_amd64.whl", hash = "sha256:4adf97bb1aff8039b0bd4991e2f838be6e4fad552821b26b71a5d1a653c2582f", size = 2177866, upload-time = "2026-08-11T05:54:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/cd/79/9dc7811883e67ea771057e8937bc536eb3e77f3635fc5a93cd85b6fe1ea8/libcst-1.9.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8f0dd08a5773d7051e105250b2a2c73d8065ea9b2d68e7e1bdc5244660e75f93", size = 2052908, upload-time = "2026-08-11T05:54:26.835Z" }, +] + +[[package]] +name = "linkify-it-py" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/98/7a1a5f31fd5c7ba93e963b168e244b8e3dd705b3d2a718e3c3307583bf57/linkify_it_py-2.2.0.tar.gz", hash = "sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee", size = 32939, upload-time = "2026-08-29T07:07:08.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/d4/1152d1c7ab42d8b908be64fd200ddc870dc9d4925e951198702084aa1a7d/linkify_it_py-2.2.0-py3-none-any.whl", hash = "sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443", size = 21971, upload-time = "2026-08-29T07:07:07.164Z" }, ] [[package]] @@ -2083,6 +2174,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] + [[package]] name = "marko" version = "2.2.4" @@ -2202,6 +2298,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "mdit-py-plugins" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -2338,11 +2446,11 @@ wheels = [ [[package]] name = "more-itertools" -version = "11.2.0" +version = "11.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/e5/8735dc589d9e3571e76cd684e7763dc0fcc940b811f71640410458749689/more_itertools-11.2.0.tar.gz", hash = "sha256:59960f488835254863a857eb501f69e4ec92fdc8822f098c0f48dba530afd487", size = 75940, upload-time = "2026-10-13T16:01:14.428Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/b8/c63995bebbb9ac9c059350e0fb8c82c2420c85aeca7d34e40aecf3df9281/more_itertools-11.2.1.tar.gz", hash = "sha256:cbf08fd0af284dc69718b9b76a8e6203df624d70209ea511e4219d350c856f63", size = 154133, upload-time = "2026-10-14T18:50:30.407Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/f7/86f27473377a8ebdbdfedec5e62e87fbc24c4c88ed0a2a5ea0a12ee2229b/more_itertools-11.2.0-py3-none-any.whl", hash = "sha256:e4c522352a24664c974d0be8382944107ae638281a06391a5a538366b5707fe5", size = 74407, upload-time = "2026-10-13T16:01:13.424Z" }, + { url = "https://files.pythonhosted.org/packages/5d/57/3c4b1c6f2e0cc29ba023802458ea3d2a996f87085559e4a5f2192d549532/more_itertools-11.2.1-py3-none-any.whl", hash = "sha256:35a7377edd1dd6608dcb2cdf534ded55ea32d49448875ad042cd3f879fb1ded0", size = 74405, upload-time = "2026-10-14T18:50:29.116Z" }, ] [[package]] @@ -2487,6 +2595,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] +[[package]] +name = "mutmut" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "coverage" }, + { name = "libcst" }, + { name = "pytest" }, + { name = "setproctitle" }, + { name = "textual" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/30/b334a22184026d92a7549c8a76ad1d397ea65a6f8534ef486556ab861c70/mutmut-3.8.0.tar.gz", hash = "sha256:bc670276518f893ab356f1d28826c49b7e606e7cf89c300e3888c0b52fc1d2ed", size = 83631, upload-time = "2026-09-12T16:53:58.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/2a/a9904a45804c1de50c0c3baad1512b78d5b66d2befd841104b8a62bf0f35/mutmut-3.8.0-py3-none-any.whl", hash = "sha256:4c0abcef6b8d1116f2adfaad19df12e1c621e4a402d4367ef15bf8cba497b91d", size = 81469, upload-time = "2026-09-12T16:53:56.906Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -2828,11 +2953,11 @@ wheels = [ [[package]] name = "petl" -version = "1.7.29" +version = "1.7.30" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/31/8c530c6b1f6d1a40c0fd09cb9f54b6553385a8f36c248ea699c1d5f89544/petl-1.7.29.tar.gz", hash = "sha256:3877f34bbc5b0b7f0214bf219584eefd08a86c144154726f1797f2051906f9c3", size = 458989, upload-time = "2026-10-11T05:05:44.264Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/5c/ec50cc70f4d73eb86172b96c67f377d99aeece0f1db983d8aab9582c89b8/petl-1.7.30.tar.gz", hash = "sha256:9621fe7f9d81dafec3bf440496118ec1b288215400810a90f683a02c6d120792", size = 460151, upload-time = "2026-10-17T02:53:55.179Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/0f/502b19a57156c2aaeb49f455537abfff793d7459de8116c77cefb221d473/petl-1.7.29-py3-none-any.whl", hash = "sha256:a52d87a03a1390bfed16afa38dabc2e3068061b62ff8e58ce61b61cf8d46d0bc", size = 262757, upload-time = "2026-10-11T05:05:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/62/e7/67a1467722b31a8e2ee2ae9e92ae7fd1338e6f02cffa462461c49aa39fd6/petl-1.7.30-py3-none-any.whl", hash = "sha256:99b7e9a66b68908146f0f590a4bb61a270572bbe33f33dbf5036a234e4fee2de", size = 263769, upload-time = "2026-10-17T02:53:54.013Z" }, ] [[package]] @@ -2849,20 +2974,20 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.13.0" +version = "4.13.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/a8/66d45abadff219e36e2a824181b8f6a67e7ed4572934d6252c71c29d5731/platformdirs-4.13.0.tar.gz", hash = "sha256:1aa0b0d3f224c1f07c295121e312a5a24a180d6ae5a8425ea1784b3e3863e9c0", size = 61094, upload-time = "2026-10-11T02:05:24.109Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/e6/d0abbb256b44e6a05748c6942511ff67db8dbb068de0c35b24bf1b395b09/platformdirs-4.13.5.tar.gz", hash = "sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0", size = 62362, upload-time = "2026-10-19T01:02:11.778Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/15/1633010b26e88e872c93b67c0b6c5e174fb74cb6fb5c1472b4d51d4a8f22/platformdirs-4.13.0-py3-none-any.whl", hash = "sha256:3dbcf4cd708f21cf876c4eaa90e58412bc4f033d87143f41b1493ff77c25b7e1", size = 32724, upload-time = "2026-10-11T02:05:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, ] [[package]] name = "pluggy" -version = "1.6.0" +version = "1.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, ] [[package]] @@ -3224,7 +3349,7 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.16.0" +version = "2.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -3232,9 +3357,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2e/3b/a5d2294799b53b448319978cfb5bd139d5a9d45e862af91661614f14c922/pydantic_settings-2.16.0.tar.gz", hash = "sha256:5b6c578049ede4db0e2ef3b4eaa4ad4069cfa9211f83fb38df899dfade50a614", size = 301031, upload-time = "2026-10-14T12:44:09.998Z" } +sdist = { url = "https://files.pythonhosted.org/packages/05/ca/7817b4bf0c4b15c6a6ab8cdf4cb96c268335a8ca2f525de93b6ebe4c7d28/pydantic_settings-2.16.1.tar.gz", hash = "sha256:aff602a1886338c2df3d1e958e2193a8dc33cec75b91f202e83fd98484b4b0da", size = 303436, upload-time = "2026-10-20T11:37:19.024Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/f4/b987bf8c51e5b19a95fa66d1ee596074141e085d9c2ddf97920803c7029b/pydantic_settings-2.16.0-py3-none-any.whl", hash = "sha256:7e73acf7f61936a15e5a3b6eedaea29f133357faf7272f2607ba479b049dd7f2", size = 80364, upload-time = "2026-10-14T12:44:08.233Z" }, + { url = "https://files.pythonhosted.org/packages/3c/74/fdf926afe145d8629c4b2ff57f4ea1d64ed7abc4d84d97463b2517a2547e/pydantic_settings-2.16.1-py3-none-any.whl", hash = "sha256:30f5313340da4e8f1b440d1910b92a5bc3ccbb6b4ba70ec2b8395bc921f418f5", size = 80771, upload-time = "2026-10-20T11:37:17.298Z" }, ] [[package]] @@ -3325,29 +3450,36 @@ wheels = [ [[package]] name = "pyoxigraph" -version = "0.5.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/bb/df1eebcf8cfe6783a63b871f53bddeb461cac663505b18028ad44f0ccabf/pyoxigraph-0.5.11.tar.gz", hash = "sha256:2b7d9bf02e7ed89cb0cbcf6c376aef361f1c3c9de49a7a8fb3ac231544bb6ba8", size = 5303115, upload-time = "2026-09-02T20:05:42.8Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/70/470f1fd094ad6931e6c63b1130ff75000f2e01d69d75e293c0d2910bebdf/pyoxigraph-0.5.11-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f2e94296ce723ed030784a79c02f7e780522588840c5a8c44e118bd7c0d280a4", size = 7683421, upload-time = "2026-09-02T20:04:55.259Z" }, - { url = "https://files.pythonhosted.org/packages/2c/0a/4ee81724aa7817aa0d15d762c8acec8a90cc0e843f57c883d2e108afe543/pyoxigraph-0.5.11-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3de0588f90a467fe2467ec76588bccb8c18e57f05f63c89b6ea921b057b37365", size = 8186692, upload-time = "2026-09-02T20:04:57.22Z" }, - { url = "https://files.pythonhosted.org/packages/ae/29/816040a8fd51026aefa5323939424a190cd69068d5beddf91e651243f2ad/pyoxigraph-0.5.11-cp313-cp313-win_amd64.whl", hash = "sha256:8aaebe4656b9e9d7ee575dad1c1fd810bb52bfa0690f13bdd408e975ae28b868", size = 5427768, upload-time = "2026-09-02T20:04:59.647Z" }, - { url = "https://files.pythonhosted.org/packages/01/b0/bcde9432c0044369eb1b2e48c826559787ab7b1e713a38362f1e6bc1f9f2/pyoxigraph-0.5.11-cp313-cp313-win_arm64.whl", hash = "sha256:acbc9f82b75d8c39aa80fcf3c6d9f897c9bb23776af868fb6e9e39dc054e0d2e", size = 5084241, upload-time = "2026-09-02T20:05:01.934Z" }, - { url = "https://files.pythonhosted.org/packages/50/d2/873dad18e44c49c6d395c6b50e3dd97e45807dc645f46a9efa0f5c07798d/pyoxigraph-0.5.11-cp313-cp313t-win_amd64.whl", hash = "sha256:f6caa21919d0ebd4f165a4ade703e1f24cdd9cdb0a12fffa56440228d1106873", size = 5423392, upload-time = "2026-09-02T20:05:03.881Z" }, - { url = "https://files.pythonhosted.org/packages/de/9c/1618c0fd2e68608c2034d122fc620a080294b26bf3c2e039ef6706e50d91/pyoxigraph-0.5.11-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:18143baee09f6a3f17c096d6d58dbb3b1bf023ac5d6a52521cb2437cbf24b4a3", size = 7680302, upload-time = "2026-09-02T20:05:05.611Z" }, - { url = "https://files.pythonhosted.org/packages/bc/e4/9ae9d8014cf039a12c1d174e202587d2b3947226f9da173391cd3e344fa0/pyoxigraph-0.5.11-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e02906504ad2ac399d1f30cbae2e47b85932d39bf89ef5c7508268faa6ae3bc4", size = 8182140, upload-time = "2026-09-02T20:05:07.496Z" }, - { url = "https://files.pythonhosted.org/packages/5c/85/e8d325f5c001d16a67df710d14a8d8e2eb9a68c806a92dc62621ecb6bbd7/pyoxigraph-0.5.11-cp314-cp314-win_amd64.whl", hash = "sha256:81ccae2810d6f6b699c49f39a157a060b5713421e91ab7edb0ef354be04af583", size = 5421318, upload-time = "2026-09-02T20:05:09.182Z" }, - { url = "https://files.pythonhosted.org/packages/f1/8a/0a40ecae761d3e559873e20ac07f138ede16d3ce9f23f2cbd2f6a7cf239f/pyoxigraph-0.5.11-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b5167ed8771e9cdfeb8640c8f04aed06c295e5049752899d0ca221477ed327bb", size = 7674974, upload-time = "2026-09-02T20:05:10.997Z" }, - { url = "https://files.pythonhosted.org/packages/50/7b/f5582bab4d251ab9fbd4de20dfee17fe88d5fa3e73fb96683ea692c66421/pyoxigraph-0.5.11-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:13ed2633b72cf4a7cd6ef405d225e1a3e505228ffadb73c5f0aea4fd65f95cd9", size = 8178709, upload-time = "2026-09-02T20:05:12.938Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d7/ba4406bdc3d3fe7a5f0e3718e2838d1b40aa73f61090d801c36ca0591468/pyoxigraph-0.5.11-cp314-cp314t-win_amd64.whl", hash = "sha256:f58294bd2695f2fc8074f9bf8a381281c737f2903159ca602f5bfc3834559174", size = 5418434, upload-time = "2026-09-02T20:05:15.458Z" }, - { url = "https://files.pythonhosted.org/packages/38/c0/824cdec1e1ea9f6d4d05da51a843be668d3a2d02d223b780c138fcc4b2f9/pyoxigraph-0.5.11-cp38-abi3-macosx_10_14_x86_64.whl", hash = "sha256:aae8c162fd349a33255f580c665d8f950aaa875d65f64fae4a6c6fb93b5b7ccd", size = 6305518, upload-time = "2026-09-02T20:05:17.462Z" }, - { url = "https://files.pythonhosted.org/packages/18/fe/23899fc8e17fb6bfa37d606f8afc755c05dd081bd690d360d3754ea7d520/pyoxigraph-0.5.11-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:3b67839b598fc806dbed8e99eb2d75b26b0ded6d52ca8bff1496d6a3cc002036", size = 5817955, upload-time = "2026-09-02T20:05:19.199Z" }, - { url = "https://files.pythonhosted.org/packages/2c/27/175c5099548c76f85b1b80a8017ad98bff5bc92adc568b472d615e1f712d/pyoxigraph-0.5.11-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:96c9c4d117a0f4d0eae2c9092a490c6c51b0b8114ab7b126b8dfb0a8f0be2745", size = 7681128, upload-time = "2026-09-02T20:05:21.084Z" }, - { url = "https://files.pythonhosted.org/packages/9e/3a/9ec824aca0377ba56a7834222454c19392ff85b00e55fff9894f5211d655/pyoxigraph-0.5.11-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed906c05164d4766046a899f5944b4cf63309e717e3f464b2c0c80e8de91fa16", size = 8185704, upload-time = "2026-09-02T20:05:23.211Z" }, - { url = "https://files.pythonhosted.org/packages/98/25/5b0b9ecdebbd7600c3642be4b090cbe1c9ac5bae440c4bf2e5f82311cd0c/pyoxigraph-0.5.11-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c0462f03c4e3789fdee48faaab0edf780379fe812d1d70073eae14da86eadc9", size = 8879152, upload-time = "2026-09-02T20:05:25.423Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b4/fda0014c1ee5bc7950dfb7b9ce1c5f0bbb611d61560a9ef7e38dba9b83af/pyoxigraph-0.5.11-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c4f2c4c907dd751cc7f7966217dcb33ecb89c89c30b1992665ae965ec5064f01", size = 9412532, upload-time = "2026-09-02T20:05:27.772Z" }, - { url = "https://files.pythonhosted.org/packages/72/83/1588895bad95d257529a0b5bf47872f0c49602dae3cf2f6f1bbe9a5d583c/pyoxigraph-0.5.11-cp38-abi3-win_amd64.whl", hash = "sha256:1057b853663e3fa296f92dba3bb4145f545600261da0943266f4f449d8f7f0a9", size = 5424225, upload-time = "2026-09-02T20:05:29.95Z" }, - { url = "https://files.pythonhosted.org/packages/8a/61/fdb038cff915024cbfd5f6b8637e747c2e054261206a376aede1ee71588b/pyoxigraph-0.5.11-cp38-abi3-win_arm64.whl", hash = "sha256:ec99a70bfc9683dcecaea1f3000b6d6ba9c34a641dda48e660c456454f642ee6", size = 5083910, upload-time = "2026-09-02T20:05:31.573Z" }, +version = "0.5.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/00/d1d29089963a4863faf34d0659a5797c23503f1000b35202f15856a016b2/pyoxigraph-0.5.12.tar.gz", hash = "sha256:e1932585267dadeebc11ad97985a916e2cf66df81abb1e335495ed0ce2d5f0ed", size = 5300947, upload-time = "2026-10-18T20:01:58.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ce/10abfc263330577320188ca0cfaed4cd429f6d13d637698f8ca18858731c/pyoxigraph-0.5.12-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:4764a0cd745f1a3d310432a637b3ab4bc7eaaa24e294ec37b0e2a4e04d7d0ea5", size = 7700530, upload-time = "2026-10-18T20:01:01.299Z" }, + { url = "https://files.pythonhosted.org/packages/c8/68/262312b07ee1089bf929eeeb22f89967d9503b080c670ea665482111f462/pyoxigraph-0.5.12-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3c00ebb1289505cf79ee0336bae2df84f873be33f769d8f48e3d063cc4eeb60b", size = 8233189, upload-time = "2026-10-18T20:01:03.275Z" }, + { url = "https://files.pythonhosted.org/packages/7e/78/3e43c27d981f2ec549b9700e4d60d001f5ab4631806f1fa485556fa2c93f/pyoxigraph-0.5.12-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fcb9991d19f8f6566ef800c6016b566d60f7a38d4076bab7e0fbd3add32af627", size = 7699716, upload-time = "2026-10-18T20:01:05.048Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a1/edd0398096eda0522f4f49b5de50f40a7f9278dbef28aa7ba290e7454ce4/pyoxigraph-0.5.12-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b59c72eae73d4e66c6a687a510133a54ebce3e3ebbdb7c69dfa7e4de36504427", size = 8232847, upload-time = "2026-10-18T20:01:07.208Z" }, + { url = "https://files.pythonhosted.org/packages/07/97/32ed238d27872a724676f84b590c2ab64d64726c0f8105fc2a89f10ca268/pyoxigraph-0.5.12-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0077b688f6fc9cf3db4f3eee7acb5153734e958bee2be4ad7345121332c2bdfe", size = 7694160, upload-time = "2026-10-18T20:01:08.986Z" }, + { url = "https://files.pythonhosted.org/packages/3c/15/6dd1844c2f09b963303203a8f531c7cfc48cfa548beff5c19293814cf4be/pyoxigraph-0.5.12-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ad18580dc0e4ee4afb38e10f1e80f536a0a92194455b7f8c964f11fd0c64a8c9", size = 8226954, upload-time = "2026-10-18T20:01:10.982Z" }, + { url = "https://files.pythonhosted.org/packages/3a/33/6b9ccc2b9c91e8f5e59913be756483da61e98c598ed2c70267b473719562/pyoxigraph-0.5.12-cp315-abi3.abi3t-macosx_10_14_x86_64.whl", hash = "sha256:63bbb2c36f72aafa28424d6048336a6a6799236dcb80f48d3c8f00e3b7c07bfa", size = 6375614, upload-time = "2026-10-18T20:01:12.729Z" }, + { url = "https://files.pythonhosted.org/packages/0e/7a/650f2305517f193095bc5fb9069ed38044d26516c18502ad60e8c871b9dd/pyoxigraph-0.5.12-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:2f190eb24aae604dcf2739cb434c0ce87c1eafcb220de86675b1d59c73b739ca", size = 5864298, upload-time = "2026-10-18T20:01:14.439Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6a/1e318f595e5569cc0e9b5f3560f01a137a598ed1ae287a1320913bb1bb8c/pyoxigraph-0.5.12-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:fa44ed2c1b0b73a42398605534a22807762959ded229c04c7994d35777f715a7", size = 7732898, upload-time = "2026-10-18T20:01:16.503Z" }, + { url = "https://files.pythonhosted.org/packages/20/30/ef64fb7a02f5ffdfb4eb214a87995bdbbb8c4857ac6a7c454f7877d30297/pyoxigraph-0.5.12-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:060491a16a76b4dd617ceff85a25827a84183407f95308be0d5874f0df513ff2", size = 8266606, upload-time = "2026-10-18T20:01:18.42Z" }, + { url = "https://files.pythonhosted.org/packages/0e/43/b6ce713cac64dfe33a4beeb8e1f30f61a109a1e9c374de5aca5f5febadcb/pyoxigraph-0.5.12-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:5ac5d9fbfe61a3cd7551b421ce54bbd5bfd2aacf23883491594830981b76a5b9", size = 8924250, upload-time = "2026-10-18T20:01:20.374Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a1/3a29665bc035be550876890457931d6fde8940880a53dc3e7348d68cfdae/pyoxigraph-0.5.12-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:87459392fa087a14c2372826cb137f93ddfde4e7e566502619ebcf30c190ecf3", size = 9490513, upload-time = "2026-10-18T20:01:22.588Z" }, + { url = "https://files.pythonhosted.org/packages/9f/85/b06ab1e215a120924605e2f3aed723f81d568aaacd5e435cf8403b49f3ae/pyoxigraph-0.5.12-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:e8cd3698abb52e0be45db9f74827e65d5b4268b990e58a87b61c2b671c83279f", size = 5495177, upload-time = "2026-10-18T20:01:24.808Z" }, + { url = "https://files.pythonhosted.org/packages/63/03/b8c4cd224569f3ff53ec946928394f14116ef5e849cfaad6c46b562f3c77/pyoxigraph-0.5.12-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:0bc1816d12c63c5500eab5825d2d376ed55d86b66b5bffb3967daf51738bd4aa", size = 5144209, upload-time = "2026-10-18T20:01:26.758Z" }, + { url = "https://files.pythonhosted.org/packages/2e/21/f115cd9d4151f60fd2e96fadfd7103bae08d46493f6a016fb403d8483066/pyoxigraph-0.5.12-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ae8c01d2789d3e42810b8965ea18024689b9a76c54c007d00c7819ea6471bd8f", size = 7699015, upload-time = "2026-10-18T20:01:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/e09a14d2f8046481a8a926f4b6a610749d95b3fc0f585d4aabbd03cdfa29/pyoxigraph-0.5.12-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:2169f6d692b79f0ef2ab3a4f1c217a5a350a7b365d61db8b862296ef5c0ef04c", size = 8232403, upload-time = "2026-10-18T20:01:30.979Z" }, + { url = "https://files.pythonhosted.org/packages/04/3d/00674b72b2b2163eece7ebf3ed7f1740bf4275c5503c1342fafb029ca71e/pyoxigraph-0.5.12-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:11f069241d0be167a473c44f6d3184cd7560af771edb83c0d7f7c19d8c2acb94", size = 7695279, upload-time = "2026-10-18T20:01:32.993Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c0/194e4ace63625c153a1fa880887b2d250978f9a0581df19e0f98406b3ed2/pyoxigraph-0.5.12-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:8e105308a4c76ce0f98e014833f8b4181b82ada18b02900d687e57af954f8fbd", size = 8227289, upload-time = "2026-10-18T20:01:34.962Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/dc406eecec6ff6ae6b0f122a294784799a0f2e4d69c22ce460126fac3fb9/pyoxigraph-0.5.12-cp39-abi3-macosx_10_14_x86_64.whl", hash = "sha256:1a0c8931365df07fd77d5f6f7997bf0716ed5a7148d40aee2a6513101ecb6609", size = 6343151, upload-time = "2026-10-18T20:01:37.112Z" }, + { url = "https://files.pythonhosted.org/packages/d9/29/674774cc216bd671329ec1b5b60cded69c2566b40cef43eefe48de622099/pyoxigraph-0.5.12-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:b828b0e2a00c9d14725181d65677074a25f5f87e53eaf6bd89cfc772bbcf94b5", size = 5837673, upload-time = "2026-10-18T20:01:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/a0/24/9210f9d4b359347e8b4e1e484e11122646cabe8b3d64db65b4c13d9c372b/pyoxigraph-0.5.12-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9ed87da388d6cbe3ae41b4f5071c5890957d434afab872b104349a322f985e48", size = 7701997, upload-time = "2026-10-18T20:01:41.116Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/259a7296568de62f144937364d7b3e3baccfbe82ec726f13c245da2f8f7a/pyoxigraph-0.5.12-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f19c69a212771ad5368f51061731eec847cd8aa7f3075de9beee728e5c82826", size = 8234885, upload-time = "2026-10-18T20:01:43.607Z" }, + { url = "https://files.pythonhosted.org/packages/9a/b3/39eb47fff5442b9e762bae09eb7c2c496809f2b0090bdc5cb569e8f572b5/pyoxigraph-0.5.12-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d968d0d45f1f38c02a4417b3eebba3e830b2556732a67f2a528832daca10805", size = 8889557, upload-time = "2026-10-18T20:01:45.888Z" }, + { url = "https://files.pythonhosted.org/packages/a6/63/f9dd746e4095bc0704caa8ceae48b448cc3c44abdb76ebbca14cd7019166/pyoxigraph-0.5.12-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d89ef734fa4a51055bd234f1812c344fbaed7bb97d51516730cba5a517a6bac5", size = 9458815, upload-time = "2026-10-18T20:01:48.616Z" }, + { url = "https://files.pythonhosted.org/packages/7a/bf/8d9356c985516eb3c0fef581729875d01c9201a93d87c629c83d85988290/pyoxigraph-0.5.12-cp39-abi3-win_amd64.whl", hash = "sha256:41a146a5b1346c5289d06fc9c789ee045092a797a6d68b687e2afaa1714cfd2a", size = 5470372, upload-time = "2026-10-18T20:01:51.165Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2a/69dbd598932b509b25d6152fe4b2722c955ecc744e61e36e76e7de61fdd3/pyoxigraph-0.5.12-cp39-abi3-win_arm64.whl", hash = "sha256:fe34a10e262f93b4642f6e888aaa15aefd7b430c7c312ace511ccd01b1350a53", size = 5124124, upload-time = "2026-10-18T20:01:52.986Z" }, ] [[package]] @@ -3475,16 +3607,16 @@ sdist = { url = "https://files.pythonhosted.org/packages/c3/33/c987434f5d50aa802 [[package]] name = "pystow" -version = "0.10.2" +version = "0.10.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backports-zstd", marker = "python_full_version < '3.14'" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/ad/da2fd4b23ccf2fa81088a49bbc7d02f3fda7bd522d2f2a9ead77d474d2df/pystow-0.10.2.tar.gz", hash = "sha256:e30ff664a9099ab9356b25e4fa3c69a1612a32c2c5b81e8a23b4c8efc2fb8a6d", size = 58055, upload-time = "2026-10-08T08:53:44.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/a2/6e17e1fd6385b4426acfd75369593856536c04073576aa35b055bda6fc33/pystow-0.10.3.tar.gz", hash = "sha256:09276e0eeff99069eec1cc6ca24a0e90e4202981d98b2783c2a7c7bb323f8817", size = 64277, upload-time = "2026-10-17T08:12:59.548Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/05/6794a5d75724ca61afe81a1f1fe221cb018aaee5d20e42f756341fc0fee6/pystow-0.10.2-py3-none-any.whl", hash = "sha256:1f992aab1d20b8ba4b7942b3d0a0d4a84e48e0fcdb2b8781ebc7d41bf7648a2d", size = 65547, upload-time = "2026-10-08T08:53:43.008Z" }, + { url = "https://files.pythonhosted.org/packages/60/90/399349845ee1f84c182cb3f7e02ee0e89c6881ee6296571d19ee63989c7a/pystow-0.10.3-py3-none-any.whl", hash = "sha256:d70467457362a1c9373c575904bca5c63799bd011b6313ba15b4cf82f313957a", size = 65858, upload-time = "2026-10-17T08:12:58.105Z" }, ] [[package]] @@ -3531,15 +3663,28 @@ wheels = [ [[package]] name = "pytest-env" -version = "1.8.0" +version = "1.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, { name = "python-dotenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/e5/02fb78ca59d456291135b48fc95ee481a6e55dc47ad53e6cee31a7e40d54/pytest_env-1.8.0.tar.gz", hash = "sha256:e2dd383be15823a875403509949c0d975266f4f1711ccfbdcdbcbcf8fa83097c", size = 17363, upload-time = "2026-10-10T22:49:40.747Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/36/ccea73f55f269dc15c17ffb85d86beab7602bd7712f509300ef0d6bb937a/pytest_env-1.8.1.tar.gz", hash = "sha256:996f86ad78b6c7b5335e0d493f9e9ecd3e421174e056742856f91da439e1cbd9", size = 17852, upload-time = "2026-10-18T06:54:41.736Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/cd/c592fb78f8d5fc456568faf1983f2b994f00aa1690f0b3b24af95ad75121/pytest_env-1.8.0-py3-none-any.whl", hash = "sha256:43a026236949342be217f1539fa7ef32973f0d3a98d79d86a15eee973d3b1a2a", size = 10385, upload-time = "2026-10-10T22:49:39.251Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7d/d02c020f0daff882c9115c3c0381b8fac386570ef08c10d648a1a6fdf89e/pytest_env-1.8.1-py3-none-any.whl", hash = "sha256:753d3f0c646dd43f292a6d664099d1cafce39dda08217510ec5e9b6a681628a0", size = 10464, upload-time = "2026-10-18T06:54:40.508Z" }, +] + +[[package]] +name = "pytest-gremlins" +version = "1.12.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/54/01fee16e8de7a0534e79bfc20510f44e26581fbcd40bd099ff461ce1d374/pytest_gremlins-1.12.2.tar.gz", hash = "sha256:28ac7f9748ab95452c157c1d06b5a3958b3b7c34e642f75d53bea144b96899e6", size = 4295970, upload-time = "2026-10-19T04:17:10.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/65/915f8c0613f18aa91c0626629ed89979cf70d3f1462fc751d5fbef502919/pytest_gremlins-1.12.2-py3-none-any.whl", hash = "sha256:f0f75eeaa1713143de01ea516fff683bda10e24bebfb5d609d9910e02e3b2f48", size = 185299, upload-time = "2026-10-19T04:17:08.271Z" }, ] [[package]] @@ -3715,6 +3860,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "pyyaml-ft" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/eb/5a0d575de784f9a1f94e2b1288c6886f13f34185e13117ed530f32b6f8a8/pyyaml_ft-8.0.0.tar.gz", hash = "sha256:0c947dce03954c7b5d38869ed4878b2e6ff1d44b08a0d84dc83fdad205ae39ab", size = 141057, upload-time = "2025-06-10T15:32:15.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/ba/a067369fe61a2e57fb38732562927d5bae088c73cb9bb5438736a9555b29/pyyaml_ft-8.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8c1306282bc958bfda31237f900eb52c9bedf9b93a11f82e1aab004c9a5657a6", size = 187027, upload-time = "2025-06-10T15:31:48.722Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c5/a3d2020ce5ccfc6aede0d45bcb870298652ac0cf199f67714d250e0cdf39/pyyaml_ft-8.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30c5f1751625786c19de751e3130fc345ebcba6a86f6bddd6e1285342f4bbb69", size = 176146, upload-time = "2025-06-10T15:31:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/e3/bb/23a9739291086ca0d3189eac7cd92b4d00e9fdc77d722ab610c35f9a82ba/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fa992481155ddda2e303fcc74c79c05eddcdbc907b888d3d9ce3ff3e2adcfb0", size = 746792, upload-time = "2025-06-10T15:31:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c2/e8825f4ff725b7e560d62a3609e31d735318068e1079539ebfde397ea03e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cec6c92b4207004b62dfad1f0be321c9f04725e0f271c16247d8b39c3bf3ea42", size = 786772, upload-time = "2025-06-10T15:31:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/35/be/58a4dcae8854f2fdca9b28d9495298fd5571a50d8430b1c3033ec95d2d0e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06237267dbcab70d4c0e9436d8f719f04a51123f0ca2694c00dd4b68c338e40b", size = 778723, upload-time = "2025-06-10T15:31:56.093Z" }, + { url = "https://files.pythonhosted.org/packages/86/ed/fed0da92b5d5d7340a082e3802d84c6dc9d5fa142954404c41a544c1cb92/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a7f332bc565817644cdb38ffe4739e44c3e18c55793f75dddb87630f03fc254", size = 758478, upload-time = "2025-06-10T15:31:58.314Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/ac02afe286275980ecb2dcdc0156617389b7e0c0a3fcdedf155c67be2b80/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7d10175a746be65f6feb86224df5d6bc5c049ebf52b89a88cf1cd78af5a367a8", size = 799159, upload-time = "2025-06-10T15:31:59.675Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ac/c492a9da2e39abdff4c3094ec54acac9747743f36428281fb186a03fab76/pyyaml_ft-8.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:58e1015098cf8d8aec82f360789c16283b88ca670fe4275ef6c48c5e30b22a96", size = 158779, upload-time = "2025-06-10T15:32:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/5d/9b/41998df3298960d7c67653669f37710fa2d568a5fc933ea24a6df60acaf6/pyyaml_ft-8.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e64fa5f3e2ceb790d50602b2fd4ec37abbd760a8c778e46354df647e7c5a4ebb", size = 191331, upload-time = "2025-06-10T15:32:02.602Z" }, + { url = "https://files.pythonhosted.org/packages/0f/16/2710c252ee04cbd74d9562ebba709e5a284faeb8ada88fcda548c9191b47/pyyaml_ft-8.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8d445bf6ea16bb93c37b42fdacfb2f94c8e92a79ba9e12768c96ecde867046d1", size = 182879, upload-time = "2025-06-10T15:32:04.466Z" }, + { url = "https://files.pythonhosted.org/packages/9a/40/ae8163519d937fa7bfa457b6f78439cc6831a7c2b170e4f612f7eda71815/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c56bb46b4fda34cbb92a9446a841da3982cdde6ea13de3fbd80db7eeeab8b49", size = 811277, upload-time = "2025-06-10T15:32:06.214Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/28d82dbff7f87b96f0eeac79b7d972a96b4980c1e445eb6a857ba91eda00/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dab0abb46eb1780da486f022dce034b952c8ae40753627b27a626d803926483b", size = 831650, upload-time = "2025-06-10T15:32:08.076Z" }, + { url = "https://files.pythonhosted.org/packages/e8/df/161c4566facac7d75a9e182295c223060373d4116dead9cc53a265de60b9/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd48d639cab5ca50ad957b6dd632c7dd3ac02a1abe0e8196a3c24a52f5db3f7a", size = 815755, upload-time = "2025-06-10T15:32:09.435Z" }, + { url = "https://files.pythonhosted.org/packages/05/10/f42c48fa5153204f42eaa945e8d1fd7c10d6296841dcb2447bf7da1be5c4/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:052561b89d5b2a8e1289f326d060e794c21fa068aa11255fe71d65baf18a632e", size = 810403, upload-time = "2025-06-10T15:32:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d2/e369064aa51009eb9245399fd8ad2c562bd0bcd392a00be44b2a824ded7c/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3bb4b927929b0cb162fb1605392a321e3333e48ce616cdcfa04a839271373255", size = 835581, upload-time = "2025-06-10T15:32:12.897Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/26534bed77109632a956977f60d8519049f545abc39215d086e33a61f1f2/pyyaml_ft-8.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793", size = 171579, upload-time = "2025-06-10T15:32:14.34Z" }, +] + [[package]] name = "pyzmq" version = "27.2.0" @@ -4001,27 +4170,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e9/a7/70debb024dfacda67b8e560cc7511f52b34b9348a8cc8c1ec23036dcd51d/ruff-0.17.0.tar.gz", hash = "sha256:5cd03240d8208a557c2a9655a5cb07ebe36aa6bb35065f97d48c1f6adef5a322", size = 5320554, upload-time = "2026-10-09T19:47:29.248Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/f8/ee5ab9da6089eae2a33e6008b01deb1eda19992c1c8e10661e98cee1640f/ruff-0.17.0-py3-none-linux_armv6l.whl", hash = "sha256:0e271826af9a20d18c6cfae8c51e82959167c24859686ddd3eb9a7f0842ce81e", size = 10148907, upload-time = "2026-10-09T19:46:38.695Z" }, - { url = "https://files.pythonhosted.org/packages/9f/d9/2f81fb5a9d580afbb11b1c8ff915233a11f2a1b27405d7991f183c5e1976/ruff-0.17.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5f0ca4a40f81403689c04f12966e22f44e329ae362072d8f1587b7bda87f603b", size = 10224208, upload-time = "2026-10-09T19:46:41.711Z" }, - { url = "https://files.pythonhosted.org/packages/a7/20/643f3c8f75594f937b2bf74801241c56a2e2b8e139d24dff8b66b28cdd7f/ruff-0.17.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cbf7149e0927dc3295d5d64679a4765576eef71b00782b2ae969ef82274d6bb9", size = 9273324, upload-time = "2026-10-09T19:46:44.323Z" }, - { url = "https://files.pythonhosted.org/packages/ec/91/627700b233d367736cb274f1bd0b47d1f2b12f68878192812bd875adadc3/ruff-0.17.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ee90156522998c3037059d8f66885c8adeeaf7643bdce2caceee196ecd23e0", size = 9953190, upload-time = "2026-10-09T19:46:47.155Z" }, - { url = "https://files.pythonhosted.org/packages/cd/92/91f7b5ed39490f89d6cbf56e1f543c383667a725efa8e2c2dee0f01f5591/ruff-0.17.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d8e4a002a94cd9d0dc48b51dc69d807a172b5b9bf2b668e656424dc5b55ead1", size = 9961746, upload-time = "2026-10-09T19:46:50.098Z" }, - { url = "https://files.pythonhosted.org/packages/87/c5/7310f9fc63ce11ff6394edbd5e85433dfb0e14c9fbf6ccc97f1538491bc7/ruff-0.17.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c0b8a60c06a218c337e1161638d34757f83449243e2db161483ddf948e53ad14", size = 10625483, upload-time = "2026-10-09T19:46:53.379Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8d/97443f0dca4a03a0bc7629fd396fd494a1cb6666121e38c5075acb217d8f/ruff-0.17.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a330178bdffc4205dbf3bda11d93e059e388fd6546f8cdd304501a9160363c0d", size = 11350129, upload-time = "2026-10-09T19:46:56.486Z" }, - { url = "https://files.pythonhosted.org/packages/9c/0a/c525efd9777be4b6b012e6969a3012648468e7e6c4b3e5b46af69f46e8eb/ruff-0.17.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7bb08489e234876fa2da67ae3ea938e9a2156da80293e0e4365abd6973d98329", size = 10906223, upload-time = "2026-10-09T19:47:00.263Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3c/4a01195d93420cad1175bedad13a515dc8a56f95a6e39789b92e582682f5/ruff-0.17.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc73e7c133e82d55b5f15897b2a442d72c0cb4a0c886c46801ce3c247150b60c", size = 10386314, upload-time = "2026-10-09T19:47:03.057Z" }, - { url = "https://files.pythonhosted.org/packages/c7/72/1a3951665485a921f6375f91e754a1854d5a645d41acc3642668064ff64d/ruff-0.17.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:db4f74c533403ab70fe4007873f6ae0c9f94a8b03158cf48d78788e47cdbe399", size = 10565464, upload-time = "2026-10-09T19:47:05.831Z" }, - { url = "https://files.pythonhosted.org/packages/90/8c/539b4d8c082f57e18db8ae2be85a460d77861c79dcd5798e32b536a6a06f/ruff-0.17.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3d8cc360e666d1914e47b0777c6906d70cf18891a55532bd0a16844195d70859", size = 10079468, upload-time = "2026-10-09T19:47:08.617Z" }, - { url = "https://files.pythonhosted.org/packages/e0/b8/84286966db79434e8c26b585b0a0f6897cb3ab1c51a4aa4df10c28488b62/ruff-0.17.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d66de796b726c4801e05fa99a2a8d7a780e107be222486c304ab61765561e866", size = 9958556, upload-time = "2026-10-09T19:47:11.324Z" }, - { url = "https://files.pythonhosted.org/packages/69/50/27b6eed27b83fcdd5bfa0d52b83231e29094754374698da404d094487ae3/ruff-0.17.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c3f268baf004aea944f040623327119527ea231af15f7fb7890e82cea0679589", size = 10352173, upload-time = "2026-10-09T19:47:14.188Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fa/955399fd13044cd827862044117d784a59e3196f6cce7424908ac9a7f914/ruff-0.17.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:864b6c1acb6b0bccf94b5a3938a1531fd09aaca5e5659a2e7bf0f3cf2a685540", size = 10791761, upload-time = "2026-10-09T19:47:16.931Z" }, - { url = "https://files.pythonhosted.org/packages/ae/bf/024e01e1f6aec87768696725e648ed5b438941341eea8f8100beb681961f/ruff-0.17.0-py3-none-win32.whl", hash = "sha256:5e50aa5b84decd9fe5b0bb0e6f71c3b592f1767ed09faa4b7207d933961e35cd", size = 9904112, upload-time = "2026-10-09T19:47:19.75Z" }, - { url = "https://files.pythonhosted.org/packages/cc/77/1ee73df41dcc8d1cdb686ee4bc46ea29704ea175feb6b95c78420f631ab8/ruff-0.17.0-py3-none-win_amd64.whl", hash = "sha256:8ab76bcda86dfd28e13776cb5de3c7bcdcf1ae3d37ed761113d1a5a415dc134c", size = 10115585, upload-time = "2026-10-09T19:47:22.698Z" }, - { url = "https://files.pythonhosted.org/packages/fd/71/eb4f0ccc844aece56963e8578df9c95d4c00f580547d52329d4035d3af18/ruff-0.17.0-py3-none-win_arm64.whl", hash = "sha256:c154c73ff43f9854395e24cac507af13078962e53d2b511605058d22af1fdb88", size = 9872035, upload-time = "2026-10-09T19:47:26.306Z" }, +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, ] [[package]] @@ -4040,14 +4209,14 @@ wheels = [ [[package]] name = "s3transfer" -version = "0.19.2" +version = "0.19.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, ] [[package]] @@ -4059,13 +4228,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/02/bce7a1d70e9d0f6ea8623c16386138c251b5855af6cd3a11433d3fba4e66/semver-3.1.0-py3-none-any.whl", hash = "sha256:6e4998f8b4762acdfea85f9bfd5b4032e0bb50038dc04bf82011b42d5d1145c5", size = 18783, upload-time = "2026-09-12T18:38:12.457Z" }, ] +[[package]] +name = "setproctitle" +version = "1.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/b0/6b8a516c5a9e9630bd5293db78314ac012f690305fe93beadea388626efb/setproctitle-1.3.8.tar.gz", hash = "sha256:cafe209d064a6efb88cb45a03e97981ff8832802b2b5d009dde0197a3b7b41c8", size = 27214, upload-time = "2026-10-01T21:09:27.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/46/125791eb19c85340c6d43de8c821b143aecbfa0882343f244f5744474229/setproctitle-1.3.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a4a782f9ab26e662a890c1a7d865c452c83ed554de44aa83da35fa5bdcc79921", size = 18100, upload-time = "2026-10-01T20:59:33.41Z" }, + { url = "https://files.pythonhosted.org/packages/47/51/73b1e62c3e019e683e58d31f9f8af57e6750f36ce079004d29b7790e22ce/setproctitle-1.3.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63adfa4be8f9dbac4be6b359e2a8eb15a733545ef613d02a12ae27a4d40e975", size = 13116, upload-time = "2026-10-01T20:59:35.159Z" }, + { url = "https://files.pythonhosted.org/packages/78/ae/095c6ad159e3a7b95bc6d9a3eae18abfea1ebc79ed3377617406a1668150/setproctitle-1.3.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d4289e209e805d7d276a534f062092974fbe6814889d4c132c247205ebb49a65", size = 33275, upload-time = "2026-10-01T20:59:37.235Z" }, + { url = "https://files.pythonhosted.org/packages/f7/6e/5196258efd941cb874b3f4da91460601b867fdc434eba48a70455736d4be/setproctitle-1.3.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bb543b4b58ad839f7f0d335a7f4b40cef6989952e7239de3532de0b7771607e", size = 34124, upload-time = "2026-10-01T20:59:39.529Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/e9754e0916e6dda232a8c337bda69d2fea33acee0c5d3bdf580b8155a50d/setproctitle-1.3.8-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b705f01a531dba8fbbab90e30d432af4cc70a79c0a33b238e2c061816140cad0", size = 36006, upload-time = "2026-10-01T20:59:41.501Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a9/c63d2d745f18423937429a8611743e7b3e17259173150fbcdaa59daa4a51/setproctitle-1.3.8-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8e286e4c4e90063072a6e088a8d6526a7ffceb47429db88ba50ba58eeebcc758", size = 32268, upload-time = "2026-10-01T20:59:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/36/0c/a72818902f32ea9b848db5ccc828eb9cd9bc8661975610e1d46ea4aacbe2/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7b77c8671974288cf192e481ed63e928f1086762e97e8845d09d65ba121f8aaa", size = 33179, upload-time = "2026-10-01T20:59:45.809Z" }, + { url = "https://files.pythonhosted.org/packages/bb/55/fa59463ae6b957048e0e842d4b5513eced1b71554d8af0f5d91cd23cde23/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0e85b55eac9abfa3166ab00dac21e718ac2624a1df7d0b75559a2d3e3781cb33", size = 34472, upload-time = "2026-10-01T20:59:47.858Z" }, + { url = "https://files.pythonhosted.org/packages/b7/77/1f0cf55055788bf7eb9a146040a64d7ec42cd56153894856effc70e943ec/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f215ec9237d5f8e96d0205be6e2141292c20b133343c2153b6d4934f523ef422", size = 31428, upload-time = "2026-10-01T20:59:49.753Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/d1d8a6cd620d72a04f5d3335125e5b2fc5667bd06a76bb5bbc50b101e804/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0e70a5d294e4c8cf7dc94e015fca6d4913779a424937e66b429fba82d8eb5e2", size = 32776, upload-time = "2026-10-01T20:59:51.763Z" }, + { url = "https://files.pythonhosted.org/packages/4b/34/8b9eea1e80ac7a7f622eead8a0e4d73a60098344ed6f373e111f7ce8d29d/setproctitle-1.3.8-cp313-cp313-win32.whl", hash = "sha256:f85b4d87273e0ec63568433bbb8ccf160b548c49b53576fe010bad9f698b8586", size = 12105, upload-time = "2026-10-01T20:59:53.444Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f0/98883c658859d331faf91eccac8946cb2f8feb2590905015b129fdeb09cd/setproctitle-1.3.8-cp313-cp313-win_amd64.whl", hash = "sha256:bcdf74fe8a6e3b31bed1bc7d3e2a38cfbda3263d2750492ea0725dbc2db5aacf", size = 12933, upload-time = "2026-10-01T20:59:55.045Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1b/15043f8886aecfb31759c4a7c2cda9f85971242a10723049fc0fea34629f/setproctitle-1.3.8-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f5f1d7e0d57cd75febd70c6fc26c2ea06024ab2006011abb3094b9fea54801ee", size = 18104, upload-time = "2026-10-01T20:59:57.014Z" }, + { url = "https://files.pythonhosted.org/packages/0d/be/c2a6880394bef30aafe274d00ca0c7dac34fff1fb867f6c51abf876a640f/setproctitle-1.3.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b8954fc72fdd49588c8b8d615eabc27dfd8eb6d0cffe987fc5df02934816fb19", size = 13124, upload-time = "2026-10-01T20:59:58.611Z" }, + { url = "https://files.pythonhosted.org/packages/a7/96/6b3818e7f63a30f13bbe97b1ea0a51509a20b5b2d48c95e526c3054343d5/setproctitle-1.3.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4d6a0245679b74e0fda974edb672cd1f2351b0d762279b6776ca3a065c9322a4", size = 33180, upload-time = "2026-10-01T21:04:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/4f/43/fd3220b4bb2169d8723b3561ae7f8e1ee6f2e13e2da38b3a7a150004d2aa/setproctitle-1.3.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:394aca04e9f11c469f4f29d25a40a79eb4b8c9a72f438ba6390b776590f89757", size = 34071, upload-time = "2026-10-01T21:07:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/58/ee9eeebeb74125723b60eada6e6b2def747815228193327ce470da088f2b/setproctitle-1.3.8-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f92f6ded50c0ae7b0fed612be77c5345ed4a4578f99b98c7f8258d0e599eff16", size = 35923, upload-time = "2026-10-01T21:07:19.054Z" }, + { url = "https://files.pythonhosted.org/packages/f6/fe/b339602e96d73b29fd123c8b9c886e8f9a5afa12a18218eece56bd99c668/setproctitle-1.3.8-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c09f15930910dd895ac113b16c0b3ac2051dcbd44b0f017f94cb51349ba11583", size = 32116, upload-time = "2026-10-01T21:07:25.121Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5e/8d46d34648d8dfdc3f3e941c149ae316b6c6f417391018e279a1bb733c42/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4756a96435917494ec2f27f0da46bc05eecaa1a7e072d57220ecb2d46ea8a3d3", size = 32980, upload-time = "2026-10-01T21:07:27.826Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6a/10720d27d5f4a47c9af51652b49dd22313be20b9d542e72f80b4b363702b/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:65279b2df2951b64878da5382363e730d242486179c440c8653fe39aa3cc9e62", size = 34307, upload-time = "2026-10-01T21:07:30.62Z" }, + { url = "https://files.pythonhosted.org/packages/6e/65/95dd7b0245fdb7d2f6633613c559ecd9f0b2afeb6e3d962c218b6a1722d4/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3d2ced815b940c88097e07182c57cbfe1a0156839d1edc9be2bc0e4c74ab4eed", size = 31126, upload-time = "2026-10-01T21:07:33.59Z" }, + { url = "https://files.pythonhosted.org/packages/fd/96/4d62c4ba70cf8fa305f1dcbe35e93449819bb27b451fcb473556acf92650/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:37a91697c3b96bac58b6936b3bf0366af96a2a6cef963fdd9aaf0593a4bd1ff2", size = 32468, upload-time = "2026-10-01T21:07:36.312Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4a/1ec517435d4b572f6c468795331c7b376ec310eca5b1321504c6647b85a9/setproctitle-1.3.8-cp314-cp314-win32.whl", hash = "sha256:7cdd3d6e5e8abcee233157615d88dbb80527a5a4eccb95a93fbf92e6d2c8f9dd", size = 12255, upload-time = "2026-10-01T21:08:12.203Z" }, + { url = "https://files.pythonhosted.org/packages/7c/13/e74a4fa864a0a694104ff74fcb3bef523be94321bac1a8ece64ad04ef233/setproctitle-1.3.8-cp314-cp314-win_amd64.whl", hash = "sha256:5cff3bdbe6be556cc19bd072ff1ed9b5a8f4cbf356cd19b1ca94506b3e2b29f6", size = 13159, upload-time = "2026-10-01T21:08:14.231Z" }, + { url = "https://files.pythonhosted.org/packages/38/d8/019116e7defcd5d2c9264da4a82d369de6806a9161adf73e5f73e63fb715/setproctitle-1.3.8-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:84320e61f3906cc550b900ab1ba02ec53edd2d2e1e917f0873cba16592801c89", size = 18105, upload-time = "2026-10-01T21:07:38.655Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f5/bfa9d6b5574903c17e02caba0e25815bb1c7e7b895d594475676baaa35b4/setproctitle-1.3.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8086259943fc75eb07e11c1d08600709b9ce4fb2ec9bbd2490556c47d3ead135", size = 13125, upload-time = "2026-10-01T21:07:40.774Z" }, + { url = "https://files.pythonhosted.org/packages/d3/29/c7febeedee66b929c335d7b8ce83b1c4baccc65c94a4112fd8c1ea32b57d/setproctitle-1.3.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8879d303d4e3559efdba033ff198c6869010f7958412ec2047674881859fdd7f", size = 34818, upload-time = "2026-10-01T21:07:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/36/b9/90fea18bcb74f19d411df7e9e2d981c4d3e99817870acb0b5cac6540629f/setproctitle-1.3.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7de4da27fa4481e8a7a4ef58c908ffd2b996847c3d7fbf27b5c53ce0984c408", size = 35622, upload-time = "2026-10-01T21:07:51.03Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e9/5899eab6e0b4a127e98a268a157b6bdba4f8ad18d0ed90fdf3e081c20a3b/setproctitle-1.3.8-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:741b1ae7e1cd68745b8b677daba73685dab2d1b8bb12500e6537f3482a91150f", size = 37762, upload-time = "2026-10-01T21:07:53.758Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cd/190748a1f68afe6154b62517d2288b7d4c2092b0efa6e61d044c0f99d17b/setproctitle-1.3.8-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c85ff5b000eef8a280333ca158483a76f86bd374816add5c016a9f4294471289", size = 33548, upload-time = "2026-10-01T21:07:56.148Z" }, + { url = "https://files.pythonhosted.org/packages/26/db/889062b0970f04bc76d8eed03165dc17de8df282399ad931cb10a23b33a3/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:656ffe282ec0ca18308eebb100898c79b4f5698a93228d50d2f2a4a09474d7e0", size = 34298, upload-time = "2026-10-01T21:07:58.898Z" }, + { url = "https://files.pythonhosted.org/packages/6d/bc/adedb2c18eb7bb152b6bc59187f46fc328e7bb729bf28a7e4e43521c4c7a/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ae87e7b3e6414b3dd887adb72d2c4ea9f210c5f48efafd295ade4a58c760d29d", size = 35897, upload-time = "2026-10-01T21:08:01.561Z" }, + { url = "https://files.pythonhosted.org/packages/38/8e/f1769dcd1eb668b01ad251eb6780b030752d18dabc1b82789b255189772e/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:09cb645c6ae440286391e256203fc03522b3b0f55fda8219f698abb5c447cce7", size = 32435, upload-time = "2026-10-01T21:08:04.112Z" }, + { url = "https://files.pythonhosted.org/packages/79/e5/68be1eed21bba3ce27c6907ddcf0559cb99e4de3aa3ee000a38994371b73/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:84003eeaf2d7460390bbff1dbd7bee5b1213e7ae47a489f0fb7aad6acd2b045c", size = 34024, upload-time = "2026-10-01T21:08:06.514Z" }, + { url = "https://files.pythonhosted.org/packages/40/7e/f275783d6915fac997dbfa3729330552c4f2ddcd4f1bba7ff522f5f05c28/setproctitle-1.3.8-cp314-cp314t-win32.whl", hash = "sha256:ac4e037d276ce27069cad0083c4611b45232278bd2fbac33f06ef9438c171a1e", size = 12259, upload-time = "2026-10-01T21:08:08.432Z" }, + { url = "https://files.pythonhosted.org/packages/86/59/3347f13aa38c81d78f5c93bc55e4355c43550f47348e5de48c5080f3a1b2/setproctitle-1.3.8-cp314-cp314t-win_amd64.whl", hash = "sha256:c284642b229ba7b7b927fb044aba03694b880ab572db3f07a10adc786db1ff7b", size = 13163, upload-time = "2026-10-01T21:08:10.087Z" }, + { url = "https://files.pythonhosted.org/packages/92/e4/e9866831fd0cff677c67488059fab3786a843ff0bd86f4b3c44bea8dd763/setproctitle-1.3.8-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:63348d86ae60b071a898e4241b41da1346d8c36f1a60bf3b698ae5eb35b79865", size = 18107, upload-time = "2026-10-01T21:08:16.029Z" }, + { url = "https://files.pythonhosted.org/packages/ff/79/5c11a8c8cf3b91e54fdf4c93d1c79d99a491e04d77d34c80a90552fe6b46/setproctitle-1.3.8-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:59239b6e5007b1bec79ef444a390b3bcd59cbcb2686c9910da5a42f1e99f738c", size = 13132, upload-time = "2026-10-01T21:08:17.756Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/47dea0a33ec67a2813e40bd92deea74c3732fd598bd71b636a92897eaf17/setproctitle-1.3.8-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:38310d9cf94959e39d22fdc0154c66c8bdbadfa790c524b04cb36da6333c7507", size = 33509, upload-time = "2026-10-01T21:08:20.254Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7c/5d6945012077e6c393d59432df42d281d213f508bcf6d83a1287ab2eb9ae/setproctitle-1.3.8-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a279b136ed97407209d2dc9c30d1e65405d1a4a06ab162cedd9c4159b51f8f20", size = 34405, upload-time = "2026-10-01T21:08:22.796Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d9/543a4470bfd01529dc35f0014dc68c70665db8b790e3647b41d155203b87/setproctitle-1.3.8-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26997a85aa70bb088e86f0b7899fb0c909be824d22fc025e87b0005afbe59261", size = 36167, upload-time = "2026-10-01T21:08:25.35Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/fc55041b162833c4bbbbfe78ed417bcab7b22f9fa72aff1722ff6af46fae/setproctitle-1.3.8-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35fc5e10fcba6df25d676ef908b44a36c17e7f167738ed016561185e24f4d274", size = 32623, upload-time = "2026-10-01T21:08:27.778Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e2/8c73c990c338b9ca5bf4fc6167ac864a80a53a10e71fd2755ba146f1fce6/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:f5df0503093ec8c3c7e1da53920e79c62ab653b51b136c4736caa252552a8732", size = 33510, upload-time = "2026-10-01T21:08:30.431Z" }, + { url = "https://files.pythonhosted.org/packages/cd/61/8e10270d3a7326dc4e8a1408c631e6ae912625b91f38adaf04794a19e07b/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0a582308ba43afa7233d28243d9d1212e11c81d3c187a43929ec0cb70c4781c5", size = 34528, upload-time = "2026-10-01T21:08:37.161Z" }, + { url = "https://files.pythonhosted.org/packages/77/d4/00773ca8c876123348e65edfd9eec47464bbfa1a31067c6e1cd382356a0f/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ce48dbf74e8ec618d74cbaffe1159a4c32a4428de598f0ccdb0782bae618a72a", size = 31681, upload-time = "2026-10-01T21:08:39.874Z" }, + { url = "https://files.pythonhosted.org/packages/d5/42/19c6b634cca492575f9d99868bc35134528fb390d683d6a0396f61df0b8d/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:dd3a8fcee7d8a23e44781144830dee97b53ca4e0380bdd2a0e05b13913ce5725", size = 32999, upload-time = "2026-10-01T21:08:42.36Z" }, + { url = "https://files.pythonhosted.org/packages/94/34/09f653f687e4a1a6f515642519cc65f0e6bcf34631386380924fc9785b44/setproctitle-1.3.8-cp315-cp315-win32.whl", hash = "sha256:50beee618377ce6fd35767a9327f2da594b36778eb10b503ee2dc9b7c801f433", size = 12251, upload-time = "2026-10-01T21:09:17.504Z" }, + { url = "https://files.pythonhosted.org/packages/89/69/ab9afe8c198594533ad5eed6dcea647e1743ac8196a6c6dea974724b72e6/setproctitle-1.3.8-cp315-cp315-win_amd64.whl", hash = "sha256:f5916010de8b6045d7c63d98bb3f7a1e13f8c9c1046e859f2fd8de79c4e2d865", size = 13161, upload-time = "2026-10-01T21:09:19.295Z" }, + { url = "https://files.pythonhosted.org/packages/78/90/da93e1ae012f2515473a45fb33f5ba2ddbbfcce1381c5e41005b31ab3230/setproctitle-1.3.8-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:e9f643bd97374aeb7e4274e5c361277bf1428e8e55246cee6823fa3525dc2c04", size = 18119, upload-time = "2026-10-01T21:08:44.296Z" }, + { url = "https://files.pythonhosted.org/packages/af/35/41b28fa8eb8f1bee9c5516b99262a5df413937bf186e9d50ec3ef812fc61/setproctitle-1.3.8-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:fbd9ceb8c091c125e5892757e4d16af06674dc96edd7b5bf1e627b1ae7650c66", size = 13140, upload-time = "2026-10-01T21:08:46.797Z" }, + { url = "https://files.pythonhosted.org/packages/0c/19/089c878e6b8a3a391ceb03e1653ad6f33f27a5f7e204cf4a54b6a053b83d/setproctitle-1.3.8-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:322b4643f98c824b17dbf2ddb07ea7a37def8cc9e18adee320befce4bdffa271", size = 35122, upload-time = "2026-10-01T21:08:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/88/a7/f419b87fcb6202ef0a94f7cd6bdecb23c2d01f2f07b38f92d3f0c5dfb518/setproctitle-1.3.8-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e36ee7300584a53fd7ac5f3853323411bde1cf6c081921bd3754a73b4071245", size = 35944, upload-time = "2026-10-01T21:08:52.612Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/fb181112f3c09a8c8962bd110f538c8a9d287a0c6a8bfd5b4dc39d54e0df/setproctitle-1.3.8-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c05aff52e6eb9c4d2ea86cf0d0a60b1487ed570263a1b13bb30d9f96aa8ba97", size = 37981, upload-time = "2026-10-01T21:08:55.136Z" }, + { url = "https://files.pythonhosted.org/packages/d8/8f/934846cf63b5ae2e9a88dcac6ddc3ec813e6b9e0919994684f7674155b8a/setproctitle-1.3.8-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df807e7f036c19235320b6b5137291891d660f10ccb0b50dc46bc8385be18379", size = 34229, upload-time = "2026-10-01T21:08:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/c9/04/c11f73d8aa0ebe5d63866130ad14ca3789a4dfb9fc617c3a612af9c177f9/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:bef8c5f8820c70e8761171409052056ffd247023b001a6254287a0694c3d8b1c", size = 34743, upload-time = "2026-10-01T21:09:04.088Z" }, + { url = "https://files.pythonhosted.org/packages/38/50/8c780488db26e8f51af91465bda6ee224522248dc5220b282791dacbbe5e/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:7855290194637a3c5a2e37c423efbf5813fde97a6b63ab99feade5e05d73dc6c", size = 36094, upload-time = "2026-10-01T21:09:06.67Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e6/13d541653607b3b7232dc9f8baae05b24112ac9a3756c3653903c624c4a9/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:8c0a7753f5ddaf0ecd1fc8df3e2106611d4290f0508e643e723cb5000b7d6c2f", size = 33148, upload-time = "2026-10-01T21:09:09.064Z" }, + { url = "https://files.pythonhosted.org/packages/42/94/f3e16c3a7949fa28a777aa30830bf89098026e13669144b0ec1b147f17f4/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:97b35b15f81363a314baca76674c19027dce19eed1590abd51c2d285830efd50", size = 34488, upload-time = "2026-10-01T21:09:11.756Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e7/9baa826fc44c6f607a39ad35d419da7f51dec2df31ef169e7a4a34e1cc75/setproctitle-1.3.8-cp315-cp315t-win32.whl", hash = "sha256:ecf44731e351c6100885120cecf53a0f43a1bb975a92480de1f9e022ab542bc1", size = 12260, upload-time = "2026-10-01T21:09:13.6Z" }, + { url = "https://files.pythonhosted.org/packages/71/fb/08f55fab7294d18a6b1bc6d13340706c998b4641bd5f99ed9c7a626fcbf0/setproctitle-1.3.8-cp315-cp315t-win_amd64.whl", hash = "sha256:61aac898128dade36438da9387fea570f00cd519ad5231d3b268913afc2fc329", size = 13170, upload-time = "2026-10-01T21:09:15.351Z" }, +] + [[package]] name = "setuptools" -version = "84.0.0" +version = "84.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/e2c41d00a4bfea5548c31420a74c6cf149cc0e7c36b45c4c82202ce867aa/setuptools-84.2.0.tar.gz", hash = "sha256:6f445ca2ce864605f81e223f262b4c94ffcb9a4a63473bae0ed7be5aab0291da", size = 1227656, upload-time = "2026-10-19T20:38:03.407Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a2/2e1af99d56c8145aa69496026c7e5fd7b6ec759b500c3ad697e2062832dc/setuptools-84.2.0-py3-none-any.whl", hash = "sha256:8cc107ac7a403105dc8d6477664171f36135fab26adf8b5e0edaebc313f39b21", size = 875816, upload-time = "2026-10-19T20:38:01.629Z" }, ] [[package]] @@ -4186,14 +4423,14 @@ wheels = [ [[package]] name = "smart-open" -version = "8.0.3" +version = "8.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/5f/4c6d2be8dc4737d209f583a9c61e40c7b90d86313308e1d82e79a1b857b4/smart_open-8.0.3.tar.gz", hash = "sha256:398eb1d6af1e887ccd30347af540793235369bd6602dc42f72c44f939ae9c80f", size = 61941, upload-time = "2026-10-12T20:21:50.154Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/e3/1fb5c310b1f55d99ba51cb81b8f1a0e4b5f89be44de69405aeeb12814778/smart_open-8.0.4.tar.gz", hash = "sha256:8cff8321f0b0ae4bd9cb0cd0c507c135ad6aa980171248fca520b645fd6ff717", size = 62316, upload-time = "2026-10-17T19:34:45.106Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/79/60e14c5a19f39ebcf4fecd8ae8b0b01418ff5839bd7e66cf02b019d31826/smart_open-8.0.3-py3-none-any.whl", hash = "sha256:c3f67795b1a2bdb62dd9aca9ba4d4018e8f11e4188700b3246ea5ae049f65887", size = 73451, upload-time = "2026-10-12T20:21:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6d/3a95e71d597ee03b3ddcf0ff154f2841b84d8724df2bb35a350493793ae9/smart_open-8.0.4-py3-none-any.whl", hash = "sha256:8a3a67679942fa40d1a93ed3aea37619b4b4ec5458f1d5a9228dfc301db11b4c", size = 73881, upload-time = "2026-10-17T19:34:43.624Z" }, ] [[package]] @@ -4251,7 +4488,7 @@ wheels = [ [[package]] name = "sphinx" -version = "9.1.0" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alabaster" }, @@ -4272,9 +4509,9 @@ dependencies = [ { name = "sphinxcontrib-qthelp" }, { name = "sphinxcontrib-serializinghtml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/73/567727fc8c8d85bc3adc88a246e12937cfda8d565cf1d80e85bceaa61fe5/sphinx-9.1.1.tar.gz", hash = "sha256:48b627f02908f22c601b3ba9c7b17265b4f315dd05c8bc64cd4da41183f44fbc", size = 8725638, upload-time = "2026-10-20T00:41:13.96Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, + { url = "https://files.pythonhosted.org/packages/47/18/3c66ea76da7de47e716b181c35bdbfafbd5e9af0bb247b14578e9de348e9/sphinx-9.1.1-py3-none-any.whl", hash = "sha256:f8b60013ce25017be41a509e5c2d265bc1918410061315c36b809c7f335646c0", size = 3923559, upload-time = "2026-10-20T00:41:11.288Z" }, ] [[package]] @@ -4347,73 +4584,73 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "2.1.4" +version = "2.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/44/311bac6b6ef81e4dfd0287d04900108b1f5c00c9761dd3c0a2b7b9d0f86b/sqlalchemy-2.1.4.tar.gz", hash = "sha256:7bd7ad604487daa7eab8716471c29a7185f17b5287ce73bb7bc79fea050d8cfd", size = 10544216, upload-time = "2026-10-07T17:33:59.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/e4/23174288ed2c03d6dbd5dfacd69e28303ee95f49642a8ed0544932999fb6/sqlalchemy-2.1.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:70006e9e6157200b795beeee04bd5cb15bccb40a14de595eb9f5dcf5945ed244", size = 2460507, upload-time = "2026-10-07T18:04:40.044Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ac/254fadc98bfd600445b976e81c6d777b08a728a415c3b77a8c8d35b89a83/sqlalchemy-2.1.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3341ddc430733cd961bc064889f42712a0b4056733a21c83176842aad67d12a6", size = 4594505, upload-time = "2026-10-07T18:16:58.768Z" }, - { url = "https://files.pythonhosted.org/packages/83/6f/ac7beddc57c9c87bd77bc1c158fcbcdc20822f1873bf33ea3480d04e865f/sqlalchemy-2.1.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98f7a4bfeaed3722804f737ae2bd4077b35e57d6f4531fe612bac8160cda5acd", size = 4647364, upload-time = "2026-10-07T18:34:51.721Z" }, - { url = "https://files.pythonhosted.org/packages/0a/82/fc3891f261c4738a8b90cfdd805fe292d1af3b77f680a63b7349304c74e5/sqlalchemy-2.1.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ec5d079935f67febe0ab8a3a203ad591b99508adc34ae0027f696dcb20373537", size = 4311619, upload-time = "2026-10-07T18:38:44.002Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1a/160c1320ab20e764a29721dc3fe7c31af34e291c652dca875d1ca6022b9a/sqlalchemy-2.1.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3d675b0856b6703b29d023517a4c19fecfbb55214ff5c72cd813527e40aed9b4", size = 4518031, upload-time = "2026-10-07T18:17:05.615Z" }, - { url = "https://files.pythonhosted.org/packages/30/2c/15a204333896e5dc63cb089ea20ca3ebc3c892bedf9fa00cc1a65e20d7b5/sqlalchemy-2.1.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a0bb9ee6a38cb36240dc88da11888348f61506047be54de3f09496c3b0ead6f5", size = 4312818, upload-time = "2026-10-07T18:38:46.541Z" }, - { url = "https://files.pythonhosted.org/packages/a6/55/5e78d288f198598f278b4b7baef42f18e039b14b1e1045e9df3cf571300d/sqlalchemy-2.1.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:61a2c48771cf314b6613d327c795902bbc0eb6d6169deb23b35004ba6ad6cc0d", size = 4608584, upload-time = "2026-10-07T18:34:53.69Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f6/e83b93ecc6e6528623fd7aa2af27ff0660d22354b78fe6ccad03f9ecbd9f/sqlalchemy-2.1.4-cp313-cp313-win32.whl", hash = "sha256:3fd608a06bafa768ad5711df4e17eb058bdc490e9df7d39b12a90947471e8712", size = 2373880, upload-time = "2026-10-07T18:22:11.722Z" }, - { url = "https://files.pythonhosted.org/packages/8f/46/afb02975023db6aa4b8608177c2fae17d0b435d9cbfcb5df4fa6e65a8078/sqlalchemy-2.1.4-cp313-cp313-win_amd64.whl", hash = "sha256:b756d74527c56a7e4cfae297f7930c1d75bdf4b23f214c8c13779746d28060cb", size = 2424430, upload-time = "2026-10-07T18:22:23.688Z" }, - { url = "https://files.pythonhosted.org/packages/21/e5/76dc82d59186b98b27589b33b01175c0d49512679276170271d9384418e2/sqlalchemy-2.1.4-cp313-cp313-win_arm64.whl", hash = "sha256:a64d54015233f824f171009977bfbb6b08bd0347b700cf17cb047ffb94c4148f", size = 2385057, upload-time = "2026-10-07T18:11:48.248Z" }, - { url = "https://files.pythonhosted.org/packages/43/b0/6675a01f4e6215e0a809d28a800953294ab31370fe8c4bb3eb9e28c0b5a6/sqlalchemy-2.1.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a2f6164c0527cd8fc4cea79a5c9d8369ffee417b8ba444a42342f36b91deb75", size = 2463645, upload-time = "2026-10-07T18:04:41.615Z" }, - { url = "https://files.pythonhosted.org/packages/7e/24/4630a4009ea08a0769d5ff6517c7fc978f6a63eba32e08c44b98c284d7e4/sqlalchemy-2.1.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6929a11ad26a91a4efd891c1252b373c2e88f056910b83ec6030ed3f2cbcb734", size = 4587911, upload-time = "2026-10-07T18:17:12.512Z" }, - { url = "https://files.pythonhosted.org/packages/0e/02/953686f44448b92cc628245687a242799b6eb11ef30ad2bc7adacd51986d/sqlalchemy-2.1.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14528d37d7d46a92f2a483f188f7fecd86cdd789254a0412b960c9fc5e9efd6d", size = 4617539, upload-time = "2026-10-07T18:34:55.826Z" }, - { url = "https://files.pythonhosted.org/packages/13/23/a44288ab4fa12e51c9d390e7d798d70a45669ddcbddc9dd9b5948eb1aa3f/sqlalchemy-2.1.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d2cb669c6bd1f19caf51db6e3c4fdd4cbb76f9db3ef81c3aeb5e288d9bae101b", size = 4310883, upload-time = "2026-10-07T18:38:50.265Z" }, - { url = "https://files.pythonhosted.org/packages/a3/39/1c441ac015767f619a9e6cc306905bb042f94b84f2a1e930e989e9c6e209/sqlalchemy-2.1.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:63dc25b21fd9a41dc09b7aada4b3b0d97cf4b6414f74bced6ac45326bc799ac9", size = 4516511, upload-time = "2026-10-07T18:17:14.368Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b9/f54ea5ccb27d9a712d90d1617050bee761df25dc1fb5e0b7d2aa867deb51/sqlalchemy-2.1.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:308f96d24e773d64609a2a0d1161a068f9f6e9165523bc4e07aa9c45f0c4213f", size = 4312314, upload-time = "2026-10-07T18:38:53.249Z" }, - { url = "https://files.pythonhosted.org/packages/df/9a/c1e39287ee988e4c2e25c619959b8fb15b297734be040653fe85b57517ee/sqlalchemy-2.1.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93b9416b9011a3b7689a933e04ac9f61d15686b6cb1948ebc1f41467153116c3", size = 4577833, upload-time = "2026-10-07T18:34:57.829Z" }, - { url = "https://files.pythonhosted.org/packages/41/78/5f1ae1911d2b20ccdb39ee522118533a4b5262b6e5e06bbcbb1ebd1f4617/sqlalchemy-2.1.4-cp314-cp314-win32.whl", hash = "sha256:89db94855287fdac98d74595cf13ea59fbffa608d6400ff972b0fd4c036d873f", size = 2380724, upload-time = "2026-10-07T18:22:25.374Z" }, - { url = "https://files.pythonhosted.org/packages/ca/93/4dfa4ce15d082011fb94e06e7c6b4c2957a3f0ddeb8fe9b89d007bc058d7/sqlalchemy-2.1.4-cp314-cp314-win_amd64.whl", hash = "sha256:080f8d853aac5bb5620f0ae6f46527397cf18dce0ec2b478b478469ef3cae2c4", size = 2431689, upload-time = "2026-10-07T18:22:27.144Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c4/6f6c29eaf459c4c2d9b7d24e300bab32043f8f8a936df863f3b886b5564a/sqlalchemy-2.1.4-cp314-cp314-win_arm64.whl", hash = "sha256:64d41be1dd88f184de1931f0173f4827122a1b49fd1150656641200c0bdf640c", size = 2396106, upload-time = "2026-10-07T18:11:49.528Z" }, - { url = "https://files.pythonhosted.org/packages/a5/e9/48f851411665e394f60c669d1f9494d660f5f1fe46e275f9615cfc812a98/sqlalchemy-2.1.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:84272f329c15081a1e09b4a7261118b4e8a547f43e00fca98e55bbdf19eff3be", size = 2503038, upload-time = "2026-10-07T18:19:41.094Z" }, - { url = "https://files.pythonhosted.org/packages/41/ed/bf83068bda4051d7fd719c14cefc15d8466ef1e3656b9f4401b0509b11e0/sqlalchemy-2.1.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b3f58bd26fc010ea28976d401845e4e6ce02e1b7c0288b3ea9c9a3c396f0bcc", size = 4919623, upload-time = "2026-10-07T18:16:45.399Z" }, - { url = "https://files.pythonhosted.org/packages/56/de/57eb70d56b70d22a9360d658b195834ecfdeff7a7bc5c2e3a7fa7a8f7823/sqlalchemy-2.1.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82d728075d42bd457d09655cf22e99d772a648c6f67e86743a4f05b7d063ca18", size = 4830238, upload-time = "2026-10-07T18:37:04.468Z" }, - { url = "https://files.pythonhosted.org/packages/70/3d/c410e9e79a53fff4c04444da609fed6404868d250f11fe8bc53d827bfb0e/sqlalchemy-2.1.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0970394ec5d9e397aafc5bc5fa2b7f8b58cb191f2703006b19a96ef4bf00b8d9", size = 4508380, upload-time = "2026-10-07T18:38:44.277Z" }, - { url = "https://files.pythonhosted.org/packages/1f/c3/01b93821ba35b5b162e79c613279d960a120767694f656da1c1374dd3ed3/sqlalchemy-2.1.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6005f2f5fcd67fdd721446128e6a2a1d18f77387a604fbd26b0006a086b33096", size = 4775311, upload-time = "2026-10-07T18:16:47.724Z" }, - { url = "https://files.pythonhosted.org/packages/c7/88/0b40754e4d851d33548792062c23467a3d8dc07f2eff90cb19e4c404fb4c/sqlalchemy-2.1.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:0e01a3e199ae219381c4889993c5584b1b905fffe6830f639adb6770036a8913", size = 4508558, upload-time = "2026-10-07T18:38:47.857Z" }, - { url = "https://files.pythonhosted.org/packages/d3/2f/3916954eca5596d9e93fccd2ec0e45fd8c65981debac0ec4617639ded6ba/sqlalchemy-2.1.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:22129e7d00ac66b291840c4dc83a9c497456ab5bffa682dcbfdc2356f9e49e5a", size = 4767114, upload-time = "2026-10-07T18:37:06.792Z" }, - { url = "https://files.pythonhosted.org/packages/6b/d6/6a29716aec6ae17cd77e27b5e0dedc68cf9068594f2b601806c1d146427a/sqlalchemy-2.1.4-cp314-cp314t-win32.whl", hash = "sha256:bc33d3e59d4e84b8866cc9ba13732585e37212dbe3542cb09f232682b36f47a5", size = 2439584, upload-time = "2026-10-07T18:22:44.434Z" }, - { url = "https://files.pythonhosted.org/packages/34/79/2f0b33647d2d26f098269096c1864c0b4e81095354cdedb95192647f47cd/sqlalchemy-2.1.4-cp314-cp314t-win_amd64.whl", hash = "sha256:346d144e8912ae087b10d3c2081657cb634728600693eee6dbb71d7eb4768101", size = 2507217, upload-time = "2026-10-07T18:22:46.176Z" }, - { url = "https://files.pythonhosted.org/packages/93/e5/869c1ac0a21e17e4617b6a7828b50320bedb7074b6d67aec59299be5cdba/sqlalchemy-2.1.4-cp314-cp314t-win_arm64.whl", hash = "sha256:3e5de57c71b3460e2ca6137e82cd3cb8c9f711f301f50d5c77156fdb9c822999", size = 2425329, upload-time = "2026-10-07T18:12:20.595Z" }, - { url = "https://files.pythonhosted.org/packages/2b/8e/a082a165b473dae45d2f2f79be15f5c405ac579830c64253efbf04695177/sqlalchemy-2.1.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:418786f05387ddb66ee683a1d016c5a8d9bf7be921e6ee8f285c7b6ac961a731", size = 2463572, upload-time = "2026-10-07T18:11:12.053Z" }, - { url = "https://files.pythonhosted.org/packages/d1/35/74db254005ecb384533973b157ba1fc3fe5bc41a5bc6e0500ab8369c49e6/sqlalchemy-2.1.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:283914efed30e4d44301e36ac90ad048570538b8a70f072fe01578d9b205d09c", size = 4591740, upload-time = "2026-10-07T18:01:00.314Z" }, - { url = "https://files.pythonhosted.org/packages/70/81/5cadd72b0c26b6ee7c1e6950cb9f0cfc383246a842314a1b2a87f455db25/sqlalchemy-2.1.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d2eacdbeb990b80235763860923c60a8393745b66f7149a734980c65896da72", size = 4627040, upload-time = "2026-10-07T18:09:24.836Z" }, - { url = "https://files.pythonhosted.org/packages/8e/78/aed93cc373f61b57625e1f9f84bbf12358e32e935e64fa098f3a446e1203/sqlalchemy-2.1.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e43fca5fdd5f34a3f8c54107a3648d3139de8bbf596a189f3f0de94bd84949bb", size = 4333103, upload-time = "2026-10-07T18:33:48.275Z" }, - { url = "https://files.pythonhosted.org/packages/e0/31/ecc6bbd365671cdc512a59d42afa7c34b2833a8d841754918ae3f62d36dd/sqlalchemy-2.1.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2e1b5343d315b10a4a71da481729f66f830a561595e02b61e8a5a65d658325ac", size = 4519962, upload-time = "2026-10-07T18:01:02.268Z" }, - { url = "https://files.pythonhosted.org/packages/58/58/9f8f6157c2252aefe73f4a0b3859413bb720d14321aa7f367c691949aaf8/sqlalchemy-2.1.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:42c37c06adcecf444e8c981f7e9237a41bdd445c83da0df9e08b4ad958becbbc", size = 4331935, upload-time = "2026-10-07T18:33:50.334Z" }, - { url = "https://files.pythonhosted.org/packages/97/de/a4ae4b95d17607004f01e9a085fb221087c557bbad77a3d87d5d0a5fd8bc/sqlalchemy-2.1.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:bab7f51d38766d6a64da2b41976f1b3f9cc2ff37d3f2f63bdbac876199f3a48e", size = 4589275, upload-time = "2026-10-07T18:09:26.872Z" }, - { url = "https://files.pythonhosted.org/packages/65/27/56f69293a01279ac0e6077b8c358eb0f1c2afc6aa17428414a86c8871042/sqlalchemy-2.1.4-cp315-cp315-win32.whl", hash = "sha256:1541ba5bf0f232cd61f9ef3df78c93977c72ba6031506a0e6d057b2a3ddb76e9", size = 2380494, upload-time = "2026-10-07T18:04:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/2c/7c/ff7e29f95996ed49b950afd531b89e7c8d15addb41735643d07090550090/sqlalchemy-2.1.4-cp315-cp315-win_amd64.whl", hash = "sha256:596a95611c217cb19c21f02f43c637cb507cab71dcf0467c5c7d98fcdd703007", size = 2431760, upload-time = "2026-10-07T18:04:27.275Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/4eaa4978760cd632093ea272e7c4f88223619202f5481f897e67d4377409/sqlalchemy-2.1.4-cp315-cp315-win_arm64.whl", hash = "sha256:0d1ca95e42ce3c18818f170b741d30a33b292c6f6b9a202ffd717e28fc99b8c7", size = 2395807, upload-time = "2026-10-07T18:30:54.962Z" }, - { url = "https://files.pythonhosted.org/packages/be/7b/b806fbfc61ade37c4f3aecec0874c345fb297b56a3743116dcefa3e4700d/sqlalchemy-2.1.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0f672ed6972164fec94a8f0b21dcf8545080d0727866335fb8adf9f4764ce6ec", size = 2499195, upload-time = "2026-10-07T18:19:42.835Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ba/4f9fba8340222f09287e936d7b76e6911a4e507c7d6373ada770e8f697d5/sqlalchemy-2.1.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72e3fa41d1fdab87d4e88bbdd69c9522e2795549fbe7b07bcf4ae9ec175f4b11", size = 4896914, upload-time = "2026-10-07T18:16:53.18Z" }, - { url = "https://files.pythonhosted.org/packages/55/34/c4aeec7bee453badd8b0e02c2021a13bd70ef01038303d05326e99f595b6/sqlalchemy-2.1.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cb2cb98d056e63e353ed697750004e07c79b054d73059ba3184ca3bb07296bea", size = 4846593, upload-time = "2026-10-07T18:37:08.766Z" }, - { url = "https://files.pythonhosted.org/packages/82/54/6dd8504364e5f5efd328e98fea963e5a2e978ff8dcba70d95231314f82a9/sqlalchemy-2.1.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d66fdcc5506e0f8bb8d3f4f95125220a7cd6c46e8b1762750f01e9639973dd8", size = 4494821, upload-time = "2026-10-07T18:38:51.166Z" }, - { url = "https://files.pythonhosted.org/packages/df/42/dc584c098bce29578fd0611cd6f36830e06b4dd2505d3020a0b592f4cf08/sqlalchemy-2.1.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:81f802c96dbf96e59c6982fa1b87da7868920fb0c27b9b81e560a62f57c2ccfb", size = 4752545, upload-time = "2026-10-07T18:16:55.711Z" }, - { url = "https://files.pythonhosted.org/packages/8c/41/69a70c1419bea97e80f65ce09f4f626df464752b276f4f3d69ff6fbf2325/sqlalchemy-2.1.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:acf8982c70471a68aa90d1aba08b48860c55b3357ec84ccb0f09368ead2ce099", size = 4494093, upload-time = "2026-10-07T18:38:54.37Z" }, - { url = "https://files.pythonhosted.org/packages/ef/bd/d296c2223e8417b350db215d94dcd344bc0dfe9deb7d810a21f7d8cd0b14/sqlalchemy-2.1.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:778094c83e36c430756a7e1a1ac66fc3cffb2c6a1067958fe6b920abcec7bc5a", size = 4771973, upload-time = "2026-10-07T18:37:10.93Z" }, - { url = "https://files.pythonhosted.org/packages/13/4c/c3a10d9da10e4e60808ffd1825547b383c0d7ca9e56d15cdae47c04e752e/sqlalchemy-2.1.4-cp315-cp315t-win32.whl", hash = "sha256:963348422b22f760e9462e56bc32bf4d95d224cc5b8c79a3c6e3b786d3d2a2b2", size = 2436383, upload-time = "2026-10-07T18:22:48.162Z" }, - { url = "https://files.pythonhosted.org/packages/51/de/8045d4ad1fd3a66c3b9bb576f3734c86015e19ae2f1617af92eb63cf9e58/sqlalchemy-2.1.4-cp315-cp315t-win_amd64.whl", hash = "sha256:fba3500e170d25f581e053009edeb0b158116084d91d465de218718d336b67c3", size = 2502237, upload-time = "2026-10-07T18:22:50.196Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4b/245e2315d331cc15765a2373e068445fbd28eb63beb23ea862828808c0bf/sqlalchemy-2.1.4-cp315-cp315t-win_arm64.whl", hash = "sha256:0a9a464bc360856b7ea9bf8aa26aab92ca115dd08149cb0e004063d5db13584b", size = 2420917, upload-time = "2026-10-07T18:12:21.876Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/dbf11a262f6fbb41390cab2d8e47a30ec0961018b68201607b599dd489f5/sqlalchemy-2.1.4-py3-none-any.whl", hash = "sha256:0b96edcc2cd60fe1e35f67a46f4eb076e57297841b9eae949ac5f196593f00a7", size = 2054935, upload-time = "2026-10-07T18:01:16.403Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/08/bb/28e3106ee586e5b2f5ecf211eec1348055f0f926314b13fe23b9cffa9039/sqlalchemy-2.1.5.tar.gz", hash = "sha256:0df76cf1ca680ad745bd378a6299a058611a8a784eeaeb4ed78ad4a0b7ce2003", size = 10569593, upload-time = "2026-10-16T20:52:20.57Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/aa/f10fc142d2db255249107ee8b21bfbabc10b90f7fadf13c3cf898acc6179/sqlalchemy-2.1.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dc1c86db97372e6bba18a7bcf0f6150dbdd808237f0f52802ef0bec676a94642", size = 2465792, upload-time = "2026-10-16T22:42:39.038Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0d/d395e310e36b5bd39a121cd455504aa1e9111f419efaa88be6b814b4e54d/sqlalchemy-2.1.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c229afac414d691d2f69a6f4e294ac5b5fd334c49da1f4fb86ab7f2a466271d", size = 4599703, upload-time = "2026-10-16T22:55:25.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6d/04f4ffc5038571e712c4f88f20d7022091bafab082e85ea5b37a7e8270f1/sqlalchemy-2.1.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cf433bcb77696cc915552d7f1e8d9117a5dae5aff9fceb06a69d74b3b3c2bd1", size = 4652645, upload-time = "2026-10-16T22:59:23.273Z" }, + { url = "https://files.pythonhosted.org/packages/11/85/08bc336576755daf087e0888e44657f4f72b6676a99522e7a612f0c2481e/sqlalchemy-2.1.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:399d3e212cb557d5bd1ca6f79981e8c3f45c0f6be60b63a9ded790396a81af08", size = 4316833, upload-time = "2026-10-16T23:15:10.011Z" }, + { url = "https://files.pythonhosted.org/packages/a7/5c/476441255cc7afa25e8c75e5a61d3c5c3af7cf61c951ebed2ee9873ec4b1/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a51c56afdb880e9f83736a2e81c02dd002fe1c5ad68346a35d97a2e135b1568", size = 4523134, upload-time = "2026-10-16T22:55:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/63/ca/064e5e3ca7c8f256c8f9b5206eafd410a39226c2ab6fe7e8c1e70725e432/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:17f8c68bd35bb76b2d9ef1097941fb783ef9ef6604c1d2a72c2e276430aeffca", size = 4318058, upload-time = "2026-10-16T23:15:12.139Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/e38956834896e21038a67193f91b189b9a61555b92fac42368027d5a2524/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:59dbd7412531da0124df8667a244b5155773632e9bd79a5c2ead393b11b59de5", size = 4613916, upload-time = "2026-10-16T22:59:25.19Z" }, + { url = "https://files.pythonhosted.org/packages/14/b3/5531eb9d10d4bd38bd7e6c505776ac73be259dc117c9f07a8c46db4e9a2b/sqlalchemy-2.1.5-cp313-cp313-win32.whl", hash = "sha256:3f566787cb3a6a25ad8c5170e90167d4e8ea103d412aa4ea95021a6df0b95964", size = 2379136, upload-time = "2026-10-16T22:48:37.111Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/c48924aafae616d59a2eb0375feeaf4bc4356327a442bda3b9205861b543/sqlalchemy-2.1.5-cp313-cp313-win_amd64.whl", hash = "sha256:fb8dcd5e1cc12fdd4d490819ec7a56e8c02e86ca9087a472e28b422ff932bae0", size = 2429688, upload-time = "2026-10-16T22:48:39.038Z" }, + { url = "https://files.pythonhosted.org/packages/51/84/0855a7d42b700b091af6f817c171c2ff97478b0d5143fe61963adbff9201/sqlalchemy-2.1.5-cp313-cp313-win_arm64.whl", hash = "sha256:65eece9d9494e46de1eb3d49eb438acfcecf059e7a92a2f3daf72ec26d371757", size = 2390320, upload-time = "2026-10-16T22:31:43.871Z" }, + { url = "https://files.pythonhosted.org/packages/01/b0/59b26375cd618e861a9295d855b866ca85be654d20ed47e494e33c3bf26e/sqlalchemy-2.1.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5541a110ac4bb6377c785aea1b8f0b4d6658ed0ca5061ed1115d0725062b026b", size = 2468925, upload-time = "2026-10-16T22:42:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a1/01931ba38770f169ce70e4034b0dcf2c1a776fe7d8dde93a34b6355510b1/sqlalchemy-2.1.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f029b2f7114623c03f762596f313515a2358004c44eb18924a15940f2e27d30f", size = 4593220, upload-time = "2026-10-16T22:55:29.253Z" }, + { url = "https://files.pythonhosted.org/packages/42/9e/fa478482503a69dfb3b0cb82f3f80fe3816026530dd34b5359e2acd9f30d/sqlalchemy-2.1.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc44dcc80e89a006af415edffc7409b1f2976a4597065449c166987f95801ab4", size = 4622806, upload-time = "2026-10-16T22:59:27.153Z" }, + { url = "https://files.pythonhosted.org/packages/fc/01/357ff85423013de8316316752c36a6bd1b7f3830c30dc63754afe06ba589/sqlalchemy-2.1.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:72ddd7a8ec187d02de0ef316a15aaf381ddcc594958247b02a3dcdb4c42f0b02", size = 4316247, upload-time = "2026-10-16T23:15:14.27Z" }, + { url = "https://files.pythonhosted.org/packages/43/53/88c5af325c81c929d0ed5b7ec15b087173bd3587d93366c5e336b9647267/sqlalchemy-2.1.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f5173a6e60a15090f9e2348a8ca94cc13de72796c16733f3b62fbb22acba07b8", size = 4522522, upload-time = "2026-10-16T22:55:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/87/29/f6f01f121d5e590c568f405f78b8eb3edace446c114126fc428536446dd6/sqlalchemy-2.1.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:09b10bd68223d57cce39c08947efae3f91ac40b552e07c7c869c3350a0fc4c23", size = 4317593, upload-time = "2026-10-16T23:15:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/0c/6f/14863fc9aea433645851330242823b1879cdaac71c118f7c9366e7caf4da/sqlalchemy-2.1.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ea4cb5022462d9f15e7b72a52c267385aa97ac224c8503ac63ba90c0067ad0d9", size = 4583207, upload-time = "2026-10-16T22:59:29.322Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e9/6907fc009d9d06e8d03d6bf286df46e878ea2a85c839dcb65ef11ceb3a39/sqlalchemy-2.1.5-cp314-cp314-win32.whl", hash = "sha256:c3ffbe6c680c626403b7e9697d92465501078731b9e8863b1410826e66b950e1", size = 2386018, upload-time = "2026-10-16T22:48:40.692Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/dea47b4b361c427d390edc10d73050e6d11d905c56ef26861ff5f94aa8fa/sqlalchemy-2.1.5-cp314-cp314-win_amd64.whl", hash = "sha256:061d089901319463a1160e4596191e355d2a9b4cce4852dac5adc301b085ac2f", size = 2436983, upload-time = "2026-10-16T22:48:42.2Z" }, + { url = "https://files.pythonhosted.org/packages/be/28/305a91768a1ca0980c6ead455dcda5a2c1e616022901898e966ccf913ba9/sqlalchemy-2.1.5-cp314-cp314-win_arm64.whl", hash = "sha256:f5bc7fb93f1c17efefeb062bc4d2742ad1b71d56c36e6e2898a371ac6a56839a", size = 2401394, upload-time = "2026-10-16T22:31:45.275Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6d/d4bab269d94ed1017950873643b5bd3eb271a5b8d5b0451250f0173f90fc/sqlalchemy-2.1.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9bb993b1d0890a2da90589684c3d09de75df82c06aa8f7eda246c7cd0ea3e820", size = 2508315, upload-time = "2026-10-16T22:55:01.45Z" }, + { url = "https://files.pythonhosted.org/packages/d9/80/aeab72b51fceb71b1e1d62cabc6f51c9d8f20588165aa54e0d0d6cc77c05/sqlalchemy-2.1.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df7e71ed051e4003e73484ef6547aacfb3a2322aac0c230d530c97e48e77b4d7", size = 4924848, upload-time = "2026-10-16T22:41:10.671Z" }, + { url = "https://files.pythonhosted.org/packages/05/8b/7318ca0ae828fa1605dd4481f8eeecb5325a7ead6289c1206fb1ff1e2e1d/sqlalchemy-2.1.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c76d4cb9c76a6906563133449a76d9eae6c8529ca09484b8615ca604442b442e", size = 4835557, upload-time = "2026-10-16T23:06:18.482Z" }, + { url = "https://files.pythonhosted.org/packages/35/17/7a5a196b0af72d65cbcd3a6fb7c4128693d039faf58a682ffe2d19329420/sqlalchemy-2.1.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:59a56be2e4e972061f9d5c1d5b01763b8b2207b48b956fbb6879c7443f5817f7", size = 4513707, upload-time = "2026-10-16T23:28:18.969Z" }, + { url = "https://files.pythonhosted.org/packages/91/50/ca323ed15fc06c73a7fd85a3a339749994c560c12312ee8a53abf0f9a7ec/sqlalchemy-2.1.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc1a61d7f42e71746b5a2e10e90a6b174b4add441b7b6cf14278f1e029b2b800", size = 4781127, upload-time = "2026-10-16T22:41:12.784Z" }, + { url = "https://files.pythonhosted.org/packages/26/3d/c56a75d0a2f6af08f6bb28d0b69e129911a2fc52b24dbafd7a7e5ac5c1aa/sqlalchemy-2.1.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:740e96dfb99a615c8056070d3747efe8bce4c145e5953ccc9e4b004c139de0d2", size = 4513816, upload-time = "2026-10-16T23:28:21.378Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dd/84e0cd627bbd3e35b5f033191a4f597e611ff96e424db36a444a8d6755cc/sqlalchemy-2.1.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:92a0bb0595331c8e40bc615a67cc7bca9b761487409d1bf46c598b6ba5c0754d", size = 4772462, upload-time = "2026-10-16T23:06:20.876Z" }, + { url = "https://files.pythonhosted.org/packages/b7/47/cf0a920fc40344f8a1806d661e7c702341aebd871e31e1d1d6e0fbc9d1e0/sqlalchemy-2.1.5-cp314-cp314t-win32.whl", hash = "sha256:11185c72b5372369166cb721537943f8e1e6e42a399e9bac19e02f3ee20592a4", size = 2444861, upload-time = "2026-10-16T22:51:51.169Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/4c5dc7e30fe740a01df4ba5ab800be2d7674b87f042e4c21c7ca89b36f9d/sqlalchemy-2.1.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5173207b8fe1ddfc1d451c420a2a9cbbd0734b4a19896b8080e3744497e1c20c", size = 2512497, upload-time = "2026-10-16T22:51:53.014Z" }, + { url = "https://files.pythonhosted.org/packages/5f/2f/d3e9bfb1faabea302b98ba8c9dd6252bc362c6afeabe28ffcc1eea8a6bd2/sqlalchemy-2.1.5-cp314-cp314t-win_arm64.whl", hash = "sha256:def2f117a6ec0a25202ee36a148bbcfad3d43e2f2bb2e698e6d4448c6db5cb19", size = 2430598, upload-time = "2026-10-16T22:32:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/ad/7c/17f07378eea7ec2fac8ac60f99fe3c474ef5c19a3adede70ae9886eb82d7/sqlalchemy-2.1.5-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a0b80f91d9ed92be7a134278ea2e83a693c07d9986824ca51d859af9e57e18f5", size = 2468856, upload-time = "2026-10-16T22:27:43.754Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c0/4fef29827b198937e09c40d0fa36b9738bd3914f0e5040927dfac8dfee21/sqlalchemy-2.1.5-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3033ddeeeb34c6c536622c07e932a20f8c9f58d6dd76e56595a76b55a25011ff", size = 4597018, upload-time = "2026-10-16T22:41:59.749Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d8/37645f8445ac3774e18f8dc72e0fa0ac868e0dff0e987ff3bec644f55102/sqlalchemy-2.1.5-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4d30723171d511ec16ac057872a38b9b395420c0aaf81b2f94dd7bf1345c5fe", size = 4632322, upload-time = "2026-10-16T22:44:53.83Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c9/15466cd08a6c329acc4f730dff7041e84122456fe38c518c93a07d6d82ca/sqlalchemy-2.1.5-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cefa5badc13df1c9082bacdfa435c8eb6dcb9b32717f51d09d1e160a599646f", size = 4338381, upload-time = "2026-10-16T23:01:22.714Z" }, + { url = "https://files.pythonhosted.org/packages/12/2b/a93b7c8f6886138dac91f66fccd3b42473c365408d9a4c1b69403cf206f8/sqlalchemy-2.1.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2549e503185897de1fc5ca7730d5daed0d27d0b26df4a059dcdd33ebe81b7b4b", size = 4525244, upload-time = "2026-10-16T22:42:01.631Z" }, + { url = "https://files.pythonhosted.org/packages/b3/47/d3926cacd194be073a2053a05470f20ed7da623f399e95213c1e497954fd/sqlalchemy-2.1.5-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:5a6871ee159a943d874012de835d56d0544ee871a99dbae05bc053da315b2c23", size = 4337223, upload-time = "2026-10-16T23:01:24.702Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/b15ab4712342a18fb3949c77687c24d3749e86079c9905cc2cbebbd44da3/sqlalchemy-2.1.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:51e290080415325b66a94d20f6360e310606acb832f2bd677d4fcb95728e7ce1", size = 4594562, upload-time = "2026-10-16T22:44:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ab/3b6ef6c77740d7fbf5a3eb45f95768fb809b0769bd21df81d5f8c002e5a5/sqlalchemy-2.1.5-cp315-cp315-win32.whl", hash = "sha256:c228084025a28aa038aef9b821d842508650fe12fe51f06e9cdaa4393be7812c", size = 2385770, upload-time = "2026-10-16T22:41:04.484Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c1/0b3256bc11487b2846a26d58946caea0ed395fd500951a16fb82d31a20ff/sqlalchemy-2.1.5-cp315-cp315-win_amd64.whl", hash = "sha256:bc09dd22c3fc491a2df7d16ca4167de35573f40c2e82dd4d6ddae589a73e7032", size = 2437043, upload-time = "2026-10-16T22:41:06.12Z" }, + { url = "https://files.pythonhosted.org/packages/e5/a0/09eaa7897f6f745de16220b20c63660948a95716be90d2f405af08222e57/sqlalchemy-2.1.5-cp315-cp315-win_arm64.whl", hash = "sha256:375ef1bf78bc269a220963163d2580d5377898ed23a00bdd98a02d755421573d", size = 2401090, upload-time = "2026-10-16T22:28:58.035Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c2/4c747dcc6fe329dd07ab57233f334eca19a3ff05934e0aba4f25a0977a67/sqlalchemy-2.1.5-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:7af1c92255cf8d50e87a808ce41096907332acdd09f285426a52f7c300e152db", size = 2504475, upload-time = "2026-10-16T22:55:02.946Z" }, + { url = "https://files.pythonhosted.org/packages/07/f1/183446b1bc29484087ba7a6bb199a265c0559bf8ff201c1ee54bf1fe0936/sqlalchemy-2.1.5-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:85891e0828e76a77fe912995dbc3ee949099488ba0a2ab323535d2e767446b20", size = 4902197, upload-time = "2026-10-16T22:41:14.703Z" }, + { url = "https://files.pythonhosted.org/packages/31/0b/b24baf343461eefdecc672df1ccbb14c8c64028a11625dabc97b0bb9791b/sqlalchemy-2.1.5-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a72e062d2d38826dd89231df692fb40f12d7d1b9d118be23bcc1cf93c6e31782", size = 4851871, upload-time = "2026-10-16T23:06:22.861Z" }, + { url = "https://files.pythonhosted.org/packages/da/27/f688b34ea3d85b38e01b7463874f1f4b91619dfed4e6cac88c27caface73/sqlalchemy-2.1.5-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b79ee943e2e12246f873345762b83dbadd1fcfc3e832eab83272d672704dbdea", size = 4500095, upload-time = "2026-10-16T23:28:23.431Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/d8e13be8d0c25ba49244a9c24579ef61d5a6b57f98e55fbb173b7923ec3a/sqlalchemy-2.1.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:e9418d87b55c469c2f1328246d4533238cdc43134c695d6b9a6b7a87da49d463", size = 4757826, upload-time = "2026-10-16T22:41:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/a0bd2ead949cc605ce3ad5d38e2fe84963b2aa7f89e8648c1bbad13a3679/sqlalchemy-2.1.5-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:898d86c22640fe52ffe02b4abc043a3c6754b4875837599e739be440966dffa1", size = 4499377, upload-time = "2026-10-16T23:28:25.203Z" }, + { url = "https://files.pythonhosted.org/packages/b9/dd/c1ca0713a1a7d8d6eb5864edd2d11d7a6a327aaba664645733cb53662425/sqlalchemy-2.1.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b3ad20cd035392f2b91a6466e24643990c1378beda80c9bda87a3a41e841e838", size = 4777264, upload-time = "2026-10-16T23:06:25.203Z" }, + { url = "https://files.pythonhosted.org/packages/5a/dc/9dec72cea689f16bc8ecfcd62b4aa2796009a7cef6be85a9f70310ecf71b/sqlalchemy-2.1.5-cp315-cp315t-win32.whl", hash = "sha256:db34eefaf992c84460ab26ecf2fc2bdcc68965d6becfae8182cae0efeaa3e8f8", size = 2441666, upload-time = "2026-10-16T22:51:54.58Z" }, + { url = "https://files.pythonhosted.org/packages/dc/28/f44c4524ce8a68935bd6f23ee801165072fb8c1e477fc9855efd9eaa2360/sqlalchemy-2.1.5-cp315-cp315t-win_amd64.whl", hash = "sha256:173054351d863ddada188284e309e87a9f030132c8d534e05458971ff061aa1e", size = 2507526, upload-time = "2026-10-16T22:51:56.262Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fc/e90c91f6c2709a431f9b75e41bd31216337859411359f4a4fc2615c989e3/sqlalchemy-2.1.5-cp315-cp315t-win_arm64.whl", hash = "sha256:22c8d0873168a54f614e381559ea83d547b2cc5488b95b049bdc92667586bd5f", size = 2426204, upload-time = "2026-10-16T22:32:56.011Z" }, + { url = "https://files.pythonhosted.org/packages/de/e9/61176e4753f74ab5f6e72a56314ee654d25c31e6fc98c99060442998cccb/sqlalchemy-2.1.5-py3-none-any.whl", hash = "sha256:2f77b77b716f005b2dc3cc059d56df6a919bfb5b439e4149890a9df6c36be9e3", size = 2060214, upload-time = "2026-10-16T22:43:30.801Z" }, ] [[package]] name = "sqlglot" -version = "30.22.0" +version = "30.24.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e0/db58fbf2527426758dc1e862ce538736978e100e4e78fc9657e9661826ee/sqlglot-30.22.0.tar.gz", hash = "sha256:ec4b83ca8236ea8867f574a382dc15ce35b071c977fecfcc66482d9a3f500661", size = 6088770, upload-time = "2026-10-09T16:09:01.04Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/7c/9fd907983ff041692399bf855d344e1fcc9a02c8d92a5fee6dbdf139d19e/sqlglot-30.24.0.tar.gz", hash = "sha256:6451aabd1c9994d0f73355c962a5eaf9258381b3f2554e53db916cd9a605552a", size = 6131857, upload-time = "2026-10-20T17:14:03.987Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/4c/b8474b02b572d9c7a2903e364335d566d52b6128b834b92a7cdfe5597823/sqlglot-30.22.0-py3-none-any.whl", hash = "sha256:90aa461490fcd95d14ec3842a97506ae20f6d3e9313307ad31be793d479cca65", size = 777816, upload-time = "2026-10-09T16:08:59.07Z" }, + { url = "https://files.pythonhosted.org/packages/68/5f/d78f74306cc026c5f40f8808a47356dfd37461eb15eb759a20e5e9efe7d7/sqlglot-30.24.0-py3-none-any.whl", hash = "sha256:9488828cc018453b7a3ba01e65d2749c9a731314c14583695163ca1f358203fe", size = 792162, upload-time = "2026-10-20T17:14:02.053Z" }, ] [[package]] @@ -4485,13 +4722,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "textual" +version = "8.2.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify"] }, + { name = "mdit-py-plugins" }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/21/39a76b01bd5eea82a04baaca7580e105d8c59450df03998345bb2cfb307b/textual-8.2.8.tar.gz", hash = "sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b", size = 1860502, upload-time = "2026-06-30T06:51:24.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/be/35261223d9416a0751cdff1c7b4a6f881387218a12d439fe22fefebc8c04/textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a", size = 731418, upload-time = "2026-06-30T06:51:26.364Z" }, +] + [[package]] name = "tomlkit" -version = "0.15.1" +version = "0.15.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, ] [[package]] @@ -4570,15 +4824,15 @@ wheels = [ [[package]] name = "types-boto3" -version = "1.43.113" +version = "1.43.115" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore-stubs" }, { name = "types-s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/4c/1708fc45a80ad1314f60f8c7fdedc8c17cde501890c373e41da13aa7af86/types_boto3-1.43.113.tar.gz", hash = "sha256:56d93beada6caef7b8162f0481c56e3c34b0be5f488298c5399b89cfdb1a812f", size = 106097, upload-time = "2026-10-13T21:52:30.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/49/0050624e4285f2de32d1091b81dc6900be3c9295226e2b8ef42580dc106c/types_boto3-1.43.115.tar.gz", hash = "sha256:0787cf5bcd91e8481c8640aefef02d7a43a36255547f568aa6a5ca610a52d72f", size = 106082, upload-time = "2026-10-19T20:43:45.489Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/fd/31c34565b3734f5563649e1f38ef43c122cd8dee3ec55a9d01442c43c42f/types_boto3-1.43.113-py3-none-any.whl", hash = "sha256:c3f5a5ede13a909c4b5e2c78a786f11beeb11e2f98df862fd136ff20fc13a980", size = 72085, upload-time = "2026-10-13T21:52:26.388Z" }, + { url = "https://files.pythonhosted.org/packages/43/65/a79f624585182bcfac2672611a50bd1f22ba2e0201eaabfd31b06b35e914/types_boto3-1.43.115-py3-none-any.whl", hash = "sha256:f97254fafc0fe66005ba885793c202972e813b3b3090e0ef40aaa5a5c8432fd3", size = 72086, upload-time = "2026-10-19T20:43:40.558Z" }, ] [package.optional-dependencies] @@ -4597,11 +4851,11 @@ wheels = [ [[package]] name = "types-s3transfer" -version = "0.19.2" +version = "0.19.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/61/29/4df7e8eccfdaac9308e06ae94271eb0e5d2324cbe6df241363c44b70e08c/types_s3transfer-0.19.2.tar.gz", hash = "sha256:2a78a806c09b11fc6d59756402ade26b49b93a85ac6f209a39e2498211c6a41b", size = 13766, upload-time = "2026-10-13T01:42:25.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/b2/4f2d72b5d234c81caa07dee688cacee2c8dc5bd793e9ae70a573f00d6425/types_s3transfer-0.19.3.tar.gz", hash = "sha256:99422ef028a5f854cd92d1e72cf4208bd0569f68a87936ca02fb9f91a68cdcd3", size = 13780, upload-time = "2026-10-20T02:24:19.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/02/3fa02c57a65f8721030247866d28c1a6601d7826297e83059c50b7d6ce7e/types_s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:f1167b9a082a49fc55f12700dcbd3a8b540b9e6fec4e1d782ff44f3eef0bdbff", size = 19473, upload-time = "2026-10-13T01:42:23.813Z" }, + { url = "https://files.pythonhosted.org/packages/f5/98/972fa03c9a6e1cc9184a28ea0354af50a6764bb172fe2d0751798a295f07/types_s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:6843143f3b79e3cb73e58b6f45738e28e88d9f681fd32cc5ef7fa16f0c193356", size = 19476, upload-time = "2026-10-20T02:24:17.682Z" }, ] [[package]] @@ -4706,38 +4960,38 @@ wheels = [ [[package]] name = "wcwidth" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/b4/7830542634bb2d3e62aa3b586a72d5b3b6c91c3168929e7000ef3fed041d/wcwidth-0.9.2.tar.gz", hash = "sha256:ae0ef90b90f6af38b54f1fe6d58662ec33b3cb4b8391958a62416d654231727b", size = 955039, upload-time = "2026-10-05T00:24:05.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/1e/4532a81fb9dfbf4114a816775e0a36c3a64ee1d1f4bba2094e2da50be5dc/wcwidth-0.9.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:7ef5a940bd5e30bac6e721f1a48fce0cd7bb3ece19e9c5d139e72c76c35cfd07", size = 605343, upload-time = "2026-10-05T00:23:22.649Z" }, - { url = "https://files.pythonhosted.org/packages/a0/07/cb6940e81134b7ed25fa312ee9ab536a63db0793b149f88a90e603ceace9/wcwidth-0.9.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ae0800c5339423cc53d33a266ad264b42ba8aaa16d4464f6e6b1bee607f50b17", size = 609960, upload-time = "2026-10-05T00:23:27.049Z" }, - { url = "https://files.pythonhosted.org/packages/a4/80/15ad05d40bfa99155639fb9e13b3d77083aa0fab893c816db2543d29005c/wcwidth-0.9.2-cp310-abi3-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:9e542f1f8475b78452a295495d7a5bc3ead565112e9446a64dc93462a41c2a79", size = 762566, upload-time = "2026-10-05T00:23:38.322Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f0/b8ef7758003d66b60f093695831a86dcc726aac01ee6446ffcbda27b61e3/wcwidth-0.9.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:674b518af28d38ee645ff97b74f5760abee5fad4bac74413bfc4b881ef2ce724", size = 771632, upload-time = "2026-10-05T00:23:32.448Z" }, - { url = "https://files.pythonhosted.org/packages/db/6c/f940133c71427c208575910e981942bd78c98b1f7cd0d1425ca4b7457c04/wcwidth-0.9.2-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:751bef0ab404b6a1dc028b56b4b85d46486be1c55833f80da533e42dc691f389", size = 769763, upload-time = "2026-10-05T00:23:40.175Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/285f862826f721964ec7c42f81dc53d23afbd723a0f4cd989651f8218e25/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c3d80f39ba4653a595edae9aa46a509d14883790a8fc23c5db221ceb207f64b7", size = 782134, upload-time = "2026-10-05T00:23:33.926Z" }, - { url = "https://files.pythonhosted.org/packages/c2/2d/64aa54882a5d556d3654c1f926d9118b797461033e23a158409941a37c8f/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:0a47e03d8293590ecce66c45dc20ff7b4b885e3c78093722239585eca0d77ab2", size = 782851, upload-time = "2026-10-05T00:23:41.974Z" }, - { url = "https://files.pythonhosted.org/packages/59/39/52389f6de7fe2e9c14ceb8253dd99034bd86e1c87847ea3c100a97dded9a/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67d901a4ad99249eb775b4ee4769ca97fa405d35a75f46e83166910a47003f04", size = 784621, upload-time = "2026-10-05T00:23:43.449Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/20225500a076ace27bbcc8a6fd7c55125133c57a618816c7b7b8b73070b1/wcwidth-0.9.2-cp310-abi3-win32.whl", hash = "sha256:ee1fd0db9d9fd711a70f3e7765e0e04c05d26982fa05361456163062549d7da4", size = 593123, upload-time = "2026-10-05T00:23:55.953Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d6/b0690f55ea0483530a18bac917fbadbf54f35122510446fc370f5f1c2453/wcwidth-0.9.2-cp310-abi3-win_amd64.whl", hash = "sha256:2a9746de704242bd4fdaabb31dd46b82f694a56a8d21081ad89b679a89da9fec", size = 597827, upload-time = "2026-10-05T00:23:57.489Z" }, - { url = "https://files.pythonhosted.org/packages/e5/11/6ecf4e9e268ab1a4ec617ffcccc2ee4a71301625f5490912dbaba462fa9c/wcwidth-0.9.2-cp310-abi3-win_arm64.whl", hash = "sha256:b9c6ab615e03723b7f8760ea2f27758d656e7e13b51515c9dca5c3e8b04612fa", size = 598415, upload-time = "2026-10-05T00:23:51.517Z" }, - { url = "https://files.pythonhosted.org/packages/4e/41/549eef1ab767032bdbdc1f0ab655d404b082b1e9a1dab1361dbba90f64ed/wcwidth-0.9.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eda88ffdc97c0fbf193d407114f2c7a54b379f67f6e52a7531ee3b9fe749eca7", size = 606555, upload-time = "2026-10-05T00:23:24.188Z" }, - { url = "https://files.pythonhosted.org/packages/9b/64/a875ed7ea71cacadc0ae11b5fd3fac3486efd58bb25e67a7344248dceadd/wcwidth-0.9.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bf361c8705576760623b4724ae564666d73b016f9a778bcfd1c7345378ef4ec", size = 610829, upload-time = "2026-10-05T00:23:28.563Z" }, - { url = "https://files.pythonhosted.org/packages/c6/98/513095e484fe79b6f2613d6a72f855f5d56b65e15c215c2a6746fbc638f5/wcwidth-0.9.2-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:97b878d1e158da5ed9ac5aac53fa3a55e282103af6a09ec353865613d1a31a76", size = 775819, upload-time = "2026-10-05T00:23:45.116Z" }, - { url = "https://files.pythonhosted.org/packages/22/fc/c02f3eec57224731e78f84b68e272250f784b6205acc7e0dcef6a7c23a0e/wcwidth-0.9.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:59dab4049cbd982b478bca098528df2c79a9160636a3a163ffebffcbd7d1b892", size = 786915, upload-time = "2026-10-05T00:23:35.323Z" }, - { url = "https://files.pythonhosted.org/packages/3d/6f/b0529a79bac3fe8d94f32b4237a13dbc3f955508753f6a6f06c73d679dc2/wcwidth-0.9.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb08ceb501d6aaf94066c3ee122dd825b152df40ff0bd0df4dc27126233b948e", size = 784186, upload-time = "2026-10-05T00:23:46.366Z" }, - { url = "https://files.pythonhosted.org/packages/d5/bd/6357c84ca9a734bfc735b7c48dbe21336b3777fab8a4101d14976dfe49a7/wcwidth-0.9.2-cp314-cp314t-win32.whl", hash = "sha256:8b4e381590b9b7390e07e22b2c0c1bb96ce50e1d2243c866d9387600362d51ed", size = 600791, upload-time = "2026-10-05T00:23:59.398Z" }, - { url = "https://files.pythonhosted.org/packages/98/de/037591ca18d897cc2179559dde72e6efc6ce0c90e9cd1e6bca4e87c38b4b/wcwidth-0.9.2-cp314-cp314t-win_amd64.whl", hash = "sha256:f2f7b3bba5a5d5f31fc350fd36ce5b84b693c83b7eb95ee630b720da5a5ce06f", size = 605463, upload-time = "2026-10-05T00:24:01.049Z" }, - { url = "https://files.pythonhosted.org/packages/d0/07/c9d96e106d938d26f7ab639bc80b8199359a1645ba6e3498413313ab6f38/wcwidth-0.9.2-cp314-cp314t-win_arm64.whl", hash = "sha256:734aa9405b321d1042301aa19c943c4731ee9e3460e4f8feea3299c064c97a14", size = 605936, upload-time = "2026-10-05T00:23:52.765Z" }, - { url = "https://files.pythonhosted.org/packages/82/8a/a28d61d910005ac93dfe48be3a0ebaa49352d88cebd25323e69e6ff2f4a8/wcwidth-0.9.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:42dbcb76ce8af39e2c9db410ac3f9bdf4e47eb41d6f44525952f172d3d98f724", size = 606575, upload-time = "2026-10-05T00:23:25.663Z" }, - { url = "https://files.pythonhosted.org/packages/01/c2/a3c66bd32766c8f4d6dc47d572532ba014fe5be30489f2576aff7cada363/wcwidth-0.9.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:138e1f8898e431b2f2d7881f8ca8d75591c1d3c21aa53f54e989bd6b39811da2", size = 611045, upload-time = "2026-10-05T00:23:30.421Z" }, - { url = "https://files.pythonhosted.org/packages/ec/8a/d39964f8f8c019d7d439b9b501d3e7bb42fee69f00354040ba0b27b5824c/wcwidth-0.9.2-cp315-cp315t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5175609bf8cc7398a5f48aa35207bd64ebf9f45e4c70df65f7fdc7a988041a3c", size = 778143, upload-time = "2026-10-05T00:23:47.7Z" }, - { url = "https://files.pythonhosted.org/packages/2f/53/525da13e8f9ff7b5b4e74ec6f8d68bdee63905796972e086c6b1b96670d2/wcwidth-0.9.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5f669ae8c3d969c72032f9cdee019674b666e522d45e1e2099a2e9dda4a341d", size = 786630, upload-time = "2026-10-05T00:23:36.967Z" }, - { url = "https://files.pythonhosted.org/packages/ef/9f/d6a0c6df354b9d93466548a65cbf4ffcb48c719bbd307504cf3e76740837/wcwidth-0.9.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:196b47cf32f9df27ccda6dc513237f3c2429c4c659db428d60a5bc443d10f270", size = 784839, upload-time = "2026-10-05T00:23:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/bf/d7/3021feed1ed7926021ec134943ad3b24a2f7ea742cc9976461171482ed77/wcwidth-0.9.2-cp315-cp315t-win32.whl", hash = "sha256:0cd4f7f2e53905dcb110d213a4c8529b6733fa3d232d8c717f946cc69a10349b", size = 600845, upload-time = "2026-10-05T00:24:02.497Z" }, - { url = "https://files.pythonhosted.org/packages/63/80/6a03356d8ee38261e3a78cf89ee03d8e7f12c572d969237be00869e2dc73/wcwidth-0.9.2-cp315-cp315t-win_amd64.whl", hash = "sha256:33df042f96c61ed3cd5fb3742fba427553a635bc578799857a48aa79f774a0b9", size = 605441, upload-time = "2026-10-05T00:24:04.052Z" }, - { url = "https://files.pythonhosted.org/packages/0c/48/1a308a86a833fd12ff7a08d0d2491ff4a72c8a92d12f5ead8317630f771e/wcwidth-0.9.2-cp315-cp315t-win_arm64.whl", hash = "sha256:48719a9bc76c2f84238693fe5013571fa5beffa3621cf228f1f3a9e30dae84b8", size = 605885, upload-time = "2026-10-05T00:23:54.274Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b4/0bfa065af506540d9d558e3e5548cff00bc1f9b24e6e2a8512498e8628de/wcwidth-0.9.2-py3-none-any.whl", hash = "sha256:89ca642c5bf0101157a09366be69fad0379db1f700ae39a920e103234573670e", size = 301667, upload-time = "2026-10-05T00:23:21.097Z" }, +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/4a/82d61476a1ccc20d43d0805724c5de4baa4971131a8db1816705ef53fe16/wcwidth-0.9.3.tar.gz", hash = "sha256:dfe7cfbd4cb59db9fe0931bd6091ab4a95b3acc5d56bfb50fa9fef0ccba5dc5c", size = 956320, upload-time = "2026-10-20T16:02:43.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/6d/eec61b5edeb88e3ea2a714184eade38335722c4246d94be567977fc20caa/wcwidth-0.9.3-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3b119ef60177a92b4f298b80e98464c2922a1fba3ed77fe9a2bcb69edf1b9af4", size = 605400, upload-time = "2026-10-20T16:02:02.523Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3c/714eba03905a23a46277cc96d3db8da6bd22d2cc0d09a4dead60a2bd1757/wcwidth-0.9.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:aab90ab489850f87633922c3064f6d0442d8f6ae5407a62f82bdb1f424179c05", size = 610124, upload-time = "2026-10-20T16:02:07.256Z" }, + { url = "https://files.pythonhosted.org/packages/90/ad/8566e7cc712bea55eab0bb6d19581b7c727a58fcdd55f02365b309aaaccd/wcwidth-0.9.3-cp310-abi3-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:9e1cbe18c18d152e1e68e4e2561dab6c2bc988f8fd337a1fb2e1e0809c3fa568", size = 762700, upload-time = "2026-10-20T16:02:17.508Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7a/4e26574d032a1709bb1fd7d95d406b7b8f87363c60305a101e6e2d9285f5/wcwidth-0.9.3-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c12abd75ffcbad417e7770a6feb82413c102ccce5c435068c39a82e2218b00b3", size = 771850, upload-time = "2026-10-20T16:02:11.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d6/48dad66d577417ef8594000506fa8619d45d7282e20b4cdc916fcc68f95a/wcwidth-0.9.3-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ab8bfee107e08de09e15f9f6afb9fe0f64c8d64fd296d0d8557bafe71ea3f93", size = 770047, upload-time = "2026-10-20T16:02:19.092Z" }, + { url = "https://files.pythonhosted.org/packages/3f/61/5f1e02e588d289837d5312478cc9f0e7bca93f4c8a77807741f79fb6a31c/wcwidth-0.9.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f875a4c9e62df2ecfa70d99b24c551f2245609858e179c3a40e74631b2d08a4", size = 782210, upload-time = "2026-10-20T16:02:12.753Z" }, + { url = "https://files.pythonhosted.org/packages/e9/11/0da2720cba3f917148448c9bf73f3079d71f918308ee9e4d737a0b7acfd8/wcwidth-0.9.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:70a2ca239b85a219324968993c76c117fe423d4e1a7c9d860c999c66192f2570", size = 783172, upload-time = "2026-10-20T16:02:20.663Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/b79407036ca4543bf708a6d4ca561f78620dc1e5443e685e094480c37753/wcwidth-0.9.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:537ea00b6792bb243cd09582322c54dbc848879e5d34ea4c5fee0147529685ab", size = 784806, upload-time = "2026-10-20T16:02:22.164Z" }, + { url = "https://files.pythonhosted.org/packages/3c/86/ada6c2244b08a86c871d42f7cd85b5226515b5f3fa7e604d1e6e45590275/wcwidth-0.9.3-cp310-abi3-win32.whl", hash = "sha256:f2f943c4ef14e9baba730740265f4e5bf5fc8473550e36cd92828cd5663561a6", size = 593154, upload-time = "2026-10-20T16:02:34.301Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6e/46363d972b2b9f002fbfc3bf8e9ab0ff89f139c935a5f21d22016e7a2018/wcwidth-0.9.3-cp310-abi3-win_amd64.whl", hash = "sha256:569a0cdb826d3097b6dbacfaa45aa0002d72a2ba6fa8a63251ea43ac4dd7b6a4", size = 597685, upload-time = "2026-10-20T16:02:35.748Z" }, + { url = "https://files.pythonhosted.org/packages/69/3d/5583cac85e233514c3906d5476b6a74160bc4dc82729dabef3485dbf3d52/wcwidth-0.9.3-cp310-abi3-win_arm64.whl", hash = "sha256:80557db1f77148b78e1b3c2a9cc4ebc4feabef42aaf69a555565057031c32367", size = 598774, upload-time = "2026-10-20T16:02:30.016Z" }, + { url = "https://files.pythonhosted.org/packages/91/b2/53e0838babc6b87505b9504ce62b72c79403642f88d3e568e05d0ef04825/wcwidth-0.9.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6f3e855d3a09af8b7b6d56e76bc270681032f8a37d527a8d5707a441958e3730", size = 606640, upload-time = "2026-10-20T16:02:04.01Z" }, + { url = "https://files.pythonhosted.org/packages/f0/fb/27c94facff5c4a39b954a41fac20c238647f0aad32452d0d9e33fc05e852/wcwidth-0.9.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1bbc8ebf133422cadfefea95a9f89d6c9e9a1ae19930bef7862a9b59b20bbad", size = 610956, upload-time = "2026-10-20T16:02:08.764Z" }, + { url = "https://files.pythonhosted.org/packages/38/61/38e6236356d25966d34636ef6f49ee83a66513e051c4a3b1e19c8b9ec806/wcwidth-0.9.3-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7a95d8c784cbccbbcb808eaadfd156cd282b2bdd95e89b32505e3443381ce61d", size = 775967, upload-time = "2026-10-20T16:02:23.765Z" }, + { url = "https://files.pythonhosted.org/packages/7b/87/c8f9295cd14a058dc27be0360fcc5ec70ce71b0d2e6a018871f585f23126/wcwidth-0.9.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a44acbb23b525070a0b8c343dd7cab7a4020aff73b38804f7a66d6a66e16e3fc", size = 787147, upload-time = "2026-10-20T16:02:14.095Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c0/238c135e283b1b0e16fef8b082d432dda1fe7e568d6dd85d5c7fc5e83859/wcwidth-0.9.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a7c0405a3bf3c564e481d6b2d55c99cc231bb68cf8f4a0995944bbce5ebf1a5d", size = 784548, upload-time = "2026-10-20T16:02:25.189Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ed/33e3080fe32f6f8f86334b2c5adde243b9a3a3a46cef97694dbed73b9dc4/wcwidth-0.9.3-cp314-cp314t-win32.whl", hash = "sha256:457f986e4867111cb037ccbf0d97ae100b1ecad030d43a020e7207f1e29649d5", size = 600761, upload-time = "2026-10-20T16:02:37.097Z" }, + { url = "https://files.pythonhosted.org/packages/27/62/6684a59147c1f86d58d0138f3109f506a6761b815dcb3d86b74d9af3a723/wcwidth-0.9.3-cp314-cp314t-win_amd64.whl", hash = "sha256:f11789e520ade8017fa93b01a4b8985571bb433eaefc2f857b57e36875e2e1d8", size = 605316, upload-time = "2026-10-20T16:02:38.513Z" }, + { url = "https://files.pythonhosted.org/packages/94/24/3ebcfd772d273e43e28447dd1946d88b8625db57b6a23496ea4eb9e1dbb7/wcwidth-0.9.3-cp314-cp314t-win_arm64.whl", hash = "sha256:eef39f3260991d8767189d296f0116fa4b45ad9c3a9a16d5af9187fb66b27564", size = 606392, upload-time = "2026-10-20T16:02:31.333Z" }, + { url = "https://files.pythonhosted.org/packages/76/fe/f90b9444bbc2d566634f91756bd11c2bd39d50c2c9077551711b10d8d03f/wcwidth-0.9.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:52a13869704d43a4add59c68ebc892198e53063883114783bf0b7457e250e3f4", size = 606655, upload-time = "2026-10-20T16:02:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/ab/63/edffb23938f4e27c28400c7d8d5a8715381d77f5db96bf80cf5dd6687f88/wcwidth-0.9.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:9470169fcb4e5340269175ebb3642dd5bfc8f82da8c86f29fc1fefc6c2b9bc1b", size = 611217, upload-time = "2026-10-20T16:02:10.252Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e6/0e4b5c81c238fac47eb66902f058a39ccdeacd7b2c5ab0c559066f203262/wcwidth-0.9.3-cp315-cp315t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c4531d93344441a7948e665854f0c40e3a8342ee106436d1531efa7a9fa7641e", size = 778155, upload-time = "2026-10-20T16:02:27.141Z" }, + { url = "https://files.pythonhosted.org/packages/f2/16/745439326544b2a6d65facbdf8462ee9d31e09c64830a764d2526c6c80d6/wcwidth-0.9.3-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:658dcbce546e7f28ee361f88e4a8e591dd56f46bff6084e11b4ad8310ed99946", size = 786757, upload-time = "2026-10-20T16:02:15.569Z" }, + { url = "https://files.pythonhosted.org/packages/08/a4/47193dc9902c30e187d81feaa7a5bc2672c41fe07093ca1ce5381395b85c/wcwidth-0.9.3-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9362be098c43d076aa43754face3cc0180fa81cea9ba304240b15490f2617e8", size = 785088, upload-time = "2026-10-20T16:02:28.784Z" }, + { url = "https://files.pythonhosted.org/packages/11/6a/9727002b938c28ab4456fc43ba31a3159ebff040021ce8dbd70e904c7cc9/wcwidth-0.9.3-cp315-cp315t-win32.whl", hash = "sha256:16d09df90b3a8f799ea14c726a41d9b906c3104f95f46d158e1ebe45cf4d547b", size = 600787, upload-time = "2026-10-20T16:02:39.766Z" }, + { url = "https://files.pythonhosted.org/packages/7c/db/0e24c2ff1e82cf298dd40a29e022da31b79be2589f0b8d865488e49140e1/wcwidth-0.9.3-cp315-cp315t-win_amd64.whl", hash = "sha256:3e3e31d6c17d6da3051f7eb854d5e4bebdc52a08a7a52adc2da6f15af51304e8", size = 605298, upload-time = "2026-10-20T16:02:41.479Z" }, + { url = "https://files.pythonhosted.org/packages/84/71/b470bd1dec14cbe6cee0c2e142787a8d99ac01711d5a1b533dda90754b48/wcwidth-0.9.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5d3b0b6c37f40e8d8298941ec373a426a892bb9a514328abcc9226cdc60ae6eb", size = 606346, upload-time = "2026-10-20T16:02:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/bf/12/e10aad98050389ef674c9f69409e38ae086c2104912644e1f9143b394301/wcwidth-0.9.3-py3-none-any.whl", hash = "sha256:e3c9069c54a2da0bf4f6c7e65a735cb1da0f47601a15cb5fef7e4b6ffca4cf60", size = 301629, upload-time = "2026-10-20T16:02:00.861Z" }, ] [[package]] @@ -4863,14 +5117,14 @@ wheels = [ [[package]] name = "xsdata" -version = "26.2" +version = "26.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/c9/71e9e8eac669091fd434ed494d806c8cc37614aecb34ce4c62c283f99abf/xsdata-26.2.tar.gz", hash = "sha256:c631af71aaa75734f8ce92a08fcf8389d905dee2aab0b5032c9032e9071009a6", size = 349690, upload-time = "2026-02-15T16:13:31.274Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/b8/7b301b80494a750dba431d3d0a800e0e80c91903e8891eef78ead8f95120/xsdata-26.10.tar.gz", hash = "sha256:4bc944d9fa3955f84ecebb83c677e2f681687c002a777731a21104e8360a7522", size = 356001, upload-time = "2026-10-16T10:06:30.516Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/92/f0edcbc2f895ecea14a68e492b24c157625e251279a94b172a6b263290e7/xsdata-26.2-py3-none-any.whl", hash = "sha256:85a591a4405d903416afbd4a917e8dda8ea44641a3e66d72134bc2a31b3c16b0", size = 235561, upload-time = "2026-02-15T16:13:29.614Z" }, + { url = "https://files.pythonhosted.org/packages/07/fe/5a9698cef0850317de2a8f3c50519fbaa2cc4225d15bcfe9922e4dff8380/xsdata-26.10-py3-none-any.whl", hash = "sha256:37a697d60db1396a0431750486744a7e0491fd60669751d95e3ede08f034e0c6", size = 243015, upload-time = "2026-10-16T10:06:28.782Z" }, ] [package.optional-dependencies] From 9e677dc946f2c55e9fb67c9bf8146cb33e4bc576 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 09:49:03 -0700 Subject: [PATCH 2/6] Unify JSONL, dest table reading; create generic CtsSettings builder; add shared FileItemDict and duckdb helpers --- tests/cdm_data_loaders/pipelines/conftest.py | 16 +-- tests/cdm_data_loaders/pipelines/framework.py | 15 +-- tests/cdm_data_loaders/pipelines/helpers.py | 116 ++++++++++++++++-- tests/cdm_data_loaders/pipelines/test_core.py | 13 +- 4 files changed, 113 insertions(+), 47 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index 20a58e88..8c9a242f 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -2,7 +2,6 @@ import copy import gzip -import json import logging import os from collections.abc import Callable, Iterable, Iterator @@ -12,7 +11,6 @@ from unittest.mock import MagicMock, patch import dlt -import pyarrow.parquet as pq import pytest from dlt.common.pipeline import LoadInfo @@ -28,6 +26,7 @@ PipelineTestRunner, ensure_docker_image, ) +from tests.cdm_data_loaders.pipelines.helpers import read_destination_rows as _read_destination_rows from tests.conftest import LOCAL_FS, S3 DEFAULT_DLT_TABLES = {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", LOAD_INFO_TABLE_NAME} @@ -263,18 +262,7 @@ def read_destination_rows(base_dir: Path, table_name: str) -> list[dict[str, Any Handles parquet, jsonl and gzipped jsonl files. Returns an empty list if the table has no files. """ - rows: list[dict[str, Any]] = [] - for data_file in sorted(base_dir.glob(f"**/{table_name}/*")): - if data_file.name.endswith(".parquet"): - rows.extend(pq.read_table(data_file).to_pylist()) - elif ".jsonl" in data_file.name: - if data_file.name.endswith(GZIP_SUFFIX): - with gzip.open(data_file, "rt") as f: - content = f.read() - else: - content = data_file.read_text() - rows.extend(json.loads(line) for line in content.splitlines() if line) - return [{key: value for key, value in row.items() if not key.startswith("_dlt")} for row in rows] + return _read_destination_rows(base_dir, table_name) def sort_rows(rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 825abffe..47661346 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -1,7 +1,5 @@ """Unified test execution framework for running CDM data ingestion pipelines in core, cli, or docker mode.""" -import gzip -import json import logging import os import shutil @@ -27,7 +25,7 @@ from cdm_data_loaders.pipelines import core from cdm_data_loaders.pipelines.core import LOAD_INFO_TABLE_NAME, PipelineCli from cdm_data_loaders.utils import cdm_logger -from tests.cdm_data_loaders.pipelines.helpers import settings_to_argv +from tests.cdm_data_loaders.pipelines.helpers import read_jsonl_file, settings_to_argv from tests.conftest import BASE_DIR, CONFIG_BUCKET, LOCAL_FS DEFAULT_DOCKER_IMAGE_TAG: Final[str] = "cdm-data-loaders-test:latest" @@ -264,14 +262,9 @@ def _read_saved_load_info_row(dataset_dir: Path) -> dict[str, Any]: for file_path in sorted(load_info_dir.glob("*.jsonl*")): if not file_path.is_file(): continue - with file_path.open("rb") as fh: - is_gzipped = fh.read(2) == b"\x1f\x8b" - opener = gzip.open if is_gzipped else open - with opener(file_path, "rt", encoding="utf-8") as fh: - for line in fh: - stripped = line.strip() - if stripped: - return json.loads(stripped) + rows = read_jsonl_file(file_path) + if rows: + return rows[0] return {} diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index b8d827be..4c7bf4ed 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -3,7 +3,8 @@ import gzip import json from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from datetime import datetime from logging import Logger, getLogger from pathlib import Path @@ -11,12 +12,16 @@ import dlt import duckdb +import pendulum import pyarrow.parquet as pq from dlt.common.pipeline import LoadInfo +from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.destinations import filesystem from cdm_data_loaders.core.fields import PARQUET, LoaderFileFormatEnum +from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.core import LOAD_INFO_TABLE_NAME +from tests.conftest import LOCAL_FS logger: Logger = getLogger(__name__) @@ -47,6 +52,25 @@ def is_metadata_table(table_name: str) -> bool: return table_name.startswith("_dlt") or table_name == LOAD_INFO_TABLE_NAME +def read_jsonl_file(file_path: Path) -> list[dict[str, Any]]: + """Read a plain or gzipped JSONLines file, detecting gzip by magic bytes.""" + with file_path.open("rb") as f: + is_gzipped = f.read(2) == b"\x1f\x8b" + + opener = gzip.open if is_gzipped else open + rows: list[dict[str, Any]] = [] + with opener(file_path, "rt", encoding="utf-8") as f: + for raw_line in f: + stripped = raw_line.strip() + if not stripped: + continue + try: + rows.append(json.loads(stripped)) + except json.JSONDecodeError: + logger.warning("Skipping unparseable line in %s: %r", file_path, raw_line[:200]) + return rows + + def read_parquet_tables(output_dir: Path) -> dict[str, list[dict[str, Any]]]: """Read all parquet files under a pipeline output dataset dir, grouped by table name.""" tables: dict[str, list[dict[str, Any]]] = defaultdict(list) @@ -71,22 +95,88 @@ def read_pipeline_tables(output_dir: Path, output_format: str | None = None) -> # ignore metadata dirs if is_metadata_table(json_file.parent.name): continue - # Detect gzip by magic bytes rather than relying on the extension, - # in case a .json file is actually gzipped or vice versa. - with json_file.open("rb") as f: - is_gzipped = f.read(2) == b"\x1f\x8b" - - opener = gzip.open if is_gzipped else open - with opener(json_file, "rt", encoding="utf-8") as f: - for raw_line in f: - try: - tables[json_file.parent.name].append(json.loads(raw_line.strip())) - except json.JSONDecodeError: - logger.warning("Skipping unparseable line in %s: %r", json_file, raw_line[:200]) + tables[json_file.parent.name].extend(read_jsonl_file(json_file)) return dict(tables) +def read_destination_rows(base_dir: Path, table_name: str) -> list[dict[str, Any]]: + """Read the rows the filesystem destination wrote for table_name under base_dir, without dlt columns. + + Handles parquet, jsonl and gzipped jsonl files. Returns an empty list if the table has no files. + """ + rows: list[dict[str, Any]] = [] + for data_file in sorted(base_dir.glob(f"**/{table_name}/*")): + if data_file.name.endswith(".parquet"): + rows.extend(pq.read_table(data_file).to_pylist()) + elif ".jsonl" in data_file.name: + rows.extend(read_jsonl_file(data_file)) + return [strip_metadata_columns(row) for row in rows] + + +def make_cts_settings[S: CtsSettings]( + settings_cls: type[S], + base_dir: Path, + **overrides: Any, # noqa: ANN401 +) -> S: + """Create standard input/output directories and log config under base_dir and instantiate settings_cls.""" + input_dir = base_dir / "input" + input_dir.mkdir(parents=True, exist_ok=True) + output_dir = base_dir / "output" + output_dir.mkdir(parents=True, exist_ok=True) + log_config_file = base_dir / "logging.json" + if not log_config_file.exists(): + log_config_file.write_text('{"version": 1}', encoding="utf-8") + + kwargs: dict[str, Any] = { + "input_dir": str(input_dir), + "output_dir": str(output_dir), + "log_config_file": str(log_config_file), + "use_destination": LOCAL_FS, + "dlt_dev_mode": False, + "use_output_dir_for_pipeline_metadata": False, + } + kwargs.update(overrides) + return settings_cls(**kwargs) + + +def make_file_item(file_path: Path, mime_type: str = "application/octet-stream") -> FileItemDict: + """Build a dlt FileItemDict for a local file.""" + return FileItemDict( + { + "file_url": file_path.as_uri(), + "file_name": file_path.name, + "relative_path": file_path.name, + "mime_type": mime_type, + "modification_date": pendulum.now(), + "size_in_bytes": file_path.stat().st_size, + } + ) + + +def fetch_all_duckdb(pipeline: dlt.Pipeline, query: str) -> list[tuple[Any, ...]]: + """Run a query against a pipeline's DuckDB destination and return all rows.""" + with ( + pipeline.sql_client() as client, + client.execute_query(query) as cur, + ): + return list(cur.fetchall()) + + +@contextmanager +def duckdb_reference_from_jsonl(jsonl_path: Path) -> Iterator[duckdb.DuckDBPyConnection]: + """Load a reference JSONL file into an in-memory DuckDB connection with a `reference` table.""" + connection = duckdb.connect() + connection.execute( + "CREATE TABLE reference AS SELECT * FROM read_json_auto(?, sample_size=200)", + [str(jsonl_path)], + ) + try: + yield connection + finally: + connection.close() + + def sorted_json(entries: list[dict[str, Any]]) -> list[str]: """Entries as sorted JSON strings for order-insensitive comparison.""" return sorted(json.dumps(entry, sort_keys=True, default=str) for entry in entries) diff --git a/tests/cdm_data_loaders/pipelines/test_core.py b/tests/cdm_data_loaders/pipelines/test_core.py index 8800aa92..76d59009 100644 --- a/tests/cdm_data_loaders/pipelines/test_core.py +++ b/tests/cdm_data_loaders/pipelines/test_core.py @@ -6,7 +6,6 @@ from collections.abc import Callable, Iterator from dataclasses import FrozenInstanceError from datetime import UTC, datetime -from json import loads from pathlib import Path from typing import Any, Final, cast from unittest.mock import MagicMock, call, patch @@ -39,6 +38,7 @@ write_pipeline_report, ) from tests.cdm_data_loaders.pipelines.conftest import HOOK_ENV_VAR +from tests.cdm_data_loaders.pipelines.helpers import read_destination_rows, read_jsonl_file from tests.conftest import LOCAL_FS, S3 from tests.dlt_config_isolation import dlt_config_unset, isolated_dlt_config @@ -101,20 +101,15 @@ def read_jsonl_rows(base_dir: Path, table_name: str) -> list[dict[str, Any]]: """Read every jsonl row written for table_name anywhere under base_dir.""" rows: list[dict[str, Any]] = [] for jsonl_file in sorted(base_dir.glob(f"**/{table_name}/*.jsonl*")): - if jsonl_file.name.endswith(".gz"): - with gzip.open(jsonl_file, "rt") as f: - content = f.read() - else: - content = jsonl_file.read_text() - rows.extend(loads(record) for record in content.splitlines() if record) + rows.extend(read_jsonl_file(jsonl_file)) return rows def read_output_jsonl_records(base_dir: Path) -> list[dict[str, Any]]: """Read the tiny table records under base_dir, minus dlt's internal columns.""" - records = read_jsonl_rows(base_dir, TINY_TABLE_NAME) + records = read_destination_rows(base_dir, TINY_TABLE_NAME) assert records - return [{key: value for key, value in record.items() if not key.startswith("_dlt")} for record in records] + return records def make_mock_pipeline(save_run_result: object, slack_hook: str | None = None) -> MagicMock: From 5647808d26bea4eb799a597ec40f80ddf83c22e3 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 10:34:21 -0700 Subject: [PATCH 3/6] Consolidate fixtures and remove duplicated functions --- tests/cdm_data_loaders/pipelines/conftest.py | 37 +++-- .../pipelines/jsonlines/conftest.py | 55 ++++---- .../jsonlines/integration/conftest.py | 34 ++--- .../jsonlines/test_jsonschema_pipeline.py | 14 +- .../jsonlines/test_make_validator.py | 18 +-- .../pipelines/ncbi_rest_api/conftest.py | 70 ++++++++++ .../ncbi_rest_api/test_ncbi_rest_api.py | 70 ++-------- .../test_ncbi_rest_api_settings.py | 60 --------- tests/cdm_data_loaders/pipelines/test_core.py | 82 ++++++------ .../pipelines/xml2db/integration/conftest.py | 52 ++------ .../pipelines/xml2db/test_xml2db_pipeline.py | 22 +-- .../pipelines/xml2db/test_xml2db_settings.py | 16 +-- .../pipelines/xmltodict/conftest.py | 22 +-- .../xmltodict/integration/conftest.py | 70 +++------- .../pipelines/xsv/conftest.py | 76 +++-------- .../pipelines/xsv/integration/conftest.py | 126 ------------------ .../xsv/integration/test_xsv_end_to_end.py | 2 +- .../pipelines/xsv/test_xsv_pipeline.py | 3 +- .../pipelines/xsv/test_xsv_settings.py | 67 +--------- tests/jsonschema_fixtures.py | 14 +- 20 files changed, 250 insertions(+), 660 deletions(-) create mode 100644 tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py delete mode 100644 tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index 8c9a242f..c7ad2963 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -26,8 +26,13 @@ PipelineTestRunner, ensure_docker_image, ) -from tests.cdm_data_loaders.pipelines.helpers import read_destination_rows as _read_destination_rows -from tests.conftest import LOCAL_FS, S3 +from tests.cdm_data_loaders.pipelines.helpers import ( + make_cts_settings, +) +from tests.cdm_data_loaders.pipelines.helpers import ( + read_destination_rows as _read_destination_rows, +) +from tests.conftest import LOCAL_FS, REFERENCE_XML_FIXTURE_DIR, S3, _generate_dlt_config DEFAULT_DLT_TABLES = {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", LOAD_INFO_TABLE_NAME} @@ -41,14 +46,10 @@ GZIP_SUFFIX: Final[str] = ".gz" -DLT_CONFIG: Final[dict[str, Any]] = { - "destination": { - LOCAL_FS: {"bucket_url": "/configured/out/", "destination_type": "filesystem"}, - S3: {"bucket_url": "s3://bucket/prefix", "destination_type": "filesystem"}, - }, - f"destination.{LOCAL_FS}.destination_type": "filesystem", - f"destination.{S3}.destination_type": "filesystem", -} +@pytest.fixture(scope="session") +def reference_xml_data_dir() -> Path: + """Path to the uniref fixture directory.""" + return REFERENCE_XML_FIXTURE_DIR def duckdb_pipeline(tmp_path: Path, pipeline_name: str, dataset_name: str, db_name: str | None = None) -> dlt.Pipeline: @@ -72,8 +73,6 @@ def clean_env() -> Iterator[None]: with patch.dict(os.environ): for name in (*HOOK_PART_ENV_VARS, HOOK_ENV_VAR, DISABLE_COMPRESSION_ENV_VAR): os.environ.pop(name, None) - for name in [key for key in os.environ if key.startswith("CDL_")]: - del os.environ[name] os.environ["RUNTIME__DLTHUB_TELEMETRY"] = "false" yield @@ -99,7 +98,7 @@ def mock_init_logger(monkeypatch: pytest.MonkeyPatch) -> MagicMock: """Stop run_cli reconfiguring logging and point dlt.config at a plain dict.""" mock = MagicMock() monkeypatch.setattr(core, "init_logger", mock) - monkeypatch.setattr(dlt, "config", copy.deepcopy(DLT_CONFIG)) + monkeypatch.setattr(dlt, "config", copy.deepcopy(_generate_dlt_config())) return mock @@ -172,21 +171,17 @@ def isolated_run( """ run_counter = count() - def _factory(**overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(**overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 run_index = next(run_counter) - output_dir = tmp_path / f"output_{run_index}" - output_dir.mkdir() - log_config_file = tmp_path / f"logging_{run_index}.conf" - log_config_file.touch() kwargs: dict[str, Any] = { "dataset_name": f"{dataset_prefix}_{run_index}", - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), + "log_config_file": str(tmp_path / f"logging_{run_index}.conf"), + "output_dir": str(tmp_path / f"output_{run_index}"), **base_kwargs, } kwargs.update(overrides) settings_cls = kwargs.pop("settings_cls") - settings = settings_cls(**kwargs) + settings = make_cts_settings(settings_cls, tmp_path, **kwargs) with dlt.config.values({"destination.local_fs.bucket_url": str(settings.output_dir)}): load_info = pipeline_runner.run(run_fn, settings) if pipeline_runner is not None else run_fn(settings) return load_info, settings.output_dir diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py index 6ad3892e..c7bc8c2d 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py @@ -1,5 +1,6 @@ """Shared fixtures for jsonlines pipeline tests.""" +import json from collections.abc import Callable from pathlib import Path from typing import Any @@ -7,6 +8,26 @@ import pytest from cdm_data_loaders.pipelines.jsonlines.settings import JsonlExtractSettings, JsonlPydanticIngestSettings +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings + + +def make_item( + record: object = None, + line_no: int = 1, + raw_record: str | None = None, + parse_error: str | None = None, + source_file: str = "file.jsonl", +) -> dict[str, Any]: + """Build a line record matching stream_jsonl_lines' output shape.""" + if raw_record is None: + raw_record = "" if parse_error is not None and record is None else json.dumps(record, separators=(",", ":")) + return { + "record": record, + "raw_record": raw_record, + "parse_error": parse_error, + "source_file": source_file, + "line_no": line_no, + } @pytest.fixture @@ -16,25 +37,14 @@ def settings_factory( """Return a factory that builds a valid JsonlPydanticIngestSettings, with fields open to override.""" def _factory(**overrides: Any) -> JsonlPydanticIngestSettings: # noqa: ANN401 - log_config_file = tmp_path / "logging.conf" - log_config_file.touch() - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { + defaults: dict[str, Any] = { "dataset_name": "test_dataset", "dlt_dev_mode": True, "entity_models_module": entity_models_module, - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, } - kwargs.update(overrides) - return JsonlPydanticIngestSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(JsonlPydanticIngestSettings, tmp_path, **defaults) return _factory @@ -44,24 +54,13 @@ def extract_settings_factory(tmp_path: Path, dlt_destination_config: str) -> Cal """Return a factory that builds a valid JsonlExtractSettings, with fields open to override.""" def _factory(**overrides: Any) -> JsonlExtractSettings: # noqa: ANN401 - log_config_file = tmp_path / "logging.conf" - log_config_file.touch() - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { + defaults: dict[str, Any] = { "dataset_name": "test_dataset", "dlt_dev_mode": True, - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), "table_name": "widget", "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, } - kwargs.update(overrides) - return JsonlExtractSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(JsonlExtractSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py index 4a021cf1..a352c32f 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py @@ -25,6 +25,7 @@ JsonlJsonschemaIngestSettings, JsonlPydanticIngestSettings, ) +from tests.cdm_data_loaders.pipelines.helpers import duckdb_reference_from_jsonl, make_cts_settings from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models import ENTITY_MODELS from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas import ENTITY_SCHEMAS from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( @@ -81,7 +82,7 @@ def run_reference_pipeline( } runner = isolated_run_factory(dataset_prefix=dataset_prefix, run_fn=run_fn, base_kwargs=base_kwargs) - def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 kwargs = {"input_dir": str(reference_data_dir / chunk_dir), **overrides} return runner(**kwargs) @@ -147,16 +148,8 @@ def reference_dataset(reference_jsonl: Path) -> Generator[duckdb.DuckDBPyConnect read_json_auto infers native STRUCT/STRUCT[] types for the nested content. """ - connection = duckdb.connect() - jsonl_path = str(reference_jsonl) - connection.execute( - "CREATE TABLE reference AS SELECT * FROM read_json_auto(?, sample_size=200)", - [jsonl_path], - ) - try: - yield connection - finally: - connection.close() + with duckdb_reference_from_jsonl(reference_jsonl) as conn: + yield conn @pytest.fixture @@ -177,26 +170,15 @@ def settings_factory( ) -> Callable[..., JsonlPydanticIngestSettings]: """Return a factory that builds a valid JsonlPydanticIngestSettings, with fields open to override.""" - def _factory(**overrides: Any) -> JsonlPydanticIngestSettings: - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { + def _factory(**overrides: Any) -> JsonlPydanticIngestSettings: # noqa: ANN401 + defaults: dict[str, Any] = { "dataset_name": "test_dataset", "dlt_dev_mode": False, "entity_models_module": entity_models_module, - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, } - kwargs.update(overrides) - return JsonlPydanticIngestSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(JsonlPydanticIngestSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py index 769b2523..a956a842 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py @@ -20,17 +20,7 @@ run_jsonlines_pipeline, ) from cdm_data_loaders.pipelines.jsonlines.settings import JsonlJsonschemaIngestSettings - - -def make_item(record: object, line_no: int = 1) -> dict[str, Any]: - """Build the shared reader's parsed-line envelope.""" - return { - "record": record, - "raw_record": json.dumps(record, separators=(",", ":")), - "parse_error": None, - "source_file": "data.jsonl", - "line_no": line_no, - } +from tests.cdm_data_loaders.pipelines.jsonlines.conftest import make_item def make_router(schema: dict[str, Any], buffer_size: int = 2) -> Callable[[list[dict[str, Any]]], Any]: @@ -187,7 +177,7 @@ def test_make_page_router_fail_reports_exact_errors( "widget_rejected", [ { - "source_file": "data.jsonl", + "source_file": "file.jsonl", "line_no": 7, "raw_record": item["raw_record"], "error_detail": json.dumps(messages), diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py index 4bf6ae42..d66254c5 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py @@ -9,23 +9,7 @@ from pydantic import ValidationError from cdm_data_loaders.pipelines.jsonlines.base import make_page_router - - -def make_item( - record: dict[str, Any] | None = None, - raw_record: str = "", - parse_error: str | None = None, - source_file: str = "file.jsonl", - line_no: int = 1, -) -> dict[str, Any]: - """Build a line record matching stream_jsonl_lines' output shape.""" - return { - "record": record, - "raw_record": raw_record, - "parse_error": parse_error, - "source_file": source_file, - "line_no": line_no, - } +from tests.cdm_data_loaders.pipelines.jsonlines.conftest import make_item def run_validator(validator: Callable[[list[dict[str, Any]]], Any], item: dict[str, Any]) -> list[DataItemWithMeta]: diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py new file mode 100644 index 00000000..e13dfef1 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py @@ -0,0 +1,70 @@ +"""Shared fixtures and constants for NCBI REST API pipeline tests.""" + +from typing import Any + +import pytest + +from cdm_data_loaders.pipelines import ncbi_rest_api as ncbi_module +from cdm_data_loaders.pipelines.ncbi_rest_api import NcbiRestApiSettings +from tests.conftest import DEFAULT_VCR_CONFIG + +ID_WITH_2K_ANNOTS = "GCF_000003135.1" +ID_WITH_500_ANNOTS = "GCF_000007725.1" +ID_TRIGGERS_500_ERR = "GCF_500_ERROR" +VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] +INVALID_ID = "invalid_id" +ALL_IDS = [*VALID_IDS, INVALID_ID] + + +@pytest.fixture(autouse=True) +def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: + """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" + monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) + monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.ncbi_genome_client._config.http_show_error_body", True) + + +@pytest.fixture(scope="module") +def vcr_config() -> dict[str, Any]: + """VCR config for tests that make HTTP requests.""" + return {**DEFAULT_VCR_CONFIG} + + +@pytest.fixture(scope="module") +def valid_assembly_ids() -> list[str]: + """A list of assembly IDs.""" + return VALID_IDS + + +@pytest.fixture(scope="module") +def assembly_id(valid_assembly_ids: list[str]) -> str: + """Single valid assembly ID.""" + return valid_assembly_ids[0] + + +@pytest.fixture(scope="module") +def invalid_assembly_id() -> str: + """Invalid assembly ID.""" + return INVALID_ID + + +@pytest.fixture(scope="module") +def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> list[str]: + """List of assembly IDs including both valid and invalid IDs.""" + return [*valid_assembly_ids, invalid_assembly_id] + + +@pytest.fixture +def test_settings(test_defaults: dict[str, Any]) -> NcbiRestApiSettings: + """Generate a test settings class.""" + return NcbiRestApiSettings(**test_defaults) + + +@pytest.fixture(autouse=True) +def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: + """Reset the pipeline settings prior to running tests. + + :param monkeypatch: monkeypatch + :type monkeypatch: pytest.MonkeyPatch + """ + monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) + diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py index 0c35d0dc..cd627ece 100644 --- a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py @@ -28,67 +28,15 @@ set_settings, ) from tests.cdm_data_loaders.conftest import check_settings -from tests.conftest import DEFAULT_VCR_CONFIG, LOCAL_FS - - -@pytest.fixture(autouse=True) -def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: - """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" - monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) - - -ID_WITH_2K_ANNOTS = "GCF_000003135.1" -ID_WITH_500_ANNOTS = "GCF_000007725.1" -ID_TRIGGERS_500_ERR = "GCF_500_ERROR" -VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] -INVALID_ID = "invalid_id" -ALL_IDS = [*VALID_IDS, INVALID_ID] - - -@pytest.fixture(scope="module") -def vcr_config() -> dict[str, Any]: - """VCR config for tests that make HTTP requests.""" - return {**DEFAULT_VCR_CONFIG} - - -@pytest.fixture(scope="module") -def valid_assembly_ids() -> list[str]: - """A list of assembly IDs.""" - return VALID_IDS - - -@pytest.fixture(scope="module") -def assembly_id(valid_assembly_ids: list[str]) -> str: - """Single valid assembly ID.""" - return valid_assembly_ids[0] - - -@pytest.fixture(scope="module") -def invalid_assembly_id() -> str: - """Invalid assembly ID.""" - return INVALID_ID - - -@pytest.fixture(scope="module") -def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> list[str]: - """List of assembly IDs including both valid and invalid IDs.""" - return [*valid_assembly_ids, invalid_assembly_id] - - -@pytest.fixture -def test_settings(test_defaults: dict[str, Any]) -> NcbiRestApiSettings: - """Generate a test settings class.""" - return NcbiRestApiSettings(**test_defaults) - - -@pytest.fixture(autouse=True) -def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: - """Reset the pipeline settings prior to running tests. - - :param monkeypatch: monkeypatch - :type monkeypatch: pytest.MonkeyPatch - """ - monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) +from tests.cdm_data_loaders.pipelines.ncbi_rest_api.conftest import ( + ALL_IDS, + ID_TRIGGERS_500_ERR, + ID_WITH_2K_ANNOTS, + ID_WITH_500_ANNOTS, + INVALID_ID, + VALID_IDS, +) +from tests.conftest import LOCAL_FS def test_cli_passes_settings_class_to_run_cli() -> None: diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py index 24e95fab..c8552e18 100644 --- a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py @@ -19,22 +19,8 @@ set_settings, ) from tests.cdm_data_loaders.conftest import check_settings -from tests.conftest import DEFAULT_VCR_CONFIG from tests.helpers import assert_cli_field_roundtrips, assert_no_cli_clashes - -@pytest.fixture(autouse=True) -def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: - """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" - monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) - - -ID_WITH_2K_ANNOTS = "GCF_000003135.1" -ID_WITH_500_ANNOTS = "GCF_000007725.1" -ID_TRIGGERS_500_ERR = "GCF_500_ERROR" -VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] -INVALID_ID = "invalid_id" -ALL_IDS = [*VALID_IDS, INVALID_ID] BATCH_SIZE = 200 BATCH_SIZE_STRING = str(BATCH_SIZE) @@ -68,52 +54,6 @@ def ncbi_settings_pair( return settings, reconciled -@pytest.fixture(scope="module") -def vcr_config() -> dict[str, Any]: - """VCR config for tests that make HTTP requests.""" - return {**DEFAULT_VCR_CONFIG} - - -@pytest.fixture(scope="module") -def valid_assembly_ids() -> list[str]: - """A list of assembly IDs.""" - return VALID_IDS - - -@pytest.fixture(scope="module") -def assembly_id(valid_assembly_ids: list[str]) -> str: - """Single valid assembly ID.""" - return valid_assembly_ids[0] - - -@pytest.fixture(scope="module") -def invalid_assembly_id() -> str: - """Invalid assembly ID.""" - return INVALID_ID - - -@pytest.fixture(scope="module") -def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> list[str]: - """List of assembly IDs including both valid and invalid IDs.""" - return [*valid_assembly_ids, invalid_assembly_id] - - -@pytest.fixture -def test_settings(test_defaults: dict[str, Any]) -> NcbiRestApiSettings: - """Generate a test settings class.""" - return NcbiRestApiSettings(**test_defaults) - - -@pytest.fixture(autouse=True) -def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: - """Reset the pipeline settings prior to running tests. - - :param monkeypatch: monkeypatch - :type monkeypatch: pytest.MonkeyPatch - """ - monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) - - def test_get_set_settings(test_settings: NcbiRestApiSettings) -> None: """Test setting and retrieval of settings.""" assert ncbi_module.PIPELINE_SETTINGS is None diff --git a/tests/cdm_data_loaders/pipelines/test_core.py b/tests/cdm_data_loaders/pipelines/test_core.py index 76d59009..07f221e7 100644 --- a/tests/cdm_data_loaders/pipelines/test_core.py +++ b/tests/cdm_data_loaders/pipelines/test_core.py @@ -62,12 +62,6 @@ def make_cts_settings(**kwargs: str | int | bool | Path) -> CtsSettings: return CtsSettings.model_validate(kwargs) -@pytest.fixture -def bucket_dir(tmp_path: Path, dlt_destination_config: str) -> Path: # noqa: ARG001 - """The bucket_url of the local_fs destination in dlt.config, where run_pipeline loads data.""" - return tmp_path / "bucket" - - @pytest.fixture def dlt_test_settings(tmp_path: Path, dlt_destination_config: str) -> CtsSettings: """Settings that load into the local_fs destination and keep pipeline metadata in a separate output_dir.""" @@ -476,13 +470,13 @@ def test_run_pipeline_no_slack_env_var_when_vars_missing( # run_pipeline tests -def test_run_pipeline_minimal(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_minimal(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """A tiny pipeline loads its records into the destination bucket, not into output_dir.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA assert read_jsonl_rows(Path(dlt_test_settings.output_dir), TINY_TABLE_NAME) == [] @@ -605,7 +599,7 @@ def test_run_pipeline_pipelines_dir_used_for_pipeline_metadata(dlt_test_settings assert any(p.name == "state.json" for p in pipeline_state_files) -def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """dev_mode=True is forwarded to dlt.pipeline(); the pipeline loads into a suffixed dataset.""" settings = dlt_test_settings.model_copy(update={"dlt_dev_mode": True}) @@ -616,7 +610,7 @@ def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: # dataset_name is `dlt_pipeline_dataset_\d+`, where \d+ is YYYYMMDDHHMMSS assert load_info.dataset_name.startswith(f"{load_info.pipeline.pipeline_name}_dataset_") assert load_info.dataset_name != f"{load_info.pipeline.pipeline_name}_dataset" - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA @pytest.mark.parametrize( @@ -630,7 +624,7 @@ def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: ) def test_run_pipeline_passes_kwargs_to_real_pipeline( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, pipeline_kwargs: dict[str, Any], pipeline_run_kwargs: dict[str, Any], ) -> None: @@ -647,17 +641,17 @@ def test_run_pipeline_passes_kwargs_to_real_pipeline( if pipeline_kwargs: assert load_info.pipeline.pipeline_name == TINY_PIPELINE_NAME assert load_info.dataset_name == "core_test_dataset" - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA # load_info saved as part of the dataset -def test_run_pipeline_saves_load_info_to_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_saves_load_info_to_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """The pipeline's load_info is saved as a table in the same dataset.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 row = rows[0] assert row["dataset_name"] == load_info.dataset_name @@ -669,26 +663,26 @@ def test_run_pipeline_saves_load_info_to_dataset(dlt_test_settings: CtsSettings, assert set(row["pipeline"]) == {"pipeline_name"} -def test_run_pipeline_load_info_saved_to_same_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_saved_to_same_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """The load info table lands in the same dataset directory as the resource tables.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None - dataset_dir = bucket_dir / load_info.dataset_name + dataset_dir = destination_bucket / load_info.dataset_name assert (dataset_dir / TINY_TABLE_NAME).is_dir() assert (dataset_dir / LOAD_INFO_TABLE_NAME).is_dir() - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA - assert read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA + assert read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) -def test_run_pipeline_load_info_flat_structure(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_flat_structure(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """max_table_nesting=0 keeps nested structures as json columns; no child tables.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None - dataset_dir = bucket_dir / load_info.dataset_name + dataset_dir = destination_bucket / load_info.dataset_name assert list(dataset_dir.glob("_dlt_load_info__*")) == [] - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 row = rows[0] for nested in ("load_packages", "outputs", "job_metrics"): @@ -697,7 +691,7 @@ def test_run_pipeline_load_info_flat_structure(dlt_test_settings: CtsSettings, b assert all(isinstance(item, dict) for item in row[nested]) -def test_run_pipeline_load_info_runs_accumulate(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_runs_accumulate(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """Each pipeline run appends its own load_info row; earlier rows are preserved.""" first = run_pipeline(dlt_test_settings, tiny_resource()) assert first is not None @@ -705,30 +699,30 @@ def test_run_pipeline_load_info_runs_accumulate(dlt_test_settings: CtsSettings, second = run_pipeline(dlt_test_settings, tiny_resource()) assert second is not None - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 2 saved_load_ids = {tuple(row["loads_ids"]) for row in rows} assert len(saved_load_ids) == 2 # each accumulated row references one of the entity data loads - tiny_load_ids = {row["_dlt_load_id"] for row in read_jsonl_rows(bucket_dir, TINY_TABLE_NAME)} + tiny_load_ids = {row["_dlt_load_id"] for row in read_jsonl_rows(destination_bucket, TINY_TABLE_NAME)} assert saved_load_ids == {(load_id,) for load_id in tiny_load_ids} -def test_run_pipeline_load_info_in_dev_mode(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_in_dev_mode(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """dev_mode runs save the load_info into the same suffixed dataset as the resource data.""" settings = dlt_test_settings.model_copy(update={"dlt_dev_mode": True}) load_info = run_pipeline(settings, tiny_resource()) assert load_info is not None assert load_info.dataset_name != "dlt_pipeline_dataset" - dataset_dir = bucket_dir / load_info.dataset_name + dataset_dir = destination_bucket / load_info.dataset_name assert (dataset_dir / TINY_TABLE_NAME).is_dir() - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 assert rows[0]["dataset_name"] == load_info.dataset_name -def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """A named pipeline/dataset save lands in the named dataset with a load_info row.""" load_info = run_pipeline( dlt_test_settings, @@ -737,22 +731,22 @@ def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: Ct ) assert load_info is not None - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 assert rows[0]["dataset_name"] == "core_test_dataset" assert rows[0]["pipeline"] == {"pipeline_name": TINY_PIPELINE_NAME} - assert (bucket_dir / "core_test_dataset" / LOAD_INFO_TABLE_NAME).is_dir() + assert (destination_bucket / "core_test_dataset" / LOAD_INFO_TABLE_NAME).is_dir() def test_run_pipeline_load_info_returns_original_import_result( - dlt_test_settings: CtsSettings, bucket_dir: Path + dlt_test_settings: CtsSettings, destination_bucket: Path ) -> None: """run_pipeline returns the original resource load; the saved row references it, not the save load.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 # loads_ids is the original payload; _dlt_load_id is the save run's own load id assert rows[0]["loads_ids"] != [rows[0]["_dlt_load_id"]] @@ -762,7 +756,7 @@ def test_run_pipeline_load_info_returns_original_import_result( def test_run_pipeline_load_info_save_resource_error_still_returns_load_info( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: @@ -781,8 +775,8 @@ def resource_boom(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 assert load_info is not None assert not load_info.has_failed_jobs - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA - assert read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) == [] + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA + assert read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) == [] error_messages = [record.getMessage() for record in caplog.records if record.levelno == logging.ERROR] assert error_messages == [f"{LOAD_INFO_SAVE_FAILED}load info save failed"] assert ALL_JOBS_COMPLETED in caplog.messages @@ -1026,7 +1020,7 @@ def test_run_pipeline_pass_failed_jobs_reported( def test_run_pipeline_pass_partial_load_worker_failure_filesystem( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, monkeypatch: pytest.MonkeyPatch, slack_mock: MagicMock, caplog: pytest.LogCaptureFixture, @@ -1060,15 +1054,15 @@ def fail_second_tiny_record(self: FilesystemLoadJob) -> None: ] # The record from the succeeding worker thread is in the tiny table; the failed worker's record is missing - assert read_output_jsonl_records(bucket_dir) == [TINY_RESOURCE_DATA[0]] + assert read_output_jsonl_records(destination_bucket) == [TINY_RESOURCE_DATA[0]] # Both the partially-failed data load and the _load_info metadata load are recorded in _dlt_loads as status 0 - dlt_loads = read_jsonl_rows(bucket_dir, "_dlt_loads") + dlt_loads = read_jsonl_rows(destination_bucket, "_dlt_loads") assert [row["status"] for row in dlt_loads] == [0, 0] - assert len(read_jsonl_rows(bucket_dir, "_dlt_pipeline_state")) == 1 + assert len(read_jsonl_rows(destination_bucket, "_dlt_pipeline_state")) == 1 # _load_info is saved to the filesystem destination and records both completed and failed worker jobs - load_info_rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + load_info_rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(load_info_rows) == 1 package = load_info_rows[0]["load_packages"][0] assert package["state"] == "loaded" @@ -1109,7 +1103,7 @@ def fail_second_tiny_record(self: FilesystemLoadJob) -> None: def test_run_pipeline_fail_partial_extract_worker_failure_filesystem( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, monkeypatch: pytest.MonkeyPatch, slack_mock: MagicMock, caplog: pytest.LogCaptureFixture, @@ -1136,9 +1130,9 @@ def parallel_transform(item: dict[str, Any]) -> dict[str, Any]: ) assert load_info is None - assert list(bucket_dir.iterdir()) == [] - assert read_jsonl_rows(bucket_dir, TINY_TABLE_NAME) == [] - assert read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) == [] + assert list(destination_bucket.iterdir()) == [] + assert read_jsonl_rows(destination_bucket, TINY_TABLE_NAME) == [] + assert read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) == [] assert caplog.records[-1].levelno == logging.ERROR assert caplog.records[-1].getMessage().startswith("Pipeline failed: ") assert SINK_ERROR in caplog.records[-1].getMessage() diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py index 65c1973d..34b09b33 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py @@ -10,10 +10,8 @@ from cdm_data_loaders.pipelines.xml2db.pipeline import run_xml2db_pipeline from cdm_data_loaders.pipelines.xml2db.settings import Xml2DbSettings -from tests.conftest import ( - REFERENCE_XML_FIXTURE_DIR, - REFERENCE_XSD, -) +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings +from tests.conftest import REFERENCE_XSD from tests.xml_samples import LIBRARY_XSD DEFAULT_XML2DB_SETTINGS: frozendict = frozendict( @@ -35,50 +33,26 @@ def library_xsd(tmp_path: Path) -> Path: return xsd_path -@pytest.fixture(scope="session") -def reference_xml_data_dir() -> Path: - """Path to the uniref fixture directory (shared with the xmltodict reference tests).""" - return REFERENCE_XML_FIXTURE_DIR - - @pytest.fixture(scope="session") def reference_xsd() -> Path: """Path to the uniref.xsd schema.""" return REFERENCE_XSD -def make_settings( - tmp_path: Path, dlt_destination_config: str, xsd_file: str | Path, **overrides: Any -) -> dict[str, Any]: - """Build a valid xml2db settings dict, with fields open to override.""" - log_config_file = overrides.get("log_config_file", tmp_path / "logging.conf") - log_config_file.touch() - output_dir = overrides.get("output_dir", tmp_path / "bucket") - output_dir.mkdir(exist_ok=True) - input_dir = overrides.get("input_dir", tmp_path / "input") - - kwargs: dict[str, Any] = { - **DEFAULT_XML2DB_SETTINGS, - "dataset_name": "xml2db_test_dataset", - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), - "use_destination": dlt_destination_config, - "xsd_file": str(xsd_file), - } - kwargs.update(overrides) - return kwargs - - @pytest.fixture def settings_factory(tmp_path: Path, dlt_destination_config: str, library_xsd: Path) -> Callable[..., Xml2DbSettings]: """Return a factory that builds a valid Xml2DbSettings using the library.xsd schema.""" - def _factory(**overrides: Any) -> Xml2DbSettings: - settings_dict = make_settings(tmp_path, dlt_destination_config, xsd_file=library_xsd, **overrides) - input_dir = Path(settings_dict["input_dir"]) - input_dir.mkdir(exist_ok=True) - return Xml2DbSettings(**settings_dict) + def _factory(**overrides: Any) -> Xml2DbSettings: # noqa: ANN401 + defaults: dict[str, Any] = { + **DEFAULT_XML2DB_SETTINGS, + "dataset_name": "xml2db_test_dataset", + "output_dir": str(tmp_path / "bucket"), + "use_destination": dlt_destination_config, + "xsd_file": str(library_xsd), + } + defaults.update(overrides) + return make_cts_settings(Xml2DbSettings, tmp_path, **defaults) return _factory @@ -114,7 +88,7 @@ def run_xml2db_pipeline_fxt( }, ) - def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 return run(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py index b273474f..09fa3859 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py @@ -20,6 +20,7 @@ from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from cdm_data_loaders.readers.xml2db_doc import build_xml2db_model from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings SIMPLE_LIBRARY_XML = """ @@ -82,28 +83,13 @@ def fake_settings( **overrides: Any, # noqa: ANN401 ) -> Xml2DbSettings: """Build a valid Xml2DbSettings with fields open to override.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}', encoding="utf-8") - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { - "buffer_size": 100, - "log_interval": 1000, + defaults: dict[str, Any] = { "dataset_name": "xml2db_test_dataset", "loader_file_format": "parquet", - "log_config_file": str(log_config_file), - "input_dir": str(input_dir), - "output_dir": str(output_dir), - "dlt_dev_mode": False, - "use_destination": "local_fs", - "use_output_dir_for_pipeline_metadata": False, "xsd_file": str(xsd_file), } - kwargs.update(overrides) - return Xml2DbSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(Xml2DbSettings, tmp_path, **defaults) # _read_items diff --git a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py index 972adc6e..5ee2981b 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py @@ -15,6 +15,7 @@ PIPELINE_NAME, Xml2DbSettings, ) +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings from tests.conftest import S3 MINIMAL_SETTINGS_KWARGS: dict[str, object] = { @@ -44,23 +45,16 @@ def settings_factory(tmp_path: Path, test_data_dir: Path) -> Callable[..., Xml2D """Return a factory that builds a valid Xml2DbSettings with fields open to override.""" def _factory(**overrides: object) -> Xml2DbSettings: - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}', encoding="utf-8") input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) xsd_file = _write_xsd_fixture(input_dir, test_data_dir, "uniref.xsd") - - kwargs: dict[str, object] = { + defaults: dict[str, object] = { **MINIMAL_SETTINGS_KWARGS, - "log_config_file": str(log_config_file), "input_dir": str(input_dir), - "output_dir": str(output_dir), + "output_dir": str(tmp_path / "output"), "xsd_file": str(input_dir / xsd_file), } - kwargs.update(overrides) - return Xml2DbSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(Xml2DbSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py index dfe75e69..dfa561bc 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py @@ -7,7 +7,7 @@ import pytest from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings -from tests.conftest import LOCAL_FS +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings @pytest.fixture @@ -15,28 +15,14 @@ def settings_factory(tmp_path: Path) -> Callable[..., XmlToDictSettings]: """Return a factory that builds a valid XmlToDictSettings, with fields open to override.""" def _factory(**overrides: Any) -> XmlToDictSettings: # noqa: ANN401 - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { - "log_config_file": str(log_config_file), - "input_dir": str(input_dir), - "output_dir": str(output_dir), + defaults: dict[str, Any] = { "dlt_dev_mode": True, - "use_destination": LOCAL_FS, - "use_output_dir_for_pipeline_metadata": False, - "buffer_size": 100, - "log_interval": 1000, "dataset_name": "xml_test_dataset", "table_name": "book", "xml_tag": "book", "file_glob": "*.xml*", } - kwargs.update(overrides) - return XmlToDictSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(XmlToDictSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py index 583017c7..1a47b253 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py @@ -2,7 +2,7 @@ import gzip import json -from collections.abc import Callable +from collections.abc import Callable, Generator from pathlib import Path from typing import Any @@ -15,7 +15,12 @@ from cdm_data_loaders.pipelines.xmltodict.pipeline import run_xmltodict_pipeline from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings from cdm_data_loaders.readers.xml import DEFAULT_XMLTODICT_ARGS -from tests.conftest import N_REFERENCE_XML_ENTRIES, REFERENCE_XML_FIXTURE_DIR, REFERENCE_XML_TAG +from tests.cdm_data_loaders.pipelines.helpers import ( + duckdb_reference_from_jsonl, + make_cts_settings, + read_jsonl_file, +) +from tests.conftest import N_REFERENCE_XML_ENTRIES, REFERENCE_XML_TAG DEFAULT_XMLTODICT_SETTINGS: frozendict = frozendict( { @@ -28,39 +33,20 @@ ) -def make_settings(tmp_path: Path, dlt_destination_config: str, **overrides: Any) -> dict[str, Any]: - """Generate a dict of settings for the XmlToDict ingestion route.""" - log_config_file = overrides.get("log_config_file", tmp_path / "logging.conf") - log_config_file.touch() - output_dir = overrides.get("output_dir", tmp_path / "output") - output_dir.mkdir(exist_ok=True) - input_dir = overrides.get("input_dir", tmp_path / "input") - - kwargs: dict[str, Any] = { - **DEFAULT_XMLTODICT_SETTINGS, - "dataset_name": "xml_test_dataset", - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), - "table_name": "book", - "use_destination": dlt_destination_config, - "xml_tag": "book", - } - kwargs.update(overrides) - return kwargs - - @pytest.fixture def settings_factory(tmp_path: Path, dlt_destination_config: str) -> Callable[..., XmlToDictSettings]: """Return a factory that builds a valid XmlToDictSettings, with fields open to override.""" - def _factory(**overrides: Any) -> XmlToDictSettings: - settings_dict = make_settings(tmp_path, dlt_destination_config, **overrides) - # ensure that the input dir exists - input_dir = Path(settings_dict["input_dir"]) - input_dir.mkdir(exist_ok=True) - - return XmlToDictSettings(**settings_dict) + def _factory(**overrides: Any) -> XmlToDictSettings: # noqa: ANN401 + defaults: dict[str, Any] = { + **DEFAULT_XMLTODICT_SETTINGS, + "dataset_name": "xml_test_dataset", + "table_name": "book", + "use_destination": dlt_destination_config, + "xml_tag": "book", + } + defaults.update(overrides) + return make_cts_settings(XmlToDictSettings, tmp_path, **defaults) return _factory @@ -77,16 +63,9 @@ def _resolve(scenario: str) -> str: return _resolve -@pytest.fixture(scope="session") -def reference_xml_data_dir() -> Path: - """Path to the uniref fixture directory.""" - return REFERENCE_XML_FIXTURE_DIR - - def import_jsonl_data(jsonl_path: Path) -> list[dict[str, Any]]: """Read the reference JSONL into a list of canonical nested entry dicts.""" - with jsonl_path.open(encoding="utf-8") as fh: - return [json.loads(line) for line in fh if line.strip()] + return read_jsonl_file(jsonl_path) def write_reference_xml_as_jsonl(source_xml: Path, jsonl_path: Path) -> int: @@ -140,19 +119,14 @@ def sorted_reference_xml_entries(reference_xml_entries: list[dict[str, Any]]) -> @pytest.fixture(scope="session") -def reference_xml_dataset(reference_xml_as_jsonl: Path) -> duckdb.DuckDBPyConnection: +def reference_xml_dataset(reference_xml_as_jsonl: Path) -> Generator[duckdb.DuckDBPyConnection]: """Load the reference JSONL into an in-memory DuckDB connection with nested structures. read_json_auto infers native STRUCT/STRUCT[] types for the nested content; `member` infers as JSON because its shape varies across entries. """ - connection = duckdb.connect() - jsonl_path = str(reference_xml_as_jsonl) - connection.execute( - "CREATE TABLE reference AS SELECT * FROM read_json_auto(?, sample_size=200)", - [jsonl_path], - ) - return connection + with duckdb_reference_from_jsonl(reference_xml_as_jsonl) as conn: + yield conn @pytest.fixture @@ -179,7 +153,7 @@ def run_xmltodict_pipeline_fxt( base_kwargs=base_kwargs, ) - def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 return runner(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xsv/conftest.py b/tests/cdm_data_loaders/pipelines/xsv/conftest.py index 7be34c86..bcd9ac69 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xsv/conftest.py @@ -12,6 +12,7 @@ from cdm_data_loaders.pipelines.xsv.pipeline import XsvWorkPaths from cdm_data_loaders.pipelines.xsv.settings import XsvIngestSettings from cdm_data_loaders.readers.jsonschema_xsv.xsv_validator.schema_utils import generate_header +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings VALID_SCHEMA_URI: Final[str] = "https://json-schema.org/draft/2020-12/schema" COLUMNS: Final[list[str]] = ["number", "date", "float", "boolean", "string"] @@ -71,71 +72,34 @@ def qsv_cmd() -> str: return cmd -def make_settings( - tmp_path: Path, - dlt_destination_config: str, - schema: dict[str, Any], - **overrides: Any, -) -> dict[str, Any]: - """Build a valid xsv ingest settings dict, with fields open to override.""" - log_config_file = overrides.pop("log_config_file", tmp_path / "logging.conf") - Path(log_config_file).touch(exist_ok=True) - output_dir = overrides.pop("output_dir", tmp_path / "output") - Path(output_dir).mkdir(exist_ok=True) - input_dir = overrides.pop("input_dir", tmp_path / "input") - Path(input_dir).mkdir(exist_ok=True) - (Path(input_dir) / "schema.json").write_text(json.dumps(schema)) - - kwargs: dict[str, Any] = { - "buffer_size": 10, - "dataset_name": "xsv_test_dataset", - "dlt_dev_mode": False, - "file_glob": "*", - "input_dir": str(input_dir), - "loader_file_format": "jsonl", - "log_config_file": str(log_config_file), - "log_interval": 1000, - "output_dir": str(output_dir), - "schema_file": str(Path(input_dir) / "schema.json"), - "table_name": "my_table", - "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, - } - kwargs.update(overrides) - return kwargs - - @pytest.fixture -def settings_factory(tmp_path: Path) -> Callable[..., XsvIngestSettings]: +def settings_factory(tmp_path: Path, dlt_destination_config: str) -> Callable[..., XsvIngestSettings]: """Return a factory that builds a valid XsvIngestSettings, with fields open to override.""" - def _factory(schema: dict[str, Any], data_files: dict[str, str], **overrides: object) -> XsvIngestSettings: - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - (input_dir / "schema.json").write_text(json.dumps(schema)) - for file_name, content in data_files.items(): - (input_dir / file_name).write_text(content) - - kwargs: dict[str, object] = { - "buffer_size": 100, + def _factory( + schema: dict[str, Any] | None = DEFAULT_XSV_SCHEMA, + data_files: dict[str, str] | None = None, + **overrides: Any, # noqa: ANN401 + ) -> XsvIngestSettings: + input_dir = Path(overrides.get("input_dir", tmp_path / "input")) + input_dir.mkdir(parents=True, exist_ok=True) + + if schema is not None: + (input_dir / "schema.json").write_text(json.dumps(schema), encoding="utf-8") + if data_files: + for file_name, content in data_files.items(): + (input_dir / file_name).write_text(content, encoding="utf-8") + + defaults: dict[str, Any] = { "dataset_name": "xsv_test_dataset", - "dlt_dev_mode": False, "file_glob": "*", - "input_dir": str(input_dir), "loader_file_format": "jsonl", - "log_config_file": None, - "log_interval": 1000, - "output_dir": str(output_dir), "schema_file": str(input_dir / "schema.json"), "table_name": "my_table", - "use_destination": "local_fs", - "use_output_dir_for_pipeline_metadata": False, + "use_destination": dlt_destination_config, } - kwargs.update(overrides) - return XsvIngestSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(XsvIngestSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py deleted file mode 100644 index bd960c93..00000000 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Shared fixtures for xsv pipeline integration tests.""" - -import json -import os -import shutil -from collections.abc import Callable -from pathlib import Path -from typing import Any, Final - -import pytest - -from cdm_data_loaders.pipelines.xsv.settings import XsvIngestSettings - -VALID_SCHEMA_URI: Final[str] = "https://json-schema.org/draft/2020-12/schema" -COLUMNS: Final[list[str]] = ["number", "date", "float", "boolean", "string"] - -# The `float` column is deliberately typed `["string", "null"]`, not `"number"`, matching the -# convention used throughout the xsv_validator test fixtures: it lets a field hold either a -# numeric-looking string or a null placeholder without needing float-parsing semantics. -DEFAULT_XSV_SCHEMA: Final[dict[str, Any]] = { - "$schema": VALID_SCHEMA_URI, - "title": "XSV integration test schema", - "type": "object", - "required": COLUMNS, - "properties": { - "number": {"type": "integer"}, - "date": {"type": "string"}, - "float": {"type": ["string", "null"]}, - "boolean": {"type": "boolean"}, - "string": {"type": "string"}, - }, - "x-xsv-config": {"x-delimiter": ",", "x-has-header": True}, -} - -VALID_ROWS: Final[list[list[str]]] = [ - ["2", "2023-01-15", "3.14", "true", "key:value1"], - ["3", "2023-02-20", "1.11", "false", "key:value2"], -] - -# One ragged row (missing the trailing `string` field) among otherwise-valid rows: recoverable. -PARTIAL_RAGGED_ROWS: Final[list[list[str]]] = [ - ["2", "2023-01-15", "3.14", "true", "key:value1"], - ["3", "2023-02-20", "1.11", "false"], - ["4", "2023-03-25", "2.22", "true", "key:value3"], -] - -# Every row here is ragged (too few or too many fields): unrecoverable. -ALL_RAGGED_ROWS: Final[list[list[str]]] = [ - ["2", "2023-01-15", "3.14", "true"], - ["3", "2023-02-20", "1.11", "false", "key:value2", "extra"], -] - - -def rows_to_csv(rows: list[list[str]], header: list[str] = COLUMNS, delimiter: str = ",") -> str: - """Render rows (plus header) as delimited text content, one row per line.""" - lines = [delimiter.join(header), *(delimiter.join(row) for row in rows)] - return "\n".join(lines) + "\n" - - -@pytest.fixture(scope="session") -def qsv_cmd() -> str: - """Provide the path/name of the qsv binary; override via the QSV_BIN env var if not on PATH. - - Tests using this fixture are skipped when the qsv binary is not available. - """ - cmd = shutil.which("qsv") or os.environ.get("QSV_BIN") - if not cmd or not Path(cmd).is_file(): - pytest.skip(f"qsv binary not found at {cmd!r}; set QSV_BIN to point at a real binary") - return cmd - - -def make_settings( - tmp_path: Path, - dlt_destination_config: str, - schema: dict[str, Any], - **overrides: Any, -) -> dict[str, Any]: - """Build a valid xsv ingest settings dict, with fields open to override.""" - log_config_file = overrides.pop("log_config_file", tmp_path / "logging.conf") - Path(log_config_file).touch(exist_ok=True) - output_dir = overrides.pop("output_dir", tmp_path / "output") - Path(output_dir).mkdir(exist_ok=True) - input_dir = overrides.pop("input_dir", tmp_path / "input") - Path(input_dir).mkdir(exist_ok=True) - (Path(input_dir) / "schema.json").write_text(json.dumps(schema)) - - kwargs: dict[str, Any] = { - "buffer_size": 10, - "dataset_name": "xsv_test_dataset", - "dlt_dev_mode": False, - "file_glob": "*", - "input_dir": str(input_dir), - "loader_file_format": "jsonl", - "log_config_file": str(log_config_file), - "log_interval": 1000, - "output_dir": str(output_dir), - "schema_file": str(Path(input_dir) / "schema.json"), - "table_name": "my_table", - "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, - } - kwargs.update(overrides) - return kwargs - - -@pytest.fixture -def settings_factory(tmp_path: Path, dlt_destination_config: str) -> Callable[..., XsvIngestSettings]: - """Return a factory that builds a valid XsvIngestSettings, with fields open to override.""" - - def _factory(schema: dict[str, Any] = DEFAULT_XSV_SCHEMA, **overrides: Any) -> XsvIngestSettings: - settings_dict = make_settings(tmp_path, dlt_destination_config, schema, **overrides) - return XsvIngestSettings(**settings_dict) - - return _factory - - -@pytest.fixture -def write_xsv_file() -> Callable[[XsvIngestSettings, str, str], Path]: - """Return a function that writes text content to a new file inside settings.input_dir.""" - - def _write(settings: XsvIngestSettings, file_name: str, content: str) -> Path: - path = Path(settings.input_dir) / file_name - path.write_text(content) - return path - - return _write diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py index 425fe9fd..490734ff 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py @@ -11,7 +11,7 @@ from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written -from tests.cdm_data_loaders.pipelines.xsv.integration.conftest import ( +from tests.cdm_data_loaders.pipelines.xsv.conftest import ( ALL_RAGGED_ROWS, COLUMNS, DEFAULT_XSV_SCHEMA, diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py index 964ae24e..acd948ed 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py @@ -26,8 +26,7 @@ run_xsv_pipeline, ) from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings - -VALID_SCHEMA_URI = "https://json-schema.org/draft/2020-12/schema" +from tests.cdm_data_loaders.pipelines.xsv.conftest import VALID_SCHEMA_URI SIMPLE_SCHEMA: dict[str, Any] = { "$schema": VALID_SCHEMA_URI, diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py index a9bff090..e035b9b4 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py @@ -1,71 +1,14 @@ """Tests for XsvIngestSettings field validation, schema loading, and derived properties.""" -import json from collections.abc import Callable -from pathlib import Path -from typing import Any, Final +from typing import Any import pytest from pydantic import ValidationError from cdm_data_loaders.core.fields import DEFAULTS from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings - -VALID_SCHEMA_URI: Final[str] = "https://json-schema.org/draft/2020-12/schema" -COLUMNS: Final[list[str]] = ["number", "date", "float", "boolean", "string"] - -VALID_SCHEMA: Final[dict[str, Any]] = { - "$schema": VALID_SCHEMA_URI, - "title": "test schema", - "type": "object", - "required": COLUMNS, - "properties": { - "number": {"type": "integer"}, - "date": {"type": "string"}, - "float": {"type": ["string", "null"]}, - "boolean": {"type": "boolean"}, - "string": {"type": "string"}, - }, -} - -MINIMAL_SETTINGS_KWARGS: dict[str, object] = { - "buffer_size": 100, - "log_interval": 1000, - "dataset_name": "xsv_test_dataset", - "loader_file_format": "parquet", - "table_name": "my_table", - "log_config_file": None, - "dlt_dev_mode": False, - "use_destination": "local_fs", - "use_output_dir_for_pipeline_metadata": False, -} - - -@pytest.fixture -def settings_factory(tmp_path: Path) -> Callable[..., XsvIngestSettings]: - """Return a factory that builds a valid XsvIngestSettings, with fields open to override.""" - - def _factory(schema: dict[str, Any] | None = VALID_SCHEMA, **overrides: object) -> XsvIngestSettings: - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - if schema is not None: - (input_dir / "schema.json").write_text(json.dumps(schema)) - - kwargs: dict[str, object] = { - **MINIMAL_SETTINGS_KWARGS, - "input_dir": str(input_dir), - "output_dir": str(output_dir), - "file_glob": DEFAULTS["file_glob"], - "schema_file": str(input_dir / "schema.json"), - } - kwargs.update(overrides) - return XsvIngestSettings(**kwargs) - - return _factory - +from tests.cdm_data_loaders.pipelines.xsv.conftest import COLUMNS, DEFAULT_XSV_SCHEMA, VALID_SCHEMA_URI """defaults and basic construction""" @@ -149,7 +92,8 @@ def test_xsv_settings_pass_parsing_config_empty_without_xsv_config_block( settings_factory: Callable[..., XsvIngestSettings], ) -> None: """parsing_config is empty when the schema carries no x-xsv-config block.""" - settings = settings_factory() + schema_without_config = {k: v for k, v in DEFAULT_XSV_SCHEMA.items() if k != "x-xsv-config"} + settings = settings_factory(schema=schema_without_config) assert settings.parsing_config == {} @@ -157,8 +101,7 @@ def test_xsv_settings_pass_parsing_config_resolved_from_xsv_config_block( settings_factory: Callable[..., XsvIngestSettings], ) -> None: """parsing_config resolves qsv kwargs from the schema's x-xsv-config block.""" - schema_with_config = {**VALID_SCHEMA, "x-xsv-config": {"x-delimiter": ",", "x-has-header": True}} - settings = settings_factory(schema=schema_with_config) + settings = settings_factory() assert settings.parsing_config == {"delimiter": ",", "missing_header": False} diff --git a/tests/jsonschema_fixtures.py b/tests/jsonschema_fixtures.py index c8e2abef..bc4f1273 100644 --- a/tests/jsonschema_fixtures.py +++ b/tests/jsonschema_fixtures.py @@ -12,6 +12,7 @@ import pytest from cdm_data_loaders.pipelines.jsonlines.settings import JsonlJsonschemaIngestSettings +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings @pytest.fixture @@ -59,27 +60,20 @@ def schema_settings_factory( def _factory(**overrides: object) -> JsonlJsonschemaIngestSettings: run_index = next(run_counter) - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) output_dir = tmp_path / f"output_{run_index}" output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1, "disable_existing_loggers": false}', encoding="utf-8") - kwargs = { - "input_dir": str(input_dir), + defaults: dict[str, Any] = { "output_dir": str(output_dir), - "log_config_file": str(log_config_file), "use_destination": dlt_destination_config, "use_output_dir_for_pipeline_metadata": True, - "dlt_dev_mode": False, "dataset_name": "schema_test", "file_glob": "*.jsonl*", "loader_file_format": "parquet", "buffer_size": 2, "schema_files_module": schema_module, } - kwargs.update(overrides) - settings = JsonlJsonschemaIngestSettings(**kwargs) + defaults.update(overrides) + settings = make_cts_settings(JsonlJsonschemaIngestSettings, tmp_path, **defaults) dlt.config["destination.local_fs.bucket_url"] = str(settings.output_dir) return settings From ae3fe49ba2b696fb0e72d7f35f445862f5183867 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 10:47:09 -0700 Subject: [PATCH 4/6] remove make_file_item dupes and manual duckdb pipeline boilerplate; remove duped duckdb helpers; standardize helpers --- .../integration/test_jsonlines_end_to_end.py | 51 +++----- .../integration/test_jsonschema_end_to_end.py | 60 +++++----- .../integration/test_reference_end_to_end.py | 11 +- .../jsonlines/test_extract_pipeline.py | 53 ++------- .../uniprot/test_uniprot_end_to_end.py | 30 ++--- .../uniref/test_uniref_end_to_end.py | 32 ++--- .../integration/test_xml2db_end_to_end.py | 52 +++------ .../pipelines/xml2db/test_xml2db_pipeline.py | 67 +++-------- .../integration/test_xmltodict_end_to_end.py | 54 +++------ .../xsv/integration/test_xsv_end_to_end.py | 60 +++------- .../pipelines/xsv/test_xsv_pipeline.py | 110 ++++-------------- 11 files changed, 165 insertions(+), 415 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py index ec0b0e3e..8b62a569 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py @@ -10,7 +10,11 @@ ) from cdm_data_loaders.pipelines.jsonlines.settings import PYDANTIC_PIPELINE_NAME, JsonlPydanticIngestSettings from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) def test_run_jsonlines_pipeline_pass_writes_expected_parquet( @@ -21,13 +25,13 @@ def test_run_jsonlines_pipeline_pass_writes_expected_parquet( """Running the pipeline on a mixed batch puts the correct rows in both output tables.""" settings = settings_factory(input_dir=scenario_input_dir("mixed")) - load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings) - assert load_info is not None + load_info = assert_run_ok(pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings)) dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "c"] assert len(dataset.widget_rejected.df()) == 2 assert_pipeline_report_written(Path(settings.output_dir), load_info) + # FIXME: remove or improve # get stats on tables # dataset.row_counts(table_names=dataset.tables).fetchall() @@ -57,40 +61,19 @@ def test_run_jsonlines_pipeline_pass_table_names_none_processes_every_registered def test_cli_pass_runs_end_to_end_from_command_line_arguments( scenario_input_dir: Callable[[str], str], - dlt_destination_config: str, - entity_models_module: str, - tmp_path: Path, + settings_factory: Callable[..., JsonlPydanticIngestSettings], pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - output_dir = tmp_path / "output" - output_dir.mkdir() - - argv = [ - PYDANTIC_PIPELINE_NAME, - "--input-dir", - scenario_input_dir("happy_path"), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - dlt_destination_config, - "--entity-models-module", - entity_models_module, - "--dataset-name", - "some_cool_dataset", - "--use-output-dir-for-pipeline-metadata", - "true", - ] - - load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, cli_args=argv[1:]) - - assert load_info is not None - assert not load_info.has_failed_jobs + settings = settings_factory( + input_dir=scenario_input_dir("happy_path"), + dataset_name="some_cool_dataset", + dlt_dev_mode=False, + use_output_dir_for_pipeline_metadata=True, + ) + + load_info = assert_run_ok(pipeline_runner.run(PYDANTIC_PIPELINE_NAME, cli_args=settings_to_argv(settings))) assert load_info.pipeline.pipeline_name == PYDANTIC_PIPELINE_NAME dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "b", "c"] - assert_pipeline_report_written(output_dir, load_info, metadata_moved=True) + assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py index dfa873de..c67487e3 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py @@ -13,6 +13,7 @@ from tests.cdm_data_loaders.pipelines.helpers import ( LOADER_FILE_FORMATS, assert_pipeline_report_written, + assert_run_ok, parse_json_string, read_data_rows, ) @@ -64,10 +65,7 @@ def test_run_jsonlines_pipeline_pass_persists_valid_and_rejected_rows( input_dir=str(schema_input_dir), loader_file_format=loader_file_format, buffer_size=buffer_size ) - load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings)) assert load_info.pipeline.pipeline_name == JSONSCHEMA_PIPELINE_NAME assert load_info.dataset_name == "schema_test" assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) @@ -111,10 +109,7 @@ def test_run_jsonlines_pipeline_pass_all_invalid( table_dir.mkdir() (table_dir / "invalid.jsonl").write_text(f"{MALFORMED_RAW}\n{TYPE_ERROR_RAW}", encoding="utf-8") - load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings)) assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) assert read_data_tables(settings, load_info.dataset_name) == { "widget_rejected": [ @@ -138,32 +133,31 @@ def test_cli_pass_loads_selected_entity( """The real CTS CLI accepts a registry shortcut, table selection, and the requested file format.""" settings = schema_settings_factory(input_dir=str(schema_input_dir), loader_file_format=loader_file_format) - load_info = pipeline_runner.run( - JSONSCHEMA_PIPELINE_NAME, - cli_args=[ - "--input-dir", - str(settings.input_dir), - "--output-dir", - str(settings.output_dir), - "--log-config-file", - str(settings.log_config_file), - "--use-destination", - str(settings.use_destination), - "--use-output-dir-for-pipeline-metadata", - "true", - "--loader-file-format", - loader_file_format, - "--dataset-name", - "schema_cli", - "-m", - settings.schema_files_module, - "--table-names", - "gadget", - ], + load_info = assert_run_ok( + pipeline_runner.run( + JSONSCHEMA_PIPELINE_NAME, + cli_args=[ + "--input-dir", + str(settings.input_dir), + "--output-dir", + str(settings.output_dir), + "--log-config-file", + str(settings.log_config_file), + "--use-destination", + str(settings.use_destination), + "--use-output-dir-for-pipeline-metadata", + "true", + "--loader-file-format", + loader_file_format, + "--dataset-name", + "schema_cli", + "-m", + settings.schema_files_module, + "--table-names", + "gadget", + ], + ) ) - - assert load_info is not None - assert not load_info.has_failed_jobs assert load_info.dataset_name == "schema_cli" assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) assert read_data_tables(settings, load_info.dataset_name) == {"gadget": [GADGET_RECORD]} diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 31140384..36ffd705 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -13,7 +13,12 @@ PYDANTIC_PIPELINE_NAME, ) from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, read_pipeline_tables, run_and_read +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + assert_run_ok, + read_pipeline_tables, + run_and_read, +) from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( canonicalize_pipeline_entry, canonicalize_reference_entry, @@ -303,9 +308,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( ] if registry_flag: argv.extend([f"--{registry_flag}", registry_module]) - load_info = pipeline_runner.run(pipeline_kind, cli_args=argv[1:]) - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run(pipeline_kind, cli_args=argv[1:])) tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) assert ( sorted( diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py index 3c7513a1..25ed1c9f 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py @@ -16,7 +16,11 @@ from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import cli, run_jsonlines_pipeline from cdm_data_loaders.pipelines.jsonlines.settings import EXTRACT_PIPELINE_NAME, JsonlExtractSettings from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) SIMPLE_JSONL = """{"widget_id": "a", "count": 1} {"widget_id": "b", "count": 2} @@ -70,10 +74,8 @@ def test_run_jsonlines_pipeline_pass_routes_lines_by_directory( (input_dir / "gadget").mkdir() (input_dir / "gadget" / "data.jsonl").write_text('{"widget_id": "g", "count": 0}\n', encoding="utf-8") - load_info = run_jsonlines_pipeline(settings) + load_info = assert_run_ok(run_jsonlines_pipeline(settings)) - assert load_info is not None - assert not load_info.has_failed_jobs dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "b", "g"] assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -88,10 +90,8 @@ def test_run_jsonlines_pipeline_pass_invalid_rows_routed_to_invalid_table( (input_dir / "widget").mkdir(parents=True) (input_dir / "widget" / "data.jsonl").write_text('{"widget_id": "a"}\n{broken\n', encoding="utf-8") - load_info = run_jsonlines_pipeline(settings) + load_info = assert_run_ok(run_jsonlines_pipeline(settings)) - assert load_info is not None - assert not load_info.has_failed_jobs dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a"] invalid_df = dataset.widget_invalid.df() @@ -107,10 +107,8 @@ def test_run_jsonlines_pipeline_pass_no_matching_files_yields_no_tables( settings = extract_settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = run_jsonlines_pipeline(settings) + load_info = assert_run_ok(run_jsonlines_pipeline(settings)) - assert load_info is not None - assert not load_info.has_failed_jobs dataset = load_info.pipeline.dataset() assert set(dataset.tables) == {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", "_load_info"} assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -122,34 +120,9 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - settings = extract_settings_factory() - input_dir = Path(settings.input_dir) - (input_dir / "data.jsonl").write_text(SIMPLE_JSONL, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - EXTRACT_PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - str(settings.use_destination), - "--dataset-name", - "cli_dataset", - "--table-name", - "widget", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = extract_settings_factory(dataset_name="cli_dataset", dlt_dev_mode=False) + (Path(settings.input_dir) / "data.jsonl").write_text(SIMPLE_JSONL, encoding="utf-8") + monkeypatch.setattr(sys, "argv", [EXTRACT_PIPELINE_NAME, *settings_to_argv(settings)]) def fake_run_pipeline( *, resource: DltResource, pipeline_kwargs: dict[str, Any], **_: dict[str, Any] @@ -159,10 +132,8 @@ def fake_run_pipeline( return pipeline.run(resource) with patch.object(extract_pipeline_module, "run_pipeline", fake_run_pipeline): - load_info = cli.run() + load_info = assert_run_ok(cli.run()) - assert load_info is not None - assert not load_info.has_failed_jobs # n.b. no _load_info due to this being a monkeypatched pipeline assert set(load_info.pipeline.dataset().tables) == {"widget", "_dlt_version", "_dlt_loads", "_dlt_pipeline_state"} assert sorted(load_info.pipeline.dataset().widget.df()["widget_id"].tolist()) == ["a", "b"] diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py index 49103b80..0b3e4fe9 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py @@ -16,7 +16,11 @@ ) from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + fetch_all_duckdb, +) from tests.conftest import LOCAL_FS, TEST_DATA_DIR UNIPROT_FIXTURE_DIR = TEST_DATA_DIR / "uniprot" / "uniprot_kb" / "chunk_4" @@ -63,15 +67,6 @@ def _run_uniprot_duckdb_pipeline(settings: UniProtSettings, tmp_path: Path, name return pipeline, load_info -def _fetch_all(pipeline: Any, query: str) -> list[tuple[Any, ...]]: - """Run a query against the pipeline's DuckDB destination and return all rows.""" - with ( - pipeline.sql_client() as client, - client.execute_query(query) as cur, - ): - return list(cur.fetchall()) - - def test_integration_uniprot_pipeline_loads_into_duckdb( duckdb_uniprot_settings: UniProtSettings, tmp_path: Path, @@ -83,10 +78,10 @@ def test_integration_uniprot_pipeline_loads_into_duckdb( """ pipeline, load_info = _run_uniprot_duckdb_pipeline(duckdb_uniprot_settings, tmp_path, "loads") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): - ((count,),) = _fetch_all(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 + ((count,),) = fetch_all_duckdb(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 assert count == expected_count, f"expected {expected_count} rows in '{table_name}', got {count}" @@ -97,16 +92,16 @@ def test_integration_uniprot_pipeline_entity_and_source_file_content( """Entity rows are uniprot proteins and source-file rows reference every fixture file.""" pipeline, load_info = _run_uniprot_duckdb_pipeline(duckdb_uniprot_settings, tmp_path, "content") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) # every entity must have a "uniprot:"-prefixed entity_id and be a protein. - rows = _fetch_all(pipeline, "SELECT entity_id, entity_type FROM entity") + rows = fetch_all_duckdb(pipeline, "SELECT entity_id, entity_type FROM entity") assert len(rows) == UNIPROT_ENTRY_COUNT assert all(entity_id.startswith("uniprot:") for entity_id, _ in rows) assert all(entity_type == "protein" for _, entity_type in rows) # entity_x_source_file must reference exactly the fixture files we loaded from - source_files = {row[0] for row in _fetch_all(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} assert source_files == {str(UNIPROT_FIXTURE_DIR.resolve() / name) for name in UNIPROT_FIXTURE_FILES} @@ -115,10 +110,7 @@ def test_integration_cli_uniprot_pipeline_output_validated( pipeline_runner: PipelineTestRunner, ) -> None: """Exercise the UniProt pipeline end-to-end across core, cli, and docker execution modes.""" - load_info = pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings)) assert load_info.pipeline.pipeline_name == "uniprot" assert load_info.dataset_name == "uniprot" assert_pipeline_report_written(Path(duckdb_uniprot_settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py index cf047063..6c80fd39 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py @@ -17,7 +17,11 @@ ) from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + fetch_all_duckdb, +) from tests.conftest import LOCAL_FS, TEST_DATA_DIR TEST_DEFAULT_UNIREF_VARIANT = "50" @@ -64,15 +68,6 @@ def _run_uniref_duckdb_pipeline(settings: UnirefSettings, tmp_path: Path, name: return pipeline, load_info -def _fetch_all(pipeline: Any, query: str) -> list[tuple[Any, ...]]: - """Run a query against the pipeline's DuckDB destination and return all rows.""" - with ( - pipeline.sql_client() as client, - client.execute_query(query) as cur, - ): - return list(cur.fetchall()) - - def test_integration_uniref_pipeline_loads_into_duckdb( duckdb_uniref_settings: UnirefSettings, tmp_path: Path, @@ -84,10 +79,10 @@ def test_integration_uniref_pipeline_loads_into_duckdb( """ pipeline, load_info = _run_uniref_duckdb_pipeline(duckdb_uniref_settings, tmp_path, "loads") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): - ((count,),) = _fetch_all(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 + ((count,),) = fetch_all_duckdb(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 assert count == expected_count, f"expected {expected_count} rows in '{table_name}', got {count}" @@ -98,20 +93,20 @@ def test_integration_uniref_pipeline_entity_and_cluster_content( """Entity rows are UniRef clusters and every cluster carries the injected protocol label.""" pipeline, load_info = _run_uniref_duckdb_pipeline(duckdb_uniref_settings, tmp_path, "content") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) # every entity must have a "uniref:"-prefixed entity_id and be a Cluster. - rows = _fetch_all(pipeline, "SELECT entity_id, entity_type FROM entity") + rows = fetch_all_duckdb(pipeline, "SELECT entity_id, entity_type FROM entity") assert len(rows) == UNIREF_CLUSTER_COUNT assert all(entity_id.startswith("uniref:") for entity_id, _ in rows) assert all(entity_type == "Cluster" for _, entity_type in rows) # every cluster must carry the injected "UniRef " protocol label - protocols = {row[0] for row in _fetch_all(pipeline, "SELECT DISTINCT protocol FROM cluster")} + protocols = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT protocol FROM cluster")} assert protocols == {f"UniRef {duckdb_uniref_settings.variant}"} # entity_x_source_file must reference exactly the fixture file we loaded from - source_files = {row[0] for row in _fetch_all(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} assert source_files == {str(UNIREF_FIXTURE_DIR.resolve() / UNIREF_FIXTURE_FILE)} @@ -120,10 +115,7 @@ def test_integration_cli_uniref_pipeline_output_validated( pipeline_runner: PipelineTestRunner, ) -> None: """Exercise the UniRef pipeline end-to-end across core, cli, and docker execution modes.""" - load_info = pipeline_runner.run("uniref", settings=duckdb_uniref_settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run("uniref", settings=duckdb_uniref_settings)) assert load_info.pipeline.pipeline_name == "uniref" assert load_info.dataset_name == f"uniref_{duckdb_uniref_settings.variant}" assert_pipeline_report_written(Path(duckdb_uniref_settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py index 7e963918..0b4d715d 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py @@ -9,7 +9,11 @@ from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) from tests.xml_samples import SIMPLE_LIBRARY_XML EXPECTED_BOOK_COUNT = 3 @@ -18,8 +22,7 @@ def check_book_list_results(load_info: LoadInfo | None) -> tuple[DataFrame, DataFrame]: """Check over the output of parsing SIMPLE_LIBRARY_XML: one root row, three book rows.""" - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(load_info) dataset = load_info.pipeline.dataset() assert {"library", "book", "library_book", *DEFAULT_DLT_TABLES} == set(dataset.tables) book_df = dataset.table("book").df() @@ -80,9 +83,7 @@ def test_run_xml2db_pipeline_pass_no_matching_files_yields_no_data_table( settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) dataset = load_info.pipeline.dataset() # only dlt's own metadata tables exist; the data tables are never created assert set(dataset.tables) == DEFAULT_DLT_TABLES @@ -111,42 +112,17 @@ def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( def test_cli_pass_runs_end_to_end_from_command_line_arguments( - tmp_path: Path, - dlt_destination_config: str, - library_xsd: Path, + settings_factory: Callable[..., Xml2DbSettings], pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - input_dir = tmp_path / "cli_input" - input_dir.mkdir() + settings = settings_factory(dataset_name="cli_dataset", use_output_dir_for_pipeline_metadata=False) + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "bucket" - output_dir.mkdir(exist_ok=True) - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - dlt_destination_config, - "--dataset-name", - "cli_dataset", - "--xsd-file", - str(library_xsd), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - - load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] - assert_pipeline_report_written(output_dir, load_info) + assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py index 09fa3859..a3d390af 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py @@ -7,10 +7,8 @@ from unittest.mock import MagicMock, patch from uuid import uuid4 -import pendulum import pytest from dlt.common.pipeline import LoadInfo -from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.extract import DltResource from xml2db import DataModel @@ -20,7 +18,13 @@ from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from cdm_data_loaders.readers.xml2db_doc import build_xml2db_model from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline -from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_run_ok, + fetch_all_duckdb, + make_cts_settings, + make_file_item, + settings_to_argv, +) SIMPLE_LIBRARY_XML = """ @@ -63,20 +67,6 @@ def library_model(library_xsd: Path) -> DataModel: return build_xml2db_model(library_xsd, short_name="library_test") -def make_file_item(file_path: Path) -> FileItemDict: - """Build a dlt FileItemDict for a local file, as the filesystem source would.""" - return FileItemDict( - { - "file_url": file_path.as_uri(), - "file_name": file_path.name, - "relative_path": file_path.name, - "mime_type": "application/xml", - "modification_date": pendulum.now(), - "size_in_bytes": file_path.stat().st_size, - } - ) - - def fake_settings( tmp_path: Path, xsd_file: Path, @@ -321,39 +311,13 @@ def test_cli_pass_calls_run_cli_with_settings_class_and_pipeline_fn() -> None: def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, + library_xsd: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and loads rows via a real DuckDB pipeline.""" - input_dir = tmp_path / "cli_input" - input_dir.mkdir() - (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - xsd_file = tmp_path / "library.xsd" - xsd_file.write_text(LIBRARY_XSD, encoding="utf-8") - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}', encoding="utf-8") - - argv = [ - "xml2db", - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - "--dataset-name", - "cli_dataset", - "--xsd-file", - str(xsd_file), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = fake_settings(tmp_path, library_xsd, dataset_name="cli_dataset") + (Path(settings.input_dir) / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["xml2db", *settings_to_argv(settings)]) # unique pipeline name: the duckdb database file is derived from it, so # repeated runs of the suite do not see rows from previous runs @@ -373,13 +337,8 @@ def fake_run_pipeline( with patch.object(xml2db_pipeline_module, "run_pipeline", fake_run_pipeline): cli() - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + assert_run_ok(captured["load_info"]) pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM book") as cur, - ): - (row_count,) = cur.fetchone() + ((row_count,),) = fetch_all_duckdb(pipeline, "SELECT COUNT(*) FROM book") assert row_count == 2 # noqa: PLR2004 diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py index b76d2803..4db0e66d 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py @@ -10,14 +10,18 @@ from cdm_data_loaders.pipelines.xmltodict.settings import PIPELINE_NAME, XmlToDictSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) from tests.xml_samples import SIMPLE_LIBRARY_XML def check_book_list_results(load_info: LoadInfo | None, table_name: str | None = None) -> DataFrame: """Check over the output of parsing SIMPLE_LIBRARY_XML.""" - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(load_info) dataset = load_info.pipeline.dataset() table_name = table_name or "book" assert set(dataset.tables) == {table_name, *DEFAULT_DLT_TABLES} @@ -70,9 +74,7 @@ def test_run_xmltodict_pipeline_pass_no_matching_files_yields_no_data_table( settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) dataset = load_info.pipeline.dataset() # only dlt's own metadata tables exist; the data table is never created assert set(dataset.tables) == DEFAULT_DLT_TABLES @@ -96,42 +98,16 @@ def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( def test_cli_pass_runs_end_to_end_from_command_line_arguments( - tmp_path: Path, - dlt_destination_config: str, + settings_factory: Callable[..., XmlToDictSettings], pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - input_dir = tmp_path / "cli_input" - input_dir.mkdir() + settings = settings_factory(dataset_name="cli_dataset") + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - dlt_destination_config, - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - - load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] - assert_pipeline_report_written(output_dir, load_info) + assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py index 490734ff..17e7ef49 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py @@ -10,7 +10,11 @@ from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) from tests.cdm_data_loaders.pipelines.xsv.conftest import ( ALL_RAGGED_ROWS, COLUMNS, @@ -34,10 +38,8 @@ def test_run_xsv_pipeline_pass_writes_expected_output( settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() assert {"my_table", *DEFAULT_DLT_TABLES} == set(dataset.tables) @@ -59,10 +61,8 @@ def test_run_xsv_pipeline_pass_multiple_files_loaded_together( write_xsv_file(settings, "part_1.csv", rows_to_csv([VALID_ROWS[0]])) write_xsv_file(settings, "part_2.csv", rows_to_csv([VALID_ROWS[1]])) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df() assert sorted(df["number"].tolist()) == [2, 3] @@ -81,10 +81,8 @@ def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(PARTIAL_RAGGED_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() assert {"my_table", REJECTED_TABLE, *DEFAULT_DLT_TABLES} == set(dataset.tables) @@ -106,10 +104,8 @@ def test_run_xsv_pipeline_pass_fully_ragged_file_is_entirely_rejected( settings = settings_factory() write_xsv_file(settings, "bad.csv", rows_to_csv(ALL_RAGGED_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() assert set(dataset.tables) == {REJECTED_TABLE, *DEFAULT_DLT_TABLES} @@ -126,10 +122,8 @@ def test_run_xsv_pipeline_pass_no_matching_files_yields_no_data_table( """An input dir with no matching data files does not fail the run and creates no data table.""" settings = settings_factory() - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False assert set(load_info.pipeline.dataset().tables) == DEFAULT_DLT_TABLES assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -153,10 +147,8 @@ def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( ), ) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df().sort_values("number") floats = df["float"].tolist() assert pd.isna(floats[0]) @@ -177,10 +169,8 @@ def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( settings = settings_factory(loader_file_format="parquet") write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df() assert len(df) == len(VALID_ROWS) @@ -191,33 +181,11 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - settings = settings_factory(dataset_name=f"cli_dataset_{uuid4().hex}") + settings = settings_factory(dataset_name=f"cli_dataset_{uuid4().hex}", dlt_dev_mode=False) write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - argv: list[str] = [ - PIPELINE_NAME, - "--input-dir", - str(settings.input_dir), - "--output-dir", - str(settings.output_dir), - "--use-destination", - settings.use_destination, - "--dataset-name", - settings.dataset_name, - "--schema-file", - str(settings.schema_file), - "--table-name", - "my_table", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - - load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df() assert sorted(df["number"].tolist()) == [2, 3] assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py index acd948ed..7d917138 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py @@ -11,10 +11,7 @@ from unittest.mock import patch from uuid import uuid4 -import dlt -import pendulum import pytest -from dlt.common.storages.fsspec_filesystem import FileItemDict import cdm_data_loaders.pipelines.xsv.pipeline as xsv_pipeline_module from cdm_data_loaders.pipelines.core import PipelineCli @@ -26,6 +23,13 @@ run_xsv_pipeline, ) from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_run_ok, + fetch_all_duckdb, + make_file_item, + settings_to_argv, +) from tests.cdm_data_loaders.pipelines.xsv.conftest import VALID_SCHEMA_URI SIMPLE_SCHEMA: dict[str, Any] = { @@ -39,20 +43,6 @@ """_process_xsv_file""" -def make_file_item(file_path: Path) -> FileItemDict: - """Build a dlt FileItemDict for a local file.""" - return FileItemDict( - { - "file_url": file_path.as_uri(), - "file_name": file_path.name, - "relative_path": file_path.name, - "mime_type": "text/csv", - "modification_date": pendulum.now(), - "size_in_bytes": file_path.stat().st_size, - } - ) - - def test_process_xsv_file_pass_routes_valid_rows_to_table( settings_factory: Callable[..., XsvIngestSettings], work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths], @@ -123,8 +113,6 @@ def test_run_xsv_pipeline_pass_end_to_end_via_duckdb( pipeline_name = f"test_xsv_pipeline_{uuid4().hex}" captured: dict[str, Any] = {} - output_dir = tmp_path / "duckdb_output" - output_dir.mkdir() def fake_run_pipeline( *, @@ -133,31 +121,16 @@ def fake_run_pipeline( **_: Any, # noqa: ANN401 ) -> None: assert pipeline_kwargs == {"pipeline_name": PIPELINE_NAME, "dataset_name": "xsv_test_dataset"} - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{output_dir!s}/{pipeline_name}.db"), - dataset_name="xsv_test_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "xsv_test_dataset") captured["load_info"] = pipeline.run(resource) with patch.object(xsv_pipeline_module, "run_pipeline", fake_run_pipeline): run_xsv_pipeline(settings) - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + assert_run_ok(captured["load_info"]) - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{output_dir!s}/{pipeline_name}.db"), - dataset_name="xsv_test_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT number, name FROM my_table ORDER BY number") as cur, - ): - rows = cur.fetchall() + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "xsv_test_dataset") + rows = fetch_all_duckdb(pipeline, "SELECT number, name FROM my_table ORDER BY number") assert [tuple(row) for row in rows] == [(1, "alice"), (2, "bob")] @@ -173,19 +146,13 @@ def test_run_xsv_pipeline_pass_no_matching_files_loads_nothing( captured: dict[str, Any] = {} def fake_run_pipeline(*, resource: Any, **_: Any) -> None: # noqa: ANN401 - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{settings.output_dir}/{pipeline_name}.db"), - dataset_name="xsv_test_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "xsv_test_dataset") captured["load_info"] = pipeline.run(resource) with patch.object(xsv_pipeline_module, "run_pipeline", fake_run_pipeline): run_xsv_pipeline(settings) - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + load_info = assert_run_ok(captured["load_info"]) assert "my_table" not in load_info.pipeline.default_schema.tables @@ -206,28 +173,12 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and loads rows via a real DuckDB pipeline.""" - settings = settings_factory(SIMPLE_SCHEMA, {"data.csv": "number,name\n1,alice\n2,bob\n"}) - - argv = [ - "xsv", - "--input-dir", - str(settings.input_dir), - "--output-dir", - str(settings.output_dir), - "--use-destination", - "local_fs", - "--dataset-name", - "cli_dataset", - "--schema-file", - str(settings.schema_file), - "--table-name", - "my_table", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory( + SIMPLE_SCHEMA, + {"data.csv": "number,name\n1,alice\n2,bob\n"}, + dataset_name="cli_dataset", + ) + monkeypatch.setattr(sys, "argv", ["xsv", *settings_to_argv(settings)]) pipeline_name = f"test_xsv_cli_pipeline_{uuid4().hex}" captured: dict[str, Any] = {} @@ -239,29 +190,14 @@ def fake_run_pipeline( **_: Any, # noqa: ANN401 ) -> None: assert pipeline_kwargs == {"pipeline_name": PIPELINE_NAME, "dataset_name": "cli_dataset"} - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{settings.output_dir}/{pipeline_name}.db"), - dataset_name="cli_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") captured["load_info"] = pipeline.run(resource) with patch.object(xsv_pipeline_module, "run_pipeline", fake_run_pipeline): cli.run() - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + assert_run_ok(captured["load_info"]) - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{settings.output_dir}/{pipeline_name}.db"), - dataset_name="cli_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM my_table") as cur, - ): - (row_count,) = cur.fetchone() + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") + ((row_count,),) = fetch_all_duckdb(pipeline, "SELECT COUNT(*) FROM my_table") assert row_count == 2 # noqa: PLR2004 From a6c785595a4c25dff9c816c2e4b025016408917e Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 11:14:49 -0700 Subject: [PATCH 5/6] consolidate entry canonicalisation and variation helps --- tests/cdm_data_loaders/pipelines/helpers.py | 27 +++++++ .../jsonlines/integration/conftest.py | 4 +- .../jsonlines_reference_helpers.py | 14 +--- .../test_jsonlines_reference_tests.py | 78 ++++++++----------- .../integration/test_reference_end_to_end.py | 13 ++-- .../test_xml2db_reference_tests.py | 53 +++++-------- .../test_xmltodict_reference_tests.py | 52 ++++++------- 7 files changed, 119 insertions(+), 122 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index 4c7bf4ed..dd02defe 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -501,3 +501,30 @@ def assert_dataset_matches_reference( for value in reconstructed.values(): assert value == sorted_reference_json + + +def run_and_compare_variations( + run_pipeline: Callable[..., Any], + variations: dict[str, tuple[str, dict[str, Any]]], + *, + sorted_reference_json: list[str] | None = None, + output_format: str | None = None, + reconstruct: Callable[[dict[str, list[dict[str, Any]]]], list[dict[str, Any]]] = reconstruct_entries, + canonicalize: Callable[[dict[str, Any]], dict[str, Any]] | None = None, +) -> dict[str, tuple[LoadInfo, Path]]: + """Run a pipeline across named (chunk_dir, overrides) variations, assert equal datasets, and optionally match reference.""" + load_info_and_dir = { + key: run_and_read(run_pipeline, chunk_dir, **overrides) + for key, (chunk_dir, overrides) in variations.items() + } + if len(load_info_and_dir) >= 2: # noqa: PLR2004 + assert_datasets_equal(load_info_and_dir) + if sorted_reference_json is not None: + assert_dataset_matches_reference( + load_info_and_dir, + sorted_reference_json, + output_format, + reconstruct=reconstruct, + canonicalize=canonicalize, + ) + return load_info_and_dir diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py index a352c32f..2387264b 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py @@ -29,7 +29,7 @@ from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models import ENTITY_MODELS from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas import ENTITY_SCHEMAS from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( - canonicalize_reference_entry, + canonicalize_entry, ) from tests.conftest import TEST_DATA_DIR @@ -158,7 +158,7 @@ def sorted_reference_entities( ) -> list[str]: """Canonicalized entries as sorted JSON strings for order-insensitive comparison.""" lines_as_json = [ - json.dumps(canonicalize_reference_entry(prepare_reference(obj)), sort_keys=True, default=str) + json.dumps(canonicalize_entry(prepare_reference(obj)), sort_keys=True, default=str) for obj in jsonl_reference_entities ] return sorted(lines_as_json) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py index af7ac75a..03681fc4 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py @@ -51,8 +51,8 @@ def reconstructed_entries( return reconstruct_entries(renamed) -def canonicalize_reference_entry(entry: dict[str, Any]) -> dict[str, Any]: - """Canonicalize an entry so pipeline output and source JSON compare equal. +def canonicalize_entry(entry: dict[str, Any]) -> dict[str, Any]: + """Canonicalize a reference or reconstructed pipeline entry so both sides compare equal. Reverses the transformations dlt's normalization applies, and normalizes the two storage representations (parquet datetimes vs jsonl timestamp strings): @@ -72,14 +72,8 @@ def canonicalize_reference_entry(entry: dict[str, Any]) -> dict[str, Any]: return _canonicalize_value(entry) -def canonicalize_pipeline_entry(entry: dict[str, Any]) -> dict[str, Any]: - """Canonicalize a reconstructed pipeline entry for comparison. - - Preserved JSON lists and reconstructed child rows share the same scalar-list - representation. Null fields and empty objects disappear during flattening. - Timestamps are normalized for both JSONL strings and Parquet datetimes. - """ - return _canonicalize_value(entry) +canonicalize_reference_entry = canonicalize_entry +canonicalize_pipeline_entry = canonicalize_entry def _canonicalize_value(value: Any, key: str = "") -> Any: diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py index fa40b0f9..f2113da7 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py @@ -21,16 +21,15 @@ LOADER_FILE_FORMATS, WORKER_CONFIGS, assert_dataset_matches_reference, - assert_datasets_equal, read_pipeline_tables, + run_and_compare_variations, run_and_read, ) from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models import ENTITY_MODELS from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas import ENTITY_SCHEMAS from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( EXPECTED_ENTRY_COUNT, - canonicalize_pipeline_entry, - canonicalize_reference_entry, + canonicalize_entry, reconstructed_entries, reference_entries_from_duckdb, ) @@ -60,8 +59,8 @@ def test_jsonlines_reference_pass_assembly_model_accepts_all_records( assert [entry.model_dump()["accession"] for entry in validated] == [ entry["accession"] for entry in jsonl_reference_entities ] - assert [canonicalize_reference_entry(entry.model_dump(mode="json", exclude_unset=True)) for entry in validated] == [ - canonicalize_reference_entry(entry) for entry in jsonl_reference_entities + assert [canonicalize_entry(entry.model_dump(mode="json", exclude_unset=True)) for entry in validated] == [ + canonicalize_entry(entry) for entry in jsonl_reference_entities ] @@ -102,16 +101,13 @@ def test_jsonlines_passed_dataset_matches_reference( sorted_reference_entities: list[str], ) -> None: """Pipeline output matches the pipeline input.""" - load_info_and_dir = { - source_dir: run_and_read(run_reference_pipeline, source_dir, loader_file_format=loader_file_format) - } - - assert_dataset_matches_reference( - load_info_and_dir, - sorted_reference_entities, - loader_file_format, + run_and_compare_variations( + run_reference_pipeline, + {source_dir: (source_dir, {"loader_file_format": loader_file_format})}, + sorted_reference_json=sorted_reference_entities, + output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -120,18 +116,13 @@ def test_jsonlines_pass_output_identical_across_chunk_dirs( run_reference_pipeline: Callable[..., Any], sorted_reference_entities: list[str], loader_file_format: str ) -> None: """Pipeline output is identical across all three chunk dirs and matches the reference.""" - load_info_and_dir = { - f"{key}_{loader_file_format}": run_and_read(run_reference_pipeline, key, loader_file_format=loader_file_format) - for key in CHUNK_DIRS - } - - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference( - load_info_and_dir, - sorted_reference_entities, - loader_file_format, + run_and_compare_variations( + run_reference_pipeline, + {f"{key}_{loader_file_format}": (key, {"loader_file_format": loader_file_format}) for key in CHUNK_DIRS}, + sorted_reference_json=sorted_reference_entities, + output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -175,7 +166,7 @@ def test_jsonlines_pass_diff_format_same_results( sorted_reference_entities, output_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -185,20 +176,19 @@ def test_jsonlines_pass_output_identical_across_buffer_sizes( ) -> None: """List buffer size does not affect pipeline output for any chunk dir.""" for chunk_dir in CHUNK_DIRS: - load_info_and_dir = { - f"{buffer_size}_{loader_file_format}": run_and_read( - run_reference_pipeline, chunk_dir, buffer_size=buffer_size, loader_file_format=loader_file_format - ) - for buffer_size in BUFFER_SIZES - } - - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference( - load_info_and_dir, - sorted_reference_entities, - loader_file_format, + run_and_compare_variations( + run_reference_pipeline, + { + f"{buffer_size}_{loader_file_format}": ( + chunk_dir, + {"buffer_size": buffer_size, "loader_file_format": loader_file_format}, + ) + for buffer_size in BUFFER_SIZES + }, + sorted_reference_json=sorted_reference_entities, + output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -235,7 +225,7 @@ def test_jsonlines_pass_no_data_lost_across_worker_configs( sorted_reference_entities, output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -253,11 +243,11 @@ def test_jsonlines_pass_reconstructed_matches_duckdb_reference( from_duckdb = reference_entries_from_duckdb(reference_dataset) assert len(from_duckdb) == EXPECTED_ENTRY_COUNT from_duckdb_canonical = sorted( - json.dumps(canonicalize_reference_entry(prepare_reference(entry)), sort_keys=True, default=str) + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in from_duckdb ) from_pipeline = sorted( - json.dumps(canonicalize_pipeline_entry(entry), sort_keys=True, default=str) for entry in reconstructed + json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in reconstructed ) assert from_pipeline == from_duckdb_canonical @@ -285,7 +275,7 @@ def test_jsonlines_pass_gzip_input_matches_plain_text( sorted_reference_entities, loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -313,5 +303,5 @@ def test_jsonlines_pass_nesting_matches_pipeline_contract( sorted_reference_entities, loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 36ffd705..62e52a73 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -20,8 +20,7 @@ run_and_read, ) from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( - canonicalize_pipeline_entry, - canonicalize_reference_entry, + canonicalize_entry, reconstructed_entries, ) @@ -58,9 +57,9 @@ def test_jsonlines_fail_malformed_rows_preserve_provenance( tables = read_pipeline_tables(output_dir / result.dataset_name, loader_file_format) actual = reconstructed_entries(tables, "dataset") assert sorted( - json.dumps(canonicalize_pipeline_entry(entry), sort_keys=True, default=str) for entry in actual + json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in actual ) == sorted( - json.dumps(canonicalize_reference_entry(prepare_reference(entry)), sort_keys=True, default=str) + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in entries ) suffix, error_key = ( @@ -125,8 +124,8 @@ def test_jsonlines_pass_duplicates_blanks_and_final_line_preserved( loader_file_format=loader_file_format, ) tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) - expected = canonicalize_reference_entry(prepare_reference(entry)) - assert [canonicalize_pipeline_entry(row) for row in reconstructed_entries(tables)] == [expected, expected] + expected = canonicalize_entry(prepare_reference(entry)) + assert [canonicalize_entry(row) for row in reconstructed_entries(tables)] == [expected, expected] def test_jsonlines_pass_multiple_entities_and_file_glob( @@ -312,7 +311,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) assert ( sorted( - json.dumps(canonicalize_pipeline_entry(entry), sort_keys=True, default=str) + json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in reconstructed_entries(tables) ) == sorted_reference_entities diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py index 9c7a67e2..6b951cec 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py @@ -10,7 +10,12 @@ import xmltodict from cdm_data_loaders.core.fields import PARQUET -from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, read_pipeline_tables, sorted_json +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + read_pipeline_tables, + run_and_read, + sorted_json, +) from tests.cdm_data_loaders.pipelines.xml2db.integration.xml2db_reference_helpers import ( assert_uniref_referential_integrity, reconstruct_xml2db_entries, @@ -77,9 +82,7 @@ def run_and_reconstruct(run_pipeline: Callable[..., Any], chunk_dir: str, **over Reads back whatever `loader_file_format` the run actually used -- `PARQUET`, matching `Xml2DbSettings.loader_file_format`'s own default, unless overridden. """ - (load_info, output_dir) = run_pipeline(chunk_dir, **overrides) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_pipeline, chunk_dir, **overrides) output_format = str(overrides.get("loader_file_format", PARQUET)) tables = read_pipeline_tables(output_dir / load_info.dataset_name, output_format) return reconstruct_xml2db_entries(tables) @@ -125,9 +128,7 @@ def test_xml2db_pass_referential_integrity_across_merged_files( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Every foreign key in the merged, multi-file dataset resolves to a row in its target table.""" - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_5_el") - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_5_el") tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) assert_uniref_referential_integrity(tables) @@ -167,9 +168,7 @@ def test_xml2db_pass_chunked_referential_integrity_across_files_and_chunks( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Foreign keys still resolve correctly when both multiple files and multiple chunks per file are involved.""" - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_5_el", xml_tag="entry", chunk_size=2) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_5_el", xml_tag="entry", chunk_size=2) tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) assert_uniref_referential_integrity(tables) @@ -185,12 +184,8 @@ def test_xml2db_pass_chunked_compaction_matches_unchunked_row_counts_exactly( Not just the same *content*, which the reconstruction-based tests above already establish, but the same physical row count, with no literal duplicates left over. """ - (unchunked_info, unchunked_dir) = run_xml2db_pipeline_fxt("chunk_100_el") - (chunked_info, chunked_dir) = run_xml2db_pipeline_fxt("chunk_100_el", xml_tag="entry", chunk_size=7) - assert unchunked_info is not None - assert chunked_info is not None - assert not unchunked_info.has_failed_jobs - assert not chunked_info.has_failed_jobs + (unchunked_info, unchunked_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el") + (chunked_info, chunked_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7) unchunked_tables = read_pipeline_tables(unchunked_dir / unchunked_info.dataset_name, PARQUET) chunked_tables = read_pipeline_tables(chunked_dir / chunked_info.dataset_name, PARQUET) @@ -208,9 +203,7 @@ def test_xml2db_pass_chunked_compaction_root_table_is_left_alone( the kind of literal duplicate compaction removes, so it still gets one row per chunk after compaction runs, unlike property/representative_member. """ - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_100_el", xml_tag="entry", chunk_size=7) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7) tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) n_chunks = -(-EXPECTED_ENTRY_COUNT // 7) # ceil division @@ -221,9 +214,7 @@ def test_xml2db_pass_chunked_compaction_preserves_referential_integrity( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Every foreign key still resolves after compaction has removed literal duplicate rows.""" - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_100_el", xml_tag="entry", chunk_size=7) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7) tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) assert_uniref_referential_integrity(tables) @@ -232,12 +223,10 @@ def test_xml2db_pass_compact_reused_tables_false_keeps_literal_duplicates( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Disabling compact_reused_tables leaves the literal duplicate rows chunking can produce.""" - (unchunked_info, unchunked_dir) = run_xml2db_pipeline_fxt("chunk_100_el") - (chunked_info, chunked_dir) = run_xml2db_pipeline_fxt( - "chunk_100_el", xml_tag="entry", chunk_size=7, compact_reused_tables=False + (unchunked_info, unchunked_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el") + (chunked_info, chunked_dir) = run_and_read( + run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7, compact_reused_tables=False ) - assert unchunked_info is not None - assert chunked_info is not None unchunked_tables = read_pipeline_tables(unchunked_dir / unchunked_info.dataset_name, PARQUET) chunked_tables = read_pipeline_tables(chunked_dir / chunked_info.dataset_name, PARQUET) @@ -262,12 +251,10 @@ def test_xml2db_pass_compaction_dedupes_across_separate_unchunked_files( literal duplicate rows until compaction merges them -- exactly the same failure mode as chunking, just from multiple files instead of multiple chunks of one file. """ - (compacted_info, compacted_dir) = run_xml2db_pipeline_fxt("chunk_5_el") - (uncompacted_info, uncompacted_dir) = run_xml2db_pipeline_fxt("chunk_5_el", compact_reused_tables=False) - assert compacted_info is not None - assert uncompacted_info is not None - assert not compacted_info.has_failed_jobs - assert not uncompacted_info.has_failed_jobs + (compacted_info, compacted_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_5_el") + (uncompacted_info, uncompacted_dir) = run_and_read( + run_xml2db_pipeline_fxt, "chunk_5_el", compact_reused_tables=False + ) compacted_tables = read_pipeline_tables(compacted_dir / compacted_info.dataset_name, PARQUET) uncompacted_tables = read_pipeline_tables(uncompacted_dir / uncompacted_info.dataset_name, PARQUET) diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py index a9ed95ec..ff7e45ea 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py @@ -17,6 +17,7 @@ read_pipeline_tables, reconstruct_entries, reference_entries_from_duckdb, + run_and_compare_variations, run_and_read, sorted_json, ) @@ -58,15 +59,13 @@ def test_xml_pass_reference_dataset_has_nested_structures( def test_xml_pass_diff_format_same_results(run_xmltodict_pipeline_fxt: Callable[..., Any]) -> None: """Ensure that the save format does not affect the output.""" - load_info_and_dir = { - str(output_format): run_and_read( - run_xmltodict_pipeline_fxt, - "chunk_20_el", - loader_file_format=output_format, - ) - for output_format in LOADER_FILE_FORMATS - } - assert_datasets_equal(load_info_and_dir) + run_and_compare_variations( + run_xmltodict_pipeline_fxt, + { + str(output_format): ("chunk_20_el", {"loader_file_format": output_format}) + for output_format in LOADER_FILE_FORMATS + }, + ) @pytest.mark.xfail(reason="not yet implemented") @@ -104,15 +103,12 @@ def test_xml_pass_output_identical_across_chunk_dirs( run_xmltodict_pipeline_fxt: Callable[..., Any], sorted_reference_xml_entries: list[str], loader_file_format: str ) -> None: """Pipeline output is identical across all three chunk dirs and matches the reference.""" - load_info_and_dir = { - f"{key}_{loader_file_format}": run_and_read( - run_xmltodict_pipeline_fxt, key, loader_file_format=loader_file_format - ) - for key in CHUNK_DIRS - } - - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference(load_info_and_dir, sorted_reference_xml_entries, loader_file_format) + run_and_compare_variations( + run_xmltodict_pipeline_fxt, + {f"{key}_{loader_file_format}": (key, {"loader_file_format": loader_file_format}) for key in CHUNK_DIRS}, + sorted_reference_json=sorted_reference_xml_entries, + output_format=loader_file_format, + ) @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS) @@ -121,14 +117,18 @@ def test_xml_pass_output_identical_across_buffer_sizes( ) -> None: """List buffer size does not affect pipeline output for any chunk dir.""" for chunk_dir in CHUNK_DIRS: - load_info_and_dir = { - f"{buffer_size}_{loader_file_format}": run_and_read( - run_xmltodict_pipeline_fxt, chunk_dir, buffer_size=buffer_size, loader_file_format=loader_file_format - ) - for buffer_size in BUFFER_SIZES - } - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference(load_info_and_dir, sorted_reference_xml_entries, loader_file_format) + run_and_compare_variations( + run_xmltodict_pipeline_fxt, + { + f"{buffer_size}_{loader_file_format}": ( + chunk_dir, + {"buffer_size": buffer_size, "loader_file_format": loader_file_format}, + ) + for buffer_size in BUFFER_SIZES + }, + sorted_reference_json=sorted_reference_xml_entries, + output_format=loader_file_format, + ) @pytest.mark.parametrize( From a88e6ad16dd876b2d333c891faa280e83ee8d5da Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 12:42:08 -0700 Subject: [PATCH 6/6] Rebase on develop, incorporate report format changes --- tests/cdm_data_loaders/pipelines/framework.py | 32 ++++++++++--------- tests/cdm_data_loaders/pipelines/helpers.py | 3 +- .../test_jsonlines_reference_tests.py | 3 +- .../integration/test_reference_end_to_end.py | 7 ++-- .../pipelines/ncbi_rest_api/conftest.py | 5 +-- tests/cdm_data_loaders/pipelines/test_core.py | 8 +++-- .../pipelines/test_framework.py | 17 ++++++++-- .../uniprot/test_uniprot_end_to_end.py | 4 ++- .../uniref/test_uniref_end_to_end.py | 4 ++- 9 files changed, 50 insertions(+), 33 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 47661346..2339b3b6 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -289,31 +289,33 @@ def reconstruct_load_info(output_dir: Path, bucket_dir: Path | None = None) -> L report_file = report_files[-1] lines = report_file.read_text(encoding="utf-8").splitlines() - if len(lines) < 3: # noqa: PLR2004 + if len(lines) < 5: # noqa: PLR2004 return None - # Line 0: "Pipeline dataset " - header_prefix = "Pipeline " - dataset_sep = " dataset " - if not lines[0].startswith(header_prefix) or dataset_sep not in lines[0]: + # Line 0: "Pipeline: " + # Line 1: "Dataset: " + # Line 2: "Started: " + # Line 3: "Finished: " + # Line 4: "Status: completed" | "Status: completed with errors" | "Status: error" + expected_prefixes = ("Pipeline: ", "Dataset: ", "Started: ", "Finished: ", "Status: ") + if not all(line.startswith(prefix) for line, prefix in zip(lines[:5], expected_prefixes, strict=True)): return None - pipeline_name, dataset_name = lines[0].removeprefix(header_prefix).rsplit(dataset_sep, maxsplit=1) - pipeline_name = pipeline_name.strip() - dataset_name = dataset_name.strip() - # Line 1: "Started: " - started_str = lines[1].removeprefix("Started: ").strip() + pipeline_name = lines[0].removeprefix("Pipeline: ").strip() + dataset_name = lines[1].removeprefix("Dataset: ").strip() + started_str = lines[2].removeprefix("Started: ").strip() + finished_str = lines[3].removeprefix("Finished: ").strip() started_at = _PreservedStrDatetime(started_str) + report_finished_at = _PreservedStrDatetime(finished_str) - # Line 2: "Status: completed" | "Status: completed with errors" | "Status: error" - status_str = lines[2].removeprefix("Status: ").strip() + status_str = lines[4].removeprefix("Status: ").strip() if status_str == "error": return None has_failed_jobs = status_str != "completed" - # Line 3+: "Completed loads: , , ..." + # Line 5+: "Completed loads: , , ..." loads_ids: list[str] = [] - for line in lines[3:]: + for line in lines[5:]: if line.startswith("Completed loads:"): loads_raw = line.removeprefix("Completed loads:").strip() loads_ids = [item.strip() for item in loads_raw.split(",") if item.strip()] @@ -329,7 +331,7 @@ def reconstruct_load_info(output_dir: Path, bucket_dir: Path | None = None) -> L saved_row = _read_saved_load_info_row(actual_bucket_dir / dataset_name) finished_at_raw = saved_row.get("finished_at") - finished_at = _PreservedStrDatetime(str(finished_at_raw)) if finished_at_raw else None + finished_at = _PreservedStrDatetime(str(finished_at_raw)) if finished_at_raw else report_finished_at # Use a fresh working directory outside output_dir so dlt syncs the schema from the destination # without colliding with stale schemas in ~/.dlt/pipelines or adding files to output_dir. diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index dd02defe..9708d6a0 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -514,8 +514,7 @@ def run_and_compare_variations( ) -> dict[str, tuple[LoadInfo, Path]]: """Run a pipeline across named (chunk_dir, overrides) variations, assert equal datasets, and optionally match reference.""" load_info_and_dir = { - key: run_and_read(run_pipeline, chunk_dir, **overrides) - for key, (chunk_dir, overrides) in variations.items() + key: run_and_read(run_pipeline, chunk_dir, **overrides) for key, (chunk_dir, overrides) in variations.items() } if len(load_info_and_dir) >= 2: # noqa: PLR2004 assert_datasets_equal(load_info_and_dir) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py index f2113da7..c5fbec28 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py @@ -243,8 +243,7 @@ def test_jsonlines_pass_reconstructed_matches_duckdb_reference( from_duckdb = reference_entries_from_duckdb(reference_dataset) assert len(from_duckdb) == EXPECTED_ENTRY_COUNT from_duckdb_canonical = sorted( - json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) - for entry in from_duckdb + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in from_duckdb ) from_pipeline = sorted( json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in reconstructed diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 62e52a73..9d794789 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -56,11 +56,8 @@ def test_jsonlines_fail_malformed_rows_preserve_provenance( ) tables = read_pipeline_tables(output_dir / result.dataset_name, loader_file_format) actual = reconstructed_entries(tables, "dataset") - assert sorted( - json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in actual - ) == sorted( - json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) - for entry in entries + assert sorted(json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in actual) == sorted( + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in entries ) suffix, error_key = ( ("invalid", "parse_error") if pipeline_kind == EXTRACT_PIPELINE_NAME else ("rejected", "error_detail") diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py index e13dfef1..865789c6 100644 --- a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py @@ -20,7 +20,9 @@ def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) - monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.ncbi_genome_client._config.http_show_error_body", True) + monkeypatch.setattr( + "cdm_data_loaders.pipelines.ncbi_rest_api.ncbi_genome_client._config.http_show_error_body", True + ) @pytest.fixture(scope="module") @@ -67,4 +69,3 @@ def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: :type monkeypatch: pytest.MonkeyPatch """ monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) - diff --git a/tests/cdm_data_loaders/pipelines/test_core.py b/tests/cdm_data_loaders/pipelines/test_core.py index 07f221e7..3d442d45 100644 --- a/tests/cdm_data_loaders/pipelines/test_core.py +++ b/tests/cdm_data_loaders/pipelines/test_core.py @@ -599,7 +599,9 @@ def test_run_pipeline_pipelines_dir_used_for_pipeline_metadata(dlt_test_settings assert any(p.name == "state.json" for p in pipeline_state_files) -def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: +def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset( + dlt_test_settings: CtsSettings, destination_bucket: Path +) -> None: """dev_mode=True is forwarded to dlt.pipeline(); the pipeline loads into a suffixed dataset.""" settings = dlt_test_settings.model_copy(update={"dlt_dev_mode": True}) @@ -722,7 +724,9 @@ def test_run_pipeline_load_info_in_dev_mode(dlt_test_settings: CtsSettings, dest assert rows[0]["dataset_name"] == load_info.dataset_name -def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: +def test_run_pipeline_load_info_named_pipeline_and_dataset( + dlt_test_settings: CtsSettings, destination_bucket: Path +) -> None: """A named pipeline/dataset save lands in the named dataset with a load_info row.""" load_info = run_pipeline( dlt_test_settings, diff --git a/tests/cdm_data_loaders/pipelines/test_framework.py b/tests/cdm_data_loaders/pipelines/test_framework.py index 27116169..9085248d 100644 --- a/tests/cdm_data_loaders/pipelines/test_framework.py +++ b/tests/cdm_data_loaders/pipelines/test_framework.py @@ -127,7 +127,12 @@ def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: error_dir = tmp_path / "error_out" error_dir.mkdir() (error_dir / "report_123_456.txt").write_text( - "Pipeline test_pipe dataset test_ds\nStarted: 2026-01-01 00:00:00+00:00\nStatus: error\nPipeline failed\n", + "Pipeline: test_pipe\n" + "Dataset: test_ds\n" + "Started: 2026-01-01 00:00:00+00:00\n" + "Finished: 2026-01-01 00:00:05+00:00\n" + "Status: error\n" + "Pipeline failed\n", encoding="utf-8", ) assert reconstruct_load_info(error_dir) is None @@ -135,9 +140,12 @@ def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: completed_dir = tmp_path / "completed_out" completed_dir.mkdir() started_str = "2026-01-01 12:34:56.789000+00:00" + finished_str = "2026-01-01 12:35:00.000000+00:00" (completed_dir / "report_111_222.txt").write_text( - f"Pipeline test_pipe dataset test_ds\n" + f"Pipeline: test_pipe\n" + f"Dataset: test_ds\n" f"Started: {started_str}\n" + f"Finished: {finished_str}\n" f"Status: completed with errors\n" f"Completed loads: 111, 222\n" f"Pipeline load info saved\n", @@ -148,6 +156,7 @@ def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: assert reconstructed.pipeline.pipeline_name == "test_pipe" assert reconstructed.dataset_name == "test_ds" assert str(reconstructed.started_at) == started_str + assert str(reconstructed.finished_at) == finished_str assert reconstructed.has_failed_jobs is True assert reconstructed.loads_ids == ["111", "222"] as_dict = reconstructed.asdict() @@ -255,8 +264,10 @@ def fake_docker_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedPr captured_cmds.append(cmd) # Simulate the container writing a completed report to the mounted output_dir (output_dir / "report_999_1000.txt").write_text( - "Pipeline xmltodict dataset xml_docker_ds\n" + "Pipeline: xmltodict\n" + "Dataset: xml_docker_ds\n" "Started: 2026-02-01 00:00:00+00:00\n" + "Finished: 2026-02-01 00:00:05+00:00\n" "Status: completed\n" "Completed loads: 999\n" "Pipeline load info saved\n", diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py index 0b3e4fe9..30ca3b80 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py @@ -101,7 +101,9 @@ def test_integration_uniprot_pipeline_entity_and_source_file_content( assert all(entity_type == "protein" for _, entity_type in rows) # entity_x_source_file must reference exactly the fixture files we loaded from - source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = { + row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file") + } assert source_files == {str(UNIPROT_FIXTURE_DIR.resolve() / name) for name in UNIPROT_FIXTURE_FILES} diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py index 6c80fd39..36a67251 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py @@ -106,7 +106,9 @@ def test_integration_uniref_pipeline_entity_and_cluster_content( assert protocols == {f"UniRef {duckdb_uniref_settings.variant}"} # entity_x_source_file must reference exactly the fixture file we loaded from - source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = { + row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file") + } assert source_files == {str(UNIREF_FIXTURE_DIR.resolve() / UNIREF_FIXTURE_FILE)}