From d6619dea642c51cb019716d2be726dcc65d2fcdc Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 09:13:17 -0700 Subject: [PATCH 01/16] Preliminary test framework creation --- CLAUDE.md | 116 +++ pyproject.toml | 36 +- .../pipelines/jsonlines/settings.py | 6 +- .../pipelines/ncbi_ftp_download.py | 9 +- .../pipelines/ncbi_ftp_promote.py | 8 +- src/cdm_data_loaders/pipelines/uniprot_kb.py | 4 +- tests/cdm_data_loaders/pipelines/conftest.py | 38 +- tests/cdm_data_loaders/pipelines/framework.py | 723 ++++++++++++++++ tests/cdm_data_loaders/pipelines/helpers.py | 9 +- .../jsonlines/integration/conftest.py | 23 +- .../integration/test_jsonlines_end_to_end.py | 15 +- .../test_jsonlines_reference_tests.py | 9 +- .../integration/test_jsonschema_end_to_end.py | 28 +- .../integration/test_reference_end_to_end.py | 42 +- .../jsonlines/test_extract_pipeline.py | 2 +- .../jsonlines/test_jsonschema_pipeline.py | 2 +- .../pipelines/test_framework.py | 412 +++++++++ .../pipelines/uniprot/test_uniprot.py | 8 +- .../uniprot/test_uniprot_end_to_end.py | 58 +- .../uniref/test_uniref_end_to_end.py | 63 +- .../integration/test_xml2db_end_to_end.py | 21 +- .../integration/test_xmltodict_end_to_end.py | 27 +- .../xmltodict/test_xmltodict_pipeline.py | 6 +- .../xsv/integration/test_xsv_end_to_end.py | 29 +- tests/conftest.py | 26 + uv.lock | 788 ++++++++++++------ 26 files changed, 2024 insertions(+), 484 deletions(-) create mode 100644 CLAUDE.md create mode 100644 tests/cdm_data_loaders/pipelines/framework.py create mode 100644 tests/cdm_data_loaders/pipelines/test_framework.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..d2f1c773 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,116 @@ +# CLAUDE.md + +Guidance for working in this repository. Rules with stricter, active requirements live in `.claude/rules/` (coding-style, testing, heresy) and apply to all changes. + +## Project + +`cdm-data-loaders`: data loaders and wranglers for the KBase CDM. Python >= 3.13, managed and executed with `uv`. Package source is under `src/cdm_data_loaders/` (~115 files, 8 top-level packages). Tests mirror the source layout under `tests/cdm_data_loaders/` (including pipeline end-to-end suites under `tests/cdm_data_loaders/pipelines/`), with multi-phase NCBI FTP integration tests in `tests/integration/ncbi_ftp/` and vcrpy cassettes in `tests/cassettes/`. + +Autouse fixtures in `tests/conftest.py` (`_isolated_cli_and_env` and `_isolated_dlt_config` from `tests/dlt_config_isolation.py`) reset `sys.argv = ["pytest"]`, strip `CDL_*` environment variables, and snapshot/restore `dlt.config` between tests. Pipeline end-to-end tests use `PipelineTestRunner` (`tests/cdm_data_loaders/pipelines/framework.py`) to run pipelines across `ExecutionMode.CORE`, `ExecutionMode.CLI`, and `ExecutionMode.DOCKER` (Docker tests are marked `requires_docker` and require `PIPELINE_DOCKER_IMAGE` to be set). + +Useful invocations: + +- Standard local test run (skips Spark, Ceph, and unset Docker image tests): `uv run pytest -m "not requires_spark and not requires_ceph"` +- Full pipelines suite, excluding the bulk downloader: `uv run pytest tests/cdm_data_loaders/pipelines --ignore=tests/cdm_data_loaders/pipelines/all_the_bacteria` +- Unit vs. container test script: `./scripts/run_tests.sh [unit|container|all]` +- Lint: `uv run ruff check` (line length 120, target py313, full rule selection in `pyproject.toml`). + +## Architecture + +Domain logic lives in the top-level packages and is orchestration-free; orchestration lives in `pipelines/`. Data flows source -> reader -> intermediate representation (IR) -> emitter -> target: + +- Converters: reader (source schema -> IR) + emitter (IR -> target format) + thin facade composing them. See `docs/converters_ir.md` for the full IR contract and package layout. +- IR is a frozen, slotted node tree (`converters/core/ir.py`) that preserves source facts without choosing target policy; emitters make all target-specific decisions. +- `converters/core/` imports no dlt, PyIceberg, or PySpark modules. The only cross-package import is the `x-xsv-config` contract reusing `X_XSV_CONFIG_SCHEMA` from `cdm_data_loaders.readers.jsonschema_xsv.xsv_validator.custom_metaschema`. + +Settings hierarchy: pipeline settings in `pipelines/` extend the hierarchy in `core/settings.py`: `CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`. Settings use `env_prefix="CDL_"`, `cli_kebab_case=True`, and `default_settings_with_shortcuts(...)`. `CdmDataLoadersBase.__pydantic_init_subclass__` calls `check_aliases()` at class creation time to verify that CLI argument names, aliases, and `cli_shortcuts` do not collide or claim reserved flags (`h`, `help`). CLI parsing is invoked explicitly via `pydantic_settings.CliApp.run` in `pipelines/core.py` (`PipelineCli` / `run_cli`), not implicitly on model instantiation. Every pipeline registers a `cli` entry point in `pyproject.toml` under `[project.scripts]` (and in `scripts/entrypoint.sh` for Docker execution): `all_the_bacteria`, `jsonl_extract`, `jsonl_jsonschema`, `jsonl_pydantic`, `ncbi_ftp_promote`, `ncbi_ftp_sync`, `ncbi_rest_api`, `pdb_manifest`, `uniprot`, `uniref`, `xmltodict`, `xml2db`, `xsv`. + +## Package Map + +All paths below are relative to `src/cdm_data_loaders/`. + +### `core/` + +Framework-level foundations: + +- `fields.py`: common annotated field types (`BufferSize`, `DatasetName`, `FileGlob`, `LoaderFileFormat`, `TableName`, `WriteDisposition`, etc.), `LoaderFileFormatEnum` (`parquet`, `jsonl`), and `DEFAULTS`. +- `path_fields.py`: `LocalInputDir` and `LocalOutputDir` path types (`file://` or plain local directory paths) and URL/path validation helpers (`normalize_local_path`, `validate_directory`, `validate_url_protocol`). +- `settings.py`: `pydantic-settings` base class hierarchy (`CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`), `CLI_SHORTCUTS`, `default_settings_with_shortcuts()`, and CLI flag/shortcut collision checking (`check_aliases()`). + +### `converters/` + +Schema-format translation. See `docs/converters_ir.md` for the full contract. + +- `core/`: shared, target-independent foundation + - `ir.py` (NodeType, Field, SchemaDocument, Reader/Emitter protocols), `ir_values.py` (`freeze_value` and friends), `errors.py` (ConversionError), `extensions.py` (ExtensionRegistry, DEFAULT_EXTENSIONS), `guards.py` (input guards), `inference.py` (type inference, decimal scale), `metadata.py` (draft-specific JSON Schema keyword selection), `io.py` (schema loading), `paths.py` (`set_nested`). +- `readers/`: source -> IR + - `json_schema.py` (JsonSchemaReader), `dlt.py` (DltReader over dlt's normalized table forest), `iceberg.py` (IcebergReader). +- `emitters/`: IR -> target + - `json_schema.py`, `dlt.py`, `pyspark.py`, `pyarrow.py`, `linkml.py`. +- Facades: `jsonschema_to_dlt.py`, `jsonschema_to_pyspark.py`, `jsonschema_to_pyarrow.py`, `dlt_to_jsonschema.py`, `pyiceberg_to_jsonschema.py`, `pyiceberg_to_linkml.py`, plus CLI dumps `pyiceberg_catalog_to_jsonschema.py` and `pyiceberg_catalog_to_linkml.py`. + +### `readers/` + +File-format readers feeding pipelines: + +- `jsonlines.py`: JSONL line parsing (`parse_jsonl_line`), paged streaming (`stream_jsonl_lines`), and table routing (`route_jsonl_lines`). +- `xml.py`: shared XML streaming and `xmltodict` helpers (`parse_head_matter`, `stream_xml_file`, `process_xml_file_to_dict`, `process_xml_file`, `process_xml_file_batches`, `process_xml_file_items`, and `build_xml_file_resource` over dlt's `filesystem` source). +- `xml2db_doc.py`: `xml2db` model building (`build_xml2db_model`), bounded-memory chunk fragment streaming (`iter_xml2db_chunk_fragments`), primary/foreign key rewriting (content-addressed keys for reused tables, file/chunk-namespaced keys otherwise), document flattening (`flatten_xml2db_document`), and `process_xml_file_with_xml2db`. +- `xsd.py`: XSD schema loading (`load_schema`) and parent/child element cardinality classification (`find_list_and_single_child_paths`). +- `jsonschema_xsv/`: `xsv_reader.py` (casts validated CSV/TSV rows to JSON Schema property types and resolves `x-xsv-config` parsing parameters) plus `xsv_validator/` (`qsv.py` wraps the external `qsv` binary for two-pass validation, normalization, and null replacement; `custom_metaschema.py` defines the `https://json-schema.kbase.us/draft/2020-12-xsv/schema` metaschema and `X_XSV_CONFIG_SCHEMA`, also imported by `converters`; `schema_utils.py` and `helpers.py`). + +### `parsers/` + +Parsers for third-party outputs and formats: + +- `uniprot/`: `uniprot_kb.py` (UniProt XML), `uniref.py` (UniRef XML), `metalink.py`, `relnotes.py`, `idmapping.schema.json`; shared XML helpers in `utils/xml_utils.py`. +- `ncbi/datasets_api/openapi_pydantic.py`: pydantic models for NCBI Datasets API responses (bundled `AnnotationReportResponse.schema.json` and `DatasetReportResponse.schema.json` alongside). +- `ncbi/ftp_mod_times/`: `ftp_mod_times.schema.json` for NCBI FTP modification-time files. +- `bbmap_stats.py` (bbmap `stats.sh` output), `checkm2.py` (quality reports), `pdb_holdings_file.py`. +- `all_the_bacteria/schema/` and `gtdb/schema/`: JSON Schemas for ATB and GTDB tabular files (no Python modules). + +### `ncbi_ftp/` + +NCBI FTP assembly domain logic, phased: `manifest.py` (phase 1: assembly summary diffing and manifest generation), download is the phase 2 pipeline, `promote.py` (phase 3: promote staged files to final Lakehouse paths in S3), plus `assembly.py` (assembly-specific logic), `metadata.py` (frictionless data package descriptors), `constants.py`. + +### `pdb/` + +PDB pipeline constants, enums, and globals (`constants.py`); the pipeline itself is `pipelines/pdb_manifest.py`. + +### `pipelines/` + +dlt-based orchestrators; each exposes a `cli = PipelineCli(...)` registered in `pyproject.toml`: + +- `core.py`: `PipelineCli`, `run_cli`, `run_pipeline`, `setup_dlt_Pipeline`, `execute_dlt_pipeline`, and `configure_pipeline_logging`. +- `all_the_bacteria.py`: All The Bacteria bulk data downloader (excluded from fast test runs). +- `jsonlines/`: JSONL ingest pipelines (`settings.py`, shared machinery in `base.py`) - `extract_pipeline.py` (`jsonl_extract`, plain load into `` / `_invalid`), `extract_jsonschema_validate_pipeline.py` (`jsonl_jsonschema`, validates against `ENTITY_SCHEMAS`), `extract_pydantic_validate_pipeline.py` (`jsonl_pydantic`, validates against `ENTITY_MODELS`). See `docs/jsonschema_ingest.md`. +- `ncbi_ftp_download.py` / `ncbi_ftp_promote.py`: phases 2 (`ncbi_ftp_sync`) and 3 (`ncbi_ftp_promote`) of the NCBI FTP flow. +- `ncbi_rest_api.py`: NCBI REST API import. +- `pdb_manifest.py`: PDB holdings transfer manifest. +- `uniprot_kb.py` / `uniref.py`: UniProt (`uniprot`) and UniRef (`uniref`) dlt pipelines. +- `xml2db/` / `xmltodict/` / `xsv/`: XML-to-relational, XML-to-dictionary, and XSV (CSV/TSV) ingestion, each with `pipeline.py` and `settings.py`; `xml2db/compaction.py` runs DuckDB-based post-load deduplication of "reused" tables on filesystem destinations. + +### `utils/` + +Cross-cutting helpers: `batcher.py` (`NumericFileSequenceBatcher`, `get_file_batches`), `buffer.py` (per-table row buffers for dlt: `DictBuffer`, `ListBuffer`, `ArrowDictBuffer`, `ArrowListBuffer`), `cdm_logger.py` (structured JSON logging), `checksums.py`, `gz.py` (`open_maybe_gzip`), `helpers.py`, `progress.py` (tqdm helpers), `spark.py`, `xml_utils.py`, `jsonschema/dereferencer.py` (`$ref` resolution), `dlt/dlt_normalization.py` (lossless parent-linked normalization of dlt table definitions, used by the converters' `DltReader`), and `file_transfer/` - `core.py` (tenacity-retried downloads), `sync_client.py` / `async_client.py` / `ftp_client.py`, and `s3/` (`client.py`, `object_utils.py`, HTTP-to-S3 streaming in `streamer.py`, `transfer_config.py`, `cli.py`). + +### Other Top-Level Directories + +- `scripts/`: `entrypoint.sh` (container entrypoint dispatching to registered pipeline CLIs), `run_tests.sh` (`unit`, `container`, or `all` test suites), `filter_ncbi_updates.py` (CLI filtering NCBI assembly IDs by modification date), `s3_local.py` (thin S3 CLI for local Ceph testing), `dtd2xsd.pl` (DTD-to-XSD converter), `run_tools.sh`, `parse_index.py`. +- `docs/`: architecture and pipeline guides (`converters_ir.md`, `jsonschema_ingest.md`, `ncbi_ftp_e2e_walkthrough.md`, `pdb_e2e_walkthrough.md`, `rclone_instructions.md`, `uniprot.md`). +- `cdl_skill/`: skill definition (`SKILL.md`) and generator (`scripts/skill_gen.sh`) for packaging pipeline documentation and schemas. + +## Conventions + +- Frozen, slotted dataclasses for models; constructors validate and recursively copy mappings. +- CLI argument parsing by pydantic-settings +- Fully type-annotated code (ruff `ANN`); datetime values carry an explicit timezone (`DTZ`); use `Decimal` for exact arithmetic, never constructed from floats. +- External HTTP calls in tests are recorded and replayed with vcrpy; AWS is mocked with moto; Spark and Ceph tests run in containers via `docker-compose.yml` and are skipped by default (`-m "not requires_spark and not requires_ceph"`). +- Tests are module-based `def test_*() -> None:` functions with pytest fixtures in `conftest.py`, named `test___`. + +## Gotchas + +- dlt 1.30 DuckDB destination fails with a Binder Error "Ambiguous reference to catalog or schema" when the db filename stem equals the dataset name; keep the db filename distinct from both pipeline and dataset names. +- dlt 1.30: `AnyDestination` is a quoted TypeAlias (a str at runtime), so `AnyDestination | None` in an evaluated annotation raises TypeError on import. Use `TDestinationReferenceArg` from `dlt.common.destination.reference`. +- pyiceberg SqlCatalog engines leak sqlite connections; use the context-manager protocol (`with load_catalog(...) as catalog`) so `close()` disposes the engine. +- pipelines are run on the KBase CTS (Task Service), which only saves the contents of the output directory (or dlt destination) if the container exits with exit code 0. For any other error code, the container STDERR and STDOUT are saved, but nothing else. As a result, even if pipelines experience failure (e.g. `load_info.has_failed_packages` is true), the container must exit 0 to ensure that logs, parsed data, and other artefacts are not deleted. Errors that occur before a pipeline is run (e.g. argument parsing errors) are not caught, so they should be picked up by the container STDERR/STDOUT. diff --git a/pyproject.toml b/pyproject.toml index 9b205d15..c68d127a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "pandas>=3.0.3", "pydantic>=2.13.4", "pydantic-settings>=2.14.1", - "pyiceberg[pandas,pyarrow,pyiceberg-core,snappy]>=0.11.1", + "pyiceberg[pandas,pyarrow,pyiceberg-core,snappy]>=0.12", "smart-open>=8.0.1", "tqdm>=4.70.1", "xml2db>=0.14.0", @@ -39,9 +39,9 @@ dependencies = [ [project.scripts] all_the_bacteria = "cdm_data_loaders.pipelines.all_the_bacteria:cli" -jsonl_extract = "cdm_data_loaders.pipelines.jsonlines.extract_pipeline:cli" -jsonl_jsonschema = "cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline:cli" -jsonl_pydantic = "cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline:cli" +jsonlines_extract = "cdm_data_loaders.pipelines.jsonlines.extract_pipeline:cli" +jsonlines_jsonschema = "cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline:cli" +jsonlines_pydantic = "cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline:cli" ncbi_ftp_promote = "cdm_data_loaders.pipelines.ncbi_ftp_promote:cli" ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" @@ -51,7 +51,7 @@ uniref = "cdm_data_loaders.pipelines.uniref:cli" xmltodict = "cdm_data_loaders.pipelines.xmltodict.pipeline:cli" xml2db = "cdm_data_loaders.pipelines.xml2db.pipeline:cli" xsv = "cdm_data_loaders.pipelines.xsv.pipeline:cli" - +hello_world = "cdm_data_loaders.utils.hello_world:cli" [dependency-groups] dev = [ @@ -75,6 +75,10 @@ models = [ xml = [ "xsdata[cli,lxml]>=26.2", ] +mutants = [ + "mutmut>=3.8.0", + "pytest-gremlins>=1.12.2", +] [tool.ruff] line-length = 120 @@ -213,6 +217,12 @@ max-complexity = 15 requires = ["uv_build>=0.12.0,<0.20.0"] build-backend = "uv_build" +[tool.mutmut] +source_paths = [ "src/" ] +pytest_add_cli_args_test_selection= [ "-m", "not requires_spark", "-m", "not slow_test", "-m", "not requires_ceph", "-m", "not external_request", "-m", "not docker", "--ignore", "tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py" ] + + + [tool.pytest] pythonpath = ["src"] log_cli = true @@ -225,6 +235,7 @@ markers = [ "slow_test: does what it says on the tin", "requires_ceph: end-to-end tests requiring a running CEPH instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)", + "docker: tests that build and run the pipeline inside a Docker container", "settings_cls(cls): the pydantic-settings class under test for alias parametrization", "cli_fields: used for restricting cli-related tests to a subset of fields" ] @@ -243,13 +254,18 @@ AWS_SECRET_ACCESS_KEY = "test_access_secret" # required for KBase Lakehouse env USER = "fake_user" KBASE_AUTH_TOKEN = "test-token-123" -CDM_TASK_SERVICE_URL = "http://localhost:8080" -BERDL_POD_IP = "192.168.1.100" -SPARK_MASTER_URL = "spark://localhost:7077" + BERDL_HIVE_METASTORE_URI = "thrift://localhost:9083" -SPARK_CLUSTER_MANAGER_API_URL = "http://localhost:8000" -GOVERNANCE_API_URL = "http://localhost:8000" +# BERDL_POD_IP = "192.168.1.100" +BERDL_POD_IP = "spark-notebook" +CDM_TASK_SERVICE_URL = "http://localhost:8080" DATALAKE_MCP_SERVER_URL = "http://localhost:8080" +GOVERNANCE_API_URL = "http://localhost:8000" +POLARIS_CATALOG_URI = "http://polaris:8181/api/catalog" +SPARK_CLUSTER_MANAGER_API_URL = "http://localhost:8000" +SPARK_MASTER_URL = "spark://localhost:7077" +# SPARK_MASTER_URL = "spark://spark-master:7077" +DLT_PROJECT_DIR = "/tmp/dlt" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } diff --git a/src/cdm_data_loaders/pipelines/jsonlines/settings.py b/src/cdm_data_loaders/pipelines/jsonlines/settings.py index 0b32c74b..6a82e588 100644 --- a/src/cdm_data_loaders/pipelines/jsonlines/settings.py +++ b/src/cdm_data_loaders/pipelines/jsonlines/settings.py @@ -22,9 +22,9 @@ default_settings_with_shortcuts, ) -EXTRACT_PIPELINE_NAME: Final[str] = "jsonlines" -PYDANTIC_PIPELINE_NAME: Final[str] = "jsonlines_pydantic_validator" -JSONSCHEMA_PIPELINE_NAME: Final[str] = "jsonlines_jsonschema_validator" +EXTRACT_PIPELINE_NAME: Final[str] = "jsonlines_extract" +PYDANTIC_PIPELINE_NAME: Final[str] = "jsonlines_pydantic" +JSONSCHEMA_PIPELINE_NAME: Final[str] = "jsonlines_jsonschema" ENTITY_MODELS_MODULE: Final[str] = "entity_models_module" SCHEMA_FILES_MODULE: Final[str] = "schema_files_module" diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index b083803e..c3589a7a 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -15,14 +15,15 @@ from ftplib import error_temp from logging import Logger, getLogger from pathlib import Path, PurePosixPath -from typing import Any +from typing import Any, Final import tqdm from pydantic import AliasChoices, Field +from pydantic_settings import SettingsConfigDict from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from cdm_data_loaders.core.fields import INPUT_MOUNT, OUTPUT_MOUNT -from cdm_data_loaders.core.settings import LoggerSettings +from cdm_data_loaders.core.settings import LoggerSettings, default_settings_with_shortcuts from cdm_data_loaders.ncbi_ftp.assembly import ( FTP_HOST, build_accession_path, @@ -36,13 +37,15 @@ logger: Logger = getLogger(__name__) - +PIPELINE_NAME: Final[str] = "ncbi_ftp_sync" DEFAULT_STAGING_KEY_PREFIX: PurePosixPath = PurePosixPath("staging") class DownloadSettings(LoggerSettings): """Configuration for the NCBI FTP assembly download pipeline.""" + model_config: SettingsConfigDict = default_settings_with_shortcuts(cli_prog_name=PIPELINE_NAME) + manifest: Path = Field( default=Path(INPUT_MOUNT) / "transfer_manifest.txt", description="Path to the transfer manifest file listing FTP paths to download", diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py index 288d317b..33f17387 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py @@ -6,14 +6,16 @@ import logging from pathlib import Path, PurePosixPath +from typing import Final from pydantic import AliasChoices, Field -from pydantic_settings import CliImplicitFlag +from pydantic_settings import CliImplicitFlag, SettingsConfigDict -from cdm_data_loaders.core.settings import LoggerSettings +from cdm_data_loaders.core.settings import LoggerSettings, default_settings_with_shortcuts from cdm_data_loaders.ncbi_ftp.promote import promote_from_s3 from cdm_data_loaders.pipelines.core import PipelineCli +PIPELINE_NAME: Final[str] = "ncbi_ftp_promote" DEFAULT_STAGING_BUCKET: PurePosixPath = PurePosixPath("cts") DEFAULT_DESTINATION_BUCKET: PurePosixPath = PurePosixPath("cdm-lake") DEFAULT_DESTINATION_PREFIX: PurePosixPath = PurePosixPath("tenant-general-warehouse/kbase/datasets/ncbi") @@ -26,6 +28,8 @@ class PromoteSettings(LoggerSettings): """Configuration for the NCBI FTP file promotion pipeline.""" + model_config: SettingsConfigDict = default_settings_with_shortcuts(cli_prog_name=PIPELINE_NAME) + staging_bucket: PurePosixPath = Field( default=DEFAULT_STAGING_BUCKET, description="Bucket where staged files are located after download", diff --git a/src/cdm_data_loaders/pipelines/uniprot_kb.py b/src/cdm_data_loaders/pipelines/uniprot_kb.py index 917e7c22..7ab79185 100644 --- a/src/cdm_data_loaders/pipelines/uniprot_kb.py +++ b/src/cdm_data_loaders/pipelines/uniprot_kb.py @@ -21,7 +21,7 @@ ) from cdm_data_loaders.readers.xml import build_xml_file_resource -PIPELINE_NAME: Final[str] = "uniprot_kb" +PIPELINE_NAME: Final[str] = "uniprot" UNIPROT_LOG_INTERVAL: Final[int] = 1000 @@ -29,7 +29,7 @@ class UniProtSettings(CtsSettings): """Configuration for running the UniProt KB import pipeline.""" model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name="uniprot", cli_shortcuts=CLI_SHORTCUTS + cli_prog_name=PIPELINE_NAME, cli_shortcuts=CLI_SHORTCUTS ) log_interval: Annotated[ diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index ad71aae4..20a58e88 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -22,6 +22,12 @@ LOAD_INFO_TABLE_NAME, ) from cdm_data_loaders.utils import cdm_logger +from tests.cdm_data_loaders.pipelines.framework import ( + EXECUTION_MODES, + ExecutionMode, + PipelineTestRunner, + ensure_docker_image, +) from tests.conftest import LOCAL_FS, S3 DEFAULT_DLT_TABLES = {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", LOAD_INFO_TABLE_NAME} @@ -120,11 +126,32 @@ def mock_dlt(monkeypatch: pytest.MonkeyPatch) -> MagicMock: return mock +@pytest.fixture(params=EXECUTION_MODES) +def execution_mode(request: pytest.FixtureRequest) -> ExecutionMode: + """Execution mode for pipeline end-to-end tests (core, cli, or docker).""" + return request.param + + +@pytest.fixture(scope="session") +def docker_image(request: pytest.FixtureRequest) -> str: + """Build or resolve the Docker image tag used for @pytest.mark.docker pipeline tests.""" + configured_tag = request.config.getoption("--docker-image") + return ensure_docker_image(configured_tag) + + +@pytest.fixture +def pipeline_runner(execution_mode: ExecutionMode, request: pytest.FixtureRequest) -> PipelineTestRunner: + """Provide a PipelineTestRunner configured for the active execution_mode.""" + image_tag = request.config.getoption("--docker-image") if execution_mode == ExecutionMode.DOCKER else None + return PipelineTestRunner(mode=execution_mode, docker_image=image_tag) + + def isolated_run( tmp_path: Path, dataset_prefix: str, run_fn: Callable[..., LoadInfo | None], base_kwargs: dict[str, Any], + pipeline_runner: PipelineTestRunner | None = None, ) -> Callable[..., tuple[LoadInfo | None, Path]]: """Return a factory that runs a pipeline with per-run isolated output state. @@ -139,6 +166,8 @@ def isolated_run( :type run_fn: Callable[..., LoadInfo | None] :param base_kwargs: base settings kwargs; keys open to override per call :type base_kwargs: dict[str, Any] + :param pipeline_runner: optional PipelineTestRunner to dispatch through + :type pipeline_runner: PipelineTestRunner | None :return: factory(chunk_dir=None, **overrides) -> (load_info, output_dir) :rtype: Callable[..., tuple[LoadInfo | None, Path]] """ @@ -160,7 +189,7 @@ def _factory(**overrides: Any) -> tuple[LoadInfo | None, Path]: settings_cls = kwargs.pop("settings_cls") settings = settings_cls(**kwargs) with dlt.config.values({"destination.local_fs.bucket_url": str(settings.output_dir)}): - load_info = run_fn(settings) + load_info = pipeline_runner.run(run_fn, settings) if pipeline_runner is not None else run_fn(settings) return load_info, settings.output_dir return _factory @@ -171,9 +200,12 @@ def isolated_run_factory(tmp_path: Path) -> Callable[..., Callable[..., tuple[Lo """Return a builder for isolated pipeline run factories bound to this test's tmp_path.""" def _build( - dataset_prefix: str, run_fn: Callable[..., LoadInfo | None], base_kwargs: dict[str, Any] + dataset_prefix: str, + run_fn: Callable[..., LoadInfo | None], + base_kwargs: dict[str, Any], + pipeline_runner: PipelineTestRunner | None = None, ) -> Callable[..., tuple[LoadInfo | None, Path]]: - return isolated_run(tmp_path, dataset_prefix, run_fn, base_kwargs) + return isolated_run(tmp_path, dataset_prefix, run_fn, base_kwargs, pipeline_runner=pipeline_runner) return _build diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py new file mode 100644 index 00000000..825abffe --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -0,0 +1,723 @@ +"""Unified test execution framework for running CDM data ingestion pipelines in core, cli, or docker mode.""" + +import gzip +import json +import logging +import os +import shutil +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime +from enum import StrEnum +from importlib import import_module +from pathlib import Path +from types import ModuleType +from typing import Any, Final, Self +from unittest.mock import patch +from uuid import uuid4 + +import dlt +import pytest +from dlt.common.pipeline import LoadInfo +from dlt.destinations import filesystem + +from cdm_data_loaders.core.settings import CtsSettings, LoggerSettings +from cdm_data_loaders.pipelines import core +from cdm_data_loaders.pipelines.core import LOAD_INFO_TABLE_NAME, PipelineCli +from cdm_data_loaders.utils import cdm_logger +from tests.cdm_data_loaders.pipelines.helpers import settings_to_argv +from tests.conftest import BASE_DIR, CONFIG_BUCKET, LOCAL_FS + +DEFAULT_DOCKER_IMAGE_TAG: Final[str] = "cdm-data-loaders-test:latest" +CONTAINER_INPUT_DIR: Final[str] = "/input_dir" +CONTAINER_OUTPUT_DIR: Final[str] = "/output_dir" +CONTAINER_AUX_DIR: Final[str] = "/aux_dir" +CONTAINER_PYMODULES_DIR: Final[str] = "/pymodules" + +# Flags whose values are local file paths that must be staged into the container in Docker mode +FILE_PATH_FLAGS: Final[frozenset[str]] = frozenset( + { + "--xsd-file", + "--xml2db-config-file", + "--schema-file", + "-s", + "--log-config-file", + } +) + +# Flags whose values are Python module paths that may be dynamically created in sys.modules +MODULE_PATH_FLAGS: Final[frozenset[str]] = frozenset( + { + "--entity-models-module", + "--schema-files-module", + "-m", + } +) + +INPUT_DIR_FLAGS: Final[frozenset[str]] = frozenset({"--input-dir", "-i"}) +OUTPUT_DIR_FLAGS: Final[frozenset[str]] = frozenset({"--output-dir", "-o"}) +DESTINATION_FLAGS: Final[frozenset[str]] = frozenset({"--use-destination", "-d"}) + +# Registry mapping console-script / pipeline names to (cli_command, module_path) +PIPELINE_TARGETS: Final[dict[str, tuple[str, str]]] = { + "all_the_bacteria": ("all_the_bacteria", "cdm_data_loaders.pipelines.all_the_bacteria"), + "jsonlines_extract": ("jsonlines_extract", "cdm_data_loaders.pipelines.jsonlines.extract_pipeline"), + "jsonlines_jsonschema": ( + "jsonlines_jsonschema", + "cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline", + ), + "jsonlines_pydantic": ( + "jsonlines_pydantic", + "cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline", + ), + "ncbi_ftp_promote": ("ncbi_ftp_promote", "cdm_data_loaders.pipelines.ncbi_ftp_promote"), + "ncbi_ftp_sync": ("ncbi_ftp_sync", "cdm_data_loaders.pipelines.ncbi_ftp_download"), + "ncbi_rest_api": ("ncbi_rest_api", "cdm_data_loaders.pipelines.ncbi_rest_api"), + "pdb_manifest": ("pdb_manifest", "cdm_data_loaders.pipelines.pdb_manifest"), + "uniprot": ("uniprot", "cdm_data_loaders.pipelines.uniprot_kb"), + "uniref": ("uniref", "cdm_data_loaders.pipelines.uniref"), + "xmltodict": ("xmltodict", "cdm_data_loaders.pipelines.xmltodict.pipeline"), + "xml2db": ("xml2db", "cdm_data_loaders.pipelines.xml2db.pipeline"), + "xsv": ("xsv", "cdm_data_loaders.pipelines.xsv.pipeline"), +} + +MODULE_TO_CLI_COMMAND: Final[dict[str, str]] = { + module_path: cli_command for cli_command, module_path in PIPELINE_TARGETS.values() +} + +_built_docker_images: set[str] = set() + + +class ExecutionMode(StrEnum): + """Supported pipeline test execution modes.""" + + CORE = "core" + CLI = "cli" + DOCKER = "docker" + + +EXECUTION_MODES: Final[tuple[Any, ...]] = ( + pytest.param(ExecutionMode.CORE, id="core"), + pytest.param(ExecutionMode.CLI, id="cli"), + pytest.param(ExecutionMode.DOCKER, id="docker", marks=pytest.mark.docker), +) + + +@dataclass(frozen=True, slots=True) +class PipelineSpec: + """Resolved metadata for executing a pipeline across core, cli, and docker modes.""" + + cli_command: str + module_path: str + settings_cls: type[LoggerSettings] | None = None + pipeline_fn: Callable[[Any], LoadInfo | None] | None = None + cli_callable: Callable[[], Any] | None = None + + +class _PreservedStrDatetime(datetime): + """Datetime subclass that preserves the exact string representation written to the report file.""" + + _raw_str: str + + def __new__(cls, raw_str: str) -> Self: + """Create a datetime from an ISO-formatted string while preserving its original string spelling.""" + dt = datetime.fromisoformat(raw_str) + instance = super().__new__( + cls, + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + tzinfo=dt.tzinfo, + ) + instance._raw_str = raw_str + return instance + + def __str__(self) -> str: + """Return the verbatim timestamp string from the pipeline report.""" + return self._raw_str + + +@dataclass +class ReconstructedLoadInfo: + """LoadInfo-compatible object reconstructed from pipeline output files on disk.""" + + pipeline: dlt.Pipeline + dataset_name: str + loads_ids: list[str] + started_at: datetime + has_failed_jobs: bool = False + finished_at: datetime | None = None + first_run: bool = False + destination_name: str = LOCAL_FS + destination_type: str = "filesystem" + load_packages: list[Any] = field(default_factory=list) + metrics: dict[str, Any] = field(default_factory=dict) + _raw_dict: dict[str, Any] = field(default_factory=dict, repr=False) + + def asdict(self) -> dict[str, Any]: + """Return a dictionary representation matching dlt's LoadInfo.asdict().""" + if self._raw_dict: + return dict(self._raw_dict) + return { + "dataset_name": self.dataset_name, + "destination_name": self.destination_name, + "destination_type": self.destination_type, + "finished_at": self.finished_at, + "first_run": self.first_run, + "has_failed_jobs": self.has_failed_jobs, + "loads_ids": list(self.loads_ids), + "pipeline": {"pipeline_name": self.pipeline.pipeline_name}, + "started_at": self.started_at, + } + + +type PipelineIdentifier = str | ModuleType | PipelineCli[Any] | PipelineSpec | Callable[..., LoadInfo | None] + + +def _spec_from_module(cli_command: str, module: ModuleType) -> PipelineSpec: + """Build a PipelineSpec by inspecting a pipeline module.""" + cli_attr = getattr(module, "cli", None) + if isinstance(cli_attr, PipelineCli): + return PipelineSpec( + cli_command=cli_command, + module_path=module.__name__, + settings_cls=cli_attr.settings_cls, + pipeline_fn=cli_attr.pipeline_fn, + cli_callable=cli_attr, + ) + return PipelineSpec( + cli_command=cli_command, + module_path=module.__name__, + settings_cls=getattr(module, "Settings", None), + pipeline_fn=getattr(module, "run_pipeline", None), + cli_callable=cli_attr if callable(cli_attr) else None, + ) + + +def resolve_pipeline(pipeline: PipelineIdentifier) -> PipelineSpec: + """Resolve a pipeline name, module, PipelineCli, or callable into a PipelineSpec. + + :param pipeline: pipeline identifier (name, module, PipelineCli, PipelineSpec, or run function) + :type pipeline: PipelineIdentifier + :raises ValueError: if the pipeline cannot be resolved + :return: resolved pipeline specification + :rtype: PipelineSpec + """ + if isinstance(pipeline, PipelineSpec): + return pipeline + + if isinstance(pipeline, PipelineCli): + module_path = pipeline.pipeline_fn.__module__ + cli_command = MODULE_TO_CLI_COMMAND.get(module_path, module_path.rsplit(".", maxsplit=1)[-1]) + return PipelineSpec( + cli_command=cli_command, + module_path=module_path, + settings_cls=pipeline.settings_cls, + pipeline_fn=pipeline.pipeline_fn, + cli_callable=pipeline, + ) + + if isinstance(pipeline, ModuleType): + module_path = pipeline.__name__ + short_key = module_path.removeprefix("cdm_data_loaders.pipelines.") + cli_command = MODULE_TO_CLI_COMMAND.get( + module_path, + PIPELINE_TARGETS.get(short_key, (short_key.rsplit(".", maxsplit=1)[-1], module_path))[0], + ) + return _spec_from_module(cli_command, pipeline) + + if callable(pipeline): + module_path = getattr(pipeline, "__module__", "") + if module_path in MODULE_TO_CLI_COMMAND: + module = import_module(module_path) + return _spec_from_module(MODULE_TO_CLI_COMMAND[module_path], module) + err_msg = f"Cannot resolve pipeline from callable {pipeline!r} (module {module_path!r})" + raise ValueError(err_msg) + + if isinstance(pipeline, str): + normalized = pipeline.strip().removeprefix("cdm_data_loaders.pipelines.") + if normalized in PIPELINE_TARGETS: + cli_command, module_path = PIPELINE_TARGETS[normalized] + module = import_module(module_path) + return _spec_from_module(cli_command, module) + full_module_path = f"cdm_data_loaders.pipelines.{normalized}" + if full_module_path in MODULE_TO_CLI_COMMAND: + cli_command = MODULE_TO_CLI_COMMAND[full_module_path] + module = import_module(full_module_path) + return _spec_from_module(cli_command, module) + + err_msg = f"Unknown pipeline {pipeline!r}. Known pipelines: {', '.join(sorted(PIPELINE_TARGETS))}" + raise ValueError(err_msg) + + +def _read_saved_load_info_row(dataset_dir: Path) -> dict[str, Any]: + """Read the serialized load_info row from ``/_load_info`` if present.""" + load_info_dir = dataset_dir / LOAD_INFO_TABLE_NAME + if not load_info_dir.is_dir(): + return {} + for file_path in sorted(load_info_dir.glob("*.jsonl*")): + if not file_path.is_file(): + continue + with file_path.open("rb") as fh: + is_gzipped = fh.read(2) == b"\x1f\x8b" + opener = gzip.open if is_gzipped else open + with opener(file_path, "rt", encoding="utf-8") as fh: + for line in fh: + stripped = line.strip() + if stripped: + return json.loads(stripped) + return {} + + +def reconstruct_load_info(output_dir: Path, bucket_dir: Path | None = None) -> LoadInfo | None: + """Reconstruct a LoadInfo-compatible object and attached dlt.Pipeline from disk artifacts. + + Reads the ``report_.txt`` file written by ``save_pipeline_metadata`` in + ``output_dir`` and attaches a ``dlt.Pipeline`` pointing at the filesystem destination + containing ````. + + :param output_dir: directory containing ``report_.txt`` + :type output_dir: Path + :param bucket_dir: optional separate filesystem destination bucket directory + :type bucket_dir: Path | None + :return: reconstructed LoadInfo, or None if the report indicates an error or no report exists + :rtype: LoadInfo | None + """ + output_dir = Path(output_dir) + report_files = sorted(output_dir.glob("report_*.txt"), key=lambda path: path.stat().st_mtime) + if not report_files: + return None + + report_file = report_files[-1] + lines = report_file.read_text(encoding="utf-8").splitlines() + if len(lines) < 3: # noqa: PLR2004 + return None + + # Line 0: "Pipeline dataset " + header_prefix = "Pipeline " + dataset_sep = " dataset " + if not lines[0].startswith(header_prefix) or dataset_sep not in lines[0]: + return None + pipeline_name, dataset_name = lines[0].removeprefix(header_prefix).rsplit(dataset_sep, maxsplit=1) + pipeline_name = pipeline_name.strip() + dataset_name = dataset_name.strip() + + # Line 1: "Started: " + started_str = lines[1].removeprefix("Started: ").strip() + started_at = _PreservedStrDatetime(started_str) + + # Line 2: "Status: completed" | "Status: completed with errors" | "Status: error" + status_str = lines[2].removeprefix("Status: ").strip() + if status_str == "error": + return None + has_failed_jobs = status_str != "completed" + + # Line 3+: "Completed loads: , , ..." + loads_ids: list[str] = [] + for line in lines[3:]: + if line.startswith("Completed loads:"): + loads_raw = line.removeprefix("Completed loads:").strip() + loads_ids = [item.strip() for item in loads_raw.split(",") if item.strip()] + break + + # Determine where the dataset directory actually lives on disk + if (output_dir / dataset_name).is_dir(): + actual_bucket_dir = output_dir + elif bucket_dir is not None and (Path(bucket_dir) / dataset_name).is_dir(): + actual_bucket_dir = Path(bucket_dir) + else: + actual_bucket_dir = Path(bucket_dir) if bucket_dir is not None else output_dir + + saved_row = _read_saved_load_info_row(actual_bucket_dir / dataset_name) + finished_at_raw = saved_row.get("finished_at") + finished_at = _PreservedStrDatetime(str(finished_at_raw)) if finished_at_raw else None + + # Use a fresh working directory outside output_dir so dlt syncs the schema from the destination + # without colliding with stale schemas in ~/.dlt/pipelines or adding files to output_dir. + pipeline_e2e_id = report_file.stem.removeprefix("report_") + reconstructed_pipelines_dir = output_dir.parent / f".reconstructed_dlt_{pipeline_e2e_id}" + pipeline = dlt.pipeline( + pipeline_name=pipeline_name, + destination=filesystem(bucket_url=str(actual_bucket_dir)), + dataset_name=dataset_name, + pipelines_dir=str(reconstructed_pipelines_dir), + ) + + return ReconstructedLoadInfo( # pyright: ignore[reportReturnType] + pipeline=pipeline, + dataset_name=dataset_name, + loads_ids=loads_ids, + started_at=started_at, + has_failed_jobs=has_failed_jobs, + finished_at=finished_at, + first_run=bool(saved_row.get("first_run", False)), + destination_name=str(saved_row.get("destination_name", LOCAL_FS)), + destination_type=str(saved_row.get("destination_type", "filesystem")), + _raw_dict=saved_row, + ) + + +def ensure_docker_image(image_tag: str | None = None, *, force_rebuild: bool = False) -> str: + """Build the Docker image from the repository Dockerfile if not already built in this session. + + :param image_tag: optional image tag; defaults to ``cdm-data-loaders-test:latest`` + :type image_tag: str | None + :param force_rebuild: force running ``docker build`` even if already built in this session + :type force_rebuild: bool + :raises RuntimeError: if ``docker build`` fails + :return: Docker image tag + :rtype: str + """ + tag = image_tag or os.environ.get("CDM_TEST_DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE_TAG + if not force_rebuild and tag in _built_docker_images: + return tag + + cmd = ["docker", "build", "-t", tag, str(BASE_DIR.resolve())] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) # noqa: S603 + if result.returncode != 0: + err_msg = ( + f"Docker image build failed (exit {result.returncode}):\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + raise RuntimeError(err_msg) + + _built_docker_images.add(tag) + return tag + + +def _extract_flag_value(cli_args: list[str], flags: frozenset[str]) -> str | None: + """Extract the value immediately following any of ``flags`` in ``cli_args``.""" + for idx, arg in enumerate(cli_args): + if arg in flags and idx + 1 < len(cli_args): + return cli_args[idx + 1] + for flag in flags: + if arg.startswith(f"{flag}="): + return arg.split("=", maxsplit=1)[1] + return None + + +def _normalize_cli_args(cli_args: list[str]) -> list[str]: + """Strip 0-byte ``.conf`` placeholder log files created by test fixtures that bypass ``init_logger``.""" + normalized: list[str] = [] + idx = 0 + while idx < len(cli_args): + arg = cli_args[idx] + if arg == "--log-config-file" and idx + 1 < len(cli_args): + log_path = Path(cli_args[idx + 1]) + if log_path.suffix.lower() == ".conf" and log_path.exists() and log_path.stat().st_size == 0: + idx += 2 + continue + normalized.append(arg) + idx += 1 + return normalized + + +def _materialize_dynamic_module(module_name: str, pymodules_dir: Path) -> bool: + """If ``module_name`` is an in-memory module in ``sys.modules``, write a ``.py`` file for Docker.""" + module = sys.modules.get(module_name) + if module is None or getattr(module, "__file__", None) is not None: + return False + + pymodules_dir.mkdir(parents=True, exist_ok=True) + target_file = pymodules_dir / f"{module_name}.py" + + if hasattr(module, "ENTITY_SCHEMAS"): + schemas = module.ENTITY_SCHEMAS + target_file.write_text(f"ENTITY_SCHEMAS = {schemas!r}\n", encoding="utf-8") + return True + + if hasattr(module, "ENTITY_MODELS"): + models: dict[str, Any] = module.ENTITY_MODELS + imports: list[str] = [] + entries: list[str] = [] + for idx, (table_name, model_cls) in enumerate(models.items()): + alias = f"_Model_{idx}" + imports.append(f"from {model_cls.__module__} import {model_cls.__name__} as {alias}") + entries.append(f" {table_name!r}: {alias},") + code = "\n".join([*imports, "", "ENTITY_MODELS = {", *entries, "}", ""]) + target_file.write_text(code, encoding="utf-8") + return True + + return False + + +def _write_docker_dlt_config(config_path: Path, use_destination: str) -> None: + """Write ``config.toml`` for the container with ``/input_dir`` and ``/output_dir`` buckets.""" + config_path.parent.mkdir(parents=True, exist_ok=True) + destinations = {"cts", LOCAL_FS, use_destination} + blocks: list[str] = [] + for dest in sorted(destinations): + blocks.append(f'[destination.{dest}]\nbucket_url = "{CONTAINER_OUTPUT_DIR}"\ndestination_type = "filesystem"\n') + blocks.append(f'[sources.{dest}]\nbucket_url = "{CONTAINER_INPUT_DIR}"\n') + config_path.write_text("\n".join(blocks), encoding="utf-8") + + +class PipelineTestRunner: + """Execute any CDM ingestion pipeline in ``core``, ``cli``, or ``docker`` mode.""" + + def __init__(self, mode: ExecutionMode | str = ExecutionMode.CORE, docker_image: str | None = None) -> None: + """Initialize the runner with the target execution mode and optional Docker image tag.""" + self.mode = ExecutionMode(mode) + self.docker_image = docker_image + + def __call__( + self, + pipeline: PipelineIdentifier, + settings: CtsSettings | LoggerSettings | None = None, + *, + cli_args: list[str] | None = None, + **overrides: Any, # noqa: ANN401 + ) -> LoadInfo | None: + """Alias for :meth:`run` so the runner can be passed as a ``run_fn`` callback.""" + return self.run(pipeline, settings, cli_args=cli_args, **overrides) + + def run( + self, + pipeline: PipelineIdentifier, + settings: CtsSettings | LoggerSettings | None = None, + *, + cli_args: list[str] | None = None, + **overrides: Any, # noqa: ANN401 + ) -> LoadInfo | None: + """Execute ``pipeline`` in the configured mode and return its ``LoadInfo``. + + :param pipeline: pipeline name, module, PipelineCli, PipelineSpec, or pipeline run function + :type pipeline: PipelineIdentifier + :param settings: optional pre-built settings instance + :type settings: CtsSettings | LoggerSettings | None + :param cli_args: optional explicit CLI argument list (excluding the program name) + :type cli_args: list[str] | None + :param overrides: optional field overrides applied if building or updating settings + :type overrides: Any + :raises ValueError: if neither settings nor cli_args are provided and settings cannot be built + :return: pipeline LoadInfo (or ReconstructedLoadInfo in cli/docker modes) + :rtype: LoadInfo | None + """ + spec = resolve_pipeline(pipeline) + effective_settings = settings + if effective_settings is not None and overrides: + effective_settings = effective_settings.model_copy(update=overrides) + elif effective_settings is None and cli_args is None: + if spec.settings_cls is None: + err_msg = f"Pipeline {spec.cli_command!r} has no settings_cls; supply cli_args explicitly." + raise ValueError(err_msg) + effective_settings = spec.settings_cls(**overrides) + + raw_args = list(cli_args) if cli_args is not None else settings_to_argv(effective_settings) + # Strip leading program name if the caller passed a full sys.argv list + if raw_args and not raw_args[0].startswith("-"): + raw_args = raw_args[1:] + args = _normalize_cli_args(raw_args) + + output_dir_str = _extract_flag_value(args, OUTPUT_DIR_FLAGS) or ( + str(getattr(effective_settings, "output_dir", "")) if effective_settings else None + ) + if not output_dir_str: + err_msg = "Could not determine output_dir from settings or cli_args" + raise ValueError(err_msg) + output_dir = Path(output_dir_str) + output_dir.mkdir(parents=True, exist_ok=True) + + use_destination = ( + _extract_flag_value(args, DESTINATION_FLAGS) + or (str(getattr(effective_settings, "use_destination", LOCAL_FS)) if effective_settings else None) + or LOCAL_FS + ) + + if self.mode == ExecutionMode.CORE: + return self._run_core(spec, args, output_dir, use_destination) + if self.mode == ExecutionMode.CLI: + return self._run_cli(spec, args, output_dir, use_destination) + return self._run_docker(spec, args, output_dir, use_destination) + + def _resolve_bucket_dir(self, output_dir: Path, use_destination: str) -> Path: + """Determine the active host bucket directory for ``use_destination``.""" + configured = dlt.config.get(f"destination.{use_destination}.bucket_url") + default_buckets = {CONFIG_BUCKET.get(LOCAL_FS), "/configured/out/", None} + if configured and configured not in default_buckets and Path(str(configured)).exists(): + return Path(str(configured)) + return output_dir + + def _run_core( + self, + spec: PipelineSpec, + args: list[str], + output_dir: Path, + use_destination: str, + ) -> LoadInfo | None: + """Execute the pipeline in-process via ``cdm_data_loaders.pipelines.core.run_cli``.""" + if spec.settings_cls is None or spec.pipeline_fn is None: + err_msg = f"Pipeline {spec.cli_command!r} does not expose settings_cls and pipeline_fn for core.run_cli" + raise ValueError(err_msg) + + bucket_dir = self._resolve_bucket_dir(output_dir, use_destination) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + applied_key = cdm_logger.applied_key + try: + with dlt.config.values( + { + f"destination.{use_destination}.destination_type": "filesystem", + f"destination.{use_destination}.bucket_url": str(bucket_dir), + } + ): + return core.run_cli(spec.settings_cls, spec.pipeline_fn, cli_args=args) + finally: + for handler in root.handlers[:]: + root.removeHandler(handler) + for handler in handlers: + root.addHandler(handler) + root.setLevel(level) + cdm_logger.applied_key = applied_key + + def _run_cli( + self, + spec: PipelineSpec, + args: list[str], + output_dir: Path, + use_destination: str, + ) -> LoadInfo | None: + """Execute the pipeline via ``cdm_data_loaders.pipelines..cli()`` and reconstruct LoadInfo.""" + if spec.cli_callable is None: + err_msg = f"Pipeline {spec.cli_command!r} does not expose a cli() entry point" + raise ValueError(err_msg) + + bucket_dir = self._resolve_bucket_dir(output_dir, use_destination) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + applied_key = cdm_logger.applied_key + try: + with ( + patch.object(sys, "argv", [spec.cli_command, *args]), + dlt.config.values( + { + f"destination.{use_destination}.destination_type": "filesystem", + f"destination.{use_destination}.bucket_url": str(bucket_dir), + } + ), + ): + spec.cli_callable() + finally: + for handler in root.handlers[:]: + root.removeHandler(handler) + for handler in handlers: + root.addHandler(handler) + root.setLevel(level) + cdm_logger.applied_key = applied_key + + return reconstruct_load_info(output_dir, bucket_dir=bucket_dir) + + def _run_docker( + self, + spec: PipelineSpec, + args: list[str], + output_dir: Path, + use_destination: str, + ) -> LoadInfo | None: + """Execute the pipeline inside the Docker container and reconstruct LoadInfo from mounted output.""" + image_tag = ensure_docker_image(self.docker_image) + input_dir_str = _extract_flag_value(args, INPUT_DIR_FLAGS) + if not input_dir_str: + err_msg = "Docker execution mode requires --input-dir / -i in settings or cli_args" + raise ValueError(err_msg) + host_input_dir = Path(input_dir_str).resolve() + host_output_dir = output_dir.resolve() + + stage_dir = host_output_dir.parent / f".docker_stage_{uuid4().hex[:8]}" + staged_input_dir = stage_dir / "input_dir" + aux_dir = stage_dir / "aux" + pymodules_dir = stage_dir / "pymodules" + staged_input_dir.mkdir(parents=True, exist_ok=True) + + if host_input_dir.is_dir(): + shutil.copytree(host_input_dir, staged_input_dir, dirs_exist_ok=True) + + _write_docker_dlt_config(staged_input_dir / "dlt_conf" / "config.toml", use_destination) + + container_args = self._translate_args_for_docker(args, aux_dir, pymodules_dir) + + docker_cmd: list[str] = [ + "docker", + "run", + "--rm", + "-v", + f"{staged_input_dir}:{CONTAINER_INPUT_DIR}", + "-v", + f"{host_output_dir}:{CONTAINER_OUTPUT_DIR}", + "-v", + f"{(BASE_DIR / 'tests').resolve()}:/app/tests:ro", + ] + if aux_dir.is_dir(): + docker_cmd.extend(["-v", f"{aux_dir.resolve()}:{CONTAINER_AUX_DIR}:ro"]) + if pymodules_dir.is_dir(): + docker_cmd.extend( + [ + "-v", + f"{pymodules_dir.resolve()}:{CONTAINER_PYMODULES_DIR}:ro", + "-e", + f"PYTHONPATH={CONTAINER_PYMODULES_DIR}:/app/src:/app", + ] + ) + else: + docker_cmd.extend(["-e", "PYTHONPATH=/app/src:/app"]) + + docker_cmd.extend([image_tag, "pipeline", spec.cli_command, *container_args]) + + result = subprocess.run(docker_cmd, capture_output=True, text=True, check=False) # noqa: S603 + if result.returncode != 0: + stderr_and_stdout = f"{result.stdout}\n{result.stderr}".strip() + if "Unknown table name" in stderr_and_stdout or "ValueError" in stderr_and_stdout: + raise ValueError(stderr_and_stdout) + err_msg = ( + f"Docker pipeline '{spec.cli_command}' failed with exit code {result.returncode}:\n" + f"Command: {' '.join(docker_cmd)}\n" + f"STDOUT:\n{result.stdout}\n" + f"STDERR:\n{result.stderr}" + ) + raise RuntimeError(err_msg) + + return reconstruct_load_info(host_output_dir, bucket_dir=host_output_dir) + + def _translate_args_for_docker( + self, + args: list[str], + aux_dir: Path, + pymodules_dir: Path, + ) -> list[str]: + """Rewrite host file paths and materialize dynamic modules for container execution.""" + translated: list[str] = [] + idx = 0 + aux_counter = 0 + while idx < len(args): + token = args[idx] + if token in INPUT_DIR_FLAGS and idx + 1 < len(args): + translated.extend([token, CONTAINER_INPUT_DIR]) + idx += 2 + continue + if token in OUTPUT_DIR_FLAGS and idx + 1 < len(args): + translated.extend([token, CONTAINER_OUTPUT_DIR]) + idx += 2 + continue + if token in FILE_PATH_FLAGS and idx + 1 < len(args): + host_file = Path(args[idx + 1]).resolve() + aux_dir.mkdir(parents=True, exist_ok=True) + staged_name = f"{aux_counter}_{host_file.name}" + aux_counter += 1 + shutil.copy2(host_file, aux_dir / staged_name) + translated.extend([token, f"{CONTAINER_AUX_DIR}/{staged_name}"]) + idx += 2 + continue + if token in MODULE_PATH_FLAGS and idx + 1 < len(args): + module_name = args[idx + 1] + _materialize_dynamic_module(module_name, pymodules_dir) + translated.extend([token, module_name]) + idx += 2 + continue + translated.append(token) + idx += 1 + return translated diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index 5b78da68..b8d827be 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -96,7 +96,8 @@ def settings_to_argv(settings: Any) -> list[str]: """Derive CLI arguments from a settings instance. Every serialized field becomes a ``--kebab-case`` flag, with bools rendered - as their ``true``/``false`` CLI spelling and None-valued fields skipped. + as their ``true``/``false`` CLI spelling, sequence fields expanded after the + flag, and None-valued or computed fields skipped. :param settings: a CtsSettings (or derivative) instance :type settings: Any @@ -104,12 +105,16 @@ def settings_to_argv(settings: Any) -> list[str]: :rtype: list[str] """ argv: list[str] = [] - for name, value in settings.model_dump().items(): + computed_fields: set[str] = set(getattr(type(settings), "model_computed_fields", {}).keys()) + for name, value in settings.model_dump(exclude=computed_fields).items(): flag = f"--{name.replace('_', '-')}" if isinstance(value, bool): argv.extend([flag, "true" if value else "false"]) elif value is None: continue + elif isinstance(value, list | tuple): + if value: + argv.extend([flag, *(str(item) for item in value)]) else: argv.extend([flag, str(value)]) return argv diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py index 36cb2fa2..4a021cf1 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py @@ -18,6 +18,9 @@ run_jsonlines_with_validation_pipeline, ) from cdm_data_loaders.pipelines.jsonlines.settings import ( + EXTRACT_PIPELINE_NAME, + JSONSCHEMA_PIPELINE_NAME, + PYDANTIC_PIPELINE_NAME, JsonlExtractSettings, JsonlJsonschemaIngestSettings, JsonlPydanticIngestSettings, @@ -33,7 +36,7 @@ type PrepareReference = Callable[[dict[str, Any]], dict[str, Any]] -@pytest.fixture(params=["extract", "extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.fixture(params=[EXTRACT_PIPELINE_NAME, PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def pipeline_kind(request: pytest.FixtureRequest) -> str: """Run the same reference contract against all three JSONL pipelines.""" return request.param @@ -48,14 +51,14 @@ def run_reference_pipeline( ) -> PipelineRunner: """Run the selected JSONL pipeline on a chunk dir with per-run isolated output directories.""" settings_cls, registry_kwarg, dataset_prefix, file_glob = { - "extract": (JsonlExtractSettings, {"table_name": "dataset"}, "jsonl_extract_run", "**/*.jsonl*"), - "extract_validate": ( + EXTRACT_PIPELINE_NAME: (JsonlExtractSettings, {"table_name": "dataset"}, "jsonl_extract_run", "**/*.jsonl*"), + PYDANTIC_PIPELINE_NAME: ( JsonlPydanticIngestSettings, {"entity_models_module": "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models"}, "jsonl_extract_validate_run", "*.jsonl*", ), - "extract_jsonschema_validate": ( + JSONSCHEMA_PIPELINE_NAME: ( JsonlJsonschemaIngestSettings, {"schema_files_module": "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas"}, "jsonschema_run", @@ -63,9 +66,9 @@ def run_reference_pipeline( ), }[pipeline_kind] run_fn = { - "extract": run_jsonlines_pipeline, - "extract_validate": run_jsonlines_with_validation_pipeline, - "extract_jsonschema_validate": run_jsonschema_pipeline, + EXTRACT_PIPELINE_NAME: run_jsonlines_pipeline, + PYDANTIC_PIPELINE_NAME: run_jsonlines_with_validation_pipeline, + JSONSCHEMA_PIPELINE_NAME: run_jsonschema_pipeline, }[pipeline_kind] base_kwargs: dict[str, Any] = { "buffer_size": 10, @@ -90,7 +93,7 @@ def prepare_reference(pipeline_kind: str) -> PrepareReference: """Apply Pydantic defaults only for model validation; other pipelines preserve input.""" def _prepare(entry: dict[str, Any]) -> dict[str, Any]: - if pipeline_kind == "extract_validate": + if pipeline_kind == PYDANTIC_PIPELINE_NAME: return ENTITY_MODELS["dataset"].model_validate(entry).model_dump(mode="json") return entry @@ -106,13 +109,13 @@ def reference_registry_factory( """Register the requested tables using the selected pipeline's assembly contract.""" def _factory(table_names: list[str]) -> dict[str, str]: - if pipeline_kind == "extract_validate": + if pipeline_kind == PYDANTIC_PIPELINE_NAME: return { "entity_models_module": entity_models_module_factory( dict.fromkeys(table_names, ENTITY_MODELS["dataset"]) ) } - if pipeline_kind == "extract_jsonschema_validate": + if pipeline_kind == JSONSCHEMA_PIPELINE_NAME: return {"schema_files_module": schema_module_factory(dict.fromkeys(table_names, ENTITY_SCHEMAS["dataset"]))} return {} diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py index 4421a137..ec0b0e3e 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py @@ -1,28 +1,27 @@ """End-to-end tests for the jsonlines pipeline.""" -import sys from collections.abc import Callable from pathlib import Path import pytest from cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline import ( - cli, load_entity_models, - run_jsonlines_with_validation_pipeline, ) from cdm_data_loaders.pipelines.jsonlines.settings import PYDANTIC_PIPELINE_NAME, JsonlPydanticIngestSettings +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written def test_run_jsonlines_pipeline_pass_writes_expected_parquet( scenario_input_dir: Callable[[str], str], settings_factory: Callable[..., JsonlPydanticIngestSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """Running the pipeline on a mixed batch puts the correct rows in both output tables.""" settings = settings_factory(input_dir=scenario_input_dir("mixed")) - load_info = run_jsonlines_with_validation_pipeline(settings) + load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings) assert load_info is not None dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "c"] @@ -35,12 +34,13 @@ def test_run_jsonlines_pipeline_pass_writes_expected_parquet( def test_run_jsonlines_pipeline_fail_unknown_table_name_raises( settings_factory: Callable[..., JsonlPydanticIngestSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """A table name in table_names that is not in the entity model registry raises ValueError.""" settings = settings_factory(table_names=["not_a_real_table"]) with pytest.raises(ValueError, match="Unknown table name"): - run_jsonlines_with_validation_pipeline(settings) + pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings) def test_run_jsonlines_pipeline_pass_table_names_none_processes_every_registered_table( @@ -60,7 +60,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( dlt_destination_config: str, entity_models_module: str, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" log_config_file = tmp_path / "logging.json" @@ -85,9 +85,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--use-output-dir-for-pipeline-metadata", "true", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, cli_args=argv[1:]) assert load_info is not None assert not load_info.has_failed_jobs diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py index 3a3d4598..fa40b0f9 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py @@ -11,6 +11,11 @@ from jsonschema import FormatChecker from jsonschema.validators import validator_for +from cdm_data_loaders.pipelines.jsonlines.settings import ( + EXTRACT_PIPELINE_NAME, + JSONSCHEMA_PIPELINE_NAME, + PYDANTIC_PIPELINE_NAME, +) from tests.cdm_data_loaders.pipelines.helpers import ( BUFFER_SIZES, LOADER_FILE_FORMATS, @@ -34,7 +39,7 @@ @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str) -@pytest.mark.parametrize("pipeline_kind", ["extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.mark.parametrize("pipeline_kind", [PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def test_jsonlines_validation_pass_reference_records_load( run_reference_pipeline: Callable[..., Any], loader_file_format: str ) -> None: @@ -297,7 +302,7 @@ def test_jsonlines_pass_nesting_matches_pipeline_contract( tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) data_tables = {name for name in tables if not name.startswith("_dlt")} columns = set().union(*(row.keys() for row in tables["dataset"])) - if pipeline_kind != "extract": + if pipeline_kind != EXTRACT_PIPELINE_NAME: assert data_tables == {"dataset"} assert {"organism", "assembly_info", "additional_submitters"} <= columns else: diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py index 664d32a3..dfa873de 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py @@ -2,18 +2,14 @@ import gzip import json -import sys from collections.abc import Callable from pathlib import Path from typing import Any import pytest -from cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline import ( - cli, - run_jsonlines_pipeline, -) from cdm_data_loaders.pipelines.jsonlines.settings import JSONSCHEMA_PIPELINE_NAME, JsonlJsonschemaIngestSettings +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import ( LOADER_FILE_FORMATS, assert_pipeline_report_written, @@ -61,13 +57,14 @@ def test_run_jsonlines_pipeline_pass_persists_valid_and_rejected_rows( schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], loader_file_format: str, buffer_size: int, + pipeline_runner: PipelineTestRunner, ) -> None: """Mixed files load every row exactly once with intact nested data and rejection provenance.""" settings = schema_settings_factory( input_dir=str(schema_input_dir), loader_file_format=loader_file_format, buffer_size=buffer_size ) - load_info = run_jsonlines_pipeline(settings) + load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) assert load_info is not None assert not load_info.has_failed_jobs @@ -104,7 +101,9 @@ def test_run_jsonlines_pipeline_pass_persists_valid_and_rejected_rows( def test_run_jsonlines_pipeline_pass_all_invalid( - schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], loader_file_format: str + schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], + loader_file_format: str, + pipeline_runner: PipelineTestRunner, ) -> None: """An entirely invalid entity still produces its complete rejection table.""" settings = schema_settings_factory(loader_file_format=loader_file_format, buffer_size=1) @@ -112,7 +111,7 @@ def test_run_jsonlines_pipeline_pass_all_invalid( table_dir.mkdir() (table_dir / "invalid.jsonl").write_text(f"{MALFORMED_RAW}\n{TYPE_ERROR_RAW}", encoding="utf-8") - load_info = run_jsonlines_pipeline(settings) + load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) assert load_info is not None assert not load_info.has_failed_jobs @@ -134,15 +133,14 @@ def test_cli_pass_loads_selected_entity( schema_input_dir: Path, schema_settings_factory: Callable[..., JsonlJsonschemaIngestSettings], loader_file_format: str, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: """The real CTS CLI accepts a registry shortcut, table selection, and the requested file format.""" settings = schema_settings_factory(input_dir=str(schema_input_dir), loader_file_format=loader_file_format) - monkeypatch.setattr( - sys, - "argv", - [ - JSONSCHEMA_PIPELINE_NAME, + + load_info = pipeline_runner.run( + JSONSCHEMA_PIPELINE_NAME, + cli_args=[ "--input-dir", str(settings.input_dir), "--output-dir", @@ -164,8 +162,6 @@ def test_cli_pass_loads_selected_entity( ], ) - load_info = cli.run() - assert load_info is not None assert not load_info.has_failed_jobs assert load_info.dataset_name == "schema_cli" diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 13a6fb9b..31140384 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -1,16 +1,18 @@ """Error and boundary tests for all JSONL pipelines using assembly-report contracts.""" import json -import sys from collections.abc import Callable from pathlib import Path from typing import Any import pytest -from cdm_data_loaders.pipelines.jsonlines.extract_jsonschema_validate_pipeline import cli as jsonschema_cli -from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import cli as extract_cli -from cdm_data_loaders.pipelines.jsonlines.extract_pydantic_validate_pipeline import cli as validate_cli +from cdm_data_loaders.pipelines.jsonlines.settings import ( + EXTRACT_PIPELINE_NAME, + JSONSCHEMA_PIPELINE_NAME, + PYDANTIC_PIPELINE_NAME, +) +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, read_pipeline_tables, run_and_read from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( canonicalize_pipeline_entry, @@ -56,9 +58,11 @@ def test_jsonlines_fail_malformed_rows_preserve_provenance( json.dumps(canonicalize_reference_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in entries ) - suffix, error_key = ("invalid", "parse_error") if pipeline_kind == "extract" else ("rejected", "error_detail") + suffix, error_key = ( + ("invalid", "parse_error") if pipeline_kind == EXTRACT_PIPELINE_NAME else ("rejected", "error_detail") + ) rejected = sorted(tables[f"dataset_{suffix}"], key=lambda row: row["source_file"]) - prefix = "dataset/" if pipeline_kind == "extract" else "" + prefix = "dataset/" if pipeline_kind == EXTRACT_PIPELINE_NAME else "" with pytest.raises(json.JSONDecodeError) as error_info: json.loads(malformed) expected_error = str(error_info.value) @@ -147,7 +151,7 @@ def test_jsonlines_pass_multiple_entities_and_file_glob( tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) expected_tables = ( {"dataset": sorted(expected.values())} - if pipeline_kind == "extract" + if pipeline_kind == EXTRACT_PIPELINE_NAME else {table_name: [accession] for table_name, accession in expected.items()} ) assert { @@ -178,7 +182,7 @@ def test_jsonlines_pass_multiple_entities_and_file_glob( pytest.param("text", "model_type", [], "'text' is not of type 'object'", id="json-string"), ], ) -@pytest.mark.parametrize("pipeline_kind", ["extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.mark.parametrize("pipeline_kind", [PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def test_jsonlines_validation_fail_schema_invalid_rows_are_rejected( *, run_reference_pipeline: Callable[..., Any], @@ -214,7 +218,7 @@ def test_jsonlines_validation_fail_schema_invalid_rows_are_rejected( assert (row["source_file"], row["line_no"]) == ("records.jsonl", 2) assert row["raw_record"] == raw_record errors = json.loads(row["error_detail"]) - if pipeline_kind == "extract_validate": + if pipeline_kind == PYDANTIC_PIPELINE_NAME: assert reconstructed_entries(tables, "dataset_rejected")[0].get("record") == record assert len(errors) == 1 assert errors[0]["type"] == error_type @@ -224,7 +228,7 @@ def test_jsonlines_validation_fail_schema_invalid_rows_are_rejected( assert errors == [error_message] -@pytest.mark.parametrize("pipeline_kind", ["extract_validate", "extract_jsonschema_validate"], ids=str) +@pytest.mark.parametrize("pipeline_kind", [PYDANTIC_PIPELINE_NAME, JSONSCHEMA_PIPELINE_NAME], ids=str) def test_jsonlines_validation_pass_table_selection_skips_other_entities( *, run_reference_pipeline: Callable[..., Any], @@ -260,7 +264,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( tmp_path: Path, loader_file_format: str, dlt_destination_config: str, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, sorted_reference_entities: list[str], ) -> None: """All real CLI entry points load the reference dataset from command-line settings.""" @@ -268,18 +272,18 @@ def test_jsonlines_cli_pass_reference_dataset_loads( log_config.write_text('{"version": 1}', encoding="utf-8") output_dir = tmp_path / "bucket" registry_flag, registry_module = { - "extract": ("table-name", "dataset"), - "extract_validate": ( + EXTRACT_PIPELINE_NAME: ("table-name", "dataset"), + PYDANTIC_PIPELINE_NAME: ( "entity-models-module", "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models", ), - "extract_jsonschema_validate": ( + JSONSCHEMA_PIPELINE_NAME: ( "schema-files-module", "tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas", ), }[pipeline_kind] argv = [ - "jsonlines", + pipeline_kind, "--input-dir", str(reference_data_dir / "chunk_52"), "--output-dir", @@ -299,13 +303,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( ] if registry_flag: argv.extend([f"--{registry_flag}", registry_module]) - monkeypatch.setattr(sys, "argv", argv) - cli = { - "extract": extract_cli, - "extract_validate": validate_cli, - "extract_jsonschema_validate": jsonschema_cli, - }[pipeline_kind] - load_info = cli.run() + load_info = pipeline_runner.run(pipeline_kind, cli_args=argv[1:]) assert load_info is not None assert not load_info.has_failed_jobs tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py index 5c1d5c00..3c7513a1 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py @@ -131,7 +131,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( log_config_file.write_text('{"version": 1}') argv = [ - "jsonlines", + EXTRACT_PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py index 7727c53f..769b2523 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py @@ -287,7 +287,7 @@ def test_settings_pass_table_selection( """JSON Schema settings retain the shared table selection contract.""" settings = schema_settings_factory(table_names=table_names) assert settings.table_names == expected - assert settings.model_config["cli_prog_name"] == "jsonlines_jsonschema_validator" + assert settings.model_config["cli_prog_name"] == "jsonlines_jsonschema" @pytest.mark.parametrize("buffer_size", [0, -1], ids=["zero", "negative"]) diff --git a/tests/cdm_data_loaders/pipelines/test_framework.py b/tests/cdm_data_loaders/pipelines/test_framework.py new file mode 100644 index 00000000..27116169 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/test_framework.py @@ -0,0 +1,412 @@ +"""Unit and integration tests for the unified pipeline test execution framework.""" + +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest +from pydantic import BaseModel + +from cdm_data_loaders.pipelines import uniprot_kb +from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import ( + cli as jsonl_extract_cli, +) +from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import ( + run_jsonlines_pipeline, +) +from cdm_data_loaders.pipelines.jsonlines.settings import JsonlExtractSettings, JsonlJsonschemaIngestSettings +from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings +from tests.cdm_data_loaders.pipelines import framework +from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import ( + CONTAINER_AUX_DIR, + CONTAINER_INPUT_DIR, + CONTAINER_OUTPUT_DIR, + CONTAINER_PYMODULES_DIR, + PIPELINE_TARGETS, + ExecutionMode, + PipelineSpec, + PipelineTestRunner, + ensure_docker_image, + reconstruct_load_info, + resolve_pipeline, +) +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + read_data_rows, + settings_to_argv, +) +from tests.conftest import LOCAL_FS + + +class _SampleEntityModel(BaseModel): + """Sample Pydantic model for dynamic module materialization tests.""" + + record_id: str + + +@pytest.mark.parametrize("target_key", sorted(PIPELINE_TARGETS)) +def test_resolve_pipeline_pass_all_registered_targets(target_key: str) -> None: + """Every registered pipeline resolves to a valid PipelineSpec with a synchronized cli_prog_name.""" + spec = resolve_pipeline(target_key) + expected_cli, expected_module = PIPELINE_TARGETS[target_key] + assert spec.cli_command == expected_cli + assert spec.module_path == expected_module + assert callable(spec.cli_callable) + assert spec.settings_cls is not None + assert spec.settings_cls.model_config.get("cli_prog_name") == expected_cli + + +def test_resolve_pipeline_pass_from_module_cli_spec_and_callable() -> None: + """resolve_pipeline accepts ModuleType, PipelineCli, PipelineSpec, and pipeline run functions.""" + from_module = resolve_pipeline(uniprot_kb) + assert from_module.cli_command == "uniprot" + assert from_module.settings_cls is uniprot_kb.UniProtSettings + assert from_module.pipeline_fn is uniprot_kb.run_uniprot_pipeline + + from_cli = resolve_pipeline(jsonl_extract_cli) + assert from_cli.cli_command == "jsonlines_extract" + assert from_cli.settings_cls is JsonlExtractSettings + assert from_cli.pipeline_fn is run_jsonlines_pipeline + + from_callable = resolve_pipeline(run_jsonlines_pipeline) + assert from_callable.cli_command == "jsonlines_extract" + assert from_callable.pipeline_fn is run_jsonlines_pipeline + + from_full_module_str = resolve_pipeline("cdm_data_loaders.pipelines.uniprot_kb") + assert from_full_module_str.cli_command == "uniprot" + + from_spec = resolve_pipeline(from_cli) + assert from_spec is from_cli + + +def test_resolve_pipeline_fail_unknown_identifier() -> None: + """resolve_pipeline raises ValueError for unknown pipeline names or unmapped callables.""" + with pytest.raises(ValueError, match="Unknown pipeline 'nonexistent_pipeline'"): + resolve_pipeline("nonexistent_pipeline") + + with pytest.raises(ValueError, match="Cannot resolve pipeline from callable"): + resolve_pipeline(lambda _settings: None) + + +def test_settings_to_argv_pass_excludes_computed_fields_and_expands_lists(tmp_path: Path) -> None: + """settings_to_argv omits computed fields and None values, formats bools, and expands list fields.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + settings = JsonlJsonschemaIngestSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="custom_ds", + loader_file_format="jsonl", + schema_files_module="some.module", + table_names=["alpha", "beta"], + ) + argv = settings_to_argv(settings) + assert "--pipeline-dir" not in argv + assert "--log-config-file" not in argv + assert "--use-output-dir-for-pipeline-metadata" in argv + idx_meta = argv.index("--use-output-dir-for-pipeline-metadata") + assert argv[idx_meta + 1] == "true" + idx_tables = argv.index("--table-names") + assert argv[idx_tables + 1 : idx_tables + 3] == ["alpha", "beta"] + + +def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: + """reconstruct_load_info returns None on missing or error reports and reconstructs completed reports.""" + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + assert reconstruct_load_info(empty_dir) is None + + error_dir = tmp_path / "error_out" + error_dir.mkdir() + (error_dir / "report_123_456.txt").write_text( + "Pipeline test_pipe dataset test_ds\nStarted: 2026-01-01 00:00:00+00:00\nStatus: error\nPipeline failed\n", + encoding="utf-8", + ) + assert reconstruct_load_info(error_dir) is None + + completed_dir = tmp_path / "completed_out" + completed_dir.mkdir() + started_str = "2026-01-01 12:34:56.789000+00:00" + (completed_dir / "report_111_222.txt").write_text( + f"Pipeline test_pipe dataset test_ds\n" + f"Started: {started_str}\n" + f"Status: completed with errors\n" + f"Completed loads: 111, 222\n" + f"Pipeline load info saved\n", + encoding="utf-8", + ) + reconstructed = reconstruct_load_info(completed_dir) + assert reconstructed is not None + assert reconstructed.pipeline.pipeline_name == "test_pipe" + assert reconstructed.dataset_name == "test_ds" + assert str(reconstructed.started_at) == started_str + assert reconstructed.has_failed_jobs is True + assert reconstructed.loads_ids == ["111", "222"] + as_dict = reconstructed.asdict() + assert as_dict["dataset_name"] == "test_ds" + assert as_dict["loads_ids"] == ["111", "222"] + + +def test_ensure_docker_image_pass_caches_and_fail_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """ensure_docker_image builds once per tag and raises RuntimeError when docker build fails.""" + monkeypatch.setattr(framework, "_built_docker_images", set()) + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(cmd) + return subprocess.CompletedProcess(cmd, returncode=0, stdout="built", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + tag = ensure_docker_image("custom-image:test") + assert tag == "custom-image:test" + assert len(calls) == 1 + assert calls[0][:4] == ["docker", "build", "-t", "custom-image:test"] + + # Second call uses cache without invoking subprocess.run + assert ensure_docker_image("custom-image:test") == "custom-image:test" + assert len(calls) == 1 + + # Failing build raises RuntimeError + def fake_fail(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(cmd, returncode=1, stdout="", stderr="build boom") + + monkeypatch.setattr(subprocess, "run", fake_fail) + with pytest.raises(RuntimeError, match="Docker image build failed"): + ensure_docker_image("failing-image:test") + + +@pytest.mark.parametrize("mode", [ExecutionMode.CORE, ExecutionMode.CLI], ids=str) +def test_pipeline_test_runner_pass_core_and_cli_modes_end_to_end( + tmp_path: Path, + mode: ExecutionMode, +) -> None: + """PipelineTestRunner executes a pipeline in core and cli modes and hooks into all result assertions.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + expected_records = [{"id": "rec_1", "value": 10}, {"id": "rec_2", "value": 20}] + (input_dir / "data.jsonl").write_text( + "\n".join(f'{{"id": "{r["id"]}", "value": {r["value"]}}}' for r in expected_records) + "\n", + encoding="utf-8", + ) + output_dir = tmp_path / "output" + output_dir.mkdir() + + settings = JsonlExtractSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="framework_test_ds", + table_name="items", + loader_file_format="jsonl", + ) + + runner = PipelineTestRunner(mode=mode) + load_info = assert_run_ok( + runner( + "jsonlines_extract", + settings=settings, + ) + ) + + assert_pipeline_report_written(output_dir, load_info, metadata_moved=True) + assert set(load_info.pipeline.dataset().tables) >= set(DEFAULT_DLT_TABLES) + assert read_data_rows(output_dir / settings.dataset_name) == {"items": expected_records} + + +def test_pipeline_test_runner_pass_docker_mode_command_and_staging( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Docker mode stages input, dlt config, auxiliary files, dynamic modules, and reconstructs LoadInfo.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + (input_dir / "sample.xml").write_text("1", encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + xsd_file = tmp_path / "schema.xsd" + xsd_file.write_text("", encoding="utf-8") + + # Create dynamic in-memory modules for both ENTITY_SCHEMAS and ENTITY_MODELS + dyn_schema_mod_name = "_test_dynamic_schema_mod" + dyn_schema_mod = ModuleType(dyn_schema_mod_name) + dyn_schema_mod.ENTITY_SCHEMAS = {"widget": {"type": "object"}} # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, dyn_schema_mod_name, dyn_schema_mod) + + dyn_model_mod_name = "_test_dynamic_model_mod" + dyn_model_mod = ModuleType(dyn_model_mod_name) + dyn_model_mod.ENTITY_MODELS = {"widget": _SampleEntityModel} # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, dyn_model_mod_name, dyn_model_mod) + + monkeypatch.setattr(framework, "_built_docker_images", {"test-image:latest"}) + captured_cmds: list[list[str]] = [] + + def fake_docker_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + captured_cmds.append(cmd) + # Simulate the container writing a completed report to the mounted output_dir + (output_dir / "report_999_1000.txt").write_text( + "Pipeline xmltodict dataset xml_docker_ds\n" + "Started: 2026-02-01 00:00:00+00:00\n" + "Status: completed\n" + "Completed loads: 999\n" + "Pipeline load info saved\n", + encoding="utf-8", + ) + return subprocess.CompletedProcess(cmd, returncode=0, stdout="ok", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_docker_run) + + settings = XmlToDictSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="xml_docker_ds", + table_name="entries", + xml_tag="entry", + loader_file_format="parquet", + ) + + runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image="test-image:latest") + cli_args = [ + *settings_to_argv(settings), + "--xsd-file", + str(xsd_file), + "--schema-files-module", + dyn_schema_mod_name, + "--entity-models-module", + dyn_model_mod_name, + ] + load_info = runner.run("xmltodict", cli_args=cli_args) + + assert load_info is not None + assert load_info.pipeline.pipeline_name == "xmltodict" + assert load_info.dataset_name == "xml_docker_ds" + assert len(captured_cmds) == 1 + + docker_cmd = captured_cmds[0] + assert docker_cmd[:3] == ["docker", "run", "--rm"] + idx_image = docker_cmd.index("test-image:latest") + assert docker_cmd[idx_image : idx_image + 3] == ["test-image:latest", "pipeline", "xmltodict"] + + container_args = docker_cmd[idx_image + 3 :] + idx_in = container_args.index("--input-dir") + assert container_args[idx_in + 1] == CONTAINER_INPUT_DIR + idx_out = container_args.index("--output-dir") + assert container_args[idx_out + 1] == CONTAINER_OUTPUT_DIR + idx_xsd = container_args.index("--xsd-file") + assert container_args[idx_xsd + 1].startswith(f"{CONTAINER_AUX_DIR}/") + assert f"PYTHONPATH={CONTAINER_PYMODULES_DIR}:/app/src:/app" in docker_cmd + + +def test_pipeline_test_runner_fail_docker_mode_nonzero_exit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Docker mode raises ValueError on validation errors and RuntimeError on general container failures.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + monkeypatch.setattr(framework, "_built_docker_images", {"test-image:latest"}) + + def fake_fail_runtime(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(cmd, returncode=2, stdout="", stderr="container crashed") + + monkeypatch.setattr(subprocess, "run", fake_fail_runtime) + runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image="test-image:latest") + with pytest.raises(RuntimeError, match="failed with exit code 2"): + runner.run( + "jsonlines_extract", + cli_args=["--input-dir", str(input_dir), "--output-dir", str(output_dir)], + ) + + def fake_fail_value_error(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(cmd, returncode=1, stdout="", stderr="ValueError: Unknown table name: foo") + + monkeypatch.setattr(subprocess, "run", fake_fail_value_error) + with pytest.raises(ValueError, match="Unknown table name"): + runner.run( + "jsonlines_extract", + cli_args=["--input-dir", str(input_dir), "--output-dir", str(output_dir)], + ) + + +def test_pipeline_test_runner_fail_missing_required_arguments(tmp_path: Path) -> None: + """PipelineTestRunner raises ValueError when output_dir or input_dir (in docker mode) is missing.""" + runner = PipelineTestRunner(mode=ExecutionMode.CORE) + with pytest.raises(ValueError, match="Could not determine output_dir"): + runner.run("jsonlines_extract", cli_args=["--input-dir", str(tmp_path)]) + + spec_no_settings = PipelineSpec(cli_command="custom", module_path="custom.mod") + with pytest.raises(ValueError, match="has no settings_cls"): + runner.run(spec_no_settings) + + with pytest.raises(ValueError, match="does not expose settings_cls and pipeline_fn"): + runner.run(spec_no_settings, cli_args=["--output-dir", str(tmp_path)]) + + cli_runner = PipelineTestRunner(mode=ExecutionMode.CLI) + with pytest.raises(ValueError, match="does not expose a cli\\(\\) entry point"): + cli_runner.run(spec_no_settings, cli_args=["--output-dir", str(tmp_path)]) + + docker_runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image="test-image:latest") + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(framework, "_built_docker_images", {"test-image:latest"}) + try: + with pytest.raises(ValueError, match="Docker execution mode requires --input-dir"): + docker_runner.run("jsonlines_extract", cli_args=["--output-dir", str(tmp_path)]) + finally: + monkeypatch.undo() + + +def test_resolve_pipeline_pass_legacy_module_without_pipeline_cli() -> None: + """resolve_pipeline handles modules whose cli attribute is a plain function rather than PipelineCli.""" + legacy_mod = ModuleType("cdm_data_loaders.pipelines.custom_legacy") + legacy_mod.Settings = JsonlExtractSettings # type: ignore[attr-defined] + legacy_mod.run_pipeline = run_jsonlines_pipeline # type: ignore[attr-defined] + legacy_mod.cli = lambda: None # type: ignore[attr-defined] + spec = resolve_pipeline(legacy_mod) + assert spec.cli_command == "custom_legacy" + assert spec.settings_cls is JsonlExtractSettings + assert spec.pipeline_fn is run_jsonlines_pipeline + assert spec.cli_callable is legacy_mod.cli + + +@pytest.mark.docker +def test_pipeline_test_runner_pass_real_docker_container(tmp_path: Path, docker_image: str) -> None: + """Execute a real Docker container run via PipelineTestRunner when --run-docker is enabled.""" + input_dir = tmp_path / "input" + input_dir.mkdir() + expected_records = [{"id": "docker_1", "value": 42}] + (input_dir / "data.jsonl").write_text('{"id": "docker_1", "value": 42}\n', encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + + settings = JsonlExtractSettings( + input_dir=str(input_dir), + output_dir=str(output_dir), + use_destination=LOCAL_FS, + use_output_dir_for_pipeline_metadata=True, + dlt_dev_mode=False, + dataset_name="docker_framework_ds", + table_name="items", + loader_file_format="jsonl", + ) + + runner = PipelineTestRunner(mode=ExecutionMode.DOCKER, docker_image=docker_image) + load_info = assert_run_ok(runner.run("jsonlines_extract", settings=settings)) + + assert_pipeline_report_written(output_dir, load_info, metadata_moved=True) + assert set(load_info.pipeline.dataset().tables) >= set(DEFAULT_DLT_TABLES) + assert read_data_rows(output_dir / settings.dataset_name) == {"items": expected_records} diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py index 5c0c6b49..b52061b7 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py @@ -33,8 +33,8 @@ def test_run_uniprot_pipeline_args_set_correctly(test_settings: UniProtSettings) _, kwargs = mock_run_pipeline.call_args assert kwargs.keys() == {"settings", "resource", "pipeline_kwargs"} assert kwargs["pipeline_kwargs"] == { - "pipeline_name": "uniprot_kb", - "dataset_name": "uniprot_kb", + "pipeline_name": "uniprot", + "dataset_name": "uniprot", } assert kwargs["settings"] == test_settings assert isinstance(kwargs["resource"], Callable) @@ -57,8 +57,8 @@ def test_run_uniprot_pipeline_sets_core_run_pipeline_args_correctly( mock_dlt.destination.assert_called_once_with(test_settings.use_destination) mock_dlt.pipeline.assert_called_once_with( - pipeline_name="uniprot_kb", - dataset_name="uniprot_kb", + pipeline_name="uniprot", + dataset_name="uniprot", ) assert mock_dlt.pipeline.return_value.destination == mock_dlt.destination.return_value first_run, load_info_save_run = mock_dlt.pipeline.return_value.run.call_args_list diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py index a01d44db..49103b80 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py @@ -4,20 +4,19 @@ per-table counts below are derived from those fixtures. """ -from collections.abc import Callable from pathlib import Path from typing import Any import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines import uniprot_kb as uniprot_module from cdm_data_loaders.pipelines.uniprot_kb import ( UniProtSettings, - cli, parse_uniprot, ) -from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner +from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.conftest import LOCAL_FS, TEST_DATA_DIR UNIPROT_FIXTURE_DIR = TEST_DATA_DIR / "uniprot" / "uniprot_kb" / "chunk_4" @@ -48,7 +47,7 @@ def duckdb_uniprot_settings(tmp_path: Path) -> UniProtSettings: input_dir=str(UNIPROT_FIXTURE_DIR), output_dir=str(output_dir), use_destination=LOCAL_FS, - use_output_dir_for_pipeline_metadata=False, + use_output_dir_for_pipeline_metadata=True, ) # pyright: ignore[reportCallIssue] @@ -113,43 +112,18 @@ def test_integration_uniprot_pipeline_entity_and_source_file_content( def test_integration_cli_uniprot_pipeline_output_validated( duckdb_uniprot_settings: UniProtSettings, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: - """Exercise the real ``cli.run()`` wiring end-to-end against a DuckDB destination.""" - captured: dict[str, Any] = {} - - def fake_run_cli( - settings_cls: type[UniProtSettings], - pipeline_fn: Callable[..., Any], - cli_args: list[str] | None = None, - settings_kwargs: dict[str, Any] | None = None, - ) -> Any: - """Replacement for core.run_cli that dispatches to pipeline_fn with the fixture settings.""" - assert settings_cls is UniProtSettings - return pipeline_fn(duckdb_uniprot_settings) - - def fake_run_pipeline( - *, - settings: UniProtSettings, - resource: Any, - pipeline_kwargs: dict[str, Any], - ) -> None: - """Replacement for core.run_pipeline that runs the resource through DuckDB.""" - assert settings is duckdb_uniprot_settings - assert pipeline_kwargs == {"pipeline_name": "uniprot_kb", "dataset_name": "uniprot_kb"} - pipeline = duckdb_pipeline(tmp_path, "test_uniprot_cli_pipeline", "test_uniprot_cli") - captured["load_info"] = pipeline.run(resource) - captured["pipeline"] = pipeline - - monkeypatch.setattr("cdm_data_loaders.pipelines.core.run_cli", fake_run_cli) - monkeypatch.setattr(uniprot_module, "run_pipeline", fake_run_pipeline) - - cli.run() - - load_info = captured["load_info"] - pipeline = captured["pipeline"] + """Exercise the UniProt pipeline end-to-end across core, cli, and docker execution modes.""" + load_info = pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings) + + assert load_info is not None assert not load_info.has_failed_jobs + assert load_info.pipeline.pipeline_name == "uniprot" + assert load_info.dataset_name == "uniprot" + assert_pipeline_report_written(Path(duckdb_uniprot_settings.output_dir), load_info, metadata_moved=True) - ((entity_count,),) = _fetch_all(pipeline, "SELECT COUNT(*) FROM entity") - assert entity_count == UNIPROT_ENTRY_COUNT + dataset_dir = Path(duckdb_uniprot_settings.output_dir) / load_info.dataset_name + for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): + rows = read_destination_rows(dataset_dir, table_name) + assert len(rows) == expected_count, f"expected {expected_count} rows in '{table_name}', got {len(rows)}" diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py index 8dd9bef7..cf047063 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py @@ -4,21 +4,20 @@ derived from that fixture. Every test runs once per valid UniRef variant. """ -from collections.abc import Callable from pathlib import Path from typing import Any import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines import uniref as uniref_module from cdm_data_loaders.pipelines.uniref import ( UNIREF_VARIANTS, UnirefSettings, - cli, parse_uniref, ) -from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner +from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.conftest import LOCAL_FS, TEST_DATA_DIR TEST_DEFAULT_UNIREF_VARIANT = "50" @@ -49,7 +48,7 @@ def duckdb_uniref_settings(tmp_path: Path, request: pytest.FixtureRequest) -> Un input_dir=str(UNIREF_FIXTURE_DIR), output_dir=str(output_dir), use_destination=LOCAL_FS, - use_output_dir_for_pipeline_metadata=False, + use_output_dir_for_pipeline_metadata=True, ) # pyright: ignore[reportCallIssue] @@ -118,48 +117,18 @@ def test_integration_uniref_pipeline_entity_and_cluster_content( def test_integration_cli_uniref_pipeline_output_validated( duckdb_uniref_settings: UnirefSettings, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: - """Exercise the real ``cli.run()`` wiring end-to-end against a DuckDB destination.""" - captured: dict[str, Any] = {} - - def fake_run_cli( - settings_cls: type[UnirefSettings], - pipeline_fn: Callable[..., Any], - cli_args: list[str] | None = None, - settings_kwargs: dict[str, Any] | None = None, - ) -> Any: - """Replacement for core.run_cli that dispatches to pipeline_fn with the fixture settings.""" - assert settings_cls is UnirefSettings - return pipeline_fn(duckdb_uniref_settings) - - def fake_run_pipeline( - *, - settings: UnirefSettings, - resource: Any, - pipeline_kwargs: dict[str, Any], - ) -> None: - """Replacement for core.run_pipeline that runs the resource through DuckDB.""" - assert settings is duckdb_uniref_settings - assert pipeline_kwargs == { - "pipeline_name": "uniref", - "dataset_name": f"uniref_{duckdb_uniref_settings.variant}", - } - pipeline = duckdb_pipeline( - tmp_path, f"test_uniref_cli_pipeline_{duckdb_uniref_settings.variant}", "test_uniref_cli" - ) - captured["load_info"] = pipeline.run(resource) - captured["pipeline"] = pipeline - - monkeypatch.setattr("cdm_data_loaders.pipelines.core.run_cli", fake_run_cli) - monkeypatch.setattr(uniref_module, "run_pipeline", fake_run_pipeline) - - cli.run() - - load_info = captured["load_info"] - pipeline = captured["pipeline"] + """Exercise the UniRef pipeline end-to-end across core, cli, and docker execution modes.""" + load_info = pipeline_runner.run("uniref", settings=duckdb_uniref_settings) + + assert load_info is not None assert not load_info.has_failed_jobs + assert load_info.pipeline.pipeline_name == "uniref" + assert load_info.dataset_name == f"uniref_{duckdb_uniref_settings.variant}" + assert_pipeline_report_written(Path(duckdb_uniref_settings.output_dir), load_info, metadata_moved=True) - ((entity_count,),) = _fetch_all(pipeline, "SELECT COUNT(*) FROM entity") - assert entity_count == UNIREF_CLUSTER_COUNT + dataset_dir = Path(duckdb_uniref_settings.output_dir) / load_info.dataset_name + for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): + rows = read_destination_rows(dataset_dir, table_name) + assert len(rows) == expected_count, f"expected {expected_count} rows in '{table_name}', got {len(rows)}" diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py index eb18406a..7e963918 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py @@ -1,16 +1,14 @@ """End-to-end tests for the xml2db pipeline, using a small library.xsd schema.""" -import sys from collections.abc import Callable from pathlib import Path -import pytest from dlt.common.pipeline import LoadInfo from pandas import DataFrame -from cdm_data_loaders.pipelines.xml2db.pipeline import cli, run_xml2db_pipeline from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.xml_samples import SIMPLE_LIBRARY_XML @@ -32,6 +30,7 @@ def check_book_list_results(load_info: LoadInfo | None) -> tuple[DataFrame, Data def test_run_xml2db_pipeline_pass_writes_expected_output( settings_factory: Callable[..., Xml2DbSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """Running the pipeline loads one row per library root and one row per book.""" settings = settings_factory() @@ -39,7 +38,7 @@ def test_run_xml2db_pipeline_pass_writes_expected_output( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] @@ -50,6 +49,7 @@ def test_run_xml2db_pipeline_pass_writes_expected_output( def test_run_xml2db_pipeline_pass_gzip_files_are_loaded( settings_factory: Callable[..., Xml2DbSettings], write_gzip_file: Callable[[Path, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """Gzip-compressed xml files matching the glob are decompressed and loaded. @@ -64,7 +64,7 @@ def test_run_xml2db_pipeline_pass_gzip_files_are_loaded( (input_dir / "plain.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") write_gzip_file(input_dir, "library.xml.gz", SIMPLE_LIBRARY_XML) - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) library_df, book_df = check_book_list_results(load_info) # one root row from each of the two files: the root table is never compacted assert len(library_df) == EXPECTED_LIBRARY_ROOT_COUNT_TWO_FILES @@ -74,12 +74,13 @@ def test_run_xml2db_pipeline_pass_gzip_files_are_loaded( def test_run_xml2db_pipeline_pass_no_matching_files_yields_no_data_table( settings_factory: Callable[..., Xml2DbSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """An input dir with no matching xml files does not fail the run and creates no data table.""" settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() @@ -90,6 +91,7 @@ def test_run_xml2db_pipeline_pass_no_matching_files_yields_no_data_table( def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( settings_factory: Callable[..., Xml2DbSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """The `short_name` setting only affects the virtual root table name for multi-root schemas. @@ -102,7 +104,7 @@ def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xml2db_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] @@ -110,9 +112,9 @@ def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, dlt_destination_config: str, library_xsd: Path, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" input_dir = tmp_path / "cli_input" @@ -142,9 +144,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--dlt-dev-mode", "false", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py index 9182c7e1..b76d2803 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py @@ -1,6 +1,5 @@ """End-to-end tests for the xmltodict pipeline.""" -import sys from collections.abc import Callable from pathlib import Path @@ -8,12 +7,9 @@ from dlt.common.pipeline import LoadInfo from pandas import DataFrame -from cdm_data_loaders.pipelines.xmltodict.pipeline import ( - cli, - run_xmltodict_pipeline, -) -from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings +from cdm_data_loaders.pipelines.xmltodict.settings import PIPELINE_NAME, XmlToDictSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, assert_pipeline_report_written from tests.xml_samples import SIMPLE_LIBRARY_XML @@ -34,6 +30,7 @@ def check_book_list_results(load_info: LoadInfo | None, table_name: str | None = def test_run_xmltodict_pipeline_pass_writes_expected_output( settings_factory: Callable[..., XmlToDictSettings], loader_file_format: str, + pipeline_runner: PipelineTestRunner, ) -> None: """Running the pipeline on xml files loads one row per matching element into the configured table.""" settings = settings_factory(loader_file_format=loader_file_format) @@ -41,7 +38,7 @@ def test_run_xmltodict_pipeline_pass_writes_expected_output( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -50,6 +47,7 @@ def test_run_xmltodict_pipeline_pass_writes_expected_output( def test_run_xmltodict_pipeline_pass_gzip_files_are_loaded( settings_factory: Callable[..., XmlToDictSettings], write_gzip_file: Callable[[Path, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """Gzip-compressed xml files matching the glob are decompressed and loaded.""" settings = settings_factory() @@ -58,7 +56,7 @@ def test_run_xmltodict_pipeline_pass_gzip_files_are_loaded( (input_dir / "plain.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") write_gzip_file(input_dir, "library.xml.gz", SIMPLE_LIBRARY_XML) - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) book_df = check_book_list_results(load_info) # one copy of the three books from the plain file, one from the gzipped copy assert sorted(book_df["book___id"].tolist()) == ["1", "1", "2", "2", "3", "3"] @@ -66,12 +64,13 @@ def test_run_xmltodict_pipeline_pass_gzip_files_are_loaded( def test_run_xmltodict_pipeline_pass_no_matching_files_yields_no_data_table( settings_factory: Callable[..., XmlToDictSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """An input dir with no matching xml files does not fail the run and creates no data table.""" settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() @@ -82,6 +81,7 @@ def test_run_xmltodict_pipeline_pass_no_matching_files_yields_no_data_table( def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( settings_factory: Callable[..., XmlToDictSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """Rows land in the table named by table_name, not in a name derived from the xml tag.""" settings = settings_factory(table_name="library_entries") @@ -89,7 +89,7 @@ def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - load_info = run_xmltodict_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) book_df = check_book_list_results(load_info, "library_entries") # the record dict is keyed by the xml tag, so the flattened column prefix is `book` assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] @@ -97,8 +97,8 @@ def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, dlt_destination_config: str, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" input_dir = tmp_path / "cli_input" @@ -110,7 +110,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", @@ -130,9 +130,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--dlt-dev-mode", "false", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] assert_pipeline_report_written(output_dir, load_info) diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py index 11c1304f..74e788fb 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py @@ -93,7 +93,7 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", @@ -158,7 +158,7 @@ def test_cli_pass_writes_rows_for_matching_xml_files( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(input_dir), "--output-dir", @@ -223,7 +223,7 @@ def test_cli_fail_missing_required_args_raise( log_config_file.write_text('{"version": 1}') argv = [ - "xmltodict", + PIPELINE_NAME, "--input-dir", str(tmp_path / "input"), "--output-dir", diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py index 5424fa8a..425fe9fd 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py @@ -1,6 +1,5 @@ """End-to-end tests for the xsv pipeline, using a real qsv binary and local_fs destination.""" -import sys from collections.abc import Callable from pathlib import Path from uuid import uuid4 @@ -8,9 +7,9 @@ import pandas as pd import pytest -from cdm_data_loaders.pipelines.xsv.pipeline import cli, run_xsv_pipeline from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written from tests.cdm_data_loaders.pipelines.xsv.integration.conftest import ( ALL_RAGGED_ROWS, @@ -29,12 +28,13 @@ def test_run_xsv_pipeline_pass_writes_expected_output( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A clean, valid file is loaded, with every row cast to its declared schema types.""" settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -52,13 +52,14 @@ def test_run_xsv_pipeline_pass_writes_expected_output( def test_run_xsv_pipeline_pass_multiple_files_loaded_together( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """Every file matching file_glob is cleaned, validated, and loaded into the same table.""" settings = settings_factory() write_xsv_file(settings, "part_1.csv", rows_to_csv([VALID_ROWS[0]])) write_xsv_file(settings, "part_2.csv", rows_to_csv([VALID_ROWS[1]])) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -69,6 +70,7 @@ def test_run_xsv_pipeline_pass_multiple_files_loaded_together( def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A file with one ragged row loads its surviving valid rows AND records the row loss. @@ -79,7 +81,7 @@ def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(PARTIAL_RAGGED_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -98,12 +100,13 @@ def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( def test_run_xsv_pipeline_pass_fully_ragged_file_is_entirely_rejected( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A file where every row is ragged yields no valid rows; the failure is recorded as rejected.""" settings = settings_factory() write_xsv_file(settings, "bad.csv", rows_to_csv(ALL_RAGGED_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -118,11 +121,12 @@ def test_run_xsv_pipeline_pass_fully_ragged_file_is_entirely_rejected( def test_run_xsv_pipeline_pass_no_matching_files_yields_no_data_table( settings_factory: Callable[..., XsvIngestSettings], + pipeline_runner: PipelineTestRunner, ) -> None: """An input dir with no matching data files does not fail the run and creates no data table.""" settings = settings_factory() - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -133,6 +137,7 @@ def test_run_xsv_pipeline_pass_no_matching_files_yields_no_data_table( def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """A schema-configured null_regex placeholder is replaced with a null value before loading.""" schema = { @@ -148,7 +153,7 @@ def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( ), ) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -161,6 +166,7 @@ def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], + pipeline_runner: PipelineTestRunner, ) -> None: """The pipeline also loads successfully when loader_file_format is parquet. @@ -171,7 +177,7 @@ def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( settings = settings_factory(loader_file_format="parquet") write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = run_xsv_pipeline(settings) + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) assert load_info is not None assert load_info.has_failed_jobs is False @@ -182,7 +188,7 @@ def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( def test_cli_pass_runs_end_to_end_from_command_line_arguments( settings_factory: Callable[..., XsvIngestSettings], write_xsv_file: Callable[[XsvIngestSettings, str, str], Path], - monkeypatch: pytest.MonkeyPatch, + pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" settings = settings_factory(dataset_name=f"cli_dataset_{uuid4().hex}") @@ -207,9 +213,8 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( "--dlt-dev-mode", "false", ] - monkeypatch.setattr(sys, "argv", argv) - load_info = cli.run() + load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) assert load_info is not None assert load_info.has_failed_jobs is False diff --git a/tests/conftest.py b/tests/conftest.py index fc0f7c90..46182b59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,6 +62,32 @@ _notebook_utils_available: bool | None = None +def pytest_addoption(parser: pytest.Parser) -> None: + """Register custom command-line options for running Docker container pipeline tests.""" + parser.addoption( + "--run-docker", + action="store_true", + default=False, + help="Build and run Docker container tests marked with @pytest.mark.docker.", + ) + parser.addoption( + "--docker-image", + action="store", + default=None, + help="Pre-built Docker image tag to use for @pytest.mark.docker tests instead of building.", + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Skip tests marked with ``docker`` unless ``--run-docker`` was passed on the command line.""" + if config.getoption("--run-docker"): + return + skip_docker = pytest.mark.skip(reason="need --run-docker option to run Docker container tests") + for item in items: + if "docker" in item.keywords: + item.add_marker(skip_docker) + + def _find_notebook_utils() -> bool: """Check whether the current environment has the KBase Lakehouse notebook_utils lib available. diff --git a/uv.lock b/uv.lock index 31c28b51..26009a94 100644 --- a/uv.lock +++ b/uv.lock @@ -2,14 +2,15 @@ version = 1 revision = 5 requires-python = ">=3.13" resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'win32'", + "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -364,7 +365,7 @@ wheels = [ [[package]] name = "bioregistry" -version = "0.15.3" +version = "0.15.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -379,9 +380,9 @@ dependencies = [ { name = "tqdm" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/b5/a12e14e450da63ee293a5c3ca536461de9d6fbf2a79c716baa78f3212bfb/bioregistry-0.15.3.tar.gz", hash = "sha256:0a6e3f326c7935138434745a02d5184019e13aeaf8cf8520784d2d15fe3d1a7c", size = 7271806, upload-time = "2026-10-10T06:01:05.259Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/4ea7b1537369526482a77a635e1725c45ea3f314e29f2c1eb572bb0fa82e/bioregistry-0.15.5.tar.gz", hash = "sha256:63fa06045d906c71edf1c5a8a81afea18d083fb8613d3d81ca7485f79433764b", size = 7275690, upload-time = "2026-10-17T04:33:53.438Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/fb/549e0af1ba4597578d0d08ef21bbc823b4e760c02d6a1bf6a4814d399518/bioregistry-0.15.3-py3-none-any.whl", hash = "sha256:a714f088375adb46a78937d4906941011a4a7ebd58ed77ca6f82c76d5d89b34c", size = 7345811, upload-time = "2026-10-10T06:01:01.804Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b1/6f2e5efbbc62c2bf762b1de5433f51c004d793463fbdd2d3aafe3b515854/bioregistry-0.15.5-py3-none-any.whl", hash = "sha256:342a7a8a98aeccb70655cc4f7ba87e5c1deba16a3ccf0941a06943c662de8aaa", size = 7349749, upload-time = "2026-10-17T04:33:51.748Z" }, ] [[package]] @@ -451,20 +452,20 @@ crt = [ [[package]] name = "botocore-stubs" -version = "1.43.113" +version = "1.43.115" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/87/86c5d1ab625e8253547e2c1a967b2575c7b00df27d7158cf636d16935b39/botocore_stubs-1.43.113.tar.gz", hash = "sha256:49deda166eceee64243cca0b5dc9de5ce005949dee9ead477f8df22bafaee374", size = 42883, upload-time = "2026-10-13T22:31:59.108Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/a1/aaf4f30377a712b607a7b829e6257933a6857ef1aeafb84362d75e22314e/botocore_stubs-1.43.115.tar.gz", hash = "sha256:8dc2089a0eb8109faa6dc07b1916ddb3d8d33bd8e2c42825f13309dd4176879d", size = 42908, upload-time = "2026-10-19T20:58:21.643Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/34/047044056dfae6e5206fafdac4c359bc2055f43827252087caa4eb041efe/botocore_stubs-1.43.113-py3-none-any.whl", hash = "sha256:716fdf4650fbcca50ec5086882d749f9d0f68047370830d9203a2de297c5515f", size = 67249, upload-time = "2026-10-13T22:31:56.771Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8f/3f84f0f4af41074ce938c47c5ba8ea4c7afa7e74c07ac0d62b78fd32b076/botocore_stubs-1.43.115-py3-none-any.whl", hash = "sha256:0fa02aedd7228b4b777d61bb720ec14dff92b59413c3cc891fff05a78d66a08d", size = 67247, upload-time = "2026-10-19T20:58:19.875Z" }, ] [[package]] name = "cachetools" -version = "7.2.1" +version = "7.2.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/44/71476a5812da1ddf2c9a3efd31ae76d01480a1cf03ed13ac28aa8f2402e4/cachetools-7.2.1.tar.gz", hash = "sha256:b1a7537025c06abf96fcc1443e496af9a3fb95e774e70e1f0af226f73f7f2dcc", size = 41357, upload-time = "2026-10-05T18:40:06.361Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/5d/15ea0494ebcb9062c7ca79bf026cbba380a50538ae73753fb40bdffa79e2/cachetools-7.2.4.tar.gz", hash = "sha256:e0203b04b75fe13e62098eeb5b2018c4eb1a14c04e2b5b450e70db3ee7e92ec5", size = 44251, upload-time = "2026-10-19T19:09:44.931Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/c9/2a61d784caf0d869a3326728c57c7203f50cc53f3cca2ee76bf924769eb4/cachetools-7.2.1-py3-none-any.whl", hash = "sha256:63aa53dfe7473c10cccdd5a01dedf76ef2c4b73a58840d9396e7d0752cbdac3b", size = 17006, upload-time = "2026-10-05T18:40:04.827Z" }, + { url = "https://files.pythonhosted.org/packages/5a/4c/694f7d44f2399b3901232d6ded1c88442a02fc8141936396d9289d951848/cachetools-7.2.4-py3-none-any.whl", hash = "sha256:4cfbebf3eec4040d2073b19c460511ad47eef7b852dbf9a0d385099b4dafa31e", size = 18571, upload-time = "2026-10-19T19:09:43.296Z" }, ] [[package]] @@ -518,6 +519,10 @@ models = [ { name = "genson" }, { name = "json2python-models" }, ] +mutants = [ + { name = "mutmut" }, + { name = "pytest-gremlins" }, +] xml = [ { name = "xsdata", extra = ["cli", "lxml"] }, ] @@ -544,7 +549,7 @@ requires-dist = [ { name = "pandas", specifier = ">=3.0.3" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pydantic-settings", specifier = ">=2.14.1" }, - { name = "pyiceberg", extras = ["pandas", "pyarrow", "pyiceberg-core", "snappy"], specifier = ">=0.11.1" }, + { name = "pyiceberg", extras = ["pandas", "pyarrow", "pyiceberg-core", "snappy"], specifier = ">=0.12" }, { name = "smart-open", specifier = ">=8.0.1" }, { name = "tqdm", specifier = ">=4.70.1" }, { name = "xml2db", specifier = ">=0.14.0" }, @@ -571,15 +576,19 @@ models = [ { name = "genson", specifier = ">=1.3.0" }, { name = "json2python-models", specifier = ">=0.3.1" }, ] +mutants = [ + { name = "mutmut", specifier = ">=3.8.0" }, + { name = "pytest-gremlins", specifier = ">=1.12.2" }, +] xml = [{ name = "xsdata", extras = ["cli", "lxml"], specifier = ">=26.2" }] [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -666,37 +675,37 @@ sdist = { url = "https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d [[package]] name = "chardet" -version = "7.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/cd61c567092a6cec796144510a68aff158ebfc1df82950a45bae65f28413/chardet-7.6.0.tar.gz", hash = "sha256:93d9df6089ded42ed1fe9f57e272c0b74bd0464d45c0c7d50f09f26f31105c3c", size = 914462, upload-time = "2026-08-14T20:36:59.305Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/29/16a7419edfbd60e901e6a797cbc3e038cb2a81903bc16c029db755f0156f/chardet-7.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:57e6846cc13ce1ff59979f4ec9da770c57e12aa99046073f632de5a51d9a6f20", size = 1088449, upload-time = "2026-08-14T20:36:34.723Z" }, - { url = "https://files.pythonhosted.org/packages/1d/36/3a14b0f8ddeb302f157281ca656a3ce6874b78e2d6af03682f520b487245/chardet-7.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:089e3bb81a0a07e94f15461ded9f9ee66d349615b1a9fd557d4de1003e2fc12e", size = 1065126, upload-time = "2026-08-14T20:36:36.21Z" }, - { url = "https://files.pythonhosted.org/packages/d2/4c/f59a39c2bfe4ac99baba8da842e8d2ea0b84dff7ba53a96e7ad8c71602d7/chardet-7.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43ea433e43a23c55e8e17f3fad1e07f5cfe5450c73124b95b0d849c21ad379ee", size = 1482492, upload-time = "2026-08-14T20:36:37.649Z" }, - { url = "https://files.pythonhosted.org/packages/bd/eb/93e8036681157f2217a18769927a035984526e6dbd5e91f28a375ca41c14/chardet-7.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b5d31f9b7f793e15e81cca877e7ccd72bffffa2a3443a9d47be9dfee84fad69", size = 1514185, upload-time = "2026-08-14T20:36:39.119Z" }, - { url = "https://files.pythonhosted.org/packages/10/04/0066d7ab2c135e404a6fa166bb7fa49d1c7bf7af07b86ed95b1c48a348c7/chardet-7.6.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c54b6a8d3b219560fa5cf4c28df932c37471afe047afdc152067104e741f38c1", size = 1454420, upload-time = "2026-08-14T20:36:40.549Z" }, - { url = "https://files.pythonhosted.org/packages/6c/9f/e965f1d9eddfb86cf118f980d329cbee43c4ac4b562448b369a6b6ef36af/chardet-7.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:b3b4c96c4df93899b3c8b9e8159e06b1f55c66d7ca384d91481108e251a06eb0", size = 1159067, upload-time = "2026-08-14T20:36:41.816Z" }, - { url = "https://files.pythonhosted.org/packages/cf/78/e14991c9487277ed7d006fff58f3084ac792ed94cced081788abb95df70c/chardet-7.6.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c6061adf247ab5dda173b67010e13904c6071717660c7c8077fb50aca362b264", size = 1087678, upload-time = "2026-08-14T20:36:43.2Z" }, - { url = "https://files.pythonhosted.org/packages/7a/40/0f95e04cb1820e0a582cd6d86bbf26be8302a94ccf330f8ba5f69735389d/chardet-7.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fc1e1571321baf8927582fe34363ad7f02279f11c8c2839c14b4c76894148db6", size = 1065310, upload-time = "2026-08-14T20:36:44.489Z" }, - { url = "https://files.pythonhosted.org/packages/3b/a1/04404dcc9d6e1253b02583e562d2c7ddca50a7e91f460d62eff7e1d07c92/chardet-7.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8900f6c7cf6b015b17a51767cc6144689059ba1cdceaa383d29eb037ac28579e", size = 1485028, upload-time = "2026-08-14T20:36:45.797Z" }, - { url = "https://files.pythonhosted.org/packages/f4/88/360064c4c7d9d0664561dae03b74c871d2f5332b329f5c99f1c997fb869a/chardet-7.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cedbc584789eb2edfde20fd03669972a833ce6019e60014ae613f9bfc440e8e3", size = 1510242, upload-time = "2026-08-14T20:36:47.206Z" }, - { url = "https://files.pythonhosted.org/packages/43/4c/302869fa1c69a5a41e4e78782680b12552ffd4286c3ae3c839b7b8df53d4/chardet-7.6.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5dc835e40e0e09c2c3eab43731a8b5127834f42786dda09ba2f4b699ccd527a", size = 1456456, upload-time = "2026-08-14T20:36:48.668Z" }, - { url = "https://files.pythonhosted.org/packages/7c/a9/ff4fef15ed25fc3f945a3b981ae0f43c8559b3fbedb40267e59e583d105b/chardet-7.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:0f304de7041afaec0195ad6464937cd112392002e9d72ed15d55f20a9abd3a13", size = 1159103, upload-time = "2026-08-14T20:36:50.107Z" }, - { url = "https://files.pythonhosted.org/packages/31/44/94e6f89485ce6c630e8fec6d388e3fa747e58b7246b0cc8c5c532ba98650/chardet-7.6.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4f0a368ad04d5def08bdfaa17c7e15e71552f93923dc2aa9b2f7d9dee02fbb6", size = 1120302, upload-time = "2026-08-14T20:36:51.484Z" }, - { url = "https://files.pythonhosted.org/packages/e0/cd/8e68ba12f13aaf4ee82d54ef8a1f83d62942e7a6bc1e579dd2d530a3e26e/chardet-7.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75d6c3a4d2046d49e83d2d2206eb073a1f390743e856d90c1bbc19949b26acf4", size = 1093157, upload-time = "2026-08-14T20:36:52.987Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f6/6a35342b9efa69dcceab0ea8966571c6442a59c336bf460f0a2af95ed234/chardet-7.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:459e2b1c98f9a86a4698112aa42dffa802bbbff883c1ff144071f87224125862", size = 1521213, upload-time = "2026-08-14T20:36:54.812Z" }, - { url = "https://files.pythonhosted.org/packages/df/8c/8f09bfabbaedae45caa72b996e96d5e3f6436dfddc55c1311ffa2f6bd7d2/chardet-7.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bdb6f03107b7ace3f44e0edd91aa24456ee558787df265cc19daf45785b31c7", size = 1528427, upload-time = "2026-08-14T20:36:56.224Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3d/2540627b193112e08b8045f3cad9295570866208555aa6625b63cc69c6cd/chardet-7.6.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:0c44a32da32cc8b23d6b20d98ace15ec7600950e4955d1bf5ab1f849b0187fdb", size = 1087374, upload-time = "2026-08-14T21:04:14.994Z" }, - { url = "https://files.pythonhosted.org/packages/fe/de/dac4f550cde73c4732b4916e72ec489ae36933fbbd1697e4122d3ae2e9dd/chardet-7.6.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7bbc8a9652c7f859c593847f220c1d264f25749369abb1a267b404ee8cceb209", size = 1064688, upload-time = "2026-08-14T21:04:16.806Z" }, - { url = "https://files.pythonhosted.org/packages/06/45/f4f3f288496797d2cf1d1e9036f920ed2b4c79787b21c42a3314f4a6d532/chardet-7.6.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83512a475a2f3886166aa0bca1bbb39343a4eb3186dd5532127d6f2591d09118", size = 1486320, upload-time = "2026-08-14T21:04:18.535Z" }, - { url = "https://files.pythonhosted.org/packages/c0/81/3ca30c16e6c6015b737fca22d9342957cc617d8a65329d3e963ad754a31b/chardet-7.6.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:271ab71ec1be61dbbce0436de0848895c03eae051c379e937a39573d9ce403d9", size = 1515374, upload-time = "2026-08-14T21:04:20.724Z" }, - { url = "https://files.pythonhosted.org/packages/6e/98/163a75b8b3372a6b6503f5f5a4154a222ac135c7a706b5d176f8e4b237a2/chardet-7.6.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da86fc1b40ff5996fbb5e4c2d2dca770eac2c893cef157dacc050b8b4d929846", size = 1469504, upload-time = "2026-08-14T21:04:22.352Z" }, - { url = "https://files.pythonhosted.org/packages/d7/4f/3ad1b1bd27ae7f0d3d13cf711366525cd781a9e067950d8a09aa280916cb/chardet-7.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:b73f277c1ac09c4f8076c4214b816c7aa78a0a2f0cb7156742f4303f856bedc3", size = 1158551, upload-time = "2026-08-14T21:04:24.259Z" }, - { url = "https://files.pythonhosted.org/packages/8d/92/22a609c68c5123ebdccd8fe1a80e423609012ce20166de19a2ed3955b2f7/chardet-7.6.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:61238d5945b36af9a2ad13494f8969b7deb3c3b4abe223e54670c064e73f5328", size = 1119186, upload-time = "2026-08-14T21:04:25.761Z" }, - { url = "https://files.pythonhosted.org/packages/29/35/75a90143e4200e197f4f6cf5895d379e22bc785d0c7711120df18fa5347c/chardet-7.6.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f2ec3c78cc6b54bf8e091ec4ee885473078b5d7ef18ab1b01c86ae1e98bf88f7", size = 1092596, upload-time = "2026-08-14T21:04:27.884Z" }, - { url = "https://files.pythonhosted.org/packages/d1/2c/b6d5f47d878c46e04ae6fcb58e0969467925bc0819b333c682e0c41bbc5c/chardet-7.6.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:167d7ba3ee08b654e36d7b43ebd9a36606c9a12e2fabdb361757a095ca3b7e3d", size = 1516644, upload-time = "2026-08-14T21:04:29.624Z" }, - { url = "https://files.pythonhosted.org/packages/25/d2/2bc3f1066c6f27bc3fa3a98dfd8a2a9c1e969a9d6bdc1edcd35278791fc4/chardet-7.6.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f14f46ef1977e41ce1f4814ca6984cea7f8b6baf8cbc6626ef7bf3d13cf7ea13", size = 1533221, upload-time = "2026-08-14T21:04:31.39Z" }, - { url = "https://files.pythonhosted.org/packages/cf/6e/5a0b348fa4cd7847567a28c6e697ccf58391960bfd13a6e7473ee23ca2f2/chardet-7.6.0-py3-none-any.whl", hash = "sha256:4076d795897ce45239825956a1334e134322ecc4bfe84dbb12acd5390de0fbc1", size = 680279, upload-time = "2026-08-14T20:36:57.763Z" }, +version = "7.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/92/3d893ac5d338b9a9af34c5b517d77492dcea8a10254254eead4215a96f3f/chardet-7.7.0.tar.gz", hash = "sha256:0ba38a05201b0abf7570c12d8af45c30b2af835bd5cc52bfc7105082eda82c1e", size = 1008480, upload-time = "2026-10-18T00:06:25.352Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/73/0c434bc65adc411e49b1bbebee24ebb36a3511c80a058f1fde17f43fb537/chardet-7.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:11263887c082c3d8c32026aeddbdef37ac8feccd1faece5a22b2ff00a29b115a", size = 1202232, upload-time = "2026-10-18T00:05:45.657Z" }, + { url = "https://files.pythonhosted.org/packages/9d/be/222e5628af10d6b80750ccdda7f6d2610646dddb082c37947623d5e55bf7/chardet-7.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e8c718e6961981fe1685e1243742f326ac553f0fa0dfd0c6d3d17dd4f92cd3ff", size = 1173273, upload-time = "2026-10-18T00:05:46.984Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/889cf390db3748918789818d93ba9a7d712b326030199698432fc50bf8cd/chardet-7.7.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a76f4d1f769b4ae8888cefd46e604cf1f5769f1176b55e78b90eaeefa888ce92", size = 1616968, upload-time = "2026-10-18T00:05:48.457Z" }, + { url = "https://files.pythonhosted.org/packages/ca/04/0b5fd38375ae182cec6446dd3121aaff5ae18fd12b98b10161a58b720db9/chardet-7.7.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c519c10c315a675bcfd4b55c0e0de1859ca5a39881d8f69750500102c2fc5d2f", size = 1658830, upload-time = "2026-10-18T00:05:49.938Z" }, + { url = "https://files.pythonhosted.org/packages/99/c0/7e642d7bc6aca242ef41b4aa5f5be27dbb57912e8ce0f38043d8d1999cd1/chardet-7.7.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:46a044166fe0d99931884dc28405faac558d5eabf28316a2b3d2a1a6c7dc439a", size = 1583300, upload-time = "2026-10-18T00:05:51.567Z" }, + { url = "https://files.pythonhosted.org/packages/30/ee/53147716ced6d43837682a2acfb09dd45add8bc496107f21051c5aeae1f6/chardet-7.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:113bbd3ab85835e3e99499ae66829915ed3b4cf5c69525ab1742260f77638449", size = 1308030, upload-time = "2026-10-18T00:05:52.997Z" }, + { url = "https://files.pythonhosted.org/packages/1a/61/800a9442f352912c86f2a00de84fd6914dd9353d29091b1e223e7c08850e/chardet-7.7.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad286a0654ae7477bc454a1dee4f396cb28adffeacc6bc7d4412f607c73de0c3", size = 1200940, upload-time = "2026-10-18T00:05:54.59Z" }, + { url = "https://files.pythonhosted.org/packages/fb/92/3c6ce24dd68c626cc6aa0b5953c2aebb1d853c7b07d3535f0300b7428288/chardet-7.7.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4f67f20c9ad3f3024bf71d4aae1d25722b12da921ca9586f3cce7a4639695aa0", size = 1173087, upload-time = "2026-10-18T00:05:56.257Z" }, + { url = "https://files.pythonhosted.org/packages/4d/32/5a952b376c00dda01465b396958a8985916d9a9293a772d6bda2c37f2cc5/chardet-7.7.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a05667635c0496dab47eca8c845c044d1da6d74d34a9bf687199c9c53a38119d", size = 1624001, upload-time = "2026-10-18T00:05:57.832Z" }, + { url = "https://files.pythonhosted.org/packages/04/f7/09296ca1d4621c9c2766d074822035eee6aebb79eff80c0af3bd96283d05/chardet-7.7.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:79eec5af9d7715334695254234c8d7c29165841526c11083951d9b9f6458e372", size = 1653785, upload-time = "2026-10-18T00:05:59.379Z" }, + { url = "https://files.pythonhosted.org/packages/cb/08/80571baad57db2962d6931b3ee1828c82e3e14e9b3464f80532f536e23ac/chardet-7.7.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4167165812219e70f311083a8e06e3b627ed5b27aa8c15c3fdd56f350fd3213e", size = 1582280, upload-time = "2026-10-18T00:06:00.825Z" }, + { url = "https://files.pythonhosted.org/packages/18/82/3c07070fdcf3c4263e16680c90e9af7c2cd2e5e59cfb1d372efa60fde8cc/chardet-7.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:546180f6170da39778069d16130449a9cff49a8fe144535947d1ef26f5330b96", size = 1310108, upload-time = "2026-10-18T00:06:02.342Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b0/d703d408c7f8931b1564110078fc3dcff9825a159990fa69b80da52be160/chardet-7.7.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4c2e83b09a540a4d33279d2bee8ed3bfd57ac6645b359ae8b8afd5b4578c3c2", size = 1237782, upload-time = "2026-10-18T00:06:03.763Z" }, + { url = "https://files.pythonhosted.org/packages/d8/3e/0eafeebd53eb096400f75138ab18614ab5be12b8794b76dd4b35ef1a8e05/chardet-7.7.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:259dd55b1ba05a66868151a42663483763046a3e6579bde6b0aa65d834635b5a", size = 1207710, upload-time = "2026-10-18T00:06:05.221Z" }, + { url = "https://files.pythonhosted.org/packages/b8/8d/3c200098027def56be1b6443a10c72efe013a42eb76444d889c809890ca5/chardet-7.7.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56e5a5c782548b301b44371de327115d13a08de391ddaf9db4ac420d8511c5d7", size = 1660832, upload-time = "2026-10-18T00:06:06.619Z" }, + { url = "https://files.pythonhosted.org/packages/08/81/02f98389fa72345f40d23d69c2d857511d11a0911779a7cf8ed1cecb5037/chardet-7.7.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d227d2aa329fcf848f4640ed9a4d99e1753840c1cfa19f7ca5a48f30c03982ad", size = 1678291, upload-time = "2026-10-18T00:06:08.196Z" }, + { url = "https://files.pythonhosted.org/packages/5e/24/0ae866b759a148d0f6087ffb30920a2b956653d0d455b4be93026edbee95/chardet-7.7.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c4a591b2dbfc0a047f8d60331c2fbfb4c3589911000406666a3085fa7d5da69f", size = 1200696, upload-time = "2026-10-18T00:06:09.701Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a6/5e0a1f56aca6c44b562d05d9e360f31b5a8f4b311d940a324fa337d277ba/chardet-7.7.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:eb625c3ab2386ca7f3be9ac2b9b822f4c5b792595c8bb8100b3b6ea68120e92c", size = 1172821, upload-time = "2026-10-18T00:06:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/89/05/868f52672cde7e188a7333f3f65deaa62f5acbfcf0f81f3582013776e8ad/chardet-7.7.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddedfb252d05c86c2d6e9a3090f8bd574876bbd39679e7c588ed57d21595c097", size = 1625481, upload-time = "2026-10-18T00:06:12.499Z" }, + { url = "https://files.pythonhosted.org/packages/f9/12/2b3a006ae31a7ab4ccac11a00abb27af23b4a5d57ddb00e937375e747a2a/chardet-7.7.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c4d8a2dc8abcc067417adc13c2f64f48482e48ce48e3af769b5a44214e5a5c", size = 1660352, upload-time = "2026-10-18T00:06:13.977Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/f995a2873a78aa3ec8308f02e61f0717deb4b01c8f1a74c2da329f325925/chardet-7.7.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78db6186c1e73d38a2106cdcbbc8f43612eaed9ef8bc4adaf0fe67b6e8a2049d", size = 1597363, upload-time = "2026-10-18T00:06:15.492Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7b/22b916b52c7bc4eb076ba1736ea9b99cf1065ebb7b56d356409144887c73/chardet-7.7.0-cp315-cp315-win_amd64.whl", hash = "sha256:8f1d611f15530e1642856d91b7aa259851466d7e961f27b888921e7e29718edb", size = 1309528, upload-time = "2026-10-18T00:06:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/ccaf5ffa0c8adca280e2687d75524cc07113a0df70cbf542da3445f2b300/chardet-7.7.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:1db39957379a801cf1798a5e50cdb3d3fc32683e23c4166ecfa157b980d1931e", size = 1236698, upload-time = "2026-10-18T00:06:18.191Z" }, + { url = "https://files.pythonhosted.org/packages/0d/7e/32f45f7ddeb27d94feb529a1a23e5a38a6d3614d12e23f0c89a9eb89098b/chardet-7.7.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:7f3b2ca3bac2bdbe42c4dfe7a2888c4b98e2e86a83ddb631960b229d1980979e", size = 1207247, upload-time = "2026-10-18T00:06:19.751Z" }, + { url = "https://files.pythonhosted.org/packages/26/40/8c1ab360c39a0bb823786e8934984cbc3e83076dfce2a7f3760086ef778c/chardet-7.7.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80db1c109e6e3c102ac589897449437ea2edf26dda1042a3ff669368e21d90e5", size = 1653659, upload-time = "2026-10-18T00:06:21.189Z" }, + { url = "https://files.pythonhosted.org/packages/1b/3e/bd5c25f525ff2466e6365cbfb098f928761357331b884eebc92ac50f8878/chardet-7.7.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:091f50d3567b1a066f797835ee5367dbcf3cb77c5e56993b3f61b28264b8e6f8", size = 1677031, upload-time = "2026-10-18T00:06:22.564Z" }, + { url = "https://files.pythonhosted.org/packages/cc/75/88a3eb36264375ff951bd2f9397c409727ab6aa6a18e1bd933851571d421/chardet-7.7.0-py3-none-any.whl", hash = "sha256:11cf9b719ea3675a2a7d8a3071bbe4d7ea24b26bfe39a51588d515ddd05cacad", size = 706680, upload-time = "2026-10-18T00:06:24.018Z" }, ] [[package]] @@ -1097,21 +1106,20 @@ wheels = [ [[package]] name = "curies" -version = "0.15.3" +version = "0.15.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, - { name = "pystow" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/9a/10def0c00f9f75e21f696d33cae90a46dfb5a97f5bf2a8a812f869b43ad5/curies-0.15.3.tar.gz", hash = "sha256:6c613b63c3d3f485fcc42248309d011911cc1c51fc0c6a54a30311d4dc87a6aa", size = 74155, upload-time = "2026-10-08T17:50:22.744Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/2d/79253d3482e3d8dbeac15a973e458d3f9b261f5faf4a6f726843a6acb175/curies-0.15.4.tar.gz", hash = "sha256:0dc7d44943b9991c518942e824905c1a19bbe24f43d4a790e87d2e42472483df", size = 79920, upload-time = "2026-10-18T15:33:22.572Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/07/1e62b10fe0102b2702e8ee92e9169a29c340e32136e0ed0ba1c4ceb5ecfa/curies-0.15.3-py3-none-any.whl", hash = "sha256:7b83bc82798aac78711578fced0e99055371fbc3d43d3d4319a66e1e5598ce95", size = 83356, upload-time = "2026-10-08T17:50:21.225Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/e2f6eb642b4497a5a161df45f07dede60745b412e85f93048bd45fa0dff7/curies-0.15.4-py3-none-any.whl", hash = "sha256:25a7aaf4c58cd787887f11622ea38975bc4a847d20283bc24ede6d8af5e83687", size = 83473, upload-time = "2026-10-18T15:33:21.329Z" }, ] [[package]] name = "datamodel-code-generator" -version = "0.83.0" +version = "0.86.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "argcomplete" }, @@ -1124,9 +1132,9 @@ dependencies = [ { name = "pyyaml" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/ad/bd0c954c4c4671e586b340059139ead38570ba916a237086e245982ab14d/datamodel_code_generator-0.83.0.tar.gz", hash = "sha256:30d841e31b82095c4ef11f3dbfc99525f559a146250b9e685e2a2e4a196ca0a1", size = 3074761, upload-time = "2026-09-24T08:56:40.622Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/69/56726c1ffd4790f90dc47630f96baed1a29a8f25375982510fce3b3e61af/datamodel_code_generator-0.86.0.tar.gz", hash = "sha256:1834ba0e84307dfed625eeacfbb07d790ede01a3adf5414a1c9947c5b9139078", size = 6255349, upload-time = "2026-10-18T15:43:30.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/4e/4a0231a7d3d4043cd27a746a50be87bc7637463023d7db9991610cd5fe56/datamodel_code_generator-0.83.0-py3-none-any.whl", hash = "sha256:371d80be360b2a57bc48462169553af57fa30225c0264ad85ae5f0d59b9d143a", size = 751087, upload-time = "2026-09-24T08:56:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/f76dac052451e88fc0b622c3d996fa48637da9fa1e3c2b9ca2b84aead28a/datamodel_code_generator-0.86.0-py3-none-any.whl", hash = "sha256:1a3d6f0a14a33e334b4251d04e67b0b6ef05c96d4625fb99bd09d716e30ea084", size = 1605265, upload-time = "2026-10-18T15:43:27.808Z" }, ] [[package]] @@ -1187,7 +1195,7 @@ wheels = [ [[package]] name = "dlt" version = "1.31.0" -source = { git = "https://github.com/dlt-hub/dlt?branch=devel#7f09267b4430b5c2e95ddcdae36fc9d614dce514" } +source = { git = "https://github.com/dlt-hub/dlt?branch=devel#2c984279acff03b29b7dc2f7de496cc65898ea18" } dependencies = [ { name = "click" }, { name = "croniter" }, @@ -1326,7 +1334,7 @@ wheels = [ [[package]] name = "frictionless" -version = "5.20.0" +version = "5.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -1350,9 +1358,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "validators" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/e1/39a7624321497e037692d3675b45ad51445575e3d2ba935978701d247f1e/frictionless-5.20.0.tar.gz", hash = "sha256:d83aca7148fe146254ac00a8c9517407c8f4974afd12b09f06bdca0c407448bf", size = 74402411, upload-time = "2026-10-08T20:47:52.43Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/f2/a0978b650b0c8b19e087e545aeb992d3f0b3596c763244565f1ea9075056/frictionless-5.21.0.tar.gz", hash = "sha256:4d9848570bc682943375824a4be7633e8bbdeff228affee763cab346e48000b4", size = 74413561, upload-time = "2026-10-15T21:52:18.022Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/4c/db4275f56375169edf065cc3461091f0a0421f46ccde636de1cde49105df/frictionless-5.20.0-py3-none-any.whl", hash = "sha256:c7dbe8d27b8a21a561b622cddf764ce3d5a2ee1688aea096c5caee7b106c05c3", size = 563587, upload-time = "2026-10-08T20:47:49.597Z" }, + { url = "https://files.pythonhosted.org/packages/88/e1/8140a97773b48ceb2b8d04179a13aec003c3999be764813ad47830fab657/frictionless-5.21.0-py3-none-any.whl", hash = "sha256:c76db03abaee4862aa04116057dbc13e788a2fc18b9be37c2d35dc07a67e1b11", size = 575379, upload-time = "2026-10-15T21:52:14.863Z" }, ] [package.optional-dependencies] @@ -1790,20 +1798,20 @@ wheels = [ [[package]] name = "jsonpath-ng" -version = "1.10.1" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4c/dc/178bf7bb75d2df2532d0d1796805381f2599eb805c40eeda089538af9393/jsonpath_ng-1.10.1.tar.gz", hash = "sha256:1247d0983361ebe44f47741e759bbb76e74213c68f25abb4b65f6de21d1934d6", size = 87626, upload-time = "2026-10-12T12:57:12.048Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/2d/ebcea1c9a2338a08a5f6107bd274bd4b1e0c9a2ebc82ddbc815271ffee3b/jsonpath_ng-1.11.0.tar.gz", hash = "sha256:938b9c8d02f625c079c884f2b76241d0ac54ee7d1c8f10a502c42d8e31ff0c4b", size = 94460, upload-time = "2026-10-16T20:11:02.037Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/e6/d0f38911783aa7bc69afb0cdf5151e8cefeecd8ca3944c5453e13fc5afda/jsonpath_ng-1.10.1-py3-none-any.whl", hash = "sha256:9355047e5e6a8919f5ae0ccfd5b793bff69e4165f1248b1763e8962457b58ff5", size = 75386, upload-time = "2026-10-12T12:57:10.48Z" }, + { url = "https://files.pythonhosted.org/packages/c6/28/7bf49fd8ef55167239820e05e7bb501e0cc4cb20075c9e4da377e1616761/jsonpath_ng-1.11.0-py3-none-any.whl", hash = "sha256:84b325f791e43ae40810ff4059f50744b4c22fe53066b0884a659f0191c0b308", size = 76222, upload-time = "2026-10-16T20:11:00.574Z" }, ] [[package]] name = "jsonpointer" -version = "3.2.1" +version = "3.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/a2/c92f0a7ed439c490d2c8ad712fdb074c311afa4f77870987826a3b1483ba/jsonpointer-3.2.1.tar.gz", hash = "sha256:47c846513b3a4ec46eecef1105207fba075e2a3659048e362bd7daff0fc33342", size = 11309, upload-time = "2026-10-10T21:11:06.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/77/9be5cce647fa4d7afa9047c0c1029bc73292c27430e64fc9d2393ccf6727/jsonpointer-3.4.tar.gz", hash = "sha256:e57015ae65af3821c6a3976069860789cf326103f38b22b2178d41d021adad25", size = 13113, upload-time = "2026-10-19T10:33:24.436Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/29/accef8eea16670b88f3a23102c35dee62c6f159db30d9628908043b3e21b/jsonpointer-3.2.1-py3-none-any.whl", hash = "sha256:b19ee68644e9ffb51440448d8f7811af2b7406eea1db90603e93f5849323119a", size = 8332, upload-time = "2026-10-10T21:11:05.648Z" }, + { url = "https://files.pythonhosted.org/packages/10/d0/3d41544018b879e9c608c32bd5fb468fd87669d53c2dbf5ae67c1bad9882/jsonpointer-3.4-py3-none-any.whl", hash = "sha256:136e2e3273fd82a1502274f56ec172f331b6a1703d86a1824d503af4d9a96142", size = 9047, upload-time = "2026-10-19T10:33:22.963Z" }, ] [[package]] @@ -1892,28 +1900,111 @@ wheels = [ [[package]] name = "lazy-object-proxy" -version = "1.12.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/a2/69df9c6ba6d316cfd81fe2381e464db3e6de5db45f8c43c6a23504abf8cb/lazy_object_proxy-1.12.0.tar.gz", hash = "sha256:1f5a462d92fd0cfb82f1fab28b51bfb209fabbe6aabf7f0d51472c0c124c0c61", size = 43681, upload-time = "2025-08-22T13:50:06.783Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/26/b74c791008841f8ad896c7f293415136c66cc27e7c7577de4ee68040c110/lazy_object_proxy-1.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:86fd61cb2ba249b9f436d789d1356deae69ad3231dc3c0f17293ac535162672e", size = 26745, upload-time = "2025-08-22T13:42:44.982Z" }, - { url = "https://files.pythonhosted.org/packages/9b/52/641870d309e5d1fb1ea7d462a818ca727e43bfa431d8c34b173eb090348c/lazy_object_proxy-1.12.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:81d1852fb30fab81696f93db1b1e55a5d1ff7940838191062f5f56987d5fcc3e", size = 71537, upload-time = "2025-08-22T13:42:46.141Z" }, - { url = "https://files.pythonhosted.org/packages/47/b6/919118e99d51c5e76e8bf5a27df406884921c0acf2c7b8a3b38d847ab3e9/lazy_object_proxy-1.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be9045646d83f6c2664c1330904b245ae2371b5c57a3195e4028aedc9f999655", size = 71141, upload-time = "2025-08-22T13:42:47.375Z" }, - { url = "https://files.pythonhosted.org/packages/e5/47/1d20e626567b41de085cf4d4fb3661a56c159feaa73c825917b3b4d4f806/lazy_object_proxy-1.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:67f07ab742f1adfb3966c40f630baaa7902be4222a17941f3d85fd1dae5565ff", size = 69449, upload-time = "2025-08-22T13:42:48.49Z" }, - { url = "https://files.pythonhosted.org/packages/58/8d/25c20ff1a1a8426d9af2d0b6f29f6388005fc8cd10d6ee71f48bff86fdd0/lazy_object_proxy-1.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ba769017b944fcacbf6a80c18b2761a1795b03f8899acdad1f1c39db4409be", size = 70744, upload-time = "2025-08-22T13:42:49.608Z" }, - { url = "https://files.pythonhosted.org/packages/c0/67/8ec9abe15c4f8a4bcc6e65160a2c667240d025cbb6591b879bea55625263/lazy_object_proxy-1.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:7b22c2bbfb155706b928ac4d74c1a63ac8552a55ba7fff4445155523ea4067e1", size = 26568, upload-time = "2025-08-22T13:42:57.719Z" }, - { url = "https://files.pythonhosted.org/packages/23/12/cd2235463f3469fd6c62d41d92b7f120e8134f76e52421413a0ad16d493e/lazy_object_proxy-1.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4a79b909aa16bde8ae606f06e6bbc9d3219d2e57fb3e0076e17879072b742c65", size = 27391, upload-time = "2025-08-22T13:42:50.62Z" }, - { url = "https://files.pythonhosted.org/packages/60/9e/f1c53e39bbebad2e8609c67d0830cc275f694d0ea23d78e8f6db526c12d3/lazy_object_proxy-1.12.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:338ab2f132276203e404951205fe80c3fd59429b3a724e7b662b2eb539bb1be9", size = 80552, upload-time = "2025-08-22T13:42:51.731Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b6/6c513693448dcb317d9d8c91d91f47addc09553613379e504435b4cc8b3e/lazy_object_proxy-1.12.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c40b3c9faee2e32bfce0df4ae63f4e73529766893258eca78548bac801c8f66", size = 82857, upload-time = "2025-08-22T13:42:53.225Z" }, - { url = "https://files.pythonhosted.org/packages/12/1c/d9c4aaa4c75da11eb7c22c43d7c90a53b4fca0e27784a5ab207768debea7/lazy_object_proxy-1.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:717484c309df78cedf48396e420fa57fc8a2b1f06ea889df7248fdd156e58847", size = 80833, upload-time = "2025-08-22T13:42:54.391Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ae/29117275aac7d7d78ae4f5a4787f36ff33262499d486ac0bf3e0b97889f6/lazy_object_proxy-1.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a6b7ea5ea1ffe15059eb44bcbcb258f97bcb40e139b88152c40d07b1a1dfc9ac", size = 79516, upload-time = "2025-08-22T13:42:55.812Z" }, - { url = "https://files.pythonhosted.org/packages/19/40/b4e48b2c38c69392ae702ae7afa7b6551e0ca5d38263198b7c79de8b3bdf/lazy_object_proxy-1.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:08c465fb5cd23527512f9bd7b4c7ba6cec33e28aad36fbbe46bf7b858f9f3f7f", size = 27656, upload-time = "2025-08-22T13:42:56.793Z" }, - { url = "https://files.pythonhosted.org/packages/ef/3a/277857b51ae419a1574557c0b12e0d06bf327b758ba94cafc664cb1e2f66/lazy_object_proxy-1.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c9defba70ab943f1df98a656247966d7729da2fe9c2d5d85346464bf320820a3", size = 26582, upload-time = "2025-08-22T13:49:49.366Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b6/c5e0fa43535bb9c87880e0ba037cdb1c50e01850b0831e80eb4f4762f270/lazy_object_proxy-1.12.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6763941dbf97eea6b90f5b06eb4da9418cc088fce0e3883f5816090f9afcde4a", size = 71059, upload-time = "2025-08-22T13:49:50.488Z" }, - { url = "https://files.pythonhosted.org/packages/06/8a/7dcad19c685963c652624702f1a968ff10220b16bfcc442257038216bf55/lazy_object_proxy-1.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fdc70d81235fc586b9e3d1aeef7d1553259b62ecaae9db2167a5d2550dcc391a", size = 71034, upload-time = "2025-08-22T13:49:54.224Z" }, - { url = "https://files.pythonhosted.org/packages/12/ac/34cbfb433a10e28c7fd830f91c5a348462ba748413cbb950c7f259e67aa7/lazy_object_proxy-1.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0a83c6f7a6b2bfc11ef3ed67f8cbe99f8ff500b05655d8e7df9aab993a6abc95", size = 69529, upload-time = "2025-08-22T13:49:55.29Z" }, - { url = "https://files.pythonhosted.org/packages/6f/6a/11ad7e349307c3ca4c0175db7a77d60ce42a41c60bcb11800aabd6a8acb8/lazy_object_proxy-1.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:256262384ebd2a77b023ad02fbcc9326282bcfd16484d5531154b02bc304f4c5", size = 70391, upload-time = "2025-08-22T13:49:56.35Z" }, - { url = "https://files.pythonhosted.org/packages/59/97/9b410ed8fbc6e79c1ee8b13f8777a80137d4bc189caf2c6202358e66192c/lazy_object_proxy-1.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:7601ec171c7e8584f8ff3f4e440aa2eebf93e854f04639263875b8c2971f819f", size = 26988, upload-time = "2025-08-22T13:49:57.302Z" }, +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/8e/f26b09778fcb26261fdeb4cc2b011975292316529bff9b541a5a02acb976/lazy_object_proxy-1.13.0.tar.gz", hash = "sha256:ce34046cb021d69c72d05ddb83821ecfd4d2d0a5d43a93c5cd01c9e34779cf14", size = 45649, upload-time = "2026-10-17T16:38:01.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/20/b0258e552870cc67de8f4da00fe6608a1f1438db24252be9011d238d4fd8/lazy_object_proxy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7edc00d4792dba391667fcc5459b023b55190b9743735e7c1281855851c3d91b", size = 26810, upload-time = "2026-10-17T16:37:45.788Z" }, + { url = "https://files.pythonhosted.org/packages/6f/0d/853912d47395268e30f2f7ba9a8a9c2319e1626c4ffdfc866bae9a2e141e/lazy_object_proxy-1.13.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4046337019bd862440764930dffc47747a36b2e5d74346f0d404eadf1e3e6fa7", size = 71472, upload-time = "2026-10-20T01:05:41.44Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e0/90469d72e87eaedacb6ee7656ac99567f1880b87c2be9ce9eb72cac62ebe/lazy_object_proxy-1.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:903fc3eb8d87ff8fcb0b2f62e7d84e9808712aa085635612d2930b6ee2d8d4f5", size = 71090, upload-time = "2026-10-20T01:05:42.538Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/f5e6366b21de5860ec14c8155ee39cae25964117a8edb310dc023b562616/lazy_object_proxy-1.13.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:168e1eb2773a187c40544653346622917152c5f93163effd80dcce848888f01e", size = 65796, upload-time = "2026-10-20T01:05:43.827Z" }, + { url = "https://files.pythonhosted.org/packages/f4/69/fce5cb19ba6c1af5f3700c651ed397d306b759de4c2f7de7e4a88d0e051f/lazy_object_proxy-1.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:78a351da00aebe3ef8089f2c74bcaff8e851123419590fb0b86d8241a9ea3fc4", size = 69399, upload-time = "2026-10-20T01:05:44.991Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4a/edda1af23f22c69179591ec4b186bdd0549fa4ff5fd1b6c602dbcad4f30f/lazy_object_proxy-1.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ea9e832c3d9112992eb1f6f16543ef555d2b51ff288c5a16c0741b9a9ca88984", size = 65101, upload-time = "2026-10-20T01:05:46.231Z" }, + { url = "https://files.pythonhosted.org/packages/dd/0b/3effaa4ec12ef98c57cd39da8f36f8d93817e9b5e7875b0d57f4637ee645/lazy_object_proxy-1.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:65a369a41028bd2ba53dd31da0f6ee97682b70bcf2bf7413cb69ba11c63e4fd4", size = 70677, upload-time = "2026-10-20T01:05:47.399Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/6239ba7aa95a1a9f383c54050a37af4d6da449af2b711240154411623646/lazy_object_proxy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:54fdf11db2733dfd8dd8177f79d8d9c80dd99a912c0b1a8906412306ad18ade2", size = 25818, upload-time = "2026-10-17T16:37:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/1c/20/cd330c8425ad9e0c57b8cc3a72d4c64b60d7525d2cb946c588d3ee66f6f1/lazy_object_proxy-1.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:042284eea98fc08b9d347e43b7c258e6c862b4c03d5881d536bc61d48e6be88e", size = 24473, upload-time = "2026-10-17T16:37:47.919Z" }, + { url = "https://files.pythonhosted.org/packages/d0/0b/410ea9a56ad59421d8969535099560f0bad7162d483869e1c07ff5880f35/lazy_object_proxy-1.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:62612150e2b67d1cb26c2201d020773e9bab9c596d783d55ada42885a695f749", size = 26856, upload-time = "2026-10-17T16:37:48.864Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/79ab37585905cf6841134806e73ace576d61a87c4eee333be4622161b23f/lazy_object_proxy-1.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ad7630266b5775463f2a1806165c1d9dff07780cf1bba3c53f311e1d026e8921", size = 70985, upload-time = "2026-10-20T01:05:49.274Z" }, + { url = "https://files.pythonhosted.org/packages/18/2e/66bbb45f20aab9e81139e2bf9fea1a29850b5af769cbf7f181221ce062dc/lazy_object_proxy-1.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73f05b09e37b0eb0ca7d6e80e2b0689c49bedac8c5d69667c1ebb9f02a4ed5c7", size = 70984, upload-time = "2026-10-20T01:05:50.548Z" }, + { url = "https://files.pythonhosted.org/packages/74/e0/63dc39750c7b266e9e25022962eba32ba80e4f17fdcd674fc4ad4143fd90/lazy_object_proxy-1.13.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:91a725d3c64dfe13d365cf78c469c2a8f95884859379aeca89cb8d2ed291649a", size = 65689, upload-time = "2026-10-20T01:05:51.765Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6c/b94a8779eec1331477557ca3ea4742860d59dc9bcd4dcf21bcef3cd8c4f2/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e28a75d33378141927c08f3eb291cc98d0a22776670ae330f8347f898cd3b175", size = 69492, upload-time = "2026-10-20T01:05:53.116Z" }, + { url = "https://files.pythonhosted.org/packages/60/99/6c3ff4a32f15e807ffb868c47de136ebd4be31e7320af42b699dc43a7495/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a785fe44b61fcbffc0e264fe6b735a909e0b5ef922f0e976e281bbe8477fb47a", size = 65019, upload-time = "2026-10-20T01:05:54.462Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1a/447ff40629a11f879f194e839d59e5a026dd26f4d0eb8672de858644be85/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e321425f76451885c93083ecec0850b64de0f83bb94c113e930e8a0a630e45ab", size = 70334, upload-time = "2026-10-20T01:05:55.619Z" }, + { url = "https://files.pythonhosted.org/packages/70/52/2ca84fb914b17d4b02193cbf8b6ba9344b36459944a19711d3da1d14b03d/lazy_object_proxy-1.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:0e52094b7fff1f26f4e227b88be3f350817078e4bb79b4c80846fca0d15efae4", size = 26122, upload-time = "2026-10-17T16:37:53.02Z" }, + { url = "https://files.pythonhosted.org/packages/84/c7/ebbeb84639e84fbf6f07a48373f76fe048d23fbcfd4557509b48f35d284d/lazy_object_proxy-1.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:b2dd1c37d64cf9f3f4a2072971639408989b0803a3b7f05e75c1fe28d1a93c1f", size = 24746, upload-time = "2026-10-17T16:37:53.954Z" }, + { url = "https://files.pythonhosted.org/packages/38/ac/8ff35cc4d1316e056144edfb816dbff9b5db9ccd7e98c69f6b602c11bc29/lazy_object_proxy-1.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:348370676ea6d9ed08f9b74545c37e5be4cf009c45b9d50c6aff476fb7fcc7ec", size = 27524, upload-time = "2026-10-17T16:37:49.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/c5/8afe34ead41fdd48a207c4535192a4e931c56c791d5f0a57c0abd4558c1c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:dee2fb3326cd81384d2b752c3f6888d7ed9eba2cdbf0583097b5168b8bb4a703", size = 80523, upload-time = "2026-10-20T01:05:57.129Z" }, + { url = "https://files.pythonhosted.org/packages/38/fd/0e68a385daaf141b1b1d943b132b8a9887bd53abb88886dab286cca9b62e/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b4c446cbc8e411b116c9ee9b475cb664b8bc5c3d912bcbf1bab5c785fc69a02", size = 82823, upload-time = "2026-10-20T01:05:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/91/ce/648e09f90dc59cb6007d9093c2cc17d72e2d93c06553828037126205ea7c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1e671e0e9f2793759f382a69a3853c8742474bc011865ed3595187340ac5956d", size = 74655, upload-time = "2026-10-20T01:05:59.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/4d/811442390862dbf85e00520ecb0cb58c7dbdf824dc563ad933729f9e20f3/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a0155084a5dc8ce3e1a3d6e3f2fb4d84d5fc09e66ed84dc0e5f8a2697d421ea0", size = 80848, upload-time = "2026-10-20T01:06:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/24/4e/c30ea3678b50a7ed7774760fbde917106cbb860edb88cc31a17a8385b3d4/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2d07e0bde2738922a7117fccc1a82514aefbc9c07ee8ff0bb84a1c606dbb5ac8", size = 74127, upload-time = "2026-10-20T01:06:01.949Z" }, + { url = "https://files.pythonhosted.org/packages/19/d0/862f0f96b031dab7faa036679e2076bbd6a18ea06e34fb1400fca65b03f0/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da9f1b8780678af93c4ddba46826166c53a25d8d71133fc818542bf7b481e033", size = 79497, upload-time = "2026-10-20T01:06:03.122Z" }, + { url = "https://files.pythonhosted.org/packages/d5/50/20454483d8f95a11710b951518f86aaaaaae9195982fe27c12eeed130660/lazy_object_proxy-1.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f21b4711e6744d46a4672497ca2db390cbdb03241d9b1654bbf38dbbc4db2a5e", size = 26670, upload-time = "2026-10-17T16:37:50.977Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/c1ff843fd9c4978d9a50ef2bceda3897d71645d4127ec1413aa15204cb64/lazy_object_proxy-1.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:de295ae25e5c1fd712088547f251c4b2976d2f60df28057b6c416e771a99842b", size = 25297, upload-time = "2026-10-17T16:37:52.088Z" }, + { url = "https://files.pythonhosted.org/packages/8e/87/3ef9b53ab82ec40edbedb5487095eea5b86e8f6260f86246dbbb0021eb55/lazy_object_proxy-1.13.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:14fbc6debdf55fe6a801a6f562456c20056f1d3b0a191f64552ecc4f9bbdcc69", size = 26873, upload-time = "2026-10-17T16:37:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/b3/2f42624f4c46464bc7fd1c15d50e6efb4e013faeb18bd58b94a34b8bf26a/lazy_object_proxy-1.13.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e5090a94e1f148b1323031342920ac4416d4702db93317e238724ea5252a0e52", size = 71663, upload-time = "2026-10-20T01:06:05.465Z" }, + { url = "https://files.pythonhosted.org/packages/33/6f/018118a339833a38f3a46f249ad04d2e97fa24ef2825b0515d71522ba19b/lazy_object_proxy-1.13.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89331615366c05e30bf5c0821b4a0536b42117c9e83d2aef03d47c8beebcdf0c", size = 71249, upload-time = "2026-10-20T01:06:06.531Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/c47648b865407872079d218f248180aa21d7cdfd2eaf13b5e0080cdb8a80/lazy_object_proxy-1.13.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:90e841c41f8cdd380d38f4e64df824d92e8245ea8798bf6ee7533c941b3c2956", size = 67114, upload-time = "2026-10-20T01:06:07.877Z" }, + { url = "https://files.pythonhosted.org/packages/74/29/453db8ff486f77e21b54bd70f792416c6e2a0660e5fb7d60739976741e4a/lazy_object_proxy-1.13.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:956757a130126c1c281c1f3cdb2856cef8ab933edf61bc7931c7947c91866af9", size = 69832, upload-time = "2026-10-20T01:06:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/9e/42/cb3f9cc75dfd59754f345a610fd90b7b7955e3b00963175c1b441dc7ee16/lazy_object_proxy-1.13.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:28d3de97c2dba54e2d1da3b3f2aef5c2bfab4a37520bd5ee275f7497914593ff", size = 66318, upload-time = "2026-10-20T01:06:10.107Z" }, + { url = "https://files.pythonhosted.org/packages/7b/81/eb932ad6b975f39010da76e27099ab25ca16617317aae06f00b3b4fa4e05/lazy_object_proxy-1.13.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c81002e6284f724cc1cb616951f254bb312b8fecc185de8ee205cd6288d370ad", size = 71004, upload-time = "2026-10-20T01:06:11.299Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cc/8f1fc032a070b6bf9ebd10340bf7e081bb90f979bebda1b5e7517c2ff8e6/lazy_object_proxy-1.13.0-cp315-cp315-win_amd64.whl", hash = "sha256:ca684bb99c88693f12df2f71d3753bec784810351b9053959ea4c17a453b9269", size = 26123, upload-time = "2026-10-17T16:37:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ca/c00a8eaeaf2100c012fc211c400ae17ff6b0659479f65eca9eedac3dcd62/lazy_object_proxy-1.13.0-cp315-cp315-win_arm64.whl", hash = "sha256:32fcf9849e1a27a1e8a79a8e593860661cc93e81b090283f911d823c8c52b092", size = 24739, upload-time = "2026-10-17T16:38:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/1dacde4090f1b78500e02539d531211ecd44e776d1b019cc56f11f49c8c1/lazy_object_proxy-1.13.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:cedfa35e1d9f2b91599b5f635e36d04c9accb68e10c413cbdb7d02d4327d20fc", size = 27502, upload-time = "2026-10-17T16:37:56.398Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9e/97b53e02d0c6fd7aac6b0b69f17ad60db4b347e276df134039a221f508b5/lazy_object_proxy-1.13.0-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:666d040e5f94c0d8420d5ce3e4e8926c1a51fbb26432a60dc61440c452c9b30c", size = 81091, upload-time = "2026-10-20T01:06:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/c7/98/4abead513874ed7abf13da76d986760aaa88afba32fa115bcd873e870882/lazy_object_proxy-1.13.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c842c320aadf7524b7361df692eea9a1d76ce533e35658188c4d51bc0dd7e85e", size = 82827, upload-time = "2026-10-20T01:06:14.029Z" }, + { url = "https://files.pythonhosted.org/packages/0a/26/6568132acafc3a1902d09ce47ce2ad51e63bfbed8cb696334be7db8710d9/lazy_object_proxy-1.13.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7fd1cbcf0ac498aad0a3ff4142cc4daf70961017ac895c6b2ed92e8a128d5009", size = 75963, upload-time = "2026-10-20T01:06:15.357Z" }, + { url = "https://files.pythonhosted.org/packages/e8/73/39585234cc2cb6b97338ccff83e174f6d031b0366e454db3a8626e35fee7/lazy_object_proxy-1.13.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:1bdad8baf318ca048af58d70fa1274e31e20b0fa073fbe9fd4ad7b32a51a3e61", size = 80810, upload-time = "2026-10-20T01:06:16.567Z" }, + { url = "https://files.pythonhosted.org/packages/61/2c/8a69f4e18b056504ee6530b1c3c40d1379fe62eefdb44416db87b996a675/lazy_object_proxy-1.13.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:291263a1647524673146f10192caf27113a32c2e6d169e19c6a257f2876be343", size = 75485, upload-time = "2026-10-20T01:06:17.63Z" }, + { url = "https://files.pythonhosted.org/packages/b6/07/1acf68a04a60ab916bce01637f91f051bf1e7dd0e79a5a92b8e1afe8691d/lazy_object_proxy-1.13.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dd0c8ce17ec708cc81d541697d3746f2dcd8e45d039078aee980f865f2f5d41b", size = 79936, upload-time = "2026-10-20T01:06:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/34/72/da0ef19fd182290d4cc465a9944be676c89529569e2e1bdb158bf0312850/lazy_object_proxy-1.13.0-cp315-cp315t-win_amd64.whl", hash = "sha256:68d6e346300c136dd4714a4772e592521beb1c0744c3f18f797854affeca96b5", size = 26657, upload-time = "2026-10-17T16:37:57.699Z" }, + { url = "https://files.pythonhosted.org/packages/6d/46/e27854a795dc70349a15f30195d62d1ecf53b107d9b4fd80a3834df63b6c/lazy_object_proxy-1.13.0-cp315-cp315t-win_arm64.whl", hash = "sha256:e97aa65f98cc4521f2e366a49dfb6a3c2315dbbf5d84e97d5d1e21b649de72fa", size = 25291, upload-time = "2026-10-17T16:37:58.808Z" }, +] + +[[package]] +name = "libcst" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version >= '3.14'" }, + { name = "pyyaml-ft", marker = "python_full_version < '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/c0/098e5c91ff1537f00c85a6438b6cb1863d17144680cc91f47c87f104a200/libcst-1.9.0.tar.gz", hash = "sha256:087b58a9afe076bb08e2d726478e1f16cb928d67ffa9092817e033c335de522a", size = 914739, upload-time = "2026-07-29T21:28:43.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/b1/befc0544283bb3923a928accf79ed685e5a725524bdb3491826670affc07/libcst-1.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b9df30f524317b097dc53065b25dda33d6a4cc3c7c8bf4fc83ca7559c58cc0", size = 2043754, upload-time = "2026-07-29T19:24:59.885Z" }, + { url = "https://files.pythonhosted.org/packages/45/50/fef7c172a8457c95894edf5fb04805024899cdfea41fa01b0636587b79e1/libcst-1.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e465a7bc9c2b9533eb9e06d2391f8819f811b5112919d4064c9fb8565aaafa08", size = 2203548, upload-time = "2026-07-29T19:25:01.323Z" }, + { url = "https://files.pythonhosted.org/packages/18/ff/764cd2be1fd99d774fc44039c319dc0ed1d9d9afeaa02759a05121cccd4b/libcst-1.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8504b422c95676a8c27b517e1ac01413ece91bf356865c587ca9bdcd5708a2f7", size = 2255274, upload-time = "2026-07-29T19:25:02.764Z" }, + { url = "https://files.pythonhosted.org/packages/34/a7/474748a27a02fa83e3556b260d5f5236ca48164fa7beffecf3b2cad24dca/libcst-1.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bcb9f9d4fcfe2ec7a40d2c26e03a538d5d9dc189c38551eb3f94ab661afee7c0", size = 2269126, upload-time = "2026-07-29T19:25:04.158Z" }, + { url = "https://files.pythonhosted.org/packages/8a/b7/655e45363b8cf87b91e41e060b21c89e5316c7ef36eb14a1a498b27bb71e/libcst-1.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8d671c39a431c309476099b8ec811e412503ec0f4465f6fc907cb51c70e8e6b", size = 2378602, upload-time = "2026-07-29T19:25:06.424Z" }, + { url = "https://files.pythonhosted.org/packages/db/13/6da63f0902ece43bf9d737017251ad7ad06edd9d3c4e1856450403cb4473/libcst-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:4d382fba04077eb556a1ea4295a4482e192aa93639c5a07ba0885841965ea0c0", size = 2103486, upload-time = "2026-07-29T19:25:07.979Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ff/dccb1a55e38b4e47256a97242d61d2bef5366c744faf88fb087d4fa0f995/libcst-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:a621e261990148c1cfbe26c1798bd2375c131cf358a44a7a4659fc41c1a336e1", size = 1979907, upload-time = "2026-07-29T19:25:09.532Z" }, + { url = "https://files.pythonhosted.org/packages/65/2a/4943c71d90975bc59034a057dea346b365c276f308c1d31f5cf2bd85492d/libcst-1.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eccf4c57d273cdd3fe1c67b72cf9bb1bbd4547aa011824e96ccf5b7136057aa4", size = 2044529, upload-time = "2026-07-29T19:25:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1168b98c2a0f338be3a44753647baab051feaf6167cc887bf4447f8fd920/libcst-1.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32395244edfe6538e0ea2bf82051d60103d3f54861274805c4fb3745efb70a85", size = 2203519, upload-time = "2026-07-29T19:25:12.543Z" }, + { url = "https://files.pythonhosted.org/packages/25/7d/2eaa697a80f899bcf2245680bbfcc1e478eb3b3328c21d4b2880bdf00dac/libcst-1.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:444e84c76cd035cd2fe136838c1a524d34b08216521f6f5093df8a6f6cfa5799", size = 2255879, upload-time = "2026-07-29T19:25:14.137Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/793b9fd96dd52d202f5f2b88d7e54f05ebed767d1bb3b32395a1817bf6ab/libcst-1.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb5d0946f2b4c6711b5d69fe4f833b364f9e7a1a2b08f88b98619dc18975099a", size = 2269807, upload-time = "2026-07-29T19:25:15.647Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/1bc7aaea03971c45e8a885fed9fc1c73176dbbd00b0296a3cde9529bafd6/libcst-1.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45808c03528b3ad40b14095348a918e08d98c47b4a125d631cb78e817c0a5b16", size = 2378171, upload-time = "2026-07-29T19:25:17.289Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f7/bc49e367d2bc8817213594dd52cf6b2da2dbbda90b5382d60653999274ac/libcst-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:568288cdbfe3b4ca3ae4852cb0a439ff053dd54c841bc4995bf2b71238b5de40", size = 2177847, upload-time = "2026-07-29T19:25:19.35Z" }, + { url = "https://files.pythonhosted.org/packages/87/80/4d81577a22e6d535d1a3409f3a1c6903e09036f0e17fc47f92169dbc3501/libcst-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:107593af46945593e7825821793393262bc4fa1d3ea24c3ed487b61269bbbdf8", size = 2058134, upload-time = "2026-07-29T19:25:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7f/c3f3a0e7a1a2adaa76e815e82a7814d6bfe7d49432276bba652248c68d0b/libcst-1.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f6248cb07444ab9a6733a855737a9febed8b9adca51347019348b09a3ac7dfe9", size = 2036102, upload-time = "2026-07-29T19:25:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/4e/af/2f5543255b2c7d749b966adeccc6bfb1652cf8b425afb913d0f3f025a865/libcst-1.9.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:496c24e0d3240bc7da45dae543aff3f5b6509978c39262d6b84ed2fb999dded2", size = 2194806, upload-time = "2026-07-29T19:25:24.017Z" }, + { url = "https://files.pythonhosted.org/packages/cb/5b/03f4cddce426d005e208b39ea7b2d456e667cdee0f1891360f0fc8430f20/libcst-1.9.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ea490fa8540503db5f321f0268becab46eb50f710e8cec8041e241fd66f6874f", size = 2246762, upload-time = "2026-07-29T19:25:25.365Z" }, + { url = "https://files.pythonhosted.org/packages/d8/31/9d5fe1e43dc3dbcc74f70f3e0e73fcdd8d84effc1059d8b45974f0d0d2eb/libcst-1.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:50ab94bb2524b419056d4003032b8c67102ac800f8d85b3c4260a01746d94dbb", size = 2259633, upload-time = "2026-07-29T19:25:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2b/6752b28d88c3a19b3bc0b9e1838443b6760d5c862b4f4b37955402659e24/libcst-1.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a2faaf92500d0226358125630f5aab4758e8aad3f2d70a10892ec3c700781a54", size = 2368043, upload-time = "2026-07-29T19:25:28.344Z" }, + { url = "https://files.pythonhosted.org/packages/e2/94/775825b2637f8ab05694b6a4b3802ae6783b4e799f9b58d2400c7e2d4369/libcst-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c7b548512db25af9c2997a95fa731bd6b6928ecbad6c0915d7482d8bb42d34f", size = 2176432, upload-time = "2026-07-29T19:25:29.921Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3d/88ad67427c6fd9db929e087912b0a540e5140e5cb77e7ca4170edaac8531/libcst-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:497d5329345f1f5df84e41b0bbd00204b64a2fd30dfe3cfaeaebca633a31e877", size = 2052931, upload-time = "2026-07-29T19:25:31.397Z" }, + { url = "https://files.pythonhosted.org/packages/39/e4/ad790b043a38b10cea13166c8dd716654ad8b227c20f12eebf6326191cd9/libcst-1.9.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a5068bf6114f6f4d79af7a6c80a28d1deb50441150ea1db13b5458ab34bec159", size = 2043987, upload-time = "2026-08-11T05:54:02.122Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/d3cd8275cc54ffc9817ade91442b3e6e9edd1a4518bd4e6dda13e3152dbe/libcst-1.9.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:7acfd18adcdd32dcf41ce676cf121002afcbe9ad2c72dc0c18d78465beedc228", size = 2204003, upload-time = "2026-08-11T05:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/da/62/87eddccb11d5d221d50ac4fff4b6e9b8d48c547028d01f7d0fbc5c8a1d75/libcst-1.9.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:86361e2b426bd1b403e52375703c8b764e226e571adcb30442510710681edbf0", size = 2256053, upload-time = "2026-08-11T05:54:06.228Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/a44126aeb9fca8b4e342e0cc803ea00ca8f6cd5712619a7897b3eba13797/libcst-1.9.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8c14abe844bec8b021111b3985474e49f5af799c020e8e40dedcac87c97a40c5", size = 2270576, upload-time = "2026-08-11T05:54:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/bb/51/3af3dd44b117d66486e0ed61e43a16b7fc8cc5c372e1dd4ca0e70b888d46/libcst-1.9.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:8930971d2299b7006bb5d10c10038f44efb6da89d5bca2823595e31a9cdb96af", size = 2378573, upload-time = "2026-08-11T05:54:10.093Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e8/e545087d298dbf6494e84a8092f0f5dbd62eedacabd4ecd6b364367b4804/libcst-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5891ce9cff815077614f509e3a23888c5ddb8081d6188e8ba1172c0ccc369022", size = 2177937, upload-time = "2026-08-11T05:54:12.168Z" }, + { url = "https://files.pythonhosted.org/packages/84/17/93a00a8e03494a84db102dd0e3d35b8876b1084ca2c194b331a168d86eb1/libcst-1.9.0-cp315-cp315-win_arm64.whl", hash = "sha256:1260b5d070a3324447a53a00387225a55472a62077ce01922d30a2a78cc4b138", size = 2058633, upload-time = "2026-08-11T05:54:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/d7/00/c3751b12eb81822ea0b6131d374a98bc6c024c4b9ec5f6ea8f53dc23e967/libcst-1.9.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:02ee2dbdb5c218116f16a021350fc267a14be205b50d81c33f5d73857ab6fbf7", size = 2036178, upload-time = "2026-08-11T05:54:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/2f/12/48ff486eb5adc593f4818569e8896b74b672414ab4722a74b17e4c4cf31e/libcst-1.9.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:3e5b684191a0462b2d40a73261ea7f4da6a49e7a030b7e0fceca46bebb51dfe5", size = 2195496, upload-time = "2026-08-11T05:54:17.625Z" }, + { url = "https://files.pythonhosted.org/packages/51/18/b13a4669864d41a4801fed7b86ede1049dc9a1e1dde250a7ee1f9c3b1285/libcst-1.9.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:2b150c4f298fe54eb0fe73abf71f57db74d070796140a8c206c9615b6861f01e", size = 2245769, upload-time = "2026-08-11T05:54:19.343Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c7/cf2d46744825e84afbd7228fdeeea8d23cf6c4b2074253c27efb7705c653/libcst-1.9.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:50b913e37187f00a6fb88962009364e1cf1b1284aa1762304f34b52b972f2218", size = 2260576, upload-time = "2026-08-11T05:54:21.444Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/5433d3d62250b2e4325938db101766bcab2a006819684713bcccb6259a88/libcst-1.9.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:9f92c75283030fd58fb7d6e560168a00381e0e3368ec8a026a3ec8a828a05f93", size = 2368709, upload-time = "2026-08-11T05:54:23.276Z" }, + { url = "https://files.pythonhosted.org/packages/87/39/9f0e1690727623f99489895db0e42048a3e1e8cea0627517c593e437fd83/libcst-1.9.0-cp315-cp315t-win_amd64.whl", hash = "sha256:4adf97bb1aff8039b0bd4991e2f838be6e4fad552821b26b71a5d1a653c2582f", size = 2177866, upload-time = "2026-08-11T05:54:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/cd/79/9dc7811883e67ea771057e8937bc536eb3e77f3635fc5a93cd85b6fe1ea8/libcst-1.9.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8f0dd08a5773d7051e105250b2a2c73d8065ea9b2d68e7e1bdc5244660e75f93", size = 2052908, upload-time = "2026-08-11T05:54:26.835Z" }, +] + +[[package]] +name = "linkify-it-py" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/98/7a1a5f31fd5c7ba93e963b168e244b8e3dd705b3d2a718e3c3307583bf57/linkify_it_py-2.2.0.tar.gz", hash = "sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee", size = 32939, upload-time = "2026-08-29T07:07:08.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/d4/1152d1c7ab42d8b908be64fd200ddc870dc9d4925e951198702084aa1a7d/linkify_it_py-2.2.0-py3-none-any.whl", hash = "sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443", size = 21971, upload-time = "2026-08-29T07:07:07.164Z" }, ] [[package]] @@ -2083,6 +2174,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] + [[package]] name = "marko" version = "2.2.4" @@ -2202,6 +2298,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "mdit-py-plugins" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -2338,11 +2446,11 @@ wheels = [ [[package]] name = "more-itertools" -version = "11.2.0" +version = "11.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/e5/8735dc589d9e3571e76cd684e7763dc0fcc940b811f71640410458749689/more_itertools-11.2.0.tar.gz", hash = "sha256:59960f488835254863a857eb501f69e4ec92fdc8822f098c0f48dba530afd487", size = 75940, upload-time = "2026-10-13T16:01:14.428Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/b8/c63995bebbb9ac9c059350e0fb8c82c2420c85aeca7d34e40aecf3df9281/more_itertools-11.2.1.tar.gz", hash = "sha256:cbf08fd0af284dc69718b9b76a8e6203df624d70209ea511e4219d350c856f63", size = 154133, upload-time = "2026-10-14T18:50:30.407Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/f7/86f27473377a8ebdbdfedec5e62e87fbc24c4c88ed0a2a5ea0a12ee2229b/more_itertools-11.2.0-py3-none-any.whl", hash = "sha256:e4c522352a24664c974d0be8382944107ae638281a06391a5a538366b5707fe5", size = 74407, upload-time = "2026-10-13T16:01:13.424Z" }, + { url = "https://files.pythonhosted.org/packages/5d/57/3c4b1c6f2e0cc29ba023802458ea3d2a996f87085559e4a5f2192d549532/more_itertools-11.2.1-py3-none-any.whl", hash = "sha256:35a7377edd1dd6608dcb2cdf534ded55ea32d49448875ad042cd3f879fb1ded0", size = 74405, upload-time = "2026-10-14T18:50:29.116Z" }, ] [[package]] @@ -2487,6 +2595,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] +[[package]] +name = "mutmut" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "coverage" }, + { name = "libcst" }, + { name = "pytest" }, + { name = "setproctitle" }, + { name = "textual" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/30/b334a22184026d92a7549c8a76ad1d397ea65a6f8534ef486556ab861c70/mutmut-3.8.0.tar.gz", hash = "sha256:bc670276518f893ab356f1d28826c49b7e606e7cf89c300e3888c0b52fc1d2ed", size = 83631, upload-time = "2026-09-12T16:53:58.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/2a/a9904a45804c1de50c0c3baad1512b78d5b66d2befd841104b8a62bf0f35/mutmut-3.8.0-py3-none-any.whl", hash = "sha256:4c0abcef6b8d1116f2adfaad19df12e1c621e4a402d4367ef15bf8cba497b91d", size = 81469, upload-time = "2026-09-12T16:53:56.906Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -2828,11 +2953,11 @@ wheels = [ [[package]] name = "petl" -version = "1.7.29" +version = "1.7.30" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/31/8c530c6b1f6d1a40c0fd09cb9f54b6553385a8f36c248ea699c1d5f89544/petl-1.7.29.tar.gz", hash = "sha256:3877f34bbc5b0b7f0214bf219584eefd08a86c144154726f1797f2051906f9c3", size = 458989, upload-time = "2026-10-11T05:05:44.264Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/5c/ec50cc70f4d73eb86172b96c67f377d99aeece0f1db983d8aab9582c89b8/petl-1.7.30.tar.gz", hash = "sha256:9621fe7f9d81dafec3bf440496118ec1b288215400810a90f683a02c6d120792", size = 460151, upload-time = "2026-10-17T02:53:55.179Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/0f/502b19a57156c2aaeb49f455537abfff793d7459de8116c77cefb221d473/petl-1.7.29-py3-none-any.whl", hash = "sha256:a52d87a03a1390bfed16afa38dabc2e3068061b62ff8e58ce61b61cf8d46d0bc", size = 262757, upload-time = "2026-10-11T05:05:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/62/e7/67a1467722b31a8e2ee2ae9e92ae7fd1338e6f02cffa462461c49aa39fd6/petl-1.7.30-py3-none-any.whl", hash = "sha256:99b7e9a66b68908146f0f590a4bb61a270572bbe33f33dbf5036a234e4fee2de", size = 263769, upload-time = "2026-10-17T02:53:54.013Z" }, ] [[package]] @@ -2849,20 +2974,20 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.13.0" +version = "4.13.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/a8/66d45abadff219e36e2a824181b8f6a67e7ed4572934d6252c71c29d5731/platformdirs-4.13.0.tar.gz", hash = "sha256:1aa0b0d3f224c1f07c295121e312a5a24a180d6ae5a8425ea1784b3e3863e9c0", size = 61094, upload-time = "2026-10-11T02:05:24.109Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/e6/d0abbb256b44e6a05748c6942511ff67db8dbb068de0c35b24bf1b395b09/platformdirs-4.13.5.tar.gz", hash = "sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0", size = 62362, upload-time = "2026-10-19T01:02:11.778Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/15/1633010b26e88e872c93b67c0b6c5e174fb74cb6fb5c1472b4d51d4a8f22/platformdirs-4.13.0-py3-none-any.whl", hash = "sha256:3dbcf4cd708f21cf876c4eaa90e58412bc4f033d87143f41b1493ff77c25b7e1", size = 32724, upload-time = "2026-10-11T02:05:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, ] [[package]] name = "pluggy" -version = "1.6.0" +version = "1.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, ] [[package]] @@ -3224,7 +3349,7 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.16.0" +version = "2.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -3232,9 +3357,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2e/3b/a5d2294799b53b448319978cfb5bd139d5a9d45e862af91661614f14c922/pydantic_settings-2.16.0.tar.gz", hash = "sha256:5b6c578049ede4db0e2ef3b4eaa4ad4069cfa9211f83fb38df899dfade50a614", size = 301031, upload-time = "2026-10-14T12:44:09.998Z" } +sdist = { url = "https://files.pythonhosted.org/packages/05/ca/7817b4bf0c4b15c6a6ab8cdf4cb96c268335a8ca2f525de93b6ebe4c7d28/pydantic_settings-2.16.1.tar.gz", hash = "sha256:aff602a1886338c2df3d1e958e2193a8dc33cec75b91f202e83fd98484b4b0da", size = 303436, upload-time = "2026-10-20T11:37:19.024Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/f4/b987bf8c51e5b19a95fa66d1ee596074141e085d9c2ddf97920803c7029b/pydantic_settings-2.16.0-py3-none-any.whl", hash = "sha256:7e73acf7f61936a15e5a3b6eedaea29f133357faf7272f2607ba479b049dd7f2", size = 80364, upload-time = "2026-10-14T12:44:08.233Z" }, + { url = "https://files.pythonhosted.org/packages/3c/74/fdf926afe145d8629c4b2ff57f4ea1d64ed7abc4d84d97463b2517a2547e/pydantic_settings-2.16.1-py3-none-any.whl", hash = "sha256:30f5313340da4e8f1b440d1910b92a5bc3ccbb6b4ba70ec2b8395bc921f418f5", size = 80771, upload-time = "2026-10-20T11:37:17.298Z" }, ] [[package]] @@ -3325,29 +3450,36 @@ wheels = [ [[package]] name = "pyoxigraph" -version = "0.5.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/bb/df1eebcf8cfe6783a63b871f53bddeb461cac663505b18028ad44f0ccabf/pyoxigraph-0.5.11.tar.gz", hash = "sha256:2b7d9bf02e7ed89cb0cbcf6c376aef361f1c3c9de49a7a8fb3ac231544bb6ba8", size = 5303115, upload-time = "2026-09-02T20:05:42.8Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/70/470f1fd094ad6931e6c63b1130ff75000f2e01d69d75e293c0d2910bebdf/pyoxigraph-0.5.11-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f2e94296ce723ed030784a79c02f7e780522588840c5a8c44e118bd7c0d280a4", size = 7683421, upload-time = "2026-09-02T20:04:55.259Z" }, - { url = "https://files.pythonhosted.org/packages/2c/0a/4ee81724aa7817aa0d15d762c8acec8a90cc0e843f57c883d2e108afe543/pyoxigraph-0.5.11-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3de0588f90a467fe2467ec76588bccb8c18e57f05f63c89b6ea921b057b37365", size = 8186692, upload-time = "2026-09-02T20:04:57.22Z" }, - { url = "https://files.pythonhosted.org/packages/ae/29/816040a8fd51026aefa5323939424a190cd69068d5beddf91e651243f2ad/pyoxigraph-0.5.11-cp313-cp313-win_amd64.whl", hash = "sha256:8aaebe4656b9e9d7ee575dad1c1fd810bb52bfa0690f13bdd408e975ae28b868", size = 5427768, upload-time = "2026-09-02T20:04:59.647Z" }, - { url = "https://files.pythonhosted.org/packages/01/b0/bcde9432c0044369eb1b2e48c826559787ab7b1e713a38362f1e6bc1f9f2/pyoxigraph-0.5.11-cp313-cp313-win_arm64.whl", hash = "sha256:acbc9f82b75d8c39aa80fcf3c6d9f897c9bb23776af868fb6e9e39dc054e0d2e", size = 5084241, upload-time = "2026-09-02T20:05:01.934Z" }, - { url = "https://files.pythonhosted.org/packages/50/d2/873dad18e44c49c6d395c6b50e3dd97e45807dc645f46a9efa0f5c07798d/pyoxigraph-0.5.11-cp313-cp313t-win_amd64.whl", hash = "sha256:f6caa21919d0ebd4f165a4ade703e1f24cdd9cdb0a12fffa56440228d1106873", size = 5423392, upload-time = "2026-09-02T20:05:03.881Z" }, - { url = "https://files.pythonhosted.org/packages/de/9c/1618c0fd2e68608c2034d122fc620a080294b26bf3c2e039ef6706e50d91/pyoxigraph-0.5.11-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:18143baee09f6a3f17c096d6d58dbb3b1bf023ac5d6a52521cb2437cbf24b4a3", size = 7680302, upload-time = "2026-09-02T20:05:05.611Z" }, - { url = "https://files.pythonhosted.org/packages/bc/e4/9ae9d8014cf039a12c1d174e202587d2b3947226f9da173391cd3e344fa0/pyoxigraph-0.5.11-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e02906504ad2ac399d1f30cbae2e47b85932d39bf89ef5c7508268faa6ae3bc4", size = 8182140, upload-time = "2026-09-02T20:05:07.496Z" }, - { url = "https://files.pythonhosted.org/packages/5c/85/e8d325f5c001d16a67df710d14a8d8e2eb9a68c806a92dc62621ecb6bbd7/pyoxigraph-0.5.11-cp314-cp314-win_amd64.whl", hash = "sha256:81ccae2810d6f6b699c49f39a157a060b5713421e91ab7edb0ef354be04af583", size = 5421318, upload-time = "2026-09-02T20:05:09.182Z" }, - { url = "https://files.pythonhosted.org/packages/f1/8a/0a40ecae761d3e559873e20ac07f138ede16d3ce9f23f2cbd2f6a7cf239f/pyoxigraph-0.5.11-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b5167ed8771e9cdfeb8640c8f04aed06c295e5049752899d0ca221477ed327bb", size = 7674974, upload-time = "2026-09-02T20:05:10.997Z" }, - { url = "https://files.pythonhosted.org/packages/50/7b/f5582bab4d251ab9fbd4de20dfee17fe88d5fa3e73fb96683ea692c66421/pyoxigraph-0.5.11-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:13ed2633b72cf4a7cd6ef405d225e1a3e505228ffadb73c5f0aea4fd65f95cd9", size = 8178709, upload-time = "2026-09-02T20:05:12.938Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d7/ba4406bdc3d3fe7a5f0e3718e2838d1b40aa73f61090d801c36ca0591468/pyoxigraph-0.5.11-cp314-cp314t-win_amd64.whl", hash = "sha256:f58294bd2695f2fc8074f9bf8a381281c737f2903159ca602f5bfc3834559174", size = 5418434, upload-time = "2026-09-02T20:05:15.458Z" }, - { url = "https://files.pythonhosted.org/packages/38/c0/824cdec1e1ea9f6d4d05da51a843be668d3a2d02d223b780c138fcc4b2f9/pyoxigraph-0.5.11-cp38-abi3-macosx_10_14_x86_64.whl", hash = "sha256:aae8c162fd349a33255f580c665d8f950aaa875d65f64fae4a6c6fb93b5b7ccd", size = 6305518, upload-time = "2026-09-02T20:05:17.462Z" }, - { url = "https://files.pythonhosted.org/packages/18/fe/23899fc8e17fb6bfa37d606f8afc755c05dd081bd690d360d3754ea7d520/pyoxigraph-0.5.11-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:3b67839b598fc806dbed8e99eb2d75b26b0ded6d52ca8bff1496d6a3cc002036", size = 5817955, upload-time = "2026-09-02T20:05:19.199Z" }, - { url = "https://files.pythonhosted.org/packages/2c/27/175c5099548c76f85b1b80a8017ad98bff5bc92adc568b472d615e1f712d/pyoxigraph-0.5.11-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:96c9c4d117a0f4d0eae2c9092a490c6c51b0b8114ab7b126b8dfb0a8f0be2745", size = 7681128, upload-time = "2026-09-02T20:05:21.084Z" }, - { url = "https://files.pythonhosted.org/packages/9e/3a/9ec824aca0377ba56a7834222454c19392ff85b00e55fff9894f5211d655/pyoxigraph-0.5.11-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed906c05164d4766046a899f5944b4cf63309e717e3f464b2c0c80e8de91fa16", size = 8185704, upload-time = "2026-09-02T20:05:23.211Z" }, - { url = "https://files.pythonhosted.org/packages/98/25/5b0b9ecdebbd7600c3642be4b090cbe1c9ac5bae440c4bf2e5f82311cd0c/pyoxigraph-0.5.11-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c0462f03c4e3789fdee48faaab0edf780379fe812d1d70073eae14da86eadc9", size = 8879152, upload-time = "2026-09-02T20:05:25.423Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b4/fda0014c1ee5bc7950dfb7b9ce1c5f0bbb611d61560a9ef7e38dba9b83af/pyoxigraph-0.5.11-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c4f2c4c907dd751cc7f7966217dcb33ecb89c89c30b1992665ae965ec5064f01", size = 9412532, upload-time = "2026-09-02T20:05:27.772Z" }, - { url = "https://files.pythonhosted.org/packages/72/83/1588895bad95d257529a0b5bf47872f0c49602dae3cf2f6f1bbe9a5d583c/pyoxigraph-0.5.11-cp38-abi3-win_amd64.whl", hash = "sha256:1057b853663e3fa296f92dba3bb4145f545600261da0943266f4f449d8f7f0a9", size = 5424225, upload-time = "2026-09-02T20:05:29.95Z" }, - { url = "https://files.pythonhosted.org/packages/8a/61/fdb038cff915024cbfd5f6b8637e747c2e054261206a376aede1ee71588b/pyoxigraph-0.5.11-cp38-abi3-win_arm64.whl", hash = "sha256:ec99a70bfc9683dcecaea1f3000b6d6ba9c34a641dda48e660c456454f642ee6", size = 5083910, upload-time = "2026-09-02T20:05:31.573Z" }, +version = "0.5.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/00/d1d29089963a4863faf34d0659a5797c23503f1000b35202f15856a016b2/pyoxigraph-0.5.12.tar.gz", hash = "sha256:e1932585267dadeebc11ad97985a916e2cf66df81abb1e335495ed0ce2d5f0ed", size = 5300947, upload-time = "2026-10-18T20:01:58.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ce/10abfc263330577320188ca0cfaed4cd429f6d13d637698f8ca18858731c/pyoxigraph-0.5.12-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:4764a0cd745f1a3d310432a637b3ab4bc7eaaa24e294ec37b0e2a4e04d7d0ea5", size = 7700530, upload-time = "2026-10-18T20:01:01.299Z" }, + { url = "https://files.pythonhosted.org/packages/c8/68/262312b07ee1089bf929eeeb22f89967d9503b080c670ea665482111f462/pyoxigraph-0.5.12-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3c00ebb1289505cf79ee0336bae2df84f873be33f769d8f48e3d063cc4eeb60b", size = 8233189, upload-time = "2026-10-18T20:01:03.275Z" }, + { url = "https://files.pythonhosted.org/packages/7e/78/3e43c27d981f2ec549b9700e4d60d001f5ab4631806f1fa485556fa2c93f/pyoxigraph-0.5.12-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fcb9991d19f8f6566ef800c6016b566d60f7a38d4076bab7e0fbd3add32af627", size = 7699716, upload-time = "2026-10-18T20:01:05.048Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a1/edd0398096eda0522f4f49b5de50f40a7f9278dbef28aa7ba290e7454ce4/pyoxigraph-0.5.12-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b59c72eae73d4e66c6a687a510133a54ebce3e3ebbdb7c69dfa7e4de36504427", size = 8232847, upload-time = "2026-10-18T20:01:07.208Z" }, + { url = "https://files.pythonhosted.org/packages/07/97/32ed238d27872a724676f84b590c2ab64d64726c0f8105fc2a89f10ca268/pyoxigraph-0.5.12-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0077b688f6fc9cf3db4f3eee7acb5153734e958bee2be4ad7345121332c2bdfe", size = 7694160, upload-time = "2026-10-18T20:01:08.986Z" }, + { url = "https://files.pythonhosted.org/packages/3c/15/6dd1844c2f09b963303203a8f531c7cfc48cfa548beff5c19293814cf4be/pyoxigraph-0.5.12-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ad18580dc0e4ee4afb38e10f1e80f536a0a92194455b7f8c964f11fd0c64a8c9", size = 8226954, upload-time = "2026-10-18T20:01:10.982Z" }, + { url = "https://files.pythonhosted.org/packages/3a/33/6b9ccc2b9c91e8f5e59913be756483da61e98c598ed2c70267b473719562/pyoxigraph-0.5.12-cp315-abi3.abi3t-macosx_10_14_x86_64.whl", hash = "sha256:63bbb2c36f72aafa28424d6048336a6a6799236dcb80f48d3c8f00e3b7c07bfa", size = 6375614, upload-time = "2026-10-18T20:01:12.729Z" }, + { url = "https://files.pythonhosted.org/packages/0e/7a/650f2305517f193095bc5fb9069ed38044d26516c18502ad60e8c871b9dd/pyoxigraph-0.5.12-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:2f190eb24aae604dcf2739cb434c0ce87c1eafcb220de86675b1d59c73b739ca", size = 5864298, upload-time = "2026-10-18T20:01:14.439Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6a/1e318f595e5569cc0e9b5f3560f01a137a598ed1ae287a1320913bb1bb8c/pyoxigraph-0.5.12-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:fa44ed2c1b0b73a42398605534a22807762959ded229c04c7994d35777f715a7", size = 7732898, upload-time = "2026-10-18T20:01:16.503Z" }, + { url = "https://files.pythonhosted.org/packages/20/30/ef64fb7a02f5ffdfb4eb214a87995bdbbb8c4857ac6a7c454f7877d30297/pyoxigraph-0.5.12-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:060491a16a76b4dd617ceff85a25827a84183407f95308be0d5874f0df513ff2", size = 8266606, upload-time = "2026-10-18T20:01:18.42Z" }, + { url = "https://files.pythonhosted.org/packages/0e/43/b6ce713cac64dfe33a4beeb8e1f30f61a109a1e9c374de5aca5f5febadcb/pyoxigraph-0.5.12-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:5ac5d9fbfe61a3cd7551b421ce54bbd5bfd2aacf23883491594830981b76a5b9", size = 8924250, upload-time = "2026-10-18T20:01:20.374Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a1/3a29665bc035be550876890457931d6fde8940880a53dc3e7348d68cfdae/pyoxigraph-0.5.12-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:87459392fa087a14c2372826cb137f93ddfde4e7e566502619ebcf30c190ecf3", size = 9490513, upload-time = "2026-10-18T20:01:22.588Z" }, + { url = "https://files.pythonhosted.org/packages/9f/85/b06ab1e215a120924605e2f3aed723f81d568aaacd5e435cf8403b49f3ae/pyoxigraph-0.5.12-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:e8cd3698abb52e0be45db9f74827e65d5b4268b990e58a87b61c2b671c83279f", size = 5495177, upload-time = "2026-10-18T20:01:24.808Z" }, + { url = "https://files.pythonhosted.org/packages/63/03/b8c4cd224569f3ff53ec946928394f14116ef5e849cfaad6c46b562f3c77/pyoxigraph-0.5.12-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:0bc1816d12c63c5500eab5825d2d376ed55d86b66b5bffb3967daf51738bd4aa", size = 5144209, upload-time = "2026-10-18T20:01:26.758Z" }, + { url = "https://files.pythonhosted.org/packages/2e/21/f115cd9d4151f60fd2e96fadfd7103bae08d46493f6a016fb403d8483066/pyoxigraph-0.5.12-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ae8c01d2789d3e42810b8965ea18024689b9a76c54c007d00c7819ea6471bd8f", size = 7699015, upload-time = "2026-10-18T20:01:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/e09a14d2f8046481a8a926f4b6a610749d95b3fc0f585d4aabbd03cdfa29/pyoxigraph-0.5.12-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:2169f6d692b79f0ef2ab3a4f1c217a5a350a7b365d61db8b862296ef5c0ef04c", size = 8232403, upload-time = "2026-10-18T20:01:30.979Z" }, + { url = "https://files.pythonhosted.org/packages/04/3d/00674b72b2b2163eece7ebf3ed7f1740bf4275c5503c1342fafb029ca71e/pyoxigraph-0.5.12-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:11f069241d0be167a473c44f6d3184cd7560af771edb83c0d7f7c19d8c2acb94", size = 7695279, upload-time = "2026-10-18T20:01:32.993Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c0/194e4ace63625c153a1fa880887b2d250978f9a0581df19e0f98406b3ed2/pyoxigraph-0.5.12-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:8e105308a4c76ce0f98e014833f8b4181b82ada18b02900d687e57af954f8fbd", size = 8227289, upload-time = "2026-10-18T20:01:34.962Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/dc406eecec6ff6ae6b0f122a294784799a0f2e4d69c22ce460126fac3fb9/pyoxigraph-0.5.12-cp39-abi3-macosx_10_14_x86_64.whl", hash = "sha256:1a0c8931365df07fd77d5f6f7997bf0716ed5a7148d40aee2a6513101ecb6609", size = 6343151, upload-time = "2026-10-18T20:01:37.112Z" }, + { url = "https://files.pythonhosted.org/packages/d9/29/674774cc216bd671329ec1b5b60cded69c2566b40cef43eefe48de622099/pyoxigraph-0.5.12-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:b828b0e2a00c9d14725181d65677074a25f5f87e53eaf6bd89cfc772bbcf94b5", size = 5837673, upload-time = "2026-10-18T20:01:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/a0/24/9210f9d4b359347e8b4e1e484e11122646cabe8b3d64db65b4c13d9c372b/pyoxigraph-0.5.12-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9ed87da388d6cbe3ae41b4f5071c5890957d434afab872b104349a322f985e48", size = 7701997, upload-time = "2026-10-18T20:01:41.116Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/259a7296568de62f144937364d7b3e3baccfbe82ec726f13c245da2f8f7a/pyoxigraph-0.5.12-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f19c69a212771ad5368f51061731eec847cd8aa7f3075de9beee728e5c82826", size = 8234885, upload-time = "2026-10-18T20:01:43.607Z" }, + { url = "https://files.pythonhosted.org/packages/9a/b3/39eb47fff5442b9e762bae09eb7c2c496809f2b0090bdc5cb569e8f572b5/pyoxigraph-0.5.12-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d968d0d45f1f38c02a4417b3eebba3e830b2556732a67f2a528832daca10805", size = 8889557, upload-time = "2026-10-18T20:01:45.888Z" }, + { url = "https://files.pythonhosted.org/packages/a6/63/f9dd746e4095bc0704caa8ceae48b448cc3c44abdb76ebbca14cd7019166/pyoxigraph-0.5.12-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d89ef734fa4a51055bd234f1812c344fbaed7bb97d51516730cba5a517a6bac5", size = 9458815, upload-time = "2026-10-18T20:01:48.616Z" }, + { url = "https://files.pythonhosted.org/packages/7a/bf/8d9356c985516eb3c0fef581729875d01c9201a93d87c629c83d85988290/pyoxigraph-0.5.12-cp39-abi3-win_amd64.whl", hash = "sha256:41a146a5b1346c5289d06fc9c789ee045092a797a6d68b687e2afaa1714cfd2a", size = 5470372, upload-time = "2026-10-18T20:01:51.165Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2a/69dbd598932b509b25d6152fe4b2722c955ecc744e61e36e76e7de61fdd3/pyoxigraph-0.5.12-cp39-abi3-win_arm64.whl", hash = "sha256:fe34a10e262f93b4642f6e888aaa15aefd7b430c7c312ace511ccd01b1350a53", size = 5124124, upload-time = "2026-10-18T20:01:52.986Z" }, ] [[package]] @@ -3475,16 +3607,16 @@ sdist = { url = "https://files.pythonhosted.org/packages/c3/33/c987434f5d50aa802 [[package]] name = "pystow" -version = "0.10.2" +version = "0.10.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backports-zstd", marker = "python_full_version < '3.14'" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/ad/da2fd4b23ccf2fa81088a49bbc7d02f3fda7bd522d2f2a9ead77d474d2df/pystow-0.10.2.tar.gz", hash = "sha256:e30ff664a9099ab9356b25e4fa3c69a1612a32c2c5b81e8a23b4c8efc2fb8a6d", size = 58055, upload-time = "2026-10-08T08:53:44.245Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/a2/6e17e1fd6385b4426acfd75369593856536c04073576aa35b055bda6fc33/pystow-0.10.3.tar.gz", hash = "sha256:09276e0eeff99069eec1cc6ca24a0e90e4202981d98b2783c2a7c7bb323f8817", size = 64277, upload-time = "2026-10-17T08:12:59.548Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/05/6794a5d75724ca61afe81a1f1fe221cb018aaee5d20e42f756341fc0fee6/pystow-0.10.2-py3-none-any.whl", hash = "sha256:1f992aab1d20b8ba4b7942b3d0a0d4a84e48e0fcdb2b8781ebc7d41bf7648a2d", size = 65547, upload-time = "2026-10-08T08:53:43.008Z" }, + { url = "https://files.pythonhosted.org/packages/60/90/399349845ee1f84c182cb3f7e02ee0e89c6881ee6296571d19ee63989c7a/pystow-0.10.3-py3-none-any.whl", hash = "sha256:d70467457362a1c9373c575904bca5c63799bd011b6313ba15b4cf82f313957a", size = 65858, upload-time = "2026-10-17T08:12:58.105Z" }, ] [[package]] @@ -3531,15 +3663,28 @@ wheels = [ [[package]] name = "pytest-env" -version = "1.8.0" +version = "1.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, { name = "python-dotenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/e5/02fb78ca59d456291135b48fc95ee481a6e55dc47ad53e6cee31a7e40d54/pytest_env-1.8.0.tar.gz", hash = "sha256:e2dd383be15823a875403509949c0d975266f4f1711ccfbdcdbcbcf8fa83097c", size = 17363, upload-time = "2026-10-10T22:49:40.747Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/36/ccea73f55f269dc15c17ffb85d86beab7602bd7712f509300ef0d6bb937a/pytest_env-1.8.1.tar.gz", hash = "sha256:996f86ad78b6c7b5335e0d493f9e9ecd3e421174e056742856f91da439e1cbd9", size = 17852, upload-time = "2026-10-18T06:54:41.736Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/cd/c592fb78f8d5fc456568faf1983f2b994f00aa1690f0b3b24af95ad75121/pytest_env-1.8.0-py3-none-any.whl", hash = "sha256:43a026236949342be217f1539fa7ef32973f0d3a98d79d86a15eee973d3b1a2a", size = 10385, upload-time = "2026-10-10T22:49:39.251Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7d/d02c020f0daff882c9115c3c0381b8fac386570ef08c10d648a1a6fdf89e/pytest_env-1.8.1-py3-none-any.whl", hash = "sha256:753d3f0c646dd43f292a6d664099d1cafce39dda08217510ec5e9b6a681628a0", size = 10464, upload-time = "2026-10-18T06:54:40.508Z" }, +] + +[[package]] +name = "pytest-gremlins" +version = "1.12.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/54/01fee16e8de7a0534e79bfc20510f44e26581fbcd40bd099ff461ce1d374/pytest_gremlins-1.12.2.tar.gz", hash = "sha256:28ac7f9748ab95452c157c1d06b5a3958b3b7c34e642f75d53bea144b96899e6", size = 4295970, upload-time = "2026-10-19T04:17:10.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/65/915f8c0613f18aa91c0626629ed89979cf70d3f1462fc751d5fbef502919/pytest_gremlins-1.12.2-py3-none-any.whl", hash = "sha256:f0f75eeaa1713143de01ea516fff683bda10e24bebfb5d609d9910e02e3b2f48", size = 185299, upload-time = "2026-10-19T04:17:08.271Z" }, ] [[package]] @@ -3715,6 +3860,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "pyyaml-ft" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/eb/5a0d575de784f9a1f94e2b1288c6886f13f34185e13117ed530f32b6f8a8/pyyaml_ft-8.0.0.tar.gz", hash = "sha256:0c947dce03954c7b5d38869ed4878b2e6ff1d44b08a0d84dc83fdad205ae39ab", size = 141057, upload-time = "2025-06-10T15:32:15.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/ba/a067369fe61a2e57fb38732562927d5bae088c73cb9bb5438736a9555b29/pyyaml_ft-8.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8c1306282bc958bfda31237f900eb52c9bedf9b93a11f82e1aab004c9a5657a6", size = 187027, upload-time = "2025-06-10T15:31:48.722Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c5/a3d2020ce5ccfc6aede0d45bcb870298652ac0cf199f67714d250e0cdf39/pyyaml_ft-8.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30c5f1751625786c19de751e3130fc345ebcba6a86f6bddd6e1285342f4bbb69", size = 176146, upload-time = "2025-06-10T15:31:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/e3/bb/23a9739291086ca0d3189eac7cd92b4d00e9fdc77d722ab610c35f9a82ba/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fa992481155ddda2e303fcc74c79c05eddcdbc907b888d3d9ce3ff3e2adcfb0", size = 746792, upload-time = "2025-06-10T15:31:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c2/e8825f4ff725b7e560d62a3609e31d735318068e1079539ebfde397ea03e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cec6c92b4207004b62dfad1f0be321c9f04725e0f271c16247d8b39c3bf3ea42", size = 786772, upload-time = "2025-06-10T15:31:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/35/be/58a4dcae8854f2fdca9b28d9495298fd5571a50d8430b1c3033ec95d2d0e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06237267dbcab70d4c0e9436d8f719f04a51123f0ca2694c00dd4b68c338e40b", size = 778723, upload-time = "2025-06-10T15:31:56.093Z" }, + { url = "https://files.pythonhosted.org/packages/86/ed/fed0da92b5d5d7340a082e3802d84c6dc9d5fa142954404c41a544c1cb92/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a7f332bc565817644cdb38ffe4739e44c3e18c55793f75dddb87630f03fc254", size = 758478, upload-time = "2025-06-10T15:31:58.314Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/ac02afe286275980ecb2dcdc0156617389b7e0c0a3fcdedf155c67be2b80/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7d10175a746be65f6feb86224df5d6bc5c049ebf52b89a88cf1cd78af5a367a8", size = 799159, upload-time = "2025-06-10T15:31:59.675Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ac/c492a9da2e39abdff4c3094ec54acac9747743f36428281fb186a03fab76/pyyaml_ft-8.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:58e1015098cf8d8aec82f360789c16283b88ca670fe4275ef6c48c5e30b22a96", size = 158779, upload-time = "2025-06-10T15:32:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/5d/9b/41998df3298960d7c67653669f37710fa2d568a5fc933ea24a6df60acaf6/pyyaml_ft-8.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e64fa5f3e2ceb790d50602b2fd4ec37abbd760a8c778e46354df647e7c5a4ebb", size = 191331, upload-time = "2025-06-10T15:32:02.602Z" }, + { url = "https://files.pythonhosted.org/packages/0f/16/2710c252ee04cbd74d9562ebba709e5a284faeb8ada88fcda548c9191b47/pyyaml_ft-8.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8d445bf6ea16bb93c37b42fdacfb2f94c8e92a79ba9e12768c96ecde867046d1", size = 182879, upload-time = "2025-06-10T15:32:04.466Z" }, + { url = "https://files.pythonhosted.org/packages/9a/40/ae8163519d937fa7bfa457b6f78439cc6831a7c2b170e4f612f7eda71815/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c56bb46b4fda34cbb92a9446a841da3982cdde6ea13de3fbd80db7eeeab8b49", size = 811277, upload-time = "2025-06-10T15:32:06.214Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/28d82dbff7f87b96f0eeac79b7d972a96b4980c1e445eb6a857ba91eda00/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dab0abb46eb1780da486f022dce034b952c8ae40753627b27a626d803926483b", size = 831650, upload-time = "2025-06-10T15:32:08.076Z" }, + { url = "https://files.pythonhosted.org/packages/e8/df/161c4566facac7d75a9e182295c223060373d4116dead9cc53a265de60b9/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd48d639cab5ca50ad957b6dd632c7dd3ac02a1abe0e8196a3c24a52f5db3f7a", size = 815755, upload-time = "2025-06-10T15:32:09.435Z" }, + { url = "https://files.pythonhosted.org/packages/05/10/f42c48fa5153204f42eaa945e8d1fd7c10d6296841dcb2447bf7da1be5c4/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:052561b89d5b2a8e1289f326d060e794c21fa068aa11255fe71d65baf18a632e", size = 810403, upload-time = "2025-06-10T15:32:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d2/e369064aa51009eb9245399fd8ad2c562bd0bcd392a00be44b2a824ded7c/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3bb4b927929b0cb162fb1605392a321e3333e48ce616cdcfa04a839271373255", size = 835581, upload-time = "2025-06-10T15:32:12.897Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/26534bed77109632a956977f60d8519049f545abc39215d086e33a61f1f2/pyyaml_ft-8.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793", size = 171579, upload-time = "2025-06-10T15:32:14.34Z" }, +] + [[package]] name = "pyzmq" version = "27.2.0" @@ -4001,27 +4170,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e9/a7/70debb024dfacda67b8e560cc7511f52b34b9348a8cc8c1ec23036dcd51d/ruff-0.17.0.tar.gz", hash = "sha256:5cd03240d8208a557c2a9655a5cb07ebe36aa6bb35065f97d48c1f6adef5a322", size = 5320554, upload-time = "2026-10-09T19:47:29.248Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/f8/ee5ab9da6089eae2a33e6008b01deb1eda19992c1c8e10661e98cee1640f/ruff-0.17.0-py3-none-linux_armv6l.whl", hash = "sha256:0e271826af9a20d18c6cfae8c51e82959167c24859686ddd3eb9a7f0842ce81e", size = 10148907, upload-time = "2026-10-09T19:46:38.695Z" }, - { url = "https://files.pythonhosted.org/packages/9f/d9/2f81fb5a9d580afbb11b1c8ff915233a11f2a1b27405d7991f183c5e1976/ruff-0.17.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5f0ca4a40f81403689c04f12966e22f44e329ae362072d8f1587b7bda87f603b", size = 10224208, upload-time = "2026-10-09T19:46:41.711Z" }, - { url = "https://files.pythonhosted.org/packages/a7/20/643f3c8f75594f937b2bf74801241c56a2e2b8e139d24dff8b66b28cdd7f/ruff-0.17.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cbf7149e0927dc3295d5d64679a4765576eef71b00782b2ae969ef82274d6bb9", size = 9273324, upload-time = "2026-10-09T19:46:44.323Z" }, - { url = "https://files.pythonhosted.org/packages/ec/91/627700b233d367736cb274f1bd0b47d1f2b12f68878192812bd875adadc3/ruff-0.17.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ee90156522998c3037059d8f66885c8adeeaf7643bdce2caceee196ecd23e0", size = 9953190, upload-time = "2026-10-09T19:46:47.155Z" }, - { url = "https://files.pythonhosted.org/packages/cd/92/91f7b5ed39490f89d6cbf56e1f543c383667a725efa8e2c2dee0f01f5591/ruff-0.17.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d8e4a002a94cd9d0dc48b51dc69d807a172b5b9bf2b668e656424dc5b55ead1", size = 9961746, upload-time = "2026-10-09T19:46:50.098Z" }, - { url = "https://files.pythonhosted.org/packages/87/c5/7310f9fc63ce11ff6394edbd5e85433dfb0e14c9fbf6ccc97f1538491bc7/ruff-0.17.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c0b8a60c06a218c337e1161638d34757f83449243e2db161483ddf948e53ad14", size = 10625483, upload-time = "2026-10-09T19:46:53.379Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8d/97443f0dca4a03a0bc7629fd396fd494a1cb6666121e38c5075acb217d8f/ruff-0.17.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a330178bdffc4205dbf3bda11d93e059e388fd6546f8cdd304501a9160363c0d", size = 11350129, upload-time = "2026-10-09T19:46:56.486Z" }, - { url = "https://files.pythonhosted.org/packages/9c/0a/c525efd9777be4b6b012e6969a3012648468e7e6c4b3e5b46af69f46e8eb/ruff-0.17.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7bb08489e234876fa2da67ae3ea938e9a2156da80293e0e4365abd6973d98329", size = 10906223, upload-time = "2026-10-09T19:47:00.263Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3c/4a01195d93420cad1175bedad13a515dc8a56f95a6e39789b92e582682f5/ruff-0.17.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc73e7c133e82d55b5f15897b2a442d72c0cb4a0c886c46801ce3c247150b60c", size = 10386314, upload-time = "2026-10-09T19:47:03.057Z" }, - { url = "https://files.pythonhosted.org/packages/c7/72/1a3951665485a921f6375f91e754a1854d5a645d41acc3642668064ff64d/ruff-0.17.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:db4f74c533403ab70fe4007873f6ae0c9f94a8b03158cf48d78788e47cdbe399", size = 10565464, upload-time = "2026-10-09T19:47:05.831Z" }, - { url = "https://files.pythonhosted.org/packages/90/8c/539b4d8c082f57e18db8ae2be85a460d77861c79dcd5798e32b536a6a06f/ruff-0.17.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3d8cc360e666d1914e47b0777c6906d70cf18891a55532bd0a16844195d70859", size = 10079468, upload-time = "2026-10-09T19:47:08.617Z" }, - { url = "https://files.pythonhosted.org/packages/e0/b8/84286966db79434e8c26b585b0a0f6897cb3ab1c51a4aa4df10c28488b62/ruff-0.17.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d66de796b726c4801e05fa99a2a8d7a780e107be222486c304ab61765561e866", size = 9958556, upload-time = "2026-10-09T19:47:11.324Z" }, - { url = "https://files.pythonhosted.org/packages/69/50/27b6eed27b83fcdd5bfa0d52b83231e29094754374698da404d094487ae3/ruff-0.17.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c3f268baf004aea944f040623327119527ea231af15f7fb7890e82cea0679589", size = 10352173, upload-time = "2026-10-09T19:47:14.188Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fa/955399fd13044cd827862044117d784a59e3196f6cce7424908ac9a7f914/ruff-0.17.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:864b6c1acb6b0bccf94b5a3938a1531fd09aaca5e5659a2e7bf0f3cf2a685540", size = 10791761, upload-time = "2026-10-09T19:47:16.931Z" }, - { url = "https://files.pythonhosted.org/packages/ae/bf/024e01e1f6aec87768696725e648ed5b438941341eea8f8100beb681961f/ruff-0.17.0-py3-none-win32.whl", hash = "sha256:5e50aa5b84decd9fe5b0bb0e6f71c3b592f1767ed09faa4b7207d933961e35cd", size = 9904112, upload-time = "2026-10-09T19:47:19.75Z" }, - { url = "https://files.pythonhosted.org/packages/cc/77/1ee73df41dcc8d1cdb686ee4bc46ea29704ea175feb6b95c78420f631ab8/ruff-0.17.0-py3-none-win_amd64.whl", hash = "sha256:8ab76bcda86dfd28e13776cb5de3c7bcdcf1ae3d37ed761113d1a5a415dc134c", size = 10115585, upload-time = "2026-10-09T19:47:22.698Z" }, - { url = "https://files.pythonhosted.org/packages/fd/71/eb4f0ccc844aece56963e8578df9c95d4c00f580547d52329d4035d3af18/ruff-0.17.0-py3-none-win_arm64.whl", hash = "sha256:c154c73ff43f9854395e24cac507af13078962e53d2b511605058d22af1fdb88", size = 9872035, upload-time = "2026-10-09T19:47:26.306Z" }, +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, ] [[package]] @@ -4040,14 +4209,14 @@ wheels = [ [[package]] name = "s3transfer" -version = "0.19.2" +version = "0.19.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, ] [[package]] @@ -4059,13 +4228,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/02/bce7a1d70e9d0f6ea8623c16386138c251b5855af6cd3a11433d3fba4e66/semver-3.1.0-py3-none-any.whl", hash = "sha256:6e4998f8b4762acdfea85f9bfd5b4032e0bb50038dc04bf82011b42d5d1145c5", size = 18783, upload-time = "2026-09-12T18:38:12.457Z" }, ] +[[package]] +name = "setproctitle" +version = "1.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/b0/6b8a516c5a9e9630bd5293db78314ac012f690305fe93beadea388626efb/setproctitle-1.3.8.tar.gz", hash = "sha256:cafe209d064a6efb88cb45a03e97981ff8832802b2b5d009dde0197a3b7b41c8", size = 27214, upload-time = "2026-10-01T21:09:27.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/46/125791eb19c85340c6d43de8c821b143aecbfa0882343f244f5744474229/setproctitle-1.3.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a4a782f9ab26e662a890c1a7d865c452c83ed554de44aa83da35fa5bdcc79921", size = 18100, upload-time = "2026-10-01T20:59:33.41Z" }, + { url = "https://files.pythonhosted.org/packages/47/51/73b1e62c3e019e683e58d31f9f8af57e6750f36ce079004d29b7790e22ce/setproctitle-1.3.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63adfa4be8f9dbac4be6b359e2a8eb15a733545ef613d02a12ae27a4d40e975", size = 13116, upload-time = "2026-10-01T20:59:35.159Z" }, + { url = "https://files.pythonhosted.org/packages/78/ae/095c6ad159e3a7b95bc6d9a3eae18abfea1ebc79ed3377617406a1668150/setproctitle-1.3.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d4289e209e805d7d276a534f062092974fbe6814889d4c132c247205ebb49a65", size = 33275, upload-time = "2026-10-01T20:59:37.235Z" }, + { url = "https://files.pythonhosted.org/packages/f7/6e/5196258efd941cb874b3f4da91460601b867fdc434eba48a70455736d4be/setproctitle-1.3.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bb543b4b58ad839f7f0d335a7f4b40cef6989952e7239de3532de0b7771607e", size = 34124, upload-time = "2026-10-01T20:59:39.529Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/e9754e0916e6dda232a8c337bda69d2fea33acee0c5d3bdf580b8155a50d/setproctitle-1.3.8-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b705f01a531dba8fbbab90e30d432af4cc70a79c0a33b238e2c061816140cad0", size = 36006, upload-time = "2026-10-01T20:59:41.501Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a9/c63d2d745f18423937429a8611743e7b3e17259173150fbcdaa59daa4a51/setproctitle-1.3.8-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8e286e4c4e90063072a6e088a8d6526a7ffceb47429db88ba50ba58eeebcc758", size = 32268, upload-time = "2026-10-01T20:59:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/36/0c/a72818902f32ea9b848db5ccc828eb9cd9bc8661975610e1d46ea4aacbe2/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7b77c8671974288cf192e481ed63e928f1086762e97e8845d09d65ba121f8aaa", size = 33179, upload-time = "2026-10-01T20:59:45.809Z" }, + { url = "https://files.pythonhosted.org/packages/bb/55/fa59463ae6b957048e0e842d4b5513eced1b71554d8af0f5d91cd23cde23/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0e85b55eac9abfa3166ab00dac21e718ac2624a1df7d0b75559a2d3e3781cb33", size = 34472, upload-time = "2026-10-01T20:59:47.858Z" }, + { url = "https://files.pythonhosted.org/packages/b7/77/1f0cf55055788bf7eb9a146040a64d7ec42cd56153894856effc70e943ec/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f215ec9237d5f8e96d0205be6e2141292c20b133343c2153b6d4934f523ef422", size = 31428, upload-time = "2026-10-01T20:59:49.753Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/d1d8a6cd620d72a04f5d3335125e5b2fc5667bd06a76bb5bbc50b101e804/setproctitle-1.3.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0e70a5d294e4c8cf7dc94e015fca6d4913779a424937e66b429fba82d8eb5e2", size = 32776, upload-time = "2026-10-01T20:59:51.763Z" }, + { url = "https://files.pythonhosted.org/packages/4b/34/8b9eea1e80ac7a7f622eead8a0e4d73a60098344ed6f373e111f7ce8d29d/setproctitle-1.3.8-cp313-cp313-win32.whl", hash = "sha256:f85b4d87273e0ec63568433bbb8ccf160b548c49b53576fe010bad9f698b8586", size = 12105, upload-time = "2026-10-01T20:59:53.444Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f0/98883c658859d331faf91eccac8946cb2f8feb2590905015b129fdeb09cd/setproctitle-1.3.8-cp313-cp313-win_amd64.whl", hash = "sha256:bcdf74fe8a6e3b31bed1bc7d3e2a38cfbda3263d2750492ea0725dbc2db5aacf", size = 12933, upload-time = "2026-10-01T20:59:55.045Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1b/15043f8886aecfb31759c4a7c2cda9f85971242a10723049fc0fea34629f/setproctitle-1.3.8-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f5f1d7e0d57cd75febd70c6fc26c2ea06024ab2006011abb3094b9fea54801ee", size = 18104, upload-time = "2026-10-01T20:59:57.014Z" }, + { url = "https://files.pythonhosted.org/packages/0d/be/c2a6880394bef30aafe274d00ca0c7dac34fff1fb867f6c51abf876a640f/setproctitle-1.3.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b8954fc72fdd49588c8b8d615eabc27dfd8eb6d0cffe987fc5df02934816fb19", size = 13124, upload-time = "2026-10-01T20:59:58.611Z" }, + { url = "https://files.pythonhosted.org/packages/a7/96/6b3818e7f63a30f13bbe97b1ea0a51509a20b5b2d48c95e526c3054343d5/setproctitle-1.3.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4d6a0245679b74e0fda974edb672cd1f2351b0d762279b6776ca3a065c9322a4", size = 33180, upload-time = "2026-10-01T21:04:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/4f/43/fd3220b4bb2169d8723b3561ae7f8e1ee6f2e13e2da38b3a7a150004d2aa/setproctitle-1.3.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:394aca04e9f11c469f4f29d25a40a79eb4b8c9a72f438ba6390b776590f89757", size = 34071, upload-time = "2026-10-01T21:07:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/58/ee9eeebeb74125723b60eada6e6b2def747815228193327ce470da088f2b/setproctitle-1.3.8-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f92f6ded50c0ae7b0fed612be77c5345ed4a4578f99b98c7f8258d0e599eff16", size = 35923, upload-time = "2026-10-01T21:07:19.054Z" }, + { url = "https://files.pythonhosted.org/packages/f6/fe/b339602e96d73b29fd123c8b9c886e8f9a5afa12a18218eece56bd99c668/setproctitle-1.3.8-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c09f15930910dd895ac113b16c0b3ac2051dcbd44b0f017f94cb51349ba11583", size = 32116, upload-time = "2026-10-01T21:07:25.121Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5e/8d46d34648d8dfdc3f3e941c149ae316b6c6f417391018e279a1bb733c42/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4756a96435917494ec2f27f0da46bc05eecaa1a7e072d57220ecb2d46ea8a3d3", size = 32980, upload-time = "2026-10-01T21:07:27.826Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6a/10720d27d5f4a47c9af51652b49dd22313be20b9d542e72f80b4b363702b/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:65279b2df2951b64878da5382363e730d242486179c440c8653fe39aa3cc9e62", size = 34307, upload-time = "2026-10-01T21:07:30.62Z" }, + { url = "https://files.pythonhosted.org/packages/6e/65/95dd7b0245fdb7d2f6633613c559ecd9f0b2afeb6e3d962c218b6a1722d4/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3d2ced815b940c88097e07182c57cbfe1a0156839d1edc9be2bc0e4c74ab4eed", size = 31126, upload-time = "2026-10-01T21:07:33.59Z" }, + { url = "https://files.pythonhosted.org/packages/fd/96/4d62c4ba70cf8fa305f1dcbe35e93449819bb27b451fcb473556acf92650/setproctitle-1.3.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:37a91697c3b96bac58b6936b3bf0366af96a2a6cef963fdd9aaf0593a4bd1ff2", size = 32468, upload-time = "2026-10-01T21:07:36.312Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4a/1ec517435d4b572f6c468795331c7b376ec310eca5b1321504c6647b85a9/setproctitle-1.3.8-cp314-cp314-win32.whl", hash = "sha256:7cdd3d6e5e8abcee233157615d88dbb80527a5a4eccb95a93fbf92e6d2c8f9dd", size = 12255, upload-time = "2026-10-01T21:08:12.203Z" }, + { url = "https://files.pythonhosted.org/packages/7c/13/e74a4fa864a0a694104ff74fcb3bef523be94321bac1a8ece64ad04ef233/setproctitle-1.3.8-cp314-cp314-win_amd64.whl", hash = "sha256:5cff3bdbe6be556cc19bd072ff1ed9b5a8f4cbf356cd19b1ca94506b3e2b29f6", size = 13159, upload-time = "2026-10-01T21:08:14.231Z" }, + { url = "https://files.pythonhosted.org/packages/38/d8/019116e7defcd5d2c9264da4a82d369de6806a9161adf73e5f73e63fb715/setproctitle-1.3.8-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:84320e61f3906cc550b900ab1ba02ec53edd2d2e1e917f0873cba16592801c89", size = 18105, upload-time = "2026-10-01T21:07:38.655Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f5/bfa9d6b5574903c17e02caba0e25815bb1c7e7b895d594475676baaa35b4/setproctitle-1.3.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8086259943fc75eb07e11c1d08600709b9ce4fb2ec9bbd2490556c47d3ead135", size = 13125, upload-time = "2026-10-01T21:07:40.774Z" }, + { url = "https://files.pythonhosted.org/packages/d3/29/c7febeedee66b929c335d7b8ce83b1c4baccc65c94a4112fd8c1ea32b57d/setproctitle-1.3.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8879d303d4e3559efdba033ff198c6869010f7958412ec2047674881859fdd7f", size = 34818, upload-time = "2026-10-01T21:07:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/36/b9/90fea18bcb74f19d411df7e9e2d981c4d3e99817870acb0b5cac6540629f/setproctitle-1.3.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7de4da27fa4481e8a7a4ef58c908ffd2b996847c3d7fbf27b5c53ce0984c408", size = 35622, upload-time = "2026-10-01T21:07:51.03Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e9/5899eab6e0b4a127e98a268a157b6bdba4f8ad18d0ed90fdf3e081c20a3b/setproctitle-1.3.8-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:741b1ae7e1cd68745b8b677daba73685dab2d1b8bb12500e6537f3482a91150f", size = 37762, upload-time = "2026-10-01T21:07:53.758Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cd/190748a1f68afe6154b62517d2288b7d4c2092b0efa6e61d044c0f99d17b/setproctitle-1.3.8-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c85ff5b000eef8a280333ca158483a76f86bd374816add5c016a9f4294471289", size = 33548, upload-time = "2026-10-01T21:07:56.148Z" }, + { url = "https://files.pythonhosted.org/packages/26/db/889062b0970f04bc76d8eed03165dc17de8df282399ad931cb10a23b33a3/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:656ffe282ec0ca18308eebb100898c79b4f5698a93228d50d2f2a4a09474d7e0", size = 34298, upload-time = "2026-10-01T21:07:58.898Z" }, + { url = "https://files.pythonhosted.org/packages/6d/bc/adedb2c18eb7bb152b6bc59187f46fc328e7bb729bf28a7e4e43521c4c7a/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ae87e7b3e6414b3dd887adb72d2c4ea9f210c5f48efafd295ade4a58c760d29d", size = 35897, upload-time = "2026-10-01T21:08:01.561Z" }, + { url = "https://files.pythonhosted.org/packages/38/8e/f1769dcd1eb668b01ad251eb6780b030752d18dabc1b82789b255189772e/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:09cb645c6ae440286391e256203fc03522b3b0f55fda8219f698abb5c447cce7", size = 32435, upload-time = "2026-10-01T21:08:04.112Z" }, + { url = "https://files.pythonhosted.org/packages/79/e5/68be1eed21bba3ce27c6907ddcf0559cb99e4de3aa3ee000a38994371b73/setproctitle-1.3.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:84003eeaf2d7460390bbff1dbd7bee5b1213e7ae47a489f0fb7aad6acd2b045c", size = 34024, upload-time = "2026-10-01T21:08:06.514Z" }, + { url = "https://files.pythonhosted.org/packages/40/7e/f275783d6915fac997dbfa3729330552c4f2ddcd4f1bba7ff522f5f05c28/setproctitle-1.3.8-cp314-cp314t-win32.whl", hash = "sha256:ac4e037d276ce27069cad0083c4611b45232278bd2fbac33f06ef9438c171a1e", size = 12259, upload-time = "2026-10-01T21:08:08.432Z" }, + { url = "https://files.pythonhosted.org/packages/86/59/3347f13aa38c81d78f5c93bc55e4355c43550f47348e5de48c5080f3a1b2/setproctitle-1.3.8-cp314-cp314t-win_amd64.whl", hash = "sha256:c284642b229ba7b7b927fb044aba03694b880ab572db3f07a10adc786db1ff7b", size = 13163, upload-time = "2026-10-01T21:08:10.087Z" }, + { url = "https://files.pythonhosted.org/packages/92/e4/e9866831fd0cff677c67488059fab3786a843ff0bd86f4b3c44bea8dd763/setproctitle-1.3.8-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:63348d86ae60b071a898e4241b41da1346d8c36f1a60bf3b698ae5eb35b79865", size = 18107, upload-time = "2026-10-01T21:08:16.029Z" }, + { url = "https://files.pythonhosted.org/packages/ff/79/5c11a8c8cf3b91e54fdf4c93d1c79d99a491e04d77d34c80a90552fe6b46/setproctitle-1.3.8-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:59239b6e5007b1bec79ef444a390b3bcd59cbcb2686c9910da5a42f1e99f738c", size = 13132, upload-time = "2026-10-01T21:08:17.756Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/47dea0a33ec67a2813e40bd92deea74c3732fd598bd71b636a92897eaf17/setproctitle-1.3.8-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:38310d9cf94959e39d22fdc0154c66c8bdbadfa790c524b04cb36da6333c7507", size = 33509, upload-time = "2026-10-01T21:08:20.254Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7c/5d6945012077e6c393d59432df42d281d213f508bcf6d83a1287ab2eb9ae/setproctitle-1.3.8-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a279b136ed97407209d2dc9c30d1e65405d1a4a06ab162cedd9c4159b51f8f20", size = 34405, upload-time = "2026-10-01T21:08:22.796Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d9/543a4470bfd01529dc35f0014dc68c70665db8b790e3647b41d155203b87/setproctitle-1.3.8-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26997a85aa70bb088e86f0b7899fb0c909be824d22fc025e87b0005afbe59261", size = 36167, upload-time = "2026-10-01T21:08:25.35Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/fc55041b162833c4bbbbfe78ed417bcab7b22f9fa72aff1722ff6af46fae/setproctitle-1.3.8-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35fc5e10fcba6df25d676ef908b44a36c17e7f167738ed016561185e24f4d274", size = 32623, upload-time = "2026-10-01T21:08:27.778Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e2/8c73c990c338b9ca5bf4fc6167ac864a80a53a10e71fd2755ba146f1fce6/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:f5df0503093ec8c3c7e1da53920e79c62ab653b51b136c4736caa252552a8732", size = 33510, upload-time = "2026-10-01T21:08:30.431Z" }, + { url = "https://files.pythonhosted.org/packages/cd/61/8e10270d3a7326dc4e8a1408c631e6ae912625b91f38adaf04794a19e07b/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0a582308ba43afa7233d28243d9d1212e11c81d3c187a43929ec0cb70c4781c5", size = 34528, upload-time = "2026-10-01T21:08:37.161Z" }, + { url = "https://files.pythonhosted.org/packages/77/d4/00773ca8c876123348e65edfd9eec47464bbfa1a31067c6e1cd382356a0f/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ce48dbf74e8ec618d74cbaffe1159a4c32a4428de598f0ccdb0782bae618a72a", size = 31681, upload-time = "2026-10-01T21:08:39.874Z" }, + { url = "https://files.pythonhosted.org/packages/d5/42/19c6b634cca492575f9d99868bc35134528fb390d683d6a0396f61df0b8d/setproctitle-1.3.8-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:dd3a8fcee7d8a23e44781144830dee97b53ca4e0380bdd2a0e05b13913ce5725", size = 32999, upload-time = "2026-10-01T21:08:42.36Z" }, + { url = "https://files.pythonhosted.org/packages/94/34/09f653f687e4a1a6f515642519cc65f0e6bcf34631386380924fc9785b44/setproctitle-1.3.8-cp315-cp315-win32.whl", hash = "sha256:50beee618377ce6fd35767a9327f2da594b36778eb10b503ee2dc9b7c801f433", size = 12251, upload-time = "2026-10-01T21:09:17.504Z" }, + { url = "https://files.pythonhosted.org/packages/89/69/ab9afe8c198594533ad5eed6dcea647e1743ac8196a6c6dea974724b72e6/setproctitle-1.3.8-cp315-cp315-win_amd64.whl", hash = "sha256:f5916010de8b6045d7c63d98bb3f7a1e13f8c9c1046e859f2fd8de79c4e2d865", size = 13161, upload-time = "2026-10-01T21:09:19.295Z" }, + { url = "https://files.pythonhosted.org/packages/78/90/da93e1ae012f2515473a45fb33f5ba2ddbbfcce1381c5e41005b31ab3230/setproctitle-1.3.8-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:e9f643bd97374aeb7e4274e5c361277bf1428e8e55246cee6823fa3525dc2c04", size = 18119, upload-time = "2026-10-01T21:08:44.296Z" }, + { url = "https://files.pythonhosted.org/packages/af/35/41b28fa8eb8f1bee9c5516b99262a5df413937bf186e9d50ec3ef812fc61/setproctitle-1.3.8-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:fbd9ceb8c091c125e5892757e4d16af06674dc96edd7b5bf1e627b1ae7650c66", size = 13140, upload-time = "2026-10-01T21:08:46.797Z" }, + { url = "https://files.pythonhosted.org/packages/0c/19/089c878e6b8a3a391ceb03e1653ad6f33f27a5f7e204cf4a54b6a053b83d/setproctitle-1.3.8-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:322b4643f98c824b17dbf2ddb07ea7a37def8cc9e18adee320befce4bdffa271", size = 35122, upload-time = "2026-10-01T21:08:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/88/a7/f419b87fcb6202ef0a94f7cd6bdecb23c2d01f2f07b38f92d3f0c5dfb518/setproctitle-1.3.8-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e36ee7300584a53fd7ac5f3853323411bde1cf6c081921bd3754a73b4071245", size = 35944, upload-time = "2026-10-01T21:08:52.612Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/fb181112f3c09a8c8962bd110f538c8a9d287a0c6a8bfd5b4dc39d54e0df/setproctitle-1.3.8-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c05aff52e6eb9c4d2ea86cf0d0a60b1487ed570263a1b13bb30d9f96aa8ba97", size = 37981, upload-time = "2026-10-01T21:08:55.136Z" }, + { url = "https://files.pythonhosted.org/packages/d8/8f/934846cf63b5ae2e9a88dcac6ddc3ec813e6b9e0919994684f7674155b8a/setproctitle-1.3.8-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df807e7f036c19235320b6b5137291891d660f10ccb0b50dc46bc8385be18379", size = 34229, upload-time = "2026-10-01T21:08:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/c9/04/c11f73d8aa0ebe5d63866130ad14ca3789a4dfb9fc617c3a612af9c177f9/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:bef8c5f8820c70e8761171409052056ffd247023b001a6254287a0694c3d8b1c", size = 34743, upload-time = "2026-10-01T21:09:04.088Z" }, + { url = "https://files.pythonhosted.org/packages/38/50/8c780488db26e8f51af91465bda6ee224522248dc5220b282791dacbbe5e/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:7855290194637a3c5a2e37c423efbf5813fde97a6b63ab99feade5e05d73dc6c", size = 36094, upload-time = "2026-10-01T21:09:06.67Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e6/13d541653607b3b7232dc9f8baae05b24112ac9a3756c3653903c624c4a9/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:8c0a7753f5ddaf0ecd1fc8df3e2106611d4290f0508e643e723cb5000b7d6c2f", size = 33148, upload-time = "2026-10-01T21:09:09.064Z" }, + { url = "https://files.pythonhosted.org/packages/42/94/f3e16c3a7949fa28a777aa30830bf89098026e13669144b0ec1b147f17f4/setproctitle-1.3.8-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:97b35b15f81363a314baca76674c19027dce19eed1590abd51c2d285830efd50", size = 34488, upload-time = "2026-10-01T21:09:11.756Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e7/9baa826fc44c6f607a39ad35d419da7f51dec2df31ef169e7a4a34e1cc75/setproctitle-1.3.8-cp315-cp315t-win32.whl", hash = "sha256:ecf44731e351c6100885120cecf53a0f43a1bb975a92480de1f9e022ab542bc1", size = 12260, upload-time = "2026-10-01T21:09:13.6Z" }, + { url = "https://files.pythonhosted.org/packages/71/fb/08f55fab7294d18a6b1bc6d13340706c998b4641bd5f99ed9c7a626fcbf0/setproctitle-1.3.8-cp315-cp315t-win_amd64.whl", hash = "sha256:61aac898128dade36438da9387fea570f00cd519ad5231d3b268913afc2fc329", size = 13170, upload-time = "2026-10-01T21:09:15.351Z" }, +] + [[package]] name = "setuptools" -version = "84.0.0" +version = "84.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/e2c41d00a4bfea5548c31420a74c6cf149cc0e7c36b45c4c82202ce867aa/setuptools-84.2.0.tar.gz", hash = "sha256:6f445ca2ce864605f81e223f262b4c94ffcb9a4a63473bae0ed7be5aab0291da", size = 1227656, upload-time = "2026-10-19T20:38:03.407Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a2/2e1af99d56c8145aa69496026c7e5fd7b6ec759b500c3ad697e2062832dc/setuptools-84.2.0-py3-none-any.whl", hash = "sha256:8cc107ac7a403105dc8d6477664171f36135fab26adf8b5e0edaebc313f39b21", size = 875816, upload-time = "2026-10-19T20:38:01.629Z" }, ] [[package]] @@ -4186,14 +4423,14 @@ wheels = [ [[package]] name = "smart-open" -version = "8.0.3" +version = "8.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/5f/4c6d2be8dc4737d209f583a9c61e40c7b90d86313308e1d82e79a1b857b4/smart_open-8.0.3.tar.gz", hash = "sha256:398eb1d6af1e887ccd30347af540793235369bd6602dc42f72c44f939ae9c80f", size = 61941, upload-time = "2026-10-12T20:21:50.154Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/e3/1fb5c310b1f55d99ba51cb81b8f1a0e4b5f89be44de69405aeeb12814778/smart_open-8.0.4.tar.gz", hash = "sha256:8cff8321f0b0ae4bd9cb0cd0c507c135ad6aa980171248fca520b645fd6ff717", size = 62316, upload-time = "2026-10-17T19:34:45.106Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/79/60e14c5a19f39ebcf4fecd8ae8b0b01418ff5839bd7e66cf02b019d31826/smart_open-8.0.3-py3-none-any.whl", hash = "sha256:c3f67795b1a2bdb62dd9aca9ba4d4018e8f11e4188700b3246ea5ae049f65887", size = 73451, upload-time = "2026-10-12T20:21:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6d/3a95e71d597ee03b3ddcf0ff154f2841b84d8724df2bb35a350493793ae9/smart_open-8.0.4-py3-none-any.whl", hash = "sha256:8a3a67679942fa40d1a93ed3aea37619b4b4ec5458f1d5a9228dfc301db11b4c", size = 73881, upload-time = "2026-10-17T19:34:43.624Z" }, ] [[package]] @@ -4251,7 +4488,7 @@ wheels = [ [[package]] name = "sphinx" -version = "9.1.0" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alabaster" }, @@ -4272,9 +4509,9 @@ dependencies = [ { name = "sphinxcontrib-qthelp" }, { name = "sphinxcontrib-serializinghtml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/73/567727fc8c8d85bc3adc88a246e12937cfda8d565cf1d80e85bceaa61fe5/sphinx-9.1.1.tar.gz", hash = "sha256:48b627f02908f22c601b3ba9c7b17265b4f315dd05c8bc64cd4da41183f44fbc", size = 8725638, upload-time = "2026-10-20T00:41:13.96Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, + { url = "https://files.pythonhosted.org/packages/47/18/3c66ea76da7de47e716b181c35bdbfafbd5e9af0bb247b14578e9de348e9/sphinx-9.1.1-py3-none-any.whl", hash = "sha256:f8b60013ce25017be41a509e5c2d265bc1918410061315c36b809c7f335646c0", size = 3923559, upload-time = "2026-10-20T00:41:11.288Z" }, ] [[package]] @@ -4347,73 +4584,73 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "2.1.4" +version = "2.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/44/311bac6b6ef81e4dfd0287d04900108b1f5c00c9761dd3c0a2b7b9d0f86b/sqlalchemy-2.1.4.tar.gz", hash = "sha256:7bd7ad604487daa7eab8716471c29a7185f17b5287ce73bb7bc79fea050d8cfd", size = 10544216, upload-time = "2026-10-07T17:33:59.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/e4/23174288ed2c03d6dbd5dfacd69e28303ee95f49642a8ed0544932999fb6/sqlalchemy-2.1.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:70006e9e6157200b795beeee04bd5cb15bccb40a14de595eb9f5dcf5945ed244", size = 2460507, upload-time = "2026-10-07T18:04:40.044Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ac/254fadc98bfd600445b976e81c6d777b08a728a415c3b77a8c8d35b89a83/sqlalchemy-2.1.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3341ddc430733cd961bc064889f42712a0b4056733a21c83176842aad67d12a6", size = 4594505, upload-time = "2026-10-07T18:16:58.768Z" }, - { url = "https://files.pythonhosted.org/packages/83/6f/ac7beddc57c9c87bd77bc1c158fcbcdc20822f1873bf33ea3480d04e865f/sqlalchemy-2.1.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98f7a4bfeaed3722804f737ae2bd4077b35e57d6f4531fe612bac8160cda5acd", size = 4647364, upload-time = "2026-10-07T18:34:51.721Z" }, - { url = "https://files.pythonhosted.org/packages/0a/82/fc3891f261c4738a8b90cfdd805fe292d1af3b77f680a63b7349304c74e5/sqlalchemy-2.1.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ec5d079935f67febe0ab8a3a203ad591b99508adc34ae0027f696dcb20373537", size = 4311619, upload-time = "2026-10-07T18:38:44.002Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1a/160c1320ab20e764a29721dc3fe7c31af34e291c652dca875d1ca6022b9a/sqlalchemy-2.1.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3d675b0856b6703b29d023517a4c19fecfbb55214ff5c72cd813527e40aed9b4", size = 4518031, upload-time = "2026-10-07T18:17:05.615Z" }, - { url = "https://files.pythonhosted.org/packages/30/2c/15a204333896e5dc63cb089ea20ca3ebc3c892bedf9fa00cc1a65e20d7b5/sqlalchemy-2.1.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a0bb9ee6a38cb36240dc88da11888348f61506047be54de3f09496c3b0ead6f5", size = 4312818, upload-time = "2026-10-07T18:38:46.541Z" }, - { url = "https://files.pythonhosted.org/packages/a6/55/5e78d288f198598f278b4b7baef42f18e039b14b1e1045e9df3cf571300d/sqlalchemy-2.1.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:61a2c48771cf314b6613d327c795902bbc0eb6d6169deb23b35004ba6ad6cc0d", size = 4608584, upload-time = "2026-10-07T18:34:53.69Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f6/e83b93ecc6e6528623fd7aa2af27ff0660d22354b78fe6ccad03f9ecbd9f/sqlalchemy-2.1.4-cp313-cp313-win32.whl", hash = "sha256:3fd608a06bafa768ad5711df4e17eb058bdc490e9df7d39b12a90947471e8712", size = 2373880, upload-time = "2026-10-07T18:22:11.722Z" }, - { url = "https://files.pythonhosted.org/packages/8f/46/afb02975023db6aa4b8608177c2fae17d0b435d9cbfcb5df4fa6e65a8078/sqlalchemy-2.1.4-cp313-cp313-win_amd64.whl", hash = "sha256:b756d74527c56a7e4cfae297f7930c1d75bdf4b23f214c8c13779746d28060cb", size = 2424430, upload-time = "2026-10-07T18:22:23.688Z" }, - { url = "https://files.pythonhosted.org/packages/21/e5/76dc82d59186b98b27589b33b01175c0d49512679276170271d9384418e2/sqlalchemy-2.1.4-cp313-cp313-win_arm64.whl", hash = "sha256:a64d54015233f824f171009977bfbb6b08bd0347b700cf17cb047ffb94c4148f", size = 2385057, upload-time = "2026-10-07T18:11:48.248Z" }, - { url = "https://files.pythonhosted.org/packages/43/b0/6675a01f4e6215e0a809d28a800953294ab31370fe8c4bb3eb9e28c0b5a6/sqlalchemy-2.1.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a2f6164c0527cd8fc4cea79a5c9d8369ffee417b8ba444a42342f36b91deb75", size = 2463645, upload-time = "2026-10-07T18:04:41.615Z" }, - { url = "https://files.pythonhosted.org/packages/7e/24/4630a4009ea08a0769d5ff6517c7fc978f6a63eba32e08c44b98c284d7e4/sqlalchemy-2.1.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6929a11ad26a91a4efd891c1252b373c2e88f056910b83ec6030ed3f2cbcb734", size = 4587911, upload-time = "2026-10-07T18:17:12.512Z" }, - { url = "https://files.pythonhosted.org/packages/0e/02/953686f44448b92cc628245687a242799b6eb11ef30ad2bc7adacd51986d/sqlalchemy-2.1.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14528d37d7d46a92f2a483f188f7fecd86cdd789254a0412b960c9fc5e9efd6d", size = 4617539, upload-time = "2026-10-07T18:34:55.826Z" }, - { url = "https://files.pythonhosted.org/packages/13/23/a44288ab4fa12e51c9d390e7d798d70a45669ddcbddc9dd9b5948eb1aa3f/sqlalchemy-2.1.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d2cb669c6bd1f19caf51db6e3c4fdd4cbb76f9db3ef81c3aeb5e288d9bae101b", size = 4310883, upload-time = "2026-10-07T18:38:50.265Z" }, - { url = "https://files.pythonhosted.org/packages/a3/39/1c441ac015767f619a9e6cc306905bb042f94b84f2a1e930e989e9c6e209/sqlalchemy-2.1.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:63dc25b21fd9a41dc09b7aada4b3b0d97cf4b6414f74bced6ac45326bc799ac9", size = 4516511, upload-time = "2026-10-07T18:17:14.368Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b9/f54ea5ccb27d9a712d90d1617050bee761df25dc1fb5e0b7d2aa867deb51/sqlalchemy-2.1.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:308f96d24e773d64609a2a0d1161a068f9f6e9165523bc4e07aa9c45f0c4213f", size = 4312314, upload-time = "2026-10-07T18:38:53.249Z" }, - { url = "https://files.pythonhosted.org/packages/df/9a/c1e39287ee988e4c2e25c619959b8fb15b297734be040653fe85b57517ee/sqlalchemy-2.1.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93b9416b9011a3b7689a933e04ac9f61d15686b6cb1948ebc1f41467153116c3", size = 4577833, upload-time = "2026-10-07T18:34:57.829Z" }, - { url = "https://files.pythonhosted.org/packages/41/78/5f1ae1911d2b20ccdb39ee522118533a4b5262b6e5e06bbcbb1ebd1f4617/sqlalchemy-2.1.4-cp314-cp314-win32.whl", hash = "sha256:89db94855287fdac98d74595cf13ea59fbffa608d6400ff972b0fd4c036d873f", size = 2380724, upload-time = "2026-10-07T18:22:25.374Z" }, - { url = "https://files.pythonhosted.org/packages/ca/93/4dfa4ce15d082011fb94e06e7c6b4c2957a3f0ddeb8fe9b89d007bc058d7/sqlalchemy-2.1.4-cp314-cp314-win_amd64.whl", hash = "sha256:080f8d853aac5bb5620f0ae6f46527397cf18dce0ec2b478b478469ef3cae2c4", size = 2431689, upload-time = "2026-10-07T18:22:27.144Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c4/6f6c29eaf459c4c2d9b7d24e300bab32043f8f8a936df863f3b886b5564a/sqlalchemy-2.1.4-cp314-cp314-win_arm64.whl", hash = "sha256:64d41be1dd88f184de1931f0173f4827122a1b49fd1150656641200c0bdf640c", size = 2396106, upload-time = "2026-10-07T18:11:49.528Z" }, - { url = "https://files.pythonhosted.org/packages/a5/e9/48f851411665e394f60c669d1f9494d660f5f1fe46e275f9615cfc812a98/sqlalchemy-2.1.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:84272f329c15081a1e09b4a7261118b4e8a547f43e00fca98e55bbdf19eff3be", size = 2503038, upload-time = "2026-10-07T18:19:41.094Z" }, - { url = "https://files.pythonhosted.org/packages/41/ed/bf83068bda4051d7fd719c14cefc15d8466ef1e3656b9f4401b0509b11e0/sqlalchemy-2.1.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b3f58bd26fc010ea28976d401845e4e6ce02e1b7c0288b3ea9c9a3c396f0bcc", size = 4919623, upload-time = "2026-10-07T18:16:45.399Z" }, - { url = "https://files.pythonhosted.org/packages/56/de/57eb70d56b70d22a9360d658b195834ecfdeff7a7bc5c2e3a7fa7a8f7823/sqlalchemy-2.1.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82d728075d42bd457d09655cf22e99d772a648c6f67e86743a4f05b7d063ca18", size = 4830238, upload-time = "2026-10-07T18:37:04.468Z" }, - { url = "https://files.pythonhosted.org/packages/70/3d/c410e9e79a53fff4c04444da609fed6404868d250f11fe8bc53d827bfb0e/sqlalchemy-2.1.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0970394ec5d9e397aafc5bc5fa2b7f8b58cb191f2703006b19a96ef4bf00b8d9", size = 4508380, upload-time = "2026-10-07T18:38:44.277Z" }, - { url = "https://files.pythonhosted.org/packages/1f/c3/01b93821ba35b5b162e79c613279d960a120767694f656da1c1374dd3ed3/sqlalchemy-2.1.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6005f2f5fcd67fdd721446128e6a2a1d18f77387a604fbd26b0006a086b33096", size = 4775311, upload-time = "2026-10-07T18:16:47.724Z" }, - { url = "https://files.pythonhosted.org/packages/c7/88/0b40754e4d851d33548792062c23467a3d8dc07f2eff90cb19e4c404fb4c/sqlalchemy-2.1.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:0e01a3e199ae219381c4889993c5584b1b905fffe6830f639adb6770036a8913", size = 4508558, upload-time = "2026-10-07T18:38:47.857Z" }, - { url = "https://files.pythonhosted.org/packages/d3/2f/3916954eca5596d9e93fccd2ec0e45fd8c65981debac0ec4617639ded6ba/sqlalchemy-2.1.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:22129e7d00ac66b291840c4dc83a9c497456ab5bffa682dcbfdc2356f9e49e5a", size = 4767114, upload-time = "2026-10-07T18:37:06.792Z" }, - { url = "https://files.pythonhosted.org/packages/6b/d6/6a29716aec6ae17cd77e27b5e0dedc68cf9068594f2b601806c1d146427a/sqlalchemy-2.1.4-cp314-cp314t-win32.whl", hash = "sha256:bc33d3e59d4e84b8866cc9ba13732585e37212dbe3542cb09f232682b36f47a5", size = 2439584, upload-time = "2026-10-07T18:22:44.434Z" }, - { url = "https://files.pythonhosted.org/packages/34/79/2f0b33647d2d26f098269096c1864c0b4e81095354cdedb95192647f47cd/sqlalchemy-2.1.4-cp314-cp314t-win_amd64.whl", hash = "sha256:346d144e8912ae087b10d3c2081657cb634728600693eee6dbb71d7eb4768101", size = 2507217, upload-time = "2026-10-07T18:22:46.176Z" }, - { url = "https://files.pythonhosted.org/packages/93/e5/869c1ac0a21e17e4617b6a7828b50320bedb7074b6d67aec59299be5cdba/sqlalchemy-2.1.4-cp314-cp314t-win_arm64.whl", hash = "sha256:3e5de57c71b3460e2ca6137e82cd3cb8c9f711f301f50d5c77156fdb9c822999", size = 2425329, upload-time = "2026-10-07T18:12:20.595Z" }, - { url = "https://files.pythonhosted.org/packages/2b/8e/a082a165b473dae45d2f2f79be15f5c405ac579830c64253efbf04695177/sqlalchemy-2.1.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:418786f05387ddb66ee683a1d016c5a8d9bf7be921e6ee8f285c7b6ac961a731", size = 2463572, upload-time = "2026-10-07T18:11:12.053Z" }, - { url = "https://files.pythonhosted.org/packages/d1/35/74db254005ecb384533973b157ba1fc3fe5bc41a5bc6e0500ab8369c49e6/sqlalchemy-2.1.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:283914efed30e4d44301e36ac90ad048570538b8a70f072fe01578d9b205d09c", size = 4591740, upload-time = "2026-10-07T18:01:00.314Z" }, - { url = "https://files.pythonhosted.org/packages/70/81/5cadd72b0c26b6ee7c1e6950cb9f0cfc383246a842314a1b2a87f455db25/sqlalchemy-2.1.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d2eacdbeb990b80235763860923c60a8393745b66f7149a734980c65896da72", size = 4627040, upload-time = "2026-10-07T18:09:24.836Z" }, - { url = "https://files.pythonhosted.org/packages/8e/78/aed93cc373f61b57625e1f9f84bbf12358e32e935e64fa098f3a446e1203/sqlalchemy-2.1.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e43fca5fdd5f34a3f8c54107a3648d3139de8bbf596a189f3f0de94bd84949bb", size = 4333103, upload-time = "2026-10-07T18:33:48.275Z" }, - { url = "https://files.pythonhosted.org/packages/e0/31/ecc6bbd365671cdc512a59d42afa7c34b2833a8d841754918ae3f62d36dd/sqlalchemy-2.1.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2e1b5343d315b10a4a71da481729f66f830a561595e02b61e8a5a65d658325ac", size = 4519962, upload-time = "2026-10-07T18:01:02.268Z" }, - { url = "https://files.pythonhosted.org/packages/58/58/9f8f6157c2252aefe73f4a0b3859413bb720d14321aa7f367c691949aaf8/sqlalchemy-2.1.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:42c37c06adcecf444e8c981f7e9237a41bdd445c83da0df9e08b4ad958becbbc", size = 4331935, upload-time = "2026-10-07T18:33:50.334Z" }, - { url = "https://files.pythonhosted.org/packages/97/de/a4ae4b95d17607004f01e9a085fb221087c557bbad77a3d87d5d0a5fd8bc/sqlalchemy-2.1.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:bab7f51d38766d6a64da2b41976f1b3f9cc2ff37d3f2f63bdbac876199f3a48e", size = 4589275, upload-time = "2026-10-07T18:09:26.872Z" }, - { url = "https://files.pythonhosted.org/packages/65/27/56f69293a01279ac0e6077b8c358eb0f1c2afc6aa17428414a86c8871042/sqlalchemy-2.1.4-cp315-cp315-win32.whl", hash = "sha256:1541ba5bf0f232cd61f9ef3df78c93977c72ba6031506a0e6d057b2a3ddb76e9", size = 2380494, upload-time = "2026-10-07T18:04:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/2c/7c/ff7e29f95996ed49b950afd531b89e7c8d15addb41735643d07090550090/sqlalchemy-2.1.4-cp315-cp315-win_amd64.whl", hash = "sha256:596a95611c217cb19c21f02f43c637cb507cab71dcf0467c5c7d98fcdd703007", size = 2431760, upload-time = "2026-10-07T18:04:27.275Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/4eaa4978760cd632093ea272e7c4f88223619202f5481f897e67d4377409/sqlalchemy-2.1.4-cp315-cp315-win_arm64.whl", hash = "sha256:0d1ca95e42ce3c18818f170b741d30a33b292c6f6b9a202ffd717e28fc99b8c7", size = 2395807, upload-time = "2026-10-07T18:30:54.962Z" }, - { url = "https://files.pythonhosted.org/packages/be/7b/b806fbfc61ade37c4f3aecec0874c345fb297b56a3743116dcefa3e4700d/sqlalchemy-2.1.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0f672ed6972164fec94a8f0b21dcf8545080d0727866335fb8adf9f4764ce6ec", size = 2499195, upload-time = "2026-10-07T18:19:42.835Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ba/4f9fba8340222f09287e936d7b76e6911a4e507c7d6373ada770e8f697d5/sqlalchemy-2.1.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72e3fa41d1fdab87d4e88bbdd69c9522e2795549fbe7b07bcf4ae9ec175f4b11", size = 4896914, upload-time = "2026-10-07T18:16:53.18Z" }, - { url = "https://files.pythonhosted.org/packages/55/34/c4aeec7bee453badd8b0e02c2021a13bd70ef01038303d05326e99f595b6/sqlalchemy-2.1.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cb2cb98d056e63e353ed697750004e07c79b054d73059ba3184ca3bb07296bea", size = 4846593, upload-time = "2026-10-07T18:37:08.766Z" }, - { url = "https://files.pythonhosted.org/packages/82/54/6dd8504364e5f5efd328e98fea963e5a2e978ff8dcba70d95231314f82a9/sqlalchemy-2.1.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d66fdcc5506e0f8bb8d3f4f95125220a7cd6c46e8b1762750f01e9639973dd8", size = 4494821, upload-time = "2026-10-07T18:38:51.166Z" }, - { url = "https://files.pythonhosted.org/packages/df/42/dc584c098bce29578fd0611cd6f36830e06b4dd2505d3020a0b592f4cf08/sqlalchemy-2.1.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:81f802c96dbf96e59c6982fa1b87da7868920fb0c27b9b81e560a62f57c2ccfb", size = 4752545, upload-time = "2026-10-07T18:16:55.711Z" }, - { url = "https://files.pythonhosted.org/packages/8c/41/69a70c1419bea97e80f65ce09f4f626df464752b276f4f3d69ff6fbf2325/sqlalchemy-2.1.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:acf8982c70471a68aa90d1aba08b48860c55b3357ec84ccb0f09368ead2ce099", size = 4494093, upload-time = "2026-10-07T18:38:54.37Z" }, - { url = "https://files.pythonhosted.org/packages/ef/bd/d296c2223e8417b350db215d94dcd344bc0dfe9deb7d810a21f7d8cd0b14/sqlalchemy-2.1.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:778094c83e36c430756a7e1a1ac66fc3cffb2c6a1067958fe6b920abcec7bc5a", size = 4771973, upload-time = "2026-10-07T18:37:10.93Z" }, - { url = "https://files.pythonhosted.org/packages/13/4c/c3a10d9da10e4e60808ffd1825547b383c0d7ca9e56d15cdae47c04e752e/sqlalchemy-2.1.4-cp315-cp315t-win32.whl", hash = "sha256:963348422b22f760e9462e56bc32bf4d95d224cc5b8c79a3c6e3b786d3d2a2b2", size = 2436383, upload-time = "2026-10-07T18:22:48.162Z" }, - { url = "https://files.pythonhosted.org/packages/51/de/8045d4ad1fd3a66c3b9bb576f3734c86015e19ae2f1617af92eb63cf9e58/sqlalchemy-2.1.4-cp315-cp315t-win_amd64.whl", hash = "sha256:fba3500e170d25f581e053009edeb0b158116084d91d465de218718d336b67c3", size = 2502237, upload-time = "2026-10-07T18:22:50.196Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4b/245e2315d331cc15765a2373e068445fbd28eb63beb23ea862828808c0bf/sqlalchemy-2.1.4-cp315-cp315t-win_arm64.whl", hash = "sha256:0a9a464bc360856b7ea9bf8aa26aab92ca115dd08149cb0e004063d5db13584b", size = 2420917, upload-time = "2026-10-07T18:12:21.876Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/dbf11a262f6fbb41390cab2d8e47a30ec0961018b68201607b599dd489f5/sqlalchemy-2.1.4-py3-none-any.whl", hash = "sha256:0b96edcc2cd60fe1e35f67a46f4eb076e57297841b9eae949ac5f196593f00a7", size = 2054935, upload-time = "2026-10-07T18:01:16.403Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/08/bb/28e3106ee586e5b2f5ecf211eec1348055f0f926314b13fe23b9cffa9039/sqlalchemy-2.1.5.tar.gz", hash = "sha256:0df76cf1ca680ad745bd378a6299a058611a8a784eeaeb4ed78ad4a0b7ce2003", size = 10569593, upload-time = "2026-10-16T20:52:20.57Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/aa/f10fc142d2db255249107ee8b21bfbabc10b90f7fadf13c3cf898acc6179/sqlalchemy-2.1.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dc1c86db97372e6bba18a7bcf0f6150dbdd808237f0f52802ef0bec676a94642", size = 2465792, upload-time = "2026-10-16T22:42:39.038Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0d/d395e310e36b5bd39a121cd455504aa1e9111f419efaa88be6b814b4e54d/sqlalchemy-2.1.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c229afac414d691d2f69a6f4e294ac5b5fd334c49da1f4fb86ab7f2a466271d", size = 4599703, upload-time = "2026-10-16T22:55:25.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6d/04f4ffc5038571e712c4f88f20d7022091bafab082e85ea5b37a7e8270f1/sqlalchemy-2.1.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cf433bcb77696cc915552d7f1e8d9117a5dae5aff9fceb06a69d74b3b3c2bd1", size = 4652645, upload-time = "2026-10-16T22:59:23.273Z" }, + { url = "https://files.pythonhosted.org/packages/11/85/08bc336576755daf087e0888e44657f4f72b6676a99522e7a612f0c2481e/sqlalchemy-2.1.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:399d3e212cb557d5bd1ca6f79981e8c3f45c0f6be60b63a9ded790396a81af08", size = 4316833, upload-time = "2026-10-16T23:15:10.011Z" }, + { url = "https://files.pythonhosted.org/packages/a7/5c/476441255cc7afa25e8c75e5a61d3c5c3af7cf61c951ebed2ee9873ec4b1/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a51c56afdb880e9f83736a2e81c02dd002fe1c5ad68346a35d97a2e135b1568", size = 4523134, upload-time = "2026-10-16T22:55:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/63/ca/064e5e3ca7c8f256c8f9b5206eafd410a39226c2ab6fe7e8c1e70725e432/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:17f8c68bd35bb76b2d9ef1097941fb783ef9ef6604c1d2a72c2e276430aeffca", size = 4318058, upload-time = "2026-10-16T23:15:12.139Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/e38956834896e21038a67193f91b189b9a61555b92fac42368027d5a2524/sqlalchemy-2.1.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:59dbd7412531da0124df8667a244b5155773632e9bd79a5c2ead393b11b59de5", size = 4613916, upload-time = "2026-10-16T22:59:25.19Z" }, + { url = "https://files.pythonhosted.org/packages/14/b3/5531eb9d10d4bd38bd7e6c505776ac73be259dc117c9f07a8c46db4e9a2b/sqlalchemy-2.1.5-cp313-cp313-win32.whl", hash = "sha256:3f566787cb3a6a25ad8c5170e90167d4e8ea103d412aa4ea95021a6df0b95964", size = 2379136, upload-time = "2026-10-16T22:48:37.111Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/c48924aafae616d59a2eb0375feeaf4bc4356327a442bda3b9205861b543/sqlalchemy-2.1.5-cp313-cp313-win_amd64.whl", hash = "sha256:fb8dcd5e1cc12fdd4d490819ec7a56e8c02e86ca9087a472e28b422ff932bae0", size = 2429688, upload-time = "2026-10-16T22:48:39.038Z" }, + { url = "https://files.pythonhosted.org/packages/51/84/0855a7d42b700b091af6f817c171c2ff97478b0d5143fe61963adbff9201/sqlalchemy-2.1.5-cp313-cp313-win_arm64.whl", hash = "sha256:65eece9d9494e46de1eb3d49eb438acfcecf059e7a92a2f3daf72ec26d371757", size = 2390320, upload-time = "2026-10-16T22:31:43.871Z" }, + { url = "https://files.pythonhosted.org/packages/01/b0/59b26375cd618e861a9295d855b866ca85be654d20ed47e494e33c3bf26e/sqlalchemy-2.1.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5541a110ac4bb6377c785aea1b8f0b4d6658ed0ca5061ed1115d0725062b026b", size = 2468925, upload-time = "2026-10-16T22:42:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a1/01931ba38770f169ce70e4034b0dcf2c1a776fe7d8dde93a34b6355510b1/sqlalchemy-2.1.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f029b2f7114623c03f762596f313515a2358004c44eb18924a15940f2e27d30f", size = 4593220, upload-time = "2026-10-16T22:55:29.253Z" }, + { url = "https://files.pythonhosted.org/packages/42/9e/fa478482503a69dfb3b0cb82f3f80fe3816026530dd34b5359e2acd9f30d/sqlalchemy-2.1.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc44dcc80e89a006af415edffc7409b1f2976a4597065449c166987f95801ab4", size = 4622806, upload-time = "2026-10-16T22:59:27.153Z" }, + { url = "https://files.pythonhosted.org/packages/fc/01/357ff85423013de8316316752c36a6bd1b7f3830c30dc63754afe06ba589/sqlalchemy-2.1.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:72ddd7a8ec187d02de0ef316a15aaf381ddcc594958247b02a3dcdb4c42f0b02", size = 4316247, upload-time = "2026-10-16T23:15:14.27Z" }, + { url = "https://files.pythonhosted.org/packages/43/53/88c5af325c81c929d0ed5b7ec15b087173bd3587d93366c5e336b9647267/sqlalchemy-2.1.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f5173a6e60a15090f9e2348a8ca94cc13de72796c16733f3b62fbb22acba07b8", size = 4522522, upload-time = "2026-10-16T22:55:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/87/29/f6f01f121d5e590c568f405f78b8eb3edace446c114126fc428536446dd6/sqlalchemy-2.1.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:09b10bd68223d57cce39c08947efae3f91ac40b552e07c7c869c3350a0fc4c23", size = 4317593, upload-time = "2026-10-16T23:15:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/0c/6f/14863fc9aea433645851330242823b1879cdaac71c118f7c9366e7caf4da/sqlalchemy-2.1.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ea4cb5022462d9f15e7b72a52c267385aa97ac224c8503ac63ba90c0067ad0d9", size = 4583207, upload-time = "2026-10-16T22:59:29.322Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e9/6907fc009d9d06e8d03d6bf286df46e878ea2a85c839dcb65ef11ceb3a39/sqlalchemy-2.1.5-cp314-cp314-win32.whl", hash = "sha256:c3ffbe6c680c626403b7e9697d92465501078731b9e8863b1410826e66b950e1", size = 2386018, upload-time = "2026-10-16T22:48:40.692Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/dea47b4b361c427d390edc10d73050e6d11d905c56ef26861ff5f94aa8fa/sqlalchemy-2.1.5-cp314-cp314-win_amd64.whl", hash = "sha256:061d089901319463a1160e4596191e355d2a9b4cce4852dac5adc301b085ac2f", size = 2436983, upload-time = "2026-10-16T22:48:42.2Z" }, + { url = "https://files.pythonhosted.org/packages/be/28/305a91768a1ca0980c6ead455dcda5a2c1e616022901898e966ccf913ba9/sqlalchemy-2.1.5-cp314-cp314-win_arm64.whl", hash = "sha256:f5bc7fb93f1c17efefeb062bc4d2742ad1b71d56c36e6e2898a371ac6a56839a", size = 2401394, upload-time = "2026-10-16T22:31:45.275Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6d/d4bab269d94ed1017950873643b5bd3eb271a5b8d5b0451250f0173f90fc/sqlalchemy-2.1.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9bb993b1d0890a2da90589684c3d09de75df82c06aa8f7eda246c7cd0ea3e820", size = 2508315, upload-time = "2026-10-16T22:55:01.45Z" }, + { url = "https://files.pythonhosted.org/packages/d9/80/aeab72b51fceb71b1e1d62cabc6f51c9d8f20588165aa54e0d0d6cc77c05/sqlalchemy-2.1.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df7e71ed051e4003e73484ef6547aacfb3a2322aac0c230d530c97e48e77b4d7", size = 4924848, upload-time = "2026-10-16T22:41:10.671Z" }, + { url = "https://files.pythonhosted.org/packages/05/8b/7318ca0ae828fa1605dd4481f8eeecb5325a7ead6289c1206fb1ff1e2e1d/sqlalchemy-2.1.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c76d4cb9c76a6906563133449a76d9eae6c8529ca09484b8615ca604442b442e", size = 4835557, upload-time = "2026-10-16T23:06:18.482Z" }, + { url = "https://files.pythonhosted.org/packages/35/17/7a5a196b0af72d65cbcd3a6fb7c4128693d039faf58a682ffe2d19329420/sqlalchemy-2.1.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:59a56be2e4e972061f9d5c1d5b01763b8b2207b48b956fbb6879c7443f5817f7", size = 4513707, upload-time = "2026-10-16T23:28:18.969Z" }, + { url = "https://files.pythonhosted.org/packages/91/50/ca323ed15fc06c73a7fd85a3a339749994c560c12312ee8a53abf0f9a7ec/sqlalchemy-2.1.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc1a61d7f42e71746b5a2e10e90a6b174b4add441b7b6cf14278f1e029b2b800", size = 4781127, upload-time = "2026-10-16T22:41:12.784Z" }, + { url = "https://files.pythonhosted.org/packages/26/3d/c56a75d0a2f6af08f6bb28d0b69e129911a2fc52b24dbafd7a7e5ac5c1aa/sqlalchemy-2.1.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:740e96dfb99a615c8056070d3747efe8bce4c145e5953ccc9e4b004c139de0d2", size = 4513816, upload-time = "2026-10-16T23:28:21.378Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dd/84e0cd627bbd3e35b5f033191a4f597e611ff96e424db36a444a8d6755cc/sqlalchemy-2.1.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:92a0bb0595331c8e40bc615a67cc7bca9b761487409d1bf46c598b6ba5c0754d", size = 4772462, upload-time = "2026-10-16T23:06:20.876Z" }, + { url = "https://files.pythonhosted.org/packages/b7/47/cf0a920fc40344f8a1806d661e7c702341aebd871e31e1d1d6e0fbc9d1e0/sqlalchemy-2.1.5-cp314-cp314t-win32.whl", hash = "sha256:11185c72b5372369166cb721537943f8e1e6e42a399e9bac19e02f3ee20592a4", size = 2444861, upload-time = "2026-10-16T22:51:51.169Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/4c5dc7e30fe740a01df4ba5ab800be2d7674b87f042e4c21c7ca89b36f9d/sqlalchemy-2.1.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5173207b8fe1ddfc1d451c420a2a9cbbd0734b4a19896b8080e3744497e1c20c", size = 2512497, upload-time = "2026-10-16T22:51:53.014Z" }, + { url = "https://files.pythonhosted.org/packages/5f/2f/d3e9bfb1faabea302b98ba8c9dd6252bc362c6afeabe28ffcc1eea8a6bd2/sqlalchemy-2.1.5-cp314-cp314t-win_arm64.whl", hash = "sha256:def2f117a6ec0a25202ee36a148bbcfad3d43e2f2bb2e698e6d4448c6db5cb19", size = 2430598, upload-time = "2026-10-16T22:32:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/ad/7c/17f07378eea7ec2fac8ac60f99fe3c474ef5c19a3adede70ae9886eb82d7/sqlalchemy-2.1.5-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a0b80f91d9ed92be7a134278ea2e83a693c07d9986824ca51d859af9e57e18f5", size = 2468856, upload-time = "2026-10-16T22:27:43.754Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c0/4fef29827b198937e09c40d0fa36b9738bd3914f0e5040927dfac8dfee21/sqlalchemy-2.1.5-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3033ddeeeb34c6c536622c07e932a20f8c9f58d6dd76e56595a76b55a25011ff", size = 4597018, upload-time = "2026-10-16T22:41:59.749Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d8/37645f8445ac3774e18f8dc72e0fa0ac868e0dff0e987ff3bec644f55102/sqlalchemy-2.1.5-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4d30723171d511ec16ac057872a38b9b395420c0aaf81b2f94dd7bf1345c5fe", size = 4632322, upload-time = "2026-10-16T22:44:53.83Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c9/15466cd08a6c329acc4f730dff7041e84122456fe38c518c93a07d6d82ca/sqlalchemy-2.1.5-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cefa5badc13df1c9082bacdfa435c8eb6dcb9b32717f51d09d1e160a599646f", size = 4338381, upload-time = "2026-10-16T23:01:22.714Z" }, + { url = "https://files.pythonhosted.org/packages/12/2b/a93b7c8f6886138dac91f66fccd3b42473c365408d9a4c1b69403cf206f8/sqlalchemy-2.1.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2549e503185897de1fc5ca7730d5daed0d27d0b26df4a059dcdd33ebe81b7b4b", size = 4525244, upload-time = "2026-10-16T22:42:01.631Z" }, + { url = "https://files.pythonhosted.org/packages/b3/47/d3926cacd194be073a2053a05470f20ed7da623f399e95213c1e497954fd/sqlalchemy-2.1.5-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:5a6871ee159a943d874012de835d56d0544ee871a99dbae05bc053da315b2c23", size = 4337223, upload-time = "2026-10-16T23:01:24.702Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/b15ab4712342a18fb3949c77687c24d3749e86079c9905cc2cbebbd44da3/sqlalchemy-2.1.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:51e290080415325b66a94d20f6360e310606acb832f2bd677d4fcb95728e7ce1", size = 4594562, upload-time = "2026-10-16T22:44:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ab/3b6ef6c77740d7fbf5a3eb45f95768fb809b0769bd21df81d5f8c002e5a5/sqlalchemy-2.1.5-cp315-cp315-win32.whl", hash = "sha256:c228084025a28aa038aef9b821d842508650fe12fe51f06e9cdaa4393be7812c", size = 2385770, upload-time = "2026-10-16T22:41:04.484Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c1/0b3256bc11487b2846a26d58946caea0ed395fd500951a16fb82d31a20ff/sqlalchemy-2.1.5-cp315-cp315-win_amd64.whl", hash = "sha256:bc09dd22c3fc491a2df7d16ca4167de35573f40c2e82dd4d6ddae589a73e7032", size = 2437043, upload-time = "2026-10-16T22:41:06.12Z" }, + { url = "https://files.pythonhosted.org/packages/e5/a0/09eaa7897f6f745de16220b20c63660948a95716be90d2f405af08222e57/sqlalchemy-2.1.5-cp315-cp315-win_arm64.whl", hash = "sha256:375ef1bf78bc269a220963163d2580d5377898ed23a00bdd98a02d755421573d", size = 2401090, upload-time = "2026-10-16T22:28:58.035Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c2/4c747dcc6fe329dd07ab57233f334eca19a3ff05934e0aba4f25a0977a67/sqlalchemy-2.1.5-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:7af1c92255cf8d50e87a808ce41096907332acdd09f285426a52f7c300e152db", size = 2504475, upload-time = "2026-10-16T22:55:02.946Z" }, + { url = "https://files.pythonhosted.org/packages/07/f1/183446b1bc29484087ba7a6bb199a265c0559bf8ff201c1ee54bf1fe0936/sqlalchemy-2.1.5-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:85891e0828e76a77fe912995dbc3ee949099488ba0a2ab323535d2e767446b20", size = 4902197, upload-time = "2026-10-16T22:41:14.703Z" }, + { url = "https://files.pythonhosted.org/packages/31/0b/b24baf343461eefdecc672df1ccbb14c8c64028a11625dabc97b0bb9791b/sqlalchemy-2.1.5-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a72e062d2d38826dd89231df692fb40f12d7d1b9d118be23bcc1cf93c6e31782", size = 4851871, upload-time = "2026-10-16T23:06:22.861Z" }, + { url = "https://files.pythonhosted.org/packages/da/27/f688b34ea3d85b38e01b7463874f1f4b91619dfed4e6cac88c27caface73/sqlalchemy-2.1.5-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b79ee943e2e12246f873345762b83dbadd1fcfc3e832eab83272d672704dbdea", size = 4500095, upload-time = "2026-10-16T23:28:23.431Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/d8e13be8d0c25ba49244a9c24579ef61d5a6b57f98e55fbb173b7923ec3a/sqlalchemy-2.1.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:e9418d87b55c469c2f1328246d4533238cdc43134c695d6b9a6b7a87da49d463", size = 4757826, upload-time = "2026-10-16T22:41:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/a0bd2ead949cc605ce3ad5d38e2fe84963b2aa7f89e8648c1bbad13a3679/sqlalchemy-2.1.5-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:898d86c22640fe52ffe02b4abc043a3c6754b4875837599e739be440966dffa1", size = 4499377, upload-time = "2026-10-16T23:28:25.203Z" }, + { url = "https://files.pythonhosted.org/packages/b9/dd/c1ca0713a1a7d8d6eb5864edd2d11d7a6a327aaba664645733cb53662425/sqlalchemy-2.1.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b3ad20cd035392f2b91a6466e24643990c1378beda80c9bda87a3a41e841e838", size = 4777264, upload-time = "2026-10-16T23:06:25.203Z" }, + { url = "https://files.pythonhosted.org/packages/5a/dc/9dec72cea689f16bc8ecfcd62b4aa2796009a7cef6be85a9f70310ecf71b/sqlalchemy-2.1.5-cp315-cp315t-win32.whl", hash = "sha256:db34eefaf992c84460ab26ecf2fc2bdcc68965d6becfae8182cae0efeaa3e8f8", size = 2441666, upload-time = "2026-10-16T22:51:54.58Z" }, + { url = "https://files.pythonhosted.org/packages/dc/28/f44c4524ce8a68935bd6f23ee801165072fb8c1e477fc9855efd9eaa2360/sqlalchemy-2.1.5-cp315-cp315t-win_amd64.whl", hash = "sha256:173054351d863ddada188284e309e87a9f030132c8d534e05458971ff061aa1e", size = 2507526, upload-time = "2026-10-16T22:51:56.262Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fc/e90c91f6c2709a431f9b75e41bd31216337859411359f4a4fc2615c989e3/sqlalchemy-2.1.5-cp315-cp315t-win_arm64.whl", hash = "sha256:22c8d0873168a54f614e381559ea83d547b2cc5488b95b049bdc92667586bd5f", size = 2426204, upload-time = "2026-10-16T22:32:56.011Z" }, + { url = "https://files.pythonhosted.org/packages/de/e9/61176e4753f74ab5f6e72a56314ee654d25c31e6fc98c99060442998cccb/sqlalchemy-2.1.5-py3-none-any.whl", hash = "sha256:2f77b77b716f005b2dc3cc059d56df6a919bfb5b439e4149890a9df6c36be9e3", size = 2060214, upload-time = "2026-10-16T22:43:30.801Z" }, ] [[package]] name = "sqlglot" -version = "30.22.0" +version = "30.24.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e0/db58fbf2527426758dc1e862ce538736978e100e4e78fc9657e9661826ee/sqlglot-30.22.0.tar.gz", hash = "sha256:ec4b83ca8236ea8867f574a382dc15ce35b071c977fecfcc66482d9a3f500661", size = 6088770, upload-time = "2026-10-09T16:09:01.04Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/7c/9fd907983ff041692399bf855d344e1fcc9a02c8d92a5fee6dbdf139d19e/sqlglot-30.24.0.tar.gz", hash = "sha256:6451aabd1c9994d0f73355c962a5eaf9258381b3f2554e53db916cd9a605552a", size = 6131857, upload-time = "2026-10-20T17:14:03.987Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/4c/b8474b02b572d9c7a2903e364335d566d52b6128b834b92a7cdfe5597823/sqlglot-30.22.0-py3-none-any.whl", hash = "sha256:90aa461490fcd95d14ec3842a97506ae20f6d3e9313307ad31be793d479cca65", size = 777816, upload-time = "2026-10-09T16:08:59.07Z" }, + { url = "https://files.pythonhosted.org/packages/68/5f/d78f74306cc026c5f40f8808a47356dfd37461eb15eb759a20e5e9efe7d7/sqlglot-30.24.0-py3-none-any.whl", hash = "sha256:9488828cc018453b7a3ba01e65d2749c9a731314c14583695163ca1f358203fe", size = 792162, upload-time = "2026-10-20T17:14:02.053Z" }, ] [[package]] @@ -4485,13 +4722,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "textual" +version = "8.2.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify"] }, + { name = "mdit-py-plugins" }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/21/39a76b01bd5eea82a04baaca7580e105d8c59450df03998345bb2cfb307b/textual-8.2.8.tar.gz", hash = "sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b", size = 1860502, upload-time = "2026-06-30T06:51:24.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/be/35261223d9416a0751cdff1c7b4a6f881387218a12d439fe22fefebc8c04/textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a", size = 731418, upload-time = "2026-06-30T06:51:26.364Z" }, +] + [[package]] name = "tomlkit" -version = "0.15.1" +version = "0.15.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, ] [[package]] @@ -4570,15 +4824,15 @@ wheels = [ [[package]] name = "types-boto3" -version = "1.43.113" +version = "1.43.115" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore-stubs" }, { name = "types-s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/4c/1708fc45a80ad1314f60f8c7fdedc8c17cde501890c373e41da13aa7af86/types_boto3-1.43.113.tar.gz", hash = "sha256:56d93beada6caef7b8162f0481c56e3c34b0be5f488298c5399b89cfdb1a812f", size = 106097, upload-time = "2026-10-13T21:52:30.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/49/0050624e4285f2de32d1091b81dc6900be3c9295226e2b8ef42580dc106c/types_boto3-1.43.115.tar.gz", hash = "sha256:0787cf5bcd91e8481c8640aefef02d7a43a36255547f568aa6a5ca610a52d72f", size = 106082, upload-time = "2026-10-19T20:43:45.489Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/fd/31c34565b3734f5563649e1f38ef43c122cd8dee3ec55a9d01442c43c42f/types_boto3-1.43.113-py3-none-any.whl", hash = "sha256:c3f5a5ede13a909c4b5e2c78a786f11beeb11e2f98df862fd136ff20fc13a980", size = 72085, upload-time = "2026-10-13T21:52:26.388Z" }, + { url = "https://files.pythonhosted.org/packages/43/65/a79f624585182bcfac2672611a50bd1f22ba2e0201eaabfd31b06b35e914/types_boto3-1.43.115-py3-none-any.whl", hash = "sha256:f97254fafc0fe66005ba885793c202972e813b3b3090e0ef40aaa5a5c8432fd3", size = 72086, upload-time = "2026-10-19T20:43:40.558Z" }, ] [package.optional-dependencies] @@ -4597,11 +4851,11 @@ wheels = [ [[package]] name = "types-s3transfer" -version = "0.19.2" +version = "0.19.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/61/29/4df7e8eccfdaac9308e06ae94271eb0e5d2324cbe6df241363c44b70e08c/types_s3transfer-0.19.2.tar.gz", hash = "sha256:2a78a806c09b11fc6d59756402ade26b49b93a85ac6f209a39e2498211c6a41b", size = 13766, upload-time = "2026-10-13T01:42:25.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/b2/4f2d72b5d234c81caa07dee688cacee2c8dc5bd793e9ae70a573f00d6425/types_s3transfer-0.19.3.tar.gz", hash = "sha256:99422ef028a5f854cd92d1e72cf4208bd0569f68a87936ca02fb9f91a68cdcd3", size = 13780, upload-time = "2026-10-20T02:24:19.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/02/3fa02c57a65f8721030247866d28c1a6601d7826297e83059c50b7d6ce7e/types_s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:f1167b9a082a49fc55f12700dcbd3a8b540b9e6fec4e1d782ff44f3eef0bdbff", size = 19473, upload-time = "2026-10-13T01:42:23.813Z" }, + { url = "https://files.pythonhosted.org/packages/f5/98/972fa03c9a6e1cc9184a28ea0354af50a6764bb172fe2d0751798a295f07/types_s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:6843143f3b79e3cb73e58b6f45738e28e88d9f681fd32cc5ef7fa16f0c193356", size = 19476, upload-time = "2026-10-20T02:24:17.682Z" }, ] [[package]] @@ -4706,38 +4960,38 @@ wheels = [ [[package]] name = "wcwidth" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/b4/7830542634bb2d3e62aa3b586a72d5b3b6c91c3168929e7000ef3fed041d/wcwidth-0.9.2.tar.gz", hash = "sha256:ae0ef90b90f6af38b54f1fe6d58662ec33b3cb4b8391958a62416d654231727b", size = 955039, upload-time = "2026-10-05T00:24:05.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/1e/4532a81fb9dfbf4114a816775e0a36c3a64ee1d1f4bba2094e2da50be5dc/wcwidth-0.9.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:7ef5a940bd5e30bac6e721f1a48fce0cd7bb3ece19e9c5d139e72c76c35cfd07", size = 605343, upload-time = "2026-10-05T00:23:22.649Z" }, - { url = "https://files.pythonhosted.org/packages/a0/07/cb6940e81134b7ed25fa312ee9ab536a63db0793b149f88a90e603ceace9/wcwidth-0.9.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ae0800c5339423cc53d33a266ad264b42ba8aaa16d4464f6e6b1bee607f50b17", size = 609960, upload-time = "2026-10-05T00:23:27.049Z" }, - { url = "https://files.pythonhosted.org/packages/a4/80/15ad05d40bfa99155639fb9e13b3d77083aa0fab893c816db2543d29005c/wcwidth-0.9.2-cp310-abi3-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:9e542f1f8475b78452a295495d7a5bc3ead565112e9446a64dc93462a41c2a79", size = 762566, upload-time = "2026-10-05T00:23:38.322Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f0/b8ef7758003d66b60f093695831a86dcc726aac01ee6446ffcbda27b61e3/wcwidth-0.9.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:674b518af28d38ee645ff97b74f5760abee5fad4bac74413bfc4b881ef2ce724", size = 771632, upload-time = "2026-10-05T00:23:32.448Z" }, - { url = "https://files.pythonhosted.org/packages/db/6c/f940133c71427c208575910e981942bd78c98b1f7cd0d1425ca4b7457c04/wcwidth-0.9.2-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:751bef0ab404b6a1dc028b56b4b85d46486be1c55833f80da533e42dc691f389", size = 769763, upload-time = "2026-10-05T00:23:40.175Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/285f862826f721964ec7c42f81dc53d23afbd723a0f4cd989651f8218e25/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c3d80f39ba4653a595edae9aa46a509d14883790a8fc23c5db221ceb207f64b7", size = 782134, upload-time = "2026-10-05T00:23:33.926Z" }, - { url = "https://files.pythonhosted.org/packages/c2/2d/64aa54882a5d556d3654c1f926d9118b797461033e23a158409941a37c8f/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:0a47e03d8293590ecce66c45dc20ff7b4b885e3c78093722239585eca0d77ab2", size = 782851, upload-time = "2026-10-05T00:23:41.974Z" }, - { url = "https://files.pythonhosted.org/packages/59/39/52389f6de7fe2e9c14ceb8253dd99034bd86e1c87847ea3c100a97dded9a/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67d901a4ad99249eb775b4ee4769ca97fa405d35a75f46e83166910a47003f04", size = 784621, upload-time = "2026-10-05T00:23:43.449Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/20225500a076ace27bbcc8a6fd7c55125133c57a618816c7b7b8b73070b1/wcwidth-0.9.2-cp310-abi3-win32.whl", hash = "sha256:ee1fd0db9d9fd711a70f3e7765e0e04c05d26982fa05361456163062549d7da4", size = 593123, upload-time = "2026-10-05T00:23:55.953Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d6/b0690f55ea0483530a18bac917fbadbf54f35122510446fc370f5f1c2453/wcwidth-0.9.2-cp310-abi3-win_amd64.whl", hash = "sha256:2a9746de704242bd4fdaabb31dd46b82f694a56a8d21081ad89b679a89da9fec", size = 597827, upload-time = "2026-10-05T00:23:57.489Z" }, - { url = "https://files.pythonhosted.org/packages/e5/11/6ecf4e9e268ab1a4ec617ffcccc2ee4a71301625f5490912dbaba462fa9c/wcwidth-0.9.2-cp310-abi3-win_arm64.whl", hash = "sha256:b9c6ab615e03723b7f8760ea2f27758d656e7e13b51515c9dca5c3e8b04612fa", size = 598415, upload-time = "2026-10-05T00:23:51.517Z" }, - { url = "https://files.pythonhosted.org/packages/4e/41/549eef1ab767032bdbdc1f0ab655d404b082b1e9a1dab1361dbba90f64ed/wcwidth-0.9.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eda88ffdc97c0fbf193d407114f2c7a54b379f67f6e52a7531ee3b9fe749eca7", size = 606555, upload-time = "2026-10-05T00:23:24.188Z" }, - { url = "https://files.pythonhosted.org/packages/9b/64/a875ed7ea71cacadc0ae11b5fd3fac3486efd58bb25e67a7344248dceadd/wcwidth-0.9.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bf361c8705576760623b4724ae564666d73b016f9a778bcfd1c7345378ef4ec", size = 610829, upload-time = "2026-10-05T00:23:28.563Z" }, - { url = "https://files.pythonhosted.org/packages/c6/98/513095e484fe79b6f2613d6a72f855f5d56b65e15c215c2a6746fbc638f5/wcwidth-0.9.2-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:97b878d1e158da5ed9ac5aac53fa3a55e282103af6a09ec353865613d1a31a76", size = 775819, upload-time = "2026-10-05T00:23:45.116Z" }, - { url = "https://files.pythonhosted.org/packages/22/fc/c02f3eec57224731e78f84b68e272250f784b6205acc7e0dcef6a7c23a0e/wcwidth-0.9.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:59dab4049cbd982b478bca098528df2c79a9160636a3a163ffebffcbd7d1b892", size = 786915, upload-time = "2026-10-05T00:23:35.323Z" }, - { url = "https://files.pythonhosted.org/packages/3d/6f/b0529a79bac3fe8d94f32b4237a13dbc3f955508753f6a6f06c73d679dc2/wcwidth-0.9.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb08ceb501d6aaf94066c3ee122dd825b152df40ff0bd0df4dc27126233b948e", size = 784186, upload-time = "2026-10-05T00:23:46.366Z" }, - { url = "https://files.pythonhosted.org/packages/d5/bd/6357c84ca9a734bfc735b7c48dbe21336b3777fab8a4101d14976dfe49a7/wcwidth-0.9.2-cp314-cp314t-win32.whl", hash = "sha256:8b4e381590b9b7390e07e22b2c0c1bb96ce50e1d2243c866d9387600362d51ed", size = 600791, upload-time = "2026-10-05T00:23:59.398Z" }, - { url = "https://files.pythonhosted.org/packages/98/de/037591ca18d897cc2179559dde72e6efc6ce0c90e9cd1e6bca4e87c38b4b/wcwidth-0.9.2-cp314-cp314t-win_amd64.whl", hash = "sha256:f2f7b3bba5a5d5f31fc350fd36ce5b84b693c83b7eb95ee630b720da5a5ce06f", size = 605463, upload-time = "2026-10-05T00:24:01.049Z" }, - { url = "https://files.pythonhosted.org/packages/d0/07/c9d96e106d938d26f7ab639bc80b8199359a1645ba6e3498413313ab6f38/wcwidth-0.9.2-cp314-cp314t-win_arm64.whl", hash = "sha256:734aa9405b321d1042301aa19c943c4731ee9e3460e4f8feea3299c064c97a14", size = 605936, upload-time = "2026-10-05T00:23:52.765Z" }, - { url = "https://files.pythonhosted.org/packages/82/8a/a28d61d910005ac93dfe48be3a0ebaa49352d88cebd25323e69e6ff2f4a8/wcwidth-0.9.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:42dbcb76ce8af39e2c9db410ac3f9bdf4e47eb41d6f44525952f172d3d98f724", size = 606575, upload-time = "2026-10-05T00:23:25.663Z" }, - { url = "https://files.pythonhosted.org/packages/01/c2/a3c66bd32766c8f4d6dc47d572532ba014fe5be30489f2576aff7cada363/wcwidth-0.9.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:138e1f8898e431b2f2d7881f8ca8d75591c1d3c21aa53f54e989bd6b39811da2", size = 611045, upload-time = "2026-10-05T00:23:30.421Z" }, - { url = "https://files.pythonhosted.org/packages/ec/8a/d39964f8f8c019d7d439b9b501d3e7bb42fee69f00354040ba0b27b5824c/wcwidth-0.9.2-cp315-cp315t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5175609bf8cc7398a5f48aa35207bd64ebf9f45e4c70df65f7fdc7a988041a3c", size = 778143, upload-time = "2026-10-05T00:23:47.7Z" }, - { url = "https://files.pythonhosted.org/packages/2f/53/525da13e8f9ff7b5b4e74ec6f8d68bdee63905796972e086c6b1b96670d2/wcwidth-0.9.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5f669ae8c3d969c72032f9cdee019674b666e522d45e1e2099a2e9dda4a341d", size = 786630, upload-time = "2026-10-05T00:23:36.967Z" }, - { url = "https://files.pythonhosted.org/packages/ef/9f/d6a0c6df354b9d93466548a65cbf4ffcb48c719bbd307504cf3e76740837/wcwidth-0.9.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:196b47cf32f9df27ccda6dc513237f3c2429c4c659db428d60a5bc443d10f270", size = 784839, upload-time = "2026-10-05T00:23:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/bf/d7/3021feed1ed7926021ec134943ad3b24a2f7ea742cc9976461171482ed77/wcwidth-0.9.2-cp315-cp315t-win32.whl", hash = "sha256:0cd4f7f2e53905dcb110d213a4c8529b6733fa3d232d8c717f946cc69a10349b", size = 600845, upload-time = "2026-10-05T00:24:02.497Z" }, - { url = "https://files.pythonhosted.org/packages/63/80/6a03356d8ee38261e3a78cf89ee03d8e7f12c572d969237be00869e2dc73/wcwidth-0.9.2-cp315-cp315t-win_amd64.whl", hash = "sha256:33df042f96c61ed3cd5fb3742fba427553a635bc578799857a48aa79f774a0b9", size = 605441, upload-time = "2026-10-05T00:24:04.052Z" }, - { url = "https://files.pythonhosted.org/packages/0c/48/1a308a86a833fd12ff7a08d0d2491ff4a72c8a92d12f5ead8317630f771e/wcwidth-0.9.2-cp315-cp315t-win_arm64.whl", hash = "sha256:48719a9bc76c2f84238693fe5013571fa5beffa3621cf228f1f3a9e30dae84b8", size = 605885, upload-time = "2026-10-05T00:23:54.274Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b4/0bfa065af506540d9d558e3e5548cff00bc1f9b24e6e2a8512498e8628de/wcwidth-0.9.2-py3-none-any.whl", hash = "sha256:89ca642c5bf0101157a09366be69fad0379db1f700ae39a920e103234573670e", size = 301667, upload-time = "2026-10-05T00:23:21.097Z" }, +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/4a/82d61476a1ccc20d43d0805724c5de4baa4971131a8db1816705ef53fe16/wcwidth-0.9.3.tar.gz", hash = "sha256:dfe7cfbd4cb59db9fe0931bd6091ab4a95b3acc5d56bfb50fa9fef0ccba5dc5c", size = 956320, upload-time = "2026-10-20T16:02:43.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/6d/eec61b5edeb88e3ea2a714184eade38335722c4246d94be567977fc20caa/wcwidth-0.9.3-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3b119ef60177a92b4f298b80e98464c2922a1fba3ed77fe9a2bcb69edf1b9af4", size = 605400, upload-time = "2026-10-20T16:02:02.523Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3c/714eba03905a23a46277cc96d3db8da6bd22d2cc0d09a4dead60a2bd1757/wcwidth-0.9.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:aab90ab489850f87633922c3064f6d0442d8f6ae5407a62f82bdb1f424179c05", size = 610124, upload-time = "2026-10-20T16:02:07.256Z" }, + { url = "https://files.pythonhosted.org/packages/90/ad/8566e7cc712bea55eab0bb6d19581b7c727a58fcdd55f02365b309aaaccd/wcwidth-0.9.3-cp310-abi3-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:9e1cbe18c18d152e1e68e4e2561dab6c2bc988f8fd337a1fb2e1e0809c3fa568", size = 762700, upload-time = "2026-10-20T16:02:17.508Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7a/4e26574d032a1709bb1fd7d95d406b7b8f87363c60305a101e6e2d9285f5/wcwidth-0.9.3-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c12abd75ffcbad417e7770a6feb82413c102ccce5c435068c39a82e2218b00b3", size = 771850, upload-time = "2026-10-20T16:02:11.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d6/48dad66d577417ef8594000506fa8619d45d7282e20b4cdc916fcc68f95a/wcwidth-0.9.3-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ab8bfee107e08de09e15f9f6afb9fe0f64c8d64fd296d0d8557bafe71ea3f93", size = 770047, upload-time = "2026-10-20T16:02:19.092Z" }, + { url = "https://files.pythonhosted.org/packages/3f/61/5f1e02e588d289837d5312478cc9f0e7bca93f4c8a77807741f79fb6a31c/wcwidth-0.9.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f875a4c9e62df2ecfa70d99b24c551f2245609858e179c3a40e74631b2d08a4", size = 782210, upload-time = "2026-10-20T16:02:12.753Z" }, + { url = "https://files.pythonhosted.org/packages/e9/11/0da2720cba3f917148448c9bf73f3079d71f918308ee9e4d737a0b7acfd8/wcwidth-0.9.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:70a2ca239b85a219324968993c76c117fe423d4e1a7c9d860c999c66192f2570", size = 783172, upload-time = "2026-10-20T16:02:20.663Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/b79407036ca4543bf708a6d4ca561f78620dc1e5443e685e094480c37753/wcwidth-0.9.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:537ea00b6792bb243cd09582322c54dbc848879e5d34ea4c5fee0147529685ab", size = 784806, upload-time = "2026-10-20T16:02:22.164Z" }, + { url = "https://files.pythonhosted.org/packages/3c/86/ada6c2244b08a86c871d42f7cd85b5226515b5f3fa7e604d1e6e45590275/wcwidth-0.9.3-cp310-abi3-win32.whl", hash = "sha256:f2f943c4ef14e9baba730740265f4e5bf5fc8473550e36cd92828cd5663561a6", size = 593154, upload-time = "2026-10-20T16:02:34.301Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6e/46363d972b2b9f002fbfc3bf8e9ab0ff89f139c935a5f21d22016e7a2018/wcwidth-0.9.3-cp310-abi3-win_amd64.whl", hash = "sha256:569a0cdb826d3097b6dbacfaa45aa0002d72a2ba6fa8a63251ea43ac4dd7b6a4", size = 597685, upload-time = "2026-10-20T16:02:35.748Z" }, + { url = "https://files.pythonhosted.org/packages/69/3d/5583cac85e233514c3906d5476b6a74160bc4dc82729dabef3485dbf3d52/wcwidth-0.9.3-cp310-abi3-win_arm64.whl", hash = "sha256:80557db1f77148b78e1b3c2a9cc4ebc4feabef42aaf69a555565057031c32367", size = 598774, upload-time = "2026-10-20T16:02:30.016Z" }, + { url = "https://files.pythonhosted.org/packages/91/b2/53e0838babc6b87505b9504ce62b72c79403642f88d3e568e05d0ef04825/wcwidth-0.9.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6f3e855d3a09af8b7b6d56e76bc270681032f8a37d527a8d5707a441958e3730", size = 606640, upload-time = "2026-10-20T16:02:04.01Z" }, + { url = "https://files.pythonhosted.org/packages/f0/fb/27c94facff5c4a39b954a41fac20c238647f0aad32452d0d9e33fc05e852/wcwidth-0.9.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1bbc8ebf133422cadfefea95a9f89d6c9e9a1ae19930bef7862a9b59b20bbad", size = 610956, upload-time = "2026-10-20T16:02:08.764Z" }, + { url = "https://files.pythonhosted.org/packages/38/61/38e6236356d25966d34636ef6f49ee83a66513e051c4a3b1e19c8b9ec806/wcwidth-0.9.3-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7a95d8c784cbccbbcb808eaadfd156cd282b2bdd95e89b32505e3443381ce61d", size = 775967, upload-time = "2026-10-20T16:02:23.765Z" }, + { url = "https://files.pythonhosted.org/packages/7b/87/c8f9295cd14a058dc27be0360fcc5ec70ce71b0d2e6a018871f585f23126/wcwidth-0.9.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a44acbb23b525070a0b8c343dd7cab7a4020aff73b38804f7a66d6a66e16e3fc", size = 787147, upload-time = "2026-10-20T16:02:14.095Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c0/238c135e283b1b0e16fef8b082d432dda1fe7e568d6dd85d5c7fc5e83859/wcwidth-0.9.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a7c0405a3bf3c564e481d6b2d55c99cc231bb68cf8f4a0995944bbce5ebf1a5d", size = 784548, upload-time = "2026-10-20T16:02:25.189Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ed/33e3080fe32f6f8f86334b2c5adde243b9a3a3a46cef97694dbed73b9dc4/wcwidth-0.9.3-cp314-cp314t-win32.whl", hash = "sha256:457f986e4867111cb037ccbf0d97ae100b1ecad030d43a020e7207f1e29649d5", size = 600761, upload-time = "2026-10-20T16:02:37.097Z" }, + { url = "https://files.pythonhosted.org/packages/27/62/6684a59147c1f86d58d0138f3109f506a6761b815dcb3d86b74d9af3a723/wcwidth-0.9.3-cp314-cp314t-win_amd64.whl", hash = "sha256:f11789e520ade8017fa93b01a4b8985571bb433eaefc2f857b57e36875e2e1d8", size = 605316, upload-time = "2026-10-20T16:02:38.513Z" }, + { url = "https://files.pythonhosted.org/packages/94/24/3ebcfd772d273e43e28447dd1946d88b8625db57b6a23496ea4eb9e1dbb7/wcwidth-0.9.3-cp314-cp314t-win_arm64.whl", hash = "sha256:eef39f3260991d8767189d296f0116fa4b45ad9c3a9a16d5af9187fb66b27564", size = 606392, upload-time = "2026-10-20T16:02:31.333Z" }, + { url = "https://files.pythonhosted.org/packages/76/fe/f90b9444bbc2d566634f91756bd11c2bd39d50c2c9077551711b10d8d03f/wcwidth-0.9.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:52a13869704d43a4add59c68ebc892198e53063883114783bf0b7457e250e3f4", size = 606655, upload-time = "2026-10-20T16:02:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/ab/63/edffb23938f4e27c28400c7d8d5a8715381d77f5db96bf80cf5dd6687f88/wcwidth-0.9.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:9470169fcb4e5340269175ebb3642dd5bfc8f82da8c86f29fc1fefc6c2b9bc1b", size = 611217, upload-time = "2026-10-20T16:02:10.252Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e6/0e4b5c81c238fac47eb66902f058a39ccdeacd7b2c5ab0c559066f203262/wcwidth-0.9.3-cp315-cp315t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c4531d93344441a7948e665854f0c40e3a8342ee106436d1531efa7a9fa7641e", size = 778155, upload-time = "2026-10-20T16:02:27.141Z" }, + { url = "https://files.pythonhosted.org/packages/f2/16/745439326544b2a6d65facbdf8462ee9d31e09c64830a764d2526c6c80d6/wcwidth-0.9.3-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:658dcbce546e7f28ee361f88e4a8e591dd56f46bff6084e11b4ad8310ed99946", size = 786757, upload-time = "2026-10-20T16:02:15.569Z" }, + { url = "https://files.pythonhosted.org/packages/08/a4/47193dc9902c30e187d81feaa7a5bc2672c41fe07093ca1ce5381395b85c/wcwidth-0.9.3-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9362be098c43d076aa43754face3cc0180fa81cea9ba304240b15490f2617e8", size = 785088, upload-time = "2026-10-20T16:02:28.784Z" }, + { url = "https://files.pythonhosted.org/packages/11/6a/9727002b938c28ab4456fc43ba31a3159ebff040021ce8dbd70e904c7cc9/wcwidth-0.9.3-cp315-cp315t-win32.whl", hash = "sha256:16d09df90b3a8f799ea14c726a41d9b906c3104f95f46d158e1ebe45cf4d547b", size = 600787, upload-time = "2026-10-20T16:02:39.766Z" }, + { url = "https://files.pythonhosted.org/packages/7c/db/0e24c2ff1e82cf298dd40a29e022da31b79be2589f0b8d865488e49140e1/wcwidth-0.9.3-cp315-cp315t-win_amd64.whl", hash = "sha256:3e3e31d6c17d6da3051f7eb854d5e4bebdc52a08a7a52adc2da6f15af51304e8", size = 605298, upload-time = "2026-10-20T16:02:41.479Z" }, + { url = "https://files.pythonhosted.org/packages/84/71/b470bd1dec14cbe6cee0c2e142787a8d99ac01711d5a1b533dda90754b48/wcwidth-0.9.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5d3b0b6c37f40e8d8298941ec373a426a892bb9a514328abcc9226cdc60ae6eb", size = 606346, upload-time = "2026-10-20T16:02:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/bf/12/e10aad98050389ef674c9f69409e38ae086c2104912644e1f9143b394301/wcwidth-0.9.3-py3-none-any.whl", hash = "sha256:e3c9069c54a2da0bf4f6c7e65a735cb1da0f47601a15cb5fef7e4b6ffca4cf60", size = 301629, upload-time = "2026-10-20T16:02:00.861Z" }, ] [[package]] @@ -4863,14 +5117,14 @@ wheels = [ [[package]] name = "xsdata" -version = "26.2" +version = "26.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/c9/71e9e8eac669091fd434ed494d806c8cc37614aecb34ce4c62c283f99abf/xsdata-26.2.tar.gz", hash = "sha256:c631af71aaa75734f8ce92a08fcf8389d905dee2aab0b5032c9032e9071009a6", size = 349690, upload-time = "2026-02-15T16:13:31.274Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/b8/7b301b80494a750dba431d3d0a800e0e80c91903e8891eef78ead8f95120/xsdata-26.10.tar.gz", hash = "sha256:4bc944d9fa3955f84ecebb83c677e2f681687c002a777731a21104e8360a7522", size = 356001, upload-time = "2026-10-16T10:06:30.516Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/92/f0edcbc2f895ecea14a68e492b24c157625e251279a94b172a6b263290e7/xsdata-26.2-py3-none-any.whl", hash = "sha256:85a591a4405d903416afbd4a917e8dda8ea44641a3e66d72134bc2a31b3c16b0", size = 235561, upload-time = "2026-02-15T16:13:29.614Z" }, + { url = "https://files.pythonhosted.org/packages/07/fe/5a9698cef0850317de2a8f3c50519fbaa2cc4225d15bcfe9922e4dff8380/xsdata-26.10-py3-none-any.whl", hash = "sha256:37a697d60db1396a0431750486744a7e0491fd60669751d95e3ede08f034e0c6", size = 243015, upload-time = "2026-10-16T10:06:28.782Z" }, ] [package.optional-dependencies] From 9e677dc946f2c55e9fb67c9bf8146cb33e4bc576 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 09:49:03 -0700 Subject: [PATCH 02/16] Unify JSONL, dest table reading; create generic CtsSettings builder; add shared FileItemDict and duckdb helpers --- tests/cdm_data_loaders/pipelines/conftest.py | 16 +-- tests/cdm_data_loaders/pipelines/framework.py | 15 +-- tests/cdm_data_loaders/pipelines/helpers.py | 116 ++++++++++++++++-- tests/cdm_data_loaders/pipelines/test_core.py | 13 +- 4 files changed, 113 insertions(+), 47 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index 20a58e88..8c9a242f 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -2,7 +2,6 @@ import copy import gzip -import json import logging import os from collections.abc import Callable, Iterable, Iterator @@ -12,7 +11,6 @@ from unittest.mock import MagicMock, patch import dlt -import pyarrow.parquet as pq import pytest from dlt.common.pipeline import LoadInfo @@ -28,6 +26,7 @@ PipelineTestRunner, ensure_docker_image, ) +from tests.cdm_data_loaders.pipelines.helpers import read_destination_rows as _read_destination_rows from tests.conftest import LOCAL_FS, S3 DEFAULT_DLT_TABLES = {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", LOAD_INFO_TABLE_NAME} @@ -263,18 +262,7 @@ def read_destination_rows(base_dir: Path, table_name: str) -> list[dict[str, Any Handles parquet, jsonl and gzipped jsonl files. Returns an empty list if the table has no files. """ - rows: list[dict[str, Any]] = [] - for data_file in sorted(base_dir.glob(f"**/{table_name}/*")): - if data_file.name.endswith(".parquet"): - rows.extend(pq.read_table(data_file).to_pylist()) - elif ".jsonl" in data_file.name: - if data_file.name.endswith(GZIP_SUFFIX): - with gzip.open(data_file, "rt") as f: - content = f.read() - else: - content = data_file.read_text() - rows.extend(json.loads(line) for line in content.splitlines() if line) - return [{key: value for key, value in row.items() if not key.startswith("_dlt")} for row in rows] + return _read_destination_rows(base_dir, table_name) def sort_rows(rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 825abffe..47661346 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -1,7 +1,5 @@ """Unified test execution framework for running CDM data ingestion pipelines in core, cli, or docker mode.""" -import gzip -import json import logging import os import shutil @@ -27,7 +25,7 @@ from cdm_data_loaders.pipelines import core from cdm_data_loaders.pipelines.core import LOAD_INFO_TABLE_NAME, PipelineCli from cdm_data_loaders.utils import cdm_logger -from tests.cdm_data_loaders.pipelines.helpers import settings_to_argv +from tests.cdm_data_loaders.pipelines.helpers import read_jsonl_file, settings_to_argv from tests.conftest import BASE_DIR, CONFIG_BUCKET, LOCAL_FS DEFAULT_DOCKER_IMAGE_TAG: Final[str] = "cdm-data-loaders-test:latest" @@ -264,14 +262,9 @@ def _read_saved_load_info_row(dataset_dir: Path) -> dict[str, Any]: for file_path in sorted(load_info_dir.glob("*.jsonl*")): if not file_path.is_file(): continue - with file_path.open("rb") as fh: - is_gzipped = fh.read(2) == b"\x1f\x8b" - opener = gzip.open if is_gzipped else open - with opener(file_path, "rt", encoding="utf-8") as fh: - for line in fh: - stripped = line.strip() - if stripped: - return json.loads(stripped) + rows = read_jsonl_file(file_path) + if rows: + return rows[0] return {} diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index b8d827be..4c7bf4ed 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -3,7 +3,8 @@ import gzip import json from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from datetime import datetime from logging import Logger, getLogger from pathlib import Path @@ -11,12 +12,16 @@ import dlt import duckdb +import pendulum import pyarrow.parquet as pq from dlt.common.pipeline import LoadInfo +from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.destinations import filesystem from cdm_data_loaders.core.fields import PARQUET, LoaderFileFormatEnum +from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.core import LOAD_INFO_TABLE_NAME +from tests.conftest import LOCAL_FS logger: Logger = getLogger(__name__) @@ -47,6 +52,25 @@ def is_metadata_table(table_name: str) -> bool: return table_name.startswith("_dlt") or table_name == LOAD_INFO_TABLE_NAME +def read_jsonl_file(file_path: Path) -> list[dict[str, Any]]: + """Read a plain or gzipped JSONLines file, detecting gzip by magic bytes.""" + with file_path.open("rb") as f: + is_gzipped = f.read(2) == b"\x1f\x8b" + + opener = gzip.open if is_gzipped else open + rows: list[dict[str, Any]] = [] + with opener(file_path, "rt", encoding="utf-8") as f: + for raw_line in f: + stripped = raw_line.strip() + if not stripped: + continue + try: + rows.append(json.loads(stripped)) + except json.JSONDecodeError: + logger.warning("Skipping unparseable line in %s: %r", file_path, raw_line[:200]) + return rows + + def read_parquet_tables(output_dir: Path) -> dict[str, list[dict[str, Any]]]: """Read all parquet files under a pipeline output dataset dir, grouped by table name.""" tables: dict[str, list[dict[str, Any]]] = defaultdict(list) @@ -71,22 +95,88 @@ def read_pipeline_tables(output_dir: Path, output_format: str | None = None) -> # ignore metadata dirs if is_metadata_table(json_file.parent.name): continue - # Detect gzip by magic bytes rather than relying on the extension, - # in case a .json file is actually gzipped or vice versa. - with json_file.open("rb") as f: - is_gzipped = f.read(2) == b"\x1f\x8b" - - opener = gzip.open if is_gzipped else open - with opener(json_file, "rt", encoding="utf-8") as f: - for raw_line in f: - try: - tables[json_file.parent.name].append(json.loads(raw_line.strip())) - except json.JSONDecodeError: - logger.warning("Skipping unparseable line in %s: %r", json_file, raw_line[:200]) + tables[json_file.parent.name].extend(read_jsonl_file(json_file)) return dict(tables) +def read_destination_rows(base_dir: Path, table_name: str) -> list[dict[str, Any]]: + """Read the rows the filesystem destination wrote for table_name under base_dir, without dlt columns. + + Handles parquet, jsonl and gzipped jsonl files. Returns an empty list if the table has no files. + """ + rows: list[dict[str, Any]] = [] + for data_file in sorted(base_dir.glob(f"**/{table_name}/*")): + if data_file.name.endswith(".parquet"): + rows.extend(pq.read_table(data_file).to_pylist()) + elif ".jsonl" in data_file.name: + rows.extend(read_jsonl_file(data_file)) + return [strip_metadata_columns(row) for row in rows] + + +def make_cts_settings[S: CtsSettings]( + settings_cls: type[S], + base_dir: Path, + **overrides: Any, # noqa: ANN401 +) -> S: + """Create standard input/output directories and log config under base_dir and instantiate settings_cls.""" + input_dir = base_dir / "input" + input_dir.mkdir(parents=True, exist_ok=True) + output_dir = base_dir / "output" + output_dir.mkdir(parents=True, exist_ok=True) + log_config_file = base_dir / "logging.json" + if not log_config_file.exists(): + log_config_file.write_text('{"version": 1}', encoding="utf-8") + + kwargs: dict[str, Any] = { + "input_dir": str(input_dir), + "output_dir": str(output_dir), + "log_config_file": str(log_config_file), + "use_destination": LOCAL_FS, + "dlt_dev_mode": False, + "use_output_dir_for_pipeline_metadata": False, + } + kwargs.update(overrides) + return settings_cls(**kwargs) + + +def make_file_item(file_path: Path, mime_type: str = "application/octet-stream") -> FileItemDict: + """Build a dlt FileItemDict for a local file.""" + return FileItemDict( + { + "file_url": file_path.as_uri(), + "file_name": file_path.name, + "relative_path": file_path.name, + "mime_type": mime_type, + "modification_date": pendulum.now(), + "size_in_bytes": file_path.stat().st_size, + } + ) + + +def fetch_all_duckdb(pipeline: dlt.Pipeline, query: str) -> list[tuple[Any, ...]]: + """Run a query against a pipeline's DuckDB destination and return all rows.""" + with ( + pipeline.sql_client() as client, + client.execute_query(query) as cur, + ): + return list(cur.fetchall()) + + +@contextmanager +def duckdb_reference_from_jsonl(jsonl_path: Path) -> Iterator[duckdb.DuckDBPyConnection]: + """Load a reference JSONL file into an in-memory DuckDB connection with a `reference` table.""" + connection = duckdb.connect() + connection.execute( + "CREATE TABLE reference AS SELECT * FROM read_json_auto(?, sample_size=200)", + [str(jsonl_path)], + ) + try: + yield connection + finally: + connection.close() + + def sorted_json(entries: list[dict[str, Any]]) -> list[str]: """Entries as sorted JSON strings for order-insensitive comparison.""" return sorted(json.dumps(entry, sort_keys=True, default=str) for entry in entries) diff --git a/tests/cdm_data_loaders/pipelines/test_core.py b/tests/cdm_data_loaders/pipelines/test_core.py index 8800aa92..76d59009 100644 --- a/tests/cdm_data_loaders/pipelines/test_core.py +++ b/tests/cdm_data_loaders/pipelines/test_core.py @@ -6,7 +6,6 @@ from collections.abc import Callable, Iterator from dataclasses import FrozenInstanceError from datetime import UTC, datetime -from json import loads from pathlib import Path from typing import Any, Final, cast from unittest.mock import MagicMock, call, patch @@ -39,6 +38,7 @@ write_pipeline_report, ) from tests.cdm_data_loaders.pipelines.conftest import HOOK_ENV_VAR +from tests.cdm_data_loaders.pipelines.helpers import read_destination_rows, read_jsonl_file from tests.conftest import LOCAL_FS, S3 from tests.dlt_config_isolation import dlt_config_unset, isolated_dlt_config @@ -101,20 +101,15 @@ def read_jsonl_rows(base_dir: Path, table_name: str) -> list[dict[str, Any]]: """Read every jsonl row written for table_name anywhere under base_dir.""" rows: list[dict[str, Any]] = [] for jsonl_file in sorted(base_dir.glob(f"**/{table_name}/*.jsonl*")): - if jsonl_file.name.endswith(".gz"): - with gzip.open(jsonl_file, "rt") as f: - content = f.read() - else: - content = jsonl_file.read_text() - rows.extend(loads(record) for record in content.splitlines() if record) + rows.extend(read_jsonl_file(jsonl_file)) return rows def read_output_jsonl_records(base_dir: Path) -> list[dict[str, Any]]: """Read the tiny table records under base_dir, minus dlt's internal columns.""" - records = read_jsonl_rows(base_dir, TINY_TABLE_NAME) + records = read_destination_rows(base_dir, TINY_TABLE_NAME) assert records - return [{key: value for key, value in record.items() if not key.startswith("_dlt")} for record in records] + return records def make_mock_pipeline(save_run_result: object, slack_hook: str | None = None) -> MagicMock: From 5647808d26bea4eb799a597ec40f80ddf83c22e3 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 10:34:21 -0700 Subject: [PATCH 03/16] Consolidate fixtures and remove duplicated functions --- tests/cdm_data_loaders/pipelines/conftest.py | 37 +++-- .../pipelines/jsonlines/conftest.py | 55 ++++---- .../jsonlines/integration/conftest.py | 34 ++--- .../jsonlines/test_jsonschema_pipeline.py | 14 +- .../jsonlines/test_make_validator.py | 18 +-- .../pipelines/ncbi_rest_api/conftest.py | 70 ++++++++++ .../ncbi_rest_api/test_ncbi_rest_api.py | 70 ++-------- .../test_ncbi_rest_api_settings.py | 60 --------- tests/cdm_data_loaders/pipelines/test_core.py | 82 ++++++------ .../pipelines/xml2db/integration/conftest.py | 52 ++------ .../pipelines/xml2db/test_xml2db_pipeline.py | 22 +-- .../pipelines/xml2db/test_xml2db_settings.py | 16 +-- .../pipelines/xmltodict/conftest.py | 22 +-- .../xmltodict/integration/conftest.py | 70 +++------- .../pipelines/xsv/conftest.py | 76 +++-------- .../pipelines/xsv/integration/conftest.py | 126 ------------------ .../xsv/integration/test_xsv_end_to_end.py | 2 +- .../pipelines/xsv/test_xsv_pipeline.py | 3 +- .../pipelines/xsv/test_xsv_settings.py | 67 +--------- tests/jsonschema_fixtures.py | 14 +- 20 files changed, 250 insertions(+), 660 deletions(-) create mode 100644 tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py delete mode 100644 tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index 8c9a242f..c7ad2963 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -26,8 +26,13 @@ PipelineTestRunner, ensure_docker_image, ) -from tests.cdm_data_loaders.pipelines.helpers import read_destination_rows as _read_destination_rows -from tests.conftest import LOCAL_FS, S3 +from tests.cdm_data_loaders.pipelines.helpers import ( + make_cts_settings, +) +from tests.cdm_data_loaders.pipelines.helpers import ( + read_destination_rows as _read_destination_rows, +) +from tests.conftest import LOCAL_FS, REFERENCE_XML_FIXTURE_DIR, S3, _generate_dlt_config DEFAULT_DLT_TABLES = {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", LOAD_INFO_TABLE_NAME} @@ -41,14 +46,10 @@ GZIP_SUFFIX: Final[str] = ".gz" -DLT_CONFIG: Final[dict[str, Any]] = { - "destination": { - LOCAL_FS: {"bucket_url": "/configured/out/", "destination_type": "filesystem"}, - S3: {"bucket_url": "s3://bucket/prefix", "destination_type": "filesystem"}, - }, - f"destination.{LOCAL_FS}.destination_type": "filesystem", - f"destination.{S3}.destination_type": "filesystem", -} +@pytest.fixture(scope="session") +def reference_xml_data_dir() -> Path: + """Path to the uniref fixture directory.""" + return REFERENCE_XML_FIXTURE_DIR def duckdb_pipeline(tmp_path: Path, pipeline_name: str, dataset_name: str, db_name: str | None = None) -> dlt.Pipeline: @@ -72,8 +73,6 @@ def clean_env() -> Iterator[None]: with patch.dict(os.environ): for name in (*HOOK_PART_ENV_VARS, HOOK_ENV_VAR, DISABLE_COMPRESSION_ENV_VAR): os.environ.pop(name, None) - for name in [key for key in os.environ if key.startswith("CDL_")]: - del os.environ[name] os.environ["RUNTIME__DLTHUB_TELEMETRY"] = "false" yield @@ -99,7 +98,7 @@ def mock_init_logger(monkeypatch: pytest.MonkeyPatch) -> MagicMock: """Stop run_cli reconfiguring logging and point dlt.config at a plain dict.""" mock = MagicMock() monkeypatch.setattr(core, "init_logger", mock) - monkeypatch.setattr(dlt, "config", copy.deepcopy(DLT_CONFIG)) + monkeypatch.setattr(dlt, "config", copy.deepcopy(_generate_dlt_config())) return mock @@ -172,21 +171,17 @@ def isolated_run( """ run_counter = count() - def _factory(**overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(**overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 run_index = next(run_counter) - output_dir = tmp_path / f"output_{run_index}" - output_dir.mkdir() - log_config_file = tmp_path / f"logging_{run_index}.conf" - log_config_file.touch() kwargs: dict[str, Any] = { "dataset_name": f"{dataset_prefix}_{run_index}", - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), + "log_config_file": str(tmp_path / f"logging_{run_index}.conf"), + "output_dir": str(tmp_path / f"output_{run_index}"), **base_kwargs, } kwargs.update(overrides) settings_cls = kwargs.pop("settings_cls") - settings = settings_cls(**kwargs) + settings = make_cts_settings(settings_cls, tmp_path, **kwargs) with dlt.config.values({"destination.local_fs.bucket_url": str(settings.output_dir)}): load_info = pipeline_runner.run(run_fn, settings) if pipeline_runner is not None else run_fn(settings) return load_info, settings.output_dir diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py index 6ad3892e..c7bc8c2d 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/conftest.py @@ -1,5 +1,6 @@ """Shared fixtures for jsonlines pipeline tests.""" +import json from collections.abc import Callable from pathlib import Path from typing import Any @@ -7,6 +8,26 @@ import pytest from cdm_data_loaders.pipelines.jsonlines.settings import JsonlExtractSettings, JsonlPydanticIngestSettings +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings + + +def make_item( + record: object = None, + line_no: int = 1, + raw_record: str | None = None, + parse_error: str | None = None, + source_file: str = "file.jsonl", +) -> dict[str, Any]: + """Build a line record matching stream_jsonl_lines' output shape.""" + if raw_record is None: + raw_record = "" if parse_error is not None and record is None else json.dumps(record, separators=(",", ":")) + return { + "record": record, + "raw_record": raw_record, + "parse_error": parse_error, + "source_file": source_file, + "line_no": line_no, + } @pytest.fixture @@ -16,25 +37,14 @@ def settings_factory( """Return a factory that builds a valid JsonlPydanticIngestSettings, with fields open to override.""" def _factory(**overrides: Any) -> JsonlPydanticIngestSettings: # noqa: ANN401 - log_config_file = tmp_path / "logging.conf" - log_config_file.touch() - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { + defaults: dict[str, Any] = { "dataset_name": "test_dataset", "dlt_dev_mode": True, "entity_models_module": entity_models_module, - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, } - kwargs.update(overrides) - return JsonlPydanticIngestSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(JsonlPydanticIngestSettings, tmp_path, **defaults) return _factory @@ -44,24 +54,13 @@ def extract_settings_factory(tmp_path: Path, dlt_destination_config: str) -> Cal """Return a factory that builds a valid JsonlExtractSettings, with fields open to override.""" def _factory(**overrides: Any) -> JsonlExtractSettings: # noqa: ANN401 - log_config_file = tmp_path / "logging.conf" - log_config_file.touch() - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { + defaults: dict[str, Any] = { "dataset_name": "test_dataset", "dlt_dev_mode": True, - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), "table_name": "widget", "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, } - kwargs.update(overrides) - return JsonlExtractSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(JsonlExtractSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py index 4a021cf1..a352c32f 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py @@ -25,6 +25,7 @@ JsonlJsonschemaIngestSettings, JsonlPydanticIngestSettings, ) +from tests.cdm_data_loaders.pipelines.helpers import duckdb_reference_from_jsonl, make_cts_settings from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models import ENTITY_MODELS from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas import ENTITY_SCHEMAS from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( @@ -81,7 +82,7 @@ def run_reference_pipeline( } runner = isolated_run_factory(dataset_prefix=dataset_prefix, run_fn=run_fn, base_kwargs=base_kwargs) - def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 kwargs = {"input_dir": str(reference_data_dir / chunk_dir), **overrides} return runner(**kwargs) @@ -147,16 +148,8 @@ def reference_dataset(reference_jsonl: Path) -> Generator[duckdb.DuckDBPyConnect read_json_auto infers native STRUCT/STRUCT[] types for the nested content. """ - connection = duckdb.connect() - jsonl_path = str(reference_jsonl) - connection.execute( - "CREATE TABLE reference AS SELECT * FROM read_json_auto(?, sample_size=200)", - [jsonl_path], - ) - try: - yield connection - finally: - connection.close() + with duckdb_reference_from_jsonl(reference_jsonl) as conn: + yield conn @pytest.fixture @@ -177,26 +170,15 @@ def settings_factory( ) -> Callable[..., JsonlPydanticIngestSettings]: """Return a factory that builds a valid JsonlPydanticIngestSettings, with fields open to override.""" - def _factory(**overrides: Any) -> JsonlPydanticIngestSettings: - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { + def _factory(**overrides: Any) -> JsonlPydanticIngestSettings: # noqa: ANN401 + defaults: dict[str, Any] = { "dataset_name": "test_dataset", "dlt_dev_mode": False, "entity_models_module": entity_models_module, - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, } - kwargs.update(overrides) - return JsonlPydanticIngestSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(JsonlPydanticIngestSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py index 769b2523..a956a842 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_jsonschema_pipeline.py @@ -20,17 +20,7 @@ run_jsonlines_pipeline, ) from cdm_data_loaders.pipelines.jsonlines.settings import JsonlJsonschemaIngestSettings - - -def make_item(record: object, line_no: int = 1) -> dict[str, Any]: - """Build the shared reader's parsed-line envelope.""" - return { - "record": record, - "raw_record": json.dumps(record, separators=(",", ":")), - "parse_error": None, - "source_file": "data.jsonl", - "line_no": line_no, - } +from tests.cdm_data_loaders.pipelines.jsonlines.conftest import make_item def make_router(schema: dict[str, Any], buffer_size: int = 2) -> Callable[[list[dict[str, Any]]], Any]: @@ -187,7 +177,7 @@ def test_make_page_router_fail_reports_exact_errors( "widget_rejected", [ { - "source_file": "data.jsonl", + "source_file": "file.jsonl", "line_no": 7, "raw_record": item["raw_record"], "error_detail": json.dumps(messages), diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py index 4bf6ae42..d66254c5 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_make_validator.py @@ -9,23 +9,7 @@ from pydantic import ValidationError from cdm_data_loaders.pipelines.jsonlines.base import make_page_router - - -def make_item( - record: dict[str, Any] | None = None, - raw_record: str = "", - parse_error: str | None = None, - source_file: str = "file.jsonl", - line_no: int = 1, -) -> dict[str, Any]: - """Build a line record matching stream_jsonl_lines' output shape.""" - return { - "record": record, - "raw_record": raw_record, - "parse_error": parse_error, - "source_file": source_file, - "line_no": line_no, - } +from tests.cdm_data_loaders.pipelines.jsonlines.conftest import make_item def run_validator(validator: Callable[[list[dict[str, Any]]], Any], item: dict[str, Any]) -> list[DataItemWithMeta]: diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py new file mode 100644 index 00000000..e13dfef1 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py @@ -0,0 +1,70 @@ +"""Shared fixtures and constants for NCBI REST API pipeline tests.""" + +from typing import Any + +import pytest + +from cdm_data_loaders.pipelines import ncbi_rest_api as ncbi_module +from cdm_data_loaders.pipelines.ncbi_rest_api import NcbiRestApiSettings +from tests.conftest import DEFAULT_VCR_CONFIG + +ID_WITH_2K_ANNOTS = "GCF_000003135.1" +ID_WITH_500_ANNOTS = "GCF_000007725.1" +ID_TRIGGERS_500_ERR = "GCF_500_ERROR" +VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] +INVALID_ID = "invalid_id" +ALL_IDS = [*VALID_IDS, INVALID_ID] + + +@pytest.fixture(autouse=True) +def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: + """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" + monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) + monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.ncbi_genome_client._config.http_show_error_body", True) + + +@pytest.fixture(scope="module") +def vcr_config() -> dict[str, Any]: + """VCR config for tests that make HTTP requests.""" + return {**DEFAULT_VCR_CONFIG} + + +@pytest.fixture(scope="module") +def valid_assembly_ids() -> list[str]: + """A list of assembly IDs.""" + return VALID_IDS + + +@pytest.fixture(scope="module") +def assembly_id(valid_assembly_ids: list[str]) -> str: + """Single valid assembly ID.""" + return valid_assembly_ids[0] + + +@pytest.fixture(scope="module") +def invalid_assembly_id() -> str: + """Invalid assembly ID.""" + return INVALID_ID + + +@pytest.fixture(scope="module") +def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> list[str]: + """List of assembly IDs including both valid and invalid IDs.""" + return [*valid_assembly_ids, invalid_assembly_id] + + +@pytest.fixture +def test_settings(test_defaults: dict[str, Any]) -> NcbiRestApiSettings: + """Generate a test settings class.""" + return NcbiRestApiSettings(**test_defaults) + + +@pytest.fixture(autouse=True) +def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: + """Reset the pipeline settings prior to running tests. + + :param monkeypatch: monkeypatch + :type monkeypatch: pytest.MonkeyPatch + """ + monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) + diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py index 0c35d0dc..cd627ece 100644 --- a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api.py @@ -28,67 +28,15 @@ set_settings, ) from tests.cdm_data_loaders.conftest import check_settings -from tests.conftest import DEFAULT_VCR_CONFIG, LOCAL_FS - - -@pytest.fixture(autouse=True) -def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: - """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" - monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) - - -ID_WITH_2K_ANNOTS = "GCF_000003135.1" -ID_WITH_500_ANNOTS = "GCF_000007725.1" -ID_TRIGGERS_500_ERR = "GCF_500_ERROR" -VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] -INVALID_ID = "invalid_id" -ALL_IDS = [*VALID_IDS, INVALID_ID] - - -@pytest.fixture(scope="module") -def vcr_config() -> dict[str, Any]: - """VCR config for tests that make HTTP requests.""" - return {**DEFAULT_VCR_CONFIG} - - -@pytest.fixture(scope="module") -def valid_assembly_ids() -> list[str]: - """A list of assembly IDs.""" - return VALID_IDS - - -@pytest.fixture(scope="module") -def assembly_id(valid_assembly_ids: list[str]) -> str: - """Single valid assembly ID.""" - return valid_assembly_ids[0] - - -@pytest.fixture(scope="module") -def invalid_assembly_id() -> str: - """Invalid assembly ID.""" - return INVALID_ID - - -@pytest.fixture(scope="module") -def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> list[str]: - """List of assembly IDs including both valid and invalid IDs.""" - return [*valid_assembly_ids, invalid_assembly_id] - - -@pytest.fixture -def test_settings(test_defaults: dict[str, Any]) -> NcbiRestApiSettings: - """Generate a test settings class.""" - return NcbiRestApiSettings(**test_defaults) - - -@pytest.fixture(autouse=True) -def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: - """Reset the pipeline settings prior to running tests. - - :param monkeypatch: monkeypatch - :type monkeypatch: pytest.MonkeyPatch - """ - monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) +from tests.cdm_data_loaders.pipelines.ncbi_rest_api.conftest import ( + ALL_IDS, + ID_TRIGGERS_500_ERR, + ID_WITH_2K_ANNOTS, + ID_WITH_500_ANNOTS, + INVALID_ID, + VALID_IDS, +) +from tests.conftest import LOCAL_FS def test_cli_passes_settings_class_to_run_cli() -> None: diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py index 24e95fab..c8552e18 100644 --- a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/test_ncbi_rest_api_settings.py @@ -19,22 +19,8 @@ set_settings, ) from tests.cdm_data_loaders.conftest import check_settings -from tests.conftest import DEFAULT_VCR_CONFIG from tests.helpers import assert_cli_field_roundtrips, assert_no_cli_clashes - -@pytest.fixture(autouse=True) -def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: - """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" - monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) - - -ID_WITH_2K_ANNOTS = "GCF_000003135.1" -ID_WITH_500_ANNOTS = "GCF_000007725.1" -ID_TRIGGERS_500_ERR = "GCF_500_ERROR" -VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] -INVALID_ID = "invalid_id" -ALL_IDS = [*VALID_IDS, INVALID_ID] BATCH_SIZE = 200 BATCH_SIZE_STRING = str(BATCH_SIZE) @@ -68,52 +54,6 @@ def ncbi_settings_pair( return settings, reconciled -@pytest.fixture(scope="module") -def vcr_config() -> dict[str, Any]: - """VCR config for tests that make HTTP requests.""" - return {**DEFAULT_VCR_CONFIG} - - -@pytest.fixture(scope="module") -def valid_assembly_ids() -> list[str]: - """A list of assembly IDs.""" - return VALID_IDS - - -@pytest.fixture(scope="module") -def assembly_id(valid_assembly_ids: list[str]) -> str: - """Single valid assembly ID.""" - return valid_assembly_ids[0] - - -@pytest.fixture(scope="module") -def invalid_assembly_id() -> str: - """Invalid assembly ID.""" - return INVALID_ID - - -@pytest.fixture(scope="module") -def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> list[str]: - """List of assembly IDs including both valid and invalid IDs.""" - return [*valid_assembly_ids, invalid_assembly_id] - - -@pytest.fixture -def test_settings(test_defaults: dict[str, Any]) -> NcbiRestApiSettings: - """Generate a test settings class.""" - return NcbiRestApiSettings(**test_defaults) - - -@pytest.fixture(autouse=True) -def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: - """Reset the pipeline settings prior to running tests. - - :param monkeypatch: monkeypatch - :type monkeypatch: pytest.MonkeyPatch - """ - monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) - - def test_get_set_settings(test_settings: NcbiRestApiSettings) -> None: """Test setting and retrieval of settings.""" assert ncbi_module.PIPELINE_SETTINGS is None diff --git a/tests/cdm_data_loaders/pipelines/test_core.py b/tests/cdm_data_loaders/pipelines/test_core.py index 76d59009..07f221e7 100644 --- a/tests/cdm_data_loaders/pipelines/test_core.py +++ b/tests/cdm_data_loaders/pipelines/test_core.py @@ -62,12 +62,6 @@ def make_cts_settings(**kwargs: str | int | bool | Path) -> CtsSettings: return CtsSettings.model_validate(kwargs) -@pytest.fixture -def bucket_dir(tmp_path: Path, dlt_destination_config: str) -> Path: # noqa: ARG001 - """The bucket_url of the local_fs destination in dlt.config, where run_pipeline loads data.""" - return tmp_path / "bucket" - - @pytest.fixture def dlt_test_settings(tmp_path: Path, dlt_destination_config: str) -> CtsSettings: """Settings that load into the local_fs destination and keep pipeline metadata in a separate output_dir.""" @@ -476,13 +470,13 @@ def test_run_pipeline_no_slack_env_var_when_vars_missing( # run_pipeline tests -def test_run_pipeline_minimal(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_minimal(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """A tiny pipeline loads its records into the destination bucket, not into output_dir.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA assert read_jsonl_rows(Path(dlt_test_settings.output_dir), TINY_TABLE_NAME) == [] @@ -605,7 +599,7 @@ def test_run_pipeline_pipelines_dir_used_for_pipeline_metadata(dlt_test_settings assert any(p.name == "state.json" for p in pipeline_state_files) -def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """dev_mode=True is forwarded to dlt.pipeline(); the pipeline loads into a suffixed dataset.""" settings = dlt_test_settings.model_copy(update={"dlt_dev_mode": True}) @@ -616,7 +610,7 @@ def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: # dataset_name is `dlt_pipeline_dataset_\d+`, where \d+ is YYYYMMDDHHMMSS assert load_info.dataset_name.startswith(f"{load_info.pipeline.pipeline_name}_dataset_") assert load_info.dataset_name != f"{load_info.pipeline.pipeline_name}_dataset" - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA @pytest.mark.parametrize( @@ -630,7 +624,7 @@ def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: ) def test_run_pipeline_passes_kwargs_to_real_pipeline( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, pipeline_kwargs: dict[str, Any], pipeline_run_kwargs: dict[str, Any], ) -> None: @@ -647,17 +641,17 @@ def test_run_pipeline_passes_kwargs_to_real_pipeline( if pipeline_kwargs: assert load_info.pipeline.pipeline_name == TINY_PIPELINE_NAME assert load_info.dataset_name == "core_test_dataset" - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA # load_info saved as part of the dataset -def test_run_pipeline_saves_load_info_to_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_saves_load_info_to_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """The pipeline's load_info is saved as a table in the same dataset.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 row = rows[0] assert row["dataset_name"] == load_info.dataset_name @@ -669,26 +663,26 @@ def test_run_pipeline_saves_load_info_to_dataset(dlt_test_settings: CtsSettings, assert set(row["pipeline"]) == {"pipeline_name"} -def test_run_pipeline_load_info_saved_to_same_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_saved_to_same_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """The load info table lands in the same dataset directory as the resource tables.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None - dataset_dir = bucket_dir / load_info.dataset_name + dataset_dir = destination_bucket / load_info.dataset_name assert (dataset_dir / TINY_TABLE_NAME).is_dir() assert (dataset_dir / LOAD_INFO_TABLE_NAME).is_dir() - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA - assert read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA + assert read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) -def test_run_pipeline_load_info_flat_structure(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_flat_structure(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """max_table_nesting=0 keeps nested structures as json columns; no child tables.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None - dataset_dir = bucket_dir / load_info.dataset_name + dataset_dir = destination_bucket / load_info.dataset_name assert list(dataset_dir.glob("_dlt_load_info__*")) == [] - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 row = rows[0] for nested in ("load_packages", "outputs", "job_metrics"): @@ -697,7 +691,7 @@ def test_run_pipeline_load_info_flat_structure(dlt_test_settings: CtsSettings, b assert all(isinstance(item, dict) for item in row[nested]) -def test_run_pipeline_load_info_runs_accumulate(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_runs_accumulate(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """Each pipeline run appends its own load_info row; earlier rows are preserved.""" first = run_pipeline(dlt_test_settings, tiny_resource()) assert first is not None @@ -705,30 +699,30 @@ def test_run_pipeline_load_info_runs_accumulate(dlt_test_settings: CtsSettings, second = run_pipeline(dlt_test_settings, tiny_resource()) assert second is not None - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 2 saved_load_ids = {tuple(row["loads_ids"]) for row in rows} assert len(saved_load_ids) == 2 # each accumulated row references one of the entity data loads - tiny_load_ids = {row["_dlt_load_id"] for row in read_jsonl_rows(bucket_dir, TINY_TABLE_NAME)} + tiny_load_ids = {row["_dlt_load_id"] for row in read_jsonl_rows(destination_bucket, TINY_TABLE_NAME)} assert saved_load_ids == {(load_id,) for load_id in tiny_load_ids} -def test_run_pipeline_load_info_in_dev_mode(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_in_dev_mode(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """dev_mode runs save the load_info into the same suffixed dataset as the resource data.""" settings = dlt_test_settings.model_copy(update={"dlt_dev_mode": True}) load_info = run_pipeline(settings, tiny_resource()) assert load_info is not None assert load_info.dataset_name != "dlt_pipeline_dataset" - dataset_dir = bucket_dir / load_info.dataset_name + dataset_dir = destination_bucket / load_info.dataset_name assert (dataset_dir / TINY_TABLE_NAME).is_dir() - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 assert rows[0]["dataset_name"] == load_info.dataset_name -def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: CtsSettings, bucket_dir: Path) -> None: +def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: """A named pipeline/dataset save lands in the named dataset with a load_info row.""" load_info = run_pipeline( dlt_test_settings, @@ -737,22 +731,22 @@ def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: Ct ) assert load_info is not None - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 assert rows[0]["dataset_name"] == "core_test_dataset" assert rows[0]["pipeline"] == {"pipeline_name": TINY_PIPELINE_NAME} - assert (bucket_dir / "core_test_dataset" / LOAD_INFO_TABLE_NAME).is_dir() + assert (destination_bucket / "core_test_dataset" / LOAD_INFO_TABLE_NAME).is_dir() def test_run_pipeline_load_info_returns_original_import_result( - dlt_test_settings: CtsSettings, bucket_dir: Path + dlt_test_settings: CtsSettings, destination_bucket: Path ) -> None: """run_pipeline returns the original resource load; the saved row references it, not the save load.""" load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs - rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(rows) == 1 # loads_ids is the original payload; _dlt_load_id is the save run's own load id assert rows[0]["loads_ids"] != [rows[0]["_dlt_load_id"]] @@ -762,7 +756,7 @@ def test_run_pipeline_load_info_returns_original_import_result( def test_run_pipeline_load_info_save_resource_error_still_returns_load_info( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: @@ -781,8 +775,8 @@ def resource_boom(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 assert load_info is not None assert not load_info.has_failed_jobs - assert read_output_jsonl_records(bucket_dir) == TINY_RESOURCE_DATA - assert read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) == [] + assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA + assert read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) == [] error_messages = [record.getMessage() for record in caplog.records if record.levelno == logging.ERROR] assert error_messages == [f"{LOAD_INFO_SAVE_FAILED}load info save failed"] assert ALL_JOBS_COMPLETED in caplog.messages @@ -1026,7 +1020,7 @@ def test_run_pipeline_pass_failed_jobs_reported( def test_run_pipeline_pass_partial_load_worker_failure_filesystem( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, monkeypatch: pytest.MonkeyPatch, slack_mock: MagicMock, caplog: pytest.LogCaptureFixture, @@ -1060,15 +1054,15 @@ def fail_second_tiny_record(self: FilesystemLoadJob) -> None: ] # The record from the succeeding worker thread is in the tiny table; the failed worker's record is missing - assert read_output_jsonl_records(bucket_dir) == [TINY_RESOURCE_DATA[0]] + assert read_output_jsonl_records(destination_bucket) == [TINY_RESOURCE_DATA[0]] # Both the partially-failed data load and the _load_info metadata load are recorded in _dlt_loads as status 0 - dlt_loads = read_jsonl_rows(bucket_dir, "_dlt_loads") + dlt_loads = read_jsonl_rows(destination_bucket, "_dlt_loads") assert [row["status"] for row in dlt_loads] == [0, 0] - assert len(read_jsonl_rows(bucket_dir, "_dlt_pipeline_state")) == 1 + assert len(read_jsonl_rows(destination_bucket, "_dlt_pipeline_state")) == 1 # _load_info is saved to the filesystem destination and records both completed and failed worker jobs - load_info_rows = read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) + load_info_rows = read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) assert len(load_info_rows) == 1 package = load_info_rows[0]["load_packages"][0] assert package["state"] == "loaded" @@ -1109,7 +1103,7 @@ def fail_second_tiny_record(self: FilesystemLoadJob) -> None: def test_run_pipeline_fail_partial_extract_worker_failure_filesystem( dlt_test_settings: CtsSettings, - bucket_dir: Path, + destination_bucket: Path, monkeypatch: pytest.MonkeyPatch, slack_mock: MagicMock, caplog: pytest.LogCaptureFixture, @@ -1136,9 +1130,9 @@ def parallel_transform(item: dict[str, Any]) -> dict[str, Any]: ) assert load_info is None - assert list(bucket_dir.iterdir()) == [] - assert read_jsonl_rows(bucket_dir, TINY_TABLE_NAME) == [] - assert read_jsonl_rows(bucket_dir, LOAD_INFO_TABLE_NAME) == [] + assert list(destination_bucket.iterdir()) == [] + assert read_jsonl_rows(destination_bucket, TINY_TABLE_NAME) == [] + assert read_jsonl_rows(destination_bucket, LOAD_INFO_TABLE_NAME) == [] assert caplog.records[-1].levelno == logging.ERROR assert caplog.records[-1].getMessage().startswith("Pipeline failed: ") assert SINK_ERROR in caplog.records[-1].getMessage() diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py index 65c1973d..34b09b33 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/conftest.py @@ -10,10 +10,8 @@ from cdm_data_loaders.pipelines.xml2db.pipeline import run_xml2db_pipeline from cdm_data_loaders.pipelines.xml2db.settings import Xml2DbSettings -from tests.conftest import ( - REFERENCE_XML_FIXTURE_DIR, - REFERENCE_XSD, -) +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings +from tests.conftest import REFERENCE_XSD from tests.xml_samples import LIBRARY_XSD DEFAULT_XML2DB_SETTINGS: frozendict = frozendict( @@ -35,50 +33,26 @@ def library_xsd(tmp_path: Path) -> Path: return xsd_path -@pytest.fixture(scope="session") -def reference_xml_data_dir() -> Path: - """Path to the uniref fixture directory (shared with the xmltodict reference tests).""" - return REFERENCE_XML_FIXTURE_DIR - - @pytest.fixture(scope="session") def reference_xsd() -> Path: """Path to the uniref.xsd schema.""" return REFERENCE_XSD -def make_settings( - tmp_path: Path, dlt_destination_config: str, xsd_file: str | Path, **overrides: Any -) -> dict[str, Any]: - """Build a valid xml2db settings dict, with fields open to override.""" - log_config_file = overrides.get("log_config_file", tmp_path / "logging.conf") - log_config_file.touch() - output_dir = overrides.get("output_dir", tmp_path / "bucket") - output_dir.mkdir(exist_ok=True) - input_dir = overrides.get("input_dir", tmp_path / "input") - - kwargs: dict[str, Any] = { - **DEFAULT_XML2DB_SETTINGS, - "dataset_name": "xml2db_test_dataset", - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), - "use_destination": dlt_destination_config, - "xsd_file": str(xsd_file), - } - kwargs.update(overrides) - return kwargs - - @pytest.fixture def settings_factory(tmp_path: Path, dlt_destination_config: str, library_xsd: Path) -> Callable[..., Xml2DbSettings]: """Return a factory that builds a valid Xml2DbSettings using the library.xsd schema.""" - def _factory(**overrides: Any) -> Xml2DbSettings: - settings_dict = make_settings(tmp_path, dlt_destination_config, xsd_file=library_xsd, **overrides) - input_dir = Path(settings_dict["input_dir"]) - input_dir.mkdir(exist_ok=True) - return Xml2DbSettings(**settings_dict) + def _factory(**overrides: Any) -> Xml2DbSettings: # noqa: ANN401 + defaults: dict[str, Any] = { + **DEFAULT_XML2DB_SETTINGS, + "dataset_name": "xml2db_test_dataset", + "output_dir": str(tmp_path / "bucket"), + "use_destination": dlt_destination_config, + "xsd_file": str(library_xsd), + } + defaults.update(overrides) + return make_cts_settings(Xml2DbSettings, tmp_path, **defaults) return _factory @@ -114,7 +88,7 @@ def run_xml2db_pipeline_fxt( }, ) - def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 return run(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py index b273474f..09fa3859 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py @@ -20,6 +20,7 @@ from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from cdm_data_loaders.readers.xml2db_doc import build_xml2db_model from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings SIMPLE_LIBRARY_XML = """ @@ -82,28 +83,13 @@ def fake_settings( **overrides: Any, # noqa: ANN401 ) -> Xml2DbSettings: """Build a valid Xml2DbSettings with fields open to override.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}', encoding="utf-8") - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { - "buffer_size": 100, - "log_interval": 1000, + defaults: dict[str, Any] = { "dataset_name": "xml2db_test_dataset", "loader_file_format": "parquet", - "log_config_file": str(log_config_file), - "input_dir": str(input_dir), - "output_dir": str(output_dir), - "dlt_dev_mode": False, - "use_destination": "local_fs", - "use_output_dir_for_pipeline_metadata": False, "xsd_file": str(xsd_file), } - kwargs.update(overrides) - return Xml2DbSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(Xml2DbSettings, tmp_path, **defaults) # _read_items diff --git a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py index 972adc6e..5ee2981b 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_settings.py @@ -15,6 +15,7 @@ PIPELINE_NAME, Xml2DbSettings, ) +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings from tests.conftest import S3 MINIMAL_SETTINGS_KWARGS: dict[str, object] = { @@ -44,23 +45,16 @@ def settings_factory(tmp_path: Path, test_data_dir: Path) -> Callable[..., Xml2D """Return a factory that builds a valid Xml2DbSettings with fields open to override.""" def _factory(**overrides: object) -> Xml2DbSettings: - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}', encoding="utf-8") input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) xsd_file = _write_xsd_fixture(input_dir, test_data_dir, "uniref.xsd") - - kwargs: dict[str, object] = { + defaults: dict[str, object] = { **MINIMAL_SETTINGS_KWARGS, - "log_config_file": str(log_config_file), "input_dir": str(input_dir), - "output_dir": str(output_dir), + "output_dir": str(tmp_path / "output"), "xsd_file": str(input_dir / xsd_file), } - kwargs.update(overrides) - return Xml2DbSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(Xml2DbSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py index dfe75e69..dfa561bc 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py @@ -7,7 +7,7 @@ import pytest from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings -from tests.conftest import LOCAL_FS +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings @pytest.fixture @@ -15,28 +15,14 @@ def settings_factory(tmp_path: Path) -> Callable[..., XmlToDictSettings]: """Return a factory that builds a valid XmlToDictSettings, with fields open to override.""" def _factory(**overrides: Any) -> XmlToDictSettings: # noqa: ANN401 - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - kwargs: dict[str, Any] = { - "log_config_file": str(log_config_file), - "input_dir": str(input_dir), - "output_dir": str(output_dir), + defaults: dict[str, Any] = { "dlt_dev_mode": True, - "use_destination": LOCAL_FS, - "use_output_dir_for_pipeline_metadata": False, - "buffer_size": 100, - "log_interval": 1000, "dataset_name": "xml_test_dataset", "table_name": "book", "xml_tag": "book", "file_glob": "*.xml*", } - kwargs.update(overrides) - return XmlToDictSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(XmlToDictSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py index 583017c7..1a47b253 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py @@ -2,7 +2,7 @@ import gzip import json -from collections.abc import Callable +from collections.abc import Callable, Generator from pathlib import Path from typing import Any @@ -15,7 +15,12 @@ from cdm_data_loaders.pipelines.xmltodict.pipeline import run_xmltodict_pipeline from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings from cdm_data_loaders.readers.xml import DEFAULT_XMLTODICT_ARGS -from tests.conftest import N_REFERENCE_XML_ENTRIES, REFERENCE_XML_FIXTURE_DIR, REFERENCE_XML_TAG +from tests.cdm_data_loaders.pipelines.helpers import ( + duckdb_reference_from_jsonl, + make_cts_settings, + read_jsonl_file, +) +from tests.conftest import N_REFERENCE_XML_ENTRIES, REFERENCE_XML_TAG DEFAULT_XMLTODICT_SETTINGS: frozendict = frozendict( { @@ -28,39 +33,20 @@ ) -def make_settings(tmp_path: Path, dlt_destination_config: str, **overrides: Any) -> dict[str, Any]: - """Generate a dict of settings for the XmlToDict ingestion route.""" - log_config_file = overrides.get("log_config_file", tmp_path / "logging.conf") - log_config_file.touch() - output_dir = overrides.get("output_dir", tmp_path / "output") - output_dir.mkdir(exist_ok=True) - input_dir = overrides.get("input_dir", tmp_path / "input") - - kwargs: dict[str, Any] = { - **DEFAULT_XMLTODICT_SETTINGS, - "dataset_name": "xml_test_dataset", - "input_dir": str(input_dir), - "log_config_file": str(log_config_file), - "output_dir": str(output_dir), - "table_name": "book", - "use_destination": dlt_destination_config, - "xml_tag": "book", - } - kwargs.update(overrides) - return kwargs - - @pytest.fixture def settings_factory(tmp_path: Path, dlt_destination_config: str) -> Callable[..., XmlToDictSettings]: """Return a factory that builds a valid XmlToDictSettings, with fields open to override.""" - def _factory(**overrides: Any) -> XmlToDictSettings: - settings_dict = make_settings(tmp_path, dlt_destination_config, **overrides) - # ensure that the input dir exists - input_dir = Path(settings_dict["input_dir"]) - input_dir.mkdir(exist_ok=True) - - return XmlToDictSettings(**settings_dict) + def _factory(**overrides: Any) -> XmlToDictSettings: # noqa: ANN401 + defaults: dict[str, Any] = { + **DEFAULT_XMLTODICT_SETTINGS, + "dataset_name": "xml_test_dataset", + "table_name": "book", + "use_destination": dlt_destination_config, + "xml_tag": "book", + } + defaults.update(overrides) + return make_cts_settings(XmlToDictSettings, tmp_path, **defaults) return _factory @@ -77,16 +63,9 @@ def _resolve(scenario: str) -> str: return _resolve -@pytest.fixture(scope="session") -def reference_xml_data_dir() -> Path: - """Path to the uniref fixture directory.""" - return REFERENCE_XML_FIXTURE_DIR - - def import_jsonl_data(jsonl_path: Path) -> list[dict[str, Any]]: """Read the reference JSONL into a list of canonical nested entry dicts.""" - with jsonl_path.open(encoding="utf-8") as fh: - return [json.loads(line) for line in fh if line.strip()] + return read_jsonl_file(jsonl_path) def write_reference_xml_as_jsonl(source_xml: Path, jsonl_path: Path) -> int: @@ -140,19 +119,14 @@ def sorted_reference_xml_entries(reference_xml_entries: list[dict[str, Any]]) -> @pytest.fixture(scope="session") -def reference_xml_dataset(reference_xml_as_jsonl: Path) -> duckdb.DuckDBPyConnection: +def reference_xml_dataset(reference_xml_as_jsonl: Path) -> Generator[duckdb.DuckDBPyConnection]: """Load the reference JSONL into an in-memory DuckDB connection with nested structures. read_json_auto infers native STRUCT/STRUCT[] types for the nested content; `member` infers as JSON because its shape varies across entries. """ - connection = duckdb.connect() - jsonl_path = str(reference_xml_as_jsonl) - connection.execute( - "CREATE TABLE reference AS SELECT * FROM read_json_auto(?, sample_size=200)", - [jsonl_path], - ) - return connection + with duckdb_reference_from_jsonl(reference_xml_as_jsonl) as conn: + yield conn @pytest.fixture @@ -179,7 +153,7 @@ def run_xmltodict_pipeline_fxt( base_kwargs=base_kwargs, ) - def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 return runner(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xsv/conftest.py b/tests/cdm_data_loaders/pipelines/xsv/conftest.py index 7be34c86..bcd9ac69 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xsv/conftest.py @@ -12,6 +12,7 @@ from cdm_data_loaders.pipelines.xsv.pipeline import XsvWorkPaths from cdm_data_loaders.pipelines.xsv.settings import XsvIngestSettings from cdm_data_loaders.readers.jsonschema_xsv.xsv_validator.schema_utils import generate_header +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings VALID_SCHEMA_URI: Final[str] = "https://json-schema.org/draft/2020-12/schema" COLUMNS: Final[list[str]] = ["number", "date", "float", "boolean", "string"] @@ -71,71 +72,34 @@ def qsv_cmd() -> str: return cmd -def make_settings( - tmp_path: Path, - dlt_destination_config: str, - schema: dict[str, Any], - **overrides: Any, -) -> dict[str, Any]: - """Build a valid xsv ingest settings dict, with fields open to override.""" - log_config_file = overrides.pop("log_config_file", tmp_path / "logging.conf") - Path(log_config_file).touch(exist_ok=True) - output_dir = overrides.pop("output_dir", tmp_path / "output") - Path(output_dir).mkdir(exist_ok=True) - input_dir = overrides.pop("input_dir", tmp_path / "input") - Path(input_dir).mkdir(exist_ok=True) - (Path(input_dir) / "schema.json").write_text(json.dumps(schema)) - - kwargs: dict[str, Any] = { - "buffer_size": 10, - "dataset_name": "xsv_test_dataset", - "dlt_dev_mode": False, - "file_glob": "*", - "input_dir": str(input_dir), - "loader_file_format": "jsonl", - "log_config_file": str(log_config_file), - "log_interval": 1000, - "output_dir": str(output_dir), - "schema_file": str(Path(input_dir) / "schema.json"), - "table_name": "my_table", - "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, - } - kwargs.update(overrides) - return kwargs - - @pytest.fixture -def settings_factory(tmp_path: Path) -> Callable[..., XsvIngestSettings]: +def settings_factory(tmp_path: Path, dlt_destination_config: str) -> Callable[..., XsvIngestSettings]: """Return a factory that builds a valid XsvIngestSettings, with fields open to override.""" - def _factory(schema: dict[str, Any], data_files: dict[str, str], **overrides: object) -> XsvIngestSettings: - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - (input_dir / "schema.json").write_text(json.dumps(schema)) - for file_name, content in data_files.items(): - (input_dir / file_name).write_text(content) - - kwargs: dict[str, object] = { - "buffer_size": 100, + def _factory( + schema: dict[str, Any] | None = DEFAULT_XSV_SCHEMA, + data_files: dict[str, str] | None = None, + **overrides: Any, # noqa: ANN401 + ) -> XsvIngestSettings: + input_dir = Path(overrides.get("input_dir", tmp_path / "input")) + input_dir.mkdir(parents=True, exist_ok=True) + + if schema is not None: + (input_dir / "schema.json").write_text(json.dumps(schema), encoding="utf-8") + if data_files: + for file_name, content in data_files.items(): + (input_dir / file_name).write_text(content, encoding="utf-8") + + defaults: dict[str, Any] = { "dataset_name": "xsv_test_dataset", - "dlt_dev_mode": False, "file_glob": "*", - "input_dir": str(input_dir), "loader_file_format": "jsonl", - "log_config_file": None, - "log_interval": 1000, - "output_dir": str(output_dir), "schema_file": str(input_dir / "schema.json"), "table_name": "my_table", - "use_destination": "local_fs", - "use_output_dir_for_pipeline_metadata": False, + "use_destination": dlt_destination_config, } - kwargs.update(overrides) - return XsvIngestSettings(**kwargs) + defaults.update(overrides) + return make_cts_settings(XsvIngestSettings, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py deleted file mode 100644 index bd960c93..00000000 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/conftest.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Shared fixtures for xsv pipeline integration tests.""" - -import json -import os -import shutil -from collections.abc import Callable -from pathlib import Path -from typing import Any, Final - -import pytest - -from cdm_data_loaders.pipelines.xsv.settings import XsvIngestSettings - -VALID_SCHEMA_URI: Final[str] = "https://json-schema.org/draft/2020-12/schema" -COLUMNS: Final[list[str]] = ["number", "date", "float", "boolean", "string"] - -# The `float` column is deliberately typed `["string", "null"]`, not `"number"`, matching the -# convention used throughout the xsv_validator test fixtures: it lets a field hold either a -# numeric-looking string or a null placeholder without needing float-parsing semantics. -DEFAULT_XSV_SCHEMA: Final[dict[str, Any]] = { - "$schema": VALID_SCHEMA_URI, - "title": "XSV integration test schema", - "type": "object", - "required": COLUMNS, - "properties": { - "number": {"type": "integer"}, - "date": {"type": "string"}, - "float": {"type": ["string", "null"]}, - "boolean": {"type": "boolean"}, - "string": {"type": "string"}, - }, - "x-xsv-config": {"x-delimiter": ",", "x-has-header": True}, -} - -VALID_ROWS: Final[list[list[str]]] = [ - ["2", "2023-01-15", "3.14", "true", "key:value1"], - ["3", "2023-02-20", "1.11", "false", "key:value2"], -] - -# One ragged row (missing the trailing `string` field) among otherwise-valid rows: recoverable. -PARTIAL_RAGGED_ROWS: Final[list[list[str]]] = [ - ["2", "2023-01-15", "3.14", "true", "key:value1"], - ["3", "2023-02-20", "1.11", "false"], - ["4", "2023-03-25", "2.22", "true", "key:value3"], -] - -# Every row here is ragged (too few or too many fields): unrecoverable. -ALL_RAGGED_ROWS: Final[list[list[str]]] = [ - ["2", "2023-01-15", "3.14", "true"], - ["3", "2023-02-20", "1.11", "false", "key:value2", "extra"], -] - - -def rows_to_csv(rows: list[list[str]], header: list[str] = COLUMNS, delimiter: str = ",") -> str: - """Render rows (plus header) as delimited text content, one row per line.""" - lines = [delimiter.join(header), *(delimiter.join(row) for row in rows)] - return "\n".join(lines) + "\n" - - -@pytest.fixture(scope="session") -def qsv_cmd() -> str: - """Provide the path/name of the qsv binary; override via the QSV_BIN env var if not on PATH. - - Tests using this fixture are skipped when the qsv binary is not available. - """ - cmd = shutil.which("qsv") or os.environ.get("QSV_BIN") - if not cmd or not Path(cmd).is_file(): - pytest.skip(f"qsv binary not found at {cmd!r}; set QSV_BIN to point at a real binary") - return cmd - - -def make_settings( - tmp_path: Path, - dlt_destination_config: str, - schema: dict[str, Any], - **overrides: Any, -) -> dict[str, Any]: - """Build a valid xsv ingest settings dict, with fields open to override.""" - log_config_file = overrides.pop("log_config_file", tmp_path / "logging.conf") - Path(log_config_file).touch(exist_ok=True) - output_dir = overrides.pop("output_dir", tmp_path / "output") - Path(output_dir).mkdir(exist_ok=True) - input_dir = overrides.pop("input_dir", tmp_path / "input") - Path(input_dir).mkdir(exist_ok=True) - (Path(input_dir) / "schema.json").write_text(json.dumps(schema)) - - kwargs: dict[str, Any] = { - "buffer_size": 10, - "dataset_name": "xsv_test_dataset", - "dlt_dev_mode": False, - "file_glob": "*", - "input_dir": str(input_dir), - "loader_file_format": "jsonl", - "log_config_file": str(log_config_file), - "log_interval": 1000, - "output_dir": str(output_dir), - "schema_file": str(Path(input_dir) / "schema.json"), - "table_name": "my_table", - "use_destination": dlt_destination_config, - "use_output_dir_for_pipeline_metadata": False, - } - kwargs.update(overrides) - return kwargs - - -@pytest.fixture -def settings_factory(tmp_path: Path, dlt_destination_config: str) -> Callable[..., XsvIngestSettings]: - """Return a factory that builds a valid XsvIngestSettings, with fields open to override.""" - - def _factory(schema: dict[str, Any] = DEFAULT_XSV_SCHEMA, **overrides: Any) -> XsvIngestSettings: - settings_dict = make_settings(tmp_path, dlt_destination_config, schema, **overrides) - return XsvIngestSettings(**settings_dict) - - return _factory - - -@pytest.fixture -def write_xsv_file() -> Callable[[XsvIngestSettings, str, str], Path]: - """Return a function that writes text content to a new file inside settings.input_dir.""" - - def _write(settings: XsvIngestSettings, file_name: str, content: str) -> Path: - path = Path(settings.input_dir) / file_name - path.write_text(content) - return path - - return _write diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py index 425fe9fd..490734ff 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py @@ -11,7 +11,7 @@ from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written -from tests.cdm_data_loaders.pipelines.xsv.integration.conftest import ( +from tests.cdm_data_loaders.pipelines.xsv.conftest import ( ALL_RAGGED_ROWS, COLUMNS, DEFAULT_XSV_SCHEMA, diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py index 964ae24e..acd948ed 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py @@ -26,8 +26,7 @@ run_xsv_pipeline, ) from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings - -VALID_SCHEMA_URI = "https://json-schema.org/draft/2020-12/schema" +from tests.cdm_data_loaders.pipelines.xsv.conftest import VALID_SCHEMA_URI SIMPLE_SCHEMA: dict[str, Any] = { "$schema": VALID_SCHEMA_URI, diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py index a9bff090..e035b9b4 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_settings.py @@ -1,71 +1,14 @@ """Tests for XsvIngestSettings field validation, schema loading, and derived properties.""" -import json from collections.abc import Callable -from pathlib import Path -from typing import Any, Final +from typing import Any import pytest from pydantic import ValidationError from cdm_data_loaders.core.fields import DEFAULTS from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings - -VALID_SCHEMA_URI: Final[str] = "https://json-schema.org/draft/2020-12/schema" -COLUMNS: Final[list[str]] = ["number", "date", "float", "boolean", "string"] - -VALID_SCHEMA: Final[dict[str, Any]] = { - "$schema": VALID_SCHEMA_URI, - "title": "test schema", - "type": "object", - "required": COLUMNS, - "properties": { - "number": {"type": "integer"}, - "date": {"type": "string"}, - "float": {"type": ["string", "null"]}, - "boolean": {"type": "boolean"}, - "string": {"type": "string"}, - }, -} - -MINIMAL_SETTINGS_KWARGS: dict[str, object] = { - "buffer_size": 100, - "log_interval": 1000, - "dataset_name": "xsv_test_dataset", - "loader_file_format": "parquet", - "table_name": "my_table", - "log_config_file": None, - "dlt_dev_mode": False, - "use_destination": "local_fs", - "use_output_dir_for_pipeline_metadata": False, -} - - -@pytest.fixture -def settings_factory(tmp_path: Path) -> Callable[..., XsvIngestSettings]: - """Return a factory that builds a valid XsvIngestSettings, with fields open to override.""" - - def _factory(schema: dict[str, Any] | None = VALID_SCHEMA, **overrides: object) -> XsvIngestSettings: - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) - output_dir = tmp_path / "output" - output_dir.mkdir(exist_ok=True) - - if schema is not None: - (input_dir / "schema.json").write_text(json.dumps(schema)) - - kwargs: dict[str, object] = { - **MINIMAL_SETTINGS_KWARGS, - "input_dir": str(input_dir), - "output_dir": str(output_dir), - "file_glob": DEFAULTS["file_glob"], - "schema_file": str(input_dir / "schema.json"), - } - kwargs.update(overrides) - return XsvIngestSettings(**kwargs) - - return _factory - +from tests.cdm_data_loaders.pipelines.xsv.conftest import COLUMNS, DEFAULT_XSV_SCHEMA, VALID_SCHEMA_URI """defaults and basic construction""" @@ -149,7 +92,8 @@ def test_xsv_settings_pass_parsing_config_empty_without_xsv_config_block( settings_factory: Callable[..., XsvIngestSettings], ) -> None: """parsing_config is empty when the schema carries no x-xsv-config block.""" - settings = settings_factory() + schema_without_config = {k: v for k, v in DEFAULT_XSV_SCHEMA.items() if k != "x-xsv-config"} + settings = settings_factory(schema=schema_without_config) assert settings.parsing_config == {} @@ -157,8 +101,7 @@ def test_xsv_settings_pass_parsing_config_resolved_from_xsv_config_block( settings_factory: Callable[..., XsvIngestSettings], ) -> None: """parsing_config resolves qsv kwargs from the schema's x-xsv-config block.""" - schema_with_config = {**VALID_SCHEMA, "x-xsv-config": {"x-delimiter": ",", "x-has-header": True}} - settings = settings_factory(schema=schema_with_config) + settings = settings_factory() assert settings.parsing_config == {"delimiter": ",", "missing_header": False} diff --git a/tests/jsonschema_fixtures.py b/tests/jsonschema_fixtures.py index c8e2abef..bc4f1273 100644 --- a/tests/jsonschema_fixtures.py +++ b/tests/jsonschema_fixtures.py @@ -12,6 +12,7 @@ import pytest from cdm_data_loaders.pipelines.jsonlines.settings import JsonlJsonschemaIngestSettings +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings @pytest.fixture @@ -59,27 +60,20 @@ def schema_settings_factory( def _factory(**overrides: object) -> JsonlJsonschemaIngestSettings: run_index = next(run_counter) - input_dir = tmp_path / "input" - input_dir.mkdir(exist_ok=True) output_dir = tmp_path / f"output_{run_index}" output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1, "disable_existing_loggers": false}', encoding="utf-8") - kwargs = { - "input_dir": str(input_dir), + defaults: dict[str, Any] = { "output_dir": str(output_dir), - "log_config_file": str(log_config_file), "use_destination": dlt_destination_config, "use_output_dir_for_pipeline_metadata": True, - "dlt_dev_mode": False, "dataset_name": "schema_test", "file_glob": "*.jsonl*", "loader_file_format": "parquet", "buffer_size": 2, "schema_files_module": schema_module, } - kwargs.update(overrides) - settings = JsonlJsonschemaIngestSettings(**kwargs) + defaults.update(overrides) + settings = make_cts_settings(JsonlJsonschemaIngestSettings, tmp_path, **defaults) dlt.config["destination.local_fs.bucket_url"] = str(settings.output_dir) return settings From ae3fe49ba2b696fb0e72d7f35f445862f5183867 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 10:47:09 -0700 Subject: [PATCH 04/16] remove make_file_item dupes and manual duckdb pipeline boilerplate; remove duped duckdb helpers; standardize helpers --- .../integration/test_jsonlines_end_to_end.py | 51 +++----- .../integration/test_jsonschema_end_to_end.py | 60 +++++----- .../integration/test_reference_end_to_end.py | 11 +- .../jsonlines/test_extract_pipeline.py | 53 ++------- .../uniprot/test_uniprot_end_to_end.py | 30 ++--- .../uniref/test_uniref_end_to_end.py | 32 ++--- .../integration/test_xml2db_end_to_end.py | 52 +++------ .../pipelines/xml2db/test_xml2db_pipeline.py | 67 +++-------- .../integration/test_xmltodict_end_to_end.py | 54 +++------ .../xsv/integration/test_xsv_end_to_end.py | 60 +++------- .../pipelines/xsv/test_xsv_pipeline.py | 110 ++++-------------- 11 files changed, 165 insertions(+), 415 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py index ec0b0e3e..8b62a569 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_end_to_end.py @@ -10,7 +10,11 @@ ) from cdm_data_loaders.pipelines.jsonlines.settings import PYDANTIC_PIPELINE_NAME, JsonlPydanticIngestSettings from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) def test_run_jsonlines_pipeline_pass_writes_expected_parquet( @@ -21,13 +25,13 @@ def test_run_jsonlines_pipeline_pass_writes_expected_parquet( """Running the pipeline on a mixed batch puts the correct rows in both output tables.""" settings = settings_factory(input_dir=scenario_input_dir("mixed")) - load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings) - assert load_info is not None + load_info = assert_run_ok(pipeline_runner.run(PYDANTIC_PIPELINE_NAME, settings=settings)) dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "c"] assert len(dataset.widget_rejected.df()) == 2 assert_pipeline_report_written(Path(settings.output_dir), load_info) + # FIXME: remove or improve # get stats on tables # dataset.row_counts(table_names=dataset.tables).fetchall() @@ -57,40 +61,19 @@ def test_run_jsonlines_pipeline_pass_table_names_none_processes_every_registered def test_cli_pass_runs_end_to_end_from_command_line_arguments( scenario_input_dir: Callable[[str], str], - dlt_destination_config: str, - entity_models_module: str, - tmp_path: Path, + settings_factory: Callable[..., JsonlPydanticIngestSettings], pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - output_dir = tmp_path / "output" - output_dir.mkdir() - - argv = [ - PYDANTIC_PIPELINE_NAME, - "--input-dir", - scenario_input_dir("happy_path"), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - dlt_destination_config, - "--entity-models-module", - entity_models_module, - "--dataset-name", - "some_cool_dataset", - "--use-output-dir-for-pipeline-metadata", - "true", - ] - - load_info = pipeline_runner.run(PYDANTIC_PIPELINE_NAME, cli_args=argv[1:]) - - assert load_info is not None - assert not load_info.has_failed_jobs + settings = settings_factory( + input_dir=scenario_input_dir("happy_path"), + dataset_name="some_cool_dataset", + dlt_dev_mode=False, + use_output_dir_for_pipeline_metadata=True, + ) + + load_info = assert_run_ok(pipeline_runner.run(PYDANTIC_PIPELINE_NAME, cli_args=settings_to_argv(settings))) assert load_info.pipeline.pipeline_name == PYDANTIC_PIPELINE_NAME dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "b", "c"] - assert_pipeline_report_written(output_dir, load_info, metadata_moved=True) + assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py index dfa873de..c67487e3 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonschema_end_to_end.py @@ -13,6 +13,7 @@ from tests.cdm_data_loaders.pipelines.helpers import ( LOADER_FILE_FORMATS, assert_pipeline_report_written, + assert_run_ok, parse_json_string, read_data_rows, ) @@ -64,10 +65,7 @@ def test_run_jsonlines_pipeline_pass_persists_valid_and_rejected_rows( input_dir=str(schema_input_dir), loader_file_format=loader_file_format, buffer_size=buffer_size ) - load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings)) assert load_info.pipeline.pipeline_name == JSONSCHEMA_PIPELINE_NAME assert load_info.dataset_name == "schema_test" assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) @@ -111,10 +109,7 @@ def test_run_jsonlines_pipeline_pass_all_invalid( table_dir.mkdir() (table_dir / "invalid.jsonl").write_text(f"{MALFORMED_RAW}\n{TYPE_ERROR_RAW}", encoding="utf-8") - load_info = pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run(JSONSCHEMA_PIPELINE_NAME, settings=settings)) assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) assert read_data_tables(settings, load_info.dataset_name) == { "widget_rejected": [ @@ -138,32 +133,31 @@ def test_cli_pass_loads_selected_entity( """The real CTS CLI accepts a registry shortcut, table selection, and the requested file format.""" settings = schema_settings_factory(input_dir=str(schema_input_dir), loader_file_format=loader_file_format) - load_info = pipeline_runner.run( - JSONSCHEMA_PIPELINE_NAME, - cli_args=[ - "--input-dir", - str(settings.input_dir), - "--output-dir", - str(settings.output_dir), - "--log-config-file", - str(settings.log_config_file), - "--use-destination", - str(settings.use_destination), - "--use-output-dir-for-pipeline-metadata", - "true", - "--loader-file-format", - loader_file_format, - "--dataset-name", - "schema_cli", - "-m", - settings.schema_files_module, - "--table-names", - "gadget", - ], + load_info = assert_run_ok( + pipeline_runner.run( + JSONSCHEMA_PIPELINE_NAME, + cli_args=[ + "--input-dir", + str(settings.input_dir), + "--output-dir", + str(settings.output_dir), + "--log-config-file", + str(settings.log_config_file), + "--use-destination", + str(settings.use_destination), + "--use-output-dir-for-pipeline-metadata", + "true", + "--loader-file-format", + loader_file_format, + "--dataset-name", + "schema_cli", + "-m", + settings.schema_files_module, + "--table-names", + "gadget", + ], + ) ) - - assert load_info is not None - assert not load_info.has_failed_jobs assert load_info.dataset_name == "schema_cli" assert_pipeline_report_written(Path(settings.output_dir), load_info, metadata_moved=True) assert read_data_tables(settings, load_info.dataset_name) == {"gadget": [GADGET_RECORD]} diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 31140384..36ffd705 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -13,7 +13,12 @@ PYDANTIC_PIPELINE_NAME, ) from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, read_pipeline_tables, run_and_read +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + assert_run_ok, + read_pipeline_tables, + run_and_read, +) from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( canonicalize_pipeline_entry, canonicalize_reference_entry, @@ -303,9 +308,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( ] if registry_flag: argv.extend([f"--{registry_flag}", registry_module]) - load_info = pipeline_runner.run(pipeline_kind, cli_args=argv[1:]) - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run(pipeline_kind, cli_args=argv[1:])) tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) assert ( sorted( diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py index 3c7513a1..25ed1c9f 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py @@ -16,7 +16,11 @@ from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import cli, run_jsonlines_pipeline from cdm_data_loaders.pipelines.jsonlines.settings import EXTRACT_PIPELINE_NAME, JsonlExtractSettings from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) SIMPLE_JSONL = """{"widget_id": "a", "count": 1} {"widget_id": "b", "count": 2} @@ -70,10 +74,8 @@ def test_run_jsonlines_pipeline_pass_routes_lines_by_directory( (input_dir / "gadget").mkdir() (input_dir / "gadget" / "data.jsonl").write_text('{"widget_id": "g", "count": 0}\n', encoding="utf-8") - load_info = run_jsonlines_pipeline(settings) + load_info = assert_run_ok(run_jsonlines_pipeline(settings)) - assert load_info is not None - assert not load_info.has_failed_jobs dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a", "b", "g"] assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -88,10 +90,8 @@ def test_run_jsonlines_pipeline_pass_invalid_rows_routed_to_invalid_table( (input_dir / "widget").mkdir(parents=True) (input_dir / "widget" / "data.jsonl").write_text('{"widget_id": "a"}\n{broken\n', encoding="utf-8") - load_info = run_jsonlines_pipeline(settings) + load_info = assert_run_ok(run_jsonlines_pipeline(settings)) - assert load_info is not None - assert not load_info.has_failed_jobs dataset = load_info.pipeline.dataset() assert sorted(dataset.widget.df()["widget_id"].tolist()) == ["a"] invalid_df = dataset.widget_invalid.df() @@ -107,10 +107,8 @@ def test_run_jsonlines_pipeline_pass_no_matching_files_yields_no_tables( settings = extract_settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = run_jsonlines_pipeline(settings) + load_info = assert_run_ok(run_jsonlines_pipeline(settings)) - assert load_info is not None - assert not load_info.has_failed_jobs dataset = load_info.pipeline.dataset() assert set(dataset.tables) == {"_dlt_version", "_dlt_loads", "_dlt_pipeline_state", "_load_info"} assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -122,34 +120,9 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - settings = extract_settings_factory() - input_dir = Path(settings.input_dir) - (input_dir / "data.jsonl").write_text(SIMPLE_JSONL, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - EXTRACT_PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - str(settings.use_destination), - "--dataset-name", - "cli_dataset", - "--table-name", - "widget", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = extract_settings_factory(dataset_name="cli_dataset", dlt_dev_mode=False) + (Path(settings.input_dir) / "data.jsonl").write_text(SIMPLE_JSONL, encoding="utf-8") + monkeypatch.setattr(sys, "argv", [EXTRACT_PIPELINE_NAME, *settings_to_argv(settings)]) def fake_run_pipeline( *, resource: DltResource, pipeline_kwargs: dict[str, Any], **_: dict[str, Any] @@ -159,10 +132,8 @@ def fake_run_pipeline( return pipeline.run(resource) with patch.object(extract_pipeline_module, "run_pipeline", fake_run_pipeline): - load_info = cli.run() + load_info = assert_run_ok(cli.run()) - assert load_info is not None - assert not load_info.has_failed_jobs # n.b. no _load_info due to this being a monkeypatched pipeline assert set(load_info.pipeline.dataset().tables) == {"widget", "_dlt_version", "_dlt_loads", "_dlt_pipeline_state"} assert sorted(load_info.pipeline.dataset().widget.df()["widget_id"].tolist()) == ["a", "b"] diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py index 49103b80..0b3e4fe9 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py @@ -16,7 +16,11 @@ ) from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + fetch_all_duckdb, +) from tests.conftest import LOCAL_FS, TEST_DATA_DIR UNIPROT_FIXTURE_DIR = TEST_DATA_DIR / "uniprot" / "uniprot_kb" / "chunk_4" @@ -63,15 +67,6 @@ def _run_uniprot_duckdb_pipeline(settings: UniProtSettings, tmp_path: Path, name return pipeline, load_info -def _fetch_all(pipeline: Any, query: str) -> list[tuple[Any, ...]]: - """Run a query against the pipeline's DuckDB destination and return all rows.""" - with ( - pipeline.sql_client() as client, - client.execute_query(query) as cur, - ): - return list(cur.fetchall()) - - def test_integration_uniprot_pipeline_loads_into_duckdb( duckdb_uniprot_settings: UniProtSettings, tmp_path: Path, @@ -83,10 +78,10 @@ def test_integration_uniprot_pipeline_loads_into_duckdb( """ pipeline, load_info = _run_uniprot_duckdb_pipeline(duckdb_uniprot_settings, tmp_path, "loads") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): - ((count,),) = _fetch_all(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 + ((count,),) = fetch_all_duckdb(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 assert count == expected_count, f"expected {expected_count} rows in '{table_name}', got {count}" @@ -97,16 +92,16 @@ def test_integration_uniprot_pipeline_entity_and_source_file_content( """Entity rows are uniprot proteins and source-file rows reference every fixture file.""" pipeline, load_info = _run_uniprot_duckdb_pipeline(duckdb_uniprot_settings, tmp_path, "content") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) # every entity must have a "uniprot:"-prefixed entity_id and be a protein. - rows = _fetch_all(pipeline, "SELECT entity_id, entity_type FROM entity") + rows = fetch_all_duckdb(pipeline, "SELECT entity_id, entity_type FROM entity") assert len(rows) == UNIPROT_ENTRY_COUNT assert all(entity_id.startswith("uniprot:") for entity_id, _ in rows) assert all(entity_type == "protein" for _, entity_type in rows) # entity_x_source_file must reference exactly the fixture files we loaded from - source_files = {row[0] for row in _fetch_all(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} assert source_files == {str(UNIPROT_FIXTURE_DIR.resolve() / name) for name in UNIPROT_FIXTURE_FILES} @@ -115,10 +110,7 @@ def test_integration_cli_uniprot_pipeline_output_validated( pipeline_runner: PipelineTestRunner, ) -> None: """Exercise the UniProt pipeline end-to-end across core, cli, and docker execution modes.""" - load_info = pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings)) assert load_info.pipeline.pipeline_name == "uniprot" assert load_info.dataset_name == "uniprot" assert_pipeline_report_written(Path(duckdb_uniprot_settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py index cf047063..6c80fd39 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py @@ -17,7 +17,11 @@ ) from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline, read_destination_rows from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + fetch_all_duckdb, +) from tests.conftest import LOCAL_FS, TEST_DATA_DIR TEST_DEFAULT_UNIREF_VARIANT = "50" @@ -64,15 +68,6 @@ def _run_uniref_duckdb_pipeline(settings: UnirefSettings, tmp_path: Path, name: return pipeline, load_info -def _fetch_all(pipeline: Any, query: str) -> list[tuple[Any, ...]]: - """Run a query against the pipeline's DuckDB destination and return all rows.""" - with ( - pipeline.sql_client() as client, - client.execute_query(query) as cur, - ): - return list(cur.fetchall()) - - def test_integration_uniref_pipeline_loads_into_duckdb( duckdb_uniref_settings: UnirefSettings, tmp_path: Path, @@ -84,10 +79,10 @@ def test_integration_uniref_pipeline_loads_into_duckdb( """ pipeline, load_info = _run_uniref_duckdb_pipeline(duckdb_uniref_settings, tmp_path, "loads") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) for table_name, expected_count in EXPECTED_ROW_COUNTS.items(): - ((count,),) = _fetch_all(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 + ((count,),) = fetch_all_duckdb(pipeline, f"SELECT COUNT(*) FROM {table_name}") # noqa: S608 assert count == expected_count, f"expected {expected_count} rows in '{table_name}', got {count}" @@ -98,20 +93,20 @@ def test_integration_uniref_pipeline_entity_and_cluster_content( """Entity rows are UniRef clusters and every cluster carries the injected protocol label.""" pipeline, load_info = _run_uniref_duckdb_pipeline(duckdb_uniref_settings, tmp_path, "content") - assert not load_info.has_failed_jobs + assert_run_ok(load_info) # every entity must have a "uniref:"-prefixed entity_id and be a Cluster. - rows = _fetch_all(pipeline, "SELECT entity_id, entity_type FROM entity") + rows = fetch_all_duckdb(pipeline, "SELECT entity_id, entity_type FROM entity") assert len(rows) == UNIREF_CLUSTER_COUNT assert all(entity_id.startswith("uniref:") for entity_id, _ in rows) assert all(entity_type == "Cluster" for _, entity_type in rows) # every cluster must carry the injected "UniRef " protocol label - protocols = {row[0] for row in _fetch_all(pipeline, "SELECT DISTINCT protocol FROM cluster")} + protocols = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT protocol FROM cluster")} assert protocols == {f"UniRef {duckdb_uniref_settings.variant}"} # entity_x_source_file must reference exactly the fixture file we loaded from - source_files = {row[0] for row in _fetch_all(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} assert source_files == {str(UNIREF_FIXTURE_DIR.resolve() / UNIREF_FIXTURE_FILE)} @@ -120,10 +115,7 @@ def test_integration_cli_uniref_pipeline_output_validated( pipeline_runner: PipelineTestRunner, ) -> None: """Exercise the UniRef pipeline end-to-end across core, cli, and docker execution modes.""" - load_info = pipeline_runner.run("uniref", settings=duckdb_uniref_settings) - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(pipeline_runner.run("uniref", settings=duckdb_uniref_settings)) assert load_info.pipeline.pipeline_name == "uniref" assert load_info.dataset_name == f"uniref_{duckdb_uniref_settings.variant}" assert_pipeline_report_written(Path(duckdb_uniref_settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py index 7e963918..0b4d715d 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_end_to_end.py @@ -9,7 +9,11 @@ from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) from tests.xml_samples import SIMPLE_LIBRARY_XML EXPECTED_BOOK_COUNT = 3 @@ -18,8 +22,7 @@ def check_book_list_results(load_info: LoadInfo | None) -> tuple[DataFrame, DataFrame]: """Check over the output of parsing SIMPLE_LIBRARY_XML: one root row, three book rows.""" - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(load_info) dataset = load_info.pipeline.dataset() assert {"library", "book", "library_book", *DEFAULT_DLT_TABLES} == set(dataset.tables) book_df = dataset.table("book").df() @@ -80,9 +83,7 @@ def test_run_xml2db_pipeline_pass_no_matching_files_yields_no_data_table( settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) dataset = load_info.pipeline.dataset() # only dlt's own metadata tables exist; the data tables are never created assert set(dataset.tables) == DEFAULT_DLT_TABLES @@ -111,42 +112,17 @@ def test_run_xml2db_pipeline_pass_custom_short_name_used_for_root_table( def test_cli_pass_runs_end_to_end_from_command_line_arguments( - tmp_path: Path, - dlt_destination_config: str, - library_xsd: Path, + settings_factory: Callable[..., Xml2DbSettings], pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - input_dir = tmp_path / "cli_input" - input_dir.mkdir() + settings = settings_factory(dataset_name="cli_dataset", use_output_dir_for_pipeline_metadata=False) + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "bucket" - output_dir.mkdir(exist_ok=True) - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - dlt_destination_config, - "--dataset-name", - "cli_dataset", - "--xsd-file", - str(library_xsd), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - - load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) library_df, book_df = check_book_list_results(load_info) assert len(library_df) == 1 assert sorted(book_df["id"].tolist()) == ["1", "2", "3"] - assert_pipeline_report_written(output_dir, load_info) + assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py index 09fa3859..a3d390af 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/test_xml2db_pipeline.py @@ -7,10 +7,8 @@ from unittest.mock import MagicMock, patch from uuid import uuid4 -import pendulum import pytest from dlt.common.pipeline import LoadInfo -from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.extract import DltResource from xml2db import DataModel @@ -20,7 +18,13 @@ from cdm_data_loaders.pipelines.xml2db.settings import PIPELINE_NAME, Xml2DbSettings from cdm_data_loaders.readers.xml2db_doc import build_xml2db_model from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline -from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_run_ok, + fetch_all_duckdb, + make_cts_settings, + make_file_item, + settings_to_argv, +) SIMPLE_LIBRARY_XML = """ @@ -63,20 +67,6 @@ def library_model(library_xsd: Path) -> DataModel: return build_xml2db_model(library_xsd, short_name="library_test") -def make_file_item(file_path: Path) -> FileItemDict: - """Build a dlt FileItemDict for a local file, as the filesystem source would.""" - return FileItemDict( - { - "file_url": file_path.as_uri(), - "file_name": file_path.name, - "relative_path": file_path.name, - "mime_type": "application/xml", - "modification_date": pendulum.now(), - "size_in_bytes": file_path.stat().st_size, - } - ) - - def fake_settings( tmp_path: Path, xsd_file: Path, @@ -321,39 +311,13 @@ def test_cli_pass_calls_run_cli_with_settings_class_and_pipeline_fn() -> None: def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, + library_xsd: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and loads rows via a real DuckDB pipeline.""" - input_dir = tmp_path / "cli_input" - input_dir.mkdir() - (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - xsd_file = tmp_path / "library.xsd" - xsd_file.write_text(LIBRARY_XSD, encoding="utf-8") - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}', encoding="utf-8") - - argv = [ - "xml2db", - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - "--dataset-name", - "cli_dataset", - "--xsd-file", - str(xsd_file), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = fake_settings(tmp_path, library_xsd, dataset_name="cli_dataset") + (Path(settings.input_dir) / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["xml2db", *settings_to_argv(settings)]) # unique pipeline name: the duckdb database file is derived from it, so # repeated runs of the suite do not see rows from previous runs @@ -373,13 +337,8 @@ def fake_run_pipeline( with patch.object(xml2db_pipeline_module, "run_pipeline", fake_run_pipeline): cli() - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + assert_run_ok(captured["load_info"]) pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM book") as cur, - ): - (row_count,) = cur.fetchone() + ((row_count,),) = fetch_all_duckdb(pipeline, "SELECT COUNT(*) FROM book") assert row_count == 2 # noqa: PLR2004 diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py index b76d2803..4db0e66d 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py @@ -10,14 +10,18 @@ from cdm_data_loaders.pipelines.xmltodict.settings import PIPELINE_NAME, XmlToDictSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) from tests.xml_samples import SIMPLE_LIBRARY_XML def check_book_list_results(load_info: LoadInfo | None, table_name: str | None = None) -> DataFrame: """Check over the output of parsing SIMPLE_LIBRARY_XML.""" - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(load_info) dataset = load_info.pipeline.dataset() table_name = table_name or "book" assert set(dataset.tables) == {table_name, *DEFAULT_DLT_TABLES} @@ -70,9 +74,7 @@ def test_run_xmltodict_pipeline_pass_no_matching_files_yields_no_data_table( settings = settings_factory() Path(settings.input_dir).mkdir(exist_ok=True) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) - assert load_info is not None - assert load_info.has_failed_jobs is False + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) dataset = load_info.pipeline.dataset() # only dlt's own metadata tables exist; the data table is never created assert set(dataset.tables) == DEFAULT_DLT_TABLES @@ -96,42 +98,16 @@ def test_run_xmltodict_pipeline_pass_custom_table_name_is_respected( def test_cli_pass_runs_end_to_end_from_command_line_arguments( - tmp_path: Path, - dlt_destination_config: str, + settings_factory: Callable[..., XmlToDictSettings], pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - input_dir = tmp_path / "cli_input" - input_dir.mkdir() + settings = settings_factory(dataset_name="cli_dataset") + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--log-config-file", - str(log_config_file), - "--use-destination", - dlt_destination_config, - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - - load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] - assert_pipeline_report_written(output_dir, load_info) + assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py index 490734ff..17e7ef49 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xsv/integration/test_xsv_end_to_end.py @@ -10,7 +10,11 @@ from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner -from tests.cdm_data_loaders.pipelines.helpers import assert_pipeline_report_written +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, +) from tests.cdm_data_loaders.pipelines.xsv.conftest import ( ALL_RAGGED_ROWS, COLUMNS, @@ -34,10 +38,8 @@ def test_run_xsv_pipeline_pass_writes_expected_output( settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() assert {"my_table", *DEFAULT_DLT_TABLES} == set(dataset.tables) @@ -59,10 +61,8 @@ def test_run_xsv_pipeline_pass_multiple_files_loaded_together( write_xsv_file(settings, "part_1.csv", rows_to_csv([VALID_ROWS[0]])) write_xsv_file(settings, "part_2.csv", rows_to_csv([VALID_ROWS[1]])) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df() assert sorted(df["number"].tolist()) == [2, 3] @@ -81,10 +81,8 @@ def test_run_xsv_pipeline_pass_partial_ragged_file_recovers_valid_rows( settings = settings_factory() write_xsv_file(settings, "data.csv", rows_to_csv(PARTIAL_RAGGED_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() assert {"my_table", REJECTED_TABLE, *DEFAULT_DLT_TABLES} == set(dataset.tables) @@ -106,10 +104,8 @@ def test_run_xsv_pipeline_pass_fully_ragged_file_is_entirely_rejected( settings = settings_factory() write_xsv_file(settings, "bad.csv", rows_to_csv(ALL_RAGGED_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False dataset = load_info.pipeline.dataset() assert set(dataset.tables) == {REJECTED_TABLE, *DEFAULT_DLT_TABLES} @@ -126,10 +122,8 @@ def test_run_xsv_pipeline_pass_no_matching_files_yields_no_data_table( """An input dir with no matching data files does not fail the run and creates no data table.""" settings = settings_factory() - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False assert set(load_info.pipeline.dataset().tables) == DEFAULT_DLT_TABLES assert_pipeline_report_written(Path(settings.output_dir), load_info) @@ -153,10 +147,8 @@ def test_run_xsv_pipeline_pass_configured_null_regex_replaces_placeholder( ), ) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df().sort_values("number") floats = df["float"].tolist() assert pd.isna(floats[0]) @@ -177,10 +169,8 @@ def test_run_xsv_pipeline_pass_parquet_format_loads_expected_row_count( settings = settings_factory(loader_file_format="parquet") write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df() assert len(df) == len(VALID_ROWS) @@ -191,33 +181,11 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( pipeline_runner: PipelineTestRunner, ) -> None: """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" - settings = settings_factory(dataset_name=f"cli_dataset_{uuid4().hex}") + settings = settings_factory(dataset_name=f"cli_dataset_{uuid4().hex}", dlt_dev_mode=False) write_xsv_file(settings, "data.csv", rows_to_csv(VALID_ROWS)) - argv: list[str] = [ - PIPELINE_NAME, - "--input-dir", - str(settings.input_dir), - "--output-dir", - str(settings.output_dir), - "--use-destination", - settings.use_destination, - "--dataset-name", - settings.dataset_name, - "--schema-file", - str(settings.schema_file), - "--table-name", - "my_table", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - - load_info = pipeline_runner.run(PIPELINE_NAME, cli_args=argv[1:]) + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) - assert load_info is not None - assert load_info.has_failed_jobs is False df = load_info.pipeline.dataset().table("my_table").df() assert sorted(df["number"].tolist()) == [2, 3] assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py index acd948ed..7d917138 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py @@ -11,10 +11,7 @@ from unittest.mock import patch from uuid import uuid4 -import dlt -import pendulum import pytest -from dlt.common.storages.fsspec_filesystem import FileItemDict import cdm_data_loaders.pipelines.xsv.pipeline as xsv_pipeline_module from cdm_data_loaders.pipelines.core import PipelineCli @@ -26,6 +23,13 @@ run_xsv_pipeline, ) from cdm_data_loaders.pipelines.xsv.settings import PIPELINE_NAME, XsvIngestSettings +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.cdm_data_loaders.pipelines.helpers import ( + assert_run_ok, + fetch_all_duckdb, + make_file_item, + settings_to_argv, +) from tests.cdm_data_loaders.pipelines.xsv.conftest import VALID_SCHEMA_URI SIMPLE_SCHEMA: dict[str, Any] = { @@ -39,20 +43,6 @@ """_process_xsv_file""" -def make_file_item(file_path: Path) -> FileItemDict: - """Build a dlt FileItemDict for a local file.""" - return FileItemDict( - { - "file_url": file_path.as_uri(), - "file_name": file_path.name, - "relative_path": file_path.name, - "mime_type": "text/csv", - "modification_date": pendulum.now(), - "size_in_bytes": file_path.stat().st_size, - } - ) - - def test_process_xsv_file_pass_routes_valid_rows_to_table( settings_factory: Callable[..., XsvIngestSettings], work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths], @@ -123,8 +113,6 @@ def test_run_xsv_pipeline_pass_end_to_end_via_duckdb( pipeline_name = f"test_xsv_pipeline_{uuid4().hex}" captured: dict[str, Any] = {} - output_dir = tmp_path / "duckdb_output" - output_dir.mkdir() def fake_run_pipeline( *, @@ -133,31 +121,16 @@ def fake_run_pipeline( **_: Any, # noqa: ANN401 ) -> None: assert pipeline_kwargs == {"pipeline_name": PIPELINE_NAME, "dataset_name": "xsv_test_dataset"} - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{output_dir!s}/{pipeline_name}.db"), - dataset_name="xsv_test_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "xsv_test_dataset") captured["load_info"] = pipeline.run(resource) with patch.object(xsv_pipeline_module, "run_pipeline", fake_run_pipeline): run_xsv_pipeline(settings) - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + assert_run_ok(captured["load_info"]) - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{output_dir!s}/{pipeline_name}.db"), - dataset_name="xsv_test_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT number, name FROM my_table ORDER BY number") as cur, - ): - rows = cur.fetchall() + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "xsv_test_dataset") + rows = fetch_all_duckdb(pipeline, "SELECT number, name FROM my_table ORDER BY number") assert [tuple(row) for row in rows] == [(1, "alice"), (2, "bob")] @@ -173,19 +146,13 @@ def test_run_xsv_pipeline_pass_no_matching_files_loads_nothing( captured: dict[str, Any] = {} def fake_run_pipeline(*, resource: Any, **_: Any) -> None: # noqa: ANN401 - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{settings.output_dir}/{pipeline_name}.db"), - dataset_name="xsv_test_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "xsv_test_dataset") captured["load_info"] = pipeline.run(resource) with patch.object(xsv_pipeline_module, "run_pipeline", fake_run_pipeline): run_xsv_pipeline(settings) - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + load_info = assert_run_ok(captured["load_info"]) assert "my_table" not in load_info.pipeline.default_schema.tables @@ -206,28 +173,12 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and loads rows via a real DuckDB pipeline.""" - settings = settings_factory(SIMPLE_SCHEMA, {"data.csv": "number,name\n1,alice\n2,bob\n"}) - - argv = [ - "xsv", - "--input-dir", - str(settings.input_dir), - "--output-dir", - str(settings.output_dir), - "--use-destination", - "local_fs", - "--dataset-name", - "cli_dataset", - "--schema-file", - str(settings.schema_file), - "--table-name", - "my_table", - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory( + SIMPLE_SCHEMA, + {"data.csv": "number,name\n1,alice\n2,bob\n"}, + dataset_name="cli_dataset", + ) + monkeypatch.setattr(sys, "argv", ["xsv", *settings_to_argv(settings)]) pipeline_name = f"test_xsv_cli_pipeline_{uuid4().hex}" captured: dict[str, Any] = {} @@ -239,29 +190,14 @@ def fake_run_pipeline( **_: Any, # noqa: ANN401 ) -> None: assert pipeline_kwargs == {"pipeline_name": PIPELINE_NAME, "dataset_name": "cli_dataset"} - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{settings.output_dir}/{pipeline_name}.db"), - dataset_name="cli_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") captured["load_info"] = pipeline.run(resource) with patch.object(xsv_pipeline_module, "run_pipeline", fake_run_pipeline): cli.run() - load_info = captured["load_info"] - assert not load_info.has_failed_jobs + assert_run_ok(captured["load_info"]) - pipeline = dlt.pipeline( - pipeline_name=pipeline_name, - destination=dlt.destinations.duckdb(f"duckdb:///{settings.output_dir}/{pipeline_name}.db"), - dataset_name="cli_dataset", - pipelines_dir=str(tmp_path / "pipelines"), - ) - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM my_table") as cur, - ): - (row_count,) = cur.fetchone() + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") + ((row_count,),) = fetch_all_duckdb(pipeline, "SELECT COUNT(*) FROM my_table") assert row_count == 2 # noqa: PLR2004 From a6c785595a4c25dff9c816c2e4b025016408917e Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 11:14:49 -0700 Subject: [PATCH 05/16] consolidate entry canonicalisation and variation helps --- tests/cdm_data_loaders/pipelines/helpers.py | 27 +++++++ .../jsonlines/integration/conftest.py | 4 +- .../jsonlines_reference_helpers.py | 14 +--- .../test_jsonlines_reference_tests.py | 78 ++++++++----------- .../integration/test_reference_end_to_end.py | 13 ++-- .../test_xml2db_reference_tests.py | 53 +++++-------- .../test_xmltodict_reference_tests.py | 52 ++++++------- 7 files changed, 119 insertions(+), 122 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index 4c7bf4ed..dd02defe 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -501,3 +501,30 @@ def assert_dataset_matches_reference( for value in reconstructed.values(): assert value == sorted_reference_json + + +def run_and_compare_variations( + run_pipeline: Callable[..., Any], + variations: dict[str, tuple[str, dict[str, Any]]], + *, + sorted_reference_json: list[str] | None = None, + output_format: str | None = None, + reconstruct: Callable[[dict[str, list[dict[str, Any]]]], list[dict[str, Any]]] = reconstruct_entries, + canonicalize: Callable[[dict[str, Any]], dict[str, Any]] | None = None, +) -> dict[str, tuple[LoadInfo, Path]]: + """Run a pipeline across named (chunk_dir, overrides) variations, assert equal datasets, and optionally match reference.""" + load_info_and_dir = { + key: run_and_read(run_pipeline, chunk_dir, **overrides) + for key, (chunk_dir, overrides) in variations.items() + } + if len(load_info_and_dir) >= 2: # noqa: PLR2004 + assert_datasets_equal(load_info_and_dir) + if sorted_reference_json is not None: + assert_dataset_matches_reference( + load_info_and_dir, + sorted_reference_json, + output_format, + reconstruct=reconstruct, + canonicalize=canonicalize, + ) + return load_info_and_dir diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py index a352c32f..2387264b 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/conftest.py @@ -29,7 +29,7 @@ from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models import ENTITY_MODELS from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas import ENTITY_SCHEMAS from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( - canonicalize_reference_entry, + canonicalize_entry, ) from tests.conftest import TEST_DATA_DIR @@ -158,7 +158,7 @@ def sorted_reference_entities( ) -> list[str]: """Canonicalized entries as sorted JSON strings for order-insensitive comparison.""" lines_as_json = [ - json.dumps(canonicalize_reference_entry(prepare_reference(obj)), sort_keys=True, default=str) + json.dumps(canonicalize_entry(prepare_reference(obj)), sort_keys=True, default=str) for obj in jsonl_reference_entities ] return sorted(lines_as_json) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py index af7ac75a..03681fc4 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/jsonlines_reference_helpers.py @@ -51,8 +51,8 @@ def reconstructed_entries( return reconstruct_entries(renamed) -def canonicalize_reference_entry(entry: dict[str, Any]) -> dict[str, Any]: - """Canonicalize an entry so pipeline output and source JSON compare equal. +def canonicalize_entry(entry: dict[str, Any]) -> dict[str, Any]: + """Canonicalize a reference or reconstructed pipeline entry so both sides compare equal. Reverses the transformations dlt's normalization applies, and normalizes the two storage representations (parquet datetimes vs jsonl timestamp strings): @@ -72,14 +72,8 @@ def canonicalize_reference_entry(entry: dict[str, Any]) -> dict[str, Any]: return _canonicalize_value(entry) -def canonicalize_pipeline_entry(entry: dict[str, Any]) -> dict[str, Any]: - """Canonicalize a reconstructed pipeline entry for comparison. - - Preserved JSON lists and reconstructed child rows share the same scalar-list - representation. Null fields and empty objects disappear during flattening. - Timestamps are normalized for both JSONL strings and Parquet datetimes. - """ - return _canonicalize_value(entry) +canonicalize_reference_entry = canonicalize_entry +canonicalize_pipeline_entry = canonicalize_entry def _canonicalize_value(value: Any, key: str = "") -> Any: diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py index fa40b0f9..f2113da7 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py @@ -21,16 +21,15 @@ LOADER_FILE_FORMATS, WORKER_CONFIGS, assert_dataset_matches_reference, - assert_datasets_equal, read_pipeline_tables, + run_and_compare_variations, run_and_read, ) from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_models import ENTITY_MODELS from tests.cdm_data_loaders.pipelines.jsonlines.integration.dataset_report_schemas import ENTITY_SCHEMAS from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( EXPECTED_ENTRY_COUNT, - canonicalize_pipeline_entry, - canonicalize_reference_entry, + canonicalize_entry, reconstructed_entries, reference_entries_from_duckdb, ) @@ -60,8 +59,8 @@ def test_jsonlines_reference_pass_assembly_model_accepts_all_records( assert [entry.model_dump()["accession"] for entry in validated] == [ entry["accession"] for entry in jsonl_reference_entities ] - assert [canonicalize_reference_entry(entry.model_dump(mode="json", exclude_unset=True)) for entry in validated] == [ - canonicalize_reference_entry(entry) for entry in jsonl_reference_entities + assert [canonicalize_entry(entry.model_dump(mode="json", exclude_unset=True)) for entry in validated] == [ + canonicalize_entry(entry) for entry in jsonl_reference_entities ] @@ -102,16 +101,13 @@ def test_jsonlines_passed_dataset_matches_reference( sorted_reference_entities: list[str], ) -> None: """Pipeline output matches the pipeline input.""" - load_info_and_dir = { - source_dir: run_and_read(run_reference_pipeline, source_dir, loader_file_format=loader_file_format) - } - - assert_dataset_matches_reference( - load_info_and_dir, - sorted_reference_entities, - loader_file_format, + run_and_compare_variations( + run_reference_pipeline, + {source_dir: (source_dir, {"loader_file_format": loader_file_format})}, + sorted_reference_json=sorted_reference_entities, + output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -120,18 +116,13 @@ def test_jsonlines_pass_output_identical_across_chunk_dirs( run_reference_pipeline: Callable[..., Any], sorted_reference_entities: list[str], loader_file_format: str ) -> None: """Pipeline output is identical across all three chunk dirs and matches the reference.""" - load_info_and_dir = { - f"{key}_{loader_file_format}": run_and_read(run_reference_pipeline, key, loader_file_format=loader_file_format) - for key in CHUNK_DIRS - } - - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference( - load_info_and_dir, - sorted_reference_entities, - loader_file_format, + run_and_compare_variations( + run_reference_pipeline, + {f"{key}_{loader_file_format}": (key, {"loader_file_format": loader_file_format}) for key in CHUNK_DIRS}, + sorted_reference_json=sorted_reference_entities, + output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -175,7 +166,7 @@ def test_jsonlines_pass_diff_format_same_results( sorted_reference_entities, output_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -185,20 +176,19 @@ def test_jsonlines_pass_output_identical_across_buffer_sizes( ) -> None: """List buffer size does not affect pipeline output for any chunk dir.""" for chunk_dir in CHUNK_DIRS: - load_info_and_dir = { - f"{buffer_size}_{loader_file_format}": run_and_read( - run_reference_pipeline, chunk_dir, buffer_size=buffer_size, loader_file_format=loader_file_format - ) - for buffer_size in BUFFER_SIZES - } - - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference( - load_info_and_dir, - sorted_reference_entities, - loader_file_format, + run_and_compare_variations( + run_reference_pipeline, + { + f"{buffer_size}_{loader_file_format}": ( + chunk_dir, + {"buffer_size": buffer_size, "loader_file_format": loader_file_format}, + ) + for buffer_size in BUFFER_SIZES + }, + sorted_reference_json=sorted_reference_entities, + output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -235,7 +225,7 @@ def test_jsonlines_pass_no_data_lost_across_worker_configs( sorted_reference_entities, output_format=loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -253,11 +243,11 @@ def test_jsonlines_pass_reconstructed_matches_duckdb_reference( from_duckdb = reference_entries_from_duckdb(reference_dataset) assert len(from_duckdb) == EXPECTED_ENTRY_COUNT from_duckdb_canonical = sorted( - json.dumps(canonicalize_reference_entry(prepare_reference(entry)), sort_keys=True, default=str) + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in from_duckdb ) from_pipeline = sorted( - json.dumps(canonicalize_pipeline_entry(entry), sort_keys=True, default=str) for entry in reconstructed + json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in reconstructed ) assert from_pipeline == from_duckdb_canonical @@ -285,7 +275,7 @@ def test_jsonlines_pass_gzip_input_matches_plain_text( sorted_reference_entities, loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) @@ -313,5 +303,5 @@ def test_jsonlines_pass_nesting_matches_pipeline_contract( sorted_reference_entities, loader_file_format, reconstruct=reconstructed_entries, - canonicalize=canonicalize_pipeline_entry, + canonicalize=canonicalize_entry, ) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 36ffd705..62e52a73 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -20,8 +20,7 @@ run_and_read, ) from tests.cdm_data_loaders.pipelines.jsonlines.integration.jsonlines_reference_helpers import ( - canonicalize_pipeline_entry, - canonicalize_reference_entry, + canonicalize_entry, reconstructed_entries, ) @@ -58,9 +57,9 @@ def test_jsonlines_fail_malformed_rows_preserve_provenance( tables = read_pipeline_tables(output_dir / result.dataset_name, loader_file_format) actual = reconstructed_entries(tables, "dataset") assert sorted( - json.dumps(canonicalize_pipeline_entry(entry), sort_keys=True, default=str) for entry in actual + json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in actual ) == sorted( - json.dumps(canonicalize_reference_entry(prepare_reference(entry)), sort_keys=True, default=str) + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in entries ) suffix, error_key = ( @@ -125,8 +124,8 @@ def test_jsonlines_pass_duplicates_blanks_and_final_line_preserved( loader_file_format=loader_file_format, ) tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) - expected = canonicalize_reference_entry(prepare_reference(entry)) - assert [canonicalize_pipeline_entry(row) for row in reconstructed_entries(tables)] == [expected, expected] + expected = canonicalize_entry(prepare_reference(entry)) + assert [canonicalize_entry(row) for row in reconstructed_entries(tables)] == [expected, expected] def test_jsonlines_pass_multiple_entities_and_file_glob( @@ -312,7 +311,7 @@ def test_jsonlines_cli_pass_reference_dataset_loads( tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) assert ( sorted( - json.dumps(canonicalize_pipeline_entry(entry), sort_keys=True, default=str) + json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in reconstructed_entries(tables) ) == sorted_reference_entities diff --git a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py index 9c7a67e2..6b951cec 100644 --- a/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/xml2db/integration/test_xml2db_reference_tests.py @@ -10,7 +10,12 @@ import xmltodict from cdm_data_loaders.core.fields import PARQUET -from tests.cdm_data_loaders.pipelines.helpers import LOADER_FILE_FORMATS, read_pipeline_tables, sorted_json +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + read_pipeline_tables, + run_and_read, + sorted_json, +) from tests.cdm_data_loaders.pipelines.xml2db.integration.xml2db_reference_helpers import ( assert_uniref_referential_integrity, reconstruct_xml2db_entries, @@ -77,9 +82,7 @@ def run_and_reconstruct(run_pipeline: Callable[..., Any], chunk_dir: str, **over Reads back whatever `loader_file_format` the run actually used -- `PARQUET`, matching `Xml2DbSettings.loader_file_format`'s own default, unless overridden. """ - (load_info, output_dir) = run_pipeline(chunk_dir, **overrides) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_pipeline, chunk_dir, **overrides) output_format = str(overrides.get("loader_file_format", PARQUET)) tables = read_pipeline_tables(output_dir / load_info.dataset_name, output_format) return reconstruct_xml2db_entries(tables) @@ -125,9 +128,7 @@ def test_xml2db_pass_referential_integrity_across_merged_files( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Every foreign key in the merged, multi-file dataset resolves to a row in its target table.""" - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_5_el") - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_5_el") tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) assert_uniref_referential_integrity(tables) @@ -167,9 +168,7 @@ def test_xml2db_pass_chunked_referential_integrity_across_files_and_chunks( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Foreign keys still resolve correctly when both multiple files and multiple chunks per file are involved.""" - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_5_el", xml_tag="entry", chunk_size=2) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_5_el", xml_tag="entry", chunk_size=2) tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) assert_uniref_referential_integrity(tables) @@ -185,12 +184,8 @@ def test_xml2db_pass_chunked_compaction_matches_unchunked_row_counts_exactly( Not just the same *content*, which the reconstruction-based tests above already establish, but the same physical row count, with no literal duplicates left over. """ - (unchunked_info, unchunked_dir) = run_xml2db_pipeline_fxt("chunk_100_el") - (chunked_info, chunked_dir) = run_xml2db_pipeline_fxt("chunk_100_el", xml_tag="entry", chunk_size=7) - assert unchunked_info is not None - assert chunked_info is not None - assert not unchunked_info.has_failed_jobs - assert not chunked_info.has_failed_jobs + (unchunked_info, unchunked_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el") + (chunked_info, chunked_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7) unchunked_tables = read_pipeline_tables(unchunked_dir / unchunked_info.dataset_name, PARQUET) chunked_tables = read_pipeline_tables(chunked_dir / chunked_info.dataset_name, PARQUET) @@ -208,9 +203,7 @@ def test_xml2db_pass_chunked_compaction_root_table_is_left_alone( the kind of literal duplicate compaction removes, so it still gets one row per chunk after compaction runs, unlike property/representative_member. """ - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_100_el", xml_tag="entry", chunk_size=7) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7) tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) n_chunks = -(-EXPECTED_ENTRY_COUNT // 7) # ceil division @@ -221,9 +214,7 @@ def test_xml2db_pass_chunked_compaction_preserves_referential_integrity( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Every foreign key still resolves after compaction has removed literal duplicate rows.""" - (load_info, output_dir) = run_xml2db_pipeline_fxt("chunk_100_el", xml_tag="entry", chunk_size=7) - assert load_info is not None - assert not load_info.has_failed_jobs + (load_info, output_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7) tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) assert_uniref_referential_integrity(tables) @@ -232,12 +223,10 @@ def test_xml2db_pass_compact_reused_tables_false_keeps_literal_duplicates( run_xml2db_pipeline_fxt: Callable[..., Any], ) -> None: """Disabling compact_reused_tables leaves the literal duplicate rows chunking can produce.""" - (unchunked_info, unchunked_dir) = run_xml2db_pipeline_fxt("chunk_100_el") - (chunked_info, chunked_dir) = run_xml2db_pipeline_fxt( - "chunk_100_el", xml_tag="entry", chunk_size=7, compact_reused_tables=False + (unchunked_info, unchunked_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_100_el") + (chunked_info, chunked_dir) = run_and_read( + run_xml2db_pipeline_fxt, "chunk_100_el", xml_tag="entry", chunk_size=7, compact_reused_tables=False ) - assert unchunked_info is not None - assert chunked_info is not None unchunked_tables = read_pipeline_tables(unchunked_dir / unchunked_info.dataset_name, PARQUET) chunked_tables = read_pipeline_tables(chunked_dir / chunked_info.dataset_name, PARQUET) @@ -262,12 +251,10 @@ def test_xml2db_pass_compaction_dedupes_across_separate_unchunked_files( literal duplicate rows until compaction merges them -- exactly the same failure mode as chunking, just from multiple files instead of multiple chunks of one file. """ - (compacted_info, compacted_dir) = run_xml2db_pipeline_fxt("chunk_5_el") - (uncompacted_info, uncompacted_dir) = run_xml2db_pipeline_fxt("chunk_5_el", compact_reused_tables=False) - assert compacted_info is not None - assert uncompacted_info is not None - assert not compacted_info.has_failed_jobs - assert not uncompacted_info.has_failed_jobs + (compacted_info, compacted_dir) = run_and_read(run_xml2db_pipeline_fxt, "chunk_5_el") + (uncompacted_info, uncompacted_dir) = run_and_read( + run_xml2db_pipeline_fxt, "chunk_5_el", compact_reused_tables=False + ) compacted_tables = read_pipeline_tables(compacted_dir / compacted_info.dataset_name, PARQUET) uncompacted_tables = read_pipeline_tables(uncompacted_dir / uncompacted_info.dataset_name, PARQUET) diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py index a9ed95ec..ff7e45ea 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py @@ -17,6 +17,7 @@ read_pipeline_tables, reconstruct_entries, reference_entries_from_duckdb, + run_and_compare_variations, run_and_read, sorted_json, ) @@ -58,15 +59,13 @@ def test_xml_pass_reference_dataset_has_nested_structures( def test_xml_pass_diff_format_same_results(run_xmltodict_pipeline_fxt: Callable[..., Any]) -> None: """Ensure that the save format does not affect the output.""" - load_info_and_dir = { - str(output_format): run_and_read( - run_xmltodict_pipeline_fxt, - "chunk_20_el", - loader_file_format=output_format, - ) - for output_format in LOADER_FILE_FORMATS - } - assert_datasets_equal(load_info_and_dir) + run_and_compare_variations( + run_xmltodict_pipeline_fxt, + { + str(output_format): ("chunk_20_el", {"loader_file_format": output_format}) + for output_format in LOADER_FILE_FORMATS + }, + ) @pytest.mark.xfail(reason="not yet implemented") @@ -104,15 +103,12 @@ def test_xml_pass_output_identical_across_chunk_dirs( run_xmltodict_pipeline_fxt: Callable[..., Any], sorted_reference_xml_entries: list[str], loader_file_format: str ) -> None: """Pipeline output is identical across all three chunk dirs and matches the reference.""" - load_info_and_dir = { - f"{key}_{loader_file_format}": run_and_read( - run_xmltodict_pipeline_fxt, key, loader_file_format=loader_file_format - ) - for key in CHUNK_DIRS - } - - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference(load_info_and_dir, sorted_reference_xml_entries, loader_file_format) + run_and_compare_variations( + run_xmltodict_pipeline_fxt, + {f"{key}_{loader_file_format}": (key, {"loader_file_format": loader_file_format}) for key in CHUNK_DIRS}, + sorted_reference_json=sorted_reference_xml_entries, + output_format=loader_file_format, + ) @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS) @@ -121,14 +117,18 @@ def test_xml_pass_output_identical_across_buffer_sizes( ) -> None: """List buffer size does not affect pipeline output for any chunk dir.""" for chunk_dir in CHUNK_DIRS: - load_info_and_dir = { - f"{buffer_size}_{loader_file_format}": run_and_read( - run_xmltodict_pipeline_fxt, chunk_dir, buffer_size=buffer_size, loader_file_format=loader_file_format - ) - for buffer_size in BUFFER_SIZES - } - assert_datasets_equal(load_info_and_dir) - assert_dataset_matches_reference(load_info_and_dir, sorted_reference_xml_entries, loader_file_format) + run_and_compare_variations( + run_xmltodict_pipeline_fxt, + { + f"{buffer_size}_{loader_file_format}": ( + chunk_dir, + {"buffer_size": buffer_size, "loader_file_format": loader_file_format}, + ) + for buffer_size in BUFFER_SIZES + }, + sorted_reference_json=sorted_reference_xml_entries, + output_format=loader_file_format, + ) @pytest.mark.parametrize( From a88e6ad16dd876b2d333c891faa280e83ee8d5da Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 20 Oct 2026 12:42:08 -0700 Subject: [PATCH 06/16] Rebase on develop, incorporate report format changes --- tests/cdm_data_loaders/pipelines/framework.py | 32 ++++++++++--------- tests/cdm_data_loaders/pipelines/helpers.py | 3 +- .../test_jsonlines_reference_tests.py | 3 +- .../integration/test_reference_end_to_end.py | 7 ++-- .../pipelines/ncbi_rest_api/conftest.py | 5 +-- tests/cdm_data_loaders/pipelines/test_core.py | 8 +++-- .../pipelines/test_framework.py | 17 ++++++++-- .../uniprot/test_uniprot_end_to_end.py | 4 ++- .../uniref/test_uniref_end_to_end.py | 4 ++- 9 files changed, 50 insertions(+), 33 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 47661346..2339b3b6 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -289,31 +289,33 @@ def reconstruct_load_info(output_dir: Path, bucket_dir: Path | None = None) -> L report_file = report_files[-1] lines = report_file.read_text(encoding="utf-8").splitlines() - if len(lines) < 3: # noqa: PLR2004 + if len(lines) < 5: # noqa: PLR2004 return None - # Line 0: "Pipeline dataset " - header_prefix = "Pipeline " - dataset_sep = " dataset " - if not lines[0].startswith(header_prefix) or dataset_sep not in lines[0]: + # Line 0: "Pipeline: " + # Line 1: "Dataset: " + # Line 2: "Started: " + # Line 3: "Finished: " + # Line 4: "Status: completed" | "Status: completed with errors" | "Status: error" + expected_prefixes = ("Pipeline: ", "Dataset: ", "Started: ", "Finished: ", "Status: ") + if not all(line.startswith(prefix) for line, prefix in zip(lines[:5], expected_prefixes, strict=True)): return None - pipeline_name, dataset_name = lines[0].removeprefix(header_prefix).rsplit(dataset_sep, maxsplit=1) - pipeline_name = pipeline_name.strip() - dataset_name = dataset_name.strip() - # Line 1: "Started: " - started_str = lines[1].removeprefix("Started: ").strip() + pipeline_name = lines[0].removeprefix("Pipeline: ").strip() + dataset_name = lines[1].removeprefix("Dataset: ").strip() + started_str = lines[2].removeprefix("Started: ").strip() + finished_str = lines[3].removeprefix("Finished: ").strip() started_at = _PreservedStrDatetime(started_str) + report_finished_at = _PreservedStrDatetime(finished_str) - # Line 2: "Status: completed" | "Status: completed with errors" | "Status: error" - status_str = lines[2].removeprefix("Status: ").strip() + status_str = lines[4].removeprefix("Status: ").strip() if status_str == "error": return None has_failed_jobs = status_str != "completed" - # Line 3+: "Completed loads: , , ..." + # Line 5+: "Completed loads: , , ..." loads_ids: list[str] = [] - for line in lines[3:]: + for line in lines[5:]: if line.startswith("Completed loads:"): loads_raw = line.removeprefix("Completed loads:").strip() loads_ids = [item.strip() for item in loads_raw.split(",") if item.strip()] @@ -329,7 +331,7 @@ def reconstruct_load_info(output_dir: Path, bucket_dir: Path | None = None) -> L saved_row = _read_saved_load_info_row(actual_bucket_dir / dataset_name) finished_at_raw = saved_row.get("finished_at") - finished_at = _PreservedStrDatetime(str(finished_at_raw)) if finished_at_raw else None + finished_at = _PreservedStrDatetime(str(finished_at_raw)) if finished_at_raw else report_finished_at # Use a fresh working directory outside output_dir so dlt syncs the schema from the destination # without colliding with stale schemas in ~/.dlt/pipelines or adding files to output_dir. diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index dd02defe..9708d6a0 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -514,8 +514,7 @@ def run_and_compare_variations( ) -> dict[str, tuple[LoadInfo, Path]]: """Run a pipeline across named (chunk_dir, overrides) variations, assert equal datasets, and optionally match reference.""" load_info_and_dir = { - key: run_and_read(run_pipeline, chunk_dir, **overrides) - for key, (chunk_dir, overrides) in variations.items() + key: run_and_read(run_pipeline, chunk_dir, **overrides) for key, (chunk_dir, overrides) in variations.items() } if len(load_info_and_dir) >= 2: # noqa: PLR2004 assert_datasets_equal(load_info_and_dir) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py index f2113da7..c5fbec28 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py @@ -243,8 +243,7 @@ def test_jsonlines_pass_reconstructed_matches_duckdb_reference( from_duckdb = reference_entries_from_duckdb(reference_dataset) assert len(from_duckdb) == EXPECTED_ENTRY_COUNT from_duckdb_canonical = sorted( - json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) - for entry in from_duckdb + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in from_duckdb ) from_pipeline = sorted( json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in reconstructed diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py index 62e52a73..9d794789 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/integration/test_reference_end_to_end.py @@ -56,11 +56,8 @@ def test_jsonlines_fail_malformed_rows_preserve_provenance( ) tables = read_pipeline_tables(output_dir / result.dataset_name, loader_file_format) actual = reconstructed_entries(tables, "dataset") - assert sorted( - json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in actual - ) == sorted( - json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) - for entry in entries + assert sorted(json.dumps(canonicalize_entry(entry), sort_keys=True, default=str) for entry in actual) == sorted( + json.dumps(canonicalize_entry(prepare_reference(entry)), sort_keys=True, default=str) for entry in entries ) suffix, error_key = ( ("invalid", "parse_error") if pipeline_kind == EXTRACT_PIPELINE_NAME else ("rejected", "error_detail") diff --git a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py index e13dfef1..865789c6 100644 --- a/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py +++ b/tests/cdm_data_loaders/pipelines/ncbi_rest_api/conftest.py @@ -20,7 +20,9 @@ def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: """Ensure that the REST_CLIENT_HOOKS dict is empty for tests.""" monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) - monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.ncbi_genome_client._config.http_show_error_body", True) + monkeypatch.setattr( + "cdm_data_loaders.pipelines.ncbi_rest_api.ncbi_genome_client._config.http_show_error_body", True + ) @pytest.fixture(scope="module") @@ -67,4 +69,3 @@ def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: :type monkeypatch: pytest.MonkeyPatch """ monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) - diff --git a/tests/cdm_data_loaders/pipelines/test_core.py b/tests/cdm_data_loaders/pipelines/test_core.py index 07f221e7..3d442d45 100644 --- a/tests/cdm_data_loaders/pipelines/test_core.py +++ b/tests/cdm_data_loaders/pipelines/test_core.py @@ -599,7 +599,9 @@ def test_run_pipeline_pipelines_dir_used_for_pipeline_metadata(dlt_test_settings assert any(p.name == "state.json" for p in pipeline_state_files) -def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: +def test_run_pipeline_dev_mode_true_loads_into_fresh_dataset( + dlt_test_settings: CtsSettings, destination_bucket: Path +) -> None: """dev_mode=True is forwarded to dlt.pipeline(); the pipeline loads into a suffixed dataset.""" settings = dlt_test_settings.model_copy(update={"dlt_dev_mode": True}) @@ -722,7 +724,9 @@ def test_run_pipeline_load_info_in_dev_mode(dlt_test_settings: CtsSettings, dest assert rows[0]["dataset_name"] == load_info.dataset_name -def test_run_pipeline_load_info_named_pipeline_and_dataset(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: +def test_run_pipeline_load_info_named_pipeline_and_dataset( + dlt_test_settings: CtsSettings, destination_bucket: Path +) -> None: """A named pipeline/dataset save lands in the named dataset with a load_info row.""" load_info = run_pipeline( dlt_test_settings, diff --git a/tests/cdm_data_loaders/pipelines/test_framework.py b/tests/cdm_data_loaders/pipelines/test_framework.py index 27116169..9085248d 100644 --- a/tests/cdm_data_loaders/pipelines/test_framework.py +++ b/tests/cdm_data_loaders/pipelines/test_framework.py @@ -127,7 +127,12 @@ def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: error_dir = tmp_path / "error_out" error_dir.mkdir() (error_dir / "report_123_456.txt").write_text( - "Pipeline test_pipe dataset test_ds\nStarted: 2026-01-01 00:00:00+00:00\nStatus: error\nPipeline failed\n", + "Pipeline: test_pipe\n" + "Dataset: test_ds\n" + "Started: 2026-01-01 00:00:00+00:00\n" + "Finished: 2026-01-01 00:00:05+00:00\n" + "Status: error\n" + "Pipeline failed\n", encoding="utf-8", ) assert reconstruct_load_info(error_dir) is None @@ -135,9 +140,12 @@ def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: completed_dir = tmp_path / "completed_out" completed_dir.mkdir() started_str = "2026-01-01 12:34:56.789000+00:00" + finished_str = "2026-01-01 12:35:00.000000+00:00" (completed_dir / "report_111_222.txt").write_text( - f"Pipeline test_pipe dataset test_ds\n" + f"Pipeline: test_pipe\n" + f"Dataset: test_ds\n" f"Started: {started_str}\n" + f"Finished: {finished_str}\n" f"Status: completed with errors\n" f"Completed loads: 111, 222\n" f"Pipeline load info saved\n", @@ -148,6 +156,7 @@ def test_reconstruct_load_info_pass_and_error_cases(tmp_path: Path) -> None: assert reconstructed.pipeline.pipeline_name == "test_pipe" assert reconstructed.dataset_name == "test_ds" assert str(reconstructed.started_at) == started_str + assert str(reconstructed.finished_at) == finished_str assert reconstructed.has_failed_jobs is True assert reconstructed.loads_ids == ["111", "222"] as_dict = reconstructed.asdict() @@ -255,8 +264,10 @@ def fake_docker_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedPr captured_cmds.append(cmd) # Simulate the container writing a completed report to the mounted output_dir (output_dir / "report_999_1000.txt").write_text( - "Pipeline xmltodict dataset xml_docker_ds\n" + "Pipeline: xmltodict\n" + "Dataset: xml_docker_ds\n" "Started: 2026-02-01 00:00:00+00:00\n" + "Finished: 2026-02-01 00:00:05+00:00\n" "Status: completed\n" "Completed loads: 999\n" "Pipeline load info saved\n", diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py index 0b3e4fe9..30ca3b80 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py @@ -101,7 +101,9 @@ def test_integration_uniprot_pipeline_entity_and_source_file_content( assert all(entity_type == "protein" for _, entity_type in rows) # entity_x_source_file must reference exactly the fixture files we loaded from - source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = { + row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file") + } assert source_files == {str(UNIPROT_FIXTURE_DIR.resolve() / name) for name in UNIPROT_FIXTURE_FILES} diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py index 6c80fd39..36a67251 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py @@ -106,7 +106,9 @@ def test_integration_uniref_pipeline_entity_and_cluster_content( assert protocols == {f"UniRef {duckdb_uniref_settings.variant}"} # entity_x_source_file must reference exactly the fixture file we loaded from - source_files = {row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file")} + source_files = { + row[0] for row in fetch_all_duckdb(pipeline, "SELECT DISTINCT source_file FROM entity_x_source_file") + } assert source_files == {str(UNIREF_FIXTURE_DIR.resolve() / UNIREF_FIXTURE_FILE)} From a146cb04b25a4a7d98df9cbabe2aa312411a4c3b Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 10:12:14 -0700 Subject: [PATCH 07/16] Move uniprot/uniref pipelines to uniprot_cdm/uniref_cdm namespace --- pyproject.toml | 5 +++-- .../parsers/uniprot/uniprot_kb.py | 1 - .../{uniprot_kb.py => uniprot_kb_cdm.py} | 2 +- .../pipelines/{uniref.py => uniref_cdm.py} | 2 +- .../{uniprot => uniprot_cdm}/__init__.py | 0 .../{uniprot => uniprot_cdm}/conftest.py | 2 +- .../test_uniprot_cdm.py} | 13 +++++++------ .../test_uniprot_cdm_end_to_end.py} | 9 +++++---- .../test_uniprot_cdm_settings.py} | 2 +- .../{uniref => uniref_cdm}/__init__.py | 0 .../{uniref => uniref_cdm}/conftest.py | 2 +- .../test_uniref_cdm.py} | 19 +++++++------------ .../test_uniref_cdm_end_to_end.py} | 7 ++++--- .../test_uniref_cdm_settings.py} | 4 ++-- 14 files changed, 33 insertions(+), 35 deletions(-) rename src/cdm_data_loaders/pipelines/{uniprot_kb.py => uniprot_kb_cdm.py} (98%) rename src/cdm_data_loaders/pipelines/{uniref.py => uniref_cdm.py} (98%) rename tests/cdm_data_loaders/pipelines/{uniprot => uniprot_cdm}/__init__.py (100%) rename tests/cdm_data_loaders/pipelines/{uniprot => uniprot_cdm}/conftest.py (90%) rename tests/cdm_data_loaders/pipelines/{uniprot/test_uniprot.py => uniprot_cdm/test_uniprot_cdm.py} (91%) rename tests/cdm_data_loaders/pipelines/{uniprot/test_uniprot_end_to_end.py => uniprot_cdm/test_uniprot_cdm_end_to_end.py} (94%) rename tests/cdm_data_loaders/pipelines/{uniprot/test_uniprot_settings.py => uniprot_cdm/test_uniprot_cdm_settings.py} (95%) rename tests/cdm_data_loaders/pipelines/{uniref => uniref_cdm}/__init__.py (100%) rename tests/cdm_data_loaders/pipelines/{uniref => uniref_cdm}/conftest.py (96%) rename tests/cdm_data_loaders/pipelines/{uniref/test_uniref.py => uniref_cdm/test_uniref_cdm.py} (85%) rename tests/cdm_data_loaders/pipelines/{uniref/test_uniref_end_to_end.py => uniref_cdm/test_uniref_cdm_end_to_end.py} (95%) rename tests/cdm_data_loaders/pipelines/{uniref/test_uniref_settings.py => uniref_cdm/test_uniref_cdm_settings.py} (98%) diff --git a/pyproject.toml b/pyproject.toml index c68d127a..2efdf762 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,8 @@ ncbi_ftp_promote = "cdm_data_loaders.pipelines.ncbi_ftp_promote:cli" ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" pdb_manifest = "cdm_data_loaders.pipelines.pdb_manifest:cli" -uniprot = "cdm_data_loaders.pipelines.uniprot_kb:cli" -uniref = "cdm_data_loaders.pipelines.uniref:cli" +uniprot_cdm = "cdm_data_loaders.pipelines.uniprot_kb_cdm:cli" +uniref_cdm = "cdm_data_loaders.pipelines.uniref_cdm:cli" xmltodict = "cdm_data_loaders.pipelines.xmltodict.pipeline:cli" xml2db = "cdm_data_loaders.pipelines.xml2db.pipeline:cli" xsv = "cdm_data_loaders.pipelines.xsv.pipeline:cli" @@ -224,6 +224,7 @@ pytest_add_cli_args_test_selection= [ "-m", "not requires_spark", "-m", "not slo [tool.pytest] +testpaths = ["tests"] pythonpath = ["src"] log_cli = true log_cli_level = "WARNING" diff --git a/src/cdm_data_loaders/parsers/uniprot/uniprot_kb.py b/src/cdm_data_loaders/parsers/uniprot/uniprot_kb.py index 4855c194..bd978a85 100644 --- a/src/cdm_data_loaders/parsers/uniprot/uniprot_kb.py +++ b/src/cdm_data_loaders/parsers/uniprot/uniprot_kb.py @@ -28,7 +28,6 @@ # XML namespace mapping for UniProt entries HTTPS_NS = "https://uniprot.org/uniprot" NS = {"ns": HTTPS_NS, "": HTTPS_NS} -APP_NAME = "uniprotkb_importer" ENTRY_XML_TAG = f"{{{HTTPS_NS}}}entry" diff --git a/src/cdm_data_loaders/pipelines/uniprot_kb.py b/src/cdm_data_loaders/pipelines/uniprot_kb_cdm.py similarity index 98% rename from src/cdm_data_loaders/pipelines/uniprot_kb.py rename to src/cdm_data_loaders/pipelines/uniprot_kb_cdm.py index 7ab79185..193b4725 100644 --- a/src/cdm_data_loaders/pipelines/uniprot_kb.py +++ b/src/cdm_data_loaders/pipelines/uniprot_kb_cdm.py @@ -21,7 +21,7 @@ ) from cdm_data_loaders.readers.xml import build_xml_file_resource -PIPELINE_NAME: Final[str] = "uniprot" +PIPELINE_NAME: Final[str] = "uniprot_cdm" UNIPROT_LOG_INTERVAL: Final[int] = 1000 diff --git a/src/cdm_data_loaders/pipelines/uniref.py b/src/cdm_data_loaders/pipelines/uniref_cdm.py similarity index 98% rename from src/cdm_data_loaders/pipelines/uniref.py rename to src/cdm_data_loaders/pipelines/uniref_cdm.py index 6cb1340f..e8a2278f 100644 --- a/src/cdm_data_loaders/pipelines/uniref.py +++ b/src/cdm_data_loaders/pipelines/uniref_cdm.py @@ -25,7 +25,7 @@ ) from cdm_data_loaders.readers.xml import build_xml_file_resource -PIPELINE_NAME: Final[str] = "uniref" +PIPELINE_NAME: Final[str] = "uniref_cdm" UNIREF_LOG_INTERVAL: Final[int] = 10000 VARIANT: Final[str] = "variant" FIFTY: Final[str] = "50" diff --git a/tests/cdm_data_loaders/pipelines/uniprot/__init__.py b/tests/cdm_data_loaders/pipelines/uniprot_cdm/__init__.py similarity index 100% rename from tests/cdm_data_loaders/pipelines/uniprot/__init__.py rename to tests/cdm_data_loaders/pipelines/uniprot_cdm/__init__.py diff --git a/tests/cdm_data_loaders/pipelines/uniprot/conftest.py b/tests/cdm_data_loaders/pipelines/uniprot_cdm/conftest.py similarity index 90% rename from tests/cdm_data_loaders/pipelines/uniprot/conftest.py rename to tests/cdm_data_loaders/pipelines/uniprot_cdm/conftest.py index d0eb95bc..d8e2d703 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/conftest.py +++ b/tests/cdm_data_loaders/pipelines/uniprot_cdm/conftest.py @@ -3,7 +3,7 @@ import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines.uniprot_kb import UNIPROT_LOG_INTERVAL, UniProtSettings +from cdm_data_loaders.pipelines.uniprot_kb_cdm import UNIPROT_LOG_INTERVAL, UniProtSettings @pytest.fixture diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py b/tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm.py similarity index 91% rename from tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py rename to tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm.py index b52061b7..50431348 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot.py +++ b/tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm.py @@ -6,9 +6,10 @@ import pytest from cdm_data_loaders.parsers.uniprot.uniprot_kb import ENTRY_XML_TAG -from cdm_data_loaders.pipelines import uniprot_kb as uniprot_module +from cdm_data_loaders.pipelines import uniprot_kb_cdm as uniprot_module from cdm_data_loaders.pipelines.core import PipelineCli -from cdm_data_loaders.pipelines.uniprot_kb import ( +from cdm_data_loaders.pipelines.uniprot_kb_cdm import ( + PIPELINE_NAME, UniProtSettings, cli, parse_uniprot, @@ -33,8 +34,8 @@ def test_run_uniprot_pipeline_args_set_correctly(test_settings: UniProtSettings) _, kwargs = mock_run_pipeline.call_args assert kwargs.keys() == {"settings", "resource", "pipeline_kwargs"} assert kwargs["pipeline_kwargs"] == { - "pipeline_name": "uniprot", - "dataset_name": "uniprot", + "pipeline_name": PIPELINE_NAME, + "dataset_name": PIPELINE_NAME, } assert kwargs["settings"] == test_settings assert isinstance(kwargs["resource"], Callable) @@ -57,8 +58,8 @@ def test_run_uniprot_pipeline_sets_core_run_pipeline_args_correctly( mock_dlt.destination.assert_called_once_with(test_settings.use_destination) mock_dlt.pipeline.assert_called_once_with( - pipeline_name="uniprot", - dataset_name="uniprot", + pipeline_name=PIPELINE_NAME, + dataset_name=PIPELINE_NAME, ) assert mock_dlt.pipeline.return_value.destination == mock_dlt.destination.return_value first_run, load_info_save_run = mock_dlt.pipeline.return_value.run.call_args_list diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm_end_to_end.py similarity index 94% rename from tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py rename to tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm_end_to_end.py index 30ca3b80..b7428031 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm_end_to_end.py @@ -10,7 +10,8 @@ import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines.uniprot_kb import ( +from cdm_data_loaders.pipelines.uniprot_kb_cdm import ( + PIPELINE_NAME, UniProtSettings, parse_uniprot, ) @@ -112,9 +113,9 @@ def test_integration_cli_uniprot_pipeline_output_validated( pipeline_runner: PipelineTestRunner, ) -> None: """Exercise the UniProt pipeline end-to-end across core, cli, and docker execution modes.""" - load_info = assert_run_ok(pipeline_runner.run("uniprot", settings=duckdb_uniprot_settings)) - assert load_info.pipeline.pipeline_name == "uniprot" - assert load_info.dataset_name == "uniprot" + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=duckdb_uniprot_settings)) + assert load_info.pipeline.pipeline_name == PIPELINE_NAME + assert load_info.dataset_name == PIPELINE_NAME assert_pipeline_report_written(Path(duckdb_uniprot_settings.output_dir), load_info, metadata_moved=True) dataset_dir = Path(duckdb_uniprot_settings.output_dir) / load_info.dataset_name diff --git a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_settings.py b/tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm_settings.py similarity index 95% rename from tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_settings.py rename to tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm_settings.py index 5af1d495..0a8cddac 100644 --- a/tests/cdm_data_loaders/pipelines/uniprot/test_uniprot_settings.py +++ b/tests/cdm_data_loaders/pipelines/uniprot_cdm/test_uniprot_cdm_settings.py @@ -5,7 +5,7 @@ import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines.uniprot_kb import UniProtSettings +from cdm_data_loaders.pipelines.uniprot_kb_cdm import UniProtSettings from tests.cdm_data_loaders.conftest import check_settings from tests.helpers import assert_cli_field_roundtrips, assert_no_cli_clashes diff --git a/tests/cdm_data_loaders/pipelines/uniref/__init__.py b/tests/cdm_data_loaders/pipelines/uniref_cdm/__init__.py similarity index 100% rename from tests/cdm_data_loaders/pipelines/uniref/__init__.py rename to tests/cdm_data_loaders/pipelines/uniref_cdm/__init__.py diff --git a/tests/cdm_data_loaders/pipelines/uniref/conftest.py b/tests/cdm_data_loaders/pipelines/uniref_cdm/conftest.py similarity index 96% rename from tests/cdm_data_loaders/pipelines/uniref/conftest.py rename to tests/cdm_data_loaders/pipelines/uniref_cdm/conftest.py index 8311e9ab..cec38caf 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/conftest.py +++ b/tests/cdm_data_loaders/pipelines/uniref_cdm/conftest.py @@ -3,7 +3,7 @@ import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines.uniref import ( +from cdm_data_loaders.pipelines.uniref_cdm import ( UNIREF_VARIANTS, VARIANT, UnirefSettings, diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref.py b/tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm.py similarity index 85% rename from tests/cdm_data_loaders/pipelines/uniref/test_uniref.py rename to tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm.py index f2866863..fc7e59d5 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref.py +++ b/tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm.py @@ -6,14 +6,9 @@ import pytest from cdm_data_loaders.parsers.uniprot.uniref import ENTRY_XML_TAG -from cdm_data_loaders.pipelines import uniref as uniref_module +from cdm_data_loaders.pipelines import uniref_cdm as uniref_cdm_module from cdm_data_loaders.pipelines.core import PipelineCli -from cdm_data_loaders.pipelines.uniref import ( - UnirefSettings, - cli, - parse_uniref, - run_uniref_pipeline, -) +from cdm_data_loaders.pipelines.uniref_cdm import PIPELINE_NAME, UnirefSettings, cli, parse_uniref, run_uniref_pipeline def test_cli_passes_settings_class_to_run_cli() -> None: @@ -26,14 +21,14 @@ def test_cli_passes_settings_class_to_run_cli() -> None: # Tests for running the pipeline itself def test_run_uniref_pipeline_args_set_correctly(test_settings: UnirefSettings) -> None: """Ensure that the pipeline arguments are set correctly, and each pipeline has a different name.""" - with patch.object(uniref_module, "run_pipeline") as mock_run_pipeline: + with patch.object(uniref_cdm_module, "run_pipeline") as mock_run_pipeline: run_uniref_pipeline(test_settings) assert mock_run_pipeline.call_count == 1 _, kwargs = mock_run_pipeline.call_args assert kwargs.keys() == {"settings", "resource", "pipeline_kwargs"} assert kwargs["pipeline_kwargs"] == { - "pipeline_name": "uniref", + "pipeline_name": PIPELINE_NAME, "dataset_name": f"uniref_{test_settings.variant}", } assert kwargs["settings"] == test_settings @@ -45,7 +40,7 @@ def test_run_uniref_pipeline_sets_core_run_pipeline_args_correctly( ) -> None: """Ensure that run_uniref_pipeline calls core.run_pipeline with the correct args.""" mock_parse_uniref = MagicMock() - monkeypatch.setattr(uniref_module, "parse_uniref", mock_parse_uniref) + monkeypatch.setattr(uniref_cdm_module, "parse_uniref", mock_parse_uniref) run_uniref_pipeline(test_settings) @@ -57,7 +52,7 @@ def test_run_uniref_pipeline_sets_core_run_pipeline_args_correctly( mock_dlt.destination.assert_called_once_with(test_settings.use_destination) mock_dlt.pipeline.assert_called_once_with( - pipeline_name="uniref", + pipeline_name=PIPELINE_NAME, dataset_name=f"uniref_{test_settings.variant}", ) assert mock_dlt.pipeline.return_value.destination == mock_dlt.destination.return_value @@ -70,7 +65,7 @@ def test_run_uniref_pipeline_sets_core_run_pipeline_args_correctly( def test_parse_uniref_resource(test_settings: UnirefSettings) -> None: """Ensure that parse_uniref calls build_xml_file_resource with the namespaced UniRef XML tag.""" - with patch.object(uniref_module, "build_xml_file_resource") as mock_build: + with patch.object(uniref_cdm_module, "build_xml_file_resource") as mock_build: resource = parse_uniref(test_settings) assert resource is mock_build.return_value diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py b/tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm_end_to_end.py similarity index 95% rename from tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py rename to tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm_end_to_end.py index 36a67251..25414fab 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm_end_to_end.py @@ -10,7 +10,8 @@ import pytest from frozendict import frozendict -from cdm_data_loaders.pipelines.uniref import ( +from cdm_data_loaders.pipelines.uniref_cdm import ( + PIPELINE_NAME, UNIREF_VARIANTS, UnirefSettings, parse_uniref, @@ -117,8 +118,8 @@ def test_integration_cli_uniref_pipeline_output_validated( pipeline_runner: PipelineTestRunner, ) -> None: """Exercise the UniRef pipeline end-to-end across core, cli, and docker execution modes.""" - load_info = assert_run_ok(pipeline_runner.run("uniref", settings=duckdb_uniref_settings)) - assert load_info.pipeline.pipeline_name == "uniref" + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=duckdb_uniref_settings)) + assert load_info.pipeline.pipeline_name == PIPELINE_NAME assert load_info.dataset_name == f"uniref_{duckdb_uniref_settings.variant}" assert_pipeline_report_written(Path(duckdb_uniref_settings.output_dir), load_info, metadata_moved=True) diff --git a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_settings.py b/tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm_settings.py similarity index 98% rename from tests/cdm_data_loaders/pipelines/uniref/test_uniref_settings.py rename to tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm_settings.py index eb0b51d7..22bdfda2 100644 --- a/tests/cdm_data_loaders/pipelines/uniref/test_uniref_settings.py +++ b/tests/cdm_data_loaders/pipelines/uniref_cdm/test_uniref_cdm_settings.py @@ -7,13 +7,13 @@ from pydantic import ValidationError from pydantic_settings import CliApp -from cdm_data_loaders.pipelines.uniref import ( +from cdm_data_loaders.pipelines.uniref_cdm import ( UNIREF_VARIANTS, VARIANT, UnirefSettings, ) from tests.cdm_data_loaders.conftest import check_settings -from tests.cdm_data_loaders.pipelines.uniref.conftest import ( +from tests.cdm_data_loaders.pipelines.uniref_cdm.conftest import ( UNIREF_VARIANT_ALIASES, ) from tests.helpers import assert_cli_field_roundtrips, assert_no_cli_clashes, make_cli_arg From 21cf5564d58a3b0ad5f7155afb447d1809edad6a Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 10:37:27 -0700 Subject: [PATCH 08/16] update framework to include renamed modules --- tests/cdm_data_loaders/pipelines/framework.py | 4 ++-- tests/cdm_data_loaders/pipelines/test_framework.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 2339b3b6..6772d2ce 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -74,8 +74,8 @@ "ncbi_ftp_sync": ("ncbi_ftp_sync", "cdm_data_loaders.pipelines.ncbi_ftp_download"), "ncbi_rest_api": ("ncbi_rest_api", "cdm_data_loaders.pipelines.ncbi_rest_api"), "pdb_manifest": ("pdb_manifest", "cdm_data_loaders.pipelines.pdb_manifest"), - "uniprot": ("uniprot", "cdm_data_loaders.pipelines.uniprot_kb"), - "uniref": ("uniref", "cdm_data_loaders.pipelines.uniref"), + "uniprot_cdm": ("uniprot_cdm", "cdm_data_loaders.pipelines.uniprot_kb_cdm"), + "uniref_cdm": ("uniref_cdm", "cdm_data_loaders.pipelines.uniref_cdm"), "xmltodict": ("xmltodict", "cdm_data_loaders.pipelines.xmltodict.pipeline"), "xml2db": ("xml2db", "cdm_data_loaders.pipelines.xml2db.pipeline"), "xsv": ("xsv", "cdm_data_loaders.pipelines.xsv.pipeline"), diff --git a/tests/cdm_data_loaders/pipelines/test_framework.py b/tests/cdm_data_loaders/pipelines/test_framework.py index 9085248d..ec66cda9 100644 --- a/tests/cdm_data_loaders/pipelines/test_framework.py +++ b/tests/cdm_data_loaders/pipelines/test_framework.py @@ -8,7 +8,7 @@ import pytest from pydantic import BaseModel -from cdm_data_loaders.pipelines import uniprot_kb +from cdm_data_loaders.pipelines import uniprot_kb_cdm from cdm_data_loaders.pipelines.jsonlines.extract_pipeline import ( cli as jsonl_extract_cli, ) @@ -61,10 +61,10 @@ def test_resolve_pipeline_pass_all_registered_targets(target_key: str) -> None: def test_resolve_pipeline_pass_from_module_cli_spec_and_callable() -> None: """resolve_pipeline accepts ModuleType, PipelineCli, PipelineSpec, and pipeline run functions.""" - from_module = resolve_pipeline(uniprot_kb) - assert from_module.cli_command == "uniprot" - assert from_module.settings_cls is uniprot_kb.UniProtSettings - assert from_module.pipeline_fn is uniprot_kb.run_uniprot_pipeline + from_module = resolve_pipeline(uniprot_kb_cdm) + assert from_module.cli_command == "uniprot_cdm" + assert from_module.settings_cls is uniprot_kb_cdm.UniProtSettings + assert from_module.pipeline_fn is uniprot_kb_cdm.run_uniprot_pipeline from_cli = resolve_pipeline(jsonl_extract_cli) assert from_cli.cli_command == "jsonlines_extract" @@ -75,8 +75,8 @@ def test_resolve_pipeline_pass_from_module_cli_spec_and_callable() -> None: assert from_callable.cli_command == "jsonlines_extract" assert from_callable.pipeline_fn is run_jsonlines_pipeline - from_full_module_str = resolve_pipeline("cdm_data_loaders.pipelines.uniprot_kb") - assert from_full_module_str.cli_command == "uniprot" + from_full_module_str = resolve_pipeline("cdm_data_loaders.pipelines.uniprot_kb_cdm") + assert from_full_module_str.cli_command == "uniprot_cdm" from_spec = resolve_pipeline(from_cli) assert from_spec is from_cli From 86a0d1b710b8fc97eb23f6cf534636faac74ea20 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 12:29:42 -0700 Subject: [PATCH 09/16] Add in docker tests --- .github/workflows/tests.yml | 2 + AGENTS.md | 116 +++++++++++++++++ CLAUDE.md | 117 +----------------- tests/cdm_data_loaders/pipelines/framework.py | 20 +++ 4 files changed, 139 insertions(+), 116 deletions(-) create mode 100644 AGENTS.md diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 46222f6a..94fdf54b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,6 +85,8 @@ jobs: suite: - name: pipelines args: tests/cdm_data_loaders/pipelines/ + - name: docker + args: --run-docker -m "docker and not requires_spark and not requires_ceph" tests/cdm_data_loaders/pipelines/ - name: integration args: tests/integration/ - name: everything else diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..d2f1c773 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,116 @@ +# CLAUDE.md + +Guidance for working in this repository. Rules with stricter, active requirements live in `.claude/rules/` (coding-style, testing, heresy) and apply to all changes. + +## Project + +`cdm-data-loaders`: data loaders and wranglers for the KBase CDM. Python >= 3.13, managed and executed with `uv`. Package source is under `src/cdm_data_loaders/` (~115 files, 8 top-level packages). Tests mirror the source layout under `tests/cdm_data_loaders/` (including pipeline end-to-end suites under `tests/cdm_data_loaders/pipelines/`), with multi-phase NCBI FTP integration tests in `tests/integration/ncbi_ftp/` and vcrpy cassettes in `tests/cassettes/`. + +Autouse fixtures in `tests/conftest.py` (`_isolated_cli_and_env` and `_isolated_dlt_config` from `tests/dlt_config_isolation.py`) reset `sys.argv = ["pytest"]`, strip `CDL_*` environment variables, and snapshot/restore `dlt.config` between tests. Pipeline end-to-end tests use `PipelineTestRunner` (`tests/cdm_data_loaders/pipelines/framework.py`) to run pipelines across `ExecutionMode.CORE`, `ExecutionMode.CLI`, and `ExecutionMode.DOCKER` (Docker tests are marked `requires_docker` and require `PIPELINE_DOCKER_IMAGE` to be set). + +Useful invocations: + +- Standard local test run (skips Spark, Ceph, and unset Docker image tests): `uv run pytest -m "not requires_spark and not requires_ceph"` +- Full pipelines suite, excluding the bulk downloader: `uv run pytest tests/cdm_data_loaders/pipelines --ignore=tests/cdm_data_loaders/pipelines/all_the_bacteria` +- Unit vs. container test script: `./scripts/run_tests.sh [unit|container|all]` +- Lint: `uv run ruff check` (line length 120, target py313, full rule selection in `pyproject.toml`). + +## Architecture + +Domain logic lives in the top-level packages and is orchestration-free; orchestration lives in `pipelines/`. Data flows source -> reader -> intermediate representation (IR) -> emitter -> target: + +- Converters: reader (source schema -> IR) + emitter (IR -> target format) + thin facade composing them. See `docs/converters_ir.md` for the full IR contract and package layout. +- IR is a frozen, slotted node tree (`converters/core/ir.py`) that preserves source facts without choosing target policy; emitters make all target-specific decisions. +- `converters/core/` imports no dlt, PyIceberg, or PySpark modules. The only cross-package import is the `x-xsv-config` contract reusing `X_XSV_CONFIG_SCHEMA` from `cdm_data_loaders.readers.jsonschema_xsv.xsv_validator.custom_metaschema`. + +Settings hierarchy: pipeline settings in `pipelines/` extend the hierarchy in `core/settings.py`: `CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`. Settings use `env_prefix="CDL_"`, `cli_kebab_case=True`, and `default_settings_with_shortcuts(...)`. `CdmDataLoadersBase.__pydantic_init_subclass__` calls `check_aliases()` at class creation time to verify that CLI argument names, aliases, and `cli_shortcuts` do not collide or claim reserved flags (`h`, `help`). CLI parsing is invoked explicitly via `pydantic_settings.CliApp.run` in `pipelines/core.py` (`PipelineCli` / `run_cli`), not implicitly on model instantiation. Every pipeline registers a `cli` entry point in `pyproject.toml` under `[project.scripts]` (and in `scripts/entrypoint.sh` for Docker execution): `all_the_bacteria`, `jsonl_extract`, `jsonl_jsonschema`, `jsonl_pydantic`, `ncbi_ftp_promote`, `ncbi_ftp_sync`, `ncbi_rest_api`, `pdb_manifest`, `uniprot`, `uniref`, `xmltodict`, `xml2db`, `xsv`. + +## Package Map + +All paths below are relative to `src/cdm_data_loaders/`. + +### `core/` + +Framework-level foundations: + +- `fields.py`: common annotated field types (`BufferSize`, `DatasetName`, `FileGlob`, `LoaderFileFormat`, `TableName`, `WriteDisposition`, etc.), `LoaderFileFormatEnum` (`parquet`, `jsonl`), and `DEFAULTS`. +- `path_fields.py`: `LocalInputDir` and `LocalOutputDir` path types (`file://` or plain local directory paths) and URL/path validation helpers (`normalize_local_path`, `validate_directory`, `validate_url_protocol`). +- `settings.py`: `pydantic-settings` base class hierarchy (`CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`), `CLI_SHORTCUTS`, `default_settings_with_shortcuts()`, and CLI flag/shortcut collision checking (`check_aliases()`). + +### `converters/` + +Schema-format translation. See `docs/converters_ir.md` for the full contract. + +- `core/`: shared, target-independent foundation + - `ir.py` (NodeType, Field, SchemaDocument, Reader/Emitter protocols), `ir_values.py` (`freeze_value` and friends), `errors.py` (ConversionError), `extensions.py` (ExtensionRegistry, DEFAULT_EXTENSIONS), `guards.py` (input guards), `inference.py` (type inference, decimal scale), `metadata.py` (draft-specific JSON Schema keyword selection), `io.py` (schema loading), `paths.py` (`set_nested`). +- `readers/`: source -> IR + - `json_schema.py` (JsonSchemaReader), `dlt.py` (DltReader over dlt's normalized table forest), `iceberg.py` (IcebergReader). +- `emitters/`: IR -> target + - `json_schema.py`, `dlt.py`, `pyspark.py`, `pyarrow.py`, `linkml.py`. +- Facades: `jsonschema_to_dlt.py`, `jsonschema_to_pyspark.py`, `jsonschema_to_pyarrow.py`, `dlt_to_jsonschema.py`, `pyiceberg_to_jsonschema.py`, `pyiceberg_to_linkml.py`, plus CLI dumps `pyiceberg_catalog_to_jsonschema.py` and `pyiceberg_catalog_to_linkml.py`. + +### `readers/` + +File-format readers feeding pipelines: + +- `jsonlines.py`: JSONL line parsing (`parse_jsonl_line`), paged streaming (`stream_jsonl_lines`), and table routing (`route_jsonl_lines`). +- `xml.py`: shared XML streaming and `xmltodict` helpers (`parse_head_matter`, `stream_xml_file`, `process_xml_file_to_dict`, `process_xml_file`, `process_xml_file_batches`, `process_xml_file_items`, and `build_xml_file_resource` over dlt's `filesystem` source). +- `xml2db_doc.py`: `xml2db` model building (`build_xml2db_model`), bounded-memory chunk fragment streaming (`iter_xml2db_chunk_fragments`), primary/foreign key rewriting (content-addressed keys for reused tables, file/chunk-namespaced keys otherwise), document flattening (`flatten_xml2db_document`), and `process_xml_file_with_xml2db`. +- `xsd.py`: XSD schema loading (`load_schema`) and parent/child element cardinality classification (`find_list_and_single_child_paths`). +- `jsonschema_xsv/`: `xsv_reader.py` (casts validated CSV/TSV rows to JSON Schema property types and resolves `x-xsv-config` parsing parameters) plus `xsv_validator/` (`qsv.py` wraps the external `qsv` binary for two-pass validation, normalization, and null replacement; `custom_metaschema.py` defines the `https://json-schema.kbase.us/draft/2020-12-xsv/schema` metaschema and `X_XSV_CONFIG_SCHEMA`, also imported by `converters`; `schema_utils.py` and `helpers.py`). + +### `parsers/` + +Parsers for third-party outputs and formats: + +- `uniprot/`: `uniprot_kb.py` (UniProt XML), `uniref.py` (UniRef XML), `metalink.py`, `relnotes.py`, `idmapping.schema.json`; shared XML helpers in `utils/xml_utils.py`. +- `ncbi/datasets_api/openapi_pydantic.py`: pydantic models for NCBI Datasets API responses (bundled `AnnotationReportResponse.schema.json` and `DatasetReportResponse.schema.json` alongside). +- `ncbi/ftp_mod_times/`: `ftp_mod_times.schema.json` for NCBI FTP modification-time files. +- `bbmap_stats.py` (bbmap `stats.sh` output), `checkm2.py` (quality reports), `pdb_holdings_file.py`. +- `all_the_bacteria/schema/` and `gtdb/schema/`: JSON Schemas for ATB and GTDB tabular files (no Python modules). + +### `ncbi_ftp/` + +NCBI FTP assembly domain logic, phased: `manifest.py` (phase 1: assembly summary diffing and manifest generation), download is the phase 2 pipeline, `promote.py` (phase 3: promote staged files to final Lakehouse paths in S3), plus `assembly.py` (assembly-specific logic), `metadata.py` (frictionless data package descriptors), `constants.py`. + +### `pdb/` + +PDB pipeline constants, enums, and globals (`constants.py`); the pipeline itself is `pipelines/pdb_manifest.py`. + +### `pipelines/` + +dlt-based orchestrators; each exposes a `cli = PipelineCli(...)` registered in `pyproject.toml`: + +- `core.py`: `PipelineCli`, `run_cli`, `run_pipeline`, `setup_dlt_Pipeline`, `execute_dlt_pipeline`, and `configure_pipeline_logging`. +- `all_the_bacteria.py`: All The Bacteria bulk data downloader (excluded from fast test runs). +- `jsonlines/`: JSONL ingest pipelines (`settings.py`, shared machinery in `base.py`) - `extract_pipeline.py` (`jsonl_extract`, plain load into `` / `_invalid`), `extract_jsonschema_validate_pipeline.py` (`jsonl_jsonschema`, validates against `ENTITY_SCHEMAS`), `extract_pydantic_validate_pipeline.py` (`jsonl_pydantic`, validates against `ENTITY_MODELS`). See `docs/jsonschema_ingest.md`. +- `ncbi_ftp_download.py` / `ncbi_ftp_promote.py`: phases 2 (`ncbi_ftp_sync`) and 3 (`ncbi_ftp_promote`) of the NCBI FTP flow. +- `ncbi_rest_api.py`: NCBI REST API import. +- `pdb_manifest.py`: PDB holdings transfer manifest. +- `uniprot_kb.py` / `uniref.py`: UniProt (`uniprot`) and UniRef (`uniref`) dlt pipelines. +- `xml2db/` / `xmltodict/` / `xsv/`: XML-to-relational, XML-to-dictionary, and XSV (CSV/TSV) ingestion, each with `pipeline.py` and `settings.py`; `xml2db/compaction.py` runs DuckDB-based post-load deduplication of "reused" tables on filesystem destinations. + +### `utils/` + +Cross-cutting helpers: `batcher.py` (`NumericFileSequenceBatcher`, `get_file_batches`), `buffer.py` (per-table row buffers for dlt: `DictBuffer`, `ListBuffer`, `ArrowDictBuffer`, `ArrowListBuffer`), `cdm_logger.py` (structured JSON logging), `checksums.py`, `gz.py` (`open_maybe_gzip`), `helpers.py`, `progress.py` (tqdm helpers), `spark.py`, `xml_utils.py`, `jsonschema/dereferencer.py` (`$ref` resolution), `dlt/dlt_normalization.py` (lossless parent-linked normalization of dlt table definitions, used by the converters' `DltReader`), and `file_transfer/` - `core.py` (tenacity-retried downloads), `sync_client.py` / `async_client.py` / `ftp_client.py`, and `s3/` (`client.py`, `object_utils.py`, HTTP-to-S3 streaming in `streamer.py`, `transfer_config.py`, `cli.py`). + +### Other Top-Level Directories + +- `scripts/`: `entrypoint.sh` (container entrypoint dispatching to registered pipeline CLIs), `run_tests.sh` (`unit`, `container`, or `all` test suites), `filter_ncbi_updates.py` (CLI filtering NCBI assembly IDs by modification date), `s3_local.py` (thin S3 CLI for local Ceph testing), `dtd2xsd.pl` (DTD-to-XSD converter), `run_tools.sh`, `parse_index.py`. +- `docs/`: architecture and pipeline guides (`converters_ir.md`, `jsonschema_ingest.md`, `ncbi_ftp_e2e_walkthrough.md`, `pdb_e2e_walkthrough.md`, `rclone_instructions.md`, `uniprot.md`). +- `cdl_skill/`: skill definition (`SKILL.md`) and generator (`scripts/skill_gen.sh`) for packaging pipeline documentation and schemas. + +## Conventions + +- Frozen, slotted dataclasses for models; constructors validate and recursively copy mappings. +- CLI argument parsing by pydantic-settings +- Fully type-annotated code (ruff `ANN`); datetime values carry an explicit timezone (`DTZ`); use `Decimal` for exact arithmetic, never constructed from floats. +- External HTTP calls in tests are recorded and replayed with vcrpy; AWS is mocked with moto; Spark and Ceph tests run in containers via `docker-compose.yml` and are skipped by default (`-m "not requires_spark and not requires_ceph"`). +- Tests are module-based `def test_*() -> None:` functions with pytest fixtures in `conftest.py`, named `test___`. + +## Gotchas + +- dlt 1.30 DuckDB destination fails with a Binder Error "Ambiguous reference to catalog or schema" when the db filename stem equals the dataset name; keep the db filename distinct from both pipeline and dataset names. +- dlt 1.30: `AnyDestination` is a quoted TypeAlias (a str at runtime), so `AnyDestination | None` in an evaluated annotation raises TypeError on import. Use `TDestinationReferenceArg` from `dlt.common.destination.reference`. +- pyiceberg SqlCatalog engines leak sqlite connections; use the context-manager protocol (`with load_catalog(...) as catalog`) so `close()` disposes the engine. +- pipelines are run on the KBase CTS (Task Service), which only saves the contents of the output directory (or dlt destination) if the container exits with exit code 0. For any other error code, the container STDERR and STDOUT are saved, but nothing else. As a result, even if pipelines experience failure (e.g. `load_info.has_failed_packages` is true), the container must exit 0 to ensure that logs, parsed data, and other artefacts are not deleted. Errors that occur before a pipeline is run (e.g. argument parsing errors) are not caught, so they should be picked up by the container STDERR/STDOUT. diff --git a/CLAUDE.md b/CLAUDE.md index d2f1c773..43c994c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,116 +1 @@ -# CLAUDE.md - -Guidance for working in this repository. Rules with stricter, active requirements live in `.claude/rules/` (coding-style, testing, heresy) and apply to all changes. - -## Project - -`cdm-data-loaders`: data loaders and wranglers for the KBase CDM. Python >= 3.13, managed and executed with `uv`. Package source is under `src/cdm_data_loaders/` (~115 files, 8 top-level packages). Tests mirror the source layout under `tests/cdm_data_loaders/` (including pipeline end-to-end suites under `tests/cdm_data_loaders/pipelines/`), with multi-phase NCBI FTP integration tests in `tests/integration/ncbi_ftp/` and vcrpy cassettes in `tests/cassettes/`. - -Autouse fixtures in `tests/conftest.py` (`_isolated_cli_and_env` and `_isolated_dlt_config` from `tests/dlt_config_isolation.py`) reset `sys.argv = ["pytest"]`, strip `CDL_*` environment variables, and snapshot/restore `dlt.config` between tests. Pipeline end-to-end tests use `PipelineTestRunner` (`tests/cdm_data_loaders/pipelines/framework.py`) to run pipelines across `ExecutionMode.CORE`, `ExecutionMode.CLI`, and `ExecutionMode.DOCKER` (Docker tests are marked `requires_docker` and require `PIPELINE_DOCKER_IMAGE` to be set). - -Useful invocations: - -- Standard local test run (skips Spark, Ceph, and unset Docker image tests): `uv run pytest -m "not requires_spark and not requires_ceph"` -- Full pipelines suite, excluding the bulk downloader: `uv run pytest tests/cdm_data_loaders/pipelines --ignore=tests/cdm_data_loaders/pipelines/all_the_bacteria` -- Unit vs. container test script: `./scripts/run_tests.sh [unit|container|all]` -- Lint: `uv run ruff check` (line length 120, target py313, full rule selection in `pyproject.toml`). - -## Architecture - -Domain logic lives in the top-level packages and is orchestration-free; orchestration lives in `pipelines/`. Data flows source -> reader -> intermediate representation (IR) -> emitter -> target: - -- Converters: reader (source schema -> IR) + emitter (IR -> target format) + thin facade composing them. See `docs/converters_ir.md` for the full IR contract and package layout. -- IR is a frozen, slotted node tree (`converters/core/ir.py`) that preserves source facts without choosing target policy; emitters make all target-specific decisions. -- `converters/core/` imports no dlt, PyIceberg, or PySpark modules. The only cross-package import is the `x-xsv-config` contract reusing `X_XSV_CONFIG_SCHEMA` from `cdm_data_loaders.readers.jsonschema_xsv.xsv_validator.custom_metaschema`. - -Settings hierarchy: pipeline settings in `pipelines/` extend the hierarchy in `core/settings.py`: `CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`. Settings use `env_prefix="CDL_"`, `cli_kebab_case=True`, and `default_settings_with_shortcuts(...)`. `CdmDataLoadersBase.__pydantic_init_subclass__` calls `check_aliases()` at class creation time to verify that CLI argument names, aliases, and `cli_shortcuts` do not collide or claim reserved flags (`h`, `help`). CLI parsing is invoked explicitly via `pydantic_settings.CliApp.run` in `pipelines/core.py` (`PipelineCli` / `run_cli`), not implicitly on model instantiation. Every pipeline registers a `cli` entry point in `pyproject.toml` under `[project.scripts]` (and in `scripts/entrypoint.sh` for Docker execution): `all_the_bacteria`, `jsonl_extract`, `jsonl_jsonschema`, `jsonl_pydantic`, `ncbi_ftp_promote`, `ncbi_ftp_sync`, `ncbi_rest_api`, `pdb_manifest`, `uniprot`, `uniref`, `xmltodict`, `xml2db`, `xsv`. - -## Package Map - -All paths below are relative to `src/cdm_data_loaders/`. - -### `core/` - -Framework-level foundations: - -- `fields.py`: common annotated field types (`BufferSize`, `DatasetName`, `FileGlob`, `LoaderFileFormat`, `TableName`, `WriteDisposition`, etc.), `LoaderFileFormatEnum` (`parquet`, `jsonl`), and `DEFAULTS`. -- `path_fields.py`: `LocalInputDir` and `LocalOutputDir` path types (`file://` or plain local directory paths) and URL/path validation helpers (`normalize_local_path`, `validate_directory`, `validate_url_protocol`). -- `settings.py`: `pydantic-settings` base class hierarchy (`CdmDataLoadersBase` -> `LoggerSettings` -> `LocalInputOutputSettings` -> `CtsSettings` -> `FileFormatIngestSettings`), `CLI_SHORTCUTS`, `default_settings_with_shortcuts()`, and CLI flag/shortcut collision checking (`check_aliases()`). - -### `converters/` - -Schema-format translation. See `docs/converters_ir.md` for the full contract. - -- `core/`: shared, target-independent foundation - - `ir.py` (NodeType, Field, SchemaDocument, Reader/Emitter protocols), `ir_values.py` (`freeze_value` and friends), `errors.py` (ConversionError), `extensions.py` (ExtensionRegistry, DEFAULT_EXTENSIONS), `guards.py` (input guards), `inference.py` (type inference, decimal scale), `metadata.py` (draft-specific JSON Schema keyword selection), `io.py` (schema loading), `paths.py` (`set_nested`). -- `readers/`: source -> IR - - `json_schema.py` (JsonSchemaReader), `dlt.py` (DltReader over dlt's normalized table forest), `iceberg.py` (IcebergReader). -- `emitters/`: IR -> target - - `json_schema.py`, `dlt.py`, `pyspark.py`, `pyarrow.py`, `linkml.py`. -- Facades: `jsonschema_to_dlt.py`, `jsonschema_to_pyspark.py`, `jsonschema_to_pyarrow.py`, `dlt_to_jsonschema.py`, `pyiceberg_to_jsonschema.py`, `pyiceberg_to_linkml.py`, plus CLI dumps `pyiceberg_catalog_to_jsonschema.py` and `pyiceberg_catalog_to_linkml.py`. - -### `readers/` - -File-format readers feeding pipelines: - -- `jsonlines.py`: JSONL line parsing (`parse_jsonl_line`), paged streaming (`stream_jsonl_lines`), and table routing (`route_jsonl_lines`). -- `xml.py`: shared XML streaming and `xmltodict` helpers (`parse_head_matter`, `stream_xml_file`, `process_xml_file_to_dict`, `process_xml_file`, `process_xml_file_batches`, `process_xml_file_items`, and `build_xml_file_resource` over dlt's `filesystem` source). -- `xml2db_doc.py`: `xml2db` model building (`build_xml2db_model`), bounded-memory chunk fragment streaming (`iter_xml2db_chunk_fragments`), primary/foreign key rewriting (content-addressed keys for reused tables, file/chunk-namespaced keys otherwise), document flattening (`flatten_xml2db_document`), and `process_xml_file_with_xml2db`. -- `xsd.py`: XSD schema loading (`load_schema`) and parent/child element cardinality classification (`find_list_and_single_child_paths`). -- `jsonschema_xsv/`: `xsv_reader.py` (casts validated CSV/TSV rows to JSON Schema property types and resolves `x-xsv-config` parsing parameters) plus `xsv_validator/` (`qsv.py` wraps the external `qsv` binary for two-pass validation, normalization, and null replacement; `custom_metaschema.py` defines the `https://json-schema.kbase.us/draft/2020-12-xsv/schema` metaschema and `X_XSV_CONFIG_SCHEMA`, also imported by `converters`; `schema_utils.py` and `helpers.py`). - -### `parsers/` - -Parsers for third-party outputs and formats: - -- `uniprot/`: `uniprot_kb.py` (UniProt XML), `uniref.py` (UniRef XML), `metalink.py`, `relnotes.py`, `idmapping.schema.json`; shared XML helpers in `utils/xml_utils.py`. -- `ncbi/datasets_api/openapi_pydantic.py`: pydantic models for NCBI Datasets API responses (bundled `AnnotationReportResponse.schema.json` and `DatasetReportResponse.schema.json` alongside). -- `ncbi/ftp_mod_times/`: `ftp_mod_times.schema.json` for NCBI FTP modification-time files. -- `bbmap_stats.py` (bbmap `stats.sh` output), `checkm2.py` (quality reports), `pdb_holdings_file.py`. -- `all_the_bacteria/schema/` and `gtdb/schema/`: JSON Schemas for ATB and GTDB tabular files (no Python modules). - -### `ncbi_ftp/` - -NCBI FTP assembly domain logic, phased: `manifest.py` (phase 1: assembly summary diffing and manifest generation), download is the phase 2 pipeline, `promote.py` (phase 3: promote staged files to final Lakehouse paths in S3), plus `assembly.py` (assembly-specific logic), `metadata.py` (frictionless data package descriptors), `constants.py`. - -### `pdb/` - -PDB pipeline constants, enums, and globals (`constants.py`); the pipeline itself is `pipelines/pdb_manifest.py`. - -### `pipelines/` - -dlt-based orchestrators; each exposes a `cli = PipelineCli(...)` registered in `pyproject.toml`: - -- `core.py`: `PipelineCli`, `run_cli`, `run_pipeline`, `setup_dlt_Pipeline`, `execute_dlt_pipeline`, and `configure_pipeline_logging`. -- `all_the_bacteria.py`: All The Bacteria bulk data downloader (excluded from fast test runs). -- `jsonlines/`: JSONL ingest pipelines (`settings.py`, shared machinery in `base.py`) - `extract_pipeline.py` (`jsonl_extract`, plain load into `` / `_invalid`), `extract_jsonschema_validate_pipeline.py` (`jsonl_jsonschema`, validates against `ENTITY_SCHEMAS`), `extract_pydantic_validate_pipeline.py` (`jsonl_pydantic`, validates against `ENTITY_MODELS`). See `docs/jsonschema_ingest.md`. -- `ncbi_ftp_download.py` / `ncbi_ftp_promote.py`: phases 2 (`ncbi_ftp_sync`) and 3 (`ncbi_ftp_promote`) of the NCBI FTP flow. -- `ncbi_rest_api.py`: NCBI REST API import. -- `pdb_manifest.py`: PDB holdings transfer manifest. -- `uniprot_kb.py` / `uniref.py`: UniProt (`uniprot`) and UniRef (`uniref`) dlt pipelines. -- `xml2db/` / `xmltodict/` / `xsv/`: XML-to-relational, XML-to-dictionary, and XSV (CSV/TSV) ingestion, each with `pipeline.py` and `settings.py`; `xml2db/compaction.py` runs DuckDB-based post-load deduplication of "reused" tables on filesystem destinations. - -### `utils/` - -Cross-cutting helpers: `batcher.py` (`NumericFileSequenceBatcher`, `get_file_batches`), `buffer.py` (per-table row buffers for dlt: `DictBuffer`, `ListBuffer`, `ArrowDictBuffer`, `ArrowListBuffer`), `cdm_logger.py` (structured JSON logging), `checksums.py`, `gz.py` (`open_maybe_gzip`), `helpers.py`, `progress.py` (tqdm helpers), `spark.py`, `xml_utils.py`, `jsonschema/dereferencer.py` (`$ref` resolution), `dlt/dlt_normalization.py` (lossless parent-linked normalization of dlt table definitions, used by the converters' `DltReader`), and `file_transfer/` - `core.py` (tenacity-retried downloads), `sync_client.py` / `async_client.py` / `ftp_client.py`, and `s3/` (`client.py`, `object_utils.py`, HTTP-to-S3 streaming in `streamer.py`, `transfer_config.py`, `cli.py`). - -### Other Top-Level Directories - -- `scripts/`: `entrypoint.sh` (container entrypoint dispatching to registered pipeline CLIs), `run_tests.sh` (`unit`, `container`, or `all` test suites), `filter_ncbi_updates.py` (CLI filtering NCBI assembly IDs by modification date), `s3_local.py` (thin S3 CLI for local Ceph testing), `dtd2xsd.pl` (DTD-to-XSD converter), `run_tools.sh`, `parse_index.py`. -- `docs/`: architecture and pipeline guides (`converters_ir.md`, `jsonschema_ingest.md`, `ncbi_ftp_e2e_walkthrough.md`, `pdb_e2e_walkthrough.md`, `rclone_instructions.md`, `uniprot.md`). -- `cdl_skill/`: skill definition (`SKILL.md`) and generator (`scripts/skill_gen.sh`) for packaging pipeline documentation and schemas. - -## Conventions - -- Frozen, slotted dataclasses for models; constructors validate and recursively copy mappings. -- CLI argument parsing by pydantic-settings -- Fully type-annotated code (ruff `ANN`); datetime values carry an explicit timezone (`DTZ`); use `Decimal` for exact arithmetic, never constructed from floats. -- External HTTP calls in tests are recorded and replayed with vcrpy; AWS is mocked with moto; Spark and Ceph tests run in containers via `docker-compose.yml` and are skipped by default (`-m "not requires_spark and not requires_ceph"`). -- Tests are module-based `def test_*() -> None:` functions with pytest fixtures in `conftest.py`, named `test___`. - -## Gotchas - -- dlt 1.30 DuckDB destination fails with a Binder Error "Ambiguous reference to catalog or schema" when the db filename stem equals the dataset name; keep the db filename distinct from both pipeline and dataset names. -- dlt 1.30: `AnyDestination` is a quoted TypeAlias (a str at runtime), so `AnyDestination | None` in an evaluated annotation raises TypeError on import. Use `TDestinationReferenceArg` from `dlt.common.destination.reference`. -- pyiceberg SqlCatalog engines leak sqlite connections; use the context-manager protocol (`with load_catalog(...) as catalog`) so `close()` disposes the engine. -- pipelines are run on the KBase CTS (Task Service), which only saves the contents of the output directory (or dlt destination) if the container exits with exit code 0. For any other error code, the container STDERR and STDOUT are saved, but nothing else. As a result, even if pipelines experience failure (e.g. `load_info.has_failed_packages` is true), the container must exit 0 to ensure that logs, parsed data, and other artefacts are not deleted. Errors that occur before a pipeline is run (e.g. argument parsing errors) are not caught, so they should be picked up by the container STDERR/STDOUT. +@AGENTS.md diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 6772d2ce..e684970b 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -452,6 +452,19 @@ def _write_docker_dlt_config(config_path: Path, use_destination: str) -> None: config_path.write_text("\n".join(blocks), encoding="utf-8") +def _make_container_accessible(root: Path, *, writable: bool = True) -> None: + """Ensure bind-mounted host paths are accessible to the container's nonroot user on Linux.""" + dir_mode = 0o777 if writable else 0o755 + file_mode = 0o666 if writable else 0o644 + if root.is_dir(): + root.chmod(dir_mode) + for child in root.rglob("*"): + if not child.is_symlink(): + child.chmod(dir_mode if child.is_dir() else file_mode) + elif root.is_file() and not root.is_symlink(): + root.chmod(file_mode) + + class PipelineTestRunner: """Execute any CDM ingestion pipeline in ``core``, ``cli``, or ``docker`` mode.""" @@ -636,6 +649,13 @@ def _run_docker( container_args = self._translate_args_for_docker(args, aux_dir, pymodules_dir) + _make_container_accessible(staged_input_dir, writable=True) + _make_container_accessible(host_output_dir, writable=True) + if aux_dir.is_dir(): + _make_container_accessible(aux_dir, writable=False) + if pymodules_dir.is_dir(): + _make_container_accessible(pymodules_dir, writable=False) + docker_cmd: list[str] = [ "docker", "run", From eea3db8b3b0bd29a49edb34192a96d77bd3508e9 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 12:53:13 -0700 Subject: [PATCH 10/16] New xsd/pyarrow XML data loader --- .github/workflows/tests.yml | 4 +- docs/xsd_to_pyarrow_specification.md | 1004 ++++ pyproject.toml | 1 + .../pipelines/xmltodict/pipeline.py | 9 +- .../pipelines/xmltodict/settings.py | 50 +- .../pipelines/xmltopyarrow/__init__.py | 0 .../pipelines/xmltopyarrow/pipeline.py | 66 + .../pipelines/xmltopyarrow/settings.py | 95 + src/cdm_data_loaders/readers/xml.py | 72 +- src/cdm_data_loaders/readers/xsd.py | 1004 +++- tests/cdm_data_loaders/pipelines/framework.py | 1 + .../pipelines/xmltodict/conftest.py | 1 + .../xmltodict/integration/conftest.py | 2 + .../integration/test_xmltodict_end_to_end.py | 2 + .../test_xmltodict_reference_tests.py | 9 +- .../xmltodict/test_xmltodict_pipeline.py | 12 - .../xmltodict/test_xmltodict_settings.py | 117 - .../pipelines/xmltopyarrow/__init__.py | 1 + .../pipelines/xmltopyarrow/conftest.py | 54 + .../xmltopyarrow/integration/__init__.py | 1 + .../xmltopyarrow/integration/conftest.py | 207 + .../test_xmltopyarrow_end_to_end.py | 145 + .../test_xmltopyarrow_reference_tests.py | 171 + .../test_xmltopyarrow_pipeline.py | 241 + .../test_xmltopyarrow_settings.py | 214 + .../test_xsd_to_arrow_end_to_end.py | 198 + tests/cdm_data_loaders/readers/conftest.py | 17 - tests/cdm_data_loaders/readers/test_xsd.py | 203 - .../cdm_data_loaders/readers/test_xsd_spec.py | 536 ++ .../test_xml_process_xml_file_and_batches.py | 344 +- .../readers/xml/test_xml_stream_xml_file.py | 75 +- tests/data/xsd/attribute_only.xsd | 7 - tests/data/xsd/cdata_variety.xsd | 63 + tests/data/xsd/combinations.xsd | 245 + .../pubmed_article/expected_schema.py | 1584 ++++++ .../xsd/examples/pubmed_article/maximal.xml | 433 ++ .../pubmed_article/maximal_expected.json | 894 ++++ .../xsd/examples/pubmed_article/minimal.xml | 26 + .../pubmed_article/minimal_expected.json | 45 + .../pubmed_book_article/expected_schema.py | 1600 ++++++ .../examples/pubmed_book_article/maximal.xml | 482 ++ .../pubmed_book_article/maximal_expected.json | 1069 ++++ .../examples/pubmed_book_article/minimal.xml | 20 + .../pubmed_book_article/minimal_expected.json | 28 + .../xsd/examples/uniprot/expected_schema.py | 4346 +++++++++++++++++ tests/data/xsd/examples/uniprot/maximal.xml | 693 +++ .../examples/uniprot/maximal_expected.json | 3525 +++++++++++++ tests/data/xsd/examples/uniprot/minimal.xml | 22 + .../examples/uniprot/minimal_expected.json | 57 + .../xsd/examples/uniref/expected_schema.py | 188 + tests/data/xsd/examples/uniref/maximal.xml | 29 + .../xsd/examples/uniref/maximal_expected.json | 82 + tests/data/xsd/examples/uniref/minimal.xml | 9 + .../xsd/examples/uniref/minimal_expected.json | 13 + tests/data/xsd/nested_choice.xsd | 10 - tests/data/xsd/nested_sequence_deep.xsd | 11 - tests/data/xsd/occurs_variety.xsd | 14 - tests/data/xsd/ref_element.xsd | 10 - .../spec/01_scalars_and_simple_types/data.xml | 50 + .../01_scalars_and_simple_types/expected.json | 81 + .../expected_schema.py | 61 + .../01_scalars_and_simple_types/schema.xsd | 69 + .../spec/02_namespaces_and_nillable/data.xml | 15 + .../02_namespaces_and_nillable/expected.json | 21 + .../expected_schema.py | 38 + .../02_namespaces_and_nillable/schema.xsd | 37 + .../spec/03_mixed_content_raw_text/data.xml | 13 + .../03_mixed_content_raw_text/expected.json | 21 + .../expected_schema.py | 27 + .../spec/03_mixed_content_raw_text/schema.xsd | 35 + .../data/xsd/spec/04_recursion_depth/data.xml | 22 + .../xsd/spec/04_recursion_depth/expected.json | 29 + .../04_recursion_depth/expected_depth_1.json | 18 + .../04_recursion_depth/expected_schema.py | 121 + .../xsd/spec/04_recursion_depth/schema.xsd | 35 + .../spec/05_groups_and_compositors/data.xml | 29 + .../05_groups_and_compositors/expected.json | 36 + .../expected_schema.py | 70 + .../spec/05_groups_and_compositors/schema.xsd | 61 + .../spec/06_context_dependent_paths/data.xml | 14 + .../06_context_dependent_paths/expected.json | 22 + .../expected_schema.py | 69 + .../06_context_dependent_paths/schema.xsd | 43 + .../spec/07_multi_root_shared_type/data.xml | 9 + .../07_multi_root_shared_type/expected.json | 10 + .../expected_schema.py | 12 + .../spec/07_multi_root_shared_type/schema.xsd | 19 + .../08_nullability_and_empty_tags/data.xml | 9 + .../expected.json | 11 + .../expected_schema.py | 27 + .../08_nullability_and_empty_tags/schema.xsd | 32 + .../spec/09_empty_marker_elements/data.xml | 12 + .../09_empty_marker_elements/expected.json | 13 + .../expected_schema.py | 19 + .../spec/09_empty_marker_elements/schema.xsd | 20 + tests/data/xsd/spec/10_combinations/data.xml | 240 + .../xsd/spec/10_combinations/expected.json | 377 ++ .../spec/10_combinations/expected_schema.py | 421 ++ .../data/xsd/spec/10_combinations/schema.xsd | 235 + .../data/xsd/spec/errors/abstract_element.xsd | 11 + tests/data/xsd/spec/errors/abstract_type.xsd | 15 + .../xsd/spec/errors/ambiguous_xml_tag.xsd | 29 + .../spec/errors/attr_child_name_collision.xsd | 11 + .../spec/errors/empty_marker_top_level.xsd | 6 + .../data/xsd/spec/errors/scalar_top_level.xsd | 4 + .../xsd/spec/errors/substitution_group.xsd | 12 + .../spec/errors/wildcard_any_attribute.xsd | 11 + .../xsd/spec/errors/wildcard_any_element.xsd | 11 + 108 files changed, 22407 insertions(+), 567 deletions(-) create mode 100644 docs/xsd_to_pyarrow_specification.md create mode 100644 src/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py create mode 100644 src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py create mode 100644 src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/__init__.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py create mode 100644 tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py delete mode 100644 tests/cdm_data_loaders/readers/conftest.py delete mode 100644 tests/cdm_data_loaders/readers/test_xsd.py create mode 100644 tests/cdm_data_loaders/readers/test_xsd_spec.py delete mode 100644 tests/data/xsd/attribute_only.xsd create mode 100644 tests/data/xsd/cdata_variety.xsd create mode 100644 tests/data/xsd/combinations.xsd create mode 100644 tests/data/xsd/examples/pubmed_article/expected_schema.py create mode 100644 tests/data/xsd/examples/pubmed_article/maximal.xml create mode 100644 tests/data/xsd/examples/pubmed_article/maximal_expected.json create mode 100644 tests/data/xsd/examples/pubmed_article/minimal.xml create mode 100644 tests/data/xsd/examples/pubmed_article/minimal_expected.json create mode 100644 tests/data/xsd/examples/pubmed_book_article/expected_schema.py create mode 100644 tests/data/xsd/examples/pubmed_book_article/maximal.xml create mode 100644 tests/data/xsd/examples/pubmed_book_article/maximal_expected.json create mode 100644 tests/data/xsd/examples/pubmed_book_article/minimal.xml create mode 100644 tests/data/xsd/examples/pubmed_book_article/minimal_expected.json create mode 100644 tests/data/xsd/examples/uniprot/expected_schema.py create mode 100644 tests/data/xsd/examples/uniprot/maximal.xml create mode 100644 tests/data/xsd/examples/uniprot/maximal_expected.json create mode 100644 tests/data/xsd/examples/uniprot/minimal.xml create mode 100644 tests/data/xsd/examples/uniprot/minimal_expected.json create mode 100644 tests/data/xsd/examples/uniref/expected_schema.py create mode 100644 tests/data/xsd/examples/uniref/maximal.xml create mode 100644 tests/data/xsd/examples/uniref/maximal_expected.json create mode 100644 tests/data/xsd/examples/uniref/minimal.xml create mode 100644 tests/data/xsd/examples/uniref/minimal_expected.json delete mode 100644 tests/data/xsd/nested_choice.xsd delete mode 100644 tests/data/xsd/nested_sequence_deep.xsd delete mode 100644 tests/data/xsd/occurs_variety.xsd delete mode 100644 tests/data/xsd/ref_element.xsd create mode 100644 tests/data/xsd/spec/01_scalars_and_simple_types/data.xml create mode 100644 tests/data/xsd/spec/01_scalars_and_simple_types/expected.json create mode 100644 tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py create mode 100644 tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd create mode 100644 tests/data/xsd/spec/02_namespaces_and_nillable/data.xml create mode 100644 tests/data/xsd/spec/02_namespaces_and_nillable/expected.json create mode 100644 tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py create mode 100644 tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd create mode 100644 tests/data/xsd/spec/03_mixed_content_raw_text/data.xml create mode 100644 tests/data/xsd/spec/03_mixed_content_raw_text/expected.json create mode 100644 tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py create mode 100644 tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd create mode 100644 tests/data/xsd/spec/04_recursion_depth/data.xml create mode 100644 tests/data/xsd/spec/04_recursion_depth/expected.json create mode 100644 tests/data/xsd/spec/04_recursion_depth/expected_depth_1.json create mode 100644 tests/data/xsd/spec/04_recursion_depth/expected_schema.py create mode 100644 tests/data/xsd/spec/04_recursion_depth/schema.xsd create mode 100644 tests/data/xsd/spec/05_groups_and_compositors/data.xml create mode 100644 tests/data/xsd/spec/05_groups_and_compositors/expected.json create mode 100644 tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py create mode 100644 tests/data/xsd/spec/05_groups_and_compositors/schema.xsd create mode 100644 tests/data/xsd/spec/06_context_dependent_paths/data.xml create mode 100644 tests/data/xsd/spec/06_context_dependent_paths/expected.json create mode 100644 tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py create mode 100644 tests/data/xsd/spec/06_context_dependent_paths/schema.xsd create mode 100644 tests/data/xsd/spec/07_multi_root_shared_type/data.xml create mode 100644 tests/data/xsd/spec/07_multi_root_shared_type/expected.json create mode 100644 tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py create mode 100644 tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd create mode 100644 tests/data/xsd/spec/08_nullability_and_empty_tags/data.xml create mode 100644 tests/data/xsd/spec/08_nullability_and_empty_tags/expected.json create mode 100644 tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py create mode 100644 tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd create mode 100644 tests/data/xsd/spec/09_empty_marker_elements/data.xml create mode 100644 tests/data/xsd/spec/09_empty_marker_elements/expected.json create mode 100644 tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py create mode 100644 tests/data/xsd/spec/09_empty_marker_elements/schema.xsd create mode 100644 tests/data/xsd/spec/10_combinations/data.xml create mode 100644 tests/data/xsd/spec/10_combinations/expected.json create mode 100644 tests/data/xsd/spec/10_combinations/expected_schema.py create mode 100644 tests/data/xsd/spec/10_combinations/schema.xsd create mode 100644 tests/data/xsd/spec/errors/abstract_element.xsd create mode 100644 tests/data/xsd/spec/errors/abstract_type.xsd create mode 100644 tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd create mode 100644 tests/data/xsd/spec/errors/attr_child_name_collision.xsd create mode 100644 tests/data/xsd/spec/errors/empty_marker_top_level.xsd create mode 100644 tests/data/xsd/spec/errors/scalar_top_level.xsd create mode 100644 tests/data/xsd/spec/errors/substitution_group.xsd create mode 100644 tests/data/xsd/spec/errors/wildcard_any_attribute.xsd create mode 100644 tests/data/xsd/spec/errors/wildcard_any_element.xsd diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 94fdf54b..6fcc9656 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,8 +85,8 @@ jobs: suite: - name: pipelines args: tests/cdm_data_loaders/pipelines/ - - name: docker - args: --run-docker -m "docker and not requires_spark and not requires_ceph" tests/cdm_data_loaders/pipelines/ + # - name: docker + # args: --run-docker -m "docker and not requires_spark and not requires_ceph" tests/cdm_data_loaders/pipelines/ - name: integration args: tests/integration/ - name: everything else diff --git a/docs/xsd_to_pyarrow_specification.md b/docs/xsd_to_pyarrow_specification.md new file mode 100644 index 00000000..59078030 --- /dev/null +++ b/docs/xsd_to_pyarrow_specification.md @@ -0,0 +1,1004 @@ +# XSD-to-PyArrow Specification + +This document specifies how an XML Schema (`.xsd`) and target `xml_tag` determine: +1. The **PyArrow `pa.Schema`** for the table yielded by the pipeline. +2. The **deterministic Python / JSON row dictionaries** produced by schema-driven XML element conversion (`convert_xml_element`) before constructing `pa.Table.from_pylist(rows, schema=pa_schema)`. +3. The **error conditions** raised when an XSD or `xml_tag` uses unsupported or ambiguous constructs. + +--- + +## 1. Summary of Conversion Rules + +### 1.1 Top-Level `xml_tag` Resolution & Unwrapping +- **Resolution:** Search the XSD for `xml_tag` starting at depth 0 (global root elements) and depth 1 (direct children of global root elements), deduplicating matches by `id(element.type)`. + - If **0** elements match (even after falling back to deeper elements), raise an error. + - If **>1** distinct `XsdType` definitions match at the shallowest matching depth, raise an ambiguity error. +- **Unwrapping:** Each streamed `<{xml_tag}>` element becomes one row in the output `pa.Table`. + - `xml_tag` must have a `complexType` that maps to a `pa.StructType` (at least one attribute, `#text`, or child element field). The outer `xml_tag` key is unwrapped so that the top-level columns of `pa.Schema` are the struct fields of `xml_tag`. + - If `xml_tag` maps to a scalar, list, or empty marker type with no attributes and no struct fields, schema generation raises a `ValueError` (see Error E3 in Section 3). + +### 1.2 Naming & Field Ordering +- **Attributes:** Prefixed with `_` (e.g., `id` $\rightarrow$ `_id`). Namespace declarations (`xmlns`, `xmlns:*`) and `xsi:*` attributes (`xsi:nil`, `xsi:schemaLocation`, `xsi:noNamespaceSchemaLocation`) are stripped. +- **Text content (`#text`):** When a complex type has both attributes and text (`simpleContent` with attributes, or `mixed="true"` with attributes), text is stored under `#text`. +- **Struct Field Order:** + 1. Declared attributes (`_attr`), in XSD declaration order (base type attributes before extension attributes). + 2. `#text` (if applicable). + 3. Child elements, in XSD content-model traversal order (deduplicated on first appearance if the same element name appears in multiple branches of a `` or ``). + +### 1.3 Element Classification (Shape) +| XSD Element Type | Cardinality (`overall_max_occurs`) | Python / JSON Value | PyArrow DataType | +| :--- | :--- | :--- | :--- | +| **`xs:simpleType`** (atomic) | `1` | Scalar (`str`, `int`, `float`, `bool`, `date`, `bytes`) or `None` | Scalar `pa.DataType` | +| **`xs:simpleType`** (atomic) | `> 1` or `unbounded` | `list[Scalar \| None]` or `None` (when absent) | `pa.list_(Scalar)` | +| **`xs:simpleType`** (`xs:list`) | `1` | `list[ItemScalar]` or `None` | `pa.list_(ItemScalar)` | +| **`xs:simpleType`** (`xs:list`) | `> 1` or `unbounded` | `list[list[ItemScalar] \| None]` or `None` | `pa.list_(pa.list_(ItemScalar))` | +| **`complexType` (`simpleContent`)** without attributes | `1` / `> 1` | Same as underlying `simpleType` | Same as underlying `simpleType` | +| **`complexType` (`simpleContent`)** with attributes | `1` | `{"_attr": ..., "#text": Scalar}` or `None` | `pa.struct([("_attr", ...), ("#text", Scalar)])` | +| **`complexType` (`mixed="true"`)** without attributes | `1` | Raw inner XML string `str` or `None` | `pa.string()` | +| **`complexType` (`mixed="true"`)** with attributes | `1` | `{"_attr": ..., "#text": str}` or `None` | `pa.struct([("_attr", ...), ("#text", pa.string())])` | +| **`complexType` (attribute-only)** | `1` | `{"_attr": ...}` or `None` | `pa.struct([("_attr", ...)])` | +| **`complexType` (element-only)** | `1` | `{"_attr": ..., "child": ...}` or `None` | `pa.struct([...])` | +| **`complexType` (empty: no attrs, no children, not mixed)** | `1` / `> 1` | `""` (when present) or `None` (when absent) / `["", ...]` | `pa.string()` / `pa.list_(pa.string())` | + +*(For any `complexType` row above with `overall_max_occurs > 1` or `unbounded`, the Python value is `list[...]` and the PyArrow type is `pa.list_(...)`. When structural recursion exceeds `max_recursion_depth`, a single recursive element becomes `pa.string()` and a repeated recursive element (`overall_max_occurs > 1`) becomes `pa.list_(pa.string())`, where each truncated item is serialized as a JSON string.)* + +### 1.4 Field Nullability (`nullable=False` vs `nullable=True`) & Empty Tag Coercion +- **Attributes (`_attr`):** + - `nullable=False` when `use="required"` (and the enclosing element is not a `nillable="true"` element that can be nilled without attributes). + - `nullable=True` when `use="optional"` (default). +- **Child Elements:** + - `nullable=False` when **all** of the following hold: + 1. The element particle has `minOccurs >= 1`. + 2. The element declaration has `nillable="false"` (default). + 3. Every enclosing compositor/group (``, ``, ``) up to the parent `complexType` has `minOccurs >= 1`, and the element is not conditional inside an `` branch (unless required in every branch of the choice). + 4. The parent element is not `nillable="true"` (since a parent with `xsi:nil="true"` and attributes, e.g. ``, produces a struct `{"_type": "function"}` whose child elements are `None`). + - `nullable=True` in all other cases. +- **`#text` Field Inside a Struct with Attributes:** + - `nullable=True` for `mixed="true"` types, `nillable="true"` elements, and string-based `simpleContent` types (where `` has no `#text` key). + - `nullable=False` for non-nillable `simpleContent` types with non-string atomic base types (e.g., `xs:int`, `xs:decimal`), where valid XML always provides lexical text. +- **Empty Standalone Tag Coercion (`` $\rightarrow$ `""`):** + - When a standalone element mapped to `pa.string()` (atomic string `simpleType`, string `simpleContent` without attributes, `mixed="true"` without attributes, or empty `complexType` marker) is **present** in the XML as `` or `` (without `xsi:nil="true"`), its value is coerced to `""` so that present empty elements satisfy `nullable=False` and are distinguishable from omitted optional elements (`None`). + +### 1.5 Built-In XSD Scalar Type Mapping & Field Metadata +| XSD Built-In Type(s) | Python Type (Coerced) | PyArrow `DataType` | +| :--- | :--- | :--- | +| `xs:string`, `xs:normalizedString`, `xs:token`, `xs:language`, `xs:Name`, `xs:NCName`, `xs:NMTOKEN`, `xs:ID`, `xs:IDREF`, `xs:ENTITY`, `xs:anyURI`, `xs:QName`, `xs:NOTATION` | `str` | `pa.string()` | +| `xs:boolean` (`"true"`/`"1"` $\rightarrow$ `True`, `"false"`/`"0"` $\rightarrow$ `False`) | `bool` | `pa.bool_()` | +| `xs:byte` | `int` | `pa.int8()` | +| `xs:short` | `int` | `pa.int16()` | +| `xs:int` | `int` | `pa.int32()` | +| `xs:long`, `xs:integer`, `xs:positiveInteger`, `xs:nonNegativeInteger`, `xs:negativeInteger`, `xs:nonPositiveInteger` | `int` | `pa.int64()` | +| `xs:unsignedByte` | `int` | `pa.uint8()` | +| `xs:unsignedShort` | `int` | `pa.uint16()` | +| `xs:unsignedInt` | `int` | `pa.uint32()` | +| `xs:unsignedLong` | `int` | `pa.uint64()` | +| `xs:float`, `xs:double`, `xs:decimal` | `float` | `pa.float64()` | +| `xs:date` | `datetime.date` | `pa.date32()` | +| `xs:dateTime`, `xs:time`, `xs:duration`, `xs:gYear`, `xs:gYearMonth`, `xs:gMonthDay`, `xs:gDay`, `xs:gMonth` | `str` | `pa.string()` | +| `xs:base64Binary` | `bytes` (`base64.b64decode`) | `pa.binary()` | +| `xs:hexBinary` | `bytes` (`bytes.fromhex`) | `pa.binary()` | +| `xs:NMTOKENS`, `xs:IDREFS`, `xs:ENTITIES` | `list[str]` (`.split()`) | `pa.list_(pa.string())` | + +- **`xs:restriction`:** Inherits the primitive/built-in base type mapping above. +- **`xs:list`:** Maps to `pa.list_(item_type)` and coerces by splitting whitespace (`value.split()`) and coercing each item. +- **`xs:union`:** If all `memberTypes` map to the same `pa.DataType`, uses that type; otherwise falls back to `pa.string()` (`str`). +- **`pa.Field.metadata`:** + - **`"xsd_type"`:** Recorded on every leaf scalar `pa.Field` (including `#text` and list `"item"` fields when list elements are scalars) with the prefixed built-in XSD type name (e.g., `"xs:string"`, `"xs:int"`, `"xs:float"`, `"xs:double"`, `"xs:decimal"`, or `"xs:date | xs:gYearMonth | xs:gYear"` for unions). Omitted on `pa.struct(...)`, outer `pa.list_(...)`, empty-marker `pa.string()`, and recursion-truncated JSON `pa.string()` fields. + - **`"comment"`:** Extracted from `` on the element or attribute declaration (falling back to its referenced `type`'s `` if the declaration itself has none), whitespace-normalized with `" ".join(doc.split())`. + +--- + +## 2. Specification Examples + +### Example 1: Scalar Types, Restrictions, Lists (`xs:list`), and Unions (`xs:union`) +Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumerations, `xs:list` (`intListType` from [`uniprot.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd#L1122-L1124)), and `xs:union` (`date | gYearMonth | gYear` from [`uniprot.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd#L341-L345)). + +#### XSD +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +#### Sample XML (`xml_tag="record"`) +```xml + + + true + -5 + 1024 + 100000 + 999999999999 + 18446744073709551615 + 1.5 + 2.718281828 + 3.14 + 2025-06-15 + 2025-06 + 42 + 10 20 30 + 48656c6c6f + SGVsbG8= + + + 0 + + +``` + +#### Python / JSON Translation +```python +[ + { + "_id": "rec1", + "_status": "active", + "_evidence": [1, 4, 9], + "_version": 2, + "flag": True, + "smallInt": -5, + "medInt": 1024, + "stdInt": 100000, + "bigInt": 999999999999, + "ulong": 18446744073709551615, + "f32": 1.5, + "f64": 2.718281828, + "dec": 3.14, + "created": datetime.date(2025, 6, 15), + "published": "2025-06", + "code": "42", # int + short -> different Arrow types (int32 vs int16) -> falls back to str + "scores": [10, 20, 30], + "rawHex": b"Hello", + "rawB64": b"Hello", + }, + { + "_id": "rec2", + "_status": "archived", + "_version": 1, + "flag": False, + }, +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:ID"}), + pa.field("_status", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_evidence", pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), nullable=True), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("flag", pa.bool_(), nullable=False, metadata={"xsd_type": "xs:boolean"}), + pa.field("smallInt", pa.int8(), nullable=True, metadata={"xsd_type": "xs:byte"}), + pa.field("medInt", pa.int16(), nullable=True, metadata={"xsd_type": "xs:short"}), + pa.field("stdInt", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field("bigInt", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"}), + pa.field("ulong", pa.uint64(), nullable=True, metadata={"xsd_type": "xs:unsignedLong"}), + pa.field("f32", pa.float64(), nullable=True, metadata={"xsd_type": "xs:float"}), + pa.field("f64", pa.float64(), nullable=True, metadata={"xsd_type": "xs:double"}), + pa.field("dec", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), + pa.field("created", pa.date32(), nullable=True, metadata={"xsd_type": "xs:date"}), + pa.field("published", pa.string(), nullable=True, metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}), + pa.field("code", pa.string(), nullable=True, metadata={"xsd_type": "xs:int | xs:short"}), + pa.field("scores", pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), nullable=True), + pa.field("rawHex", pa.binary(), nullable=True, metadata={"xsd_type": "xs:hexBinary"}), + pa.field("rawB64", pa.binary(), nullable=True, metadata={"xsd_type": "xs:base64Binary"}), +]) +``` +*(In Examples 2–9 below, `metadata={"xsd_type": ...}` on scalar fields is omitted for brevity.)* + +--- + +### Example 2: Namespaces & `nillable="true"` (`xsi:nil="true"`) +Covers `targetNamespace`, `elementFormDefault="qualified"`, stripping `xmlns` / `xmlns:xsi`, and the 3 cases of `xsi:nil="true"`: +1. Nillable element with **no other attributes** $\rightarrow$ `None`. +2. Nillable complex element **with attributes** (like `` in [`uniprot.xsd` L47](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd#L47)) $\rightarrow$ `{"_type": "function"}` (with `_xsi:nil` stripped and child fields `null`). +3. Nillable `simpleContent` element **with attributes** $\rightarrow$ `{"_lang": "en"}` (`#text` is `null`). + +#### XSD +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +#### Sample XML (`xml_tag="entry"`) +```xml + + + + <label lang="en" xsi:nil="true"/> + <comment type="function" xsi:nil="true"/> + <comment type="subunit"> + <text>Homodimer</text> + </comment> + </entry> + <entry id="e2"> + <title>Normal Title + + +``` + +#### Python / JSON Translation +```python +[ + { + "_id": "e1", + "title": None, + "label": {"_lang": "en"}, + "comment": [ + {"_type": "function"}, + {"_type": "subunit", "text": "Homodimer"}, + ], + }, + { + "_id": "e2", + "title": "Normal Title", + "label": None, + }, +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("_id", pa.string(), nullable=False), + pa.field("title", pa.string(), nullable=True), + pa.field("label", pa.struct([ + pa.field("_lang", pa.string(), nullable=True), + pa.field("#text", pa.string(), nullable=True), + ]), nullable=True), + pa.field("comment", pa.list_(pa.struct([ + pa.field("_type", pa.string(), nullable=False), + pa.field("text", pa.string(), nullable=True), + ])), nullable=True), +]) +``` + +--- + +### Example 3: Mixed Content Kept as Raw Text +Covers `mixed="true"` elements with inline HTML/formatting tags (``, ``, ``, ``, ``), both with and without attributes on the mixed element itself (as in [`pubmed.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd) `ArticleTitle`, `AbstractText`, and [`10_combinations/schema.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd) `ParaType`, `NoteType`, `RemarkType`). + +- Child elements declared inside a `mixed="true"` complexType are **not** emitted as struct fields; instead, the mixed element's entire inner XML is captured as a raw text string. +- Character entities in text nodes (`&`, `<`, `>`) remain XML-escaped in the captured raw text string so that literal `<` or `>` is distinguishable from inline HTML tags like ``. +- If the `mixed="true"` type has **no attributes**, its PyArrow type is `pa.string()` (and an empty tag `` without `xsi:nil="true"` coerces to `""`). +- If the `mixed="true"` type **has attributes**, its PyArrow type is `pa.struct([("_attr", ...), ("#text", pa.string())])`. + +#### XSD +```xml +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:group name="inlineFormatting"> + <xs:choice> + <xs:element name="b" type="formattedText"/> + <xs:element name="i" type="formattedText"/> + <xs:element name="sup" type="formattedText"/> + </xs:choice> + </xs:group> + + <xs:complexType name="formattedText" mixed="true"> + <xs:group ref="inlineFormatting" minOccurs="0" maxOccurs="unbounded"/> + </xs:complexType> + + <xs:complexType name="abstractTextType" mixed="true"> + <xs:group ref="inlineFormatting" minOccurs="0" maxOccurs="unbounded"/> + <xs:attribute name="Label" type="xs:string"/> + </xs:complexType> + + <xs:element name="articles"> + <xs:complexType> + <xs:sequence> + <xs:element name="article" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="title" type="formattedText"/> + <xs:element name="abstractText" type="abstractTextType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> +``` + +#### Sample XML (`xml_tag="article"`) +```xml +<articles> + <article> + <title> Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells + Studied bold & italic text. + Unlabelled paragraph with tags. + + + +
+ + </article> +</articles> +``` + +#### Python / JSON Translation +```python +[ + { + "title": "Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells", + "abstractText": [ + {"_Label": "BACKGROUND", "#text": "Studied <b>bold</b> & <i>italic</i> text."}, + {"#text": "Unlabelled paragraph with <b>tags</b>."}, + {"_Label": "EMPTY"}, + None, + ], + }, + { + "title": "", + }, +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("title", pa.string(), nullable=False), + pa.field("abstractText", pa.list_(pa.struct([ + pa.field("_Label", pa.string(), nullable=True), + pa.field("#text", pa.string(), nullable=True), + ])), nullable=True), +]) +``` + +--- + +### Example 4: Structural Recursion with Configurable `max_recursion_depth` +Covers both: +1. **Type self-recursion** (`TreeType` $\rightarrow$ `branch: TreeType`) +2. **Element-ref self-recursion** (`Section` $\rightarrow$ `<xs:element ref="Section"/>` as in [`pubmed.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd)) +3. **Mutual recursion** (`NodeA` $\rightarrow$ `NodeB` $\rightarrow$ `NodeA`) + +With `max_recursion_depth = 2`: +- Occurrence 1 (top-level `Section` or `tree`) = depth 1 (`pa.struct`) +- Occurrence 2 (child `Section` or `branch`) = depth 2 (`pa.struct`) +- Occurrence 3 (grandchild `Section` or `branch` beyond `max_recursion_depth=2`) = flattened to JSON string (`pa.string()` when `overall_max_occurs == 1`, or `pa.list_(pa.string())` when `overall_max_occurs > 1`). + +#### XSD +```xml +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="TreeType"> + <xs:sequence> + <xs:element name="branch" type="TreeType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + <xs:attribute name="label" type="xs:string"/> + </xs:complexType> + + <xs:element name="doc"> + <xs:complexType> + <xs:sequence> + <xs:element name="record" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="tree" type="TreeType" minOccurs="0"/> + </xs:sequence> + <xs:attribute name="id" type="xs:string" use="required"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> +``` + +#### Sample XML (`xml_tag="record"`, `max_recursion_depth=2`) +```xml +<doc> + <record id="r1"> + <tree label="L1"> + <branch label="L2"> + <branch label="L3"> + <branch label="L4"/> + </branch> + <branch label="L3_sibling"/> + </branch> + </tree> + </record> +</doc> +``` + +#### Python / JSON Translation (`max_recursion_depth=2`) +```python +[ + { + "_id": "r1", + "tree": { + "_label": "L1", + "branch": [ + { + "_label": "L2", + "branch": [ + '{"_label": "L3", "branch": [{"_label": "L4"}]}', + '{"_label": "L3_sibling"}', + ], + } + ], + }, + } +] +``` + +#### PyArrow Schema (`max_recursion_depth=2`) +```python +pa.schema([ + pa.field("_id", pa.string(), nullable=False), + pa.field("tree", pa.struct([ # depth 1 of TreeType + pa.field("_label", pa.string(), nullable=True), + pa.field("branch", pa.list_(pa.struct([ # depth 2 of TreeType + pa.field("_label", pa.string(), nullable=True), + pa.field("branch", pa.list_(pa.string()), nullable=True), # depth > 2 -> JSON string per item + ])), nullable=True), + ]), nullable=True), +]) +``` + +--- + +### Example 5: Model Groups (`xs:group`), Attribute Groups (`xs:attributeGroup`), `<xs:all>`, and Compositors with Duplicate Element Names +Covers: +- `<xs:group ref="..."/>` (including with `maxOccurs="unbounded"`) +- `<xs:attributeGroup ref="..."/>` +- Unrepeated `<xs:choice>` where the **same element name** appears in multiple branches (e.g., `(StartPage, EndPage?, MedlinePgn?) | MedlinePgn` from [`pubmed.xsd` L672–679](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd#L672-L679)) $\rightarrow$ `MedlinePgn` has `overall_max_occurs == 1`, so it is a **single** `pa.string()` field emitted once. +- `<xs:sequence>` where the **same element name** appears twice (`<xs:element name="interactant"/><xs:element name="interactant"/>`) $\rightarrow$ `overall_max_occurs == 2`, so it is a **list** field emitted once. +- `<xs:all>` compositor. + +#### XSD +```xml +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:attributeGroup name="auditAttrs"> + <xs:attribute name="createdBy" type="xs:string"/> + <xs:attribute name="version" type="xs:int"/> + </xs:attributeGroup> + + <xs:group name="kvGroup"> + <xs:sequence> + <xs:element name="k" type="xs:string"/> + <xs:element name="v" type="xs:int"/> + </xs:sequence> + </xs:group> + + <xs:complexType name="paginationType"> + <xs:choice> + <xs:sequence> + <xs:element name="StartPage" type="xs:string"/> + <xs:element name="EndPage" type="xs:string" minOccurs="0"/> + <xs:element name="MedlinePgn" type="xs:string" minOccurs="0"/> + </xs:sequence> + <xs:element name="MedlinePgn" type="xs:string"/> + </xs:choice> + </xs:complexType> + + <xs:complexType name="metadataAllType"> + <xs:all> + <xs:element name="author" type="xs:string"/> + <xs:element name="year" type="xs:int" minOccurs="0"/> + </xs:all> + </xs:complexType> + + <xs:element name="dataset"> + <xs:complexType> + <xs:sequence> + <xs:element name="item" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="pagination" type="paginationType" minOccurs="0"/> + <xs:element name="meta" type="metadataAllType" minOccurs="0"/> + <xs:element name="pairs" minOccurs="0"> + <xs:complexType> + <xs:group ref="kvGroup" maxOccurs="unbounded"/> + </xs:complexType> + </xs:element> + </xs:sequence> + <xs:attributeGroup ref="auditAttrs"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> +``` + +#### Sample XML (`xml_tag="item"`) +```xml +<dataset> + <item createdBy="alice" version="1"> + <pagination> + <StartPage>10</StartPage> + <EndPage>20</EndPage> + <MedlinePgn>10-20</MedlinePgn> + </pagination> + <meta> + <year>2025</year> + <author>Bob</author> + </meta> + <pairs> + <k>a</k> + <v>1</v> + <k>b</k> + <v>2</v> + </pairs> + </item> + <item> + <pagination> + <MedlinePgn>e12345</MedlinePgn> + </pagination> + </item> +</dataset> +``` + +#### Python / JSON Translation +```python +[ + { + "_createdBy": "alice", + "_version": 1, + "pagination": { + "StartPage": "10", + "EndPage": "20", + "MedlinePgn": "10-20", + }, + "meta": { + "author": "Bob", + "year": 2025, + }, + "pairs": { + "k": ["a", "b"], + "v": [1, 2], + }, + }, + { + "pagination": { + "MedlinePgn": "e12345", + }, + }, +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("_createdBy", pa.string(), nullable=True), + pa.field("_version", pa.int32(), nullable=True), + pa.field("pagination", pa.struct([ + pa.field("StartPage", pa.string(), nullable=True), + pa.field("EndPage", pa.string(), nullable=True), + pa.field("MedlinePgn", pa.string(), nullable=True), + ]), nullable=True), + pa.field("meta", pa.struct([ + pa.field("author", pa.string(), nullable=False), + pa.field("year", pa.int32(), nullable=True), + ]), nullable=True), + pa.field("pairs", pa.struct([ + pa.field("k", pa.list_(pa.string()), nullable=False), + pa.field("v", pa.list_(pa.int32()), nullable=False), + ]), nullable=True), +]) +``` + +--- + +### Example 6: Context-Dependent Same `(parent, child)` Names with Different Types +In [`06_context_dependent_paths/schema.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd) and [`10_combinations/schema.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd#L195-L226), `left/node/leaf` is a single `xs:string` while `right/node/leaf` is an unbounded `LabelType` (complex `simpleContent` with attribute `lang`). +Because the full ancestor path from `xml_tag` (`("root", "left", "node")` vs `("root", "right", "node")`) distinguishes them, each gets its exact PyArrow type instead of merging `("node", "leaf")` globally. + +#### XSD +```xml +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:element name="root"> + <xs:complexType> + <xs:sequence> + <xs:element name="left"> + <xs:complexType> + <xs:sequence> + <xs:element name="node"> + <xs:complexType> + <xs:sequence> + <xs:element name="leaf" type="xs:string"/> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + <xs:element name="right"> + <xs:complexType> + <xs:sequence> + <xs:element name="node"> + <xs:complexType> + <xs:sequence> + <xs:element name="leaf" maxOccurs="unbounded"> + <xs:complexType> + <xs:simpleContent> + <xs:extension base="xs:string"> + <xs:attribute name="lang" type="xs:string"/> + </xs:extension> + </xs:simpleContent> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> +``` + +#### Sample XML (`xml_tag="root"`) +```xml +<root> + <left> + <node> + <leaf>l1</leaf> + </node> + </left> + <right> + <node> + <leaf>r1</leaf> + <leaf lang="en">r2</leaf> + </node> + </right> +</root> +``` + +#### Python / JSON Translation +```python +[ + { + "left": { + "node": { + "leaf": "l1", + } + }, + "right": { + "node": { + "leaf": [ + {"#text": "r1"}, + {"_lang": "en", "#text": "r2"}, + ] + } + }, + } +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("left", pa.struct([ + pa.field("node", pa.struct([ + pa.field("leaf", pa.string(), nullable=False), + ]), nullable=False), + ]), nullable=False), + pa.field("right", pa.struct([ + pa.field("node", pa.struct([ + pa.field("leaf", pa.list_(pa.struct([ + pa.field("_lang", pa.string(), nullable=True), + pa.field("#text", pa.string(), nullable=True), + ])), nullable=False), + ]), nullable=False), + ]), nullable=False), +]) +``` + +--- + +### Example 7: Multiple Root Elements Sharing the Same Child `xml_tag` (`uniref.xsd` Pattern) +In [`uniref.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniref/uniref.xsd), four root elements (`UniRef100`, `UniRef90`, `UniRef50`, `UniRef`) each declare a child `<xs:element name="entry" type="entryType"/>`. Because all four `entry` declarations share the same `entryType`, resolving `xml_tag="entry"` succeeds unambiguously. + +--- + +### Example 8: Field Nullability & Empty String Tag Coercion +Covers: +1. Required attributes (`use="required"` $\rightarrow$ `nullable=False`) vs optional attributes (`use="optional"` $\rightarrow$ `nullable=True`). +2. Required non-nillable elements (`minOccurs="1"`, `nillable="false"` $\rightarrow$ `nullable=False`), with present empty string tags `<summary/>` coerced to `""`. +3. Elements inside `<xs:choice>` branches (`nullable=True`). +4. Children of `nillable="true"` complex elements (`nullable=True` so `<comment type="function" xsi:nil="true"/>` can omit child `text`). + +#### XSD +```xml +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="commentType"> + <xs:sequence> + <xs:element name="text" type="xs:string"/> + </xs:sequence> + <xs:attribute name="type" type="xs:string" use="required"/> + </xs:complexType> + + <xs:element name="root"> + <xs:complexType> + <xs:sequence> + <xs:element name="entry" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="name" type="xs:string"/> + <xs:element name="summary" type="xs:string"/> + <xs:element name="optionalNote" type="xs:string" minOccurs="0"/> + <xs:choice> + <xs:element name="geneId" type="xs:int"/> + <xs:element name="proteinId" type="xs:string"/> + </xs:choice> + <xs:element name="comment" type="commentType" nillable="true"/> + </xs:sequence> + <xs:attribute name="id" type="xs:string" use="required"/> + <xs:attribute name="source" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> +``` + +#### Sample XML (`xml_tag="entry"`) +```xml +<root xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <entry id="e1"> + <name>BRCA1</name> + <summary/> + <geneId>672</geneId> + <comment type="function" xsi:nil="true"/> + </entry> +</root> +``` + +#### Python / JSON Translation +```python +[ + { + "_id": "e1", + "name": "BRCA1", + "summary": "", # coerced from None -> "" to satisfy nullable=False + "geneId": 672, + "comment": {"_type": "function"}, + } +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("_id", pa.string(), nullable=False), # use="required" + pa.field("_source", pa.string(), nullable=True), # use="optional" + pa.field("name", pa.string(), nullable=False), # minOccurs=1, non-nillable + pa.field("summary", pa.string(), nullable=False), # minOccurs=1, non-nillable + pa.field("optionalNote", pa.string(), nullable=True), # minOccurs=0 + pa.field("geneId", pa.int32(), nullable=True), # inside xs:choice -> nullable=True + pa.field("proteinId", pa.string(), nullable=True), # inside xs:choice -> nullable=True + pa.field("comment", pa.struct([ + pa.field("_type", pa.string(), nullable=False), # use="required" + pa.field("text", pa.string(), nullable=True), # parent is nillable="true" -> nullable=True + ]), nullable=True), # nillable="true" -> nullable=True +]) +``` + +--- + +### Example 9: Purely Empty `complexType` Marker Elements +Covers DTD-derived empty marker elements like [`pubmed.xsd` L1312](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd#L1312) (`<BookInProcess/>`) and [`10_combinations/schema.xsd` L11–12](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd#L11-L12) (`EmptyType`): +- Mapped to `pa.string()` (or `pa.list_(pa.string())` when `overall_max_occurs > 1`). +- Coerced to `""` when present in the XML, and `None` when omitted. + +#### XSD +```xml +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="EmptyType"/> + + <xs:element name="books"> + <xs:complexType> + <xs:sequence> + <xs:element name="book" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="title" type="xs:string"/> + <xs:element name="BookInProcess" type="EmptyType" minOccurs="0"/> + <xs:element name="flag" type="EmptyType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> +``` + +#### Sample XML (`xml_tag="book"`) +```xml +<books> + <book> + <title>In-Process Book + + + + + + Completed Book + + +``` + +#### Python / JSON Translation +```python +[ + { + "title": "In-Process Book", + "BookInProcess": "", + "flag": ["", ""], + }, + { + "title": "Completed Book", + }, +] +``` + +#### PyArrow Schema +```python +pa.schema([ + pa.field("title", pa.string(), nullable=False), + pa.field("BookInProcess", pa.string(), nullable=True), + pa.field("flag", pa.list_(pa.string()), nullable=True), +]) +``` + +--- + +## 3. Error Specification (Unsupported / Invalid XSD Features) + +The converter raises a descriptive `ValueError` during schema generation when encountering any of the following: + +| # | Error Case | Example XSD Pattern | Reason | +| :--- | :--- | :--- | :--- | +| **E1** | **`xml_tag` Not Found** | `` or `xml_tag="nonexistent"` | Target element does not exist in the XSD. | +| **E2** | **Ambiguous `xml_tag` Type** | Root `A` has `` and root `B` has `` | Cannot determine a single `pa.Schema` for `xml_tag="entry"` when declarations at the same resolution depth have different types. | +| **E3** | **Non-Struct Top-Level `xml_tag`** | `` or empty `` with no attributes/children | A `pa.Table` schema requires top-level columns (at least one attribute, child element, or `#text` field). | +| **E4** | **Element Wildcard (``)** | `` | Arbitrary undeclared child elements cannot be represented as fixed fields in a `pa.struct`. | +| **E5** | **Attribute Wildcard (``)** | `` | Arbitrary undeclared attributes cannot be represented as fixed fields in a `pa.struct`. | +| **E6** | **Abstract Element or Abstract Type (`abstract="true"`)** | `` used as an element's type, or `` | Concrete runtime instances require `xsi:type` or `substitutionGroup` polymorphism with varying schemas. | +| **E7** | **Substitution Group with Heterogeneous Types** | `` | Substituted elements introduce different element names/types not declared in the parent's content model. | +| **E8** | **Prefixed Attribute / Child Name Collision** | Element has both `` (mapped to `_id`) and `` | Both map to field name `"_id"` in the same `pa.struct`. | +| **E9** | **Non-JSON-Serializable Scalar Type Under Recursion** | Recursive element subtree contains `xs:date`, `xs:dateTime`, `xs:hexBinary`, or `xs:base64Binary` | Elements truncated beyond `max_recursion_depth` are serialized to JSON (`json.dumps`), which does not natively serialize `datetime.date` or `bytes` without lossy/ambiguous string encoding. | + +--- + +## 4. Generated Specification Fixtures, Real-World Examples & Tests + +### 4.1 Specification Fixtures (`tests/data/xsd/spec/`) + +| Suite | `xml_tag` | XSD | Expected PyArrow Schema | Sample XML | Expected JSON | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **`01_scalars_and_simple_types`** | `record` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/expected.json) | +| **`02_namespaces_and_nillable`** | `entry` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/expected.json) | +| **`03_mixed_content_raw_text`** | `article` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/expected.json) | +| **`04_recursion_depth`** | `record` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/expected.json), [expected_depth_1.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/expected_depth_1.json) | +| **`05_groups_and_compositors`** | `item` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/expected.json) | +| **`06_context_dependent_paths`** | `root` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/expected.json) | +| **`07_multi_root_shared_type`** | `entry` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/expected.json) | +| **`08_nullability_and_empty_tags`** | `entry` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/expected.json) | +| **`09_empty_marker_elements`** | `book` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/expected.json) | +| **`10_combinations`** | `record` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/expected.json) | + +### 4.2 Error Specification Fixtures (`tests/data/xsd/spec/errors/`) + +- [ambiguous_xml_tag.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd) (E2) +- [scalar_top_level.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/scalar_top_level.xsd) (E3) +- [empty_marker_top_level.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/empty_marker_top_level.xsd) (E3) +- [wildcard_any_element.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/wildcard_any_element.xsd) (E4) +- [wildcard_any_attribute.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/wildcard_any_attribute.xsd) (E5) +- [abstract_element.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/abstract_element.xsd) (E6) +- [abstract_type.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/abstract_type.xsd) (E6) +- [substitution_group.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/substitution_group.xsd) (E7) +- [attr_child_name_collision.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/attr_child_name_collision.xsd) (E8) +- [nested_date_under_recursion.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/nested_date_under_recursion.xsd) (E9) +- [nested_binary_under_recursion.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd) (E9) + +### 4.3 Real-World Minimal & Maximal Examples (`tests/data/xsd/examples/`) + +| Dataset | `xml_tag` | XSD | Expected PyArrow Schema | Minimal XML & Expected JSON | Maximal XML & Expected JSON | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **UniRef** | `entry` | [uniref.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniref/uniref.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/maximal_expected.json) | +| **UniProtKB** | `entry` | [uniprot.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/maximal_expected.json) | +| **PubMed (`PubmedArticle`)** | `PubmedArticle` | [pubmed.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/maximal_expected.json) | +| **PubMed (`PubmedBookArticle`)** | `PubmedBookArticle` | [pubmed.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/maximal_expected.json) | + +### 4.4 Specification Test Suite + +- [test_xsd_spec.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/cdm_data_loaders/readers/test_xsd_spec.py): Validates all fixtures against their XSDs, tests `generate_pyarrow_schema`, tests error cases E1–E9, tests `convert_xml_element` + PyArrow Table / Parquet round-trips, and tests `process_xml_file_to_dict` yielding `pa.Table` batches. + diff --git a/pyproject.toml b/pyproject.toml index 2efdf762..76631030 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ pdb_manifest = "cdm_data_loaders.pipelines.pdb_manifest:cli" uniprot_cdm = "cdm_data_loaders.pipelines.uniprot_kb_cdm:cli" uniref_cdm = "cdm_data_loaders.pipelines.uniref_cdm:cli" xmltodict = "cdm_data_loaders.pipelines.xmltodict.pipeline:cli" +xmltopyarrow = "cdm_data_loaders.pipelines.xmltopyarrow.pipeline:cli" xml2db = "cdm_data_loaders.pipelines.xml2db.pipeline:cli" xsv = "cdm_data_loaders.pipelines.xsv.pipeline:cli" hello_world = "cdm_data_loaders.utils.hello_world:cli" diff --git a/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py b/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py index dfe57fa7..ea22f315 100644 --- a/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py @@ -21,12 +21,14 @@ logger: Logger = getLogger(__name__) -def _read_items(items: Iterator[FileItemDict], settings: XmlToDictSettings) -> Generator[TDataItems, Any, Any]: +def _read_xmltodict_items( + items: Iterator[FileItemDict], settings: XmlToDictSettings +) -> Generator[TDataItems, Any, Any]: """Read each file in items. Yields one dict per `xml_tag` element. :param items: file items to read :type items: Iterator[FileItemDict] - :param settings: pipeline config, including xml_tag and xsd-derived paths + :param settings: pipeline config, including xml_tag :type settings: XmlToDictSettings :yield: one dict per matched element :rtype: Generator[TDataItems, Any, Any] @@ -41,8 +43,7 @@ def run_xmltodict_pipeline(settings: XmlToDictSettings) -> LoadInfo | None: :param settings: pipeline configuration :type settings: XmlToDictSettings """ - reader = dlt.transformer(_read_items, name="xmltodict_reader", parallelized=True) - + reader = dlt.transformer(_read_xmltodict_items, name="xmltodict_reader", parallelized=True) files = filesystem(bucket_url=str(settings.input_dir), file_glob=settings.file_glob) xmltodict_resource: DltResource = files | reader(settings) diff --git a/src/cdm_data_loaders/pipelines/xmltodict/settings.py b/src/cdm_data_loaders/pipelines/xmltodict/settings.py index 22eed4cb..11436f5f 100644 --- a/src/cdm_data_loaders/pipelines/xmltodict/settings.py +++ b/src/cdm_data_loaders/pipelines/xmltodict/settings.py @@ -1,9 +1,8 @@ """Settings class for the XML to dictionary pipeline for the KBase CTS.""" -from functools import cached_property -from typing import Annotated, Any, Final, Self +from typing import Annotated, Final -from pydantic import Field, FilePath, PrivateAttr, computed_field, model_validator +from pydantic import Field from pydantic_settings import SettingsConfigDict from cdm_data_loaders.core.fields import ( @@ -17,7 +16,6 @@ FileFormatIngestSettings, default_settings_with_shortcuts, ) -from cdm_data_loaders.readers.xsd import find_list_and_single_child_paths, load_schema PIPELINE_NAME: Final[str] = "xmltodict" @@ -33,9 +31,6 @@ class XmlToDictSettings(FileFormatIngestSettings): }, ) - _paths: Annotated[dict[str, Any], Field(description="parent-child paths in the schema")] = PrivateAttr( - default_factory=dict - ) file_glob: Annotated[ FileGlob, Field( @@ -50,45 +45,4 @@ class XmlToDictSettings(FileFormatIngestSettings): description="XML tag to capture the contents of", ), ] - xsd_file: Annotated[ - FilePath | None, - Field( - None, - description="Path to the XSD file describing the XML documents to be imported. Including the schema will ensure that the appropriate relationships (1-to-1 vs 1-to-many) are created between parent and child elements.", - ), - ] - - @model_validator(mode="after") - def check_xsd_file(self) -> Self: - """Check whether the schema file is present and correct; if so, init self._paths.""" - if self.xsd_file: - try: - schema = load_schema(self.xsd_file) - list_paths, single_paths = find_list_and_single_child_paths(schema) - self._paths = {"list": list_paths, "single": single_paths} - except Exception as e: - err_msg = f"Could not generate parent-child relationships for schema: {e!s}" - raise ValueError(err_msg) from e - - return self - - @computed_field - @cached_property - def xmltodict_args(self) -> dict[str, Any]: - """Extra parser configuration arguments for running xmltodict over a dataset. - - :return: kwargs for configuring xmltodict.parse() - :rtype: dict[str, Any] - """ - if not self._paths: - return {} - - single_paths = self._paths["single"] - - def force_list(path: list[tuple[str, str | None]], key: str, _: Any) -> bool: - if not path: - return False - - return (path[-1][0], key) not in single_paths - return {"force_list": force_list} diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py new file mode 100644 index 00000000..78f80769 --- /dev/null +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py @@ -0,0 +1,66 @@ +"""XML to pyarrow pipeline for the KBase CTS.""" + +from collections.abc import Generator, Iterator +from logging import Logger, getLogger +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import dlt +from dlt.common.pipeline import LoadInfo +from dlt.common.storages.fsspec_filesystem import FileItemDict +from dlt.common.typing import TDataItems +from dlt.sources.filesystem import filesystem + +from cdm_data_loaders.pipelines.core import PipelineCli, run_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings +from cdm_data_loaders.readers.xml import process_xml_file_to_pyarrow + +if TYPE_CHECKING: + from dlt.extract import DltResource + +logger: Logger = getLogger(__name__) + + +def _read_items(items: Iterator[FileItemDict], settings: XmlToPyArrowSettings) -> Generator[TDataItems, Any, Any]: + """Read each file in items. Yields PyArrow tables for matched `xml_tag` elements. + + :param items: file items to read + :type items: Iterator[FileItemDict] + :param settings: pipeline config, including xml_tag and XSD schema + :type settings: XmlToPyArrowSettings + :yield: PyArrow tables of matched elements + :rtype: Generator[TDataItems, Any, Any] + """ + for file_item in items: + yield from process_xml_file_to_pyarrow(settings, file_path=Path(file_item.local_file_path)) + + +def run_xmltopyarrow_pipeline(settings: XmlToPyArrowSettings) -> LoadInfo | None: + """Run the XML to PyArrow pipeline on matching files in settings.input_dir. + + :param settings: pipeline configuration + :type settings: XmlToPyArrowSettings + """ + reader = dlt.transformer(_read_items, name="xmltopyarrow_reader", parallelized=True) + files = filesystem(bucket_url=str(settings.input_dir), file_glob=settings.file_glob) + + xmltopyarrow_resource: DltResource = files | reader(settings) + + return run_pipeline( + settings=settings, + resource=xmltopyarrow_resource, + pipeline_kwargs={ + "pipeline_name": PIPELINE_NAME, + "dataset_name": settings.dataset_name, + }, + pipeline_run_kwargs={ + "loader_file_format": str(settings.loader_file_format), + }, + ) + + +cli = PipelineCli(XmlToPyArrowSettings, run_xmltopyarrow_pipeline) + + +if __name__ == "__main__": + cli() diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py new file mode 100644 index 00000000..e5c1f2e4 --- /dev/null +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py @@ -0,0 +1,95 @@ +"""Settings class for the XML to PyArrow pipeline for the KBase CTS.""" + +from typing import Annotated, Final, Self + +import pyarrow as pa +import xmlschema +from pydantic import Field, FilePath, PositiveInt, PrivateAttr, model_validator +from pydantic_settings import SettingsConfigDict + +from cdm_data_loaders.core.fields import ( + DEFAULT_XML_FILE_GLOB, + FILE_GLOB, + FileGlob, + NonEmptyStr, + TableName, +) +from cdm_data_loaders.core.settings import ( + FileFormatIngestSettings, + default_settings_with_shortcuts, +) +from cdm_data_loaders.readers.xsd import ( + DEFAULT_MAX_RECURSION_DEPTH, + generate_pyarrow_schema, + load_schema, +) + +PIPELINE_NAME: Final[str] = "xmltopyarrow" + + +class XmlToPyArrowSettings(FileFormatIngestSettings): + """Settings for the XML to PyArrow ingestion pipeline.""" + + model_config: SettingsConfigDict = default_settings_with_shortcuts( + cli_prog_name=PIPELINE_NAME, + cli_shortcuts={ + **FileFormatIngestSettings.model_config["cli_shortcuts"], + FILE_GLOB: "g", + }, + ) + + _xsd_schema: xmlschema.XMLSchema = PrivateAttr() + _pyarrow_schema: pa.Schema = PrivateAttr() + file_glob: Annotated[ + FileGlob, + Field( + default=DEFAULT_XML_FILE_GLOB, + description="Glob pattern for XML files inside each entity's input subdirectory.", + ), + ] + table_name: TableName + xml_tag: Annotated[ + NonEmptyStr, + Field( + description="XML tag to capture the contents of", + ), + ] + xsd_file: Annotated[ + FilePath, + Field( + description="Path to the XSD file describing the XML documents to be imported. Including the schema will ensure that the appropriate relationships (1-to-1 vs 1-to-many) are created between parent and child elements, and that element text is stored consistently.", + ), + ] + max_recursion_depth: Annotated[ + PositiveInt, + Field( + default=DEFAULT_MAX_RECURSION_DEPTH, + description="Maximum recursion depth for self-referential XSD types before serializing deeper content as JSON.", + ), + ] + + @model_validator(mode="after") + def check_xsd_file(self) -> Self: + """Check whether the schema file is present and valid; load the XSD and generate the PyArrow schema.""" + try: + self._xsd_schema = load_schema(self.xsd_file) + self._pyarrow_schema = generate_pyarrow_schema( + self._xsd_schema, + self.xml_tag, + max_recursion_depth=self.max_recursion_depth, + ) + except Exception as e: + err_msg = f"Could not generate PyArrow schema from XSD: {e!s}" + raise ValueError(err_msg) from e + + return self + + @property + def xsd_schema(self) -> xmlschema.XMLSchema: + """Return the loaded XMLSchema instance.""" + return self._xsd_schema + + @property + def pyarrow_schema(self) -> pa.Schema: + """Return the generated PyArrow schema for xml_tag.""" + return self._pyarrow_schema diff --git a/src/cdm_data_loaders/readers/xml.py b/src/cdm_data_loaders/readers/xml.py index bb40f84e..05683f62 100644 --- a/src/cdm_data_loaders/readers/xml.py +++ b/src/cdm_data_loaders/readers/xml.py @@ -16,8 +16,10 @@ from cdm_data_loaders.core.fields import DEFAULT_XML_FILE_GLOB from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings +from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from cdm_data_loaders.readers.xsd import convert_xml_element from cdm_data_loaders.utils.batcher import get_file_batches -from cdm_data_loaders.utils.buffer import DictBuffer, ListBuffer +from cdm_data_loaders.utils.buffer import ArrowListBuffer, DictBuffer, ListBuffer from cdm_data_loaders.utils.gz import open_maybe_gzip logger: Logger = getLogger(__name__) @@ -64,7 +66,9 @@ def parse_head_matter(file_path: str | Path) -> dict[str, str]: def stream_xml_file(file_path: str | Path, element_with_ns: str) -> Generator[Element, Any]: - """Stream XML elements from a file. + """Stream the outermost matching XML elements from a file. + + A matching element nested inside another matching element is yielded as part of the outer element. :param file_path: path to the XML file; file can be gzipped or not. :type file_path: str | Path @@ -76,10 +80,21 @@ def stream_xml_file(file_path: str | Path, element_with_ns: str) -> Generator[El logger.debug("Streaming XML from %s", file_path) with open_maybe_gzip(file_path) as f: - for _, elem in iterparse(f, tag=(element_with_ns), remove_blank_text=True): + # number of open matching elements; 0 means the parser is outside any match + depth = 0 + for event, elem in iterparse(f, events=("start", "end"), tag=element_with_ns, remove_blank_text=True): + if event == "start": + depth += 1 + continue + depth -= 1 + if depth: + # nested match: leave it in place as part of the enclosing match + continue logger.debug(elem) yield elem elem.clear() + while elem.getprevious() is not None: + del elem.getparent()[0] def _log_interval_progress(n_entries: int, log_interval: int) -> None: @@ -94,16 +109,59 @@ def _log_interval_final(n_entries: int, file_path: Path, log_interval: int) -> N logger.debug("Processed %d entries from %s", n_entries + 1, file_path.name) +def process_xml_file_to_pyarrow(settings: XmlToPyArrowSettings, file_path: Path) -> Generator[DataItemWithMeta]: + """Convert XML elements to PyArrow table batches using an XSD schema. + + Each matching element is converted using ``settings.xsd_schema`` and buffered into PyArrow + tables matching ``settings.pyarrow_schema``. + + Use process_xml_file for parsing functions that return data spread across several tables. + + :param settings: pipeline config, including xml_tag, table_name, xsd_schema, pyarrow_schema, and buffer_size + :type settings: XmlToPyArrowSettings + :param file_path: file path, as a Path + :type file_path: Path + :yield: pages of PyArrow tables + :rtype: Generator[DataItemWithMeta] + """ + logger.info("Reading from %s", str(file_path)) + n_entries = -1 + + stream_tag = settings.xml_tag + if not stream_tag.startswith("{"): + default_ns = parse_head_matter(file_path).get("") + if default_ns: + stream_tag = f"{{{default_ns}}}{stream_tag}" + + arrow_buffer = ArrowListBuffer( + table_to_schema_map={settings.table_name: settings.pyarrow_schema}, + max_items=settings.buffer_size, + ) + for n_entries, element in enumerate(stream_xml_file(file_path, stream_tag)): + parsed_row = convert_xml_element( + element, + settings.xsd_schema, + settings.xml_tag, + max_recursion_depth=settings.max_recursion_depth, + ) + yield from arrow_buffer.add_item(parsed_row) + _log_interval_progress(n_entries, settings.log_interval) + + _log_interval_final(n_entries, file_path, settings.log_interval) + yield from arrow_buffer.flush() + return + + def process_xml_file_to_dict(settings: XmlToDictSettings, file_path: Path) -> Generator[DataItemWithMeta]: - """Convert XML to dictionary form. + """Convert XML elements to dictionary form using ``xmltodict.parse``. - Note: each incoming element produces a single dictionary as output. + Each matching element is converted with ``xmltodict.parse`` and buffered as dicts. Use process_xml_file for parsing functions that return data spread across several tables. :param settings: pipeline config, including xml_tag, table_name, and buffer_size :type settings: XmlToDictSettings - :param file_path: file path, as a string + :param file_path: file path, as a Path :type file_path: Path :yield: pages of dictionary items :rtype: Generator[DataItemWithMeta] @@ -113,7 +171,7 @@ def process_xml_file_to_dict(settings: XmlToDictSettings, file_path: Path) -> Ge buffer = ListBuffer(table_name=settings.table_name, max_items=settings.buffer_size) for n_entries, element in enumerate(stream_xml_file(file_path, settings.xml_tag)): - parsed_element = xmltodict.parse(tostring(element), **DEFAULT_XMLTODICT_ARGS, **settings.xmltodict_args) + parsed_element = xmltodict.parse(tostring(element), **DEFAULT_XMLTODICT_ARGS) if parsed_element: for k, v in parsed_element.items(): # remove the xmlns declarations diff --git a/src/cdm_data_loaders/readers/xsd.py b/src/cdm_data_loaders/readers/xsd.py index 7e2cb24d..624aac8a 100644 --- a/src/cdm_data_loaders/readers/xsd.py +++ b/src/cdm_data_loaders/readers/xsd.py @@ -1,76 +1,976 @@ -"""Functions for reading XSD files.""" +"""Functions for reading XSD files, generating PyArrow schemas, and converting XML elements.""" +import base64 +import datetime as dt +import json +from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path +from typing import Any, Final +import pyarrow as pa import xmlschema +from lxml.etree import Element +from xmlschema.validators import ( + XsdAnyAttribute, + XsdAnyElement, + XsdAttribute, + XsdComplexType, + XsdElement, + XsdGroup, + XsdSimpleType, + XsdType, +) + +DEFAULT_MAX_RECURSION_DEPTH: Final[int] = 2 + +XSD_NAMESPACE: Final[str] = "{http://www.w3.org/2001/XMLSchema}" +XSI_NAMESPACE: Final[str] = "{http://www.w3.org/2001/XMLSchema-instance}" +XSI_NIL: Final[str] = f"{XSI_NAMESPACE}nil" + +FORBIDDEN_RECURSIVE_XSD_TYPES: Final[frozenset[str]] = frozenset( + { + "xs:date", + "xs:dateTime", + "xs:hexBinary", + "xs:base64Binary", + } +) + +BUILTIN_LIST_ITEM_TYPES: Final[Mapping[str, str]] = { + "xs:NMTOKENS": "xs:NMTOKEN", + "xs:IDREFS": "xs:IDREF", + "xs:ENTITIES": "xs:ENTITY", +} + +SCALAR_ARROW_TYPES: Final[Mapping[str, pa.DataType]] = { + "xs:string": pa.string(), + "xs:normalizedString": pa.string(), + "xs:token": pa.string(), + "xs:language": pa.string(), + "xs:Name": pa.string(), + "xs:NCName": pa.string(), + "xs:NMTOKEN": pa.string(), + "xs:ID": pa.string(), + "xs:IDREF": pa.string(), + "xs:ENTITY": pa.string(), + "xs:anyURI": pa.string(), + "xs:QName": pa.string(), + "xs:NOTATION": pa.string(), + "xs:dateTime": pa.string(), + "xs:time": pa.string(), + "xs:duration": pa.string(), + "xs:gYear": pa.string(), + "xs:gYearMonth": pa.string(), + "xs:gMonthDay": pa.string(), + "xs:gDay": pa.string(), + "xs:gMonth": pa.string(), + "xs:boolean": pa.bool_(), + "xs:byte": pa.int8(), + "xs:short": pa.int16(), + "xs:int": pa.int32(), + "xs:long": pa.int64(), + "xs:integer": pa.int64(), + "xs:positiveInteger": pa.int64(), + "xs:nonNegativeInteger": pa.int64(), + "xs:negativeInteger": pa.int64(), + "xs:nonPositiveInteger": pa.int64(), + "xs:unsignedByte": pa.uint8(), + "xs:unsignedShort": pa.uint16(), + "xs:unsignedInt": pa.uint32(), + "xs:unsignedLong": pa.uint64(), + "xs:float": pa.float64(), + "xs:double": pa.float64(), + "xs:decimal": pa.float64(), + "xs:date": pa.date32(), + "xs:base64Binary": pa.binary(), + "xs:hexBinary": pa.binary(), +} def load_schema(schema_path: str | Path) -> xmlschema.XMLSchema: """Load an XML schema document into an XMLSchema object. :param schema_path: path to the schema file - :type schema_path: str | Path + :type schema_path: str | Path :return: loaded schema - :rtype: xmlschema.XMLSchema + :rtype : xmlschema.XMLSchema """ return xmlschema.XMLSchema(str(schema_path)) -def find_list_and_single_child_paths( - schema: xmlschema.XMLSchema, -) -> tuple[set[tuple[str, str]], set[tuple[str, str]]]: - """Traverse an xmlschema XMLSchema and classify parent/child element paths. - - A child is put in list_paths if it can occur zero, one, or more times - (maxOccurs is unbounded or greater than 1), meaning it should be - represented as a list. Otherwise it goes in single_paths (occurs at most - once, whether required or optional). - - :param schema: the schema to traverse - :type schema: xmlschema.XMLSchema - :return: tuple of (list_paths, single_paths) - :rtype: tuple[set[tuple[str, str]], set[tuple[str, str]]] +def _local_name(tag: str) -> str: + """Strip namespace URI from a Clark-notation tag or prefixed attribute name.""" + if "}" in tag: + return tag.rsplit("}", maxsplit=1)[-1] + return tag + + +def _extract_doc_comment(component: object, fallback_type: object = None) -> str | None: + """Extract whitespace-normalized ```` text from an XSD component or its type.""" + candidates = [component] + ref = getattr(component, "ref", None) + if ref is not None and ref is not component: + candidates.append(ref) + if fallback_type is not None: + candidates.append(fallback_type) + + for candidate in candidates: + annotation = getattr(candidate, "annotation", None) + docs = getattr(annotation, "documentation", None) + if not docs: + continue + parts = [" ".join(doc.text.split()) for doc in docs if getattr(doc, "text", None) and doc.text.strip()] + if parts: + return " ".join(parts) + return None + + +def _build_field_metadata( + *, + xsd_type: str | None = None, + comment: str | None = None, +) -> dict[str, str] | None: + """Construct the ``pa.Field`` metadata dictionary from optional comment and XSD type strings.""" + metadata: dict[str, str] = {} + if comment: + metadata["comment"] = comment + if xsd_type: + metadata["xsd_type"] = xsd_type + return metadata or None + + +def _builtin_xsd_name(simple_type: XsdType) -> str: + """Resolve an atomic simpleType or restriction to its built-in ``xs:`` identifier.""" + cur: Any = simple_type + while cur is not None: + name = getattr(cur, "name", None) + if isinstance(name, str) and name.startswith(XSD_NAMESPACE): + return f"xs:{cur.local_name}" + cur = getattr(cur, "base_type", None) + return "xs:string" + + +def _unwrap_member_types(simple_type: XsdType) -> list[XsdSimpleType]: + """Return the member types of an ``xs:union`` simpleType (unwrapping restrictions).""" + cur: Any = simple_type + while cur is not None: + members = getattr(cur, "member_types", None) + if members: + return list(members) + cur = getattr(cur, "base_type", None) + return [] + + +def _unwrap_item_type(simple_type: XsdType) -> XsdSimpleType | None: + """Return the item type of an ``xs:list`` simpleType (unwrapping restrictions).""" + cur: Any = simple_type + while cur is not None: + item_type = getattr(cur, "item_type", None) + if item_type is not None: + return item_type + cur = getattr(cur, "base_type", None) + return None + + +def _resolve_scalar_type(simple_type: XsdType) -> tuple[pa.DataType, str]: + """Map a non-list XSD simpleType (atomic or union) to ``(pa.DataType, xsd_type_str)``.""" + if simple_type.is_union(): + members = _unwrap_member_types(simple_type) + member_pairs = [_resolve_scalar_type(m) for m in members] + xsd_type_str = " | ".join(xsd_name for _, xsd_name in member_pairs) + unique_arrow_types = {arrow_t for arrow_t, _ in member_pairs} + arrow_type = next(iter(unique_arrow_types)) if len(unique_arrow_types) == 1 else pa.string() + return arrow_type, xsd_type_str + + xsd_name = _builtin_xsd_name(simple_type) + arrow_type = SCALAR_ARROW_TYPES.get(xsd_name, pa.string()) + return arrow_type, xsd_name + + +def _simple_type_to_arrow(simple_type: XsdType) -> tuple[pa.DataType, str | None]: + """Map any XSD simpleType (atomic, union, or list) to ``(pa.DataType, leaf_xsd_type_or_None)``. + + When ``simple_type`` is a list type (``xs:list``, ``xs:NMTOKENS``, ``xs:IDREFS``, ``xs:ENTITIES``), + the returned ``pa.DataType`` is ``pa.list_(pa.field("item", item_arrow_type, nullable=True, metadata={"xsd_type": ...}))`` + and the second element is ``None`` because ``"xsd_type"`` is attached to the inner ``"item"`` field. """ - list_paths: set[tuple[str, str]] = set() - single_paths: set[tuple[str, str]] = set() - visiting: set[int] = set() - - def walk(element: xmlschema.XsdElement) -> None: - elem_type = element.type - if elem_type is None: - return - - model_group = elem_type.model_group - if model_group is None: - return - - # guards against infinite recursion for self-referential types - type_key = id(elem_type) - if type_key in visiting: - return - visiting.add(type_key) - - parent_name = element.local_name - for child in model_group.iter_elements(): - if not isinstance(child, xmlschema.XsdElement): + builtin_name = _builtin_xsd_name(simple_type) + if builtin_name in BUILTIN_LIST_ITEM_TYPES: + item_xsd = BUILTIN_LIST_ITEM_TYPES[builtin_name] + item_field = pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": item_xsd}) + return pa.list_(item_field), None + + if simple_type.is_list(): + item_type = _unwrap_item_type(simple_type) + if item_type is not None: + item_arrow, item_xsd = _resolve_scalar_type(item_type) + else: + item_arrow, item_xsd = pa.string(), "xs:string" + item_field = pa.field("item", item_arrow, nullable=True, metadata={"xsd_type": item_xsd}) + return pa.list_(item_field), None + + return _resolve_scalar_type(simple_type) + + +def _iter_simple_builtin_names(simple_type: XsdType) -> list[str]: + """Return all constituent built-in ``xs:*`` type names for an atomic, list, or union simpleType.""" + builtin_name = _builtin_xsd_name(simple_type) + if builtin_name in BUILTIN_LIST_ITEM_TYPES: + return [BUILTIN_LIST_ITEM_TYPES[builtin_name]] + if simple_type.is_list(): + item_type = _unwrap_item_type(simple_type) + return _iter_simple_builtin_names(item_type) if item_type is not None else ["xs:string"] + if simple_type.is_union(): + names: list[str] = [] + for member in _unwrap_member_types(simple_type): + names.extend(_iter_simple_builtin_names(member)) + return names + return [builtin_name] + + +@dataclass(frozen=True, slots=True) +class _SchemaContext: + """Immutable traversal context for PyArrow schema generation.""" + + schema: xmlschema.XMLSchema + type_counts: Mapping[int, int] + max_recursion_depth: int + + +@dataclass(frozen=True, slots=True) +class _ConvertContext: + """Immutable traversal context for XML-to-dict conversion.""" + + schema: xmlschema.XMLSchema + type_counts: Mapping[int, int] + max_recursion_depth: int | None + + +def _check_complex_type_forbidden_recursion(root_type: XsdType, current: XsdComplexType) -> list[XsdType]: + """Validate attributes on ``current`` and return child ``XsdType``s to inspect for forbidden recursive scalars.""" + for attr in current.attributes.values(): + if isinstance(attr, XsdAttribute) and attr.type is not None: + for xsd_name in _iter_simple_builtin_names(attr.type): + if xsd_name in FORBIDDEN_RECURSIVE_XSD_TYPES: + msg = ( + f"Unsupported XSD feature: recursive type {root_type.local_name or root_type.name!r} " + f"contains attribute @{attr.local_name} of type {xsd_name!r}" + ) + raise ValueError(msg) + + if current.has_simple_content() and isinstance(current.content, XsdType): + return [current.content] + if not current.mixed and current.model_group is not None: + return [ + child.type + for child in current.model_group.iter_elements() + if isinstance(child, XsdElement) and child.type is not None and child.max_occurs != 0 + ] + return [] + + +def _check_no_forbidden_types_under_recursion(root_type: XsdType) -> None: + """Raise ValueError if a recursive type or any descendant type uses date, dateTime, or binary scalars.""" + visited: set[int] = set() + stack: list[XsdType] = [root_type] + + while stack: + current = stack.pop() + if id(current) in visited: + continue + visited.add(id(current)) + + if not isinstance(current, XsdComplexType): + for xsd_name in _iter_simple_builtin_names(current): + if xsd_name in FORBIDDEN_RECURSIVE_XSD_TYPES: + msg = ( + f"Unsupported XSD feature: recursive type {root_type.local_name or root_type.name!r} " + f"contains {xsd_name!r}, which cannot be serialized to JSON when recursion is truncated" + ) + raise ValueError(msg) + continue + + stack.extend(_check_complex_type_forbidden_recursion(root_type, current)) + + +def _group_min_occurs_by_name(group: XsdGroup, local_name: str) -> int: + """Compute minimum occurrences of elements with ``local_name`` across a model group.""" + if group.min_occurs == 0 or group.max_occurs == 0: + return 0 + target_group = group.ref if group.ref is not None else group + counts: list[int] = [] + for item in target_group.content: + if isinstance(item, XsdGroup): + counts.append(_group_min_occurs_by_name(item, local_name)) + elif isinstance(item, XsdElement) and item.max_occurs != 0 and item.local_name == local_name: + counts.append(item.min_occurs) + else: + counts.append(0) + if not counts: + return 0 + branch_min = min(counts) if target_group.model == "choice" else sum(counts) + return group.min_occurs * branch_min + + +def _particle_max_occurs_by_name(item: object, local_name: str) -> int | None: + """Compute maximum occurrences (None for unbounded) of ``local_name`` for a single group particle.""" + if isinstance(item, XsdGroup): + return _group_max_occurs_by_name(item, local_name) + if isinstance(item, XsdElement) and item.max_occurs != 0 and item.local_name == local_name: + return item.max_occurs + return 0 + + +def _group_max_occurs_by_name(group: XsdGroup, local_name: str) -> int | None: + """Compute maximum occurrences (None for unbounded) of elements with ``local_name`` across a model group.""" + if group.max_occurs == 0: + return 0 + target_group = group.ref if group.ref is not None else group + branch_maxes = [_particle_max_occurs_by_name(item, local_name) for item in target_group.content] + if not branch_maxes: + return 0 + if any(m is None for m in branch_maxes): + inner_max: int | None = None + elif target_group.model == "choice": + inner_max = max(m for m in branch_maxes if m is not None) + else: + inner_max = sum(m for m in branch_maxes if m is not None) + if inner_max == 0: + return 0 + if inner_max is None or group.max_occurs is None: + return None + return group.max_occurs * inner_max + + +def _is_any_element(item: object) -> bool: + """Return True if ``item`` is an ``xs:any`` wildcard particle.""" + return isinstance(item, XsdAnyElement) + + +def _is_any_attribute(attr: object) -> bool: + """Return True if ``attr`` is an ``xs:anyAttribute`` wildcard declaration.""" + return isinstance(attr, XsdAnyAttribute) + + +def _validate_group_particles(schema: xmlschema.XMLSchema, group: XsdGroup) -> None: + """Validate that a model group contains no wildcards, abstract elements/types, or substitution groups.""" + target_group = group.ref if group.ref is not None else group + for item in target_group.content: + if _is_any_element(item): + msg = "Unsupported XSD feature: xs:any wildcard element is not supported" + raise ValueError(msg) + if isinstance(item, XsdGroup): + _validate_group_particles(schema, item) + elif isinstance(item, XsdElement): + target_el = item.ref if item.ref is not None else item + if getattr(item, "abstract", False) or getattr(target_el, "abstract", False): + msg = f"Unsupported XSD feature: abstract element {item.local_name!r} is not supported" + raise ValueError(msg) + if ( + item.name in schema.substitution_groups + or target_el.name in schema.substitution_groups + or getattr(item, "substitution_group", None) is not None + or getattr(target_el, "substitution_group", None) is not None + ): + msg = f"Unsupported XSD feature: substitutionGroup on element {item.local_name!r} is not supported" + raise ValueError(msg) + if item.type is not None and getattr(item.type, "abstract", False): + msg = f"Unsupported XSD feature: abstract type {item.type.local_name!r} is not supported" + raise ValueError(msg) + + +def _collect_unique_children(schema: xmlschema.XMLSchema, complex_type: XsdComplexType) -> list[XsdElement]: + """Validate and return unique child ``XsdElement`` declarations in content-model order.""" + model_group = complex_type.model_group + if model_group is None: + return [] + _validate_group_particles(schema, model_group) + seen_names: set[str] = set() + children: list[XsdElement] = [] + for child in model_group.iter_elements(): + if not isinstance(child, XsdElement): + continue + max_occurs = _group_max_occurs_by_name(model_group, child.local_name) + if max_occurs == 0 or child.local_name in seen_names: + continue + seen_names.add(child.local_name) + children.append(child) + return children + + +def _collect_attributes(complex_type: XsdComplexType) -> list[XsdAttribute]: + """Validate and return declared attributes on ``complex_type`` in declaration order.""" + attrs: list[XsdAttribute] = [] + for attr in complex_type.attributes.values(): + if _is_any_attribute(attr): + msg = "Unsupported XSD feature: xs:anyAttribute wildcard is not supported" + raise ValueError(msg) + if isinstance(attr, XsdAttribute) and getattr(attr, "use", None) != "prohibited": + if attr.local_name is None: continue + attrs.append(attr) + return attrs + + +def _is_empty_marker_complex_type(complex_type: XsdComplexType) -> bool: + """Return True if ``complex_type`` has no attributes, no child elements, not simpleContent, and not mixed.""" + if complex_type.has_simple_content() or complex_type.mixed: + return False + if any(isinstance(a, XsdAttribute) and getattr(a, "use", None) != "prohibited" for a in complex_type.attributes.values()): + return False + if complex_type.model_group is None: + return True + return not any( + isinstance(el, XsdElement) and _group_max_occurs_by_name(complex_type.model_group, el.local_name) != 0 + for el in complex_type.model_group.iter_elements() + ) + + +def _build_complex_struct_fields( + ctx: _SchemaContext, + complex_type: XsdComplexType, + *, + parent_nillable: bool, +) -> list[pa.Field]: + """Build the ordered list of ``pa.Field``s for a struct-mapped ``XsdComplexType``.""" + fields: list[pa.Field] = [] + attr_field_names: set[str] = set() + + for attr in _collect_attributes(complex_type): + field_name = f"_{attr.local_name}" + attr_field_names.add(field_name) + attr_type = attr.type + if attr_type is not None: + pa_type, xsd_type_str = _simple_type_to_arrow(attr_type) + else: + pa_type, xsd_type_str = pa.string(), "xs:string" + comment = _extract_doc_comment(attr, attr_type) + nullable = parent_nillable or getattr(attr, "use", "optional") != "required" + fields.append( + pa.field( + field_name, + pa_type, + nullable=nullable, + metadata=_build_field_metadata(xsd_type=xsd_type_str, comment=comment), + ) + ) + + if complex_type.has_simple_content(): + content_type = complex_type.content + if isinstance(content_type, XsdType): + pa_type, xsd_type_str = _simple_type_to_arrow(content_type) + else: + pa_type, xsd_type_str = pa.string(), "xs:string" + fields.append( + pa.field( + "#text", + pa_type, + nullable=True, + metadata=_build_field_metadata(xsd_type=xsd_type_str), + ) + ) + return fields + + if complex_type.mixed: + fields.append( + pa.field( + "#text", + pa.string(), + nullable=True, + metadata=_build_field_metadata(xsd_type="xs:string"), + ) + ) + return fields + + model_group = complex_type.model_group + if model_group is None: + return fields + + for child in _collect_unique_children(ctx.schema, complex_type): + child_name = child.local_name + if child_name in attr_field_names: + msg = ( + f"Field name collision in {complex_type.local_name or 'complexType'!r}: " + f"attribute and child element both map to {child_name!r}" + ) + raise ValueError(msg) + + max_occurs = _group_max_occurs_by_name(model_group, child_name) + min_occurs = _group_min_occurs_by_name(model_group, child_name) + is_list = max_occurs is None or max_occurs > 1 + nullable = parent_nillable or child.nillable or min_occurs < 1 + + fields.append( + _build_element_field( + ctx, + child, + is_list=is_list, + nullable=nullable, + ) + ) + + return fields + + +def _resolve_non_struct_element_type(elem_type: XsdType | None) -> tuple[pa.DataType, str | None] | None: + """Return ``(pa_type, xsd_type_or_None)`` if ``elem_type`` maps to a leaf scalar rather than a struct.""" + if not isinstance(elem_type, XsdComplexType): + return _simple_type_to_arrow(elem_type) if elem_type is not None else (pa.string(), "xs:string") + + attrs = _collect_attributes(elem_type) + if elem_type.has_simple_content() and not attrs: + content_type = elem_type.content + return _simple_type_to_arrow(content_type) if isinstance(content_type, XsdType) else (pa.string(), "xs:string") + + if elem_type.mixed and not attrs: + return pa.string(), "xs:string" - max_occurs = elem_type.overall_max_occurs(child) - # never occurs => skip it - if max_occurs == 0: + if _is_empty_marker_complex_type(elem_type): + return pa.string(), None + + return None + + +def _build_element_field( + ctx: _SchemaContext, + element: XsdElement, + *, + is_list: bool, + nullable: bool, +) -> pa.Field: + """Build a ``pa.Field`` for a child ``XsdElement`` given its cardinality and nullability.""" + elem_type = element.type + comment = _extract_doc_comment(element, elem_type) + + leaf_type_info = _resolve_non_struct_element_type(elem_type) + if leaf_type_info is not None: + return _wrap_scalar_or_list_field( + element.local_name, + leaf_type_info, + comment=comment, + is_list=is_list, + nullable=nullable, + ) + + assert isinstance(elem_type, XsdComplexType) # noqa: S101 + type_id = id(elem_type) + if type_id in ctx.type_counts: + _check_no_forbidden_types_under_recursion(elem_type) + + next_count = ctx.type_counts.get(type_id, 0) + 1 + if next_count > ctx.max_recursion_depth: + return _wrap_scalar_or_list_field( + element.local_name, + (pa.string(), None), + comment=comment, + is_list=is_list, + nullable=nullable, + ) + + next_ctx = _SchemaContext( + schema=ctx.schema, + type_counts={**ctx.type_counts, type_id: next_count}, + max_recursion_depth=ctx.max_recursion_depth, + ) + struct_fields = _build_complex_struct_fields( + next_ctx, + elem_type, + parent_nillable=element.nillable, + ) + struct_type = pa.struct(struct_fields) + field_type = pa.list_(pa.field("item", struct_type, nullable=True)) if is_list else struct_type + return pa.field( + element.local_name, + field_type, + nullable=nullable, + metadata=_build_field_metadata(comment=comment), + ) + + +def _wrap_scalar_or_list_field( + name: str, + type_info: tuple[pa.DataType, str | None], + *, + comment: str | None, + is_list: bool, + nullable: bool, +) -> pa.Field: + """Create a scalar or list ``pa.Field`` with ``xsd_type`` on leaf scalar fields and ``comment`` on the outer field.""" + pa_type, xsd_type = type_info + if is_list: + item_field = pa.field("item", pa_type, nullable=True, metadata=_build_field_metadata(xsd_type=xsd_type)) + return pa.field( + name, + pa.list_(item_field), + nullable=nullable, + metadata=_build_field_metadata(comment=comment), + ) + return pa.field( + name, + pa_type, + nullable=nullable, + metadata=_build_field_metadata(xsd_type=xsd_type, comment=comment), + ) + + +def _resolve_target_element(schema: xmlschema.XMLSchema, xml_tag: str) -> XsdElement: + """Resolve ``xml_tag`` (local name or Clark notation) to a unique ``XsdElement`` declaration.""" + if xml_tag.startswith("{") and "}" in xml_tag: + ns_filter: str | None = xml_tag + local_target = xml_tag.rsplit("}", maxsplit=1)[-1] + else: + ns_filter = None + local_target = xml_tag + + def _matches(el: XsdElement) -> bool: + if el.local_name != local_target: + return False + return ns_filter is None or el.name == ns_filter + + # Depth 0: global elements + depth0 = [el for el in schema.elements.values() if isinstance(el, XsdElement) and _matches(el)] + if depth0: + return _deduplicate_target_elements(depth0, xml_tag) + + # BFS from depth 1 onwards through complexType model groups + current_level: list[XsdElement] = [el for el in schema.elements.values() if isinstance(el, XsdElement)] + visited_types: set[int] = set() + while current_level: + matches: list[XsdElement] = [] + next_level: list[XsdElement] = [] + for parent in current_level: + p_type = parent.type + if not isinstance(p_type, XsdComplexType) or id(p_type) in visited_types: + continue + visited_types.add(id(p_type)) + if p_type.model_group is None: continue + for child in p_type.model_group.iter_elements(): + if not isinstance(child, XsdElement): + continue + if _matches(child): + matches.append(child) + next_level.append(child) + if matches: + return _deduplicate_target_elements(matches, xml_tag) + current_level = next_level + + msg = f"Element {xml_tag!r} not found in XSD schema" + raise ValueError(msg) + + +def _deduplicate_target_elements(matches: list[XsdElement], xml_tag: str) -> XsdElement: + """Verify all matching ``XsdElement`` declarations at the resolution depth share a single ``XsdType``.""" + by_type: dict[int, XsdElement] = {id(el.type): el for el in matches} + if len(by_type) > 1: + msg = f"Ambiguous xml_tag {xml_tag!r}: found {len(by_type)} declarations with different types" + raise ValueError(msg) + target = next(iter(by_type.values())) + target_ref = target.ref if target.ref is not None else target + if getattr(target, "abstract", False) or getattr(target_ref, "abstract", False): + msg = f"Unsupported XSD feature: abstract element {xml_tag!r} is not supported" + raise ValueError(msg) + if target.type is not None and getattr(target.type, "abstract", False): + msg = f"Unsupported XSD feature: abstract type on {xml_tag!r} is not supported" + raise ValueError(msg) + return target + + +def _require_complex_type(elem_type: XsdType | None, xml_tag: str) -> XsdComplexType: + """Ensure that the resolved target element has an ``XsdComplexType``.""" + if elem_type is None or not elem_type.is_complex(): + msg = f"Top-level xml_tag {xml_tag!r} must have a complex type with attributes or child elements" + raise ValueError(msg) + assert isinstance(elem_type, XsdComplexType) # noqa: S101 + return elem_type + + +def generate_pyarrow_schema( + schema: xmlschema.XMLSchema, + xml_tag: str, + *, + max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH, +) -> pa.Schema: + """Generate a PyArrow schema for the target XML element from an XSD schema. + + :param schema: loaded XMLSchema instance + :type schema: xmlschema.XMLSchema + :param xml_tag: target XML element name (local name or Clark-notation ``{ns}local``) + :type xml_tag: str + :param max_recursion_depth: maximum structural recursion depth before JSON serialization + :type max_recursion_depth: int + :return: PyArrow schema whose top-level fields correspond to the target element's fields + :rtype : pa.Schema + """ + target_elem = _resolve_target_element(schema, xml_tag) + elem_type = _require_complex_type(target_elem.type, xml_tag) + + if _is_empty_marker_complex_type(elem_type): + msg = f"Top-level xml_tag {xml_tag!r} has an empty complex type with no attributes or child elements" + raise ValueError(msg) + + attrs = _collect_attributes(elem_type) + if (elem_type.has_simple_content() or elem_type.mixed) and not attrs: + msg = f"Top-level xml_tag {xml_tag!r} must have a complex type that resolves to a struct" + raise ValueError(msg) + + ctx = _SchemaContext( + schema=schema, + type_counts={id(elem_type): 1}, + max_recursion_depth=max_recursion_depth, + ) + fields = _build_complex_struct_fields( + ctx, + elem_type, + parent_nillable=target_elem.nillable, + ) + if not fields: + msg = f"Top-level xml_tag {xml_tag!r} must have a complex type with at least one field" + raise ValueError(msg) + return pa.schema(fields) + + +def _is_ignored_attr(attr_name: str) -> bool: + """Return True if an XML attribute is an ``xmlns`` or ``xsi:*`` attribute that should be stripped.""" + return attr_name.startswith((XSI_NAMESPACE, "xsi:", "xmlns", "{http://www.w3.org/2000/xmlns/}")) + + +def _is_xsi_nil(element: Element) -> bool: + """Return True if ``element`` carries ``xsi:nil='true'`` or ``xsi:nil='1'``.""" + val = element.attrib.get(XSI_NIL) or element.attrib.get("xsi:nil") + return val is not None and val.strip() in {"true", "1"} + + +def _escape_xml_text(text: str) -> str: + """Escape XML character entities ``&``, ``<``, and ``>`` in mixed-content text.""" + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + +def _escape_xml_attr(value: str) -> str: + """Escape XML character entities and double quotes in mixed-content attribute values.""" + return _escape_xml_text(value).replace('"', """) + - path = (parent_name, child.local_name) - if max_occurs is None or max_occurs > 1: - list_paths.add(path) - else: - single_paths.add(path) +def _serialize_mixed_node(node: Element) -> str: + """Serialize an inline child element inside a ``mixed='true'`` element without namespace declarations.""" + local = _local_name(node.tag) + attrs = "".join( + f' {_local_name(k)}="{_escape_xml_attr(v)}"' + for k, v in node.attrib.items() + if not _is_ignored_attr(k) + ) + inner = _serialize_inner_xml(node) + if not inner and len(node) == 0 and node.text is None: + return f"<{local}{attrs}/>" + return f"<{local}{attrs}>{inner}" - walk(child) - visiting.discard(type_key) +def _serialize_inner_xml(element: Element) -> str: + """Serialize the raw inner XML (text and inline child elements) of a ``mixed='true'`` element.""" + parts: list[str] = [] + if element.text: + parts.append(_escape_xml_text(element.text)) + for child in element: + if isinstance(child.tag, str): + parts.append(_serialize_mixed_node(child)) + if child.tail: + parts.append(_escape_xml_text(child.tail)) + return "".join(parts) - for element in schema.elements.values(): - walk(element) - return list_paths, single_paths +def _coerce_resolved_scalar(stripped: str, arrow_type: pa.DataType, xsd_name: str) -> Any: # noqa: ANN401 + """Coerce a stripped lexical string using its resolved ``arrow_type`` and ``xsd_name``.""" + if pa.types.is_boolean(arrow_type): + return stripped in {"true", "1"} + if pa.types.is_integer(arrow_type): + return int(stripped) + if pa.types.is_floating(arrow_type): + return float(stripped) + if pa.types.is_date32(arrow_type): + return dt.date.fromisoformat(stripped[:10]) + if pa.types.is_binary(arrow_type): + return bytes.fromhex(stripped) if xsd_name == "xs:hexBinary" else base64.b64decode(stripped) + return stripped + + +def _coerce_scalar_text(raw_text: str, simple_type: XsdType | None) -> Any: # noqa: ANN401 + """Coerce a non-empty lexical XML string according to its XSD ``simple_type``.""" + if simple_type is None: + return raw_text + + builtin_name = _builtin_xsd_name(simple_type) + if builtin_name in BUILTIN_LIST_ITEM_TYPES: + return raw_text.split() + + if simple_type.is_list(): + item_type = _unwrap_item_type(simple_type) + return [_coerce_scalar_text(token, item_type) for token in raw_text.split()] + + arrow_type, xsd_name = _resolve_scalar_type(simple_type) + return _coerce_resolved_scalar(raw_text.strip(), arrow_type, xsd_name) + + +def _extract_struct_attributes(element: Element, complex_type: XsdComplexType) -> dict[str, Any]: + """Extract and coerce declared XML attributes from ``element`` into ``_`` dictionary entries.""" + declared_attrs = {attr.local_name: attr for attr in _collect_attributes(complex_type) if attr.local_name} + raw_attrs = { + _local_name(k): v + for k, v in element.attrib.items() + if not _is_ignored_attr(k) + } + return { + f"_{attr_name}": _coerce_scalar_text(raw_attrs[attr_name], attr_decl.type) + for attr_name, attr_decl in declared_attrs.items() + if attr_name in raw_attrs + } + + +def _group_child_elements_by_name(element: Element) -> dict[str, list[Element]]: + """Group direct XML child elements by their local tag name in document order.""" + children_by_name: dict[str, list[Element]] = {} + for child_el in element: + if isinstance(child_el.tag, str): + children_by_name.setdefault(_local_name(child_el.tag), []).append(child_el) + return children_by_name + + +def _convert_complex_struct( + ctx: _ConvertContext, + element: Element, + complex_type: XsdComplexType, +) -> dict[str, Any] | None: + """Convert an XML ``element`` with a struct-mapped ``complex_type`` into a sparse dictionary.""" + result = _extract_struct_attributes(element, complex_type) + + if _is_xsi_nil(element): + return result or None + + if complex_type.has_simple_content(): + text_val = (element.text or "").strip() + if text_val: + content_type = complex_type.content if isinstance(complex_type.content, XsdType) else None + result["#text"] = _coerce_scalar_text(text_val, content_type) + return result or None + + if complex_type.mixed: + inner_xml = _serialize_inner_xml(element).strip() + if inner_xml: + result["#text"] = inner_xml + return result or None + + model_group = complex_type.model_group + if model_group is None: + return result or None + + children_by_name = _group_child_elements_by_name(element) + + for child_decl in _collect_unique_children(ctx.schema, complex_type): + child_name = child_decl.local_name + matching_elements = children_by_name.get(child_name) + if not matching_elements: + continue + + max_occurs = _group_max_occurs_by_name(model_group, child_name) + min_occurs = _group_min_occurs_by_name(model_group, child_name) + is_list = max_occurs is None or max_occurs > 1 + nullable = child_decl.nillable or min_occurs < 1 + + if is_list: + result[child_name] = [ + _convert_child_occurrence(ctx, el, child_decl, nullable=True) + for el in matching_elements + ] + else: + val = _convert_child_occurrence(ctx, matching_elements[0], child_decl, nullable=nullable) + if val is not None: + result[child_name] = val + + return result or None + + +def _convert_text_or_empty_leaf( + element: Element, + *, + is_mixed: bool, + simple_type: XsdType | None, + nullable: bool, +) -> Any: # noqa: ANN401 + """Convert an XML element that maps to a scalar or mixed-text leaf field.""" + if _is_xsi_nil(element): + return None + raw_val = _serialize_inner_xml(element).strip() if is_mixed else (element.text or "").strip() + if not raw_val: + return "" if not nullable else None + return raw_val if is_mixed else _coerce_scalar_text(raw_val, simple_type) + + +def _convert_child_occurrence( + ctx: _ConvertContext, + element: Element, + xsd_element: XsdElement, + *, + nullable: bool, +) -> Any: # noqa: ANN401 + """Convert a single XML element occurrence according to ``xsd_element``.""" + elem_type = xsd_element.type + + if not isinstance(elem_type, XsdComplexType): + return _convert_text_or_empty_leaf(element, is_mixed=False, simple_type=elem_type, nullable=nullable) + + attrs = _collect_attributes(elem_type) + if elem_type.has_simple_content() and not attrs: + content_type = elem_type.content if isinstance(elem_type.content, XsdType) else None + return _convert_text_or_empty_leaf(element, is_mixed=False, simple_type=content_type, nullable=nullable) + + if elem_type.mixed and not attrs: + return _convert_text_or_empty_leaf(element, is_mixed=True, simple_type=None, nullable=nullable) + + if _is_empty_marker_complex_type(elem_type): + return None if _is_xsi_nil(element) else "" + + type_id = id(elem_type) + next_count = ctx.type_counts.get(type_id, 0) + 1 + if ctx.max_recursion_depth is not None and next_count > ctx.max_recursion_depth: + unbounded_ctx = _ConvertContext(schema=ctx.schema, type_counts={}, max_recursion_depth=None) + nested = _convert_complex_struct(unbounded_ctx, element, elem_type) + return json.dumps(nested) if nested is not None else None + + next_ctx = _ConvertContext( + schema=ctx.schema, + type_counts={**ctx.type_counts, type_id: next_count}, + max_recursion_depth=ctx.max_recursion_depth, + ) + return _convert_complex_struct(next_ctx, element, elem_type) + + +def convert_xml_element( + element: Element, + schema: xmlschema.XMLSchema, + xml_tag: str, + *, + max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH, +) -> dict[str, Any]: + """Convert an lxml Element matching ``xml_tag`` into a typed dictionary according to the XSD. + + :param element: XML element to convert + :type element: Element + :param schema: loaded XMLSchema instance + :type schema: xmlschema.XMLSchema + :param xml_tag: target XML element name (local name or Clark-notation ``{ns}local``) + :type xml_tag: str + :param max_recursion_depth: maximum structural recursion depth before JSON serialization + :type max_recursion_depth: int + :return: sparse dictionary of coerced field values matching the generated PyArrow schema + :rtype : dict[str, Any] + """ + target_elem = _resolve_target_element(schema, xml_tag) + elem_type = _require_complex_type(target_elem.type, xml_tag) + + ctx = _ConvertContext( + schema=schema, + type_counts={id(elem_type): 1}, + max_recursion_depth=max_recursion_depth, + ) + converted = _convert_complex_struct(ctx, element, elem_type) + return converted or {} diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index e684970b..66d11659 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -77,6 +77,7 @@ "uniprot_cdm": ("uniprot_cdm", "cdm_data_loaders.pipelines.uniprot_kb_cdm"), "uniref_cdm": ("uniref_cdm", "cdm_data_loaders.pipelines.uniref_cdm"), "xmltodict": ("xmltodict", "cdm_data_loaders.pipelines.xmltodict.pipeline"), + "xmltopyarrow": ("xmltopyarrow", "cdm_data_loaders.pipelines.xmltopyarrow.pipeline"), "xml2db": ("xml2db", "cdm_data_loaders.pipelines.xml2db.pipeline"), "xsv": ("xsv", "cdm_data_loaders.pipelines.xsv.pipeline"), } diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py index dfa561bc..c584d11f 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py @@ -26,3 +26,4 @@ def _factory(**overrides: Any) -> XmlToDictSettings: # noqa: ANN401 return make_cts_settings(XmlToDictSettings, tmp_path, **defaults) return _factory + diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py index 1a47b253..2c277cb6 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py @@ -157,3 +157,5 @@ def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: return runner(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) return _factory + + diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py index 4db0e66d..3e8957b2 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py @@ -111,3 +111,5 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] assert_pipeline_report_written(Path(settings.output_dir), load_info) + + diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py index ff7e45ea..9727c356 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py @@ -45,6 +45,14 @@ def test_xml_pass_reference_dataset_has_nested_structures( ) -> None: """The DuckDB reference stores nested JSON as STRUCT/LIST types, not JSON text columns.""" reference_xml_table = reference_xml_dataset.table("reference") + reference_xml_df = reference_xml_table.df() + entry = reference_xml_df["entry"].iloc[0] + assert isinstance(entry, dict) + assert isinstance(entry["property"], list) + assert isinstance(entry["representativeMember"], dict) + assert len(reference_xml_df) == EXPECTED_ENTRY_COUNT + + # same thing, but using native duckdb entry_type = reference_xml_table.types[0] entry_fields = dict(entry_type.children) @@ -52,7 +60,6 @@ def test_xml_pass_reference_dataset_has_nested_structures( member_type = entry_fields["representativeMember"] assert property_type.id == "list" assert member_type.id == "struct" - count = reference_xml_table.count("*").fetchone()[0] assert count == EXPECTED_ENTRY_COUNT diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py index 74e788fb..d39392af 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py @@ -59,18 +59,6 @@ def test_run_xmltodict_pipeline_pass_binds_reader_before_run_pipeline( resource = mock_run_pipeline.call_args.kwargs["resource"] assert resource.name == "xmltodict_reader" assert resource._pipe.parent.name == "filesystem" # noqa: SLF001 - - -def test_run_xmltodict_pipeline_pass_resource_is_reader_piped_into_source( - settings_factory: Callable[..., XmlToDictSettings], -) -> None: - """The resource passed to run_pipeline combines the filesystem source with the bound reader.""" - settings = settings_factory() - - with patch.object(xmltodict_module, "run_pipeline") as mock_run_pipeline: - run_xmltodict_pipeline(settings) - - resource = mock_run_pipeline.call_args.kwargs["resource"] assert isinstance(resource, dlt.sources.DltResource) diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_settings.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_settings.py index 437f5d0a..ae0e53b6 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_settings.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_settings.py @@ -1,6 +1,5 @@ """Tests for XmlToDictSettings field validation and defaults.""" -import shutil from collections.abc import Callable from pathlib import Path @@ -12,19 +11,6 @@ from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings from tests.conftest import S3 -# All schema-driven tests below use tests/data/xsd/uniref_like.xsd, which is already exercised -# (and its list_paths/single_paths independently verified) by test_xsd.py: -# test_find_list_and_single_child_paths_pass_uniref_like_schema. -UNIREF_LIKE_XSD: str = "uniref_like.xsd" - - -def _copy_xsd_fixture(test_data_dir: Path, input_dir: Path, filename: str) -> str: - """Copy an XSD fixture from tests/data/xsd into input_dir; return its path.""" - input_dir.mkdir(parents=True, exist_ok=True) - dest = input_dir / filename - shutil.copy(test_data_dir / "xsd" / filename, dest) - return str(dest) - def test_xmltodict_settings_pass_defaults(settings_factory: Callable[..., XmlToDictSettings]) -> None: """file_glob defaults to '*.xml*'. buffer_size and log_interval default to the common CTS defaults.""" @@ -113,109 +99,6 @@ def test_xmltodict_settings_pass_unknown_use_destination_rejected( assert settings.use_destination == "not_a_destination" -def test_xmltodict_settings_pass_xmltodict_args_empty_without_xsd_file( - settings_factory: Callable[..., XmlToDictSettings], -) -> None: - """Without xsd_file (the default), xmltodict_args contributes no extra xmltodict.parse() kwargs.""" - settings = settings_factory() - assert settings.xsd_file is None - assert settings.xmltodict_args == {} - - -def test_xmltodict_settings_pass_xmltodict_args_has_force_list_callable_when_xsd_file_set( - tmp_path: Path, test_data_dir: Path, settings_factory: Callable[..., XmlToDictSettings] -) -> None: - """With xsd_file set, xmltodict_args exposes exactly one force_list callable kwarg.""" - xsd_file = _copy_xsd_fixture(test_data_dir, tmp_path / "input", UNIREF_LIKE_XSD) - settings = settings_factory(xsd_file=xsd_file) - - assert settings.xmltodict_args.keys() == {"force_list"} - assert callable(settings.xmltodict_args["force_list"]) - - -def test_xmltodict_settings_pass_xmltodict_args_is_cached( - tmp_path: Path, test_data_dir: Path, settings_factory: Callable[..., XmlToDictSettings] -) -> None: - """xmltodict_args is a cached_property: repeated access returns the same dict, not a rebuild.""" - xsd_file = _copy_xsd_fixture(test_data_dir, tmp_path / "input", UNIREF_LIKE_XSD) - settings = settings_factory(xsd_file=xsd_file) - assert settings.xmltodict_args is settings.xmltodict_args - - -def test_xmltodict_settings_fail_xsd_file_not_found_raises_runtime_error( - settings_factory: Callable[..., XmlToDictSettings], -) -> None: - """An xsd_file that does not exist raises ValidationError at construction.""" - with pytest.raises(ValidationError, match="Path does not point to a file"): - settings_factory(xsd_file="does_not_exist.xsd") - - -@pytest.fixture -def force_list_from_uniref_like_xsd( - tmp_path: Path, test_data_dir: Path, settings_factory: Callable[..., XmlToDictSettings] -) -> Callable[..., bool]: - """Build a real XmlToDictSettings from uniref_like.xsd and return its force_list callable.""" - xsd_file = _copy_xsd_fixture(test_data_dir, tmp_path / "input", UNIREF_LIKE_XSD) - settings = settings_factory(xsd_file=xsd_file) - return settings.xmltodict_args["force_list"] - - -@pytest.mark.parametrize( - ("path", "key", "expected"), - [ - pytest.param([], "UniRef50", False, id="document-root-is-never-forced-into-a-list"), - pytest.param([("UniRef50", None)], "entry", True, id="direct-child-of-root-list-path"), - pytest.param([("UniRef50", None), ("entry", None)], "property", True, id="two-levels-deep-list-path"), - pytest.param( - [("UniRef50", None), ("entry", None)], - "representativeMember", - False, - id="two-levels-deep-single-path", - ), - pytest.param( - [("UniRef50", None), ("entry", None), ("representativeMember", None)], - "dbReference", - False, - id="three-levels-deep-single-path-uses-immediate-parent-not-full-ancestor-chain", - ), - pytest.param( - [("UniRef50", None), ("entry", None), ("representativeMember", None), ("dbReference", None)], - "property", - True, - id="four-levels-deep-list-path-uses-immediate-parent-not-full-ancestor-chain", - ), - pytest.param( - [("UniRef50", None)], - "totallyUnmodelledChild", - True, - id="child-not-covered-by-schema-defaults-to-list", - ), - ], -) -def test_xmltodict_settings_pass_force_list_classifies_by_immediate_parent( - force_list_from_uniref_like_xsd: Callable[..., bool], - path: list[tuple[str, dict[str, str] | None]], - key: str, - expected: bool, -) -> None: - """force_list classifies every (parent, key) pair correctly, using only the immediate parent. - - xmltodict calls force_list(path, key, value) with `path` set to the full ancestor chain (a - list of (name, attrs) tuples) down to and including the immediate parent -- not just the - parent's own name -- and with an empty path for the top-level parsed element itself. A - correct implementation must: - - - Extract the immediate parent's name from the end of that chain, not compare the whole - chain (which would never match at any nesting depth beyond the root's direct children, - since find_list_and_single_child_paths stores plain (parent_name, child_name) pairs). - - Never force the top-level element into a list, since a parsed document only ever has one - root. - - Not raise: the ancestor chain is a list, so comparing it directly (unconverted) against a - set of (parent, key) string pairs raises `TypeError: unhashable type: 'list'`. - """ - assert force_list_from_uniref_like_xsd(path, key, "some-value") is expected - - @pytest.mark.parametrize("file_glob_arg", ["--file-glob", "-g"]) def test_xmltodict_settings_pass_xml_tag_shortcut(tmp_path: Path, file_glob_arg: str) -> None: """The file_glob field is exposed as both --file-glob and -g on the command line.""" diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py new file mode 100644 index 00000000..d5808b27 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py @@ -0,0 +1,54 @@ +"""Shared fixtures for xmltopyarrow pipeline tests.""" + +import shutil +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings +from tests.xml_samples import LIBRARY_XSD + + +@pytest.fixture +def library_xsd(tmp_path: Path) -> Path: + """Write LIBRARY_XSD to tmp_path and return its path.""" + xsd_path = tmp_path / "library.xsd" + xsd_path.write_text(LIBRARY_XSD, encoding="utf-8") + return xsd_path + + +@pytest.fixture +def settings_factory(tmp_path: Path, library_xsd: Path) -> Callable[..., XmlToPyArrowSettings]: + """Return a factory that builds a valid XmlToPyArrowSettings, with fields open to override.""" + + def _factory(**overrides: Any) -> XmlToPyArrowSettings: # noqa: ANN401 + defaults: dict[str, Any] = { + "dlt_dev_mode": True, + "dataset_name": "xml_test_dataset", + "table_name": "book", + "xml_tag": "book", + "xsd_file": str(library_xsd), + "file_glob": "*.xml*", + } + defaults.update(overrides) + return make_cts_settings(XmlToPyArrowSettings, tmp_path, **defaults) + + return _factory + + +@pytest.fixture +def copy_xsd_fixture(tmp_path: Path, test_data_dir: Path) -> Callable[[str], str]: + """Return a function that copies an XSD from tests/data/xsd into the input dir and returns its path.""" + + def _copy(filename: str) -> str: + input_dir = tmp_path / "input" + input_dir.mkdir(exist_ok=True) + dest = input_dir / filename + shutil.copy(test_data_dir / "xsd" / filename, dest) + return str(dest) + + return _copy + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/__init__.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py new file mode 100644 index 00000000..7dcbf0dd --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py @@ -0,0 +1,207 @@ +"""Shared fixtures for xmltopyarrow pipeline integration tests.""" + +import gzip +import json +from collections.abc import Callable, Generator +from pathlib import Path +from typing import Any + +import duckdb +import pytest +import xmltodict +from dlt.common.pipeline import LoadInfo +from frozendict import frozendict + +from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from cdm_data_loaders.readers.xml import DEFAULT_XMLTODICT_ARGS +from tests.cdm_data_loaders.pipelines.helpers import ( + denormalize_identifier, + duckdb_reference_from_jsonl, + make_cts_settings, + read_jsonl_file, + strip_metadata_columns, +) +from tests.conftest import N_REFERENCE_XML_ENTRIES, REFERENCE_XML_TAG +from tests.xml_samples import LIBRARY_XSD + +DEFAULT_XMLTOPYARROW_SETTINGS: frozendict = frozendict( + { + "buffer_size": 10, + "dlt_dev_mode": False, + "file_glob": "*.xml*", + "log_interval": 1000, + "use_output_dir_for_pipeline_metadata": False, + } +) + + +@pytest.fixture +def library_xsd(tmp_path: Path) -> Path: + """Write LIBRARY_XSD to tmp_path and return its path.""" + xsd_path = tmp_path / "library.xsd" + xsd_path.write_text(LIBRARY_XSD, encoding="utf-8") + return xsd_path + + +@pytest.fixture +def settings_factory( + tmp_path: Path, + dlt_destination_config: str, + library_xsd: Path, +) -> Callable[..., XmlToPyArrowSettings]: + """Return a factory that builds a valid XmlToPyArrowSettings, with fields open to override.""" + + def _factory(**overrides: Any) -> XmlToPyArrowSettings: # noqa: ANN401 + defaults: dict[str, Any] = { + **DEFAULT_XMLTOPYARROW_SETTINGS, + "dataset_name": "xml_test_dataset", + "table_name": "book", + "use_destination": dlt_destination_config, + "xml_tag": "book", + "xsd_file": str(library_xsd), + } + defaults.update(overrides) + return make_cts_settings(XmlToPyArrowSettings, tmp_path, **defaults) + + return _factory + + +def import_jsonl_data(jsonl_path: Path) -> list[dict[str, Any]]: + """Read the reference JSONL into a list of canonical nested entry dicts.""" + return read_jsonl_file(jsonl_path) + + +def write_reference_xml_as_jsonl(source_xml: Path, jsonl_path: Path) -> int: + """Parse the UniRef XML with xmltodict and write the reference JSONL.""" + with gzip.open(source_xml, "rb") as fh: + document = xmltodict.parse(fh.read(), **DEFAULT_XMLTODICT_ARGS) + + entries = document["UniRef50"]["entry"] + with jsonl_path.open("w", encoding="utf-8") as out: + for entry in entries: + out.write(json.dumps({"entry": entry}, ensure_ascii=False) + "\n") + return len(entries) + + +@pytest.fixture(scope="session") +def reference_xml_as_jsonl(reference_xml_data_dir: Path) -> Path: + """Generate the reference JSONL from the chunk_100_el source, if not present.""" + jsonl_path = reference_xml_data_dir / "jsonl" / "uniref_100.jsonl" + if not jsonl_path.exists(): + source_xml = reference_xml_data_dir / "chunk_100_el" / "uniref_100_part_01.xml.gz" + n_entries = write_reference_xml_as_jsonl(source_xml, jsonl_path) + assert n_entries == N_REFERENCE_XML_ENTRIES + + return jsonl_path + + +@pytest.fixture(scope="session") +def reference_xml_entries(reference_xml_as_jsonl: Path) -> list[dict[str, Any]]: + """The canonical nested entry dicts read back from the reference JSONL.""" + ref_json = import_jsonl_data(reference_xml_as_jsonl) + assert isinstance(ref_json, list) + for line in ref_json: + assert list(line.keys()) == ["entry"] + entry = line.get("entry", {}) + for key in ("_id", "name", "property", "representativeMember"): + assert key in entry + assert entry.get("_id") is not None + assert entry.get("_id").startswith("UniRef50_") + assert len(ref_json) == N_REFERENCE_XML_ENTRIES + return ref_json + + +@pytest.fixture(scope="session") +def sorted_reference_xml_entries(reference_xml_entries: list[dict[str, Any]]) -> list[str]: + """Sorted XSD-normalized entries for the uniref reference dataset.""" + return sorted( + json.dumps(normalize_uniref_entry_for_xsd(entry), sort_keys=True, default=str) + for entry in reference_xml_entries + ) + + +@pytest.fixture(scope="session") +def reference_xml_dataset(reference_xml_as_jsonl: Path) -> Generator[duckdb.DuckDBPyConnection]: + """Load the reference JSONL into an in-memory DuckDB connection with nested structures.""" + with duckdb_reference_from_jsonl(reference_xml_as_jsonl) as conn: + yield conn + + +@pytest.fixture +def run_xmltopyarrow_pipeline_fxt( + reference_xml_data_dir: Path, + dlt_destination_config: str, + isolated_run_factory: Callable[..., Callable[..., tuple[LoadInfo | None, Path]]], +) -> Callable[..., Any]: + """Return a factory that runs the xmltopyarrow pipeline on a chunk dir with full isolation. + + Each run gets a unique output_dir and dataset_name under tmp_path, so + parallel-safe repeated runs never collide or accumulate rows. + """ + base_kwargs = { + **DEFAULT_XMLTOPYARROW_SETTINGS, + "settings_cls": XmlToPyArrowSettings, + "table_name": "entry", + "use_destination": dlt_destination_config, + "xml_tag": REFERENCE_XML_TAG, + "xsd_file": str(reference_xml_data_dir / "uniref.xsd"), + } + runner = isolated_run_factory( + dataset_prefix="reference_run", + run_fn=run_xmltopyarrow_pipeline, + base_kwargs=base_kwargs, + ) + + def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: # noqa: ANN401 + return runner(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) + + return _factory + + +def _strip_none_values(value: Any) -> Any: # noqa: ANN401 + """Recursively remove keys with None values from nested dicts and lists.""" + if isinstance(value, dict): + return {k: _strip_none_values(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [_strip_none_values(item) for item in value] + return value + + +def strip_xsd_arrow_row(row: dict[str, Any]) -> dict[str, Any]: + """Strip dlt metadata columns, unmangle top-level column names, and drop None struct fields.""" + unmangled = {denormalize_identifier(k): v for k, v in strip_metadata_columns(row).items()} + return _strip_none_values(unmangled) + + +def _normalize_xsd_reference_member(member: dict[str, Any]) -> dict[str, Any]: + """Normalize an xmltodict member dict to match XSD-typed cardinality and scalar types.""" + normalized = dict(member) + db_ref = dict(normalized["dbReference"]) + if "property" in db_ref and isinstance(db_ref["property"], dict): + db_ref["property"] = [db_ref["property"]] + normalized["dbReference"] = db_ref + if "sequence" in normalized and isinstance(normalized["sequence"], dict): + seq = dict(normalized["sequence"]) + if "_length" in seq: + seq["_length"] = int(seq["_length"]) + normalized["sequence"] = seq + return normalized + + +def normalize_uniref_entry_for_xsd(raw_entry: dict[str, Any]) -> dict[str, Any]: + """Normalize a raw xmltodict UniRef entry dict to match XSD-backed Arrow output.""" + entry = dict(raw_entry.get("entry", raw_entry)) + if "property" in entry and isinstance(entry["property"], dict): + entry["property"] = [entry["property"]] + entry["representativeMember"] = _normalize_xsd_reference_member(entry["representativeMember"]) + if "member" in entry: + members = entry["member"] if isinstance(entry["member"], list) else [entry["member"]] + entry["member"] = [_normalize_xsd_reference_member(m) for m in members] + return entry + + +def reconstruct_xsd_entries(tables: dict[str, list[dict[str, Any]]]) -> list[dict[str, Any]]: + """Extract normalized XSD entry dicts from the single Arrow-backed entry table.""" + return [strip_xsd_arrow_row(row) for row in tables.get("entry", [])] + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py new file mode 100644 index 00000000..f39b7b89 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py @@ -0,0 +1,145 @@ +"""End-to-end tests for the xmltopyarrow pipeline.""" + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from dlt.common.pipeline import LoadInfo +from pandas import DataFrame + +from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings +from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner +from tests.cdm_data_loaders.pipelines.helpers import ( + LOADER_FILE_FORMATS, + assert_pipeline_report_written, + assert_run_ok, + settings_to_argv, + sorted_json, +) +from tests.cdm_data_loaders.pipelines.xmltopyarrow.integration.conftest import ( + normalize_uniref_entry_for_xsd, + strip_xsd_arrow_row, +) +from tests.conftest import N_REFERENCE_XML_ENTRIES +from tests.xml_samples import SIMPLE_LIBRARY_XML + + +def check_book_list_results(load_info: LoadInfo | None, table_name: str | None = None) -> DataFrame: + """Check over the output of parsing SIMPLE_LIBRARY_XML with LIBRARY_XSD.""" + load_info = assert_run_ok(load_info) + dataset = load_info.pipeline.dataset() + table_name = table_name or "book" + assert set(dataset.tables) == {table_name, *DEFAULT_DLT_TABLES} + book_df = dataset.table(table_name).df() + assert set(book_df.columns.tolist()) >= {"_id", "title"} + return book_df + + +@pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS) +def test_run_xmltopyarrow_pipeline_pass_writes_expected_output( + settings_factory: Callable[..., XmlToPyArrowSettings], + loader_file_format: str, + pipeline_runner: PipelineTestRunner, +) -> None: + """Running the pipeline on xml files loads one row per matching element into the configured table.""" + settings = settings_factory(loader_file_format=loader_file_format) + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) + (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") + + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + book_df = check_book_list_results(load_info) + assert sorted(book_df["_id"].tolist()) == ["1", "2", "3"] + assert_pipeline_report_written(Path(settings.output_dir), load_info) + + +def test_run_xmltopyarrow_pipeline_pass_gzip_files_are_loaded( + settings_factory: Callable[..., XmlToPyArrowSettings], + write_gzip_file: Callable[[Path, str, str], Path], + pipeline_runner: PipelineTestRunner, +) -> None: + """Gzip-compressed xml files matching the glob are decompressed and loaded.""" + settings = settings_factory() + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) + (input_dir / "plain.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") + write_gzip_file(input_dir, "library.xml.gz", SIMPLE_LIBRARY_XML) + + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + book_df = check_book_list_results(load_info) + # one copy of the three books from the plain file, one from the gzipped copy + assert sorted(book_df["_id"].tolist()) == ["1", "1", "2", "2", "3", "3"] + + +def test_run_xmltopyarrow_pipeline_pass_no_matching_files_yields_no_data_table( + settings_factory: Callable[..., XmlToPyArrowSettings], + pipeline_runner: PipelineTestRunner, +) -> None: + """An input dir with no matching xml files does not fail the run and creates no data table.""" + settings = settings_factory() + Path(settings.input_dir).mkdir(exist_ok=True) + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) + dataset = load_info.pipeline.dataset() + # only dlt's own metadata tables exist; the data table is never created + assert set(dataset.tables) == DEFAULT_DLT_TABLES + assert_pipeline_report_written(Path(settings.output_dir), load_info) + + +def test_run_xmltopyarrow_pipeline_pass_custom_table_name_is_respected( + settings_factory: Callable[..., XmlToPyArrowSettings], + pipeline_runner: PipelineTestRunner, +) -> None: + """Rows land in the table named by table_name, not in a name derived from the xml tag.""" + settings = settings_factory(table_name="library_entries") + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) + (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") + + load_info = pipeline_runner.run(PIPELINE_NAME, settings=settings) + book_df = check_book_list_results(load_info, "library_entries") + assert sorted(book_df["_id"].tolist()) == ["1", "2", "3"] + + +def test_cli_pass_runs_end_to_end_from_command_line_arguments( + settings_factory: Callable[..., XmlToPyArrowSettings], + pipeline_runner: PipelineTestRunner, +) -> None: + """cli() reads command-line arguments and runs the pipeline, producing the expected output.""" + settings = settings_factory(dataset_name="cli_dataset") + input_dir = Path(settings.input_dir) + input_dir.mkdir(exist_ok=True) + (input_dir / "library.xml").write_text(SIMPLE_LIBRARY_XML, encoding="utf-8") + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, cli_args=settings_to_argv(settings))) + book_df = check_book_list_results(load_info) + assert sorted(book_df["_id"].tolist()) == ["1", "2", "3"] + assert_pipeline_report_written(Path(settings.output_dir), load_info) + + +def test_run_xmltopyarrow_pipeline_pass_uniref_chunk_5_el( + settings_factory: Callable[..., XmlToPyArrowSettings], + reference_xml_data_dir: Path, + reference_xml_entries: list[dict[str, Any]], + pipeline_runner: PipelineTestRunner, +) -> None: + """Running the pipeline on uniref chunk_5_el with uniref.xsd loads all 100 entries with Arrow types.""" + settings = settings_factory( + input_dir=str(reference_xml_data_dir / "chunk_5_el"), + xsd_file=str(reference_xml_data_dir / "uniref.xsd"), + xml_tag="entry", + table_name="entry", + ) + + load_info = assert_run_ok(pipeline_runner.run(PIPELINE_NAME, settings=settings)) + dataset = load_info.pipeline.dataset() + assert set(dataset.tables) == {"entry", *DEFAULT_DLT_TABLES} + + loaded_rows = [strip_xsd_arrow_row(row) for row in dataset.table("entry").arrow().to_pylist()] + expected_rows = [normalize_uniref_entry_for_xsd(entry) for entry in reference_xml_entries] + assert len(loaded_rows) == N_REFERENCE_XML_ENTRIES + assert sorted_json(loaded_rows) == sorted_json(expected_rows) + assert_pipeline_report_written(Path(settings.output_dir), load_info) + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py new file mode 100644 index 00000000..8c598fb0 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py @@ -0,0 +1,171 @@ +"""Integration tests for the xmltopyarrow pipeline against the uniref reference dataset.""" + +from collections.abc import Callable +from typing import Any + +import dlt +import duckdb +import pytest + +from cdm_data_loaders.core.fields import PARQUET +from tests.cdm_data_loaders.pipelines.helpers import ( + BUFFER_SIZES, + LOADER_FILE_FORMATS, + WORKER_CONFIGS, + assert_dataset_matches_reference, + assert_datasets_equal, + read_pipeline_tables, + reference_entries_from_duckdb, + run_and_compare_variations, + run_and_read, + sorted_json, +) +from tests.cdm_data_loaders.pipelines.xmltopyarrow.integration.conftest import ( + normalize_uniref_entry_for_xsd, + reconstruct_xsd_entries, + strip_xsd_arrow_row, +) + +CHUNK_DIRS = ("chunk_5_el", "chunk_20_el", "chunk_100_el") +EXPECTED_ENTRY_COUNT = 100 + + +def test_xmltopyarrow_pass_diff_format_same_results( + run_xmltopyarrow_pipeline_fxt: Callable[..., Any], + sorted_reference_xml_entries: list[str], +) -> None: + """Ensure that the save format does not affect the reconstructed output.""" + reconstructed_by_format: dict[str, list[str]] = {} + for output_format in LOADER_FILE_FORMATS: + fmt = str(output_format) + load_info, output_dir = run_and_read( + run_xmltopyarrow_pipeline_fxt, + "chunk_20_el", + loader_file_format=output_format, + ) + tables = read_pipeline_tables(output_dir / load_info.dataset_name, fmt) + reconstructed_by_format[fmt] = sorted_json(reconstruct_xsd_entries(tables)) + + formats = list(reconstructed_by_format.keys()) + for fmt in formats[1:]: + assert reconstructed_by_format[fmt] == reconstructed_by_format[formats[0]] + assert reconstructed_by_format[formats[0]] == sorted_reference_xml_entries + + +@pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS) +def test_xmltopyarrow_pass_output_identical_across_chunk_dirs( + run_xmltopyarrow_pipeline_fxt: Callable[..., Any], + sorted_reference_xml_entries: list[str], + loader_file_format: str, +) -> None: + """Pipeline output is identical across all three chunk dirs and matches the reference.""" + run_and_compare_variations( + run_xmltopyarrow_pipeline_fxt, + {f"{key}_{loader_file_format}": (key, {"loader_file_format": loader_file_format}) for key in CHUNK_DIRS}, + sorted_reference_json=sorted_reference_xml_entries, + output_format=loader_file_format, + reconstruct=reconstruct_xsd_entries, + ) + + +@pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS) +def test_xmltopyarrow_pass_output_identical_across_buffer_sizes( + run_xmltopyarrow_pipeline_fxt: Callable[..., Any], + sorted_reference_xml_entries: list[str], + loader_file_format: str, +) -> None: + """ArrowListBuffer size does not affect pipeline output for any chunk dir.""" + for chunk_dir in CHUNK_DIRS: + run_and_compare_variations( + run_xmltopyarrow_pipeline_fxt, + { + f"{buffer_size}_{loader_file_format}": ( + chunk_dir, + {"buffer_size": buffer_size, "loader_file_format": loader_file_format}, + ) + for buffer_size in BUFFER_SIZES + }, + sorted_reference_json=sorted_reference_xml_entries, + output_format=loader_file_format, + reconstruct=reconstruct_xsd_entries, + ) + + +@pytest.mark.parametrize( + ("extract_workers", "normalize_workers", "load_workers"), + WORKER_CONFIGS, + ids=[f"extract_{e}-normalize_{n}-load_{l}" for e, n, l in WORKER_CONFIGS], # noqa: E741 +) +@pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str) +def test_xmltopyarrow_pass_no_data_lost_across_worker_configs( + *, + run_xmltopyarrow_pipeline_fxt: Callable[..., Any], + sorted_reference_xml_entries: list[str], + extract_workers: int, + normalize_workers: int, + load_workers: int, + loader_file_format: str, +) -> None: + """No data is lost regardless of how the job is split across extract/normalize/load workers.""" + load_info_and_dir = {} + for worker_key in ("baseline", "current"): + worker_config = ( + {} + if worker_key == "baseline" + else { + "extract.workers": extract_workers, + "normalize.workers": normalize_workers, + "load.workers": load_workers, + } + ) + with dlt.config.values(worker_config): + load_info_and_dir[worker_key] = run_and_read( + run_xmltopyarrow_pipeline_fxt, "chunk_5_el", loader_file_format=loader_file_format + ) + + assert_datasets_equal(load_info_and_dir) + assert_dataset_matches_reference( + load_info_and_dir, + sorted_reference_xml_entries, + loader_file_format, + reconstruct=reconstruct_xsd_entries, + ) + + +def test_xmltopyarrow_pass_uniref_chunk_5_el_matches_reference( + run_xmltopyarrow_pipeline_fxt: Callable[..., Any], + reference_xml_entries: list[dict[str, Any]], +) -> None: + """Running on chunk_5_el with uniref.xsd produces a single Arrow-typed entry table matching the 100 reference entries.""" + (load_info, output_dir) = run_and_read( + run_xmltopyarrow_pipeline_fxt, + "chunk_5_el", + ) + tables = read_pipeline_tables(output_dir / load_info.dataset_name, PARQUET) + assert set(tables) == {"entry"} + loaded_rows = [strip_xsd_arrow_row(row) for row in tables["entry"]] + expected_rows = [normalize_uniref_entry_for_xsd(entry) for entry in reference_xml_entries] + assert len(loaded_rows) == EXPECTED_ENTRY_COUNT + assert sorted_json(loaded_rows) == sorted_json(expected_rows) + + +@pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS) +def test_xmltopyarrow_pass_reconstructed_matches_duckdb_reference( + run_xmltopyarrow_pipeline_fxt: Callable[..., Any], + reference_xml_dataset: duckdb.DuckDBPyConnection, + loader_file_format: str, +) -> None: + """Reconstructed pipeline entries equal the entries loaded from the DuckDB reference.""" + (load_info, output_dir) = run_and_read( + run_xmltopyarrow_pipeline_fxt, + "chunk_100_el", + loader_file_format=loader_file_format, + ) + tables = read_pipeline_tables(output_dir / load_info.dataset_name, loader_file_format) + reconstructed = reconstruct_xsd_entries(tables) + from_duckdb = [ + normalize_uniref_entry_for_xsd(entry) for entry in reference_entries_from_duckdb(reference_xml_dataset) + ] + assert len(from_duckdb) == EXPECTED_ENTRY_COUNT + assert sorted_json(reconstructed) == sorted_json(from_duckdb) + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py new file mode 100644 index 00000000..dfce58ec --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py @@ -0,0 +1,241 @@ +"""Unit tests for run_xmltopyarrow_pipeline and cli wiring.""" + +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any +from unittest.mock import patch +from uuid import uuid4 + +import dlt +import pytest +from dlt.common.pipeline import LoadInfo +from dlt.extract import DltResource + +import cdm_data_loaders.pipelines.xmltopyarrow.pipeline as xmltopyarrow_module +from cdm_data_loaders.core.fields import LoaderFileFormatEnum +from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import ( + cli, + run_xmltopyarrow_pipeline, +) +from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings +from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from tests.xml_samples import TWO_BOOK_LIBRARY_XML + + +@pytest.mark.parametrize("loader_file_format", list(LoaderFileFormatEnum), ids=str) +def test_run_xmltopyarrow_pipeline_pass_sets_core_run_pipeline_args_correctly( + settings_factory: Callable[..., XmlToPyArrowSettings], + loader_file_format: str, +) -> None: + """run_xmltopyarrow_pipeline builds the reader and delegates to run_pipeline with the correct args.""" + settings = settings_factory(loader_file_format=loader_file_format) + + with patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline: + run_xmltopyarrow_pipeline(settings) + + assert mock_run_pipeline.call_count == 1 + _, kwargs = mock_run_pipeline.call_args + assert kwargs.keys() == {"settings", "resource", "pipeline_kwargs", "pipeline_run_kwargs"} + assert kwargs["settings"] == settings + assert kwargs["pipeline_kwargs"] == { + "pipeline_name": PIPELINE_NAME, + "dataset_name": settings.dataset_name, + } + assert kwargs["pipeline_run_kwargs"] == {"loader_file_format": loader_file_format} + assert isinstance(kwargs["resource"], DltResource) + + +def test_run_xmltopyarrow_pipeline_pass_binds_reader_before_run_pipeline( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """The resource passed to run_pipeline is the bound reader piped into the filesystem source.""" + settings = settings_factory() + + with patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline: + run_xmltopyarrow_pipeline(settings) + + mock_run_pipeline.assert_called_once() + resource = mock_run_pipeline.call_args.kwargs["resource"] + assert resource.name == "xmltopyarrow_reader" + assert resource._pipe.parent.name == "filesystem" # noqa: SLF001 + assert isinstance(resource, dlt.sources.DltResource) + + +def test_cli_pass_runs_end_to_end_from_command_line_arguments( + tmp_path: Path, + library_xsd: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """cli() reads command-line arguments and runs the pipeline via run_xmltopyarrow_pipeline. + + The real XmlToPyArrowSettings parses argv (seeded by the autouse dlt + config isolation fixture), and core.run_pipeline is redirected to a real + DuckDB pipeline so the full flow is validated. + """ + input_dir = tmp_path / "cli_input" + input_dir.mkdir() + (input_dir / "library.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") + output_dir = tmp_path / "cli_output" + output_dir.mkdir() + log_config_file = tmp_path / "logging.json" + log_config_file.write_text('{"version": 1}') + + argv = [ + PIPELINE_NAME, + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + "--use-destination", + "local_fs", + "--log-config-file", + str(log_config_file), + "--dataset-name", + "cli_dataset", + "--table-name", + "book", + "--xml-tag", + "book", + "--xsd-file", + str(library_xsd), + "--use-output-dir-for-pipeline-metadata", + "false", + "--dlt-dev-mode", + "false", + ] + monkeypatch.setattr(sys, "argv", argv) + + # unique pipeline name: the duckdb database file is derived from it, so + # repeated runs of the suite do not see rows from previous runs + pipeline_name = f"test_xmltopyarrow_cli_pipeline_{uuid4().hex}" + + def fake_run_pipeline( + *, + resource: DltResource, + pipeline_kwargs: dict[str, Any], + **_: dict[str, Any], + ) -> LoadInfo | None: + assert pipeline_kwargs == {"pipeline_name": PIPELINE_NAME, "dataset_name": "cli_dataset"} + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") + return pipeline.run(resource) + + with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): + load_info = cli.run() + + assert load_info is not None + assert not load_info.has_failed_jobs + + with ( + load_info.pipeline.sql_client() as client, + client.execute_query("SELECT COUNT(*) FROM book") as cur, + ): + (row_count,) = cur.fetchone() + assert row_count == 2 # noqa: PLR2004 + + +def test_cli_pass_writes_rows_for_matching_xml_files( + tmp_path: Path, + library_xsd: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """cli() with xml files in the input dir loads one row per matching element.""" + input_dir = tmp_path / "xml_input" + input_dir.mkdir() + (input_dir / "data.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") + + output_dir = tmp_path / "cli_output" + output_dir.mkdir() + log_config_file = tmp_path / "logging.json" + log_config_file.write_text('{"version": 1}') + + argv = [ + PIPELINE_NAME, + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + "--use-destination", + "local_fs", + "--log-config-file", + str(log_config_file), + "--dataset-name", + "cli_dataset", + "--table-name", + "book", + "--xml-tag", + "book", + "--xsd-file", + str(library_xsd), + "--use-output-dir-for-pipeline-metadata", + "false", + "--dlt-dev-mode", + "false", + ] + monkeypatch.setattr(sys, "argv", argv) + + pipeline_name = f"test_xmltopyarrow_cli_rows_pipeline_{uuid4().hex}" + + def fake_run_pipeline( + *, + resource: Any, # noqa: ANN401 + **_: Any, # noqa: ANN401 + ) -> LoadInfo | None: + pipeline = duckdb_pipeline(tmp_path, pipeline_name, "cli_dataset") + return pipeline.run(resource) + + with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): + load_info = cli.run() + + assert load_info is not None + assert not load_info.has_failed_jobs + pipeline = load_info.pipeline + with ( + pipeline.sql_client() as client, + client.execute_query("SELECT COUNT(*) FROM book") as cur, + ): + (row_count,) = cur.fetchone() + assert row_count == 2 # noqa: PLR2004 + + +@pytest.mark.parametrize( + ("missing_args", "expected_fragment"), + [ + (["--table-name", "book", "--xml-tag", "book", "--xsd-file", "{xsd}"], "dataset_name"), + (["--dataset-name", "cli_dataset", "--xml-tag", "book", "--xsd-file", "{xsd}"], "table_name"), + (["--dataset-name", "cli_dataset", "--table-name", "book", "--xsd-file", "{xsd}"], "xml_tag"), + (["--dataset-name", "cli_dataset", "--table-name", "book", "--xml-tag", "book"], "xsd_file"), + ], +) +def test_cli_fail_missing_required_args_raise( + tmp_path: Path, + library_xsd: Path, + monkeypatch: pytest.MonkeyPatch, + missing_args: list[str], + expected_fragment: str, +) -> None: + """cli() without required pipeline-specific args raises before running the pipeline.""" + log_config_file = tmp_path / "logging.json" + log_config_file.write_text('{"version": 1}') + + resolved_args = [arg.format(xsd=library_xsd) for arg in missing_args] + argv = [ + PIPELINE_NAME, + "--input-dir", + str(tmp_path / "input"), + "--output-dir", + str(tmp_path / "output"), + "--log-config-file", + str(log_config_file), + *resolved_args, + ] + monkeypatch.setattr(sys, "argv", argv) + + with ( + patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline, + pytest.raises(Exception, match=expected_fragment), + ): + cli() + + mock_run_pipeline.assert_not_called() + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py new file mode 100644 index 00000000..c7958156 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py @@ -0,0 +1,214 @@ +"""Tests for XmlToPyArrowSettings field validation and defaults.""" + +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest +import xmlschema +from pydantic import ValidationError +from pydantic_settings import CliApp + +from cdm_data_loaders.core.fields import DEFAULTS +from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from cdm_data_loaders.readers.xsd import DEFAULT_MAX_RECURSION_DEPTH, generate_pyarrow_schema, load_schema +from tests.conftest import S3 + +UNIREF_LIKE_XSD: str = "uniref_like.xsd" + + +def test_xmltopyarrow_settings_pass_defaults(settings_factory: Callable[..., XmlToPyArrowSettings]) -> None: + """file_glob defaults to '*.xml*'. buffer_size, log_interval, and max_recursion_depth default to expected values.""" + settings = settings_factory() + assert settings.file_glob == "*.xml*" + assert settings.buffer_size == DEFAULTS["buffer_size"] + assert settings.log_interval == DEFAULTS["log_interval"] + assert settings.max_recursion_depth == DEFAULT_MAX_RECURSION_DEPTH + + +def test_xmltopyarrow_settings_pass_custom_file_glob( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """file_glob accepts a custom glob pattern.""" + assert settings_factory(file_glob="*.rdf").file_glob == "*.rdf" + + +def test_xmltopyarrow_settings_pass_custom_xml_tag( + settings_factory: Callable[..., XmlToPyArrowSettings], + copy_xsd_fixture: Callable[[str], str], +) -> None: + """xml_tag accepts any tag name present in the XSD.""" + xsd_file = copy_xsd_fixture(UNIREF_LIKE_XSD) + settings = settings_factory(xsd_file=xsd_file, xml_tag="entry") + assert settings.xml_tag == "entry" + + +def test_xmltopyarrow_settings_fail_missing_xml_tag( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """Omitting xml_tag raises ValidationError.""" + with pytest.raises(ValidationError): + settings_factory(xml_tag=None) + + +def test_xmltopyarrow_settings_fail_missing_table_name( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """Omitting table_name raises ValidationError.""" + with pytest.raises(ValidationError): + settings_factory(table_name=None) + + +def test_xmltopyarrow_settings_fail_missing_dataset_name( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """Omitting dataset_name raises ValidationError.""" + with pytest.raises(ValidationError): + settings_factory(dataset_name=None) + + +def test_xmltopyarrow_settings_fail_missing_xsd_file( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """Omitting xsd_file raises ValidationError.""" + with pytest.raises(ValidationError): + settings_factory(xsd_file=None) + + +def test_xmltopyarrow_settings_fail_non_positive_buffer_size( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """buffer_size must be a positive integer.""" + with pytest.raises(ValidationError): + settings_factory(buffer_size=0) + + +def test_xmltopyarrow_settings_fail_non_positive_log_interval( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """log_interval must be a positive integer.""" + with pytest.raises(ValidationError): + settings_factory(log_interval=0) + + +def test_xmltopyarrow_settings_fail_local_destination_with_s3_output_dir( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """An s3:// output_dir raises ValidationError at construction because output_dir must be a local path.""" + with pytest.raises(ValidationError): + settings_factory(output_dir="s3://some-bucket/path", use_destination="local_fs") + + +def test_xmltopyarrow_settings_fail_local_destination_with_s3_flag( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """An s3 destination with a local output_dir is accepted at construction.""" + settings = settings_factory(use_destination=S3) + assert settings.use_destination == S3 + + +def test_xmltopyarrow_settings_pass_unknown_use_destination_rejected( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """use_destination not present in the dlt config is accepted at construction.""" + settings = settings_factory(use_destination="not_a_destination") + assert settings.use_destination == "not_a_destination" + + +def test_xmltopyarrow_settings_pass_schema_properties_populated_when_xsd_file_set( + settings_factory: Callable[..., XmlToPyArrowSettings], + copy_xsd_fixture: Callable[[str], str], +) -> None: + """With xsd_file set, xsd_schema and pyarrow_schema are populated for xml_tag.""" + xsd_file = copy_xsd_fixture(UNIREF_LIKE_XSD) + settings = settings_factory(xsd_file=xsd_file, xml_tag="entry", table_name="entry") + + assert isinstance(settings.xsd_schema, xmlschema.XMLSchema) + assert isinstance(settings.pyarrow_schema, pa.Schema) + expected_schema = generate_pyarrow_schema( + load_schema(xsd_file), + "entry", + max_recursion_depth=DEFAULT_MAX_RECURSION_DEPTH, + ) + assert settings.pyarrow_schema.equals(expected_schema, check_metadata=True) + + +def test_xmltopyarrow_settings_pass_custom_max_recursion_depth_controls_pyarrow_schema( + settings_factory: Callable[..., XmlToPyArrowSettings], + copy_xsd_fixture: Callable[[str], str], +) -> None: + """Custom max_recursion_depth is forwarded to generate_pyarrow_schema.""" + xsd_file = copy_xsd_fixture("self_referential.xsd") + settings_d1 = settings_factory(xsd_file=xsd_file, xml_tag="node", table_name="node", max_recursion_depth=1) + settings_d2 = settings_factory(xsd_file=xsd_file, xml_tag="node", table_name="node", max_recursion_depth=2) + + assert settings_d1.pyarrow_schema is not None + assert settings_d2.pyarrow_schema is not None + assert not settings_d1.pyarrow_schema.equals(settings_d2.pyarrow_schema) + + +def test_xmltopyarrow_settings_fail_non_positive_max_recursion_depth( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """max_recursion_depth must be a positive integer.""" + with pytest.raises(ValidationError): + settings_factory(max_recursion_depth=0) + + +def test_xmltopyarrow_settings_fail_xsd_file_not_found_raises_validation_error( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """An xsd_file that does not exist raises ValidationError at construction.""" + with pytest.raises(ValidationError, match="Path does not point to a file"): + settings_factory(xsd_file="does_not_exist.xsd") + + +def test_xmltopyarrow_settings_fail_xml_tag_not_in_xsd_raises_validation_error( + settings_factory: Callable[..., XmlToPyArrowSettings], + copy_xsd_fixture: Callable[[str], str], +) -> None: + """An xml_tag not present in xsd_file raises ValidationError at construction.""" + xsd_file = copy_xsd_fixture(UNIREF_LIKE_XSD) + with pytest.raises(ValidationError, match="Could not generate PyArrow schema from XSD"): + settings_factory(xsd_file=xsd_file, xml_tag="nonExistentTag") + + +@pytest.mark.parametrize("file_glob_arg", ["--file-glob", "-g"]) +def test_xmltopyarrow_settings_pass_xml_tag_shortcut( + tmp_path: Path, + library_xsd: Path, + file_glob_arg: str, +) -> None: + """The file_glob field is exposed as both --file-glob and -g on the command line.""" + log_config_file = tmp_path / "logging.json" + log_config_file.write_text('{"version": 1}') + input_dir = tmp_path / "input" + input_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + + argv = [ + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + "--use-destination", + "local_fs", + "--log-config-file", + str(log_config_file), + file_glob_arg, + "*.xml.gz", + "--dataset-name", + "cli_dataset", + "--table-name", + "book", + "--xml-tag", + "book", + "--xsd-file", + str(library_xsd), + ] + + settings = CliApp.run(XmlToPyArrowSettings, cli_args=argv) + assert settings.file_glob == "*.xml.gz" + assert settings.xml_tag == "book" + diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py new file mode 100644 index 00000000..31b2f8ef --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py @@ -0,0 +1,198 @@ +"""End-to-end specification tests: XSD + XML -> process_xml_file_to_pyarrow -> PyArrow Table batches.""" + +import datetime as dt +import json +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest + +from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from cdm_data_loaders.readers.xml import process_xml_file_to_pyarrow +from cdm_data_loaders.readers.xsd import generate_pyarrow_schema, load_schema + + +def _json_default(obj: object) -> str: + """Serialize Python date and bytes values from Arrow tables for comparison with expected JSON.""" + if isinstance(obj, dt.date): + return obj.isoformat() + if isinstance(obj, bytes): + return obj.decode("utf-8") + msg = f"Object of type {type(obj)} is not JSON serializable" + raise TypeError(msg) + + +def _strip_none(value: object) -> object: + """Recursively remove keys with None values from dicts to compare Arrow-materialized rows with sparse expected.json.""" + if isinstance(value, dict): + return {k: _strip_none(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [_strip_none(item) for item in value] + return value + + +@pytest.mark.parametrize( + ("xsd_rel_path", "xml_rel_path", "expected_rel_path", "xml_tag"), + [ + pytest.param( + "xsd/spec/01_scalars_and_simple_types/schema.xsd", + "xsd/spec/01_scalars_and_simple_types/data.xml", + "xsd/spec/01_scalars_and_simple_types/expected.json", + "record", + id="spec-01-scalars-and-simple-types", + ), + pytest.param( + "xsd/spec/02_namespaces_and_nillable/schema.xsd", + "xsd/spec/02_namespaces_and_nillable/data.xml", + "xsd/spec/02_namespaces_and_nillable/expected.json", + "entry", + id="spec-02-namespaces-and-nillable", + ), + pytest.param( + "xsd/spec/03_mixed_content_raw_text/schema.xsd", + "xsd/spec/03_mixed_content_raw_text/data.xml", + "xsd/spec/03_mixed_content_raw_text/expected.json", + "article", + id="spec-03-mixed-content-raw-text", + ), + pytest.param( + "xsd/spec/04_recursion_depth/schema.xsd", + "xsd/spec/04_recursion_depth/data.xml", + "xsd/spec/04_recursion_depth/expected.json", + "record", + id="spec-04-recursion-depth", + ), + pytest.param( + "xsd/spec/05_groups_and_compositors/schema.xsd", + "xsd/spec/05_groups_and_compositors/data.xml", + "xsd/spec/05_groups_and_compositors/expected.json", + "item", + id="spec-05-groups-and-compositors", + ), + pytest.param( + "xsd/spec/06_context_dependent_paths/schema.xsd", + "xsd/spec/06_context_dependent_paths/data.xml", + "xsd/spec/06_context_dependent_paths/expected.json", + "root", + id="spec-06-context-dependent-paths", + ), + pytest.param( + "xsd/spec/07_multi_root_shared_type/schema.xsd", + "xsd/spec/07_multi_root_shared_type/data.xml", + "xsd/spec/07_multi_root_shared_type/expected.json", + "entry", + id="spec-07-multi-root-shared-type", + ), + pytest.param( + "xsd/spec/08_nullability_and_empty_tags/schema.xsd", + "xsd/spec/08_nullability_and_empty_tags/data.xml", + "xsd/spec/08_nullability_and_empty_tags/expected.json", + "entry", + id="spec-08-nullability-and-empty-tags", + ), + pytest.param( + "xsd/spec/09_empty_marker_elements/schema.xsd", + "xsd/spec/09_empty_marker_elements/data.xml", + "xsd/spec/09_empty_marker_elements/expected.json", + "book", + id="spec-09-empty-marker-elements", + ), + pytest.param( + "xsd/spec/10_combinations/schema.xsd", + "xsd/spec/10_combinations/data.xml", + "xsd/spec/10_combinations/expected.json", + "record", + id="spec-10-combinations", + ), + pytest.param( + "uniprot/uniref/uniref.xsd", + "xsd/examples/uniref/minimal.xml", + "xsd/examples/uniref/minimal_expected.json", + "entry", + id="example-uniref-minimal", + ), + pytest.param( + "uniprot/uniref/uniref.xsd", + "xsd/examples/uniref/maximal.xml", + "xsd/examples/uniref/maximal_expected.json", + "entry", + id="example-uniref-maximal", + ), + pytest.param( + "uniprot/uniprot_kb/uniprot.xsd", + "xsd/examples/uniprot/minimal.xml", + "xsd/examples/uniprot/minimal_expected.json", + "entry", + id="example-uniprot-minimal", + ), + pytest.param( + "uniprot/uniprot_kb/uniprot.xsd", + "xsd/examples/uniprot/maximal.xml", + "xsd/examples/uniprot/maximal_expected.json", + "entry", + id="example-uniprot-maximal", + ), + pytest.param( + "pubmed/pubmed.xsd", + "xsd/examples/pubmed_article/minimal.xml", + "xsd/examples/pubmed_article/minimal_expected.json", + "PubmedArticle", + id="example-pubmed-article-minimal", + ), + pytest.param( + "pubmed/pubmed.xsd", + "xsd/examples/pubmed_article/maximal.xml", + "xsd/examples/pubmed_article/maximal_expected.json", + "PubmedArticle", + id="example-pubmed-article-maximal", + ), + pytest.param( + "pubmed/pubmed.xsd", + "xsd/examples/pubmed_book_article/minimal.xml", + "xsd/examples/pubmed_book_article/minimal_expected.json", + "PubmedBookArticle", + id="example-pubmed-book-article-minimal", + ), + pytest.param( + "pubmed/pubmed.xsd", + "xsd/examples/pubmed_book_article/maximal.xml", + "xsd/examples/pubmed_book_article/maximal_expected.json", + "PubmedBookArticle", + id="example-pubmed-book-article-maximal", + ), + ], +) +def test_process_xml_file_to_pyarrow_pass_yields_arrow_tables_for_spec_and_examples( + settings_factory: Callable[..., XmlToPyArrowSettings], + test_data_dir: Path, + *, + xsd_rel_path: str, + xml_rel_path: str, + expected_rel_path: str, + xml_tag: str, +) -> None: + """process_xml_file_to_pyarrow yields pa.Table batches matching the XSD PyArrow schema and expected JSON.""" + xsd_path = test_data_dir / xsd_rel_path + xml_path = test_data_dir / xml_rel_path + expected = json.loads((test_data_dir / expected_rel_path).read_text(encoding="utf-8")) + + settings = settings_factory( + table_name=xml_tag, + xml_tag=xml_tag, + xsd_file=str(xsd_path), + ) + + items = list(process_xml_file_to_pyarrow(settings, file_path=xml_path)) + assert len(items) == 1 + assert items[0].meta.table_name == xml_tag + assert isinstance(items[0].data, pa.Table) + + table: pa.Table = items[0].data + expected_pa_schema = generate_pyarrow_schema(load_schema(xsd_path), xml_tag, max_recursion_depth=2) + assert table.schema.equals(expected_pa_schema, check_metadata=True) + assert table.num_rows == len(expected) + + actual_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) + assert actual_rows == expected + diff --git a/tests/cdm_data_loaders/readers/conftest.py b/tests/cdm_data_loaders/readers/conftest.py deleted file mode 100644 index 02a9d990..00000000 --- a/tests/cdm_data_loaders/readers/conftest.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Shared fixtures and functions for cdm_data_loaders.readers.""" - -from collections.abc import Callable -from pathlib import Path - -import pytest -import xmlschema - - -@pytest.fixture -def load_schema(test_data_dir: Path) -> Callable[[str], xmlschema.XMLSchema]: - """Return a function that builds an XMLSchema from a file name in the data directory.""" - - def _load(filename: str) -> xmlschema.XMLSchema: - return xmlschema.XMLSchema(str(test_data_dir / "xsd" / filename)) - - return _load diff --git a/tests/cdm_data_loaders/readers/test_xsd.py b/tests/cdm_data_loaders/readers/test_xsd.py deleted file mode 100644 index 2fe2529d..00000000 --- a/tests/cdm_data_loaders/readers/test_xsd.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for the xsd reader module. - -Test XSDs live under tests/data/xsd and are loaded through the load_schema -fixture defined in conftest.py. -""" - -from collections.abc import Callable -from pathlib import Path - -import pytest -import xmlschema - -from cdm_data_loaders.readers.xsd import find_list_and_single_child_paths - - -def test_find_list_and_single_child_paths_pass_no_children( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A root element with a simple type has no children, so both result sets are empty.""" - schema = load_schema("no_children.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == set() - assert single_paths == set() - - -def test_find_list_and_single_child_paths_pass_empty_schema( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A schema with no global elements yields no paths at all.""" - schema = load_schema("empty_schema.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == set() - assert single_paths == set() - - -def test_find_list_and_single_child_paths_pass_attribute_only_type_no_children( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A complex type with only attributes and no child elements yields no paths.""" - schema = load_schema("attribute_only.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == set() - assert single_paths == set() - - -@pytest.mark.parametrize( - ("child_name", "expected_set_name"), - [ - pytest.param("requiredSingle", "single", id="required-single-occurs-once"), - pytest.param("optionalSingle", "single", id="optional-single-occurs-zero-or-one"), - pytest.param("optionalRepeated", "list", id="optional-repeated-occurs-zero-or-more"), - pytest.param("requiredRepeated", "list", id="required-repeated-occurs-one-or-more"), - pytest.param("boundedRepeated", "list", id="bounded-repeated-max-occurs-two"), - ], -) -def test_find_list_and_single_child_paths_pass_occurs_variety( - load_schema: Callable[[str], xmlschema.XMLSchema], - child_name: str, - expected_set_name: str, -) -> None: - """Children are sorted into list_paths or single_paths based on their maxOccurs.""" - schema = load_schema("occurs_variety.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - path = ("root", child_name) - if expected_set_name == "list": - assert path in list_paths - assert path not in single_paths - else: - assert path in single_paths - assert path not in list_paths - - -def test_find_list_and_single_child_paths_pass_occurs_variety_excludes_impossible_child( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A child with maxOccurs=0 can never appear, so it is excluded from both result sets.""" - schema = load_schema("occurs_variety.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - path = ("root", "neverOccurs") - assert path not in list_paths - assert path not in single_paths - - -def test_find_list_and_single_child_paths_pass_choice_group_repeated( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """Children of a repeatable choice group are list paths even with their own maxOccurs=1.""" - schema = load_schema("nested_choice.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == {("root", "optionA"), ("root", "optionB")} - assert single_paths == set() - - -def test_find_list_and_single_child_paths_pass_nested_sequence_flattened( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A child nested two sequence levels deep is still reported against its direct parent element.""" - schema = load_schema("nested_sequence_deep.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == {("root", "deepChild")} - assert single_paths == set() - - -def test_find_list_and_single_child_paths_pass_ref_element_resolved( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A child declared with ref= resolves to the referenced global element's local name.""" - schema = load_schema("ref_element.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert ("root", "shared") in list_paths - assert ("root", "shared") not in single_paths - - -def test_find_list_and_single_child_paths_pass_self_referential_type_no_infinite_recursion( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """A type that nests itself as its own child is recorded once and does not recurse forever.""" - schema = load_schema("self_referential.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == {("node", "child")} - assert single_paths == set() - - -def test_find_list_and_single_child_paths_pass_mutual_recursion_no_infinite_recursion( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """Two types that reference each other are both traversed once without infinite recursion.""" - schema = load_schema("mutual_recursion.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == {("a", "b")} - assert single_paths == {("b", "a")} - - -def test_find_list_and_single_child_paths_pass_uniref_like_schema( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """Test the function on a more realistic schema.""" - schema = load_schema("uniref_like.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths == { - ("UniRef50", "entry"), - ("UniRef90", "entry"), - ("UniRef100", "entry"), - ("entry", "property"), - ("entry", "member"), - ("dbReference", "property"), - } - assert single_paths == { - ("entry", "representativeMember"), - ("member", "dbReference"), - ("member", "sequence"), - ("representativeMember", "dbReference"), - ("representativeMember", "sequence"), - } - - -def test_find_list_and_single_child_paths_pass_returns_disjoint_sets( - load_schema: Callable[[str], xmlschema.XMLSchema], -) -> None: - """No path can be classified as both a list path and a single path at the same time.""" - schema = load_schema("uniref_like.xsd") - - list_paths, single_paths = find_list_and_single_child_paths(schema) - - assert list_paths.isdisjoint(single_paths) - - -def test_find_list_and_single_child_paths_fail_invalid_schema_type() -> None: - """Passing an object without a schema-like elements mapping raises AttributeError.""" - not_a_schema = "this is a string, not an XMLSchema" - - with pytest.raises(AttributeError): - find_list_and_single_child_paths(not_a_schema) # type: ignore[arg-type] - - -def test_find_list_and_single_child_paths_fail_nonexistent_file_path(test_data_dir: Path) -> None: - """Loading a schema from a path that does not exist raises an OSError from xmlschema.""" - missing_path = test_data_dir / "does_not_exist.xsd" - - with pytest.raises(OSError): - xmlschema.XMLSchema(str(missing_path)) diff --git a/tests/cdm_data_loaders/readers/test_xsd_spec.py b/tests/cdm_data_loaders/readers/test_xsd_spec.py new file mode 100644 index 00000000..ca4358b8 --- /dev/null +++ b/tests/cdm_data_loaders/readers/test_xsd_spec.py @@ -0,0 +1,536 @@ +"""Specification tests for XSD-to-PyArrow schema generation and deterministic XML-to-Arrow conversion.""" + +import copy +import datetime as dt +import importlib.util +import io +import json +from collections import defaultdict +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import xmlschema +from _pytest.mark.structures import ParameterSet +from lxml import etree + +from cdm_data_loaders.readers.xsd import ( + convert_xml_element, + generate_pyarrow_schema, + load_schema, +) + + +def _list_field(name: str, item_type: pa.DataType, *, nullable: bool = True) -> pa.Field: + """Build a PyArrow list field with standard nullable item field.""" + return pa.field(name, pa.list_(pa.field("item", item_type, nullable=True)), nullable=nullable) + + +def _load_expected_schema(schema_file: Path, attr_name: str = "EXPECTED_SCHEMA") -> pa.Schema: + """Load a pa.Schema constant from an expected_schema.py fixture file.""" + spec = importlib.util.spec_from_file_location(schema_file.stem, schema_file) + if spec is None or spec.loader is None: + msg = f"Could not load schema module from {schema_file}" + raise RuntimeError(msg) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + schema = getattr(module, attr_name) + if not isinstance(schema, pa.Schema): + msg = f"Expected {attr_name} in {schema_file} to be a pa.Schema, got {type(schema)}" + raise TypeError(msg) + return schema + + +def _coerce_json_rows_for_arrow(value: Any, dtype: pa.DataType) -> Any: # noqa: ANN401 + """Coerce ISO date strings and binary strings from expected JSON into Python objects for pa.Table.from_pylist.""" + if value is None: + return None + if pa.types.is_date32(dtype) and isinstance(value, str): + return dt.date.fromisoformat(value) + if pa.types.is_binary(dtype) and isinstance(value, str): + return value.encode("utf-8") + if pa.types.is_list(dtype) and isinstance(value, list): + return [_coerce_json_rows_for_arrow(item, dtype.value_type) for item in value] + if pa.types.is_struct(dtype) and isinstance(value, dict): + return { + field.name: _coerce_json_rows_for_arrow(value[field.name], field.type) + for field in dtype + if field.name in value + } + return value + + +def _coerce_rows_to_schema(rows: list[dict[str, Any]], schema: pa.Schema) -> list[dict[str, Any]]: + """Coerce JSON-loaded rows using field types in schema so pa.Table.from_pylist can materialize them.""" + struct_type = pa.struct(list(schema)) + return [_coerce_json_rows_for_arrow(row, struct_type) for row in rows] + + +def _json_default(obj: object) -> str: + """Serialize Python date and bytes values from Arrow tables for comparison with expected JSON.""" + if isinstance(obj, dt.date): + return obj.isoformat() + if isinstance(obj, bytes): + return obj.decode("utf-8") + msg = f"Object of type {type(obj)} is not JSON serializable" + raise TypeError(msg) + + +def _strip_none(value: object) -> object: + """Recursively remove keys with None values from dicts to compare Arrow-materialized rows with sparse expected.json.""" + if isinstance(value, dict): + return {k: _strip_none(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [_strip_none(item) for item in value] + return value + + +def _extract_matching_elements(xml_path: Path, xml_tag: str) -> list[etree._Element]: + """Parse xml_path and return all elements matching xml_tag at the root or as a direct child of root.""" + local_tag = xml_tag.rsplit("}", maxsplit=1)[-1] if "}" in xml_tag else xml_tag + tree = etree.parse(str(xml_path)) + root = tree.getroot() + root_local = root.tag.rsplit("}", maxsplit=1)[-1] if "}" in root.tag else root.tag + if root_local == local_tag: + return [root] + return [ + child + for child in root + if isinstance(child.tag, str) + and (child.tag.rsplit("}", maxsplit=1)[-1] if "}" in child.tag else child.tag) == local_tag + ] + + +SPEC_CASES: list[ParameterSet] = [ + pytest.param("01_scalars_and_simple_types", "record", id="01-scalars-and-simple-types"), + pytest.param("02_namespaces_and_nillable", "entry", id="02-namespaces-and-nillable"), + pytest.param("03_mixed_content_raw_text", "article", id="03-mixed-content-raw-text"), + pytest.param("04_recursion_depth", "record", id="04-recursion-depth"), + pytest.param("05_groups_and_compositors", "item", id="05-groups-and-compositors"), + pytest.param("06_context_dependent_paths", "root", id="06-context-dependent-paths"), + pytest.param("07_multi_root_shared_type", "entry", id="07-multi-root-shared-type"), + pytest.param("08_nullability_and_empty_tags", "entry", id="08-nullability-and-empty-tags"), + pytest.param("09_empty_marker_elements", "book", id="09-empty-marker-elements"), + pytest.param("10_combinations", "record", id="10-combinations"), +] + +REAL_WORLD_SCHEMAS: list[ParameterSet] = [ + pytest.param("uniref", "uniprot/uniref/uniref.xsd", "entry", id="uniref"), + pytest.param("uniprot", "uniprot/uniprot_kb/uniprot.xsd", "entry", id="uniprot"), + pytest.param("pubmed_article", "pubmed/pubmed.xsd", "PubmedArticle", id="pubmed-article"), + pytest.param("pubmed_book_article", "pubmed/pubmed.xsd", "PubmedBookArticle", id="pubmed-book-article"), +] + +REAL_WORLD_EXAMPLES: list[ParameterSet] = [ + pytest.param("uniref", "uniprot/uniref/uniref.xsd", "entry", "minimal", id="uniref-minimal"), + pytest.param("uniref", "uniprot/uniref/uniref.xsd", "entry", "maximal", id="uniref-maximal"), + pytest.param("uniprot", "uniprot/uniprot_kb/uniprot.xsd", "entry", "minimal", id="uniprot-minimal"), + pytest.param("uniprot", "uniprot/uniprot_kb/uniprot.xsd", "entry", "maximal", id="uniprot-maximal"), + pytest.param("pubmed_article", "pubmed/pubmed.xsd", "PubmedArticle", "minimal", id="pubmed-article-minimal"), + pytest.param("pubmed_article", "pubmed/pubmed.xsd", "PubmedArticle", "maximal", id="pubmed-article-maximal"), + pytest.param( + "pubmed_book_article", + "pubmed/pubmed.xsd", + "PubmedBookArticle", + "minimal", + id="pubmed-book-article-minimal", + ), + pytest.param( + "pubmed_book_article", + "pubmed/pubmed.xsd", + "PubmedBookArticle", + "maximal", + id="pubmed-book-article-maximal", + ), +] + + +@pytest.mark.parametrize(("case_dir", "xml_tag"), SPEC_CASES) +def test_spec_fixtures_pass_xml_and_expected_schema_validate( + test_data_dir: Path, + case_dir: str, + xml_tag: str, +) -> None: + """Every specification data.xml validates against schema.xsd and expected.json materializes with expected_schema.py.""" + fixture_dir = test_data_dir / "xsd" / "spec" / case_dir + schema = load_schema(fixture_dir / "schema.xsd") + schema.validate(str(fixture_dir / "data.xml")) + + elements = _extract_matching_elements(fixture_dir / "data.xml", xml_tag) + expected = json.loads((fixture_dir / "expected.json").read_text(encoding="utf-8")) + assert len(elements) == len(expected) + + expected_pa_schema = _load_expected_schema(fixture_dir / "expected_schema.py") + coerced_rows = _coerce_rows_to_schema(expected, expected_pa_schema) + table = pa.Table.from_pylist(coerced_rows, schema=expected_pa_schema) + table.validate(full=True) + assert table.num_rows == len(expected) + + dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) + assert dumped_rows == expected + + buf = io.BytesIO() + pq.write_table(table, buf) + buf.seek(0) + roundtrip_table = pq.read_table(buf) + roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) + assert roundtrip_rows == expected + + +@pytest.mark.parametrize( + ("example_dir", "xsd_rel_path", "xml_tag", "variant"), + REAL_WORLD_EXAMPLES, +) +def test_real_world_examples_pass_xml_and_expected_schema_validate( + test_data_dir: Path, + example_dir: str, + xsd_rel_path: str, + xml_tag: str, + variant: str, +) -> None: + """Every minimal and maximal real-world XML validates against its XSD and materializes with expected_schema.py.""" + schema = load_schema(test_data_dir / xsd_rel_path) + example_base = test_data_dir / "xsd" / "examples" / example_dir + xml_path = example_base / f"{variant}.xml" + expected_path = example_base / f"{variant}_expected.json" + + schema.validate(str(xml_path)) + elements = _extract_matching_elements(xml_path, xml_tag) + expected = json.loads(expected_path.read_text(encoding="utf-8")) + assert len(elements) == 1 + assert len(expected) == 1 + + expected_pa_schema = _load_expected_schema(example_base / "expected_schema.py") + coerced_rows = _coerce_rows_to_schema(expected, expected_pa_schema) + table = pa.Table.from_pylist(coerced_rows, schema=expected_pa_schema) + table.validate(full=True) + assert table.num_rows == 1 + + dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) + assert dumped_rows == expected + + buf = io.BytesIO() + pq.write_table(table, buf) + buf.seek(0) + roundtrip_table = pq.read_table(buf) + roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) + assert roundtrip_rows == expected + + +@pytest.mark.parametrize( + "example_dir", + [ + pytest.param("uniref", id="uniref-maximal"), + pytest.param("uniprot", id="uniprot-maximal"), + pytest.param("pubmed_article", id="pubmed-article-maximal"), + pytest.param("pubmed_book_article", id="pubmed-book-article-maximal"), + ], +) +def test_real_world_maximal_examples_pass_repeated_elements_have_distinct_content( + test_data_dir: Path, + example_dir: str, +) -> None: + """Every repeated element in a maximal XML example contains distinct data.""" + xml_path = test_data_dir / "xsd" / "examples" / example_dir / "maximal.xml" + root = etree.parse(str(xml_path)).getroot() + + by_tag: dict[str, list[str]] = defaultdict(list) + for elem in root.iter(): + if elem is root or not isinstance(elem.tag, str): + continue + clone = copy.deepcopy(elem) + for node in clone.iter(): + if node.text is not None: + node.text = node.text.strip() + if node.tail is not None: + node.tail = node.tail.strip() + local = elem.tag.rsplit("}", maxsplit=1)[-1] if "}" in elem.tag else elem.tag + by_tag[local].append(etree.tostring(clone, encoding="unicode")) + + duplicates = {tag: sigs for tag, sigs in by_tag.items() if len(sigs) > 1 and len(set(sigs)) < len(sigs)} + assert not duplicates + + +@pytest.mark.parametrize(("case_dir", "xml_tag"), SPEC_CASES) +def test_generate_pyarrow_schema_pass_spec_cases( + test_data_dir: Path, + case_dir: str, + xml_tag: str, +) -> None: + """Verify XSD specification schemas generate the exact expected PyArrow schema in expected_schema.py.""" + fixture_dir = test_data_dir / "xsd" / "spec" / case_dir + schema = load_schema(fixture_dir / "schema.xsd") + pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) + expected_schema = _load_expected_schema(fixture_dir / "expected_schema.py") + assert pa_schema.equals(expected_schema, check_metadata=True) + + +def test_generate_pyarrow_schema_pass_clark_notation_tag(test_data_dir: Path) -> None: + """Verify Clark-notation {namespace}local_name resolves identically to local_name.""" + fixture_dir = test_data_dir / "xsd" / "spec" / "02_namespaces_and_nillable" + schema = load_schema(fixture_dir / "schema.xsd") + pa_clark = generate_pyarrow_schema(schema, "{http://example.org/ns}entry") + expected_schema = _load_expected_schema(fixture_dir / "expected_schema.py") + assert pa_clark.equals(expected_schema, check_metadata=True) + + +def test_generate_pyarrow_schema_pass_04_recursion_depth_1(test_data_dir: Path) -> None: + """Verify structural recursion with max_recursion_depth=1 matches EXPECTED_SCHEMA_DEPTH_1.""" + fixture_dir = test_data_dir / "xsd" / "spec" / "04_recursion_depth" + schema = load_schema(fixture_dir / "schema.xsd") + schema_d1 = generate_pyarrow_schema(schema, "record", max_recursion_depth=1) + expected_d1 = _load_expected_schema(fixture_dir / "expected_schema.py", "EXPECTED_SCHEMA_DEPTH_1") + assert schema_d1.equals(expected_d1, check_metadata=True) + + +def test_generate_pyarrow_schema_pass_mutual_and_self_referential_xsds(test_data_dir: Path) -> None: + """Verify existing self_referential.xsd and mutual_recursion.xsd schemas terminate at max_recursion_depth.""" + self_ref_schema = load_schema(test_data_dir / "xsd" / "self_referential.xsd") + pa_self = generate_pyarrow_schema(self_ref_schema, "node", max_recursion_depth=2) + expected_self = pa.schema( + [ + _list_field( + "child", + pa.struct( + [ + _list_field("child", pa.string(), nullable=True), + ] + ), + nullable=True, + ), + ] + ) + assert pa_self.equals(expected_self, check_metadata=True) + + mutual_schema = load_schema(test_data_dir / "xsd" / "mutual_recursion.xsd") + pa_mutual = generate_pyarrow_schema(mutual_schema, "a", max_recursion_depth=1) + expected_mutual = pa.schema( + [ + _list_field( + "b", + pa.struct( + [ + pa.field("a", pa.string(), nullable=True), + ] + ), + nullable=True, + ), + ] + ) + assert pa_mutual.equals(expected_mutual, check_metadata=True) + + +@pytest.mark.parametrize( + ("example_dir", "xsd_rel_path", "xml_tag"), + REAL_WORLD_SCHEMAS, +) +def test_generate_pyarrow_schema_pass_real_world_schemas( + test_data_dir: Path, + example_dir: str, + xsd_rel_path: str, + xml_tag: str, +) -> None: + """Verify PyArrow schema generation on UniRef, UniProtKB, and PubMed XSDs matches expected_schema.py.""" + schema = load_schema(test_data_dir / xsd_rel_path) + pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) + expected_schema = _load_expected_schema(test_data_dir / "xsd" / "examples" / example_dir / "expected_schema.py") + assert pa_schema.equals(expected_schema, check_metadata=True) + + +@pytest.mark.parametrize( + ("error_xsd_filename", "xml_tag", "expected_match"), + [ + pytest.param("ambiguous_xml_tag.xsd", "entry", "Ambiguous", id="E2-ambiguous-xml-tag"), + pytest.param("scalar_top_level.xsd", "leaf", "complex type", id="E3-scalar-top-level"), + pytest.param("empty_marker_top_level.xsd", "marker", "complex type", id="E3-empty-marker-top-level"), + pytest.param("wildcard_any_element.xsd", "entry", "xs:any", id="E4-wildcard-any-element"), + pytest.param("wildcard_any_attribute.xsd", "entry", "xs:anyAttribute", id="E5-wildcard-any-attribute"), + pytest.param("abstract_element.xsd", "entry", "abstract", id="E6-abstract-element"), + pytest.param("abstract_type.xsd", "entry", "abstract", id="E6-abstract-type"), + pytest.param("substitution_group.xsd", "entry", "substitutionGroup", id="E7-substitution-group"), + pytest.param("attr_child_name_collision.xsd", "entry", "collision", id="E8-attr-child-name-collision"), + ], +) +def test_generate_pyarrow_schema_fail_unsupported_or_invalid_xsd( + test_data_dir: Path, + error_xsd_filename: str, + xml_tag: str, + expected_match: str, +) -> None: + """Verify unsupported or ambiguous XSD features raise a descriptive ValueError.""" + schema = load_schema(test_data_dir / "xsd" / "spec" / "errors" / error_xsd_filename) + with pytest.raises(ValueError, match=expected_match): + generate_pyarrow_schema(schema, xml_tag) + + +@pytest.mark.parametrize( + ("xsd_filename", "xml_tag", "expected_match"), + [ + pytest.param("empty_schema.xsd", "root", "not found", id="E1-empty-schema"), + pytest.param("no_children.xsd", "leaf", "complex type", id="E3-no-children-scalar-root"), + pytest.param("cdata_variety.xsd", "root", "xs:anyAttribute", id="E5-cdata-variety-has-anyAttribute"), + pytest.param("combinations.xsd", "record", "xs:anyAttribute", id="E5-legacy-combinations-has-anyAttribute"), + ], +) +def test_generate_pyarrow_schema_fail_existing_invalid_schemas( + test_data_dir: Path, + xsd_filename: str, + xml_tag: str, + expected_match: str, +) -> None: + """Verify existing XSDs with scalar roots, empty schemas, or xs:anyAttribute raise ValueError.""" + schema = load_schema(test_data_dir / "xsd" / xsd_filename) + with pytest.raises(ValueError, match=expected_match): + generate_pyarrow_schema(schema, xml_tag) + + +def test_generate_pyarrow_schema_fail_nonexistent_xml_tag(test_data_dir: Path) -> None: + """Verify requesting an xml_tag that does not exist in the XSD raises ValueError.""" + schema = load_schema(test_data_dir / "uniprot" / "uniref" / "uniref.xsd") + with pytest.raises(ValueError, match="not found"): + generate_pyarrow_schema(schema, "nonExistentElement") + + +@pytest.mark.parametrize( + "forbidden_xsd_type", + [ + pytest.param("xs:date", id="xs-date"), + pytest.param("xs:dateTime", id="xs-dateTime"), + pytest.param("xs:hexBinary", id="xs-hexBinary"), + pytest.param("xs:base64Binary", id="xs-base64Binary"), + ], +) +def test_generate_pyarrow_schema_fail_non_json_serializable_type_under_recursion( + forbidden_xsd_type: str, +) -> None: + """Verify xs:date, xs:dateTime, xs:hexBinary, and xs:base64Binary under a recursive element raise ValueError.""" + xsd_with_child = f""" + + + + + + + + + + """ + schema_child = xmlschema.XMLSchema(xsd_with_child) + with pytest.raises(ValueError, match="recursive type"): + generate_pyarrow_schema(schema_child, "root") + + xsd_with_attr = f""" + + + + + + + + + + """ + schema_attr = xmlschema.XMLSchema(xsd_with_attr) + with pytest.raises(ValueError, match="recursive type"): + generate_pyarrow_schema(schema_attr, "root") + + +@pytest.mark.parametrize(("case_dir", "xml_tag"), SPEC_CASES) +def test_convert_xml_element_and_arrow_table_pass_spec_fixtures( + test_data_dir: Path, + case_dir: str, + xml_tag: str, +) -> None: + """Verify XML elements convert to expected JSON and build a valid PyArrow Table and Parquet file.""" + fixture_dir = test_data_dir / "xsd" / "spec" / case_dir + schema = load_schema(fixture_dir / "schema.xsd") + pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) + + elements = _extract_matching_elements(fixture_dir / "data.xml", xml_tag) + rows = [convert_xml_element(elem, schema, xml_tag, max_recursion_depth=2) for elem in elements] + + expected = json.loads((fixture_dir / "expected.json").read_text(encoding="utf-8")) + serialized_rows = json.loads(json.dumps(rows, default=_json_default)) + assert serialized_rows == expected + + table = pa.Table.from_pylist(rows, schema=pa_schema) + table.validate(full=True) + assert table.num_rows == len(expected) + + dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) + assert dumped_rows == expected + + buf = io.BytesIO() + pq.write_table(table, buf) + buf.seek(0) + roundtrip_table = pq.read_table(buf) + roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) + assert roundtrip_rows == expected + + +def test_convert_xml_element_and_arrow_table_pass_recursion_depth_1(test_data_dir: Path) -> None: + """Verify max_recursion_depth=1 truncates recursive structures one level earlier than depth=2.""" + fixture_dir = test_data_dir / "xsd" / "spec" / "04_recursion_depth" + schema = load_schema(fixture_dir / "schema.xsd") + pa_schema = generate_pyarrow_schema(schema, "record", max_recursion_depth=1) + + elements = _extract_matching_elements(fixture_dir / "data.xml", "record") + rows = [convert_xml_element(elem, schema, "record", max_recursion_depth=1) for elem in elements] + + expected = json.loads((fixture_dir / "expected_depth_1.json").read_text(encoding="utf-8")) + serialized_rows = json.loads(json.dumps(rows, default=_json_default)) + assert serialized_rows == expected + + table = pa.Table.from_pylist(rows, schema=pa_schema) + table.validate(full=True) + dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) + assert dumped_rows == expected + + buf = io.BytesIO() + pq.write_table(table, buf) + buf.seek(0) + roundtrip_table = pq.read_table(buf) + roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) + assert roundtrip_rows == expected + + +@pytest.mark.parametrize( + ("example_dir", "xsd_rel_path", "xml_tag", "variant"), + REAL_WORLD_EXAMPLES, +) +def test_convert_xml_element_and_arrow_table_pass_real_world_examples( + test_data_dir: Path, + example_dir: str, + xsd_rel_path: str, + xml_tag: str, + variant: str, +) -> None: + """Verify minimal and maximal UniRef, UniProt, and PubMed entries convert to expected JSON and Arrow tables.""" + schema = load_schema(test_data_dir / xsd_rel_path) + xml_path = test_data_dir / "xsd" / "examples" / example_dir / f"{variant}.xml" + expected_path = test_data_dir / "xsd" / "examples" / example_dir / f"{variant}_expected.json" + + pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) + elements = _extract_matching_elements(xml_path, xml_tag) + rows = [convert_xml_element(elem, schema, xml_tag, max_recursion_depth=2) for elem in elements] + + expected = json.loads(expected_path.read_text(encoding="utf-8")) + serialized_rows = json.loads(json.dumps(rows, default=_json_default)) + assert serialized_rows == expected + + table = pa.Table.from_pylist(rows, schema=pa_schema) + table.validate(full=True) + assert table.num_rows == 1 + + dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) + assert dumped_rows == expected + + buf = io.BytesIO() + pq.write_table(table, buf) + buf.seek(0) + roundtrip_table = pq.read_table(buf) + assert roundtrip_table.num_rows == 1 + roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) + assert roundtrip_rows == expected diff --git a/tests/cdm_data_loaders/readers/xml/test_xml_process_xml_file_and_batches.py b/tests/cdm_data_loaders/readers/xml/test_xml_process_xml_file_and_batches.py index 8659678f..e7caac24 100644 --- a/tests/cdm_data_loaders/readers/xml/test_xml_process_xml_file_and_batches.py +++ b/tests/cdm_data_loaders/readers/xml/test_xml_process_xml_file_and_batches.py @@ -1,4 +1,4 @@ -"""Tests for `process_xml_file`, `process_xml_file_items`, `build_xml_file_resource`, and `process_xml_file_to_dict` from cdm_data_loaders.readers.xml.""" +"""Tests for `process_xml_file`, `process_xml_file_items`, `build_xml_file_resource`, `process_xml_file_to_dict`, and `process_xml_file_to_pyarrow` from cdm_data_loaders.readers.xml.""" import gzip import logging @@ -7,7 +7,9 @@ from typing import Any, Final from unittest.mock import MagicMock +import pyarrow as pa import pytest +import xmlschema import xmltodict from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.extract.items import DataItemWithMeta, TableNameMeta @@ -19,13 +21,16 @@ from cdm_data_loaders.core.fields import BUFFER_SIZE, DEFAULT_XML_FILE_GLOB, DEFAULTS, LOG_INTERVAL from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings +from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings from cdm_data_loaders.readers.xml import ( DEFAULT_XMLTODICT_ARGS, process_xml_file, process_xml_file_batches, process_xml_file_items, process_xml_file_to_dict, + process_xml_file_to_pyarrow, ) +from cdm_data_loaders.readers.xsd import DEFAULT_MAX_RECURSION_DEPTH, generate_pyarrow_schema ParseFn = Callable[..., dict[str, list[dict[str, Any]]]] @@ -84,6 +89,28 @@ {"person": {"_id": p["id"], "name": p["name"], "email": p["email"]}} for p in PEOPLE_PARSED ] +PEOPLE_XML_TO_PYARROW_PARSED = [{"_id": p["id"], "name": p["name"], "email": p["email"]} for p in PEOPLE_PARSED] + +PEOPLE_XSD: Final[str] = """ + + + + + + + + + + + + + + + + + +""" + PEOPLE_XML_EMPTY: Final[str] = """ @@ -248,6 +275,32 @@ def fake_xmltodict_settings( return settings +def fake_xmltopyarrow_settings( + *, + table_name: str = "people", + xml_tag: str = "person", + xsd_text: str = PEOPLE_XSD, + buffer_size: int = DEFAULTS[BUFFER_SIZE], + log_interval: int = DEFAULTS[LOG_INTERVAL], + max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH, +) -> XmlToPyArrowSettings: + """Build a MagicMock XmlToPyArrowSettings with xsd_schema/pyarrow_schema set, for process_xml_file_to_pyarrow tests.""" + settings: XmlToPyArrowSettings = fake_settings( # pyright: ignore[reportAssignmentType] + buffer_size=buffer_size, log_interval=log_interval, settings_class=XmlToPyArrowSettings + ) + settings.table_name = table_name + settings.xml_tag = xml_tag + settings.max_recursion_depth = max_recursion_depth + schema = xmlschema.XMLSchema(xsd_text) + settings.xsd_schema = schema # pyright: ignore[reportAttributeAccessIssue] + settings.pyarrow_schema = generate_pyarrow_schema( # pyright: ignore[reportAttributeAccessIssue] + schema, + xml_tag, + max_recursion_depth=max_recursion_depth, + ) + return settings + + def test_process_xml_file_pass_single_table_single_entry(tmp_path: Path) -> None: """Verify a single matching element is parsed into one correctly table-tagged row.""" file_path = _write_xml(tmp_path, "one.xml", PEOPLE_XML_2) @@ -660,33 +713,44 @@ def test_process_xml_file_to_dict_fail_malformed_xml(tmp_path: Path) -> None: """ -def _build_real_xmltodict_settings(tmp_path: Path, test_data_dir: Path, xsd_filename: str | None) -> XmlToDictSettings: - """Build a real (non-mock) XmlToDictSettings, optionally with an xsd_file copied from tests/data/xsd. +def _build_real_xmltodict_settings(tmp_path: Path) -> XmlToDictSettings: + """Build a real (non-mock) XmlToDictSettings.""" + input_dir = tmp_path / "input" + input_dir.mkdir(exist_ok=True) + log_config_file = tmp_path / "logging.json" + log_config_file.write_text('{"version": 1}', encoding="utf-8") - A real settings object is required here, rather than fake_xmltodict_settings, because - xmltodict_args is a computed property with real schema-derived logic to exercise: a mock - would either need that same logic re-implemented by hand (proving nothing about the actual - implementation) or would unpack to `{}` by default, silently skipping the feature entirely. - """ + return XmlToDictSettings( + input_dir=str(input_dir), + log_config_file=str(log_config_file), + dataset_name="xsd_test_dataset", + table_name="entry", + xml_tag="entry", + buffer_size=100, + log_interval=1000, + output_dir=tmp_path, + use_destination="whatever", + ) # pyright: ignore[reportCallIssue] + + +def _build_real_xmltopyarrow_settings(tmp_path: Path, test_data_dir: Path, xsd_filename: str) -> XmlToPyArrowSettings: + """Build a real (non-mock) XmlToPyArrowSettings with an xsd_file copied from tests/data/xsd.""" input_dir = tmp_path / "input" input_dir.mkdir(exist_ok=True) log_config_file = tmp_path / "logging.json" log_config_file.write_text('{"version": 1}', encoding="utf-8") - xsd_file = None - if xsd_filename is not None: - xsd_content = (test_data_dir / "xsd" / xsd_filename).read_text(encoding="utf-8") - xsd_path = input_dir / xsd_filename - xsd_path.write_text(xsd_content, encoding="utf-8") - xsd_file = str(xsd_path) + xsd_content = (test_data_dir / "xsd" / xsd_filename).read_text(encoding="utf-8") + xsd_path = input_dir / xsd_filename + xsd_path.write_text(xsd_content, encoding="utf-8") - return XmlToDictSettings( + return XmlToPyArrowSettings( input_dir=str(input_dir), log_config_file=str(log_config_file), dataset_name="xsd_test_dataset", table_name="entry", xml_tag="entry", - xsd_file=xsd_file, + xsd_file=str(xsd_path), buffer_size=100, log_interval=1000, output_dir=tmp_path, @@ -694,51 +758,229 @@ def _build_real_xmltodict_settings(tmp_path: Path, test_data_dir: Path, xsd_file ) # pyright: ignore[reportCallIssue] +def test_process_xml_file_to_dict_pass_keeps_xmltodict_default_dicts( + tmp_path: Path, +) -> None: + """process_xml_file_to_dict uses xmltodict's ordinary single-occurrence dicts.""" + file_path = _write_xml(tmp_path, "entry.xml", UNIREF_LIKE_ENTRY_XML) + settings = _build_real_xmltodict_settings(tmp_path) + + items = _table_and_data(process_xml_file_to_dict(settings, file_path=file_path)) + + assert items == [ + ( + "entry", + [ + { + "entry": { + "property": {"_type": "member count", "_value": "1"}, + "representativeMember": { + "dbReference": {"property": {"_type": "isSeed", "_value": "true"}}, + }, + } + } + ], + ) + ] + + +"""process_xml_file_to_pyarrow""" + + +def test_process_xml_file_to_pyarrow_pass_custom_table_name_and_xml_tag_are_respected(tmp_path: Path) -> None: + """Verify rows land under settings.table_name as a PyArrow Table, matched by settings.xml_tag.""" + file_path = _write_xml(tmp_path, "two.xml", PEOPLE_XML_2) + settings = fake_xmltopyarrow_settings(table_name="library_entries", xml_tag="person") + + tagged = _table_and_data(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + assert [table_name for table_name, _ in tagged] == ["library_entries"] + assert isinstance(tagged[0][1], pa.Table) + assert tagged[0][1].to_pylist() == PEOPLE_XML_TO_PYARROW_PARSED[:2] + + +def test_process_xml_file_to_pyarrow_pass_resolves_default_namespace_on_unprefixed_tag(tmp_path: Path) -> None: + """Verify an unprefixed xml_tag resolves the root default namespace via parse_head_matter.""" + namespace = "http://example.com/people" + namespaced_xsd = f""" + + + + + + + + + + + + + + + + + +""" + namespaced_xml = f""" + + Anne Examplea@example.com + Belinda Carlisleb@example.com + +""" + file_path = _write_xml(tmp_path, "namespaced.xml", namespaced_xml) + settings = fake_xmltopyarrow_settings(table_name="people", xml_tag="person", xsd_text=namespaced_xsd) + + tagged = _table_and_data(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + assert [table_name for table_name, _ in tagged] == ["people"] + assert tagged[0][1].to_pylist() == PEOPLE_XML_TO_PYARROW_PARSED[:2] + + @pytest.mark.parametrize( - ("xsd_filename", "expected_entry"), + ("buffer_size", "expected_batch_sizes"), [ - pytest.param( - "uniref_like.xsd", - { - "property": [{"_type": "member count", "_value": "1"}], - "representativeMember": { - "dbReference": {"property": [{"_type": "isSeed", "_value": "true"}]}, - }, - }, - id="with-xsd-file-forces-repeatable-schema-children-into-lists", - ), - pytest.param( - None, - { - "property": {"_type": "member count", "_value": "1"}, - "representativeMember": { - "dbReference": {"property": {"_type": "isSeed", "_value": "true"}}, - }, - }, - id="without-xsd-file-keeps-xmltodicts-default-single-occurrence-dicts", - ), + pytest.param(1, [1, 1, 1, 1, 1], id="one-row-batches"), + pytest.param(2, [2, 2, 1], id="two-row-batches-with-remainder"), + pytest.param(3, [3, 2], id="three-row-batches-with-remainder"), + pytest.param(10, [5], id="larger-than-input"), + ], +) +def test_process_xml_file_to_pyarrow_pass_buffer_size_controls_batching( + tmp_path: Path, buffer_size: int, expected_batch_sizes: list[int] +) -> None: + """Verify rows are yielded in pa.Table batches of at most `buffer_size`, with a final partial batch.""" + file_path = _write_xml(tmp_path, "five.xml", PEOPLE_XML_5) + settings = fake_xmltopyarrow_settings(buffer_size=buffer_size) + + items = _table_and_data(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + assert [table.num_rows for _, table in items] == expected_batch_sizes + assert [row for _, table in items for row in table.to_pylist()] == PEOPLE_XML_TO_PYARROW_PARSED + assert {table_name for table_name, _ in items} == {"people"} + + +def test_process_xml_file_to_pyarrow_pass_no_matching_elements_yields_nothing_and_logs_no_progress( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Verify a well-formed file with no elements matching xml_tag yields nothing and logs no progress.""" + caplog.set_level(logging.DEBUG) + file_path = _write_xml(tmp_path, "empty.xml", PEOPLE_XML_EMPTY) + settings = fake_xmltopyarrow_settings() + + items = list(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + assert items == [] + assert [r for r in caplog.records if r.msg.startswith("Processed")] == [] + + +def test_process_xml_file_to_pyarrow_pass_logs_reading_info_message_once( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Verify the INFO-level 'Reading from ' message is logged exactly once with the correct path.""" + caplog.set_level(logging.INFO) + file_path = _write_xml(tmp_path, "five.xml", PEOPLE_XML_5) + settings = fake_xmltopyarrow_settings() + + list(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + info_records = [r for r in caplog.records if r.levelno == logging.INFO] + assert len(info_records) == 1 + assert info_records[0].msg == "Reading from %s" + assert info_records[0].args == (str(file_path),) + + +@pytest.mark.parametrize( + ("log_interval", "expected_interim_counts", "expected_final_count"), + [ + pytest.param(1, [1, 2, 3, 4, 5], None, id="divides_evenly_every_entry"), + pytest.param(5, [5], None, id="divides_evenly_equal_to_count"), + pytest.param(2, [2, 4], 5, id="remainder_leaves_trailing_entries"), + pytest.param(10, [], 5, id="interval_larger_than_entry_count"), ], ) -def test_process_xml_file_to_dict_pass_force_list_shapes_children_by_schema_occurs( +def test_process_xml_file_to_pyarrow_pass_log_interval_boundary( tmp_path: Path, - test_data_dir: Path, - xsd_filename: str | None, - expected_entry: dict[str, Any], + caplog: pytest.LogCaptureFixture, + log_interval: int, + expected_interim_counts: list[int], + expected_final_count: int | None, ) -> None: - """Verify children the schema allows to repeat become lists even with a single occurrence. + """Verify the exact entry counts logged at each interim checkpoint and in the trailing summary.""" + caplog.set_level(logging.DEBUG) + file_path = _write_xml(tmp_path, "five.xml", PEOPLE_XML_5) + settings = fake_xmltopyarrow_settings(log_interval=log_interval) - With xsd_file set, every child the schema allows to repeat must be a list, regardless of how - many times it actually occurs in this particular document, while children the schema allows - only once stay plain dicts -- at every nesting depth. Without xsd_file, xmltodict's ordinary - behaviour applies instead: a lone occurrence is never turned into a list, whether or not the - schema would allow more. - """ + list(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + interim_counts = [r.args[0] for r in caplog.records if r.msg == "Processed %d entries"] + final_records = [r for r in caplog.records if r.msg == "Processed %d entries from %s"] + + assert interim_counts == expected_interim_counts + if expected_final_count is None: + assert final_records == [] + else: + assert len(final_records) == 1 + assert final_records[0].args == (expected_final_count, file_path.name) + + +def test_process_xml_file_to_pyarrow_pass_gzip_file(tmp_path: Path) -> None: + """Verify gzip-compressed XML input (.gz suffix) is transparently decompressed and parsed.""" + plain_path = _write_xml(tmp_path, "plain.xml", PEOPLE_XML_5) + gz_path = _write_xml(tmp_path, "compressed.xml.gz", PEOPLE_XML_5, gzip_compress=True) + + plain_items = _table_and_data(process_xml_file_to_pyarrow(fake_xmltopyarrow_settings(), file_path=plain_path)) + gz_items = _table_and_data(process_xml_file_to_pyarrow(fake_xmltopyarrow_settings(), file_path=gz_path)) + + plain_rows = [row for _, table in plain_items for row in table.to_pylist()] + gz_rows = [row for _, table in gz_items for row in table.to_pylist()] + assert gz_rows == plain_rows == PEOPLE_XML_TO_PYARROW_PARSED + + +def test_process_xml_file_to_pyarrow_fail_missing_file(tmp_path: Path) -> None: + """Verify a nonexistent file path raises FileNotFoundError when the generator is consumed.""" + missing_path = tmp_path / "does_not_exist.xml" + settings = fake_xmltopyarrow_settings() + with pytest.raises(FileNotFoundError, match="No such file or directory"): + list(process_xml_file_to_pyarrow(settings, file_path=missing_path)) + + +def test_process_xml_file_to_pyarrow_fail_malformed_xml(tmp_path: Path) -> None: + """Verify malformed (not well-formed) XML raises lxml's XMLSyntaxError during streaming.""" + file_path = _write_xml(tmp_path, "malformed.xml", PEOPLE_XML_MALFORMED) + settings = fake_xmltopyarrow_settings() + with pytest.raises(XMLSyntaxError, match="Opening and ending tag mismatch"): + list(process_xml_file_to_pyarrow(settings, file_path=file_path)) + + +def test_process_xml_file_to_pyarrow_pass_with_xsd_file_yields_arrow_table_with_schema_lists( + tmp_path: Path, + test_data_dir: Path, +) -> None: + """process_xml_file_to_pyarrow yields a PyArrow Table with schema-driven list and struct fields.""" file_path = _write_xml(tmp_path, "entry.xml", UNIREF_LIKE_ENTRY_XML) - settings = _build_real_xmltodict_settings(tmp_path, test_data_dir, xsd_filename) + settings = _build_real_xmltopyarrow_settings(tmp_path, test_data_dir, "uniref_like.xsd") - items = _table_and_data(process_xml_file_to_dict(settings, file_path=file_path)) + items = _table_and_data(process_xml_file_to_pyarrow(settings, file_path=file_path)) - assert items == [("entry", [{"entry": expected_entry}])] + assert len(items) == 1 + table_name, table = items[0] + assert table_name == "entry" + assert isinstance(table, pa.Table) + assert settings.pyarrow_schema is not None + assert table.schema.equals(settings.pyarrow_schema, check_metadata=True) + assert table.to_pylist() == [ + { + "property": [{"_type": "member count", "_value": "1"}], + "representativeMember": { + "dbReference": {"property": [{"_type": "isSeed", "_value": "true"}]}, + "sequence": None, + }, + "member": None, + } + ] """process_xml_file_items""" diff --git a/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py b/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py index 956ec4ba..ec0b68e5 100644 --- a/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py +++ b/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest -from lxml.etree import XMLSyntaxError +from lxml.etree import XMLSyntaxError, tostring from cdm_data_loaders.readers.xml import stream_xml_file from tests.xml_samples import SIMPLE_LIBRARY_XML @@ -205,3 +205,76 @@ def test_stream_xml_file_fail_non_gzip_content_with_gz_extension_raises_error(tm with pytest.raises(OSError, match="Not a gzipped file"): list(stream_xml_file(fake_gz_path, "book")) + + +@pytest.mark.parametrize( + ("xml", "tag", "expected"), + [ + pytest.param("", "node", [], id="no-match"), + pytest.param( + '', + "node", + [b'', b''], + id="siblings", + ), + pytest.param( + '', + "node", + [b'', b''], + id="matches-under-different-parents", + ), + pytest.param( + '', + "node", + [b''], + id="nested-once", + ), + pytest.param( + 'x', + "node", + [b'x'], + id="nested-with-earlier-siblings", + ), + pytest.param( + '', + "node", + [b''], + id="nested-three-levels", + ), + pytest.param( + '', + "node", + [b'', b''], + id="nested-then-sibling", + ), + pytest.param( + '', + "node", + [b''], + id="root-is-a-match", + ), + pytest.param( + '', + "{urn:x}node", + [b''], + id="nested-in-default-namespace", + ), + ], +) +@pytest.mark.parametrize("compressed", [False, True], ids=["plain", "gzip"]) +def test_stream_xml_file_pass_yields_outermost_matches( # noqa: PLR0917 + tmp_path: Path, + write_gzip_file: Callable[[Path, str, str], Path], + xml: str, + tag: str, + expected: list[bytes], + compressed: bool, +) -> None: + """Only the outermost matching elements are yielded, each with all of its content.""" + if compressed: + file_path = write_gzip_file(tmp_path, "doc.xml.gz", xml) + else: + file_path = tmp_path / "doc.xml" + file_path.write_text(xml, encoding="utf-8") + + assert [tostring(elem, with_tail=False) for elem in stream_xml_file(file_path, tag)] == expected diff --git a/tests/data/xsd/attribute_only.xsd b/tests/data/xsd/attribute_only.xsd deleted file mode 100644 index 516e2f56..00000000 --- a/tests/data/xsd/attribute_only.xsd +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/tests/data/xsd/cdata_variety.xsd b/tests/data/xsd/cdata_variety.xsd new file mode 100644 index 00000000..5310cf36 --- /dev/null +++ b/tests/data/xsd/cdata_variety.xsd @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/combinations.xsd b/tests/data/xsd/combinations.xsd new file mode 100644 index 00000000..6ae69962 --- /dev/null +++ b/tests/data/xsd/combinations.xsd @@ -0,0 +1,245 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/examples/pubmed_article/expected_schema.py b/tests/data/xsd/examples/pubmed_article/expected_schema.py new file mode 100644 index 00000000..a5dad39c --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/expected_schema.py @@ -0,0 +1,1584 @@ +"""Expected PyArrow schema for pubmed_article (xml_tag='PubmedArticle').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "MedlineCitation", + pa.struct( + [ + pa.field("_Owner", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_Status", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_VersionID", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_VersionDate", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_IndexingMethod", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "PMID", + pa.struct( + [ + pa.field("_Version", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + pa.field( + "DateCompleted", + pa.struct( + [ + pa.field("Year", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Month", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Day", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "DateRevised", + pa.struct( + [ + pa.field("Year", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Month", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Day", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "Article", + pa.struct( + [ + pa.field("_PubModel", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "Journal", + pa.struct( + [ + pa.field( + "ISSN", + pa.struct( + [ + pa.field( + "_IssnType", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "JournalIssue", + pa.struct( + [ + pa.field( + "_CitedMedium", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Volume", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Issue", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "PubDate", + pa.struct( + [ + pa.field( + "Year", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Month", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Day", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Season", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "MedlineDate", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "Title", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "ISOAbbreviation", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ArticleTitle", + pa.struct( + [ + pa.field( + "_book", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_part", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_sec", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=False, + ), + pa.field( + "Pagination", + pa.struct( + [ + pa.field( + "StartPage", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "EndPage", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "MedlinePgn", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "ELocationID", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_EIdType", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "Abstract", + pa.struct( + [ + pa.field( + "AbstractText", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Label", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_NlmCategory", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "CopyrightInformation", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "AuthorList", + pa.struct( + [ + pa.field( + "_CompleteYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_Type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Author", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_EqualContrib", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "LastName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "CollectiveName", + pa.struct( + [ + pa.field( + "_Investigators", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:IDREF"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "AffiliationInfo", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Affiliation", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "Language", + pa.list_( + pa.field( + "item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ), + nullable=False, + ), + pa.field( + "DataBankList", + pa.struct( + [ + pa.field( + "_CompleteYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "DataBank", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "DataBankName", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "AccessionNumberList", + pa.struct( + [ + pa.field( + "AccessionNumber", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "GrantList", + pa.struct( + [ + pa.field( + "_CompleteYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Grant", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "GrantID", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Acronym", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Agency", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Country", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "PublicationTypeList", + pa.struct( + [ + pa.field( + "PublicationType", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_UI", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "VernacularTitle", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "ArticleDate", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_DateType", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Year", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Month", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Day", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + pa.field( + "MedlineJournalInfo", + pa.struct( + [ + pa.field("Country", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("MedlineTA", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("NlmUniqueID", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("ISSNLinking", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + pa.field( + "ChemicalList", + pa.struct( + [ + pa.field( + "Chemical", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "RegistryNumber", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "NameOfSubstance", + pa.struct( + [ + pa.field( + "_UI", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "SupplMeshList", + pa.struct( + [ + pa.field( + "SupplMeshName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_UI", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "CitationSubset", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "CommentsCorrectionsList", + pa.struct( + [ + pa.field( + "CommentsCorrections", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_RefType", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "RefSource", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "PMID", + pa.struct( + [ + pa.field( + "_Version", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "Note", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "GeneSymbolList", + pa.struct( + [ + pa.field( + "GeneSymbol", + pa.list_( + pa.field( + "item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "MeshHeadingList", + pa.struct( + [ + pa.field( + "MeshHeading", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "DescriptorName", + pa.struct( + [ + pa.field( + "_MajorTopicYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_AutoHM", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_Type", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_UI", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "QualifierName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_AutoHM", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_MajorTopicYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_UI", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field("NumberOfReferences", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "PersonalNameSubjectList", + pa.struct( + [ + pa.field( + "PersonalNameSubject", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "LastName", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "OtherID", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "OtherAbstract", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_Language", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "AbstractText", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Label", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_NlmCategory", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "CopyrightInformation", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "KeywordList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Owner", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Keyword", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_MajorTopicYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("CoiStatement", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "SpaceFlightMission", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "InvestigatorList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_ID", pa.string(), nullable=True, metadata={"xsd_type": "xs:ID"}), + pa.field( + "Investigator", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "LastName", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "AffiliationInfo", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Affiliation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "GeneralNote", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Owner", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + pa.field( + "PubmedData", + pa.struct( + [ + pa.field( + "History", + pa.struct( + [ + pa.field( + "PubMedPubDate", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_PubStatus", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Year", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Month", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Day", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Hour", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Minute", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Second", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field("PublicationStatus", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "ObjectList", + pa.struct( + [ + pa.field( + "Object", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Param", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "ReferenceList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Title", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Citation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ReferenceList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Title", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Citation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ReferenceList", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/examples/pubmed_article/maximal.xml b/tests/data/xsd/examples/pubmed_article/maximal.xml new file mode 100644 index 00000000..2e4c380c --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/maximal.xml @@ -0,0 +1,433 @@ + + + + + 38123456 + + 2024 + 05 + 29 + + + 2025 + 01 + 16 + +
+ + 1476-4687 + + 629 + 8012 + + 2023 + 09 + 15 + + + Nature + Nature + + Structural basis of TP53 tetramerization and Ca2+ signaling + + 410 + 418 + 410-418 + + 10.1038/s41586-024-07123-x + S0028-0836(24)00123-4 + + The tumor suppressor p53 regulates transcription in response to <5 Gy stress and H2O2. + We undertook 20 semi-structured interviews with participants aged >18 years studying Ca2+ signaling & DNA repair and underlined text. + + Copyright (c) 2024 Springer Nature Limited. + + + + Morris + Charlotte + C + Jr + 0000-0002-9047-0481 + 0000-0001-6450-5815 + + The University of Manchester, Division of Population Health, Manchester, United Kingdom. + 0000-0003-2804-1181 + 0000-0002-0939-9842 + + + National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. + 0000-0001-5784-7403 + 0000-0002-1825-0097 + + + + International Cancer Genome Consortium + https://ror.org/027m9bs27 + https://ror.org/01ej9dk98 + + Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. + https://ror.org/00za53h95 + https://ror.org/03vek6s52 + + + Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. + 0000-0002-9047-0481 (11) + 0000-0001-6450-5815 (12) + + + + eng + fre + + + PDB + + 1TUP + 2OCJ + + + + GenBank + + NM_000546 + AF307851 + + + + + + R01-CA043460 + CA + NCI NIH HHS + United States + + + MR/T021975/1 + MRC + Medical Research Council + United Kingdom + + + + Journal Article + Research Support, Non-U.S. Gov't + + Bases structurales de la tetramerisation de TP53 + + 2022 + 11 + 04 + + + 2021 + 03 + 21 + +
+ + Belgium + Nature + 0410462 + 0028-0836 + + + + EC 2.7.11.1 + Tumor Suppressor Protein p53 + + + EC 6.3.2.19 + Proto-Oncogene Proteins c-mdm2 + + + + Li-Fraumeni Syndrome + Chromatin Immunoprecipitation + + IM + AIM + + + Nature. 2000 Nov 16;408(6810):307-310. + 11099028 + Foundational review of the p53 signaling network. + + + Science. 2024 Jun 7;384(6700):1020-1021. + 38843210 + Perspective highlighting TP53 tetramerization dynamics. + + + + TP53 + MDM2 + + + + Tumor Suppressor Protein p53 + genetics + metabolism + + + DNA Repair + physiology + drug effects + + + 42 + + + Kontopantelis + Evangelos + E + III + + + Robinson + Louise + L + Sr + + + NTRS-20240001234 + PIP-987654 + + Tetramerization affinity increased in vitro by 3.5-fold (p < 0.001) upon phosphorylation. + Clarifying TP53 & MDM2 kinetics improves targeted cancer therapy. + Copyright (c) 2024 The Authors. Published under CC-BY 4.0. + + + French summary: Le facteur de transcription p53 maintient la stabilite du genome. + Plain-language summary: How cells repair damaged DNA when exposed to &gt;2 Gy radiation. + Copyright (c) 2024 National Library of Medicine. + + + p53 signaling + tetramerization + + + DNA damage response + apoptosis + + The authors declare no competing financial or non-financial interests. + ISS Expedition 68 + Artemis I BioSentinel + + + Ashcroft + Darren + D + II + 0000-0003-2804-1181 (13) + 0000-0002-0939-9842 (14) + + MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. + 0000-0001-5784-7403 (15) + 0000-0002-1825-0097 (16) + + + Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. + https://ror.org/027m9bs27 (17) + https://ror.org/01ej9dk98 (18) + + + + Blakeman + Thomas + T + IV + https://ror.org/00za53h95 (19) + https://ror.org/03vek6s52 (20) + + Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. + 0000-0002-9047-0481 (21) + 0000-0001-6450-5815 (22) + + + Department of Molecular Biology, Princeton University, Princeton, NJ, USA. + 0000-0003-2804-1181 (23) + 0000-0002-0939-9842 (24) + + + + + + Tucker + Joseph + J + PhD + 0000-0001-5784-7403 (25) + 0000-0002-1825-0097 (26) + + The University of Manchester, Division of Population Health, Manchester, United Kingdom. (9) + https://ror.org/027m9bs27 (27) + https://ror.org/01ej9dk98 (28) + + + National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (10) + https://ror.org/00za53h95 (29) + https://ror.org/03vek6s52 (30) + + + + Mullens + Amy + A + MD + 0000-0002-9047-0481 (31) + 0000-0001-6450-5815 (32) + + Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (11) + 0000-0003-2804-1181 (33) + 0000-0002-0939-9842 (34) + + + Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (12) + 0000-0001-5784-7403 (35) + 0000-0002-1825-0097 (36) + + + + Indexed by automated curation. + Supported by NASA Space Biology grant 80NSSC22K0123. + + + + + 2020 + 07 + 12 + 14 + 30 + 00 + + + 2019 + 02 + 08 + 09 + 15 + 25 + + + ppublish + + 38123456 + 10.1038/s41586-024-07123-x + + + + supp_table_1.pdf + Crystallographic data collection and refinement statistics + + + GSE245678 + NCBI Gene Expression Omnibus + + + + Primary Literature + + Vogelstein B, Lane D, Levine AJ. Surfing the p53 network. Nature. 2000;408(6810):307-310. + + PMC10987654 + S0028-0836(24)00123-4 + + + + Cho Y, Gorina S, Jeffrey PD, Pavletich NP. Crystal structure of a p53 tumor suppressor-DNA complex. Science. 1994;265(5170):346-355. + + 11099028 + 10.1038/35042675 + + + + Structural Biology References + + Baker SJ, Fearon ER, Nigro JM, et al. Chromosome 17 deletions and p53 gene mutations in colorectal carcinomas. Science. 1989;244(4901):217-221. + + 7585406 + 10.1126/science.7878469 + + + + Momand J, Zambetti GP, Olson DC, George D, Levine AJ. The mdm-2 oncogene product forms a complex with the p53 protein. Cell. 1992;69(7):1237-1245. + + 2694717 + 10.1016/0092-8674(89)90984-7 + + + + Clinical &amp; Genetic Studies + + Kussie PH, Gorina S, Marechal V, et al. Structure of the MDM2 oncoprotein bound to the p53 tumor suppressor transactivation domain. Science. 1996;274(5289):948-953. + + 8183552 + 10.1126/science.8183552 + + + + Haupt Y, Maya R, Kazaz A, Oren M. Mdm2 promotes the rapid degradation of p53. Nature. 1997;387(6630):296-299. + + 9367981 + 10.1038/387296a0 + + + + + + + Historical Background + + Cohen GN, Barbe V, Flament D, et al. An integrated analysis of the genome of Pyrococcus abyssi. Mol Microbiol. 2003;47(6):1495-1512. + + 10430560 + 10.1093/genetics/152.4.1299 + + + + Erkel C, Kube M, Reinhardt R, Liesack W. Genome of rice cluster I archaea. Science. 2006;313(5785):370-372. + + 12622808 + 10.1046/j.1365-2958.2003.03381.x + + + + Methodological References + + Anderson IJ, Sieprawska-Lupa M, Lapidus A, et al. Complete genome sequence of Methanoculleus marisnigri JR1. Stand Genomic Sci. 2009;1(2):189-196. + + 16857943 + 10.1126/science.1127062 + + + + Gao J, Wang J. Re-annotation of two hyperthermophilic archaea Pyrococcus abyssi & P. furiosus. Curr Microbiol. 2012;64(2):118-129. + + 21304656 + 10.4056/sigs.32535 + + + + Supplementary Citations + + Morris C, Robinson L, Ashcroft D, et al. Primary care experiences of people with dementia. Br J Gen Pract. 2025;75(751):e102-e110. + + 22057919 + 10.1007/s00284-011-0035-x + + + + Tucker J, Mullens A, Morroni C, Ong J. The HIV response at an inflection point: resilience &amp; equity. Sex Health. 2026;23(1):SH25236. + + 31234567 + 10.1016/j.cell.2019.05.012 + + + + + + + + diff --git a/tests/data/xsd/examples/pubmed_article/maximal_expected.json b/tests/data/xsd/examples/pubmed_article/maximal_expected.json new file mode 100644 index 00000000..fdbdb6c3 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/maximal_expected.json @@ -0,0 +1,894 @@ +[ + { + "MedlineCitation": { + "_Owner": "NLM", + "_Status": "MEDLINE", + "_VersionID": "1", + "_VersionDate": "2024-05-29", + "_IndexingMethod": "Automated", + "PMID": { + "_Version": "1", + "#text": "38123456" + }, + "DateCompleted": { + "Year": "2024", + "Month": "05", + "Day": "29" + }, + "DateRevised": { + "Year": "2025", + "Month": "01", + "Day": "16" + }, + "Article": { + "_PubModel": "Print-Electronic", + "Journal": { + "ISSN": { + "_IssnType": "Electronic", + "#text": "1476-4687" + }, + "JournalIssue": { + "_CitedMedium": "Internet", + "Volume": "629", + "Issue": "8012", + "PubDate": { + "Year": "2023", + "Month": "09", + "Day": "15" + } + }, + "Title": "Nature", + "ISOAbbreviation": "Nature" + }, + "ArticleTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Structural basis of TP53 tetramerization and Ca2+ signaling" + }, + "Pagination": { + "StartPage": "410", + "EndPage": "418", + "MedlinePgn": "410-418" + }, + "ELocationID": [ + { + "_EIdType": "doi", + "_ValidYN": "Y", + "#text": "10.1038/s41586-024-07123-x" + }, + { + "_EIdType": "pii", + "_ValidYN": "N", + "#text": "S0028-0836(24)00123-4" + } + ], + "Abstract": { + "AbstractText": [ + { + "_Label": "BACKGROUND", + "_NlmCategory": "BACKGROUND", + "#text": "The tumor suppressor p53 regulates transcription in response to <5 Gy stress and H2O2." + }, + { + "_Label": "METHODS", + "_NlmCategory": "METHODS", + "#text": "We undertook 20 semi-structured interviews with participants aged >18 years studying Ca2+ signaling & DNA repair and underlined text." + }, + { + "_Label": "UNLABELLED", + "_NlmCategory": "UNASSIGNED" + } + ], + "CopyrightInformation": "Copyright (c) 2024 Springer Nature Limited." + }, + "AuthorList": { + "_CompleteYN": "Y", + "_Type": "authors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "LastName": "Morris", + "ForeName": "Charlotte", + "Initials": "C", + "Suffix": "Jr", + "Identifier": [ + { + "_Source": "ORCID", + "#text": "0000-0002-9047-0481" + }, + { + "_Source": "ISNI", + "#text": "0000-0001-6450-5815" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom.", + "Identifier": [ + { + "_Source": "VIAF", + "#text": "0000-0003-2804-1181" + }, + { + "_Source": "GND", + "#text": "0000-0002-0939-9842" + } + ] + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom.", + "Identifier": [ + { + "_Source": "RINGGOLD", + "#text": "0000-0001-5784-7403" + }, + { + "_Source": "GRID", + "#text": "0000-0002-1825-0097" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "International Cancer Genome Consortium" + }, + "Identifier": [ + { + "_Source": "ROR", + "#text": "https://ror.org/027m9bs27" + }, + { + "_Source": "ORCID (8)", + "#text": "https://ror.org/01ej9dk98" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom.", + "Identifier": [ + { + "_Source": "ISNI (9)", + "#text": "https://ror.org/00za53h95" + }, + { + "_Source": "VIAF (10)", + "#text": "https://ror.org/03vek6s52" + } + ] + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA.", + "Identifier": [ + { + "_Source": "GND (11)", + "#text": "0000-0002-9047-0481 (11)" + }, + { + "_Source": "RINGGOLD (12)", + "#text": "0000-0001-6450-5815 (12)" + } + ] + } + ] + } + ] + }, + "Language": [ + "eng", + "fre" + ], + "DataBankList": { + "_CompleteYN": "Y", + "DataBank": [ + { + "DataBankName": "PDB", + "AccessionNumberList": { + "AccessionNumber": [ + "1TUP", + "2OCJ" + ] + } + }, + { + "DataBankName": "GenBank", + "AccessionNumberList": { + "AccessionNumber": [ + "NM_000546", + "AF307851" + ] + } + } + ] + }, + "GrantList": { + "_CompleteYN": "Y", + "Grant": [ + { + "GrantID": "R01-CA043460", + "Acronym": "CA", + "Agency": "NCI NIH HHS", + "Country": "United States" + }, + { + "GrantID": "MR/T021975/1", + "Acronym": "MRC", + "Agency": "Medical Research Council", + "Country": "United Kingdom" + } + ] + }, + "PublicationTypeList": { + "PublicationType": [ + { + "_UI": "D016428", + "#text": "Journal Article" + }, + { + "_UI": "D013485", + "#text": "Research Support, Non-U.S. Gov't" + } + ] + }, + "VernacularTitle": "Bases structurales de la tetramerisation de TP53", + "ArticleDate": [ + { + "_DateType": "Electronic", + "Year": "2022", + "Month": "11", + "Day": "04" + }, + { + "_DateType": "Electronic", + "Year": "2021", + "Month": "03", + "Day": "21" + } + ] + }, + "MedlineJournalInfo": { + "Country": "Belgium", + "MedlineTA": "Nature", + "NlmUniqueID": "0410462", + "ISSNLinking": "0028-0836" + }, + "ChemicalList": { + "Chemical": [ + { + "RegistryNumber": "EC 2.7.11.1", + "NameOfSubstance": { + "_UI": "Sample_UI_1", + "#text": "Tumor Suppressor Protein p53" + } + }, + { + "RegistryNumber": "EC 6.3.2.19", + "NameOfSubstance": { + "_UI": "Sample_UI_2", + "#text": "Proto-Oncogene Proteins c-mdm2" + } + } + ] + }, + "SupplMeshList": { + "SupplMeshName": [ + { + "_Type": "Disease", + "_UI": "C536599", + "#text": "Li-Fraumeni Syndrome" + }, + { + "_Type": "Protocol", + "_UI": "C045812", + "#text": "Chromatin Immunoprecipitation" + } + ] + }, + "CitationSubset": [ + "IM", + "AIM" + ], + "CommentsCorrectionsList": { + "CommentsCorrections": [ + { + "_RefType": "Cites", + "RefSource": "Nature. 2000 Nov 16;408(6810):307-310.", + "PMID": { + "_Version": "2", + "#text": "11099028" + }, + "Note": "Foundational review of the p53 signaling network." + }, + { + "_RefType": "CommentIn", + "RefSource": "Science. 2024 Jun 7;384(6700):1020-1021.", + "PMID": { + "_Version": "3", + "#text": "38843210" + }, + "Note": "Perspective highlighting TP53 tetramerization dynamics." + } + ] + }, + "GeneSymbolList": { + "GeneSymbol": [ + "TP53", + "MDM2" + ] + }, + "MeshHeadingList": { + "MeshHeading": [ + { + "DescriptorName": { + "_MajorTopicYN": "Y", + "_AutoHM": "Y", + "_Type": "Geographic", + "_UI": "D016159", + "#text": "Tumor Suppressor Protein p53" + }, + "QualifierName": [ + { + "_AutoHM": "Y", + "_MajorTopicYN": "N", + "_UI": "Q000235", + "#text": "genetics" + }, + { + "_AutoHM": "Y", + "_MajorTopicYN": "Y", + "_UI": "Q000378", + "#text": "metabolism" + } + ] + }, + { + "DescriptorName": { + "_MajorTopicYN": "N", + "_AutoHM": "Y", + "_Type": "Geographic", + "_UI": "D004260", + "#text": "DNA Repair" + }, + "QualifierName": [ + { + "_AutoHM": "Y", + "_MajorTopicYN": "N", + "_UI": "Q000502", + "#text": "physiology" + }, + { + "_AutoHM": "Y", + "_MajorTopicYN": "Y", + "_UI": "Q000187", + "#text": "drug effects" + } + ] + } + ] + }, + "NumberOfReferences": "42", + "PersonalNameSubjectList": { + "PersonalNameSubject": [ + { + "LastName": "Kontopantelis", + "ForeName": "Evangelos", + "Initials": "E", + "Suffix": "III" + }, + { + "LastName": "Robinson", + "ForeName": "Louise", + "Initials": "L", + "Suffix": "Sr" + } + ] + }, + "OtherID": [ + { + "_Source": "NASA", + "#text": "NTRS-20240001234" + }, + { + "_Source": "PIP", + "#text": "PIP-987654" + } + ], + "OtherAbstract": [ + { + "_Type": "Publisher", + "_Language": "eng", + "AbstractText": [ + { + "_Label": "RESULTS", + "_NlmCategory": "RESULTS", + "#text": "Tetramerization affinity increased in vitro by 3.5-fold (p < 0.001) upon phosphorylation." + }, + { + "_Label": "CONCLUSIONS", + "_NlmCategory": "CONCLUSIONS", + "#text": "Clarifying TP53 & MDM2 kinetics improves targeted cancer therapy." + } + ], + "CopyrightInformation": "Copyright (c) 2024 The Authors. Published under CC-BY 4.0." + }, + { + "_Type": "AAMC", + "_Language": "fre", + "AbstractText": [ + { + "_Label": "OBJECTIVE", + "_NlmCategory": "OBJECTIVE", + "#text": "French summary: Le facteur de transcription p53 maintient la stabilite du genome." + }, + { + "_Label": "BACKGROUND (6)", + "_NlmCategory": "UNASSIGNED", + "#text": "Plain-language summary: How cells repair damaged DNA when exposed to &gt;2 Gy radiation." + } + ], + "CopyrightInformation": "Copyright (c) 2024 National Library of Medicine." + } + ], + "KeywordList": [ + { + "_Owner": "NOTNLM", + "Keyword": [ + { + "_MajorTopicYN": "Y", + "#text": "p53 signaling" + }, + { + "_MajorTopicYN": "N", + "#text": "tetramerization" + } + ] + }, + { + "_Owner": "NLM", + "Keyword": [ + { + "_MajorTopicYN": "Y", + "#text": "DNA damage response" + }, + { + "_MajorTopicYN": "N", + "#text": "apoptosis" + } + ] + } + ], + "CoiStatement": "The authors declare no competing financial or non-financial interests.", + "SpaceFlightMission": [ + "ISS Expedition 68", + "Artemis I BioSentinel" + ], + "InvestigatorList": [ + { + "_ID": "INV_1", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Ashcroft", + "ForeName": "Darren", + "Initials": "D", + "Suffix": "II", + "Identifier": [ + { + "_Source": "GRID (13)", + "#text": "0000-0003-2804-1181 (13)" + }, + { + "_Source": "ROR (14)", + "#text": "0000-0002-0939-9842 (14)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom.", + "Identifier": [ + { + "_Source": "ORCID (15)", + "#text": "0000-0001-5784-7403 (15)" + }, + { + "_Source": "ISNI (16)", + "#text": "0000-0002-1825-0097 (16)" + } + ] + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA.", + "Identifier": [ + { + "_Source": "VIAF (17)", + "#text": "https://ror.org/027m9bs27 (17)" + }, + { + "_Source": "GND (18)", + "#text": "https://ror.org/01ej9dk98 (18)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "LastName": "Blakeman", + "ForeName": "Thomas", + "Initials": "T", + "Suffix": "IV", + "Identifier": [ + { + "_Source": "RINGGOLD (19)", + "#text": "https://ror.org/00za53h95 (19)" + }, + { + "_Source": "GRID (20)", + "#text": "https://ror.org/03vek6s52 (20)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA.", + "Identifier": [ + { + "_Source": "ROR (21)", + "#text": "0000-0002-9047-0481 (21)" + }, + { + "_Source": "ORCID (22)", + "#text": "0000-0001-6450-5815 (22)" + } + ] + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA.", + "Identifier": [ + { + "_Source": "ISNI (23)", + "#text": "0000-0003-2804-1181 (23)" + }, + { + "_Source": "VIAF (24)", + "#text": "0000-0002-0939-9842 (24)" + } + ] + } + ] + } + ] + }, + { + "_ID": "INV_2", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Tucker", + "ForeName": "Joseph", + "Initials": "J", + "Suffix": "PhD", + "Identifier": [ + { + "_Source": "GND (25)", + "#text": "0000-0001-5784-7403 (25)" + }, + { + "_Source": "RINGGOLD (26)", + "#text": "0000-0002-1825-0097 (26)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom. (9)", + "Identifier": [ + { + "_Source": "GRID (27)", + "#text": "https://ror.org/027m9bs27 (27)" + }, + { + "_Source": "ROR (28)", + "#text": "https://ror.org/01ej9dk98 (28)" + } + ] + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (10)", + "Identifier": [ + { + "_Source": "ORCID (29)", + "#text": "https://ror.org/00za53h95 (29)" + }, + { + "_Source": "ISNI (30)", + "#text": "https://ror.org/03vek6s52 (30)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "LastName": "Mullens", + "ForeName": "Amy", + "Initials": "A", + "Suffix": "MD", + "Identifier": [ + { + "_Source": "VIAF (31)", + "#text": "0000-0002-9047-0481 (31)" + }, + { + "_Source": "GND (32)", + "#text": "0000-0001-6450-5815 (32)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (11)", + "Identifier": [ + { + "_Source": "RINGGOLD (33)", + "#text": "0000-0003-2804-1181 (33)" + }, + { + "_Source": "GRID (34)", + "#text": "0000-0002-0939-9842 (34)" + } + ] + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (12)", + "Identifier": [ + { + "_Source": "ROR (35)", + "#text": "0000-0001-5784-7403 (35)" + }, + { + "_Source": "ORCID (36)", + "#text": "0000-0002-1825-0097 (36)" + } + ] + } + ] + } + ] + } + ], + "GeneralNote": [ + { + "_Owner": "NLM", + "#text": "Indexed by automated curation." + }, + { + "_Owner": "NASA", + "#text": "Supported by NASA Space Biology grant 80NSSC22K0123." + } + ] + }, + "PubmedData": { + "History": { + "PubMedPubDate": [ + { + "_PubStatus": "received", + "Year": "2020", + "Month": "07", + "Day": "12", + "Hour": "14", + "Minute": "30", + "Second": "00" + }, + { + "_PubStatus": "revised", + "Year": "2019", + "Month": "02", + "Day": "08", + "Hour": "09", + "Minute": "15", + "Second": "25" + } + ] + }, + "PublicationStatus": "ppublish", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pubmed", + "#text": "38123456" + }, + { + "_IdType": "doi", + "#text": "10.1038/s41586-024-07123-x" + } + ] + }, + "ObjectList": { + "Object": [ + { + "_Type": "SupplementaryMaterial", + "Param": [ + { + "_Name": "id", + "#text": "supp_table_1.pdf" + }, + { + "_Name": "caption", + "#text": "Crystallographic data collection and refinement statistics" + } + ] + }, + { + "_Type": "Dataset", + "Param": [ + { + "_Name": "accession", + "#text": "GSE245678" + }, + { + "_Name": "repository", + "#text": "NCBI Gene Expression Omnibus" + } + ] + } + ] + }, + "ReferenceList": [ + { + "Title": "Primary Literature", + "Reference": [ + { + "Citation": "Vogelstein B, Lane D, Levine AJ. Surfing the p53 network. Nature. 2000;408(6810):307-310.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmc", + "#text": "PMC10987654" + }, + { + "_IdType": "pii", + "#text": "S0028-0836(24)00123-4" + } + ] + } + }, + { + "Citation": "Cho Y, Gorina S, Jeffrey PD, Pavletich NP. Crystal structure of a p53 tumor suppressor-DNA complex. Science. 1994;265(5170):346-355.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "mid", + "#text": "11099028" + }, + { + "_IdType": "pmcid", + "#text": "10.1038/35042675" + } + ] + } + } + ], + "ReferenceList": [ + { + "Title": "Structural Biology References", + "Reference": [ + { + "Citation": "Baker SJ, Fearon ER, Nigro JM, et al. Chromosome 17 deletions and p53 gene mutations in colorectal carcinomas. Science. 1989;244(4901):217-221.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmcpid", + "#text": "7585406" + }, + { + "_IdType": "pmpid", + "#text": "10.1126/science.7878469" + } + ] + } + }, + { + "Citation": "Momand J, Zambetti GP, Olson DC, George D, Levine AJ. The mdm-2 oncogene product forms a complex with the p53 protein. Cell. 1992;69(7):1237-1245.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "sici", + "#text": "2694717" + }, + { + "_IdType": "medline", + "#text": "10.1016/0092-8674(89)90984-7" + } + ] + } + } + ], + "ReferenceList": [ + "{\"Title\": \"Clinical &amp; Genetic Studies\", \"Reference\": [{\"Citation\": \"Kussie PH, Gorina S, Marechal V, et al. Structure of the MDM2 oncoprotein bound to the p53 tumor suppressor transactivation domain. Science. 1996;274(5289):948-953.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pmcbook\", \"#text\": \"8183552\"}, {\"_IdType\": \"bookaccession\", \"#text\": \"10.1126/science.8183552\"}]}}, {\"Citation\": \"Haupt Y, Maya R, Kazaz A, Oren M. Mdm2 promotes the rapid degradation of p53. Nature. 1997;387(6630):296-299.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pubmed\", \"#text\": \"9367981\"}, {\"_IdType\": \"doi\", \"#text\": \"10.1038/387296a0\"}]}}]}" + ] + } + ] + }, + { + "Title": "Historical Background", + "Reference": [ + { + "Citation": "Cohen GN, Barbe V, Flament D, et al. An integrated analysis of the genome of Pyrococcus abyssi. Mol Microbiol. 2003;47(6):1495-1512.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmc", + "#text": "10430560" + }, + { + "_IdType": "pii", + "#text": "10.1093/genetics/152.4.1299" + } + ] + } + }, + { + "Citation": "Erkel C, Kube M, Reinhardt R, Liesack W. Genome of rice cluster I archaea. Science. 2006;313(5785):370-372.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "mid", + "#text": "12622808" + }, + { + "_IdType": "pmcid", + "#text": "10.1046/j.1365-2958.2003.03381.x" + } + ] + } + } + ], + "ReferenceList": [ + { + "Title": "Methodological References", + "Reference": [ + { + "Citation": "Anderson IJ, Sieprawska-Lupa M, Lapidus A, et al. Complete genome sequence of Methanoculleus marisnigri JR1. Stand Genomic Sci. 2009;1(2):189-196.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmcpid", + "#text": "16857943" + }, + { + "_IdType": "pmpid", + "#text": "10.1126/science.1127062" + } + ] + } + }, + { + "Citation": "Gao J, Wang J. Re-annotation of two hyperthermophilic archaea Pyrococcus abyssi & P. furiosus. Curr Microbiol. 2012;64(2):118-129.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "sici", + "#text": "21304656" + }, + { + "_IdType": "medline", + "#text": "10.4056/sigs.32535" + } + ] + } + } + ], + "ReferenceList": [ + "{\"Title\": \"Supplementary Citations\", \"Reference\": [{\"Citation\": \"Morris C, Robinson L, Ashcroft D, et al. Primary care experiences of people with dementia. Br J Gen Pract. 2025;75(751):e102-e110.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pmcbook\", \"#text\": \"22057919\"}, {\"_IdType\": \"bookaccession\", \"#text\": \"10.1007/s00284-011-0035-x\"}]}}, {\"Citation\": \"Tucker J, Mullens A, Morroni C, Ong J. The HIV response at an inflection point: resilience &amp; equity. Sex Health. 2026;23(1):SH25236.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pubmed\", \"#text\": \"31234567\"}, {\"_IdType\": \"doi\", \"#text\": \"10.1016/j.cell.2019.05.012\"}]}}]}" + ] + } + ] + } + ] + } + } +] diff --git a/tests/data/xsd/examples/pubmed_article/minimal.xml b/tests/data/xsd/examples/pubmed_article/minimal.xml new file mode 100644 index 00000000..ef0c1aef --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/minimal.xml @@ -0,0 +1,26 @@ + + + + + 38123456 +
+ + + + Sample_MedlineDate_1 + + + + Structural basis of TP53 tetramerization and Ca2+ signaling + 10.1038/s41586-024-07123-x + eng + + Journal Article + +
+ + Nature + + + + diff --git a/tests/data/xsd/examples/pubmed_article/minimal_expected.json b/tests/data/xsd/examples/pubmed_article/minimal_expected.json new file mode 100644 index 00000000..e5b4d516 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/minimal_expected.json @@ -0,0 +1,45 @@ +[ + { + "MedlineCitation": { + "_Status": "MEDLINE", + "PMID": { + "_Version": "1", + "#text": "38123456" + }, + "Article": { + "_PubModel": "Print-Electronic", + "Journal": { + "JournalIssue": { + "_CitedMedium": "Internet", + "PubDate": { + "MedlineDate": "Sample_MedlineDate_1" + } + } + }, + "ArticleTitle": { + "#text": "Structural basis of TP53 tetramerization and Ca2+ signaling" + }, + "ELocationID": [ + { + "_EIdType": "doi", + "#text": "10.1038/s41586-024-07123-x" + } + ], + "Language": [ + "eng" + ], + "PublicationTypeList": { + "PublicationType": [ + { + "_UI": "D016428", + "#text": "Journal Article" + } + ] + } + }, + "MedlineJournalInfo": { + "MedlineTA": "Nature" + } + } + } +] diff --git a/tests/data/xsd/examples/pubmed_book_article/expected_schema.py b/tests/data/xsd/examples/pubmed_book_article/expected_schema.py new file mode 100644 index 00000000..47354c9c --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/expected_schema.py @@ -0,0 +1,1600 @@ +"""Expected PyArrow schema for pubmed_book_article (xml_tag='PubmedBookArticle').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "BookDocument", + pa.struct( + [ + pa.field( + "PMID", + pa.struct( + [ + pa.field("_Version", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "Book", + pa.struct( + [ + pa.field( + "Publisher", + pa.struct( + [ + pa.field( + "PublisherName", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "PublisherLocation", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "BookTitle", + pa.struct( + [ + pa.field( + "_book", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_part", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_sec", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=False, + ), + pa.field( + "PubDate", + pa.struct( + [ + pa.field( + "Year", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Month", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Day", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Season", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "MedlineDate", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "BeginningDate", + pa.struct( + [ + pa.field( + "Year", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Month", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Day", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Season", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + pa.field( + "EndingDate", + pa.struct( + [ + pa.field( + "Year", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Month", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Day", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Season", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + pa.field( + "AuthorList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_CompleteYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_Type", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Author", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_EqualContrib", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "LastName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "CollectiveName", + pa.struct( + [ + pa.field( + "_Investigators", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:IDREF" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "AffiliationInfo", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Affiliation", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "InvestigatorList", + pa.struct( + [ + pa.field("_ID", pa.string(), nullable=True, metadata={"xsd_type": "xs:ID"}), + pa.field( + "Investigator", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "LastName", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "AffiliationInfo", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Affiliation", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field("Volume", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("VolumeTitle", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("Edition", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "CollectionTitle", + pa.struct( + [ + pa.field( + "_book", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_part", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_sec", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + pa.field( + "Isbn", + pa.list_( + pa.field( + "item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ), + nullable=True, + ), + pa.field( + "ELocationID", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_EIdType", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("Medium", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "ReportNumber", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=False, + ), + pa.field( + "LocationLabel", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ArticleTitle", + pa.struct( + [ + pa.field("_book", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_part", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_sec", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field("VernacularTitle", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "Pagination", + pa.struct( + [ + pa.field("StartPage", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("EndPage", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("MedlinePgn", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "Language", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "AuthorList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_CompleteYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_Type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Author", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_EqualContrib", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "LastName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "CollectiveName", + pa.struct( + [ + pa.field( + "_Investigators", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:IDREF"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "AffiliationInfo", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Affiliation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "InvestigatorList", + pa.struct( + [ + pa.field("_ID", pa.string(), nullable=True, metadata={"xsd_type": "xs:ID"}), + pa.field( + "Investigator", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ValidYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "LastName", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ForeName", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Initials", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Suffix", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "AffiliationInfo", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Affiliation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Identifier", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Source", + pa.string(), + nullable=False, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "PublicationType", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_UI", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "Abstract", + pa.struct( + [ + pa.field( + "AbstractText", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Label", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_NlmCategory", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "CopyrightInformation", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "Sections", + pa.struct( + [ + pa.field( + "Section", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "LocationLabel", + pa.struct( + [ + pa.field( + "_Type", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "SectionTitle", + pa.struct( + [ + pa.field( + "_book", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_part", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_sec", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "Section", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "LocationLabel", + pa.struct( + [ + pa.field( + "_Type", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + pa.field( + "SectionTitle", + pa.struct( + [ + pa.field( + "_book", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "_part", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "_sec", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=False, + ), + pa.field( + "Section", + pa.list_( + pa.field( + "item", pa.string(), nullable=True + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "KeywordList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Owner", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Keyword", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_MajorTopicYN", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ContributionDate", + pa.struct( + [ + pa.field("Year", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Month", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("Day", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("Season", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "DateRevised", + pa.struct( + [ + pa.field("Year", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Month", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("Day", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "GrantList", + pa.struct( + [ + pa.field("_CompleteYN", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "Grant", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "GrantID", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Acronym", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Agency", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Country", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "ItemList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_ListType", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Item", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ReferenceList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Title", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "Reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Citation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ReferenceList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Title", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "Citation", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={ + "xsd_type": "xs:string" + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ReferenceList", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + pa.field( + "PubmedBookData", + pa.struct( + [ + pa.field( + "History", + pa.struct( + [ + pa.field( + "PubMedPubDate", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_PubStatus", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Year", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Month", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Day", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Hour", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Minute", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Second", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field("PublicationStatus", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "ArticleIdList", + pa.struct( + [ + pa.field( + "ArticleId", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_IdType", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "ObjectList", + pa.struct( + [ + pa.field( + "Object", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Param", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_Name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/examples/pubmed_book_article/maximal.xml b/tests/data/xsd/examples/pubmed_book_article/maximal.xml new file mode 100644 index 00000000..47126081 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/maximal.xml @@ -0,0 +1,482 @@ + + + + + 38123456 + + 38123456 + 10.1038/s41586-024-07123-x + + + + Cold Spring Harbor Laboratory Press + Cold Spring Harbor (NY) + + Molecular Biology of the Cell, 7th Edition + + 2024 + 05 + 29 + + + 2025 + 01 + 16 + + + 2023 + 09 + 15 + + + + Morris + Charlotte + C + Jr + 0000-0002-9047-0481 + 0000-0001-6450-5815 + + The University of Manchester, Division of Population Health, Manchester, United Kingdom. + 0000-0003-2804-1181 + 0000-0002-0939-9842 + + + National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. + 0000-0001-5784-7403 + 0000-0002-1825-0097 + + + + International Cancer Genome Consortium + https://ror.org/027m9bs27 + https://ror.org/01ej9dk98 + + Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. + https://ror.org/00za53h95 + https://ror.org/03vek6s52 + + + Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. + 0000-0002-9047-0481 (11) + 0000-0001-6450-5815 (12) + + + + + + UK Primary Care Dementia Research Group + 0000-0003-2804-1181 (13) + 0000-0002-0939-9842 (14) + + MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. + 0000-0001-5784-7403 (15) + 0000-0002-1825-0097 (16) + + + Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. + https://ror.org/027m9bs27 (17) + https://ror.org/01ej9dk98 (18) + + + + Kontopantelis + Evangelos + E + III + https://ror.org/00za53h95 (19) + https://ror.org/03vek6s52 (20) + + Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. + 0000-0002-9047-0481 (21) + 0000-0001-6450-5815 (22) + + + Department of Molecular Biology, Princeton University, Princeton, NJ, USA. + 0000-0003-2804-1181 (23) + 0000-0002-0939-9842 (24) + + + + + + Robinson + Louise + L + Sr + 0000-0001-5784-7403 (25) + 0000-0002-1825-0097 (26) + + The University of Manchester, Division of Population Health, Manchester, United Kingdom. (9) + https://ror.org/027m9bs27 (27) + https://ror.org/01ej9dk98 (28) + + + National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (10) + https://ror.org/00za53h95 (29) + https://ror.org/03vek6s52 (30) + + + + Ashcroft + Darren + D + II + 0000-0002-9047-0481 (31) + 0000-0001-6450-5815 (32) + + Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (11) + 0000-0003-2804-1181 (33) + 0000-0002-0939-9842 (34) + + + Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (12) + 0000-0001-5784-7403 (35) + 0000-0002-1825-0097 (36) + + + + 629 + Part I: Introduction to the Cell + 7th + NCBI Bookshelf Classics + 978-0-8153-4432-2 + 978-0-8153-4464-3 + 10.1038/s41586-024-07123-x + S0028-0836(24)00123-4 + Internet + NBK21054 + + Chapter 1 + Part I + Structural basis of TP53 tetramerization and Ca2+ signaling + Bases structurales de la tetramerisation de TP53 + + 410 + 418 + 410-418 + + eng + fre + + + Blakeman + Thomas + T + IV + https://ror.org/027m9bs27 (37) + https://ror.org/01ej9dk98 (38) + + MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. (13) + https://ror.org/00za53h95 (39) + https://ror.org/03vek6s52 (40) + + + Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. (14) + 0000-0002-9047-0481 (41) + 0000-0001-6450-5815 (42) + + + + Asia-Pacific HIV Prevention Network + 0000-0003-2804-1181 (43) + 0000-0002-0939-9842 (44) + + Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. (15) + 0000-0001-5784-7403 (45) + 0000-0002-1825-0097 (46) + + + Department of Molecular Biology, Princeton University, Princeton, NJ, USA. (16) + https://ror.org/027m9bs27 (47) + https://ror.org/01ej9dk98 (48) + + + + + + European TP53 Structural Biology Consortium + https://ror.org/00za53h95 (49) + https://ror.org/03vek6s52 (50) + + The University of Manchester, Division of Population Health, Manchester, United Kingdom. (17) + 0000-0002-9047-0481 (51) + 0000-0001-6450-5815 (52) + + + National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (18) + 0000-0003-2804-1181 (53) + 0000-0002-0939-9842 (54) + + + + Tucker + Joseph + J + PhD + 0000-0001-5784-7403 (55) + 0000-0002-1825-0097 (56) + + Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (19) + https://ror.org/027m9bs27 (57) + https://ror.org/01ej9dk98 (58) + + + Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (20) + https://ror.org/00za53h95 (59) + https://ror.org/03vek6s52 (60) + + + + + + Mullens + Amy + A + MD + 0000-0002-9047-0481 (61) + 0000-0001-6450-5815 (62) + + MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. (21) + 0000-0003-2804-1181 (63) + 0000-0002-0939-9842 (64) + + + Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. (22) + 0000-0001-5784-7403 (65) + 0000-0002-1825-0097 (66) + + + + Morroni + Chelsea + CM + FRS + https://ror.org/027m9bs27 (67) + https://ror.org/01ej9dk98 (68) + + Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. (23) + https://ror.org/00za53h95 (69) + https://ror.org/03vek6s52 (70) + + + Department of Molecular Biology, Princeton University, Princeton, NJ, USA. (24) + 0000-0002-9047-0481 (71) + 0000-0001-6450-5815 (72) + + + + Journal Article + Research Support, Non-U.S. Gov't + + The tumor suppressor p53 regulates transcription in response to <5 Gy stress and H2O2. + We undertook 20 semi-structured interviews with participants aged >18 years studying Ca2+ signaling & DNA repair and underlined text. + + Copyright (c) 2024 Springer Nature Limited. + + +
+ Chapter 1 + Chapter 1: Cells and Genomes & Evolution +
+ Section 1.1 + Section 1.1: The Universal Features of Cells on Earth +
+ Subsection 1.1.1 + Subsection 1.1.1: All Cells Store Their Hereditary Information in DNA +
+
+
+
+ Chapter 2 + Chapter 2: Cell Chemistry and Bioenergetics +
+ Section 2.1 + Section 2.1: The Chemical Components of a Cell & CO2 Fixation +
+ Subsection 2.1.1 + Subsection 2.1.1: Chemical Bonds and Ca2+ Ions +
+
+
+ + + p53 signaling + tetramerization + + + DNA damage response + apoptosis + + + 2022 + 11 + 04 + + + 2021 + 03 + 21 + + + + R01-CA043460 + CA + NCI NIH HHS + United States + + + MR/T021975/1 + MRC + Medical Research Council + United Kingdom + + + + G1/S DNA damage checkpoint + G2/M mitotic spindle checkpoint + + + Ataxia-telangiectasia mutated (ATM) kinase activation + CHK2-mediated phosphorylation of Ser-20 + + + Nature + + Vogelstein B, Lane D, Levine AJ. Surfing the p53 network. Nature. 2000;408(6810):307-310. + + PMC10987654 + S0028-0836(24)00123-4 + + + + Cho Y, Gorina S, Jeffrey PD, Pavletich NP. Crystal structure of a p53 tumor suppressor-DNA complex. Science. 1994;265(5170):346-355. + + 11099028 + 10.1038/35042675 + + + + Primary Literature + + Baker SJ, Fearon ER, Nigro JM, et al. Chromosome 17 deletions and p53 gene mutations in colorectal carcinomas. Science. 1989;244(4901):217-221. + + 7585406 + 10.1126/science.7878469 + + + + Momand J, Zambetti GP, Olson DC, George D, Levine AJ. The mdm-2 oncogene product forms a complex with the p53 protein. Cell. 1992;69(7):1237-1245. + + 2694717 + 10.1016/0092-8674(89)90984-7 + + + + Structural Biology References + + Kussie PH, Gorina S, Marechal V, et al. Structure of the MDM2 oncoprotein bound to the p53 tumor suppressor transactivation domain. Science. 1996;274(5289):948-953. + + 8183552 + 10.1126/science.8183552 + + + + Haupt Y, Maya R, Kazaz A, Oren M. Mdm2 promotes the rapid degradation of p53. Nature. 1997;387(6630):296-299. + + 9367981 + 10.1038/387296a0 + + + + + + + Clinical &amp; Genetic Studies + + Cohen GN, Barbe V, Flament D, et al. An integrated analysis of the genome of Pyrococcus abyssi. Mol Microbiol. 2003;47(6):1495-1512. + + 10430560 + 10.1093/genetics/152.4.1299 + + + + Erkel C, Kube M, Reinhardt R, Liesack W. Genome of rice cluster I archaea. Science. 2006;313(5785):370-372. + + 12622808 + 10.1046/j.1365-2958.2003.03381.x + + + + Historical Background + + Anderson IJ, Sieprawska-Lupa M, Lapidus A, et al. Complete genome sequence of Methanoculleus marisnigri JR1. Stand Genomic Sci. 2009;1(2):189-196. + + 16857943 + 10.1126/science.1127062 + + + + Gao J, Wang J. Re-annotation of two hyperthermophilic archaea Pyrococcus abyssi & P. furiosus. Curr Microbiol. 2012;64(2):118-129. + + 21304656 + 10.4056/sigs.32535 + + + + Methodological References + + Morris C, Robinson L, Ashcroft D, et al. Primary care experiences of people with dementia. Br J Gen Pract. 2025;75(751):e102-e110. + + 22057919 + 10.1007/s00284-011-0035-x + + + + Tucker J, Mullens A, Morroni C, Ong J. The HIV response at an inflection point: resilience &amp; equity. Sex Health. 2026;23(1):SH25236. + + 31234567 + 10.1016/j.cell.2019.05.012 + + + + + + + + + + 2020 + 07 + 12 + 14 + 30 + 00 + + + 2019 + 02 + 08 + 09 + 15 + 25 + + + ppublish + + 38123456 (27) + 10.1038/s41586-024-07123-x (28) + + + + supp_table_1.pdf + Crystallographic data collection and refinement statistics + + + GSE245678 + NCBI Gene Expression Omnibus + + + + + diff --git a/tests/data/xsd/examples/pubmed_book_article/maximal_expected.json b/tests/data/xsd/examples/pubmed_book_article/maximal_expected.json new file mode 100644 index 00000000..28ff4c61 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/maximal_expected.json @@ -0,0 +1,1069 @@ +[ + { + "BookDocument": { + "PMID": { + "_Version": "1", + "#text": "38123456" + }, + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pubmed", + "#text": "38123456" + }, + { + "_IdType": "doi", + "#text": "10.1038/s41586-024-07123-x" + } + ] + }, + "Book": { + "Publisher": { + "PublisherName": "Cold Spring Harbor Laboratory Press", + "PublisherLocation": "Cold Spring Harbor (NY)" + }, + "BookTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Molecular Biology of the Cell, 7th Edition" + }, + "PubDate": { + "Year": "2024", + "Month": "05", + "Day": "29" + }, + "BeginningDate": { + "Year": "2025", + "Month": "01", + "Day": "16" + }, + "EndingDate": { + "Year": "2023", + "Month": "09", + "Day": "15" + }, + "AuthorList": [ + { + "_CompleteYN": "Y", + "_Type": "authors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "LastName": "Morris", + "ForeName": "Charlotte", + "Initials": "C", + "Suffix": "Jr", + "Identifier": [ + { + "_Source": "ORCID", + "#text": "0000-0002-9047-0481" + }, + { + "_Source": "ISNI", + "#text": "0000-0001-6450-5815" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom.", + "Identifier": [ + { + "_Source": "VIAF", + "#text": "0000-0003-2804-1181" + }, + { + "_Source": "GND", + "#text": "0000-0002-0939-9842" + } + ] + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom.", + "Identifier": [ + { + "_Source": "RINGGOLD", + "#text": "0000-0001-5784-7403" + }, + { + "_Source": "GRID", + "#text": "0000-0002-1825-0097" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "International Cancer Genome Consortium" + }, + "Identifier": [ + { + "_Source": "ROR", + "#text": "https://ror.org/027m9bs27" + }, + { + "_Source": "ORCID (8)", + "#text": "https://ror.org/01ej9dk98" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom.", + "Identifier": [ + { + "_Source": "ISNI (9)", + "#text": "https://ror.org/00za53h95" + }, + { + "_Source": "VIAF (10)", + "#text": "https://ror.org/03vek6s52" + } + ] + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA.", + "Identifier": [ + { + "_Source": "GND (11)", + "#text": "0000-0002-9047-0481 (11)" + }, + { + "_Source": "RINGGOLD (12)", + "#text": "0000-0001-6450-5815 (12)" + } + ] + } + ] + } + ] + }, + { + "_CompleteYN": "N", + "_Type": "editors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "UK Primary Care Dementia Research Group" + }, + "Identifier": [ + { + "_Source": "GRID (13)", + "#text": "0000-0003-2804-1181 (13)" + }, + { + "_Source": "ROR (14)", + "#text": "0000-0002-0939-9842 (14)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom.", + "Identifier": [ + { + "_Source": "ORCID (15)", + "#text": "0000-0001-5784-7403 (15)" + }, + { + "_Source": "ISNI (16)", + "#text": "0000-0002-1825-0097 (16)" + } + ] + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA.", + "Identifier": [ + { + "_Source": "VIAF (17)", + "#text": "https://ror.org/027m9bs27 (17)" + }, + { + "_Source": "GND (18)", + "#text": "https://ror.org/01ej9dk98 (18)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "LastName": "Kontopantelis", + "ForeName": "Evangelos", + "Initials": "E", + "Suffix": "III", + "Identifier": [ + { + "_Source": "RINGGOLD (19)", + "#text": "https://ror.org/00za53h95 (19)" + }, + { + "_Source": "GRID (20)", + "#text": "https://ror.org/03vek6s52 (20)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA.", + "Identifier": [ + { + "_Source": "ROR (21)", + "#text": "0000-0002-9047-0481 (21)" + }, + { + "_Source": "ORCID (22)", + "#text": "0000-0001-6450-5815 (22)" + } + ] + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA.", + "Identifier": [ + { + "_Source": "ISNI (23)", + "#text": "0000-0003-2804-1181 (23)" + }, + { + "_Source": "VIAF (24)", + "#text": "0000-0002-0939-9842 (24)" + } + ] + } + ] + } + ] + } + ], + "InvestigatorList": { + "_ID": "INV_1", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Robinson", + "ForeName": "Louise", + "Initials": "L", + "Suffix": "Sr", + "Identifier": [ + { + "_Source": "GND (25)", + "#text": "0000-0001-5784-7403 (25)" + }, + { + "_Source": "RINGGOLD (26)", + "#text": "0000-0002-1825-0097 (26)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom. (9)", + "Identifier": [ + { + "_Source": "GRID (27)", + "#text": "https://ror.org/027m9bs27 (27)" + }, + { + "_Source": "ROR (28)", + "#text": "https://ror.org/01ej9dk98 (28)" + } + ] + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (10)", + "Identifier": [ + { + "_Source": "ORCID (29)", + "#text": "https://ror.org/00za53h95 (29)" + }, + { + "_Source": "ISNI (30)", + "#text": "https://ror.org/03vek6s52 (30)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "LastName": "Ashcroft", + "ForeName": "Darren", + "Initials": "D", + "Suffix": "II", + "Identifier": [ + { + "_Source": "VIAF (31)", + "#text": "0000-0002-9047-0481 (31)" + }, + { + "_Source": "GND (32)", + "#text": "0000-0001-6450-5815 (32)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (11)", + "Identifier": [ + { + "_Source": "RINGGOLD (33)", + "#text": "0000-0003-2804-1181 (33)" + }, + { + "_Source": "GRID (34)", + "#text": "0000-0002-0939-9842 (34)" + } + ] + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (12)", + "Identifier": [ + { + "_Source": "ROR (35)", + "#text": "0000-0001-5784-7403 (35)" + }, + { + "_Source": "ORCID (36)", + "#text": "0000-0002-1825-0097 (36)" + } + ] + } + ] + } + ] + }, + "Volume": "629", + "VolumeTitle": "Part I: Introduction to the Cell", + "Edition": "7th", + "CollectionTitle": { + "_book": "nbk21054", + "_part": "seriesA", + "_sec": "vol1", + "#text": "NCBI Bookshelf Classics" + }, + "Isbn": [ + "978-0-8153-4432-2", + "978-0-8153-4464-3" + ], + "ELocationID": [ + { + "_EIdType": "doi", + "_ValidYN": "Y", + "#text": "10.1038/s41586-024-07123-x" + }, + { + "_EIdType": "pii", + "_ValidYN": "N", + "#text": "S0028-0836(24)00123-4" + } + ], + "Medium": "Internet", + "ReportNumber": "NBK21054" + }, + "LocationLabel": [ + { + "_Type": "chapter", + "#text": "Chapter 1" + }, + { + "_Type": "section", + "#text": "Part I" + } + ], + "ArticleTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Structural basis of TP53 tetramerization and Ca2+ signaling" + }, + "VernacularTitle": "Bases structurales de la tetramerisation de TP53", + "Pagination": { + "StartPage": "410", + "EndPage": "418", + "MedlinePgn": "410-418" + }, + "Language": [ + "eng", + "fre" + ], + "AuthorList": [ + { + "_CompleteYN": "Y", + "_Type": "authors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "LastName": "Blakeman", + "ForeName": "Thomas", + "Initials": "T", + "Suffix": "IV", + "Identifier": [ + { + "_Source": "ISNI (37)", + "#text": "https://ror.org/027m9bs27 (37)" + }, + { + "_Source": "VIAF (38)", + "#text": "https://ror.org/01ej9dk98 (38)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. (13)", + "Identifier": [ + { + "_Source": "GND (39)", + "#text": "https://ror.org/00za53h95 (39)" + }, + { + "_Source": "RINGGOLD (40)", + "#text": "https://ror.org/03vek6s52 (40)" + } + ] + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. (14)", + "Identifier": [ + { + "_Source": "GRID (41)", + "#text": "0000-0002-9047-0481 (41)" + }, + { + "_Source": "ROR (42)", + "#text": "0000-0001-6450-5815 (42)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "Asia-Pacific HIV Prevention Network" + }, + "Identifier": [ + { + "_Source": "ORCID (43)", + "#text": "0000-0003-2804-1181 (43)" + }, + { + "_Source": "ISNI (44)", + "#text": "0000-0002-0939-9842 (44)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. (15)", + "Identifier": [ + { + "_Source": "VIAF (45)", + "#text": "0000-0001-5784-7403 (45)" + }, + { + "_Source": "GND (46)", + "#text": "0000-0002-1825-0097 (46)" + } + ] + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA. (16)", + "Identifier": [ + { + "_Source": "RINGGOLD (47)", + "#text": "https://ror.org/027m9bs27 (47)" + }, + { + "_Source": "GRID (48)", + "#text": "https://ror.org/01ej9dk98 (48)" + } + ] + } + ] + } + ] + }, + { + "_CompleteYN": "N", + "_Type": "editors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "European TP53 Structural Biology Consortium" + }, + "Identifier": [ + { + "_Source": "ROR (49)", + "#text": "https://ror.org/00za53h95 (49)" + }, + { + "_Source": "ORCID (50)", + "#text": "https://ror.org/03vek6s52 (50)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom. (17)", + "Identifier": [ + { + "_Source": "ISNI (51)", + "#text": "0000-0002-9047-0481 (51)" + }, + { + "_Source": "VIAF (52)", + "#text": "0000-0001-6450-5815 (52)" + } + ] + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (18)", + "Identifier": [ + { + "_Source": "GND (53)", + "#text": "0000-0003-2804-1181 (53)" + }, + { + "_Source": "RINGGOLD (54)", + "#text": "0000-0002-0939-9842 (54)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "LastName": "Tucker", + "ForeName": "Joseph", + "Initials": "J", + "Suffix": "PhD", + "Identifier": [ + { + "_Source": "GRID (55)", + "#text": "0000-0001-5784-7403 (55)" + }, + { + "_Source": "ROR (56)", + "#text": "0000-0002-1825-0097 (56)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (19)", + "Identifier": [ + { + "_Source": "ORCID (57)", + "#text": "https://ror.org/027m9bs27 (57)" + }, + { + "_Source": "ISNI (58)", + "#text": "https://ror.org/01ej9dk98 (58)" + } + ] + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (20)", + "Identifier": [ + { + "_Source": "VIAF (59)", + "#text": "https://ror.org/00za53h95 (59)" + }, + { + "_Source": "GND (60)", + "#text": "https://ror.org/03vek6s52 (60)" + } + ] + } + ] + } + ] + } + ], + "InvestigatorList": { + "_ID": "INV_2", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Mullens", + "ForeName": "Amy", + "Initials": "A", + "Suffix": "MD", + "Identifier": [ + { + "_Source": "RINGGOLD (61)", + "#text": "0000-0002-9047-0481 (61)" + }, + { + "_Source": "GRID (62)", + "#text": "0000-0001-6450-5815 (62)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. (21)", + "Identifier": [ + { + "_Source": "ROR (63)", + "#text": "0000-0003-2804-1181 (63)" + }, + { + "_Source": "ORCID (64)", + "#text": "0000-0002-0939-9842 (64)" + } + ] + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. (22)", + "Identifier": [ + { + "_Source": "ISNI (65)", + "#text": "0000-0001-5784-7403 (65)" + }, + { + "_Source": "VIAF (66)", + "#text": "0000-0002-1825-0097 (66)" + } + ] + } + ] + }, + { + "_ValidYN": "N", + "LastName": "Morroni", + "ForeName": "Chelsea", + "Initials": "CM", + "Suffix": "FRS", + "Identifier": [ + { + "_Source": "GND (67)", + "#text": "https://ror.org/027m9bs27 (67)" + }, + { + "_Source": "RINGGOLD (68)", + "#text": "https://ror.org/01ej9dk98 (68)" + } + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. (23)", + "Identifier": [ + { + "_Source": "GRID (69)", + "#text": "https://ror.org/00za53h95 (69)" + }, + { + "_Source": "ROR (70)", + "#text": "https://ror.org/03vek6s52 (70)" + } + ] + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA. (24)", + "Identifier": [ + { + "_Source": "ORCID (71)", + "#text": "0000-0002-9047-0481 (71)" + }, + { + "_Source": "ISNI (72)", + "#text": "0000-0001-6450-5815 (72)" + } + ] + } + ] + } + ] + }, + "PublicationType": [ + { + "_UI": "D016428", + "#text": "Journal Article" + }, + { + "_UI": "D013485", + "#text": "Research Support, Non-U.S. Gov't" + } + ], + "Abstract": { + "AbstractText": [ + { + "_Label": "BACKGROUND", + "_NlmCategory": "BACKGROUND", + "#text": "The tumor suppressor p53 regulates transcription in response to <5 Gy stress and H2O2." + }, + { + "_Label": "METHODS", + "_NlmCategory": "METHODS", + "#text": "We undertook 20 semi-structured interviews with participants aged >18 years studying Ca2+ signaling & DNA repair and underlined text." + }, + { + "_Label": "UNLABELLED", + "_NlmCategory": "UNASSIGNED" + } + ], + "CopyrightInformation": "Copyright (c) 2024 Springer Nature Limited." + }, + "Sections": { + "Section": [ + { + "LocationLabel": { + "_Type": "part", + "#text": "Chapter 1" + }, + "SectionTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1", + "#text": "Chapter 1: Cells and Genomes & Evolution" + }, + "Section": [ + { + "LocationLabel": { + "_Type": "appendix", + "#text": "Section 1.1" + }, + "SectionTitle": { + "_book": "nbk21055", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Section 1.1: The Universal Features of Cells on Earth" + }, + "Section": [ + "{\"LocationLabel\": {\"_Type\": \"table\", \"#text\": \"Subsection 1.1.1\"}, \"SectionTitle\": {\"_book\": \"nbk21056\", \"_part\": \"part1\", \"_sec\": \"sec1.1.1\", \"#text\": \"Subsection 1.1.1: All Cells Store Their Hereditary Information in DNA\"}}" + ] + } + ] + }, + { + "LocationLabel": { + "_Type": "figure", + "#text": "Chapter 2" + }, + "SectionTitle": { + "_book": "nbk21057", + "_part": "part2", + "_sec": "sec2", + "#text": "Chapter 2: Cell Chemistry and Bioenergetics" + }, + "Section": [ + { + "LocationLabel": { + "_Type": "box", + "#text": "Section 2.1" + }, + "SectionTitle": { + "_book": "nbk21058", + "_part": "part2", + "_sec": "sec2.1", + "#text": "Section 2.1: The Chemical Components of a Cell & CO2 Fixation" + }, + "Section": [ + "{\"LocationLabel\": {\"_Type\": \"chapter\", \"#text\": \"Subsection 2.1.1\"}, \"SectionTitle\": {\"_book\": \"nbk21059\", \"_part\": \"part2\", \"_sec\": \"sec2.1.1\", \"#text\": \"Subsection 2.1.1: Chemical Bonds and Ca2+ Ions\"}}" + ] + } + ] + } + ] + }, + "KeywordList": [ + { + "_Owner": "NOTNLM", + "Keyword": [ + { + "_MajorTopicYN": "Y", + "#text": "p53 signaling" + }, + { + "_MajorTopicYN": "N", + "#text": "tetramerization" + } + ] + }, + { + "_Owner": "NLM", + "Keyword": [ + { + "_MajorTopicYN": "Y", + "#text": "DNA damage response" + }, + { + "_MajorTopicYN": "N", + "#text": "apoptosis" + } + ] + } + ], + "ContributionDate": { + "Year": "2022", + "Month": "11", + "Day": "04" + }, + "DateRevised": { + "Year": "2021", + "Month": "03", + "Day": "21" + }, + "GrantList": { + "_CompleteYN": "Y", + "Grant": [ + { + "GrantID": "R01-CA043460", + "Acronym": "CA", + "Agency": "NCI NIH HHS", + "Country": "United States" + }, + { + "GrantID": "MR/T021975/1", + "Acronym": "MRC", + "Agency": "Medical Research Council", + "Country": "United Kingdom" + } + ] + }, + "ItemList": [ + { + "_ListType": "list", + "Item": [ + "G1/S DNA damage checkpoint", + "G2/M mitotic spindle checkpoint" + ] + }, + { + "_ListType": "ordered", + "Item": [ + "Ataxia-telangiectasia mutated (ATM) kinase activation", + "CHK2-mediated phosphorylation of Ser-20" + ] + } + ], + "ReferenceList": [ + { + "Title": "Nature", + "Reference": [ + { + "Citation": "Vogelstein B, Lane D, Levine AJ. Surfing the p53 network. Nature. 2000;408(6810):307-310.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmc", + "#text": "PMC10987654" + }, + { + "_IdType": "pii", + "#text": "S0028-0836(24)00123-4" + } + ] + } + }, + { + "Citation": "Cho Y, Gorina S, Jeffrey PD, Pavletich NP. Crystal structure of a p53 tumor suppressor-DNA complex. Science. 1994;265(5170):346-355.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "mid", + "#text": "11099028" + }, + { + "_IdType": "pmcid", + "#text": "10.1038/35042675" + } + ] + } + } + ], + "ReferenceList": [ + { + "Title": "Primary Literature", + "Reference": [ + { + "Citation": "Baker SJ, Fearon ER, Nigro JM, et al. Chromosome 17 deletions and p53 gene mutations in colorectal carcinomas. Science. 1989;244(4901):217-221.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmcpid", + "#text": "7585406" + }, + { + "_IdType": "pmpid", + "#text": "10.1126/science.7878469" + } + ] + } + }, + { + "Citation": "Momand J, Zambetti GP, Olson DC, George D, Levine AJ. The mdm-2 oncogene product forms a complex with the p53 protein. Cell. 1992;69(7):1237-1245.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "sici", + "#text": "2694717" + }, + { + "_IdType": "medline", + "#text": "10.1016/0092-8674(89)90984-7" + } + ] + } + } + ], + "ReferenceList": [ + "{\"Title\": \"Structural Biology References\", \"Reference\": [{\"Citation\": \"Kussie PH, Gorina S, Marechal V, et al. Structure of the MDM2 oncoprotein bound to the p53 tumor suppressor transactivation domain. Science. 1996;274(5289):948-953.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pmcbook\", \"#text\": \"8183552\"}, {\"_IdType\": \"bookaccession\", \"#text\": \"10.1126/science.8183552\"}]}}, {\"Citation\": \"Haupt Y, Maya R, Kazaz A, Oren M. Mdm2 promotes the rapid degradation of p53. Nature. 1997;387(6630):296-299.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pubmed\", \"#text\": \"9367981\"}, {\"_IdType\": \"doi\", \"#text\": \"10.1038/387296a0\"}]}}]}" + ] + } + ] + }, + { + "Title": "Clinical &amp; Genetic Studies", + "Reference": [ + { + "Citation": "Cohen GN, Barbe V, Flament D, et al. An integrated analysis of the genome of Pyrococcus abyssi. Mol Microbiol. 2003;47(6):1495-1512.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmc", + "#text": "10430560" + }, + { + "_IdType": "pii", + "#text": "10.1093/genetics/152.4.1299" + } + ] + } + }, + { + "Citation": "Erkel C, Kube M, Reinhardt R, Liesack W. Genome of rice cluster I archaea. Science. 2006;313(5785):370-372.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "mid", + "#text": "12622808" + }, + { + "_IdType": "pmcid", + "#text": "10.1046/j.1365-2958.2003.03381.x" + } + ] + } + } + ], + "ReferenceList": [ + { + "Title": "Historical Background", + "Reference": [ + { + "Citation": "Anderson IJ, Sieprawska-Lupa M, Lapidus A, et al. Complete genome sequence of Methanoculleus marisnigri JR1. Stand Genomic Sci. 2009;1(2):189-196.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmcpid", + "#text": "16857943" + }, + { + "_IdType": "pmpid", + "#text": "10.1126/science.1127062" + } + ] + } + }, + { + "Citation": "Gao J, Wang J. Re-annotation of two hyperthermophilic archaea Pyrococcus abyssi & P. furiosus. Curr Microbiol. 2012;64(2):118-129.", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "sici", + "#text": "21304656" + }, + { + "_IdType": "medline", + "#text": "10.4056/sigs.32535" + } + ] + } + } + ], + "ReferenceList": [ + "{\"Title\": \"Methodological References\", \"Reference\": [{\"Citation\": \"Morris C, Robinson L, Ashcroft D, et al. Primary care experiences of people with dementia. Br J Gen Pract. 2025;75(751):e102-e110.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pmcbook\", \"#text\": \"22057919\"}, {\"_IdType\": \"bookaccession\", \"#text\": \"10.1007/s00284-011-0035-x\"}]}}, {\"Citation\": \"Tucker J, Mullens A, Morroni C, Ong J. The HIV response at an inflection point: resilience &amp; equity. Sex Health. 2026;23(1):SH25236.\", \"ArticleIdList\": {\"ArticleId\": [{\"_IdType\": \"pubmed\", \"#text\": \"31234567\"}, {\"_IdType\": \"doi\", \"#text\": \"10.1016/j.cell.2019.05.012\"}]}}]}" + ] + } + ] + } + ] + }, + "PubmedBookData": { + "History": { + "PubMedPubDate": [ + { + "_PubStatus": "received", + "Year": "2020", + "Month": "07", + "Day": "12", + "Hour": "14", + "Minute": "30", + "Second": "00" + }, + { + "_PubStatus": "revised", + "Year": "2019", + "Month": "02", + "Day": "08", + "Hour": "09", + "Minute": "15", + "Second": "25" + } + ] + }, + "PublicationStatus": "ppublish", + "ArticleIdList": { + "ArticleId": [ + { + "_IdType": "pmc", + "#text": "38123456 (27)" + }, + { + "_IdType": "pii", + "#text": "10.1038/s41586-024-07123-x (28)" + } + ] + }, + "ObjectList": { + "Object": [ + { + "_Type": "SupplementaryMaterial", + "Param": [ + { + "_Name": "id", + "#text": "supp_table_1.pdf" + }, + { + "_Name": "caption", + "#text": "Crystallographic data collection and refinement statistics" + } + ] + }, + { + "_Type": "Dataset", + "Param": [ + { + "_Name": "accession", + "#text": "GSE245678" + }, + { + "_Name": "repository", + "#text": "NCBI Gene Expression Omnibus" + } + ] + } + ] + } + } + } +] diff --git a/tests/data/xsd/examples/pubmed_book_article/minimal.xml b/tests/data/xsd/examples/pubmed_book_article/minimal.xml new file mode 100644 index 00000000..e4688d09 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/minimal.xml @@ -0,0 +1,20 @@ + + + + + 38123456 + + 38123456 + + + + Cold Spring Harbor Laboratory Press + + Molecular Biology of the Cell, 7th Edition + + Sample_MedlineDate_1 + + + + + diff --git a/tests/data/xsd/examples/pubmed_book_article/minimal_expected.json b/tests/data/xsd/examples/pubmed_book_article/minimal_expected.json new file mode 100644 index 00000000..77d6ab90 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/minimal_expected.json @@ -0,0 +1,28 @@ +[ + { + "BookDocument": { + "PMID": { + "_Version": "1", + "#text": "38123456" + }, + "ArticleIdList": { + "ArticleId": [ + { + "#text": "38123456" + } + ] + }, + "Book": { + "Publisher": { + "PublisherName": "Cold Spring Harbor Laboratory Press" + }, + "BookTitle": { + "#text": "Molecular Biology of the Cell, 7th Edition" + }, + "PubDate": { + "MedlineDate": "Sample_MedlineDate_1" + } + } + } + } +] diff --git a/tests/data/xsd/examples/uniprot/expected_schema.py b/tests/data/xsd/examples/uniprot/expected_schema.py new file mode 100644 index 00000000..ceb7a289 --- /dev/null +++ b/tests/data/xsd/examples/uniprot/expected_schema.py @@ -0,0 +1,4346 @@ +"""Expected PyArrow schema for uniprot (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_dataset", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_created", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_modified", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field( + "accession", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "name", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "protein", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "domain", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": 'Describes names of "domains". Equivalent to the flat file DE-line Includes: section.' + }, + ), + pa.field( + "component", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes names of processed products. Equivalent to the flat file DE-line Contains: section." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes the names for the protein and parts thereof. Equivalent to the flat file DE-line." + }, + ), + pa.field( + "gene", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes different types of gene designations. Equivalent to the flat file GN-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a gene. Equivalent to the flat file GN-line."}, + ), + pa.field( + "organism", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes the names of the source organism. Equivalent to the flat file OS-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a cross-reference to the NCBI taxonomy database. Equivalent to the flat file OX-line." + }, + ), + pa.field( + "lineage", + pa.struct( + [ + pa.field( + "taxon", + pa.list_( + pa.field( + "item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the lineage of the source organism. Equivalent to the flat file OC-line." + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Describes the source organism."}, + ), + pa.field( + "organismHost", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes the names of the source organism. Equivalent to the flat file OS-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a cross-reference to the NCBI taxonomy database. Equivalent to the flat file OX-line." + }, + ), + pa.field( + "lineage", + pa.struct( + [ + pa.field( + "taxon", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the lineage of the source organism. Equivalent to the flat file OC-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the source organism."}, + ), + pa.field( + "geneLocation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Indicates whether the name of a plasmid is known or unknown."}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes non-nuclear gene locations (organelles and plasmids). Equivalent to the flat file OG-line." + }, + ), + pa.field( + "reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_key", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "citation", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of a citation.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_date", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}, + ), + pa.field( + "_name", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the name of an (online) journal or book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_volume", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the volume of a journal or book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_first", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the first page of an article.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_last", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the last page of an article.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_publisher", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the publisher of a book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_city", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the city where a book was published.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_db", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the database name of submissions.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_number", + pa.string(), + nullable=True, + metadata={"comment": "Describes a patent number.", "xsd_type": "xs:string"}, + ), + pa.field( + "_institute", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the institute where a thesis was made.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_country", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the country where a thesis was made.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "title", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the title of a citation. Equivalent to the flat file RT-line.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "editorList", + pa.struct( + [ + pa.field( + "consortium", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation when these are represented by a consortium. Equivalent to the flat file RG-line." + }, + ), + pa.field( + "person", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the editors of a book (only used for books). Equivalent to part of the flat file RL-line of books." + }, + ), + pa.field( + "authorList", + pa.struct( + [ + pa.field( + "consortium", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation when these are represented by a consortium. Equivalent to the flat file RG-line." + }, + ), + pa.field( + "person", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation. Equivalent to the flat file RA-line." + }, + ), + pa.field( + "locator", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the location (URL) of an online journal article. No flat file equivalent.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes cross-references to bibliography databases (MEDLINE, PubMed, AGRICOLA) or other online resources (DOI). Equivalent to the flat file RX-line." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes different types of citations. Equivalent to the flat file RX-, RG-, RA-, RT- and RL-lines." + }, + ), + pa.field( + "scope", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + metadata={ + "comment": "Describes the scope of a citation. Equivalent to the flat file RP-line." + }, + ), + pa.field( + "source", + pa.struct( + [ + pa.field( + "strain", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "plasmid", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "transposon", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "tissue", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the sequence according to the citation. Equivalent to the flat file RC-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a citation and a summary of its content. Equivalent to the flat file RN-, RP-, RC-, RX-, RG-, RA-, RT- and RL-lines." + }, + ), + pa.field( + "comment", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=True, + metadata={ + "comment": 'Describes the type of a general annotation. Equivalent to the flat file CC comment topics (except for "DATABASE" which is translated to "online information").', + "xsd_type": "xs:string", + }, + ), + pa.field( + "_locationType", + pa.string(), + nullable=True, + metadata={ + "comment": 'Describes the type of sequence location in \'RNA editing\' annotations. Common values are "Not_applicable" and "Undetermined".', + "xsd_type": "xs:string", + }, + ), + pa.field( + "_name", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes an optional name for an 'online information'.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_mass", + pa.float64(), + nullable=True, + metadata={ + "comment": "Describes the molecular mass in 'mass spectrometry' annotations.", + "xsd_type": "xs:float", + }, + ), + pa.field( + "_error", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the error of the mass measurement in 'mass spectrometry' annotations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_method", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the experimental method in 'mass spectrometry' annotations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "absorption", + pa.struct( + [ + pa.field( + "max", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "kinetics", + pa.struct( + [ + pa.field( + "KM", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "Vmax", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "phDependence", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "redoxPotential", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "temperatureDependence", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "reaction", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "text", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a chemical reaction."}, + ), + pa.field( + "physiologicalReaction", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_direction", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a physiological reaction."}, + ), + pa.field( + "cofactor", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a cofactor."}, + ), + pa.field( + "subcellularLocation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "location", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "topology", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "orientation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the subcellular location and optionally the topology and orientation of a molecule." + }, + ), + pa.field( + "conflict", + pa.struct( + [ + pa.field( + "_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_ref", + pa.string(), + nullable=True, + metadata={ + "comment": "Refers to the 'key' attribute of a 'reference' element.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_resource", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_version", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={"comment": "Used in 'sequence caution' annotations."}, + ), + pa.field( + "link", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_uri", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:anyURI"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Used in 'online information' annotations."}, + ), + pa.field( + "event", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the type of events that cause alternative products."}, + ), + pa.field( + "isoform", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "id", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_ref", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes isoforms in 'alternative products' annotations."}, + ), + pa.field( + "interactant", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_intactId", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("organismsDiffer", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + pa.field("experiments", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "disease", + pa.struct( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "acronym", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "description", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={"comment": "Used in 'disease' annotations."}, + ), + pa.field( + "location", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_sequence", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "begin", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "position", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Used in 'mass spectrometry' and 'sequence caution' annotations."}, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes different types of general annotations. Equivalent to the flat file CC-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"comment": "Describes the name of the database.", "xsd_type": "xs:string"}, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a database cross-reference. Equivalent to the flat file DR-line."}, + ), + pa.field( + "proteinExistence", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + metadata={ + "comment": "Describes the evidence for the protein's existence. Equivalent to the flat file PE-line." + }, + ), + pa.field( + "keyword", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "feature", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of a sequence annotation. Equivalent to the flat file FT feature keys, but using full terms instead of acronyms.", + "xsd_type": "xs:string", + }, + ), + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_description", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_ref", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "original", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the original sequence in annotations that describe natural or artifical sequence variations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "variation", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + metadata={ + "comment": "Describes the variant sequence in annotations that describe natural or artifical sequence variations." + }, + ), + pa.field( + "location", + pa.struct( + [ + pa.field( + "_sequence", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "begin", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "position", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Describes the sequence coordinates of the annotation."}, + ), + pa.field( + "ligand", + pa.struct( + [ + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + pa.field( + "label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "note", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the chemical entity that is bound in annotations that describe binding sites." + }, + ), + pa.field( + "ligandPart", + pa.struct( + [ + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + pa.field( + "label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "note", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the specific part of a molecule that is bound in annotations that describe binding sites." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes different types of sequence annotations. Equivalent to the flat file FT-line." + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of an evidence using the Evidence Code Ontology (http://www.obofoundry.org/cgi-bin/detail.cgi?id=evidence_code).", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_key", + pa.int64(), + nullable=False, + metadata={ + "comment": "A unique key to link annotations (via 'evidence' attributes) to evidences.", + "xsd_type": "xs:integer", + }, + ), + pa.field( + "source", + pa.struct( + [ + pa.field( + "_ref", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the data using a database cross-reference (or a 'ref' attribute when the source cannot be found in a public data source, such as PubMed, and is cited only within the UniProtKB entry)." + }, + ), + pa.field( + "importedFrom", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the evidence, when it is not assigned by UniProt, but imported from an external database." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the evidence for an annotation. No flat file equivalent."}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field("_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_mass", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_modified", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_precursor", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + pa.field("_fragment", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + ] +) diff --git a/tests/data/xsd/examples/uniprot/maximal.xml b/tests/data/xsd/examples/uniprot/maximal.xml new file mode 100644 index 00000000..4567e786 --- /dev/null +++ b/tests/data/xsd/examples/uniprot/maximal.xml @@ -0,0 +1,693 @@ + + + + P04637 + Q9V2L2 + P53_HUMAN + 1A1D_PYRAB + + + Cellular tumor antigen p53 + p53 + TP53-alpha + 2.7.11.1 + 3.5.99.7 + + + Antigen NY-CO-13 + NY-CO-13 + Pp53 + 3.1.3.16 + 3.1.3.48 + + + Phosphoprotein p53 + TRP53 + p53-iso1 + 2.7.10.2 + 2.7.12.1 + + + Cellular phosphoprotein p53 + 2.7.11.22 + 2.7.11.24 + + + Mutant-type p53 antigen + 3.2.2.6 + 3.2.2.20 + + Hom s p53 + Recombinant human p53 (rHu-p53) + CD-p53a + CD-p53b + p53-alfa + p53-beta + + + DNA-binding core domain + p53-DBD + DBD-core + 3.2.2.- + 3.1.3.16 + + + Tumor suppressor p53 + CD-A1 + CD-A2 + 3.4.19.12 + 3.4.22.68 + + + Transformation-related protein 53 + CD-B1 + CD-B2 + 2.4.2.30 + 2.1.1.43 + + + Zinc-coordinated DNA binding region + 3.5.1.98 + 3.5.1.104 + + + Conserved box II-V domain + 6.3.2.25 + 6.3.2.27 + + Hom s p53-DBD + Engineered p53 core domain + CD-DBD1 + CD-DBD2 + p53-gamma + p53-delta + + + + Tetramerization domain + p53-TD + TD-domain + 2.3.1.48 + 3.4.21.- + + + Core domain isoform A + OR-1a + OR-1b + 1.1.1.1 + 1.2.1.12 + + + Core domain isoform B + OR-2a + OR-2b + 3.1.3.16 (11) + 3.1.3.48 (12) + + + C-terminal tetramerization helix + 2.7.1.37 + 2.7.4.3 + + + Nuclear export signal region + 2.7.11.22 (11) + 2.7.11.24 (12) + + Hom s p53-TD + Stabilized p53 tetramer domain + CD-TD1 + CD-TD2 + p53-epsilon + p53-zeta + + + + Cleaved p53 N-terminal peptide + p53-N + N-pep + 2.7.7.7 + 3.6.4.12 + + + Oligomerization region 1 + TA-1a + TA-1b + 2.7.10.2 (13) + 2.7.12.1 (14) + + + Oligomerization region 2 + TA-2a + TA-2b + 3.4.19.12 (15) + 3.4.22.68 (16) + + + N-terminal TAD1 peptide + 3.2.2.6 (13) + 3.2.2.20 (14) + + + N-terminal TAD2 peptide + 3.5.1.98 (15) + 3.5.1.104 (16) + + Hom s p53-N + Synthetic TAD peptide + CD-TAD1 + CD-TAD2 + p53-eta + p53-theta + + + + Cleaved p53 C-terminal regulatory chain + p53-C + C-chain + 1.14.11.- + 6.3.2.19 + + + Transactivation subunit alpha + CT-1a + CT-1b + 2.4.2.30 (17) + 2.1.1.43 (18) + + + Transactivation subunit beta + CT-2a + CT-2b + 1.1.1.1 (19) + 1.2.1.12 (20) + + + Basic regulatory region + 6.3.2.25 (17) + 6.3.2.27 (18) + + + Apoptosis cofactor peptide + 2.7.1.37 (19) + 2.7.4.3 (20) + + Hom s p53-C + Fusion C-terminal peptide + CD-REG1 + CD-REG2 + p53-iota + p53-kappa + + + + TP53 + P53 + + + PYRAB00630 + PAB2303 + + + Homo sapiens + Human + + Isoform 1 + + + + + Isoform 2 + + + + + Eukaryota + Metazoa + + + + Pyrococcus abyssi (strain GE5 / Orsay) + Pyrococcus abyssi + + Isoform 3 + + + + + Chain A + + + + + Archaea + Thermococci + + + + Methanocella arvoryzae (strain DSM 22066 / NBRC 105507 / MRE50) + Rice cluster I archaeon + + Chain B + + + + + Monomer + + + + + Methanomicrobia + Methanocellales + + + + 17p13.1 + chr17:7668402-7687550 + + + pBR322 + pMRE50 + + + + An integrated analysis of the genome of the hyperthermophilic archaeon Pyrococcus abyssi. + + + + + + + + + 10.1046/j.1365-2958.2003.03381.x + + Homotetramer + + + + + Core domain + + + + + NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA] + X-RAY CRYSTALLOGRAPHY (2.2 ANGSTROMS) OF 102-292 + + GE5 / Orsay + pAB1 + Tn10 + Liver + + + + + Genome of rice cluster I archaea -- the key methane producers in the rice rhizosphere. + + + + + + + + + 10.1126/science.1127062 + + Precursor + + + + + Mature peptide + + + + + GENOME REANNOTATION + FUNCTION AND CATALYTIC ACTIVITY + + DSM 22066 / NBRC 105507 / MRE50 + pMRE50 + Tn5 + Colon epithelium + + + + + NUCLEOTIDE SEQUENCE [GENOMIC DNA] + + + Isoform 1 (11) + + 280 + Absorbs UV light with a primary peak at 280 nm. + Secondary shoulder observed at 292 nm in native buffer. + + + 12.5 uM for ATP + 48.0 uM for 1-aminocyclopropane-1-carboxylate + 450 nmol/min/mg enzyme + 1280 nmol/min/mg in presence of PLP cofactor + Follows classical Michaelis-Menten kinetics at pH 7.4. + Shows positive cooperativity upon tetramerization. + + + Optimum pH is 7.4-7.8 in phosphate buffer. + Rapidly loses DNA-binding activity below pH 6.0. + + + Midpoint potential E0 is -240 mV at pH 7.0. + Oxidation of Cys-277 abolishes sequence-specific binding. + + + Optimum temperature is 37 degrees Celsius. + Unfolds irreversibly above 44 degrees Celsius (Tm = 42.5 C). + + + + + + + + + Biophysical measurements performed on purified recombinant core domain. + Exhibits high-affinity sequence-specific DNA binding. + + + Isoform 2 (12) + + 1-aminocyclopropane-1-carboxylate + H2O = 2-oxobutanoate + NH4(+) + + Isoform 3 (13) + + + + + Chain A (14) + + + + + + + Chain B (15) + + + + + + + Monomer (16) + + + + + + + + + + + + Catalyzes the ring-opening deamination of ACC. + Activity is strictly dependent on divalent cations. + + + Homotetramer (17) + + pyridoxal 5'-phosphate + + Core domain (18) + + + + + + Zn(2+) + + Precursor (19) + + + + + + + + + + + + Zinc coordination stabilizes loop-sheet-helix motif. + Depletion of zinc induces molten-globule conformation. + + + Mature peptide (20) + + Nucleus + Nucleolus + Peripheral membrane protein + Single-pass membrane protein + Cytoplasmic side + Extracellular side + + + Cytoplasm + Mitochondrion + Multi-pass membrane protein + Lipid-anchor + Matrix side + Lumenal side + + + + + + + + + Shuttles between nucleus and cytoplasm in a CRM1-dependent manner. + Accumulates in the nucleus following ionizing radiation. + + + Isoform 1 (21) + + + + + + + + + + + Sequence differs from genomic reference at codon 112. + Verified by mass spectrometry of tryptic peptides. + + + Isoform 2 (22) + + + + + + + + + + Full structural and clinical annotations available online. + Cross-referenced with PDBe macromolecular database. + + + Isoform 3 (23) + + + + P04637-1 + P04637-alpha + Alpha + Canonical + + Chosen as the canonical sequence. + Expressed ubiquitously in normal tissues. + + + P04637-2 + P04637-beta + Beta + Alternative + + Produced by alternative splicing at intron 9. + Lacks the C-terminal oligomerization domain. + + + + + + + + + Contains 12 characterized splice isoforms. + Alternative promoter usage produces delta133 isoforms. + + + Chain A (24) + + P04637 + + + Chain B (25) + + + + + + Q00987 + + + Monomer (26) + + + + + false + 14 + + + + + + + + Forms a tight complex with E3 ubiquitin-protein ligase MDM2. + Phosphorylation at Ser-15 disrupts MDM2 binding. + + + Homotetramer (27) + + Li-Fraumeni syndrome + LFS1 + An autosomal dominant familial cancer syndrome caused by germline TP53 mutations. + + Core domain (28) + + + + + + + + + + + + Mutations in the DNA-binding domain account for 80% of LFS cases. + Associated with early-onset sarcomas and breast carcinomas. + + + + Precursor (29) + + + + + Mature peptide (30) + + + + + 3D-structure + DNA damage + + R + H + Q + + + + + + Mg(2+) + + Isoform 1 (31) + + + + + Coordinated by Cys-176, His-179, Cys-238, and Cys-242. + + + catalytic zinc site + + Isoform 2 (32) + + + + + Essential for structural integrity of L2/L3 loops. + + + + S + A + E + + + + + S-adenosyl-L-methionine + + Isoform 3 (33) + + + + + Covalently linked via Schiff base to Lys-54. + + + phosphate binding pocket + + Chain A (34) + + + + + Stabilized by hydrogen bonding with Gly-188 and Thr-190. + + + + + + Chain B (35) + + + + + + + Monomer (36) + + + + + + + + + Homotetramer (37) + + + + + + + Core domain (38) + + + + + + MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP + + diff --git a/tests/data/xsd/examples/uniprot/maximal_expected.json b/tests/data/xsd/examples/uniprot/maximal_expected.json new file mode 100644 index 00000000..4a61b48c --- /dev/null +++ b/tests/data/xsd/examples/uniprot/maximal_expected.json @@ -0,0 +1,3525 @@ +[ + { + "_dataset": "Swiss-Prot", + "_created": "1986-08-13", + "_modified": "2024-05-29", + "_version": 285, + "accession": [ + "P04637", + "Q9V2L2" + ], + "name": [ + "P53_HUMAN", + "1A1D_PYRAB" + ], + "protein": { + "recommendedName": { + "fullName": { + "_evidence": [ + 1, + 2 + ], + "#text": "Cellular tumor antigen p53" + }, + "shortName": [ + { + "_evidence": [ + 1, + 3 + ], + "#text": "p53" + }, + { + "_evidence": [ + 2, + 3 + ], + "#text": "TP53-alpha" + } + ], + "ecNumber": [ + { + "_evidence": [ + 1, + 4 + ], + "#text": "2.7.11.1" + }, + { + "_evidence": [ + 2, + 4 + ], + "#text": "3.5.99.7" + } + ] + }, + "alternativeName": [ + { + "fullName": { + "_evidence": [ + 3, + 4 + ], + "#text": "Antigen NY-CO-13" + }, + "shortName": [ + { + "_evidence": [ + 1, + 2, + 3 + ], + "#text": "NY-CO-13" + }, + { + "_evidence": [ + 1, + 2, + 4 + ], + "#text": "Pp53" + } + ], + "ecNumber": [ + { + "_evidence": [ + 2, + 3, + 4 + ], + "#text": "3.1.3.16" + }, + { + "_evidence": [ + 1, + 5 + ], + "#text": "3.1.3.48" + } + ] + }, + { + "fullName": { + "_evidence": [ + 2, + 5 + ], + "#text": "Phosphoprotein p53" + }, + "shortName": [ + { + "_evidence": [ + 3, + 5 + ], + "#text": "TRP53" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "p53-iso1" + } + ], + "ecNumber": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "2.7.10.2" + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "2.7.12.1" + } + ] + } + ], + "submittedName": [ + { + "fullName": { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Cellular phosphoprotein p53" + }, + "ecNumber": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "2.7.11.22" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "2.7.11.24" + } + ] + }, + { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Mutant-type p53 antigen" + }, + "ecNumber": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "3.2.2.6" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "3.2.2.20" + } + ] + } + ], + "allergenName": { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Hom s p53" + }, + "biotechName": { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Recombinant human p53 (rHu-p53)" + }, + "cdAntigenName": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "CD-p53a" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CD-p53b" + } + ], + "innName": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "p53-alfa" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "p53-beta" + } + ], + "domain": [ + { + "recommendedName": { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "DNA-binding core domain" + }, + "shortName": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "p53-DBD" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "DBD-core" + } + ], + "ecNumber": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "3.2.2.-" + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "3.1.3.16" + } + ] + }, + "alternativeName": [ + { + "fullName": { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Tumor suppressor p53" + }, + "shortName": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CD-A1" + }, + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "CD-A2" + } + ], + "ecNumber": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "3.4.19.12" + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "3.4.22.68" + } + ] + }, + { + "fullName": { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Transformation-related protein 53" + }, + "shortName": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "CD-B1" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "CD-B2" + } + ], + "ecNumber": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "2.4.2.30" + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "2.1.1.43" + } + ] + } + ], + "submittedName": [ + { + "fullName": { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Zinc-coordinated DNA binding region" + }, + "ecNumber": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "3.5.1.98" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "3.5.1.104" + } + ] + }, + { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Conserved box II-V domain" + }, + "ecNumber": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "6.3.2.25" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "6.3.2.27" + } + ] + } + ], + "allergenName": { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Hom s p53-DBD" + }, + "biotechName": { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Engineered p53 core domain" + }, + "cdAntigenName": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "CD-DBD1" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CD-DBD2" + } + ], + "innName": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "p53-gamma" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "p53-delta" + } + ] + }, + { + "recommendedName": { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Tetramerization domain" + }, + "shortName": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "p53-TD" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "TD-domain" + } + ], + "ecNumber": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "2.3.1.48" + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "3.4.21.-" + } + ] + }, + "alternativeName": [ + { + "fullName": { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Core domain isoform A" + }, + "shortName": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "OR-1a" + }, + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "OR-1b" + } + ], + "ecNumber": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "1.1.1.1" + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "1.2.1.12" + } + ] + }, + { + "fullName": { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Core domain isoform B" + }, + "shortName": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "OR-2a" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "OR-2b" + } + ], + "ecNumber": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "3.1.3.16 (11)" + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "3.1.3.48 (12)" + } + ] + } + ], + "submittedName": [ + { + "fullName": { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "C-terminal tetramerization helix" + }, + "ecNumber": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "2.7.1.37" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "2.7.4.3" + } + ] + }, + { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Nuclear export signal region" + }, + "ecNumber": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "2.7.11.22 (11)" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "2.7.11.24 (12)" + } + ] + } + ], + "allergenName": { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Hom s p53-TD" + }, + "biotechName": { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Stabilized p53 tetramer domain" + }, + "cdAntigenName": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "CD-TD1" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CD-TD2" + } + ], + "innName": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "p53-epsilon" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "p53-zeta" + } + ] + } + ], + "component": [ + { + "recommendedName": { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Cleaved p53 N-terminal peptide" + }, + "shortName": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "p53-N" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "N-pep" + } + ], + "ecNumber": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "2.7.7.7" + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "3.6.4.12" + } + ] + }, + "alternativeName": [ + { + "fullName": { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Oligomerization region 1" + }, + "shortName": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "TA-1a" + }, + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "TA-1b" + } + ], + "ecNumber": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "2.7.10.2 (13)" + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "2.7.12.1 (14)" + } + ] + }, + { + "fullName": { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Oligomerization region 2" + }, + "shortName": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "TA-2a" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "TA-2b" + } + ], + "ecNumber": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "3.4.19.12 (15)" + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "3.4.22.68 (16)" + } + ] + } + ], + "submittedName": [ + { + "fullName": { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "N-terminal TAD1 peptide" + }, + "ecNumber": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "3.2.2.6 (13)" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "3.2.2.20 (14)" + } + ] + }, + { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "N-terminal TAD2 peptide" + }, + "ecNumber": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "3.5.1.98 (15)" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "3.5.1.104 (16)" + } + ] + } + ], + "allergenName": { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Hom s p53-N" + }, + "biotechName": { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Synthetic TAD peptide" + }, + "cdAntigenName": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "CD-TAD1" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CD-TAD2" + } + ], + "innName": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "p53-eta" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "p53-theta" + } + ] + }, + { + "recommendedName": { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Cleaved p53 C-terminal regulatory chain" + }, + "shortName": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "p53-C" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "C-chain" + } + ], + "ecNumber": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "1.14.11.-" + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "6.3.2.19" + } + ] + }, + "alternativeName": [ + { + "fullName": { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Transactivation subunit alpha" + }, + "shortName": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CT-1a" + }, + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "CT-1b" + } + ], + "ecNumber": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "2.4.2.30 (17)" + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "2.1.1.43 (18)" + } + ] + }, + { + "fullName": { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Transactivation subunit beta" + }, + "shortName": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "CT-2a" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "CT-2b" + } + ], + "ecNumber": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "1.1.1.1 (19)" + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "1.2.1.12 (20)" + } + ] + } + ], + "submittedName": [ + { + "fullName": { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Basic regulatory region" + }, + "ecNumber": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "6.3.2.25 (17)" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "6.3.2.27 (18)" + } + ] + }, + { + "fullName": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Apoptosis cofactor peptide" + }, + "ecNumber": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "2.7.1.37 (19)" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "2.7.4.3 (20)" + } + ] + } + ], + "allergenName": { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Hom s p53-C" + }, + "biotechName": { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Fusion C-terminal peptide" + }, + "cdAntigenName": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "CD-REG1" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "CD-REG2" + } + ], + "innName": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "p53-iota" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "p53-kappa" + } + ] + } + ] + }, + "gene": [ + { + "name": [ + { + "_evidence": [ + 1, + 3, + 6 + ], + "_type": "primary", + "#text": "TP53" + }, + { + "_evidence": [ + 2, + 4, + 7 + ], + "_type": "synonym", + "#text": "P53" + } + ] + }, + { + "name": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "_type": "ordered locus", + "#text": "PYRAB00630" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "_type": "ORF", + "#text": "PAB2303" + } + ] + } + ], + "organism": { + "_evidence": [ + 1 + ], + "name": [ + { + "_type": "synonym", + "#text": "Homo sapiens" + }, + { + "_type": "synonym", + "#text": "Human" + } + ], + "dbReference": [ + { + "_type": "UniProtKB ID", + "_id": "P53_HUMAN", + "_evidence": [ + 5, + 7, + 1 + ], + "molecule": { + "_id": "P04637-1", + "#text": "Isoform 1" + }, + "property": [ + { + "_type": "member count", + "_value": "3" + }, + { + "_type": "common taxon", + "_value": "Homo sapiens" + } + ] + }, + { + "_type": "UniParc ID", + "_id": "UPI000002ED67", + "_evidence": [ + 6, + 8, + 2 + ], + "molecule": { + "_id": "P04637-2", + "#text": "Isoform 2" + }, + "property": [ + { + "_type": "common taxon ID", + "_value": "9606" + }, + { + "_type": "GO Molecular Function", + "_value": "GO:0003677" + } + ] + } + ], + "lineage": { + "taxon": [ + "Eukaryota", + "Metazoa" + ] + } + }, + "organismHost": [ + { + "_evidence": [ + 1, + 2 + ], + "name": [ + { + "_type": "synonym", + "#text": "Pyrococcus abyssi (strain GE5 / Orsay)" + }, + { + "_type": "synonym", + "#text": "Pyrococcus abyssi" + } + ], + "dbReference": [ + { + "_type": "RefSeq", + "_id": "NP_000537.3", + "_evidence": [ + 7, + 9, + 3 + ], + "molecule": { + "_id": "P04637-3", + "#text": "Isoform 3" + }, + "property": [ + { + "_type": "UniProtKB accession", + "_value": "P04637" + }, + { + "_type": "UniParc ID", + "_value": "UPI000002ED67" + } + ] + }, + { + "_type": "NCBI Taxonomy", + "_id": "9606", + "_evidence": [ + 8, + 1, + 4 + ], + "molecule": { + "_id": "Q9V2L2-1", + "#text": "Chain A" + }, + "property": [ + { + "_type": "UniRef100 ID", + "_value": "UniRef100_P04637" + }, + { + "_type": "UniRef90 ID", + "_value": "UniRef90_P04637" + } + ] + } + ], + "lineage": { + "taxon": [ + "Archaea", + "Thermococci" + ] + } + }, + { + "_evidence": [ + 2, + 3 + ], + "name": [ + { + "_type": "synonym", + "#text": "Methanocella arvoryzae (strain DSM 22066 / NBRC 105507 / MRE50)" + }, + { + "_type": "synonym", + "#text": "Rice cluster I archaeon" + } + ], + "dbReference": [ + { + "_type": "Proteomes", + "_id": "UP000005640", + "_evidence": [ + 9, + 2, + 5 + ], + "molecule": { + "_id": "Q0W5C8-1", + "#text": "Chain B" + }, + "property": [ + { + "_type": "protein name", + "_value": "Cellular tumor antigen p53" + }, + { + "_type": "source organism", + "_value": "Homo sapiens (Human)" + } + ] + }, + { + "_type": "EMBL", + "_id": "X02469", + "_evidence": [ + 1, + 3, + 6 + ], + "molecule": { + "_id": "A3CTY6-1", + "#text": "Monomer" + }, + "property": [ + { + "_type": "NCBI taxonomy", + "_value": "9606" + }, + { + "_type": "length", + "_value": "393" + } + ] + } + ], + "lineage": { + "taxon": [ + "Methanomicrobia", + "Methanocellales" + ] + } + } + ], + "geneLocation": [ + { + "_type": "mitochondrion", + "_evidence": [ + 2, + 4, + 7 + ], + "name": [ + { + "_status": "known", + "#text": "17p13.1" + }, + { + "_status": "known", + "#text": "chr17:7668402-7687550" + } + ] + }, + { + "_type": "plasmid", + "_evidence": [ + 3, + 5, + 8 + ], + "name": [ + { + "_status": "known", + "#text": "pBR322" + }, + { + "_status": "known", + "#text": "pMRE50" + } + ] + } + ], + "reference": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "_key": "1", + "citation": { + "_type": "journal article", + "_date": "2003-09-15", + "_name": "Mol. Microbiol.", + "_volume": "47", + "_first": "1495", + "_last": "1512", + "_publisher": "Wiley-Blackwell", + "_city": "Oxford", + "_db": "EMBL/GenBank/DDBJ", + "_number": "US20230123456", + "_institute": "MRC Laboratory of Molecular Biology", + "_country": "United Kingdom", + "title": "An integrated analysis of the genome of the hyperthermophilic archaeon Pyrococcus abyssi.", + "editorList": { + "consortium": [ + { + "_name": "The UniProt Consortium" + } + ], + "person": [ + { + "_name": "Cohen G.N." + } + ] + }, + "authorList": { + "consortium": [ + { + "_name": "International Cancer Genome Consortium" + } + ], + "person": [ + { + "_name": "Forterre P." + } + ] + }, + "locator": "10.1046/j.1365-2958.2003.03381.x", + "dbReference": [ + { + "_type": "GO", + "_id": "GO:0003677", + "_evidence": [ + 5, + 7, + 1 + ], + "molecule": { + "_id": "P04637-1 (7)", + "#text": "Homotetramer" + }, + "property": [ + { + "_type": "isSeed", + "_value": "true" + }, + { + "_type": "protein sequence ID", + "_value": "CAB48986.1" + } + ] + }, + { + "_type": "Pfam", + "_id": "PF00870", + "_evidence": [ + 6, + 8, + 2 + ], + "molecule": { + "_id": "P04637-2 (8)", + "#text": "Core domain" + }, + "property": [ + { + "_type": "molecule type", + "_value": "Genomic_DNA" + }, + { + "_type": "entry name", + "_value": "ACCD_DCysDesulf" + } + ] + } + ] + }, + "scope": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]", + "X-RAY CRYSTALLOGRAPHY (2.2 ANGSTROMS) OF 102-292" + ], + "source": { + "strain": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "GE5 / Orsay" + } + ], + "plasmid": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "pAB1" + } + ], + "transposon": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "Tn10" + } + ], + "tissue": [ + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Liver" + } + ] + } + }, + { + "_evidence": [ + 2, + 4, + 7 + ], + "_key": "2", + "citation": { + "_type": "book", + "_date": "1999", + "_name": "Curr. Microbiol.", + "_volume": "64", + "_first": "118", + "_last": "129", + "_publisher": "Springer", + "_city": "Heidelberg", + "_db": "UniProtKB", + "_number": "EP3987654A1", + "_institute": "Max Planck Institute for Biochemistry", + "_country": "Germany", + "title": "Genome of rice cluster I archaea -- the key methane producers in the rice rhizosphere.", + "editorList": { + "consortium": [ + { + "_name": "Archaea Genome Sequencing Project" + } + ], + "person": [ + { + "_name": "Erkel C." + } + ] + }, + "authorList": { + "consortium": [ + { + "_name": "The UniProt Consortium (4)" + } + ], + "person": [ + { + "_name": "Liesack W." + } + ] + }, + "locator": "10.1126/science.1127062", + "dbReference": [ + { + "_type": "InterPro", + "_id": "IPR011615", + "_evidence": [ + 3, + 5, + 8 + ], + "molecule": { + "_id": "P04637-3 (9)", + "#text": "Precursor" + }, + "property": [ + { + "_type": "match status", + "_value": "1" + }, + { + "_type": "nucleotide sequence ID", + "_value": "NC_000868.1" + } + ] + }, + { + "_type": "PDB", + "_id": "1TUP", + "_evidence": [ + 4, + 6, + 9 + ], + "molecule": { + "_id": "Q9V2L2-1 (10)", + "#text": "Mature peptide" + }, + "property": [ + { + "_type": "taxonomic scope", + "_value": "Archaea" + }, + { + "_type": "component", + "_value": "Chromosome" + } + ] + } + ] + }, + "scope": [ + "GENOME REANNOTATION", + "FUNCTION AND CATALYTIC ACTIVITY" + ], + "source": { + "strain": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "DSM 22066 / NBRC 105507 / MRE50" + } + ], + "plasmid": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "pMRE50" + } + ], + "transposon": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Tn5" + } + ], + "tissue": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "Colon epithelium" + } + ] + } + }, + { + "_evidence": [ + 1, + 5, + 9 + ], + "_key": "3", + "citation": { + "_type": "submission", + "_date": "2007-10", + "_db": "EMBL/GenBank/DDBJ databases" + }, + "scope": [ + "NUCLEOTIDE SEQUENCE [GENOMIC DNA]" + ] + } + ], + "comment": [ + { + "_type": "biophysicochemical properties", + "_locationType": "topology", + "_name": "UV-Vis absorption profile", + "_mass": 43653.5, + "_error": "Frameshift at position 112", + "_method": "Electrospray ionization", + "_evidence": [ + 9, + 2, + 5 + ], + "molecule": { + "_id": "Q0W5C8-1 (11)", + "#text": "Isoform 1 (11)" + }, + "absorption": { + "max": { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "280" + }, + "text": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Absorbs UV light with a primary peak at 280 nm." + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "Secondary shoulder observed at 292 nm in native buffer." + } + ] + }, + "kinetics": { + "KM": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "12.5 uM for ATP" + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "48.0 uM for 1-aminocyclopropane-1-carboxylate" + } + ], + "Vmax": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "450 nmol/min/mg enzyme" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "1280 nmol/min/mg in presence of PLP cofactor" + } + ], + "text": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "Follows classical Michaelis-Menten kinetics at pH 7.4." + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "Shows positive cooperativity upon tetramerization." + } + ] + }, + "phDependence": { + "text": [ + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Optimum pH is 7.4-7.8 in phosphate buffer." + }, + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Rapidly loses DNA-binding activity below pH 6.0." + } + ] + }, + "redoxPotential": { + "text": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "Midpoint potential E0 is -240 mV at pH 7.0." + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Oxidation of Cys-277 abolishes sequence-specific binding." + } + ] + }, + "temperatureDependence": { + "text": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Optimum temperature is 37 degrees Celsius." + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Unfolds irreversibly above 44 degrees Celsius (Tm = 42.5 C)." + } + ] + }, + "location": [ + { + "_sequence": "VSP_006535", + "begin": { + "_position": 1, + "_status": "certain", + "_evidence": [ + 7, + 9, + 3 + ] + }, + "end": { + "_position": 42, + "_status": "certain", + "_evidence": [ + 8, + 1, + 4 + ] + } + }, + { + "_sequence": "VSP_006536", + "position": { + "_position": 15, + "_status": "certain", + "_evidence": [ + 9, + 2, + 5 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Biophysical measurements performed on purified recombinant core domain." + }, + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Exhibits high-affinity sequence-specific DNA binding." + } + ] + }, + { + "_type": "catalytic activity", + "_locationType": "transmembrane", + "_name": "ACC deaminase reaction", + "_mass": 35755.25, + "_error": "Erroneous initiation at residue 1", + "_method": "MALDI-TOF", + "_evidence": [ + 3, + 5, + 8 + ], + "molecule": { + "_id": "A3CTY6-1 (12)", + "#text": "Isoform 2 (12)" + }, + "reaction": { + "_evidence": [ + 4, + 6, + 9 + ], + "text": "1-aminocyclopropane-1-carboxylate + H2O = 2-oxobutanoate + NH4(+)", + "dbReference": [ + { + "_type": "PubMed", + "_id": "12622808", + "_evidence": [ + 5, + 7, + 1 + ], + "molecule": { + "_id": "P04637-1 (13)", + "#text": "Isoform 3 (13)" + }, + "property": [ + { + "_type": "term", + "_value": "F:DNA-binding transcription factor activity" + }, + { + "_type": "evidence", + "_value": "ECO:0007669" + } + ] + }, + { + "_type": "DOI", + "_id": "10.1046/j.1365-2958.2003.03381.x", + "_evidence": [ + 6, + 8, + 2 + ], + "molecule": { + "_id": "P04637-2 (14)", + "#text": "Chain A (14)" + }, + "property": [ + { + "_type": "project", + "_value": "UniProtKB-KW" + }, + { + "_type": "chains", + "_value": "A/B=102-292" + } + ] + } + ] + }, + "physiologicalReaction": [ + { + "_direction": "left-to-right", + "_evidence": [ + 7, + 9, + 3 + ], + "dbReference": { + "_type": "Rhea", + "_id": "RHEA:16933", + "_evidence": [ + 8, + 1, + 4 + ], + "molecule": { + "_id": "P04637-3 (15)", + "#text": "Chain B (15)" + }, + "property": [ + { + "_type": "resolution", + "_value": "2.20 A" + }, + { + "_type": "method", + "_value": "X-ray" + } + ] + } + }, + { + "_direction": "right-to-left", + "_evidence": [ + 9, + 2, + 5 + ], + "dbReference": { + "_type": "ChEBI", + "_id": "CHEBI:15377", + "_evidence": [ + 1, + 3, + 6 + ], + "molecule": { + "_id": "Q9V2L2-1 (16)", + "#text": "Monomer (16)" + }, + "property": [ + { + "_type": "member count (27)", + "_value": "Q9V2L2" + }, + { + "_type": "common taxon (28)", + "_value": "UPI0000061B2A" + } + ] + } + } + ], + "location": [ + { + "_sequence": "VSP_006537", + "position": { + "_position": 20, + "_status": "uncertain", + "_evidence": [ + 2, + 4, + 7 + ] + } + }, + { + "_sequence": "VSP_006538", + "begin": { + "_position": 43, + "_status": "uncertain", + "_evidence": [ + 3, + 5, + 8 + ] + }, + "end": { + "_position": 63, + "_status": "uncertain", + "_evidence": [ + 4, + 6, + 9 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Catalyzes the ring-opening deamination of ACC." + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Activity is strictly dependent on divalent cations." + } + ] + }, + { + "_type": "cofactor", + "_locationType": "intramembrane", + "_name": "Zinc-binding site", + "_mass": 21879.75, + "_error": "Stop codon readthrough at residue 341", + "_method": "Analytical ultracentrifugation", + "_evidence": [ + 7, + 9, + 3 + ], + "molecule": { + "_id": "Q0W5C8-1 (17)", + "#text": "Homotetramer (17)" + }, + "cofactor": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "name": "pyridoxal 5'-phosphate", + "dbReference": { + "_type": "EC", + "_id": "3.5.99.7", + "_evidence": [ + 9, + 2, + 5 + ], + "molecule": { + "_id": "A3CTY6-1 (18)", + "#text": "Core domain (18)" + }, + "property": [ + { + "_type": "common taxon ID (29)", + "_value": "UniRef100_Q9V2L2" + }, + { + "_type": "GO Molecular Function (30)", + "_value": "Putative 1-aminocyclopropane-1-carboxylate deaminase" + } + ] + } + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "name": "Zn(2+)", + "dbReference": { + "_type": "IntAct", + "_id": "EBI-366083", + "_evidence": [ + 2, + 4, + 7 + ], + "molecule": { + "_id": "P04637-1 (19)", + "#text": "Precursor (19)" + }, + "property": [ + { + "_type": "UniProtKB accession (31)", + "_value": "Pyrococcus abyssi (strain GE5 / Orsay)" + }, + { + "_type": "UniParc ID (32)", + "_value": "272844" + } + ] + } + } + ], + "location": [ + { + "_sequence": "VSP_006539", + "begin": { + "_position": 65, + "_status": "less than", + "_evidence": [ + 3, + 5, + 8 + ] + }, + "end": { + "_position": 92, + "_status": "less than", + "_evidence": [ + 4, + 6, + 9 + ] + } + }, + { + "_sequence": "VSP_006540", + "position": { + "_position": 37, + "_status": "less than", + "_evidence": [ + 5, + 7, + 1 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Zinc coordination stabilizes loop-sheet-helix motif." + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Depletion of zinc induces molten-globule conformation." + } + ] + }, + { + "_type": "subcellular location", + "_locationType": "topological domain", + "_name": "Nuclear-cytoplasmic shuttling", + "_mass": 42110.125, + "_error": "Intron retention in exon 4", + "_method": "Size-exclusion chromatography", + "_evidence": [ + 8, + 1, + 4 + ], + "molecule": { + "_id": "P04637-2 (20)", + "#text": "Mature peptide (20)" + }, + "subcellularLocation": [ + { + "location": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "Nucleus" + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Nucleolus" + } + ], + "topology": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Peripheral membrane protein" + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "Single-pass membrane protein" + } + ], + "orientation": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Cytoplasmic side" + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Extracellular side" + } + ] + }, + { + "location": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Cytoplasm" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Mitochondrion" + } + ], + "topology": [ + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "Multi-pass membrane protein" + }, + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "Lipid-anchor" + } + ], + "orientation": [ + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Matrix side" + }, + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Lumenal side" + } + ] + } + ], + "location": [ + { + "_sequence": "VSP_006541", + "position": { + "_position": 54, + "_status": "greater than", + "_evidence": [ + 3, + 5, + 8 + ] + } + }, + { + "_sequence": "VSP_006542", + "begin": { + "_position": 102, + "_status": "greater than", + "_evidence": [ + 4, + 6, + 9 + ] + }, + "end": { + "_position": 292, + "_status": "greater than", + "_evidence": [ + 5, + 7, + 1 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Shuttles between nucleus and cytoplasm in a CRM1-dependent manner." + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Accumulates in the nucleus following ionizing radiation." + } + ] + }, + { + "_type": "sequence caution", + "_locationType": "chain", + "_name": "Frameshift discrepancy", + "_mass": 39850.0, + "_error": "Sequencing artifact at position 215", + "_method": "X-ray crystallography", + "_evidence": [ + 8, + 1, + 4 + ], + "molecule": { + "_id": "P04637-3 (21)", + "#text": "Isoform 1 (21)" + }, + "conflict": { + "_type": "frameshift", + "_ref": "1", + "sequence": { + "_resource": "EMBL-CDS", + "_id": "CAB48986", + "_version": 1 + } + }, + "location": [ + { + "_sequence": "VSP_006543", + "begin": { + "_position": 113, + "_status": "unknown", + "_evidence": [ + 9, + 2, + 5 + ] + }, + "end": { + "_position": 185, + "_status": "unknown", + "_evidence": [ + 1, + 3, + 6 + ] + } + }, + { + "_sequence": "VSP_006544", + "position": { + "_position": 72, + "_status": "unknown", + "_evidence": [ + 2, + 4, + 7 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "Sequence differs from genomic reference at codon 112." + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Verified by mass spectrometry of tryptic peptides." + } + ] + }, + { + "_type": "online information", + "_locationType": "domain", + "_name": "UniProtKB entry page", + "_mass": 41200.5, + "_error": "Mismatch at codon 72", + "_method": "Cryo-electron microscopy", + "_evidence": [ + 5, + 7, + 1 + ], + "molecule": { + "_id": "Q9V2L2-1 (22)", + "#text": "Isoform 2 (22)" + }, + "link": [ + { + "_uri": "https://www.uniprot.org/uniprotkb/P04637" + }, + { + "_uri": "https://www.ebi.ac.uk/pdbe/entry/pdb/1tup" + } + ], + "location": [ + { + "_sequence": "VSP_006535 (11)", + "position": { + "_position": 175, + "_status": "certain", + "_evidence": [ + 6, + 8, + 2 + ] + } + }, + { + "_sequence": "VSP_006536 (12)", + "begin": { + "_position": 195, + "_status": "certain", + "_evidence": [ + 7, + 9, + 3 + ] + }, + "end": { + "_position": 240, + "_status": "certain", + "_evidence": [ + 8, + 1, + 4 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "Full structural and clinical annotations available online." + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Cross-referenced with PDBe macromolecular database." + } + ] + }, + { + "_type": "alternative products", + "_locationType": "region", + "_name": "TP53 splice variants", + "_mass": 38915.25, + "_error": "Alternative exon boundary", + "_method": "NMR spectroscopy", + "_evidence": [ + 2, + 4, + 7 + ], + "molecule": { + "_id": "Q0W5C8-1 (23)", + "#text": "Isoform 3 (23)" + }, + "event": [ + { + "_type": "alternative splicing" + }, + { + "_type": "alternative initiation" + } + ], + "isoform": [ + { + "id": [ + "P04637-1", + "P04637-alpha" + ], + "name": [ + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "Alpha" + }, + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Canonical" + } + ], + "sequence": { + "_type": "displayed", + "_ref": "VSP_006535" + }, + "text": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Chosen as the canonical sequence." + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Expressed ubiquitously in normal tissues." + } + ] + }, + { + "id": [ + "P04637-2", + "P04637-beta" + ], + "name": [ + { + "_evidence": [ + 7, + 9, + 3 + ], + "#text": "Beta" + }, + { + "_evidence": [ + 8, + 1, + 4 + ], + "#text": "Alternative" + } + ], + "sequence": { + "_type": "external", + "_ref": "VSP_006536" + }, + "text": [ + { + "_evidence": [ + 9, + 2, + 5 + ], + "#text": "Produced by alternative splicing at intron 9." + }, + { + "_evidence": [ + 1, + 3, + 6 + ], + "#text": "Lacks the C-terminal oligomerization domain." + } + ] + } + ], + "location": [ + { + "_sequence": "VSP_006537 (13)", + "begin": { + "_position": 248, + "_status": "uncertain", + "_evidence": [ + 2, + 4, + 7 + ] + }, + "end": { + "_position": 270, + "_status": "uncertain", + "_evidence": [ + 3, + 5, + 8 + ] + } + }, + { + "_sequence": "VSP_006538 (14)", + "position": { + "_position": 245, + "_status": "uncertain", + "_evidence": [ + 4, + 6, + 9 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Contains 12 characterized splice isoforms." + }, + { + "_evidence": [ + 6, + 8, + 2 + ], + "#text": "Alternative promoter usage produces delta133 isoforms." + } + ] + }, + { + "_type": "interaction", + "_locationType": "motif", + "_name": "p53-MDM2 complex", + "_mass": 44102.75, + "_error": "Premature termination at 305", + "_method": "Surface plasmon resonance", + "_evidence": [ + 7, + 9, + 3 + ], + "molecule": { + "_id": "A3CTY6-1 (24)", + "#text": "Chain A (24)" + }, + "interactant": [ + { + "_intactId": "EBI-366083", + "id": "P04637", + "label": "TP53", + "dbReference": { + "_type": "MIM", + "_id": "151623", + "_evidence": [ + 8, + 1, + 4 + ], + "molecule": { + "_id": "P04637-1 (25)", + "#text": "Chain B (25)" + }, + "property": [ + { + "_type": "UniRef100 ID (33)", + "_value": "330" + }, + { + "_type": "UniRef90 ID (34)", + "_value": "false" + } + ] + } + }, + { + "_intactId": "EBI-389668", + "id": "Q00987", + "label": "MDM2", + "dbReference": { + "_type": "Reactome", + "_id": "R-HSA-3700989", + "_evidence": [ + 9, + 2, + 5 + ], + "molecule": { + "_id": "P04637-2 (26)", + "#text": "Monomer (26)" + }, + "property": [ + { + "_type": "protein name (35)", + "_value": "CAJ36415.1" + }, + { + "_type": "source organism (36)", + "_value": "mRNA" + } + ] + } + } + ], + "organismsDiffer": false, + "experiments": 14, + "location": [ + { + "_sequence": "VSP_006539 (15)", + "position": { + "_position": 248, + "_status": "less than", + "_evidence": [ + 1, + 3, + 6 + ] + } + }, + { + "_sequence": "VSP_006540 (16)", + "begin": { + "_position": 273, + "_status": "less than", + "_evidence": [ + 2, + 4, + 7 + ] + }, + "end": { + "_position": 299, + "_status": "less than", + "_evidence": [ + 3, + 5, + 8 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 4, + 6, + 9 + ], + "#text": "Forms a tight complex with E3 ubiquitin-protein ligase MDM2." + }, + { + "_evidence": [ + 5, + 7, + 1 + ], + "#text": "Phosphorylation at Ser-15 disrupts MDM2 binding." + } + ] + }, + { + "_type": "disease", + "_locationType": "site", + "_name": "Li-Fraumeni predisposition", + "_mass": 43590.875, + "_error": "Missense substitution R175H", + "_method": "Isothermal titration calorimetry", + "_evidence": [ + 6, + 8, + 2 + ], + "molecule": { + "_id": "P04637-3 (27)", + "#text": "Homotetramer (27)" + }, + "disease": { + "_id": "DI-00682", + "name": "Li-Fraumeni syndrome", + "acronym": "LFS1", + "description": "An autosomal dominant familial cancer syndrome caused by germline TP53 mutations.", + "dbReference": { + "_type": "AlphaFoldDB", + "_id": "AF-P04637-F1", + "_evidence": [ + 7, + 9, + 3 + ], + "molecule": { + "_id": "Q9V2L2-1 (28)", + "#text": "Core domain (28)" + }, + "property": [ + { + "_type": "NCBI taxonomy (37)", + "_value": "Pur_DNA_glyco" + }, + { + "_type": "length (38)", + "_value": "2" + } + ] + } + }, + "location": [ + { + "_sequence": "VSP_006541 (17)", + "begin": { + "_position": 301, + "_status": "greater than", + "_evidence": [ + 8, + 1, + 4 + ] + }, + "end": { + "_position": 322, + "_status": "greater than", + "_evidence": [ + 9, + 2, + 5 + ] + } + }, + { + "_sequence": "VSP_006542 (18)", + "position": { + "_position": 249, + "_status": "greater than", + "_evidence": [ + 1, + 3, + 6 + ] + } + } + ], + "text": [ + { + "_evidence": [ + 2, + 4, + 7 + ], + "#text": "Mutations in the DNA-binding domain account for 80% of LFS cases." + }, + { + "_evidence": [ + 3, + 5, + 8 + ], + "#text": "Associated with early-onset sarcomas and breast carcinomas." + } + ] + }, + { + "_type": "miscellaneous" + } + ], + "dbReference": [ + { + "_type": "SMART", + "_id": "SM00325", + "_evidence": [ + 4, + 6, + 9 + ], + "molecule": { + "_id": "Q0W5C8-1 (29)", + "#text": "Precursor (29)" + }, + "property": [ + { + "_type": "isSeed (39)", + "_value": "NC_009464.1" + }, + { + "_type": "protein sequence ID (40)", + "_value": "Eukaryota" + } + ] + }, + { + "_type": "SUPFAM", + "_id": "SSF49417", + "_evidence": [ + 5, + 7, + 1 + ], + "molecule": { + "_id": "A3CTY6-1 (30)", + "#text": "Mature peptide (30)" + }, + "property": [ + { + "_type": "molecule type (41)", + "_value": "Mitochondrion" + }, + { + "_type": "entry name (42)", + "_value": "P:regulation of cell cycle" + } + ] + } + ], + "proteinExistence": { + "_type": "evidence at protein level" + }, + "keyword": [ + { + "_evidence": [ + 6, + 8, + 2 + ], + "_id": "KW-0002", + "#text": "3D-structure" + }, + { + "_evidence": [ + 7, + 9, + 3 + ], + "_id": "KW-0227", + "#text": "DNA damage" + } + ], + "feature": [ + { + "_type": "domain", + "_id": "PRO_0000080560", + "_description": "p53 DNA-binding core domain", + "_evidence": [ + 8, + 1, + 4 + ], + "_ref": "1", + "original": "R", + "variation": [ + "H", + "Q" + ], + "location": { + "_sequence": "VSP_006543 (19)", + "begin": { + "_position": 325, + "_status": "unknown", + "_evidence": [ + 9, + 2, + 5 + ] + }, + "end": { + "_position": 355, + "_status": "unknown", + "_evidence": [ + 1, + 3, + 6 + ] + } + }, + "ligand": { + "name": "Mg(2+)", + "dbReference": { + "_type": "PANTHER", + "_id": "PTHR11447", + "_evidence": [ + 2, + 4, + 7 + ], + "molecule": { + "_id": "P04637-1 (31)", + "#text": "Isoform 1 (31)" + }, + "property": [ + { + "_type": "match status (43)", + "_value": "ECO:0000314" + }, + { + "_type": "nucleotide sequence ID (44)", + "_value": "GO_Central" + } + ] + }, + "label": "ZN-1", + "note": "Coordinated by Cys-176, His-179, Cys-238, and Cys-242." + }, + "ligandPart": { + "name": "catalytic zinc site", + "dbReference": { + "_type": "Gene3D", + "_id": "2.60.40.720", + "_evidence": [ + 3, + 5, + 8 + ], + "molecule": { + "_id": "P04637-2 (32)", + "#text": "Isoform 2 (32)" + }, + "property": [ + { + "_type": "taxonomic scope (45)", + "_value": "3 (45)" + }, + { + "_type": "component (46)", + "_value": "Homo sapiens (46)" + } + ] + }, + "label": "ZN-coord", + "note": "Essential for structural integrity of L2/L3 loops." + } + }, + { + "_type": "mutagenesis site", + "_id": "PRO_0000184521", + "_description": "Loss of sequence-specific DNA binding", + "_evidence": [ + 4, + 6, + 9 + ], + "_ref": "2", + "original": "S", + "variation": [ + "A", + "E" + ], + "location": { + "_sequence": "VSP_006544 (20)", + "position": { + "_position": 273, + "_status": "unknown", + "_evidence": [ + 5, + 7, + 1 + ] + } + }, + "ligand": { + "name": "S-adenosyl-L-methionine", + "dbReference": { + "_type": "CDD", + "_id": "cd08367", + "_evidence": [ + 6, + 8, + 2 + ], + "molecule": { + "_id": "P04637-3 (33)", + "#text": "Isoform 3 (33)" + }, + "property": [ + { + "_type": "term (47)", + "_value": "9606 (47)" + }, + { + "_type": "evidence (48)", + "_value": "GO:0003677 (48)" + } + ] + }, + "label": "PLP-54", + "note": "Covalently linked via Schiff base to Lys-54." + }, + "ligandPart": { + "name": "phosphate binding pocket", + "dbReference": { + "_type": "HAMAP-Rule", + "_id": "MF_00527", + "_evidence": [ + 7, + 9, + 3 + ], + "molecule": { + "_id": "Q9V2L2-1 (34)", + "#text": "Chain A (34)" + }, + "property": [ + { + "_type": "project (49)", + "_value": "P04637 (49)" + }, + { + "_type": "chains (50)", + "_value": "UPI000002ED67 (50)" + } + ] + }, + "label": "PLP-OPO3", + "note": "Stabilized by hydrogen bonding with Gly-188 and Thr-190." + } + } + ], + "evidence": [ + { + "_type": "ECO:0000269", + "_key": 1, + "source": { + "_ref": 1, + "dbReference": { + "_type": "PIR", + "_id": "A25224", + "_evidence": [ + 8, + 1, + 4 + ], + "molecule": { + "_id": "Q0W5C8-1 (35)", + "#text": "Chain B (35)" + }, + "property": [ + { + "_type": "resolution (51)", + "_value": "UniRef100_P04637 (51)" + }, + { + "_type": "method (52)", + "_value": "UniRef90_P04637 (52)" + } + ] + } + }, + "importedFrom": { + "dbReference": { + "_type": "KEGG", + "_id": "hsa:7157", + "_evidence": [ + 9, + 2, + 5 + ], + "molecule": { + "_id": "A3CTY6-1 (36)", + "#text": "Monomer (36)" + }, + "property": [ + { + "_type": "member count (53)", + "_value": "Cellular tumor antigen p53 (53)" + }, + { + "_type": "common taxon (54)", + "_value": "Homo sapiens (Human) (54)" + } + ] + } + } + }, + { + "_type": "ECO:0000305", + "_key": 2, + "source": { + "_ref": 2, + "dbReference": { + "_type": "STRING", + "_id": "9606.ENSP00000269305", + "_evidence": [ + 1, + 3, + 6 + ], + "molecule": { + "_id": "P04637-1 (37)", + "#text": "Homotetramer (37)" + }, + "property": [ + { + "_type": "common taxon ID (55)", + "_value": "9606 (55)" + }, + { + "_type": "GO Molecular Function (56)", + "_value": "393 (56)" + } + ] + } + }, + "importedFrom": { + "dbReference": { + "_type": "OrthoDB", + "_id": "119234at2759", + "_evidence": [ + 2, + 4, + 7 + ], + "molecule": { + "_id": "P04637-2 (38)", + "#text": "Core domain (38)" + }, + "property": [ + { + "_type": "UniProtKB accession (57)", + "_value": "true (57)" + }, + { + "_type": "UniParc ID (58)", + "_value": "CAB48986.1 (58)" + } + ] + } + } + } + ], + "sequence": { + "_length": 393, + "_mass": 43653, + "_checksum": "AD5C149FD8106131", + "_modified": "2009-11-24", + "_version": 2, + "_precursor": true, + "_fragment": "single", + "#text": "MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP" + } + } +] diff --git a/tests/data/xsd/examples/uniprot/minimal.xml b/tests/data/xsd/examples/uniprot/minimal.xml new file mode 100644 index 00000000..6fb4ac89 --- /dev/null +++ b/tests/data/xsd/examples/uniprot/minimal.xml @@ -0,0 +1,22 @@ + + + + P04637 + P53_HUMAN + + + Cellular tumor antigen p53 + + + + Homo sapiens + + + + + NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA] + + + MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP + + diff --git a/tests/data/xsd/examples/uniprot/minimal_expected.json b/tests/data/xsd/examples/uniprot/minimal_expected.json new file mode 100644 index 00000000..b30aba42 --- /dev/null +++ b/tests/data/xsd/examples/uniprot/minimal_expected.json @@ -0,0 +1,57 @@ +[ + { + "_dataset": "Swiss-Prot", + "_created": "1986-08-13", + "_modified": "2024-05-29", + "_version": 285, + "accession": [ + "P04637" + ], + "name": [ + "P53_HUMAN" + ], + "protein": { + "recommendedName": { + "fullName": { + "#text": "Cellular tumor antigen p53" + } + } + }, + "organism": { + "name": [ + { + "_type": "synonym", + "#text": "Homo sapiens" + } + ], + "dbReference": [ + { + "_type": "UniProtKB ID", + "_id": "P53_HUMAN" + } + ] + }, + "reference": [ + { + "_key": "1", + "citation": { + "_type": "journal article" + }, + "scope": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ] + } + ], + "proteinExistence": { + "_type": "evidence at protein level" + }, + "sequence": { + "_length": 393, + "_mass": 43653, + "_checksum": "AD5C149FD8106131", + "_modified": "2009-11-24", + "_version": 1, + "#text": "MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP" + } + } +] diff --git a/tests/data/xsd/examples/uniref/expected_schema.py b/tests/data/xsd/examples/uniref/expected_schema.py new file mode 100644 index 00000000..cdd3e680 --- /dev/null +++ b/tests/data/xsd/examples/uniref/expected_schema.py @@ -0,0 +1,188 @@ +"""Expected PyArrow schema for uniref (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"comment": "The entry identifier.", "xsd_type": "xs:ID"} + ), + pa.field( + "_updated", + pa.date32(), + nullable=False, + metadata={"comment": "The update date for the entry.", "xsd_type": "xs:date"}, + ), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_value", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "representativeMember", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "The name of the database this cross-reference is referring to.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"comment": "The ID referred to.", "xsd_type": "xs:string"}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Database cross-references"}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field("_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "UniRef50/90/100 cluster members"}, + ), + pa.field( + "member", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "The name of the database this cross-reference is referring to.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"comment": "The ID referred to.", "xsd_type": "xs:string"}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Database cross-references"}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"} + ), + pa.field( + "_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "UniRef50/90/100 cluster members"}, + ), + ] +) diff --git a/tests/data/xsd/examples/uniref/maximal.xml b/tests/data/xsd/examples/uniref/maximal.xml new file mode 100644 index 00000000..6e5a3d58 --- /dev/null +++ b/tests/data/xsd/examples/uniref/maximal.xml @@ -0,0 +1,29 @@ + + + + Cluster: Glycogen synthase + + + + + + + + MHPKVDALLSRFPRITLIPWETPIQYLPRISRELGVDVYVKRDDLTGLGIGGNKIRKLEF + + + + + + + MPVLPRDFYDRPTLEVARDLLGKTLVRQLPAGRVALRIVETEAYIGENDKACHASKGMTA + + + + + + + MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP + + + diff --git a/tests/data/xsd/examples/uniref/maximal_expected.json b/tests/data/xsd/examples/uniref/maximal_expected.json new file mode 100644 index 00000000..9b6883a8 --- /dev/null +++ b/tests/data/xsd/examples/uniref/maximal_expected.json @@ -0,0 +1,82 @@ +[ + { + "_id": "UniRef100_P04637", + "_updated": "2025-02-05", + "name": "Cluster: Glycogen synthase", + "property": [ + { + "_type": "member count", + "_value": "3" + }, + { + "_type": "common taxon", + "_value": "Homo sapiens" + } + ], + "representativeMember": { + "dbReference": { + "_type": "UniProtKB ID", + "_id": "P53_HUMAN", + "property": [ + { + "_type": "common taxon ID", + "_value": "9606" + }, + { + "_type": "GO Molecular Function", + "_value": "GO:0003677" + } + ] + }, + "sequence": { + "_length": 393, + "_checksum": "AD5C149FD8106131", + "#text": "MHPKVDALLSRFPRITLIPWETPIQYLPRISRELGVDVYVKRDDLTGLGIGGNKIRKLEF" + } + }, + "member": [ + { + "dbReference": { + "_type": "UniParc ID", + "_id": "UPI000002ED67", + "property": [ + { + "_type": "UniProtKB accession", + "_value": "P04637" + }, + { + "_type": "UniParc ID", + "_value": "UPI000002ED67" + } + ] + }, + "sequence": { + "_length": 330, + "_checksum": "CC8699923C0B11CD", + "#text": "MPVLPRDFYDRPTLEVARDLLGKTLVRQLPAGRVALRIVETEAYIGENDKACHASKGMTA" + } + }, + { + "dbReference": { + "_type": "RefSeq", + "_id": "NP_000537.3", + "property": [ + { + "_type": "UniRef100 ID", + "_value": "UniRef100_P04637" + }, + { + "_type": "UniRef90 ID", + "_value": "UniRef90_P04637" + } + ] + }, + "sequence": { + "_length": 200, + "_checksum": "8DA31EA44EEB8636", + "#text": "MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP" + } + } + ] + } +] diff --git a/tests/data/xsd/examples/uniref/minimal.xml b/tests/data/xsd/examples/uniref/minimal.xml new file mode 100644 index 00000000..7a1904bd --- /dev/null +++ b/tests/data/xsd/examples/uniref/minimal.xml @@ -0,0 +1,9 @@ + + + + Cluster: Glycogen synthase + + + + + diff --git a/tests/data/xsd/examples/uniref/minimal_expected.json b/tests/data/xsd/examples/uniref/minimal_expected.json new file mode 100644 index 00000000..1c78c485 --- /dev/null +++ b/tests/data/xsd/examples/uniref/minimal_expected.json @@ -0,0 +1,13 @@ +[ + { + "_id": "UniRef100_P04637", + "_updated": "2025-02-05", + "name": "Cluster: Glycogen synthase", + "representativeMember": { + "dbReference": { + "_type": "UniProtKB ID", + "_id": "P53_HUMAN" + } + } + } +] diff --git a/tests/data/xsd/nested_choice.xsd b/tests/data/xsd/nested_choice.xsd deleted file mode 100644 index 4fe1fdd0..00000000 --- a/tests/data/xsd/nested_choice.xsd +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/tests/data/xsd/nested_sequence_deep.xsd b/tests/data/xsd/nested_sequence_deep.xsd deleted file mode 100644 index 46d867bb..00000000 --- a/tests/data/xsd/nested_sequence_deep.xsd +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/tests/data/xsd/occurs_variety.xsd b/tests/data/xsd/occurs_variety.xsd deleted file mode 100644 index 52427754..00000000 --- a/tests/data/xsd/occurs_variety.xsd +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/tests/data/xsd/ref_element.xsd b/tests/data/xsd/ref_element.xsd deleted file mode 100644 index 52ab6632..00000000 --- a/tests/data/xsd/ref_element.xsd +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/tests/data/xsd/spec/01_scalars_and_simple_types/data.xml b/tests/data/xsd/spec/01_scalars_and_simple_types/data.xml new file mode 100644 index 00000000..e89fcd3d --- /dev/null +++ b/tests/data/xsd/spec/01_scalars_and_simple_types/data.xml @@ -0,0 +1,50 @@ + + + + true + -5 + 1024 + 100000 + 999999999999 + 255 + 65535 + 4000000000 + 18446744073709551615 + 1.5 + 2.718281828 + 3.14 + 2025-06-15 + 2025-06-15T12:30:00Z + 2025-06 + 42000000000 + 10 20 30 + alpha beta gamma + 48656c6c6f + SGVsbG8= + + + 0 + + + 1 + -128 + -32768 + -2147483648 + -999999999999 + 0 + 0 + 0 + 0 + -0.5 + 1.25e-4 + 42 + 2025-06-15Z + 2025-06-15T12:30:00.123+02:00 + 1999 + -42 + 100 200 + one two + 48656C6C6F + SGVsbG8= + + diff --git a/tests/data/xsd/spec/01_scalars_and_simple_types/expected.json b/tests/data/xsd/spec/01_scalars_and_simple_types/expected.json new file mode 100644 index 00000000..42319478 --- /dev/null +++ b/tests/data/xsd/spec/01_scalars_and_simple_types/expected.json @@ -0,0 +1,81 @@ +[ + { + "_id": "rec1", + "_status": "active", + "_evidence": [ + 1, + 4, + 9 + ], + "_version": 2, + "flag": true, + "smallInt": -5, + "medInt": 1024, + "stdInt": 100000, + "bigInt": 999999999999, + "uByte": 255, + "uShort": 65535, + "uInt": 4000000000, + "uLong": 18446744073709551615, + "f32": 1.5, + "f64": 2.718281828, + "dec": 3.14, + "created": "2025-06-15", + "updatedAt": "2025-06-15T12:30:00Z", + "published": "2025-06", + "homogeneousUnion": 42000000000, + "scores": [ + 10, + 20, + 30 + ], + "tokens": [ + "alpha", + "beta", + "gamma" + ], + "rawHex": "Hello", + "rawB64": "Hello" + }, + { + "_id": "rec2", + "_status": "archived", + "_version": 1, + "flag": false + }, + { + "_id": "rec3", + "_status": "active", + "_evidence": [ + 10, + 20 + ], + "_version": -42, + "flag": true, + "smallInt": -128, + "medInt": -32768, + "stdInt": -2147483648, + "bigInt": -999999999999, + "uByte": 0, + "uShort": 0, + "uInt": 0, + "uLong": 0, + "f32": -0.5, + "f64": 0.000125, + "dec": 42.0, + "created": "2025-06-15", + "updatedAt": "2025-06-15T12:30:00.123+02:00", + "published": "1999", + "homogeneousUnion": -42, + "scores": [ + 100, + 200 + ], + "tokens": [ + "one", + "two" + ], + "rawHex": "Hello", + "rawB64": "Hello" + } +] diff --git a/tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py b/tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py new file mode 100644 index 00000000..aac1da5b --- /dev/null +++ b/tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py @@ -0,0 +1,61 @@ +"""Expected PyArrow schema for 01_scalars_and_simple_types (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"comment": "Unique record identifier.", "xsd_type": "xs:ID"} + ), + pa.field( + "_status", + pa.string(), + nullable=False, + metadata={"comment": "Record lifecycle status.", "xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field( + "flag", pa.bool_(), nullable=False, metadata={"comment": "Primary boolean flag.", "xsd_type": "xs:boolean"} + ), + pa.field("smallInt", pa.int8(), nullable=True, metadata={"xsd_type": "xs:byte"}), + pa.field("medInt", pa.int16(), nullable=True, metadata={"xsd_type": "xs:short"}), + pa.field("stdInt", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field("bigInt", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"}), + pa.field("uByte", pa.uint8(), nullable=True, metadata={"xsd_type": "xs:unsignedByte"}), + pa.field("uShort", pa.uint16(), nullable=True, metadata={"xsd_type": "xs:unsignedShort"}), + pa.field("uInt", pa.uint32(), nullable=True, metadata={"xsd_type": "xs:unsignedInt"}), + pa.field("uLong", pa.uint64(), nullable=True, metadata={"xsd_type": "xs:unsignedLong"}), + pa.field("f32", pa.float64(), nullable=True, metadata={"xsd_type": "xs:float"}), + pa.field("f64", pa.float64(), nullable=True, metadata={"xsd_type": "xs:double"}), + pa.field("dec", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), + pa.field("created", pa.date32(), nullable=True, metadata={"xsd_type": "xs:date"}), + pa.field("updatedAt", pa.string(), nullable=True, metadata={"xsd_type": "xs:dateTime"}), + pa.field("published", pa.string(), nullable=True, metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}), + pa.field("homogeneousUnion", pa.int64(), nullable=True, metadata={"xsd_type": "xs:long | xs:integer"}), + pa.field( + "scores", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "tokens", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:NMTOKEN"}), + ), + nullable=True, + ), + pa.field("rawHex", pa.binary(), nullable=True, metadata={"xsd_type": "xs:hexBinary"}), + pa.field("rawB64", pa.binary(), nullable=True, metadata={"xsd_type": "xs:base64Binary"}), + ] +) diff --git a/tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd b/tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd new file mode 100644 index 00000000..31364311 --- /dev/null +++ b/tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + Record lifecycle status. + + + + + + + + + + + + + + + + Primary boolean flag. + + + + + + + + + + + + + + + + + + + + + + + + + Unique record identifier. + + + + + + + + + + + diff --git a/tests/data/xsd/spec/02_namespaces_and_nillable/data.xml b/tests/data/xsd/spec/02_namespaces_and_nillable/data.xml new file mode 100644 index 00000000..4eda01ef --- /dev/null +++ b/tests/data/xsd/spec/02_namespaces_and_nillable/data.xml @@ -0,0 +1,15 @@ + + + + + <label lang="en" xsi:nil="true"/> + <comment type="function" xsi:nil="true"/> + <comment type="subunit"> + <text>Homodimer</text> + </comment> + </entry> + <entry id="e2"> + <title>Normal Title + + diff --git a/tests/data/xsd/spec/02_namespaces_and_nillable/expected.json b/tests/data/xsd/spec/02_namespaces_and_nillable/expected.json new file mode 100644 index 00000000..58ff76bd --- /dev/null +++ b/tests/data/xsd/spec/02_namespaces_and_nillable/expected.json @@ -0,0 +1,21 @@ +[ + { + "_id": "e1", + "label": { + "_lang": "en" + }, + "comment": [ + { + "_type": "function" + }, + { + "_type": "subunit", + "text": "Homodimer" + } + ] + }, + { + "_id": "e2", + "title": "Normal Title" + } +] diff --git a/tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py b/tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py new file mode 100644 index 00000000..a2c573cf --- /dev/null +++ b/tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py @@ -0,0 +1,38 @@ +"""Expected PyArrow schema for 02_namespaces_and_nillable (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("title", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "label", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "comment", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd b/tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd new file mode 100644 index 00000000..e6058f32 --- /dev/null +++ b/tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/spec/03_mixed_content_raw_text/data.xml b/tests/data/xsd/spec/03_mixed_content_raw_text/data.xml new file mode 100644 index 00000000..b63a0ff7 --- /dev/null +++ b/tests/data/xsd/spec/03_mixed_content_raw_text/data.xml @@ -0,0 +1,13 @@ + + +
+ Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells + Studied bold & italic text where x < y. + Unlabelled paragraph with 2O and empty tag. + + +
+
+ + </article> +</articles> diff --git a/tests/data/xsd/spec/03_mixed_content_raw_text/expected.json b/tests/data/xsd/spec/03_mixed_content_raw_text/expected.json new file mode 100644 index 00000000..65f443d2 --- /dev/null +++ b/tests/data/xsd/spec/03_mixed_content_raw_text/expected.json @@ -0,0 +1,21 @@ +[ + { + "title": "Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells", + "abstractText": [ + { + "_Label": "BACKGROUND", + "#text": "Studied <b>bold</b> & <i>italic</i> text where x < y." + }, + { + "#text": "Unlabelled paragraph with <sub>2</sub>O and empty <sup/> tag." + }, + { + "_Label": "EMPTY" + }, + null + ] + }, + { + "title": "" + } +] diff --git a/tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py b/tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py new file mode 100644 index 00000000..b372cb8f --- /dev/null +++ b/tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py @@ -0,0 +1,27 @@ +"""Expected PyArrow schema for 03_mixed_content_raw_text (xml_tag='article').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("title", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "abstractText", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_Label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd b/tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd new file mode 100644 index 00000000..2f200b4b --- /dev/null +++ b/tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:group name="inlineFormatting"> + <xs:choice> + <xs:element name="b" type="formattedText"/> + <xs:element name="i" type="formattedText"/> + <xs:element name="sup" type="formattedText"/> + <xs:element name="sub" type="formattedText"/> + </xs:choice> + </xs:group> + + <xs:complexType name="formattedText" mixed="true"> + <xs:group ref="inlineFormatting" minOccurs="0" maxOccurs="unbounded"/> + </xs:complexType> + + <xs:complexType name="abstractTextType" mixed="true"> + <xs:group ref="inlineFormatting" minOccurs="0" maxOccurs="unbounded"/> + <xs:attribute name="Label" type="xs:string"/> + </xs:complexType> + + <xs:element name="articles"> + <xs:complexType> + <xs:sequence> + <xs:element name="article" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="title" type="formattedText"/> + <xs:element name="abstractText" type="abstractTextType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/data/xsd/spec/04_recursion_depth/data.xml b/tests/data/xsd/spec/04_recursion_depth/data.xml new file mode 100644 index 00000000..f330d7d4 --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/data.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="UTF-8"?> +<doc> + <record id="r1"> + <tree label="L1" weight="10"> + <branch label="L2"> + <branch label="L3" weight="30"> + <branch label="L4"/> + </branch> + <branch label="L3_sibling"/> + </branch> + </tree> + <Section> + <SectionTitle>Ch 1</SectionTitle> + <Section> + <SectionTitle>Sec 1.1</SectionTitle> + <Section> + <SectionTitle>Subsec 1.1.1</SectionTitle> + </Section> + </Section> + </Section> + </record> +</doc> diff --git a/tests/data/xsd/spec/04_recursion_depth/expected.json b/tests/data/xsd/spec/04_recursion_depth/expected.json new file mode 100644 index 00000000..44e8d693 --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/expected.json @@ -0,0 +1,29 @@ +[ + { + "_id": "r1", + "tree": { + "_label": "L1", + "_weight": 10, + "branch": [ + { + "_label": "L2", + "branch": [ + "{\"_label\": \"L3\", \"_weight\": 30, \"branch\": [{\"_label\": \"L4\"}]}", + "{\"_label\": \"L3_sibling\"}" + ] + } + ] + }, + "Section": { + "SectionTitle": "Ch 1", + "Section": [ + { + "SectionTitle": "Sec 1.1", + "Section": [ + "{\"SectionTitle\": \"Subsec 1.1.1\"}" + ] + } + ] + } + } +] diff --git a/tests/data/xsd/spec/04_recursion_depth/expected_depth_1.json b/tests/data/xsd/spec/04_recursion_depth/expected_depth_1.json new file mode 100644 index 00000000..d364b862 --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/expected_depth_1.json @@ -0,0 +1,18 @@ +[ + { + "_id": "r1", + "tree": { + "_label": "L1", + "_weight": 10, + "branch": [ + "{\"_label\": \"L2\", \"branch\": [{\"_label\": \"L3\", \"_weight\": 30, \"branch\": [{\"_label\": \"L4\"}]}, {\"_label\": \"L3_sibling\"}]}" + ] + }, + "Section": { + "SectionTitle": "Ch 1", + "Section": [ + "{\"SectionTitle\": \"Sec 1.1\", \"Section\": [{\"SectionTitle\": \"Subsec 1.1.1\"}]}" + ] + } + } +] diff --git a/tests/data/xsd/spec/04_recursion_depth/expected_schema.py b/tests/data/xsd/spec/04_recursion_depth/expected_schema.py new file mode 100644 index 00000000..d7f6d57f --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/expected_schema.py @@ -0,0 +1,121 @@ +"""Expected PyArrow schema for 04_recursion_depth (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_weight", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "branch", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field("_weight", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "Section", + pa.struct( + [ + pa.field("SectionTitle", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "Section", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "SectionTitle", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Section", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) + +EXPECTED_SCHEMA_DEPTH_1 = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_weight", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "Section", + pa.struct( + [ + pa.field("SectionTitle", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "Section", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/04_recursion_depth/schema.xsd b/tests/data/xsd/spec/04_recursion_depth/schema.xsd new file mode 100644 index 00000000..e79b9051 --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/schema.xsd @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="TreeType"> + <xs:sequence> + <xs:element name="branch" type="TreeType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + <xs:attribute name="label" type="xs:string"/> + <xs:attribute name="weight" type="xs:int"/> + </xs:complexType> + + <xs:element name="Section"> + <xs:complexType> + <xs:sequence> + <xs:element name="SectionTitle" type="xs:string"/> + <xs:element ref="Section" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:element name="doc"> + <xs:complexType> + <xs:sequence> + <xs:element name="record" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="tree" type="TreeType" minOccurs="0"/> + <xs:element ref="Section" minOccurs="0"/> + </xs:sequence> + <xs:attribute name="id" type="xs:string" use="required"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/data/xsd/spec/05_groups_and_compositors/data.xml b/tests/data/xsd/spec/05_groups_and_compositors/data.xml new file mode 100644 index 00000000..302a7b2b --- /dev/null +++ b/tests/data/xsd/spec/05_groups_and_compositors/data.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="UTF-8"?> +<dataset> + <item createdBy="alice" version="1"> + <pagination> + <StartPage>10</StartPage> + <EndPage>20</EndPage> + <MedlinePgn>10-20</MedlinePgn> + </pagination> + <interaction> + <interactant>P12345</interactant> + <interactant>Q67890</interactant> + </interaction> + <meta> + <year>2025</year> + <author>Bob</author> + </meta> + <pairs> + <k>a</k> + <v>1</v> + <k>b</k> + <v>2</v> + </pairs> + </item> + <item> + <pagination> + <MedlinePgn>e12345</MedlinePgn> + </pagination> + </item> +</dataset> diff --git a/tests/data/xsd/spec/05_groups_and_compositors/expected.json b/tests/data/xsd/spec/05_groups_and_compositors/expected.json new file mode 100644 index 00000000..538b9710 --- /dev/null +++ b/tests/data/xsd/spec/05_groups_and_compositors/expected.json @@ -0,0 +1,36 @@ +[ + { + "_createdBy": "alice", + "_version": 1, + "pagination": { + "StartPage": "10", + "EndPage": "20", + "MedlinePgn": "10-20" + }, + "interaction": { + "interactant": [ + "P12345", + "Q67890" + ] + }, + "meta": { + "author": "Bob", + "year": 2025 + }, + "pairs": { + "k": [ + "a", + "b" + ], + "v": [ + 1, + 2 + ] + } + }, + { + "pagination": { + "MedlinePgn": "e12345" + } + } +] diff --git a/tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py b/tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py new file mode 100644 index 00000000..390b5318 --- /dev/null +++ b/tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py @@ -0,0 +1,70 @@ +"""Expected PyArrow schema for 05_groups_and_compositors (xml_tag='item').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_createdBy", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_version", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "pagination", + pa.struct( + [ + pa.field("StartPage", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("EndPage", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("MedlinePgn", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "interaction", + pa.struct( + [ + pa.field( + "interactant", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "meta", + pa.struct( + [ + pa.field("author", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("year", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ] + ), + nullable=True, + ), + pa.field( + "pairs", + pa.struct( + [ + pa.field( + "k", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "v", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/05_groups_and_compositors/schema.xsd b/tests/data/xsd/spec/05_groups_and_compositors/schema.xsd new file mode 100644 index 00000000..d382efdf --- /dev/null +++ b/tests/data/xsd/spec/05_groups_and_compositors/schema.xsd @@ -0,0 +1,61 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:attributeGroup name="auditAttrs"> + <xs:attribute name="createdBy" type="xs:string"/> + <xs:attribute name="version" type="xs:int"/> + </xs:attributeGroup> + + <xs:group name="kvGroup"> + <xs:sequence> + <xs:element name="k" type="xs:string"/> + <xs:element name="v" type="xs:int"/> + </xs:sequence> + </xs:group> + + <xs:complexType name="paginationType"> + <xs:choice> + <xs:sequence> + <xs:element name="StartPage" type="xs:string"/> + <xs:element name="EndPage" type="xs:string" minOccurs="0"/> + <xs:element name="MedlinePgn" type="xs:string" minOccurs="0"/> + </xs:sequence> + <xs:element name="MedlinePgn" type="xs:string"/> + </xs:choice> + </xs:complexType> + + <xs:complexType name="interactionType"> + <xs:sequence> + <xs:element name="interactant" type="xs:string"/> + <xs:element name="interactant" type="xs:string"/> + </xs:sequence> + </xs:complexType> + + <xs:complexType name="metadataAllType"> + <xs:all> + <xs:element name="author" type="xs:string"/> + <xs:element name="year" type="xs:int" minOccurs="0"/> + </xs:all> + </xs:complexType> + + <xs:element name="dataset"> + <xs:complexType> + <xs:sequence> + <xs:element name="item" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="pagination" type="paginationType" minOccurs="0"/> + <xs:element name="interaction" type="interactionType" minOccurs="0"/> + <xs:element name="meta" type="metadataAllType" minOccurs="0"/> + <xs:element name="pairs" minOccurs="0"> + <xs:complexType> + <xs:group ref="kvGroup" maxOccurs="unbounded"/> + </xs:complexType> + </xs:element> + </xs:sequence> + <xs:attributeGroup ref="auditAttrs"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/data/xsd/spec/06_context_dependent_paths/data.xml b/tests/data/xsd/spec/06_context_dependent_paths/data.xml new file mode 100644 index 00000000..48246436 --- /dev/null +++ b/tests/data/xsd/spec/06_context_dependent_paths/data.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<root> + <left> + <node> + <leaf>l1</leaf> + </node> + </left> + <right> + <node> + <leaf>r1</leaf> + <leaf lang="en">r2</leaf> + </node> + </right> +</root> diff --git a/tests/data/xsd/spec/06_context_dependent_paths/expected.json b/tests/data/xsd/spec/06_context_dependent_paths/expected.json new file mode 100644 index 00000000..9551adce --- /dev/null +++ b/tests/data/xsd/spec/06_context_dependent_paths/expected.json @@ -0,0 +1,22 @@ +[ + { + "left": { + "node": { + "leaf": "l1" + } + }, + "right": { + "node": { + "leaf": [ + { + "#text": "r1" + }, + { + "_lang": "en", + "#text": "r2" + } + ] + } + } + } +] diff --git a/tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py b/tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py new file mode 100644 index 00000000..753f5621 --- /dev/null +++ b/tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py @@ -0,0 +1,69 @@ +"""Expected PyArrow schema for 06_context_dependent_paths (xml_tag='root').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "left", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field("leaf", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "right", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field( + "leaf", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_lang", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + ] +) diff --git a/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd b/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd new file mode 100644 index 00000000..79e9a9e6 --- /dev/null +++ b/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:element name="root"> + <xs:complexType> + <xs:sequence> + <xs:element name="left"> + <xs:complexType> + <xs:sequence> + <xs:element name="node"> + <xs:complexType> + <xs:sequence> + <xs:element name="leaf" type="xs:string"/> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + <xs:element name="right"> + <xs:complexType> + <xs:sequence> + <xs:element name="node"> + <xs:complexType> + <xs:sequence> + <xs:element name="leaf" maxOccurs="unbounded"> + <xs:complexType> + <xs:simpleContent> + <xs:extension base="xs:string"> + <xs:attribute name="lang" type="xs:string"/> + </xs:extension> + </xs:simpleContent> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/data/xsd/spec/07_multi_root_shared_type/data.xml b/tests/data/xsd/spec/07_multi_root_shared_type/data.xml new file mode 100644 index 00000000..b6946958 --- /dev/null +++ b/tests/data/xsd/spec/07_multi_root_shared_type/data.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<Cluster50> + <entry id="C1"> + <name>First Cluster</name> + </entry> + <entry id="C2"> + <name>Second Cluster</name> + </entry> +</Cluster50> diff --git a/tests/data/xsd/spec/07_multi_root_shared_type/expected.json b/tests/data/xsd/spec/07_multi_root_shared_type/expected.json new file mode 100644 index 00000000..83219793 --- /dev/null +++ b/tests/data/xsd/spec/07_multi_root_shared_type/expected.json @@ -0,0 +1,10 @@ +[ + { + "_id": "C1", + "name": "First Cluster" + }, + { + "_id": "C2", + "name": "Second Cluster" + } +] diff --git a/tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py b/tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py new file mode 100644 index 00000000..9b5f5a7f --- /dev/null +++ b/tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py @@ -0,0 +1,12 @@ +"""Expected PyArrow schema for 07_multi_root_shared_type (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] +) diff --git a/tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd b/tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd new file mode 100644 index 00000000..0f14887c --- /dev/null +++ b/tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="entryType"> + <xs:sequence> + <xs:element name="name" type="xs:string"/> + </xs:sequence> + <xs:attribute name="id" type="xs:string" use="required"/> + </xs:complexType> + + <xs:complexType name="clusterSetType"> + <xs:sequence> + <xs:element name="entry" type="entryType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + + <xs:element name="Cluster100" type="clusterSetType"/> + <xs:element name="Cluster90" type="clusterSetType"/> + <xs:element name="Cluster50" type="clusterSetType"/> +</xs:schema> diff --git a/tests/data/xsd/spec/08_nullability_and_empty_tags/data.xml b/tests/data/xsd/spec/08_nullability_and_empty_tags/data.xml new file mode 100644 index 00000000..3ce65c5a --- /dev/null +++ b/tests/data/xsd/spec/08_nullability_and_empty_tags/data.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<root xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <entry id="e1"> + <name>BRCA1</name> + <summary/> + <geneId>672</geneId> + <comment type="function" xsi:nil="true"/> + </entry> +</root> diff --git a/tests/data/xsd/spec/08_nullability_and_empty_tags/expected.json b/tests/data/xsd/spec/08_nullability_and_empty_tags/expected.json new file mode 100644 index 00000000..7404854c --- /dev/null +++ b/tests/data/xsd/spec/08_nullability_and_empty_tags/expected.json @@ -0,0 +1,11 @@ +[ + { + "_id": "e1", + "name": "BRCA1", + "summary": "", + "geneId": 672, + "comment": { + "_type": "function" + } + } +] diff --git a/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py b/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py new file mode 100644 index 00000000..12b33128 --- /dev/null +++ b/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py @@ -0,0 +1,27 @@ +"""Expected PyArrow schema for 08_nullability_and_empty_tags (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("summary", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("optionalNote", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("geneId", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field("proteinId", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "comment", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd b/tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd new file mode 100644 index 00000000..eec356df --- /dev/null +++ b/tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="commentType"> + <xs:sequence> + <xs:element name="text" type="xs:string"/> + </xs:sequence> + <xs:attribute name="type" type="xs:string" use="required"/> + </xs:complexType> + + <xs:element name="root"> + <xs:complexType> + <xs:sequence> + <xs:element name="entry" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="name" type="xs:string"/> + <xs:element name="summary" type="xs:string"/> + <xs:element name="optionalNote" type="xs:string" minOccurs="0"/> + <xs:choice> + <xs:element name="geneId" type="xs:int"/> + <xs:element name="proteinId" type="xs:string"/> + </xs:choice> + <xs:element name="comment" type="commentType" nillable="true"/> + </xs:sequence> + <xs:attribute name="id" type="xs:string" use="required"/> + <xs:attribute name="source" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/data/xsd/spec/09_empty_marker_elements/data.xml b/tests/data/xsd/spec/09_empty_marker_elements/data.xml new file mode 100644 index 00000000..a9f06029 --- /dev/null +++ b/tests/data/xsd/spec/09_empty_marker_elements/data.xml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="UTF-8"?> +<books> + <book> + <title>In-Process Book + + + + + + Completed Book + + diff --git a/tests/data/xsd/spec/09_empty_marker_elements/expected.json b/tests/data/xsd/spec/09_empty_marker_elements/expected.json new file mode 100644 index 00000000..f4b88e91 --- /dev/null +++ b/tests/data/xsd/spec/09_empty_marker_elements/expected.json @@ -0,0 +1,13 @@ +[ + { + "title": "In-Process Book", + "BookInProcess": "", + "flag": [ + "", + "" + ] + }, + { + "title": "Completed Book" + } +] diff --git a/tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py b/tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py new file mode 100644 index 00000000..529fbc96 --- /dev/null +++ b/tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py @@ -0,0 +1,19 @@ +"""Expected PyArrow schema for 09_empty_marker_elements (xml_tag='book').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("title", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("BookInProcess", pa.string(), nullable=True), + pa.field( + "flag", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/09_empty_marker_elements/schema.xsd b/tests/data/xsd/spec/09_empty_marker_elements/schema.xsd new file mode 100644 index 00000000..3ffbb59e --- /dev/null +++ b/tests/data/xsd/spec/09_empty_marker_elements/schema.xsd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/spec/10_combinations/data.xml b/tests/data/xsd/spec/10_combinations/data.xml new file mode 100644 index 00000000..64669df1 --- /dev/null +++ b/tests/data/xsd/spec/10_combinations/data.xml @@ -0,0 +1,240 @@ + + + + + + + + a + x + only + + + + + x + + + y + b1 + b2 + + + + + + + padded + Tom & Jerry + + + + + one + two + + + + + + + solo + + + + + + + + + + + + + + + + + + + + + + + + + g1 + a1 + a2 + + + + child-name + + + + + + + + + g2 + + + 1 + + + + + + + + plain text + Hello + world again + + + x + + + A + 1B + 2C + 3 + + Intro + bold + + side text + just text + + + + + only italic + + + bare note + + + + + + a1 + b1 + a2 + b2 + + + + solo + + a + 1 + + + + + + + a + 1 + b + 2 + + + + + + Book title + + Ch1 + + + Ch2 + Ch2 alt + + + + + + + Lonely + + + + + + + l1 + + + + + r1 + r2 + + + + + + + + + + + + + r + + + + + + + + + + + + + + + + + + + + + + + + + + + + s1 + + base + x1 + + + + + s1 + s2 + + x1 + x2 + + + diff --git a/tests/data/xsd/spec/10_combinations/expected.json b/tests/data/xsd/spec/10_combinations/expected.json new file mode 100644 index 00000000..0bd55d4f --- /dev/null +++ b/tests/data/xsd/spec/10_combinations/expected.json @@ -0,0 +1,377 @@ +[ + { + "_id": "r01" + }, + { + "_id": "r02", + "plain": "a", + "plainList": [ + "x" + ], + "bounded": [ + "only" + ] + }, + { + "_id": "r03", + "plainList": [ + "x", + null, + null, + "y" + ], + "bounded": [ + "b1", + "b2" + ] + }, + { + "_id": "r04", + "plain": "a < b & c", + "plainList": [ + "padded", + "Tom & Jerry" + ] + }, + { + "_id": "r05", + "label": { + "_lang": "en", + "#text": "Hello" + }, + "labelList": [ + { + "_lang": "en", + "_source": "curated", + "#text": "one" + }, + { + "#text": "two" + }, + { + "_lang": "fr" + }, + null + ] + }, + { + "_id": "r06", + "label": { + "#text": "Hello" + }, + "labelList": [ + { + "#text": "solo" + } + ] + }, + { + "_id": "r07", + "label": { + "_lang": "de" + }, + "textNoAttr": "t", + "measure": { + "_unit": "kg", + "#text": 3.5 + } + }, + { + "_id": "r08", + "measure": { + "#text": 7.0 + } + }, + { + "_id": "r09", + "flag": { + "_on": true + }, + "flagList": [ + { + "_on": false + }, + null + ], + "marker": [ + "", + "" + ] + }, + { + "_id": "r10", + "flagList": [ + { + "_on": true + } + ], + "marker": [ + "" + ] + }, + { + "_id": "r11", + "group": { + "name": "g1", + "alias": [ + "a1", + "a2" + ] + }, + "entity": [ + { + "_type": "person", + "_name": "attr-name", + "name": "child-name", + "prop": [ + { + "_key": "k1", + "_value": "v1" + }, + { + "_key": "k2" + } + ] + }, + { + "_type": "org" + } + ] + }, + { + "_id": "r12", + "group": { + "name": "g2" + }, + "box": { + "item": [ + "1" + ] + }, + "entity": [ + { + "prop": [ + { + "_key": "only" + } + ] + } + ] + }, + { + "_id": "r13", + "para": [ + { + "#text": "plain text" + }, + { + "#text": "Hello\n world again" + }, + { + "#text": "x" + }, + null, + { + "#text": "A\n 1B\n 2C\n 3" + }, + { + "_style": "em", + "#text": "Intro\n bold" + } + ], + "note": { + "_kind": "aside", + "#text": "side text" + }, + "remark": "just text" + }, + { + "_id": "r14", + "para": [ + { + "#text": "only italic" + }, + { + "_style": "x" + } + ], + "note": { + "#text": "bare note" + } + }, + { + "_id": "r15", + "note": { + "_kind": "empty" + }, + "optA": [ + "a1", + "a2" + ], + "optB": [ + { + "_lang": "en", + "#text": "b1" + }, + { + "#text": "b2" + } + ] + }, + { + "_id": "r16", + "optA": [ + "solo" + ], + "pairs": { + "k": [ + "a" + ], + "v": [ + "1" + ] + } + }, + { + "_id": "r17", + "optB": [ + null + ], + "pairs": { + "k": [ + "a", + "b" + ], + "v": [ + "1", + "2" + ] + } + }, + { + "_id": "r18", + "book": { + "title": "Book title", + "chapter": [ + { + "title": [ + { + "#text": "Ch1" + } + ] + }, + { + "title": [ + { + "_lang": "en", + "#text": "Ch2" + }, + { + "#text": "Ch2 alt" + } + ] + } + ] + } + }, + { + "_id": "r19", + "book": { + "title": "Lonely" + } + }, + { + "_id": "r20", + "left": { + "node": { + "leaf": "l1" + } + }, + "right": { + "node": { + "leaf": [ + { + "#text": "r1" + }, + { + "_lang": "en", + "#text": "r2" + } + ] + } + } + }, + { + "_id": "r21", + "left": { + "node": { + "leaf": "" + } + }, + "right": { + "node": { + "leaf": [ + { + "#text": "r" + } + ] + } + } + }, + { + "_id": "r22", + "tree": { + "_label": "root", + "branch": [ + { + "_label": "a", + "branch": [ + "{\"_label\": \"a1\"}", + "{\"_label\": \"a2\", \"branch\": [{\"_label\": \"a2x\"}]}" + ] + }, + null + ] + } + }, + { + "_id": "r23", + "tree": { + "_label": "t", + "branch": [ + { + "branch": [ + "{\"_label\": \"deep\"}" + ] + } + ] + } + }, + { + "_id": "r24", + "shared": [ + "s1" + ], + "derived": { + "_baseAttr": "b", + "_extraAttr": "e", + "baseField": "base", + "extraField": [ + "x1" + ] + } + }, + { + "_id": "r25", + "shared": [ + "s1", + "s2" + ], + "derived": { + "extraField": [ + "x1", + "x2" + ] + } + } +] diff --git a/tests/data/xsd/spec/10_combinations/expected_schema.py b/tests/data/xsd/spec/10_combinations/expected_schema.py new file mode 100644 index 00000000..dd6c1a2f --- /dev/null +++ b/tests/data/xsd/spec/10_combinations/expected_schema.py @@ -0,0 +1,421 @@ +"""Expected PyArrow schema for 10_combinations (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("plain", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "plainList", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "bounded", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "label", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "labelList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("textNoAttr", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "measure", + pa.struct( + [ + pa.field("_unit", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), + ] + ), + nullable=True, + ), + pa.field( + "flag", + pa.struct( + [ + pa.field("_on", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + ] + ), + nullable=True, + ), + pa.field( + "flagList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_on", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "marker", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + pa.field( + "group", + pa.struct( + [ + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "alias", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "box", + pa.struct( + [ + pa.field( + "item", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "entity", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_name", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("name", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "prop", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_key", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "para", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_style", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "note", + pa.struct( + [ + pa.field("_kind", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field("remark", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "optA", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "optB", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "pairs", + pa.struct( + [ + pa.field( + "k", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "v", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "book", + pa.struct( + [ + pa.field("title", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "chapter", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "title", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_lang", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_source", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "left", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field("leaf", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "right", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field( + "leaf", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_lang", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_source", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "tree", + pa.struct( + [ + pa.field("_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "branch", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "shared", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "derived", + pa.struct( + [ + pa.field("_baseAttr", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_extraAttr", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("baseField", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "extraField", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/xsd/spec/10_combinations/schema.xsd b/tests/data/xsd/spec/10_combinations/schema.xsd new file mode 100644 index 00000000..5a7e157a --- /dev/null +++ b/tests/data/xsd/spec/10_combinations/schema.xsd @@ -0,0 +1,235 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/abstract_element.xsd b/tests/data/xsd/spec/errors/abstract_element.xsd new file mode 100644 index 00000000..50de1258 --- /dev/null +++ b/tests/data/xsd/spec/errors/abstract_element.xsd @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/abstract_type.xsd b/tests/data/xsd/spec/errors/abstract_type.xsd new file mode 100644 index 00000000..7048d5c9 --- /dev/null +++ b/tests/data/xsd/spec/errors/abstract_type.xsd @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd b/tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd new file mode 100644 index 00000000..81eab948 --- /dev/null +++ b/tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/attr_child_name_collision.xsd b/tests/data/xsd/spec/errors/attr_child_name_collision.xsd new file mode 100644 index 00000000..ef3f2f1e --- /dev/null +++ b/tests/data/xsd/spec/errors/attr_child_name_collision.xsd @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/empty_marker_top_level.xsd b/tests/data/xsd/spec/errors/empty_marker_top_level.xsd new file mode 100644 index 00000000..9e640600 --- /dev/null +++ b/tests/data/xsd/spec/errors/empty_marker_top_level.xsd @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/data/xsd/spec/errors/scalar_top_level.xsd b/tests/data/xsd/spec/errors/scalar_top_level.xsd new file mode 100644 index 00000000..dc22ba2c --- /dev/null +++ b/tests/data/xsd/spec/errors/scalar_top_level.xsd @@ -0,0 +1,4 @@ + + + + diff --git a/tests/data/xsd/spec/errors/substitution_group.xsd b/tests/data/xsd/spec/errors/substitution_group.xsd new file mode 100644 index 00000000..638be6da --- /dev/null +++ b/tests/data/xsd/spec/errors/substitution_group.xsd @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/wildcard_any_attribute.xsd b/tests/data/xsd/spec/errors/wildcard_any_attribute.xsd new file mode 100644 index 00000000..e319240f --- /dev/null +++ b/tests/data/xsd/spec/errors/wildcard_any_attribute.xsd @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/tests/data/xsd/spec/errors/wildcard_any_element.xsd b/tests/data/xsd/spec/errors/wildcard_any_element.xsd new file mode 100644 index 00000000..64497408 --- /dev/null +++ b/tests/data/xsd/spec/errors/wildcard_any_element.xsd @@ -0,0 +1,11 @@ + + + + + + + + + + + From 661a4eec0fd667ddc48166d84d1beeaa91daae43 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 12:59:32 -0700 Subject: [PATCH 11/16] ruff format --- .../pipelines/xmltodict/settings.py | 1 - src/cdm_data_loaders/readers/xsd.py | 18 +++++++----------- .../pipelines/xmltodict/conftest.py | 1 - .../xmltodict/integration/conftest.py | 2 -- .../integration/test_xmltodict_end_to_end.py | 2 -- .../pipelines/xmltopyarrow/conftest.py | 1 - .../xmltopyarrow/integration/conftest.py | 1 - .../test_xmltopyarrow_end_to_end.py | 1 - .../test_xmltopyarrow_reference_tests.py | 1 - .../xmltopyarrow/test_xmltopyarrow_pipeline.py | 1 - .../xmltopyarrow/test_xmltopyarrow_settings.py | 1 - .../test_xsd_to_arrow_end_to_end.py | 1 - 12 files changed, 7 insertions(+), 24 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/xmltodict/settings.py b/src/cdm_data_loaders/pipelines/xmltodict/settings.py index 11436f5f..43b44fdf 100644 --- a/src/cdm_data_loaders/pipelines/xmltodict/settings.py +++ b/src/cdm_data_loaders/pipelines/xmltodict/settings.py @@ -45,4 +45,3 @@ class XmlToDictSettings(FileFormatIngestSettings): description="XML tag to capture the contents of", ), ] - diff --git a/src/cdm_data_loaders/readers/xsd.py b/src/cdm_data_loaders/readers/xsd.py index 624aac8a..e766fe9e 100644 --- a/src/cdm_data_loaders/readers/xsd.py +++ b/src/cdm_data_loaders/readers/xsd.py @@ -417,7 +417,10 @@ def _is_empty_marker_complex_type(complex_type: XsdComplexType) -> bool: """Return True if ``complex_type`` has no attributes, no child elements, not simpleContent, and not mixed.""" if complex_type.has_simple_content() or complex_type.mixed: return False - if any(isinstance(a, XsdAttribute) and getattr(a, "use", None) != "prohibited" for a in complex_type.attributes.values()): + if any( + isinstance(a, XsdAttribute) and getattr(a, "use", None) != "prohibited" + for a in complex_type.attributes.values() + ): return False if complex_type.model_group is None: return True @@ -756,9 +759,7 @@ def _serialize_mixed_node(node: Element) -> str: """Serialize an inline child element inside a ``mixed='true'`` element without namespace declarations.""" local = _local_name(node.tag) attrs = "".join( - f' {_local_name(k)}="{_escape_xml_attr(v)}"' - for k, v in node.attrib.items() - if not _is_ignored_attr(k) + f' {_local_name(k)}="{_escape_xml_attr(v)}"' for k, v in node.attrib.items() if not _is_ignored_attr(k) ) inner = _serialize_inner_xml(node) if not inner and len(node) == 0 and node.text is None: @@ -814,11 +815,7 @@ def _coerce_scalar_text(raw_text: str, simple_type: XsdType | None) -> Any: # n def _extract_struct_attributes(element: Element, complex_type: XsdComplexType) -> dict[str, Any]: """Extract and coerce declared XML attributes from ``element`` into ``_`` dictionary entries.""" declared_attrs = {attr.local_name: attr for attr in _collect_attributes(complex_type) if attr.local_name} - raw_attrs = { - _local_name(k): v - for k, v in element.attrib.items() - if not _is_ignored_attr(k) - } + raw_attrs = {_local_name(k): v for k, v in element.attrib.items() if not _is_ignored_attr(k)} return { f"_{attr_name}": _coerce_scalar_text(raw_attrs[attr_name], attr_decl.type) for attr_name, attr_decl in declared_attrs.items() @@ -878,8 +875,7 @@ def _convert_complex_struct( if is_list: result[child_name] = [ - _convert_child_occurrence(ctx, el, child_decl, nullable=True) - for el in matching_elements + _convert_child_occurrence(ctx, el, child_decl, nullable=True) for el in matching_elements ] else: val = _convert_child_occurrence(ctx, matching_elements[0], child_decl, nullable=nullable) diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py index c584d11f..dfa561bc 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/conftest.py @@ -26,4 +26,3 @@ def _factory(**overrides: Any) -> XmlToDictSettings: # noqa: ANN401 return make_cts_settings(XmlToDictSettings, tmp_path, **defaults) return _factory - diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py index 2c277cb6..1a47b253 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/conftest.py @@ -157,5 +157,3 @@ def _factory(chunk_dir: str, **overrides: Any) -> tuple[LoadInfo | None, Path]: return runner(input_dir=str(reference_xml_data_dir / chunk_dir), **overrides) return _factory - - diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py index 3e8957b2..4db0e66d 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_end_to_end.py @@ -111,5 +111,3 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( book_df = check_book_list_results(load_info) assert sorted(book_df["book___id"].tolist()) == ["1", "2", "3"] assert_pipeline_report_written(Path(settings.output_dir), load_info) - - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py index d5808b27..fc225dc4 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py @@ -51,4 +51,3 @@ def _copy(filename: str) -> str: return str(dest) return _copy - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py index 7dcbf0dd..7e295bd7 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py @@ -204,4 +204,3 @@ def normalize_uniref_entry_for_xsd(raw_entry: dict[str, Any]) -> dict[str, Any]: def reconstruct_xsd_entries(tables: dict[str, list[dict[str, Any]]]) -> list[dict[str, Any]]: """Extract normalized XSD entry dicts from the single Arrow-backed entry table.""" return [strip_xsd_arrow_row(row) for row in tables.get("entry", [])] - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py index f39b7b89..8c7724cb 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py @@ -142,4 +142,3 @@ def test_run_xmltopyarrow_pipeline_pass_uniref_chunk_5_el( assert len(loaded_rows) == N_REFERENCE_XML_ENTRIES assert sorted_json(loaded_rows) == sorted_json(expected_rows) assert_pipeline_report_written(Path(settings.output_dir), load_info) - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py index 8c598fb0..3372d831 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_reference_tests.py @@ -168,4 +168,3 @@ def test_xmltopyarrow_pass_reconstructed_matches_duckdb_reference( ] assert len(from_duckdb) == EXPECTED_ENTRY_COUNT assert sorted_json(reconstructed) == sorted_json(from_duckdb) - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py index dfce58ec..e6053cd9 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py @@ -238,4 +238,3 @@ def test_cli_fail_missing_required_args_raise( cli() mock_run_pipeline.assert_not_called() - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py index c7958156..df4ae695 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py @@ -211,4 +211,3 @@ def test_xmltopyarrow_settings_pass_xml_tag_shortcut( settings = CliApp.run(XmlToPyArrowSettings, cli_args=argv) assert settings.file_glob == "*.xml.gz" assert settings.xml_tag == "book" - diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py index 31b2f8ef..11bc8e59 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xsd_to_arrow_end_to_end.py @@ -195,4 +195,3 @@ def test_process_xml_file_to_pyarrow_pass_yields_arrow_tables_for_spec_and_examp actual_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) assert actual_rows == expected - From 4da806563b98017d689781601013289848401268 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 13:06:18 -0700 Subject: [PATCH 12/16] ruff format, rm unused loggers --- docs/xsd_to_pyarrow_specification.md | 318 ++++++++++++------ .../pipelines/ncbi_ftp_promote.py | 3 - .../pipelines/xmltodict/pipeline.py | 3 - .../pipelines/xmltopyarrow/pipeline.py | 3 - .../jsonschema_xsv/xsv_validator/helpers.py | 4 - 5 files changed, 218 insertions(+), 113 deletions(-) diff --git a/docs/xsd_to_pyarrow_specification.md b/docs/xsd_to_pyarrow_specification.md index 59078030..2744c0a9 100644 --- a/docs/xsd_to_pyarrow_specification.md +++ b/docs/xsd_to_pyarrow_specification.md @@ -211,27 +211,37 @@ Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumeratio #### PyArrow Schema ```python -pa.schema([ - pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:ID"}), - pa.field("_status", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), - pa.field("_evidence", pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), nullable=True), - pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), - pa.field("flag", pa.bool_(), nullable=False, metadata={"xsd_type": "xs:boolean"}), - pa.field("smallInt", pa.int8(), nullable=True, metadata={"xsd_type": "xs:byte"}), - pa.field("medInt", pa.int16(), nullable=True, metadata={"xsd_type": "xs:short"}), - pa.field("stdInt", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), - pa.field("bigInt", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"}), - pa.field("ulong", pa.uint64(), nullable=True, metadata={"xsd_type": "xs:unsignedLong"}), - pa.field("f32", pa.float64(), nullable=True, metadata={"xsd_type": "xs:float"}), - pa.field("f64", pa.float64(), nullable=True, metadata={"xsd_type": "xs:double"}), - pa.field("dec", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), - pa.field("created", pa.date32(), nullable=True, metadata={"xsd_type": "xs:date"}), - pa.field("published", pa.string(), nullable=True, metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}), - pa.field("code", pa.string(), nullable=True, metadata={"xsd_type": "xs:int | xs:short"}), - pa.field("scores", pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), nullable=True), - pa.field("rawHex", pa.binary(), nullable=True, metadata={"xsd_type": "xs:hexBinary"}), - pa.field("rawB64", pa.binary(), nullable=True, metadata={"xsd_type": "xs:base64Binary"}), -]) +pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:ID"}), + pa.field("_status", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), + nullable=True, + ), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("flag", pa.bool_(), nullable=False, metadata={"xsd_type": "xs:boolean"}), + pa.field("smallInt", pa.int8(), nullable=True, metadata={"xsd_type": "xs:byte"}), + pa.field("medInt", pa.int16(), nullable=True, metadata={"xsd_type": "xs:short"}), + pa.field("stdInt", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field("bigInt", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"}), + pa.field("ulong", pa.uint64(), nullable=True, metadata={"xsd_type": "xs:unsignedLong"}), + pa.field("f32", pa.float64(), nullable=True, metadata={"xsd_type": "xs:float"}), + pa.field("f64", pa.float64(), nullable=True, metadata={"xsd_type": "xs:double"}), + pa.field("dec", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), + pa.field("created", pa.date32(), nullable=True, metadata={"xsd_type": "xs:date"}), + pa.field("published", pa.string(), nullable=True, metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}), + pa.field("code", pa.string(), nullable=True, metadata={"xsd_type": "xs:int | xs:short"}), + pa.field( + "scores", + pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), + nullable=True, + ), + pa.field("rawHex", pa.binary(), nullable=True, metadata={"xsd_type": "xs:hexBinary"}), + pa.field("rawB64", pa.binary(), nullable=True, metadata={"xsd_type": "xs:base64Binary"}), + ] +) ``` *(In Examples 2–9 below, `metadata={"xsd_type": ...}` on scalar fields is omitted for brevity.)* @@ -324,18 +334,34 @@ Covers `targetNamespace`, `elementFormDefault="qualified"`, stripping `xmlns` / #### PyArrow Schema ```python -pa.schema([ - pa.field("_id", pa.string(), nullable=False), - pa.field("title", pa.string(), nullable=True), - pa.field("label", pa.struct([ - pa.field("_lang", pa.string(), nullable=True), - pa.field("#text", pa.string(), nullable=True), - ]), nullable=True), - pa.field("comment", pa.list_(pa.struct([ - pa.field("_type", pa.string(), nullable=False), - pa.field("text", pa.string(), nullable=True), - ])), nullable=True), -]) +pa.schema( + [ + pa.field("_id", pa.string(), nullable=False), + pa.field("title", pa.string(), nullable=True), + pa.field( + "label", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True), + pa.field("#text", pa.string(), nullable=True), + ] + ), + nullable=True, + ), + pa.field( + "comment", + pa.list_( + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False), + pa.field("text", pa.string(), nullable=True), + ] + ) + ), + nullable=True, + ), + ] +) ``` --- @@ -422,13 +448,23 @@ Covers `mixed="true"` elements with inline HTML/formatting tags (``, ``, ` #### PyArrow Schema ```python -pa.schema([ - pa.field("title", pa.string(), nullable=False), - pa.field("abstractText", pa.list_(pa.struct([ - pa.field("_Label", pa.string(), nullable=True), - pa.field("#text", pa.string(), nullable=True), - ])), nullable=True), -]) +pa.schema( + [ + pa.field("title", pa.string(), nullable=False), + pa.field( + "abstractText", + pa.list_( + pa.struct( + [ + pa.field("_Label", pa.string(), nullable=True), + pa.field("#text", pa.string(), nullable=True), + ] + ) + ), + nullable=True, + ), + ] +) ``` --- @@ -511,16 +547,34 @@ With `max_recursion_depth = 2`: #### PyArrow Schema (`max_recursion_depth=2`) ```python -pa.schema([ - pa.field("_id", pa.string(), nullable=False), - pa.field("tree", pa.struct([ # depth 1 of TreeType - pa.field("_label", pa.string(), nullable=True), - pa.field("branch", pa.list_(pa.struct([ # depth 2 of TreeType - pa.field("_label", pa.string(), nullable=True), - pa.field("branch", pa.list_(pa.string()), nullable=True), # depth > 2 -> JSON string per item - ])), nullable=True), - ]), nullable=True), -]) +pa.schema( + [ + pa.field("_id", pa.string(), nullable=False), + pa.field( + "tree", + pa.struct( + [ # depth 1 of TreeType + pa.field("_label", pa.string(), nullable=True), + pa.field( + "branch", + pa.list_( + pa.struct( + [ # depth 2 of TreeType + pa.field("_label", pa.string(), nullable=True), + pa.field( + "branch", pa.list_(pa.string()), nullable=True + ), # depth > 2 -> JSON string per item + ] + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) ``` --- @@ -648,23 +702,43 @@ Covers: #### PyArrow Schema ```python -pa.schema([ - pa.field("_createdBy", pa.string(), nullable=True), - pa.field("_version", pa.int32(), nullable=True), - pa.field("pagination", pa.struct([ - pa.field("StartPage", pa.string(), nullable=True), - pa.field("EndPage", pa.string(), nullable=True), - pa.field("MedlinePgn", pa.string(), nullable=True), - ]), nullable=True), - pa.field("meta", pa.struct([ - pa.field("author", pa.string(), nullable=False), - pa.field("year", pa.int32(), nullable=True), - ]), nullable=True), - pa.field("pairs", pa.struct([ - pa.field("k", pa.list_(pa.string()), nullable=False), - pa.field("v", pa.list_(pa.int32()), nullable=False), - ]), nullable=True), -]) +pa.schema( + [ + pa.field("_createdBy", pa.string(), nullable=True), + pa.field("_version", pa.int32(), nullable=True), + pa.field( + "pagination", + pa.struct( + [ + pa.field("StartPage", pa.string(), nullable=True), + pa.field("EndPage", pa.string(), nullable=True), + pa.field("MedlinePgn", pa.string(), nullable=True), + ] + ), + nullable=True, + ), + pa.field( + "meta", + pa.struct( + [ + pa.field("author", pa.string(), nullable=False), + pa.field("year", pa.int32(), nullable=True), + ] + ), + nullable=True, + ), + pa.field( + "pairs", + pa.struct( + [ + pa.field("k", pa.list_(pa.string()), nullable=False), + pa.field("v", pa.list_(pa.int32()), nullable=False), + ] + ), + nullable=True, + ), + ] +) ``` --- @@ -760,21 +834,55 @@ Because the full ancestor path from `xml_tag` (`("root", "left", "node")` vs `(" #### PyArrow Schema ```python -pa.schema([ - pa.field("left", pa.struct([ - pa.field("node", pa.struct([ - pa.field("leaf", pa.string(), nullable=False), - ]), nullable=False), - ]), nullable=False), - pa.field("right", pa.struct([ - pa.field("node", pa.struct([ - pa.field("leaf", pa.list_(pa.struct([ - pa.field("_lang", pa.string(), nullable=True), - pa.field("#text", pa.string(), nullable=True), - ])), nullable=False), - ]), nullable=False), - ]), nullable=False), -]) +pa.schema( + [ + pa.field( + "left", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field("leaf", pa.string(), nullable=False), + ] + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + pa.field( + "right", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field( + "leaf", + pa.list_( + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True), + pa.field("#text", pa.string(), nullable=True), + ] + ) + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + ] +) ``` --- @@ -854,19 +962,27 @@ Covers: #### PyArrow Schema ```python -pa.schema([ - pa.field("_id", pa.string(), nullable=False), # use="required" - pa.field("_source", pa.string(), nullable=True), # use="optional" - pa.field("name", pa.string(), nullable=False), # minOccurs=1, non-nillable - pa.field("summary", pa.string(), nullable=False), # minOccurs=1, non-nillable - pa.field("optionalNote", pa.string(), nullable=True), # minOccurs=0 - pa.field("geneId", pa.int32(), nullable=True), # inside xs:choice -> nullable=True - pa.field("proteinId", pa.string(), nullable=True), # inside xs:choice -> nullable=True - pa.field("comment", pa.struct([ - pa.field("_type", pa.string(), nullable=False), # use="required" - pa.field("text", pa.string(), nullable=True), # parent is nillable="true" -> nullable=True - ]), nullable=True), # nillable="true" -> nullable=True -]) +pa.schema( + [ + pa.field("_id", pa.string(), nullable=False), # use="required" + pa.field("_source", pa.string(), nullable=True), # use="optional" + pa.field("name", pa.string(), nullable=False), # minOccurs=1, non-nillable + pa.field("summary", pa.string(), nullable=False), # minOccurs=1, non-nillable + pa.field("optionalNote", pa.string(), nullable=True), # minOccurs=0 + pa.field("geneId", pa.int32(), nullable=True), # inside xs:choice -> nullable=True + pa.field("proteinId", pa.string(), nullable=True), # inside xs:choice -> nullable=True + pa.field( + "comment", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False), # use="required" + pa.field("text", pa.string(), nullable=True), # parent is nillable="true" -> nullable=True + ] + ), + nullable=True, + ), # nillable="true" -> nullable=True + ] +) ``` --- @@ -931,11 +1047,13 @@ Covers DTD-derived empty marker elements like [`pubmed.xsd` L1312](file:///Users #### PyArrow Schema ```python -pa.schema([ - pa.field("title", pa.string(), nullable=False), - pa.field("BookInProcess", pa.string(), nullable=True), - pa.field("flag", pa.list_(pa.string()), nullable=True), -]) +pa.schema( + [ + pa.field("title", pa.string(), nullable=False), + pa.field("BookInProcess", pa.string(), nullable=True), + pa.field("flag", pa.list_(pa.string()), nullable=True), + ] +) ``` --- diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py index 33f17387..8d111629 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_promote.py @@ -4,7 +4,6 @@ final location in the Lakehouse. """ -import logging from pathlib import Path, PurePosixPath from typing import Final @@ -22,8 +21,6 @@ DEFAULT_TRANSFER_MANIFEST_FILE: PurePosixPath = PurePosixPath("transfer_manifest.txt") ESTIMATED_FILES_PER_ASSEMBLY: int = 21 -logger = logging.getLogger("dlt") - class PromoteSettings(LoggerSettings): """Configuration for the NCBI FTP file promotion pipeline.""" diff --git a/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py b/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py index ea22f315..d7b4afc2 100644 --- a/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py @@ -1,7 +1,6 @@ """XML to dictionary pipeline for the KBase CTS.""" from collections.abc import Generator, Iterator -from logging import Logger, getLogger from pathlib import Path from typing import TYPE_CHECKING, Any @@ -18,8 +17,6 @@ if TYPE_CHECKING: from dlt.extract import DltResource -logger: Logger = getLogger(__name__) - def _read_xmltodict_items( items: Iterator[FileItemDict], settings: XmlToDictSettings diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py index 78f80769..4dfabd81 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py @@ -1,7 +1,6 @@ """XML to pyarrow pipeline for the KBase CTS.""" from collections.abc import Generator, Iterator -from logging import Logger, getLogger from pathlib import Path from typing import TYPE_CHECKING, Any @@ -18,8 +17,6 @@ if TYPE_CHECKING: from dlt.extract import DltResource -logger: Logger = getLogger(__name__) - def _read_items(items: Iterator[FileItemDict], settings: XmlToPyArrowSettings) -> Generator[TDataItems, Any, Any]: """Read each file in items. Yields PyArrow tables for matched `xml_tag` elements. diff --git a/src/cdm_data_loaders/readers/jsonschema_xsv/xsv_validator/helpers.py b/src/cdm_data_loaders/readers/jsonschema_xsv/xsv_validator/helpers.py index d8a7c54b..d839a49d 100644 --- a/src/cdm_data_loaders/readers/jsonschema_xsv/xsv_validator/helpers.py +++ b/src/cdm_data_loaders/readers/jsonschema_xsv/xsv_validator/helpers.py @@ -4,7 +4,6 @@ import subprocess from dataclasses import dataclass from functools import cached_property -from logging import Logger, getLogger from pathlib import Path from shutil import copy, move from typing import Annotated, Final, Self @@ -35,9 +34,6 @@ VALID_SUFFIX: Final[str] = "-valid" -logger: Logger = getLogger(__name__) - - class ErrorRecord(BaseModel): """A single error encountered while cleaning/validating a file. From 80a500ef3cfe7f7cbf6e335bfa0e7f00d79c12f0 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 13:12:29 -0700 Subject: [PATCH 13/16] add in missing resources --- tests/data/pubmed/pubmed.xsd | 1473 +++++++++++++++++++++ tests/data/uniprot/uniprot_kb/uniprot.xsd | 1125 ++++++++++++++++ 2 files changed, 2598 insertions(+) create mode 100644 tests/data/pubmed/pubmed.xsd create mode 100644 tests/data/uniprot/uniprot_kb/uniprot.xsd diff --git a/tests/data/pubmed/pubmed.xsd b/tests/data/pubmed/pubmed.xsd new file mode 100644 index 00000000..9bd92936 --- /dev/null +++ b/tests/data/pubmed/pubmed.xsd @@ -0,0 +1,1473 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/uniprot/uniprot_kb/uniprot.xsd b/tests/data/uniprot/uniprot_kb/uniprot.xsd new file mode 100644 index 00000000..e79ba5ea --- /dev/null +++ b/tests/data/uniprot/uniprot_kb/uniprot.xsd @@ -0,0 +1,1125 @@ + + + + + + + + + + Describes a collection of UniProtKB entries. + + + + + + + + + + + + + + Describes a UniProtKB entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes the names for the protein and parts thereof. + Equivalent to the flat file DE-line. + + + + + + Describes names of "domains". + Equivalent to the flat file DE-line Includes: section. + + + + + + + + Describes names of processed products. + Equivalent to the flat file DE-line Contains: section. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a gene. + Equivalent to the flat file GN-line. + + + + + + + + Describes different types of gene designations. + Equivalent to the flat file GN-line. + + + + + + + + + + + + + + + + + + + + + + + Describes the source organism. + + + + + Describes the names of the source organism. + Equivalent to the flat file OS-line. + + + + + Describes a cross-reference to the NCBI taxonomy database. + Equivalent to the flat file OX-line. + + + + + Describes the lineage of the source organism. + Equivalent to the flat file OC-line. + + + + + + + + + + + + + Describes different types of source organism names. + + + + + + + + + + + + + + + + + + + + + + + Describes non-nuclear gene locations (organelles and plasmids). + Equivalent to the flat file OG-line. + + + + + + + + + + + + + + + + + + + + + + + + + Indicates whether the name of a plasmid is known or unknown. + + + + + + + + + + + + + + + + + + + + Describes a citation and a summary of its content. + Equivalent to the flat file RN-, RP-, RC-, RX-, RG-, RA-, RT- and RL-lines. + + + + + + + + + + + + + + Describes different types of citations. + Equivalent to the flat file RX-, RG-, RA-, RT- and RL-lines. + + + + + Describes the title of a citation. + Equivalent to the flat file RT-line. + + + + + Describes the editors of a book (only used for books). + Equivalent to part of the flat file RL-line of books. + + + + + Describes the authors of a citation. + Equivalent to the flat file RA-line. + + + + + Describes the location (URL) of an online journal article. + No flat file equivalent. + + + + + Describes cross-references to bibliography databases (MEDLINE, PubMed, AGRICOLA) or other online resources (DOI). + Equivalent to the flat file RX-line. + + + + + + Describes the type of a citation. + + + + + + + + + + + + + + + + + + + + + Describes the name of an (online) journal or book. + + + + + Describes the volume of a journal or book. + + + + + Describes the first page of an article. + + + + + Describes the last page of an article. + + + + + Describes the publisher of a book. + + + + + Describes the city where a book was published. + + + + + Describes the database name of submissions. + + + + + Describes a patent number. + + + + + Describes the institute where a thesis was made. + + + + + Describes the country where a thesis was made. + + + + + + Describes the authors of a citation when these are represented by a consortium. + Equivalent to the flat file RG-line. + + + + + + + + + + + + + + + + + + Groups a citation's scope and source descriptions. + + + + + Describes the scope of a citation. + Equivalent to the flat file RP-line. + + + + + + + + Describes the source of the sequence according to the citation. + Equivalent to the flat file RC-line. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes different types of general annotations. + Equivalent to the flat file CC-line. + + + + + + + + + Used in 'catalytic activity' annotations. + + + + + + + + Used in 'cofactor' annotations. + + + + + + + Used in 'subcellular location' annotations. + + + + + + + Used in 'sequence caution' annotations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Refers to the 'key' attribute of a 'reference' element. + + + + + + + + + Used in 'online information' annotations. + + + + + + + + + + Used in 'alternative products' annotations. + + + + + + + + Used in 'interaction' annotations. + + + + + + + + + Used in 'disease' annotations. + + + + + + + + + + + + + + + + + Used in 'mass spectrometry' and 'sequence caution' annotations. + + + + + + + + + + Describes the type of a general annotation. + Equivalent to the flat file CC comment topics (except for "DATABASE" which is translated to "online information"). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes the type of sequence location in 'RNA editing' annotations. Common values are "Not_applicable" and "Undetermined". + + + + + + Describes an optional name for an 'online information'. + + + + + + Describes the molecular mass in 'mass spectrometry' annotations. + + + + + Describes the error of the mass measurement in 'mass spectrometry' annotations. + + + + + Describes the experimental method in 'mass spectrometry' annotations. + + + + + + + + + Describes different types of biophysicochemical properties. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a chemical reaction. + + + + + + + + + + + Describes a physiological reaction. + + + + + + + + + + + + + + + + + + Describes a cofactor. + + + + + + + + + + + Describes the subcellular location and optionally the topology and orientation of a molecule. + + + + + + + + + + + Describes the type of events that cause alternative products. + + + + + + + + + + + + + + + + Describes isoforms in 'alternative products' annotations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a database cross-reference. + Equivalent to the flat file DR-line. + + + + + + + + + Describes the name of the database. + + + + + Describes a unique database identifier. + + + + + + + + + + + + + + + + Describes the evidence for the protein's existence. + Equivalent to the flat file PE-line. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes different types of sequence annotations. + Equivalent to the flat file FT-line. + + + + + Describes the original sequence in annotations that describe natural or artifical sequence variations. + + + + + Describes the variant sequence in annotations that describe natural or artifical sequence variations. + + + + + Describes the sequence coordinates of the annotation. + + + + + Describes the chemical entity that is bound in annotations that describe binding sites. + + + + + Describes the specific part of a molecule that is bound in annotations that describe binding sites. + + + + + + Describes the type of a sequence annotation. + Equivalent to the flat file FT feature keys, but using full terms instead of acronyms. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a sequence location as either a range with a begin and end or as a position. The 'sequence' attribute is only used when the location is not on the canonical sequence displayed in the current entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a ligand. + + + + + + + + + + + + Describes a ligand part. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a molecule by name or unique identifier. + + + + + + + + + + + + + Describes the evidence for an annotation. + No flat file equivalent. + + + + + + + + Describes the type of an evidence using the Evidence Code Ontology (http://www.obofoundry.org/cgi-bin/detail.cgi?id=evidence_code). + + + + + A unique key to link annotations (via 'evidence' attributes) to evidences. + + + + + + Describes the source of the data using a database cross-reference (or a 'ref' attribute when the source cannot be found in a public data source, such as PubMed, and is cited only within the UniProtKB entry). + + + + + + + + + Describes the source of the evidence, when it is not assigned by UniProt, but imported from an external database. + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 533a6776385542761de8d9cec599d5fd1f31061c Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 16:31:38 -0700 Subject: [PATCH 14/16] Separating uniprot and uniref pipelines out; adding schemas. Refactoring some tests --- CHANGELOG.md | 10 + pyproject.toml | 4 +- .../pipelines/xmltopyarrow/pipeline.py | 3 +- .../pipelines/xmltopyarrow/settings.py | 119 +- .../pipelines/xmltopyarrow/uniprot.py | 33 + .../pipelines/xmltopyarrow/uniref.py | 33 + src/cdm_data_loaders/schemas/__init__.py | 0 .../schemas/uniprot/__init__.py | 0 .../schemas/uniprot/schema.py | 4344 +++++++++++++++++ .../schemas/uniprot/uniprot.xsd | 1125 +++++ .../schemas/uniref/__init__.py | 0 src/cdm_data_loaders/schemas/uniref/schema.py | 186 + .../schemas/uniref/uniref.xsd | 116 + tests/cdm_data_loaders/pipelines/framework.py | 2 + tests/cdm_data_loaders/pipelines/helpers.py | 12 +- .../pipelines/xmltopyarrow/conftest.py | 28 +- .../xmltopyarrow/integration/conftest.py | 22 +- .../test_xmltopyarrow_end_to_end.py | 39 +- .../test_xmltopyarrow_pipeline.py | 221 +- .../test_xmltopyarrow_settings.py | 106 +- 20 files changed, 6224 insertions(+), 179 deletions(-) create mode 100644 src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py create mode 100644 src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py create mode 100644 src/cdm_data_loaders/schemas/__init__.py create mode 100644 src/cdm_data_loaders/schemas/uniprot/__init__.py create mode 100644 src/cdm_data_loaders/schemas/uniprot/schema.py create mode 100644 src/cdm_data_loaders/schemas/uniprot/uniprot.xsd create mode 100644 src/cdm_data_loaders/schemas/uniref/__init__.py create mode 100644 src/cdm_data_loaders/schemas/uniref/schema.py create mode 100644 src/cdm_data_loaders/schemas/uniref/uniref.xsd diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f70e06..4a7d150e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # CDM Data Loaders Changelog - [CDM Data Loaders Changelog](#cdm-data-loaders-changelog) + - [v0.1.12](#v0112) + - [v0.1.11](#v0111) - [v0.1.10](#v0110) - [v0.1.9](#v019) - [v0.1.8](#v018) @@ -14,6 +16,14 @@ - [v0.1.0](#v010) +### v0.1.12 + +- Dedicated UniProt and UniRef XML import pipelines using custom XSD-aware XML-to-JSON converter. + +### v0.1.11 + +- Preliminary XSV, XML, and JSONLines importers + ### v0.1.10 - Add in the ability to configure loggers using a file diff --git a/pyproject.toml b/pyproject.toml index 76631030..ab5a4093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cdm-data-loaders" -version = "0.1.11" +version = "0.1.12" description = "Data loaders and wranglers for the CDM." requires-python = ">= 3.13" readme = "README.md" @@ -46,7 +46,9 @@ ncbi_ftp_promote = "cdm_data_loaders.pipelines.ncbi_ftp_promote:cli" ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" pdb_manifest = "cdm_data_loaders.pipelines.pdb_manifest:cli" +uniprot = "cdm_data_loaders.pipelines.xmltopyarrow.uniprot:cli" uniprot_cdm = "cdm_data_loaders.pipelines.uniprot_kb_cdm:cli" +uniref = "cdm_data_loaders.pipelines.xmltopyarrow.uniref:cli" uniref_cdm = "cdm_data_loaders.pipelines.uniref_cdm:cli" xmltodict = "cdm_data_loaders.pipelines.xmltodict.pipeline:cli" xmltopyarrow = "cdm_data_loaders.pipelines.xmltopyarrow.pipeline:cli" diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py index 4dfabd81..94aeb548 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py @@ -42,12 +42,13 @@ def run_xmltopyarrow_pipeline(settings: XmlToPyArrowSettings) -> LoadInfo | None files = filesystem(bucket_url=str(settings.input_dir), file_glob=settings.file_glob) xmltopyarrow_resource: DltResource = files | reader(settings) + pipeline_name = str(settings.model_config.get("cli_prog_name") or PIPELINE_NAME) return run_pipeline( settings=settings, resource=xmltopyarrow_resource, pipeline_kwargs={ - "pipeline_name": PIPELINE_NAME, + "pipeline_name": pipeline_name, "dataset_name": settings.dataset_name, }, pipeline_run_kwargs={ diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py index e5c1f2e4..4db39d25 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py @@ -1,5 +1,6 @@ """Settings class for the XML to PyArrow pipeline for the KBase CTS.""" +from pathlib import Path from typing import Annotated, Final, Self import pyarrow as pa @@ -10,6 +11,7 @@ from cdm_data_loaders.core.fields import ( DEFAULT_XML_FILE_GLOB, FILE_GLOB, + DatasetName, FileGlob, NonEmptyStr, TableName, @@ -23,8 +25,19 @@ generate_pyarrow_schema, load_schema, ) +from cdm_data_loaders.schemas.uniprot.schema import SCHEMA as UNIPROT_SCHEMA +from cdm_data_loaders.schemas.uniref.schema import SCHEMA as UNIREF_SCHEMA PIPELINE_NAME: Final[str] = "xmltopyarrow" +UNIPROT_PIPELINE_NAME: Final[str] = "uniprot" +UNIREF_PIPELINE_NAME: Final[str] = "uniref" + +ENTRY_XML_TAG: Final[str] = "entry" +ENTRY_TABLE_NAME: Final[str] = "entry" + +SCHEMAS_DIR: Final[Path] = Path(__file__).resolve().parents[2] / "schemas" +UNIPROT_XSD_FILE: Final[Path] = SCHEMAS_DIR / "uniprot" / "uniprot.xsd" +UNIREF_XSD_FILE: Final[Path] = SCHEMAS_DIR / "uniref" / "uniref.xsd" class XmlToPyArrowSettings(FileFormatIngestSettings): @@ -70,20 +83,24 @@ class XmlToPyArrowSettings(FileFormatIngestSettings): @model_validator(mode="after") def check_xsd_file(self) -> Self: - """Check whether the schema file is present and valid; load the XSD and generate the PyArrow schema.""" + """Check whether the schema file is present and valid; load the XSD and resolve the PyArrow schema.""" try: self._xsd_schema = load_schema(self.xsd_file) - self._pyarrow_schema = generate_pyarrow_schema( - self._xsd_schema, - self.xml_tag, - max_recursion_depth=self.max_recursion_depth, - ) + self._pyarrow_schema = self._resolve_pyarrow_schema() except Exception as e: err_msg = f"Could not generate PyArrow schema from XSD: {e!s}" raise ValueError(err_msg) from e return self + def _resolve_pyarrow_schema(self) -> pa.Schema: + """Return the PyArrow schema for this settings instance.""" + return generate_pyarrow_schema( + self._xsd_schema, + self.xml_tag, + max_recursion_depth=self.max_recursion_depth, + ) + @property def xsd_schema(self) -> xmlschema.XMLSchema: """Return the loaded XMLSchema instance.""" @@ -93,3 +110,93 @@ def xsd_schema(self) -> xmlschema.XMLSchema: def pyarrow_schema(self) -> pa.Schema: """Return the generated PyArrow schema for xml_tag.""" return self._pyarrow_schema + + +class UniProtXmlToPyArrowSettings(XmlToPyArrowSettings): + """Settings for the UniProt XML to PyArrow ingestion pipeline.""" + + model_config: SettingsConfigDict = default_settings_with_shortcuts( + cli_prog_name=UNIPROT_PIPELINE_NAME, + cli_shortcuts={ + **FileFormatIngestSettings.model_config["cli_shortcuts"], + FILE_GLOB: "g", + }, + ) + + dataset_name: Annotated[ + DatasetName, + Field( + default=UNIPROT_PIPELINE_NAME, + description="The name of the dataset being produced", + ), + ] + table_name: Annotated[ + TableName, + Field( + default=ENTRY_TABLE_NAME, + description="The name of the table to export parsed data to", + ), + ] + xml_tag: Annotated[ + NonEmptyStr, + Field( + default=ENTRY_XML_TAG, + description="XML tag to capture the contents of", + ), + ] + xsd_file: Annotated[ + FilePath, + Field( + default=UNIPROT_XSD_FILE, + description="Path to the UniProt XSD file describing the XML documents to be imported.", + ), + ] + + def _resolve_pyarrow_schema(self) -> pa.Schema: + """Return the precomputed UniProt PyArrow schema.""" + return UNIPROT_SCHEMA + + +class UniRefXmlToPyArrowSettings(XmlToPyArrowSettings): + """Settings for the UniRef XML to PyArrow ingestion pipeline.""" + + model_config: SettingsConfigDict = default_settings_with_shortcuts( + cli_prog_name=UNIREF_PIPELINE_NAME, + cli_shortcuts={ + **FileFormatIngestSettings.model_config["cli_shortcuts"], + FILE_GLOB: "g", + }, + ) + + dataset_name: Annotated[ + DatasetName, + Field( + default=UNIREF_PIPELINE_NAME, + description="The name of the dataset being produced", + ), + ] + table_name: Annotated[ + TableName, + Field( + default=ENTRY_TABLE_NAME, + description="The name of the table to export parsed data to", + ), + ] + xml_tag: Annotated[ + NonEmptyStr, + Field( + default=ENTRY_XML_TAG, + description="XML tag to capture the contents of", + ), + ] + xsd_file: Annotated[ + FilePath, + Field( + default=UNIREF_XSD_FILE, + description="Path to the UniRef XSD file describing the XML documents to be imported.", + ), + ] + + def _resolve_pyarrow_schema(self) -> pa.Schema: + """Return the precomputed UniRef PyArrow schema.""" + return UNIREF_SCHEMA diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py new file mode 100644 index 00000000..6ab31561 --- /dev/null +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py @@ -0,0 +1,33 @@ +"""UniProt XML to PyArrow pipeline for the KBase CTS.""" + +from dlt.common.pipeline import LoadInfo + +from cdm_data_loaders.pipelines.core import PipelineCli +from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UNIPROT_PIPELINE_NAME as PIPELINE_NAME, +) +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UniProtXmlToPyArrowSettings, +) + +__all__ = ["PIPELINE_NAME", "UniProtXmlToPyArrowSettings", "cli", "run_uniprot_pipeline"] + + +def run_uniprot_pipeline(settings: UniProtXmlToPyArrowSettings) -> LoadInfo | None: + """Run the UniProt XML to PyArrow pipeline on matching files in settings.input_dir. + + :param settings: pipeline configuration + :type settings: UniProtXmlToPyArrowSettings + :return: load information for the pipeline + :rtype: LoadInfo | None + """ + return run_xmltopyarrow_pipeline(settings) + + +cli = PipelineCli(UniProtXmlToPyArrowSettings, run_uniprot_pipeline) + + +if __name__ == "__main__": + cli() + diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py new file mode 100644 index 00000000..bf36d1ff --- /dev/null +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py @@ -0,0 +1,33 @@ +"""UniRef XML to PyArrow pipeline for the KBase CTS.""" + +from dlt.common.pipeline import LoadInfo + +from cdm_data_loaders.pipelines.core import PipelineCli +from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UNIREF_PIPELINE_NAME as PIPELINE_NAME, +) +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UniRefXmlToPyArrowSettings, +) + +__all__ = ["PIPELINE_NAME", "UniRefXmlToPyArrowSettings", "cli", "run_uniref_pipeline"] + + +def run_uniref_pipeline(settings: UniRefXmlToPyArrowSettings) -> LoadInfo | None: + """Run the UniRef XML to PyArrow pipeline on matching files in settings.input_dir. + + :param settings: pipeline configuration + :type settings: UniRefXmlToPyArrowSettings + :return: load information for the pipeline + :rtype: LoadInfo | None + """ + return run_xmltopyarrow_pipeline(settings) + + +cli = PipelineCli(UniRefXmlToPyArrowSettings, run_uniref_pipeline) + + +if __name__ == "__main__": + cli() + diff --git a/src/cdm_data_loaders/schemas/__init__.py b/src/cdm_data_loaders/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/schemas/uniprot/__init__.py b/src/cdm_data_loaders/schemas/uniprot/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/schemas/uniprot/schema.py b/src/cdm_data_loaders/schemas/uniprot/schema.py new file mode 100644 index 00000000..dfdc94fb --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniprot/schema.py @@ -0,0 +1,4344 @@ +"""PyArrow schema for uniprot (xml_tag='entry').""" + +import pyarrow as pa + +SCHEMA = pa.schema( + [ + pa.field("_dataset", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_created", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_modified", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field( + "accession", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "name", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "protein", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "domain", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": 'Describes names of "domains". Equivalent to the flat file DE-line Includes: section.' + }, + ), + pa.field( + "component", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes names of processed products. Equivalent to the flat file DE-line Contains: section." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes the names for the protein and parts thereof. Equivalent to the flat file DE-line." + }, + ), + pa.field( + "gene", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes different types of gene designations. Equivalent to the flat file GN-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a gene. Equivalent to the flat file GN-line."}, + ), + pa.field( + "organism", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes the names of the source organism. Equivalent to the flat file OS-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a cross-reference to the NCBI taxonomy database. Equivalent to the flat file OX-line." + }, + ), + pa.field( + "lineage", + pa.struct( + [ + pa.field( + "taxon", + pa.list_( + pa.field( + "item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the lineage of the source organism. Equivalent to the flat file OC-line." + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Describes the source organism."}, + ), + pa.field( + "organismHost", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes the names of the source organism. Equivalent to the flat file OS-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a cross-reference to the NCBI taxonomy database. Equivalent to the flat file OX-line." + }, + ), + pa.field( + "lineage", + pa.struct( + [ + pa.field( + "taxon", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the lineage of the source organism. Equivalent to the flat file OC-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the source organism."}, + ), + pa.field( + "geneLocation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Indicates whether the name of a plasmid is known or unknown."}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes non-nuclear gene locations (organelles and plasmids). Equivalent to the flat file OG-line." + }, + ), + pa.field( + "reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_key", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "citation", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of a citation.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_date", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}, + ), + pa.field( + "_name", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the name of an (online) journal or book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_volume", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the volume of a journal or book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_first", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the first page of an article.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_last", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the last page of an article.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_publisher", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the publisher of a book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_city", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the city where a book was published.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_db", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the database name of submissions.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_number", + pa.string(), + nullable=True, + metadata={"comment": "Describes a patent number.", "xsd_type": "xs:string"}, + ), + pa.field( + "_institute", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the institute where a thesis was made.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_country", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the country where a thesis was made.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "title", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the title of a citation. Equivalent to the flat file RT-line.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "editorList", + pa.struct( + [ + pa.field( + "consortium", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation when these are represented by a consortium. Equivalent to the flat file RG-line." + }, + ), + pa.field( + "person", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the editors of a book (only used for books). Equivalent to part of the flat file RL-line of books." + }, + ), + pa.field( + "authorList", + pa.struct( + [ + pa.field( + "consortium", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation when these are represented by a consortium. Equivalent to the flat file RG-line." + }, + ), + pa.field( + "person", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation. Equivalent to the flat file RA-line." + }, + ), + pa.field( + "locator", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the location (URL) of an online journal article. No flat file equivalent.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes cross-references to bibliography databases (MEDLINE, PubMed, AGRICOLA) or other online resources (DOI). Equivalent to the flat file RX-line." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes different types of citations. Equivalent to the flat file RX-, RG-, RA-, RT- and RL-lines." + }, + ), + pa.field( + "scope", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + metadata={ + "comment": "Describes the scope of a citation. Equivalent to the flat file RP-line." + }, + ), + pa.field( + "source", + pa.struct( + [ + pa.field( + "strain", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "plasmid", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "transposon", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "tissue", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the sequence according to the citation. Equivalent to the flat file RC-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a citation and a summary of its content. Equivalent to the flat file RN-, RP-, RC-, RX-, RG-, RA-, RT- and RL-lines." + }, + ), + pa.field( + "comment", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=True, + metadata={ + "comment": 'Describes the type of a general annotation. Equivalent to the flat file CC comment topics (except for "DATABASE" which is translated to "online information").', + "xsd_type": "xs:string", + }, + ), + pa.field( + "_locationType", + pa.string(), + nullable=True, + metadata={ + "comment": 'Describes the type of sequence location in \'RNA editing\' annotations. Common values are "Not_applicable" and "Undetermined".', + "xsd_type": "xs:string", + }, + ), + pa.field( + "_name", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes an optional name for an 'online information'.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_mass", + pa.float64(), + nullable=True, + metadata={ + "comment": "Describes the molecular mass in 'mass spectrometry' annotations.", + "xsd_type": "xs:float", + }, + ), + pa.field( + "_error", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the error of the mass measurement in 'mass spectrometry' annotations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_method", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the experimental method in 'mass spectrometry' annotations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "absorption", + pa.struct( + [ + pa.field( + "max", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "kinetics", + pa.struct( + [ + pa.field( + "KM", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "Vmax", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "phDependence", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "redoxPotential", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "temperatureDependence", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "reaction", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "text", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a chemical reaction."}, + ), + pa.field( + "physiologicalReaction", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_direction", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a physiological reaction."}, + ), + pa.field( + "cofactor", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a cofactor."}, + ), + pa.field( + "subcellularLocation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "location", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "topology", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "orientation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the subcellular location and optionally the topology and orientation of a molecule." + }, + ), + pa.field( + "conflict", + pa.struct( + [ + pa.field( + "_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_ref", + pa.string(), + nullable=True, + metadata={ + "comment": "Refers to the 'key' attribute of a 'reference' element.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_resource", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_version", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={"comment": "Used in 'sequence caution' annotations."}, + ), + pa.field( + "link", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_uri", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:anyURI"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Used in 'online information' annotations."}, + ), + pa.field( + "event", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the type of events that cause alternative products."}, + ), + pa.field( + "isoform", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "id", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_ref", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes isoforms in 'alternative products' annotations."}, + ), + pa.field( + "interactant", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_intactId", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("organismsDiffer", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + pa.field("experiments", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "disease", + pa.struct( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "acronym", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "description", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={"comment": "Used in 'disease' annotations."}, + ), + pa.field( + "location", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_sequence", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "begin", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "position", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Used in 'mass spectrometry' and 'sequence caution' annotations."}, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes different types of general annotations. Equivalent to the flat file CC-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"comment": "Describes the name of the database.", "xsd_type": "xs:string"}, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a database cross-reference. Equivalent to the flat file DR-line."}, + ), + pa.field( + "proteinExistence", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + metadata={ + "comment": "Describes the evidence for the protein's existence. Equivalent to the flat file PE-line." + }, + ), + pa.field( + "keyword", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "feature", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of a sequence annotation. Equivalent to the flat file FT feature keys, but using full terms instead of acronyms.", + "xsd_type": "xs:string", + }, + ), + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_description", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_ref", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "original", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the original sequence in annotations that describe natural or artifical sequence variations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "variation", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + metadata={ + "comment": "Describes the variant sequence in annotations that describe natural or artifical sequence variations." + }, + ), + pa.field( + "location", + pa.struct( + [ + pa.field( + "_sequence", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "begin", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "position", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Describes the sequence coordinates of the annotation."}, + ), + pa.field( + "ligand", + pa.struct( + [ + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + pa.field( + "label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "note", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the chemical entity that is bound in annotations that describe binding sites." + }, + ), + pa.field( + "ligandPart", + pa.struct( + [ + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + pa.field( + "label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "note", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the specific part of a molecule that is bound in annotations that describe binding sites." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes different types of sequence annotations. Equivalent to the flat file FT-line." + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of an evidence using the Evidence Code Ontology (http://www.obofoundry.org/cgi-bin/detail.cgi?id=evidence_code).", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_key", + pa.int64(), + nullable=False, + metadata={ + "comment": "A unique key to link annotations (via 'evidence' attributes) to evidences.", + "xsd_type": "xs:integer", + }, + ), + pa.field( + "source", + pa.struct( + [ + pa.field( + "_ref", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the data using a database cross-reference (or a 'ref' attribute when the source cannot be found in a public data source, such as PubMed, and is cited only within the UniProtKB entry)." + }, + ), + pa.field( + "importedFrom", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the evidence, when it is not assigned by UniProt, but imported from an external database." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the evidence for an annotation. No flat file equivalent."}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field("_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_mass", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_modified", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_precursor", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + pa.field("_fragment", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + ] +) diff --git a/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd b/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd new file mode 100644 index 00000000..e79ba5ea --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd @@ -0,0 +1,1125 @@ + + + + + + + + + + Describes a collection of UniProtKB entries. + + + + + + + + + + + + + + Describes a UniProtKB entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes the names for the protein and parts thereof. + Equivalent to the flat file DE-line. + + + + + + Describes names of "domains". + Equivalent to the flat file DE-line Includes: section. + + + + + + + + Describes names of processed products. + Equivalent to the flat file DE-line Contains: section. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a gene. + Equivalent to the flat file GN-line. + + + + + + + + Describes different types of gene designations. + Equivalent to the flat file GN-line. + + + + + + + + + + + + + + + + + + + + + + + Describes the source organism. + + + + + Describes the names of the source organism. + Equivalent to the flat file OS-line. + + + + + Describes a cross-reference to the NCBI taxonomy database. + Equivalent to the flat file OX-line. + + + + + Describes the lineage of the source organism. + Equivalent to the flat file OC-line. + + + + + + + + + + + + + Describes different types of source organism names. + + + + + + + + + + + + + + + + + + + + + + + Describes non-nuclear gene locations (organelles and plasmids). + Equivalent to the flat file OG-line. + + + + + + + + + + + + + + + + + + + + + + + + + Indicates whether the name of a plasmid is known or unknown. + + + + + + + + + + + + + + + + + + + + Describes a citation and a summary of its content. + Equivalent to the flat file RN-, RP-, RC-, RX-, RG-, RA-, RT- and RL-lines. + + + + + + + + + + + + + + Describes different types of citations. + Equivalent to the flat file RX-, RG-, RA-, RT- and RL-lines. + + + + + Describes the title of a citation. + Equivalent to the flat file RT-line. + + + + + Describes the editors of a book (only used for books). + Equivalent to part of the flat file RL-line of books. + + + + + Describes the authors of a citation. + Equivalent to the flat file RA-line. + + + + + Describes the location (URL) of an online journal article. + No flat file equivalent. + + + + + Describes cross-references to bibliography databases (MEDLINE, PubMed, AGRICOLA) or other online resources (DOI). + Equivalent to the flat file RX-line. + + + + + + Describes the type of a citation. + + + + + + + + + + + + + + + + + + + + + Describes the name of an (online) journal or book. + + + + + Describes the volume of a journal or book. + + + + + Describes the first page of an article. + + + + + Describes the last page of an article. + + + + + Describes the publisher of a book. + + + + + Describes the city where a book was published. + + + + + Describes the database name of submissions. + + + + + Describes a patent number. + + + + + Describes the institute where a thesis was made. + + + + + Describes the country where a thesis was made. + + + + + + Describes the authors of a citation when these are represented by a consortium. + Equivalent to the flat file RG-line. + + + + + + + + + + + + + + + + + + Groups a citation's scope and source descriptions. + + + + + Describes the scope of a citation. + Equivalent to the flat file RP-line. + + + + + + + + Describes the source of the sequence according to the citation. + Equivalent to the flat file RC-line. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes different types of general annotations. + Equivalent to the flat file CC-line. + + + + + + + + + Used in 'catalytic activity' annotations. + + + + + + + + Used in 'cofactor' annotations. + + + + + + + Used in 'subcellular location' annotations. + + + + + + + Used in 'sequence caution' annotations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Refers to the 'key' attribute of a 'reference' element. + + + + + + + + + Used in 'online information' annotations. + + + + + + + + + + Used in 'alternative products' annotations. + + + + + + + + Used in 'interaction' annotations. + + + + + + + + + Used in 'disease' annotations. + + + + + + + + + + + + + + + + + Used in 'mass spectrometry' and 'sequence caution' annotations. + + + + + + + + + + Describes the type of a general annotation. + Equivalent to the flat file CC comment topics (except for "DATABASE" which is translated to "online information"). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes the type of sequence location in 'RNA editing' annotations. Common values are "Not_applicable" and "Undetermined". + + + + + + Describes an optional name for an 'online information'. + + + + + + Describes the molecular mass in 'mass spectrometry' annotations. + + + + + Describes the error of the mass measurement in 'mass spectrometry' annotations. + + + + + Describes the experimental method in 'mass spectrometry' annotations. + + + + + + + + + Describes different types of biophysicochemical properties. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a chemical reaction. + + + + + + + + + + + Describes a physiological reaction. + + + + + + + + + + + + + + + + + + Describes a cofactor. + + + + + + + + + + + Describes the subcellular location and optionally the topology and orientation of a molecule. + + + + + + + + + + + Describes the type of events that cause alternative products. + + + + + + + + + + + + + + + + Describes isoforms in 'alternative products' annotations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a database cross-reference. + Equivalent to the flat file DR-line. + + + + + + + + + Describes the name of the database. + + + + + Describes a unique database identifier. + + + + + + + + + + + + + + + + Describes the evidence for the protein's existence. + Equivalent to the flat file PE-line. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes different types of sequence annotations. + Equivalent to the flat file FT-line. + + + + + Describes the original sequence in annotations that describe natural or artifical sequence variations. + + + + + Describes the variant sequence in annotations that describe natural or artifical sequence variations. + + + + + Describes the sequence coordinates of the annotation. + + + + + Describes the chemical entity that is bound in annotations that describe binding sites. + + + + + Describes the specific part of a molecule that is bound in annotations that describe binding sites. + + + + + + Describes the type of a sequence annotation. + Equivalent to the flat file FT feature keys, but using full terms instead of acronyms. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a sequence location as either a range with a begin and end or as a position. The 'sequence' attribute is only used when the location is not on the canonical sequence displayed in the current entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a ligand. + + + + + + + + + + + + Describes a ligand part. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a molecule by name or unique identifier. + + + + + + + + + + + + + Describes the evidence for an annotation. + No flat file equivalent. + + + + + + + + Describes the type of an evidence using the Evidence Code Ontology (http://www.obofoundry.org/cgi-bin/detail.cgi?id=evidence_code). + + + + + A unique key to link annotations (via 'evidence' attributes) to evidences. + + + + + + Describes the source of the data using a database cross-reference (or a 'ref' attribute when the source cannot be found in a public data source, such as PubMed, and is cited only within the UniProtKB entry). + + + + + + + + + Describes the source of the evidence, when it is not assigned by UniProt, but imported from an external database. + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/cdm_data_loaders/schemas/uniref/__init__.py b/src/cdm_data_loaders/schemas/uniref/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/schemas/uniref/schema.py b/src/cdm_data_loaders/schemas/uniref/schema.py new file mode 100644 index 00000000..f834378c --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniref/schema.py @@ -0,0 +1,186 @@ +"""PyArrow schema for uniref (xml_tag='entry').""" + +import pyarrow as pa + +SCHEMA = pa.schema( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"comment": "The entry identifier.", "xsd_type": "xs:ID"} + ), + pa.field( + "_updated", + pa.date32(), + nullable=False, + metadata={"comment": "The update date for the entry.", "xsd_type": "xs:date"}, + ), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_value", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "representativeMember", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "The name of the database this cross-reference is referring to.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"comment": "The ID referred to.", "xsd_type": "xs:string"}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Database cross-references"}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field("_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "UniRef50/90/100 cluster members"}, + ), + pa.field( + "member", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "The name of the database this cross-reference is referring to.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"comment": "The ID referred to.", "xsd_type": "xs:string"}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Database cross-references"}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"} + ), + pa.field( + "_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "UniRef50/90/100 cluster members"}, + ), + ] +) diff --git a/src/cdm_data_loaders/schemas/uniref/uniref.xsd b/src/cdm_data_loaders/schemas/uniref/uniref.xsd new file mode 100644 index 00000000..c6d87954 --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniref/uniref.xsd @@ -0,0 +1,116 @@ + + + + + + + + + + Database cross-references + + + + + + + The name of the database this cross-reference is referring to. + + + + + The ID referred to. + + + + + + UniRef50/90/100 cluster members + + + + + + + + + + + + + + + + + + UniRef50/90/100 entry + + + + + + + + + + The entry identifier. + + + + + The update date for the entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 66d11659..5f26debe 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -74,7 +74,9 @@ "ncbi_ftp_sync": ("ncbi_ftp_sync", "cdm_data_loaders.pipelines.ncbi_ftp_download"), "ncbi_rest_api": ("ncbi_rest_api", "cdm_data_loaders.pipelines.ncbi_rest_api"), "pdb_manifest": ("pdb_manifest", "cdm_data_loaders.pipelines.pdb_manifest"), + "uniprot": ("uniprot", "cdm_data_loaders.pipelines.xmltopyarrow.uniprot"), "uniprot_cdm": ("uniprot_cdm", "cdm_data_loaders.pipelines.uniprot_kb_cdm"), + "uniref": ("uniref", "cdm_data_loaders.pipelines.xmltopyarrow.uniref"), "uniref_cdm": ("uniref_cdm", "cdm_data_loaders.pipelines.uniref_cdm"), "xmltodict": ("xmltodict", "cdm_data_loaders.pipelines.xmltodict.pipeline"), "xmltopyarrow": ("xmltopyarrow", "cdm_data_loaders.pipelines.xmltopyarrow.pipeline"), diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index 9708d6a0..80ff6aa6 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -3,7 +3,7 @@ import gzip import json from collections import defaultdict -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from contextlib import contextmanager from datetime import datetime from logging import Logger, getLogger @@ -182,21 +182,23 @@ def sorted_json(entries: list[dict[str, Any]]) -> list[str]: return sorted(json.dumps(entry, sort_keys=True, default=str) for entry in entries) -def settings_to_argv(settings: Any) -> list[str]: +def settings_to_argv(settings: Any, *, exclude: Iterable[str] = ()) -> list[str]: # noqa: ANN401 """Derive CLI arguments from a settings instance. Every serialized field becomes a ``--kebab-case`` flag, with bools rendered as their ``true``/``false`` CLI spelling, sequence fields expanded after the - flag, and None-valued or computed fields skipped. + flag, and None-valued, excluded, or computed fields skipped. :param settings: a CtsSettings (or derivative) instance :type settings: Any + :param exclude: field names to omit from the resulting argument list + :type exclude: Iterable[str] :return: command-line argument list reproducing the settings :rtype: list[str] """ argv: list[str] = [] - computed_fields: set[str] = set(getattr(type(settings), "model_computed_fields", {}).keys()) - for name, value in settings.model_dump(exclude=computed_fields).items(): + excluded_fields: set[str] = set(getattr(type(settings), "model_computed_fields", {}).keys()) | set(exclude) + for name, value in settings.model_dump(exclude=excluded_fields).items(): flag = f"--{name.replace('_', '-')}" if isinstance(value, bool): argv.extend([flag, "true" if value else "false"]) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py index fc225dc4..67deda9d 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py @@ -7,6 +7,7 @@ import pytest +from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings from tests.xml_samples import LIBRARY_XSD @@ -21,20 +22,29 @@ def library_xsd(tmp_path: Path) -> Path: @pytest.fixture -def settings_factory(tmp_path: Path, library_xsd: Path) -> Callable[..., XmlToPyArrowSettings]: - """Return a factory that builds a valid XmlToPyArrowSettings, with fields open to override.""" +def settings_factory(tmp_path: Path, library_xsd: Path) -> Callable[..., Any]: + """Return a factory that builds a valid XmlToPyArrowSettings (or CtsSettings), with fields open to override.""" - def _factory(**overrides: Any) -> XmlToPyArrowSettings: # noqa: ANN401 + def _factory( + settings_cls: type[CtsSettings] = XmlToPyArrowSettings, + **overrides: Any, # noqa: ANN401 + ) -> Any: # noqa: ANN401 defaults: dict[str, Any] = { "dlt_dev_mode": True, - "dataset_name": "xml_test_dataset", - "table_name": "book", - "xml_tag": "book", - "xsd_file": str(library_xsd), - "file_glob": "*.xml*", } + if issubclass(settings_cls, XmlToPyArrowSettings): + defaults["file_glob"] = "*.xml*" + if settings_cls is XmlToPyArrowSettings: + defaults.update( + { + "dataset_name": "xml_test_dataset", + "table_name": "book", + "xml_tag": "book", + "xsd_file": str(library_xsd), + } + ) defaults.update(overrides) - return make_cts_settings(XmlToPyArrowSettings, tmp_path, **defaults) + return make_cts_settings(settings_cls, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py index 7e295bd7..c92fc94f 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py @@ -50,19 +50,27 @@ def settings_factory( dlt_destination_config: str, library_xsd: Path, ) -> Callable[..., XmlToPyArrowSettings]: - """Return a factory that builds a valid XmlToPyArrowSettings, with fields open to override.""" + """Return a factory that builds a valid XmlToPyArrowSettings (or subclass), with fields open to override.""" - def _factory(**overrides: Any) -> XmlToPyArrowSettings: # noqa: ANN401 + def _factory( + settings_cls: type[XmlToPyArrowSettings] = XmlToPyArrowSettings, + **overrides: Any, # noqa: ANN401 + ) -> XmlToPyArrowSettings: defaults: dict[str, Any] = { **DEFAULT_XMLTOPYARROW_SETTINGS, - "dataset_name": "xml_test_dataset", - "table_name": "book", "use_destination": dlt_destination_config, - "xml_tag": "book", - "xsd_file": str(library_xsd), } + if settings_cls is XmlToPyArrowSettings: + defaults.update( + { + "dataset_name": "xml_test_dataset", + "table_name": "book", + "xml_tag": "book", + "xsd_file": str(library_xsd), + } + ) defaults.update(overrides) - return make_cts_settings(XmlToPyArrowSettings, tmp_path, **defaults) + return make_cts_settings(settings_cls, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py index 8c7724cb..f33c5635 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py @@ -8,8 +8,15 @@ from dlt.common.pipeline import LoadInfo from pandas import DataFrame -from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings -from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + PIPELINE_NAME, + UNIPROT_PIPELINE_NAME, + UNIREF_PIPELINE_NAME, + UniProtXmlToPyArrowSettings, + UniRefXmlToPyArrowSettings, + XmlToPyArrowSettings, +) +from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES, copy_test_files from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import ( LOADER_FILE_FORMATS, @@ -142,3 +149,31 @@ def test_run_xmltopyarrow_pipeline_pass_uniref_chunk_5_el( assert len(loaded_rows) == N_REFERENCE_XML_ENTRIES assert sorted_json(loaded_rows) == sorted_json(expected_rows) assert_pipeline_report_written(Path(settings.output_dir), load_info) + + +@pytest.mark.parametrize( + ("target_name", "settings_cls", "sample_xml_rel_dir"), + [ + (UNIPROT_PIPELINE_NAME, UniProtXmlToPyArrowSettings, "xsd/examples/uniprot"), + (UNIREF_PIPELINE_NAME, UniRefXmlToPyArrowSettings, "xsd/examples/uniref"), + ], +) +def test_run_subclass_pipeline_pass_end_to_end( + settings_factory: Callable[..., XmlToPyArrowSettings], + test_data_dir: Path, + pipeline_runner: PipelineTestRunner, + *, + target_name: str, + settings_cls: type[XmlToPyArrowSettings], + sample_xml_rel_dir: str, +) -> None: + """UniProt and UniRef pipelines load XML entries into the 'entry' table using precomputed PyArrow schemas.""" + settings = settings_factory(settings_cls) + copy_test_files(test_data_dir / sample_xml_rel_dir, ["minimal.xml.gz"], Path(settings.input_dir)) + + load_info = assert_run_ok(pipeline_runner.run(target_name, settings=settings)) + dataset = load_info.pipeline.dataset() + assert set(dataset.tables) == {"entry", *DEFAULT_DLT_TABLES} + entry_df = dataset.table("entry").df() + assert len(entry_df) == 1 + assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py index e6053cd9..dbd5d649 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py @@ -14,12 +14,25 @@ import cdm_data_loaders.pipelines.xmltopyarrow.pipeline as xmltopyarrow_module from cdm_data_loaders.core.fields import LoaderFileFormatEnum +from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import ( cli, run_xmltopyarrow_pipeline, ) -from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings -from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + PIPELINE_NAME, + UNIPROT_PIPELINE_NAME, + UNIREF_PIPELINE_NAME, + UniProtXmlToPyArrowSettings, + UniRefXmlToPyArrowSettings, + XmlToPyArrowSettings, +) +from cdm_data_loaders.pipelines.xmltopyarrow.uniprot import cli as uniprot_cli +from cdm_data_loaders.pipelines.xmltopyarrow.uniprot import run_uniprot_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.uniref import cli as uniref_cli +from cdm_data_loaders.pipelines.xmltopyarrow.uniref import run_uniref_pipeline +from tests.cdm_data_loaders.pipelines.conftest import copy_test_files, duckdb_pipeline +from tests.cdm_data_loaders.pipelines.helpers import assert_run_ok, fetch_all_duckdb, settings_to_argv from tests.xml_samples import TWO_BOOK_LIBRARY_XML @@ -64,7 +77,7 @@ def test_run_xmltopyarrow_pipeline_pass_binds_reader_before_run_pipeline( def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and runs the pipeline via run_xmltopyarrow_pipeline. @@ -73,38 +86,9 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( config isolation fixture), and core.run_pipeline is redirected to a real DuckDB pipeline so the full flow is validated. """ - input_dir = tmp_path / "cli_input" - input_dir.mkdir() - (input_dir / "library.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--xsd-file", - str(library_xsd), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory(dataset_name="cli_dataset", dlt_dev_mode=False) + (Path(settings.input_dir) / "library.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") + monkeypatch.setattr(sys, "argv", [PIPELINE_NAME, *settings_to_argv(settings)]) # unique pipeline name: the duckdb database file is derived from it, so # repeated runs of the suite do not see rows from previous runs @@ -121,58 +105,20 @@ def fake_run_pipeline( return pipeline.run(resource) with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): - load_info = cli.run() - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(cli.run()) - with ( - load_info.pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM book") as cur, - ): - (row_count,) = cur.fetchone() - assert row_count == 2 # noqa: PLR2004 + assert fetch_all_duckdb(load_info.pipeline, "SELECT COUNT(*) FROM book") == [(2,)] def test_cli_pass_writes_rows_for_matching_xml_files( tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() with xml files in the input dir loads one row per matching element.""" - input_dir = tmp_path / "xml_input" - input_dir.mkdir() - (input_dir / "data.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") - - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--xsd-file", - str(library_xsd), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory(dataset_name="cli_dataset", dlt_dev_mode=False) + (Path(settings.input_dir) / "data.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") + monkeypatch.setattr(sys, "argv", [PIPELINE_NAME, *settings_to_argv(settings)]) pipeline_name = f"test_xmltopyarrow_cli_rows_pipeline_{uuid4().hex}" @@ -185,56 +131,95 @@ def fake_run_pipeline( return pipeline.run(resource) with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): - load_info = cli.run() + load_info = assert_run_ok(cli.run()) - assert load_info is not None - assert not load_info.has_failed_jobs - pipeline = load_info.pipeline - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM book") as cur, - ): - (row_count,) = cur.fetchone() - assert row_count == 2 # noqa: PLR2004 + assert fetch_all_duckdb(load_info.pipeline, "SELECT COUNT(*) FROM book") == [(2,)] -@pytest.mark.parametrize( - ("missing_args", "expected_fragment"), - [ - (["--table-name", "book", "--xml-tag", "book", "--xsd-file", "{xsd}"], "dataset_name"), - (["--dataset-name", "cli_dataset", "--xml-tag", "book", "--xsd-file", "{xsd}"], "table_name"), - (["--dataset-name", "cli_dataset", "--table-name", "book", "--xsd-file", "{xsd}"], "xml_tag"), - (["--dataset-name", "cli_dataset", "--table-name", "book", "--xml-tag", "book"], "xsd_file"), - ], -) +@pytest.mark.parametrize("missing_field", ["dataset_name", "table_name", "xml_tag", "xsd_file"]) def test_cli_fail_missing_required_args_raise( - tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], monkeypatch: pytest.MonkeyPatch, - missing_args: list[str], - expected_fragment: str, + missing_field: str, ) -> None: """cli() without required pipeline-specific args raises before running the pipeline.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - resolved_args = [arg.format(xsd=library_xsd) for arg in missing_args] - argv = [ - PIPELINE_NAME, - "--input-dir", - str(tmp_path / "input"), - "--output-dir", - str(tmp_path / "output"), - "--log-config-file", - str(log_config_file), - *resolved_args, - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory() + monkeypatch.setattr(sys, "argv", [PIPELINE_NAME, *settings_to_argv(settings, exclude=[missing_field])]) with ( patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline, - pytest.raises(Exception, match=expected_fragment), + pytest.raises(Exception, match=missing_field), ): cli() mock_run_pipeline.assert_not_called() + + +@pytest.mark.parametrize( + ("settings_cls", "run_fn", "expected_pipeline_name"), + [ + (UniProtXmlToPyArrowSettings, run_uniprot_pipeline, UNIPROT_PIPELINE_NAME), + (UniRefXmlToPyArrowSettings, run_uniref_pipeline, UNIREF_PIPELINE_NAME), + ], +) +def test_run_subclass_pipeline_pass_sets_core_run_pipeline_args_correctly( + settings_factory: Callable[..., Any], + *, + settings_cls: type[XmlToPyArrowSettings], + run_fn: Callable[[Any], LoadInfo | None], + expected_pipeline_name: str, +) -> None: + """run_uniprot_pipeline and run_uniref_pipeline delegate to run_pipeline with the subclass pipeline_name.""" + settings = settings_factory(settings_cls) + + with patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline: + run_fn(settings) + + mock_run_pipeline.assert_called_once() + _, kwargs = mock_run_pipeline.call_args + assert kwargs["settings"] == settings + assert kwargs["pipeline_kwargs"] == { + "pipeline_name": expected_pipeline_name, + "dataset_name": expected_pipeline_name, + } + assert isinstance(kwargs["resource"], DltResource) + + +@pytest.mark.parametrize( + ("prog_name", "cli_obj", "sample_xml_rel_dir"), + [ + (UNIPROT_PIPELINE_NAME, uniprot_cli, "xsd/examples/uniprot"), + (UNIREF_PIPELINE_NAME, uniref_cli, "xsd/examples/uniref"), + ], +) +def test_subclass_cli_pass_runs_end_to_end_with_default_settings( + tmp_path: Path, + test_data_dir: Path, + settings_factory: Callable[..., Any], + monkeypatch: pytest.MonkeyPatch, + *, + prog_name: str, + cli_obj: Any, # noqa: ANN401 + sample_xml_rel_dir: str, +) -> None: + """UniProt and UniRef CLIs load entry rows end-to-end using their default settings.""" + base_settings = settings_factory(CtsSettings, dlt_dev_mode=False) + copy_test_files(test_data_dir / sample_xml_rel_dir, ["minimal.xml.gz"], Path(base_settings.input_dir)) + monkeypatch.setattr(sys, "argv", [prog_name, *settings_to_argv(base_settings)]) + + pipeline_name = f"test_{prog_name}_cli_pipeline_{uuid4().hex}" + + def fake_run_pipeline( + *, + resource: DltResource, + pipeline_kwargs: dict[str, Any], + **_: dict[str, Any], + ) -> LoadInfo | None: + assert pipeline_kwargs == {"pipeline_name": prog_name, "dataset_name": prog_name} + pipeline = duckdb_pipeline(tmp_path, pipeline_name, prog_name) + return pipeline.run(resource) + + with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): + load_info = assert_run_ok(cli_obj.run()) + + assert fetch_all_duckdb(load_info.pipeline, "SELECT COUNT(*) FROM entry") == [(1,)] diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py index df4ae695..3dfd03f5 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py @@ -2,6 +2,8 @@ from collections.abc import Callable from pathlib import Path +from typing import Any +from unittest.mock import patch import pyarrow as pa import pytest @@ -9,9 +11,22 @@ from pydantic import ValidationError from pydantic_settings import CliApp +import cdm_data_loaders.pipelines.xmltopyarrow.settings as xmltopyarrow_settings_module from cdm_data_loaders.core.fields import DEFAULTS -from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from cdm_data_loaders.core.settings import CtsSettings +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UNIPROT_PIPELINE_NAME, + UNIPROT_XSD_FILE, + UNIREF_PIPELINE_NAME, + UNIREF_XSD_FILE, + UniProtXmlToPyArrowSettings, + UniRefXmlToPyArrowSettings, + XmlToPyArrowSettings, +) from cdm_data_loaders.readers.xsd import DEFAULT_MAX_RECURSION_DEPTH, generate_pyarrow_schema, load_schema +from cdm_data_loaders.schemas.uniprot.schema import SCHEMA as UNIPROT_SCHEMA +from cdm_data_loaders.schemas.uniref.schema import SCHEMA as UNIREF_SCHEMA +from tests.cdm_data_loaders.pipelines.helpers import settings_to_argv from tests.conftest import S3 UNIREF_LIKE_XSD: str = "uniref_like.xsd" @@ -175,39 +190,70 @@ def test_xmltopyarrow_settings_fail_xml_tag_not_in_xsd_raises_validation_error( @pytest.mark.parametrize("file_glob_arg", ["--file-glob", "-g"]) def test_xmltopyarrow_settings_pass_xml_tag_shortcut( - tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], file_glob_arg: str, ) -> None: """The file_glob field is exposed as both --file-glob and -g on the command line.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - input_dir = tmp_path / "input" - input_dir.mkdir() - output_dir = tmp_path / "output" - output_dir.mkdir() - - argv = [ - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - file_glob_arg, - "*.xml.gz", - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--xsd-file", - str(library_xsd), - ] + base_settings = settings_factory() + argv = [*settings_to_argv(base_settings, exclude=["file_glob"]), file_glob_arg, "*.xml.gz"] settings = CliApp.run(XmlToPyArrowSettings, cli_args=argv) assert settings.file_glob == "*.xml.gz" assert settings.xml_tag == "book" + + +def test_uniprot_xmltopyarrow_settings_pass_defaults( + settings_factory: Callable[..., Any], +) -> None: + """UniProtXmlToPyArrowSettings defaults dataset_name, table_name, xml_tag, xsd_file and uses precomputed SCHEMA.""" + with patch.object(xmltopyarrow_settings_module, "generate_pyarrow_schema") as mock_gen_schema: + settings = settings_factory(UniProtXmlToPyArrowSettings) + + mock_gen_schema.assert_not_called() + assert settings.dataset_name == UNIPROT_PIPELINE_NAME + assert settings.table_name == "entry" + assert settings.xml_tag == "entry" + assert settings.xsd_file == UNIPROT_XSD_FILE + assert isinstance(settings.xsd_schema, xmlschema.XMLSchema) + assert settings.pyarrow_schema is UNIPROT_SCHEMA + + +def test_uniref_xmltopyarrow_settings_pass_defaults( + settings_factory: Callable[..., Any], +) -> None: + """UniRefXmlToPyArrowSettings defaults dataset_name, table_name, xml_tag, xsd_file and uses precomputed SCHEMA.""" + with patch.object(xmltopyarrow_settings_module, "generate_pyarrow_schema") as mock_gen_schema: + settings = settings_factory(UniRefXmlToPyArrowSettings) + + mock_gen_schema.assert_not_called() + assert settings.dataset_name == UNIREF_PIPELINE_NAME + assert settings.table_name == "entry" + assert settings.xml_tag == "entry" + assert settings.xsd_file == UNIREF_XSD_FILE + assert isinstance(settings.xsd_schema, xmlschema.XMLSchema) + assert settings.pyarrow_schema is UNIREF_SCHEMA + + +@pytest.mark.parametrize( + ("settings_cls", "expected_dataset_name", "expected_xsd_file", "expected_schema"), + [ + (UniProtXmlToPyArrowSettings, UNIPROT_PIPELINE_NAME, UNIPROT_XSD_FILE, UNIPROT_SCHEMA), + (UniRefXmlToPyArrowSettings, UNIREF_PIPELINE_NAME, UNIREF_XSD_FILE, UNIREF_SCHEMA), + ], +) +def test_subclass_xmltopyarrow_settings_pass_cli_defaults( + settings_factory: Callable[..., Any], + *, + settings_cls: type[XmlToPyArrowSettings], + expected_dataset_name: str, + expected_xsd_file: Path, + expected_schema: pa.Schema, +) -> None: + """UniProt and UniRef settings subclasses can be parsed via CLI without specifying defaulted arguments.""" + base_settings = settings_factory(CtsSettings) + settings = CliApp.run(settings_cls, cli_args=settings_to_argv(base_settings)) + assert settings.dataset_name == expected_dataset_name + assert settings.table_name == "entry" + assert settings.xml_tag == "entry" + assert settings.xsd_file == expected_xsd_file + assert settings.pyarrow_schema is expected_schema From 6755356cf085a6fbcd2bae476d3f2c702dbaf5dd Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 16:37:39 -0700 Subject: [PATCH 15/16] formatting --- src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py | 1 - src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py index 6ab31561..1840aaf6 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py @@ -30,4 +30,3 @@ def run_uniprot_pipeline(settings: UniProtXmlToPyArrowSettings) -> LoadInfo | No if __name__ == "__main__": cli() - diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py index bf36d1ff..08f8ba03 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py @@ -30,4 +30,3 @@ def run_uniref_pipeline(settings: UniRefXmlToPyArrowSettings) -> LoadInfo | None if __name__ == "__main__": cli() - From a6744e088a301f8c81635e9ff0a71a4c190f75f8 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 21 Oct 2026 16:40:00 -0700 Subject: [PATCH 16/16] update uv lock --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 26009a94..f244ce77 100644 --- a/uv.lock +++ b/uv.lock @@ -3,12 +3,12 @@ revision = 5 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and sys_platform == 'emscripten'", ] @@ -470,7 +470,7 @@ wheels = [ [[package]] name = "cdm-data-loaders" -version = "0.1.11" +version = "0.1.12" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },