diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f70e06..4a7d150e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # CDM Data Loaders Changelog - [CDM Data Loaders Changelog](#cdm-data-loaders-changelog) + - [v0.1.12](#v0112) + - [v0.1.11](#v0111) - [v0.1.10](#v0110) - [v0.1.9](#v019) - [v0.1.8](#v018) @@ -14,6 +16,14 @@ - [v0.1.0](#v010) +### v0.1.12 + +- Dedicated UniProt and UniRef XML import pipelines using custom XSD-aware XML-to-JSON converter. + +### v0.1.11 + +- Preliminary XSV, XML, and JSONLines importers + ### v0.1.10 - Add in the ability to configure loggers using a file diff --git a/pyproject.toml b/pyproject.toml index 76631030..ab5a4093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cdm-data-loaders" -version = "0.1.11" +version = "0.1.12" description = "Data loaders and wranglers for the CDM." requires-python = ">= 3.13" readme = "README.md" @@ -46,7 +46,9 @@ ncbi_ftp_promote = "cdm_data_loaders.pipelines.ncbi_ftp_promote:cli" ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" pdb_manifest = "cdm_data_loaders.pipelines.pdb_manifest:cli" +uniprot = "cdm_data_loaders.pipelines.xmltopyarrow.uniprot:cli" uniprot_cdm = "cdm_data_loaders.pipelines.uniprot_kb_cdm:cli" +uniref = "cdm_data_loaders.pipelines.xmltopyarrow.uniref:cli" uniref_cdm = "cdm_data_loaders.pipelines.uniref_cdm:cli" xmltodict = "cdm_data_loaders.pipelines.xmltodict.pipeline:cli" xmltopyarrow = "cdm_data_loaders.pipelines.xmltopyarrow.pipeline:cli" diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py index 4dfabd81..94aeb548 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py @@ -42,12 +42,13 @@ def run_xmltopyarrow_pipeline(settings: XmlToPyArrowSettings) -> LoadInfo | None files = filesystem(bucket_url=str(settings.input_dir), file_glob=settings.file_glob) xmltopyarrow_resource: DltResource = files | reader(settings) + pipeline_name = str(settings.model_config.get("cli_prog_name") or PIPELINE_NAME) return run_pipeline( settings=settings, resource=xmltopyarrow_resource, pipeline_kwargs={ - "pipeline_name": PIPELINE_NAME, + "pipeline_name": pipeline_name, "dataset_name": settings.dataset_name, }, pipeline_run_kwargs={ diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py index e5c1f2e4..4db39d25 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py @@ -1,5 +1,6 @@ """Settings class for the XML to PyArrow pipeline for the KBase CTS.""" +from pathlib import Path from typing import Annotated, Final, Self import pyarrow as pa @@ -10,6 +11,7 @@ from cdm_data_loaders.core.fields import ( DEFAULT_XML_FILE_GLOB, FILE_GLOB, + DatasetName, FileGlob, NonEmptyStr, TableName, @@ -23,8 +25,19 @@ generate_pyarrow_schema, load_schema, ) +from cdm_data_loaders.schemas.uniprot.schema import SCHEMA as UNIPROT_SCHEMA +from cdm_data_loaders.schemas.uniref.schema import SCHEMA as UNIREF_SCHEMA PIPELINE_NAME: Final[str] = "xmltopyarrow" +UNIPROT_PIPELINE_NAME: Final[str] = "uniprot" +UNIREF_PIPELINE_NAME: Final[str] = "uniref" + +ENTRY_XML_TAG: Final[str] = "entry" +ENTRY_TABLE_NAME: Final[str] = "entry" + +SCHEMAS_DIR: Final[Path] = Path(__file__).resolve().parents[2] / "schemas" +UNIPROT_XSD_FILE: Final[Path] = SCHEMAS_DIR / "uniprot" / "uniprot.xsd" +UNIREF_XSD_FILE: Final[Path] = SCHEMAS_DIR / "uniref" / "uniref.xsd" class XmlToPyArrowSettings(FileFormatIngestSettings): @@ -70,20 +83,24 @@ class XmlToPyArrowSettings(FileFormatIngestSettings): @model_validator(mode="after") def check_xsd_file(self) -> Self: - """Check whether the schema file is present and valid; load the XSD and generate the PyArrow schema.""" + """Check whether the schema file is present and valid; load the XSD and resolve the PyArrow schema.""" try: self._xsd_schema = load_schema(self.xsd_file) - self._pyarrow_schema = generate_pyarrow_schema( - self._xsd_schema, - self.xml_tag, - max_recursion_depth=self.max_recursion_depth, - ) + self._pyarrow_schema = self._resolve_pyarrow_schema() except Exception as e: err_msg = f"Could not generate PyArrow schema from XSD: {e!s}" raise ValueError(err_msg) from e return self + def _resolve_pyarrow_schema(self) -> pa.Schema: + """Return the PyArrow schema for this settings instance.""" + return generate_pyarrow_schema( + self._xsd_schema, + self.xml_tag, + max_recursion_depth=self.max_recursion_depth, + ) + @property def xsd_schema(self) -> xmlschema.XMLSchema: """Return the loaded XMLSchema instance.""" @@ -93,3 +110,93 @@ def xsd_schema(self) -> xmlschema.XMLSchema: def pyarrow_schema(self) -> pa.Schema: """Return the generated PyArrow schema for xml_tag.""" return self._pyarrow_schema + + +class UniProtXmlToPyArrowSettings(XmlToPyArrowSettings): + """Settings for the UniProt XML to PyArrow ingestion pipeline.""" + + model_config: SettingsConfigDict = default_settings_with_shortcuts( + cli_prog_name=UNIPROT_PIPELINE_NAME, + cli_shortcuts={ + **FileFormatIngestSettings.model_config["cli_shortcuts"], + FILE_GLOB: "g", + }, + ) + + dataset_name: Annotated[ + DatasetName, + Field( + default=UNIPROT_PIPELINE_NAME, + description="The name of the dataset being produced", + ), + ] + table_name: Annotated[ + TableName, + Field( + default=ENTRY_TABLE_NAME, + description="The name of the table to export parsed data to", + ), + ] + xml_tag: Annotated[ + NonEmptyStr, + Field( + default=ENTRY_XML_TAG, + description="XML tag to capture the contents of", + ), + ] + xsd_file: Annotated[ + FilePath, + Field( + default=UNIPROT_XSD_FILE, + description="Path to the UniProt XSD file describing the XML documents to be imported.", + ), + ] + + def _resolve_pyarrow_schema(self) -> pa.Schema: + """Return the precomputed UniProt PyArrow schema.""" + return UNIPROT_SCHEMA + + +class UniRefXmlToPyArrowSettings(XmlToPyArrowSettings): + """Settings for the UniRef XML to PyArrow ingestion pipeline.""" + + model_config: SettingsConfigDict = default_settings_with_shortcuts( + cli_prog_name=UNIREF_PIPELINE_NAME, + cli_shortcuts={ + **FileFormatIngestSettings.model_config["cli_shortcuts"], + FILE_GLOB: "g", + }, + ) + + dataset_name: Annotated[ + DatasetName, + Field( + default=UNIREF_PIPELINE_NAME, + description="The name of the dataset being produced", + ), + ] + table_name: Annotated[ + TableName, + Field( + default=ENTRY_TABLE_NAME, + description="The name of the table to export parsed data to", + ), + ] + xml_tag: Annotated[ + NonEmptyStr, + Field( + default=ENTRY_XML_TAG, + description="XML tag to capture the contents of", + ), + ] + xsd_file: Annotated[ + FilePath, + Field( + default=UNIREF_XSD_FILE, + description="Path to the UniRef XSD file describing the XML documents to be imported.", + ), + ] + + def _resolve_pyarrow_schema(self) -> pa.Schema: + """Return the precomputed UniRef PyArrow schema.""" + return UNIREF_SCHEMA diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py new file mode 100644 index 00000000..1840aaf6 --- /dev/null +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniprot.py @@ -0,0 +1,32 @@ +"""UniProt XML to PyArrow pipeline for the KBase CTS.""" + +from dlt.common.pipeline import LoadInfo + +from cdm_data_loaders.pipelines.core import PipelineCli +from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UNIPROT_PIPELINE_NAME as PIPELINE_NAME, +) +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UniProtXmlToPyArrowSettings, +) + +__all__ = ["PIPELINE_NAME", "UniProtXmlToPyArrowSettings", "cli", "run_uniprot_pipeline"] + + +def run_uniprot_pipeline(settings: UniProtXmlToPyArrowSettings) -> LoadInfo | None: + """Run the UniProt XML to PyArrow pipeline on matching files in settings.input_dir. + + :param settings: pipeline configuration + :type settings: UniProtXmlToPyArrowSettings + :return: load information for the pipeline + :rtype: LoadInfo | None + """ + return run_xmltopyarrow_pipeline(settings) + + +cli = PipelineCli(UniProtXmlToPyArrowSettings, run_uniprot_pipeline) + + +if __name__ == "__main__": + cli() diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py new file mode 100644 index 00000000..08f8ba03 --- /dev/null +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/uniref.py @@ -0,0 +1,32 @@ +"""UniRef XML to PyArrow pipeline for the KBase CTS.""" + +from dlt.common.pipeline import LoadInfo + +from cdm_data_loaders.pipelines.core import PipelineCli +from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UNIREF_PIPELINE_NAME as PIPELINE_NAME, +) +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UniRefXmlToPyArrowSettings, +) + +__all__ = ["PIPELINE_NAME", "UniRefXmlToPyArrowSettings", "cli", "run_uniref_pipeline"] + + +def run_uniref_pipeline(settings: UniRefXmlToPyArrowSettings) -> LoadInfo | None: + """Run the UniRef XML to PyArrow pipeline on matching files in settings.input_dir. + + :param settings: pipeline configuration + :type settings: UniRefXmlToPyArrowSettings + :return: load information for the pipeline + :rtype: LoadInfo | None + """ + return run_xmltopyarrow_pipeline(settings) + + +cli = PipelineCli(UniRefXmlToPyArrowSettings, run_uniref_pipeline) + + +if __name__ == "__main__": + cli() diff --git a/src/cdm_data_loaders/schemas/__init__.py b/src/cdm_data_loaders/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/schemas/uniprot/__init__.py b/src/cdm_data_loaders/schemas/uniprot/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/schemas/uniprot/schema.py b/src/cdm_data_loaders/schemas/uniprot/schema.py new file mode 100644 index 00000000..dfdc94fb --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniprot/schema.py @@ -0,0 +1,4344 @@ +"""PyArrow schema for uniprot (xml_tag='entry').""" + +import pyarrow as pa + +SCHEMA = pa.schema( + [ + pa.field("_dataset", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_created", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_modified", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field( + "accession", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "name", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "protein", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "domain", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": 'Describes names of "domains". Equivalent to the flat file DE-line Includes: section.' + }, + ), + pa.field( + "component", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "alternativeName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "shortName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "submittedName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "ecNumber", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={ + "xsd_type": "xs:int" + }, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "innName", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes names of processed products. Equivalent to the flat file DE-line Contains: section." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes the names for the protein and parts thereof. Equivalent to the flat file DE-line." + }, + ), + pa.field( + "gene", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes different types of gene designations. Equivalent to the flat file GN-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a gene. Equivalent to the flat file GN-line."}, + ), + pa.field( + "organism", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes the names of the source organism. Equivalent to the flat file OS-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a cross-reference to the NCBI taxonomy database. Equivalent to the flat file OX-line." + }, + ), + pa.field( + "lineage", + pa.struct( + [ + pa.field( + "taxon", + pa.list_( + pa.field( + "item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the lineage of the source organism. Equivalent to the flat file OC-line." + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Describes the source organism."}, + ), + pa.field( + "organismHost", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes the names of the source organism. Equivalent to the flat file OS-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a cross-reference to the NCBI taxonomy database. Equivalent to the flat file OX-line." + }, + ), + pa.field( + "lineage", + pa.struct( + [ + pa.field( + "taxon", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the lineage of the source organism. Equivalent to the flat file OC-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the source organism."}, + ), + pa.field( + "geneLocation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Indicates whether the name of a plasmid is known or unknown."}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes non-nuclear gene locations (organelles and plasmids). Equivalent to the flat file OG-line." + }, + ), + pa.field( + "reference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_key", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "citation", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of a citation.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_date", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}, + ), + pa.field( + "_name", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the name of an (online) journal or book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_volume", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the volume of a journal or book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_first", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the first page of an article.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_last", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the last page of an article.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_publisher", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the publisher of a book.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_city", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the city where a book was published.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_db", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the database name of submissions.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_number", + pa.string(), + nullable=True, + metadata={"comment": "Describes a patent number.", "xsd_type": "xs:string"}, + ), + pa.field( + "_institute", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the institute where a thesis was made.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_country", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the country where a thesis was made.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "title", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the title of a citation. Equivalent to the flat file RT-line.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "editorList", + pa.struct( + [ + pa.field( + "consortium", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation when these are represented by a consortium. Equivalent to the flat file RG-line." + }, + ), + pa.field( + "person", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the editors of a book (only used for books). Equivalent to part of the flat file RL-line of books." + }, + ), + pa.field( + "authorList", + pa.struct( + [ + pa.field( + "consortium", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation when these are represented by a consortium. Equivalent to the flat file RG-line." + }, + ), + pa.field( + "person", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the authors of a citation. Equivalent to the flat file RA-line." + }, + ), + pa.field( + "locator", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the location (URL) of an online journal article. No flat file equivalent.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes cross-references to bibliography databases (MEDLINE, PubMed, AGRICOLA) or other online resources (DOI). Equivalent to the flat file RX-line." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes different types of citations. Equivalent to the flat file RX-, RG-, RA-, RT- and RL-lines." + }, + ), + pa.field( + "scope", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + metadata={ + "comment": "Describes the scope of a citation. Equivalent to the flat file RP-line." + }, + ), + pa.field( + "source", + pa.struct( + [ + pa.field( + "strain", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "plasmid", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "transposon", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "tissue", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the sequence according to the citation. Equivalent to the flat file RC-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a citation and a summary of its content. Equivalent to the flat file RN-, RP-, RC-, RX-, RG-, RA-, RT- and RL-lines." + }, + ), + pa.field( + "comment", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=True, + metadata={ + "comment": 'Describes the type of a general annotation. Equivalent to the flat file CC comment topics (except for "DATABASE" which is translated to "online information").', + "xsd_type": "xs:string", + }, + ), + pa.field( + "_locationType", + pa.string(), + nullable=True, + metadata={ + "comment": 'Describes the type of sequence location in \'RNA editing\' annotations. Common values are "Not_applicable" and "Undetermined".', + "xsd_type": "xs:string", + }, + ), + pa.field( + "_name", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes an optional name for an 'online information'.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_mass", + pa.float64(), + nullable=True, + metadata={ + "comment": "Describes the molecular mass in 'mass spectrometry' annotations.", + "xsd_type": "xs:float", + }, + ), + pa.field( + "_error", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the error of the mass measurement in 'mass spectrometry' annotations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_method", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the experimental method in 'mass spectrometry' annotations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "absorption", + pa.struct( + [ + pa.field( + "max", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "kinetics", + pa.struct( + [ + pa.field( + "KM", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "Vmax", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "phDependence", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "redoxPotential", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "temperatureDependence", + pa.struct( + [ + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "reaction", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"} + ), + ), + nullable=True, + ), + pa.field( + "text", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a chemical reaction."}, + ), + pa.field( + "physiologicalReaction", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_direction", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a physiological reaction."}, + ), + pa.field( + "cofactor", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "name", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a cofactor."}, + ), + pa.field( + "subcellularLocation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "location", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "topology", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "orientation", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes the subcellular location and optionally the topology and orientation of a molecule." + }, + ), + pa.field( + "conflict", + pa.struct( + [ + pa.field( + "_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "_ref", + pa.string(), + nullable=True, + metadata={ + "comment": "Refers to the 'key' attribute of a 'reference' element.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_resource", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_version", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={"comment": "Used in 'sequence caution' annotations."}, + ), + pa.field( + "link", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_uri", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:anyURI"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Used in 'online information' annotations."}, + ), + pa.field( + "event", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the type of events that cause alternative products."}, + ), + pa.field( + "isoform", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "id", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ), + nullable=False, + ), + pa.field( + "name", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_ref", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=False, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes isoforms in 'alternative products' annotations."}, + ), + pa.field( + "interactant", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_intactId", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("organismsDiffer", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + pa.field("experiments", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "disease", + pa.struct( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "acronym", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "description", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={"comment": "Used in 'disease' annotations."}, + ), + pa.field( + "location", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_sequence", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "begin", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "position", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Used in 'mass spectrometry' and 'sequence caution' annotations."}, + ), + pa.field( + "text", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes different types of general annotations. Equivalent to the flat file CC-line." + }, + ), + pa.field( + "dbReference", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"comment": "Describes the name of the database.", "xsd_type": "xs:string"}, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={"comment": "Describes a molecule by name or unique identifier."}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes a database cross-reference. Equivalent to the flat file DR-line."}, + ), + pa.field( + "proteinExistence", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + metadata={ + "comment": "Describes the evidence for the protein's existence. Equivalent to the flat file PE-line." + }, + ), + pa.field( + "keyword", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "feature", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of a sequence annotation. Equivalent to the flat file FT feature keys, but using full terms instead of acronyms.", + "xsd_type": "xs:string", + }, + ), + pa.field("_id", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_description", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "_evidence", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field("_ref", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "original", + pa.string(), + nullable=True, + metadata={ + "comment": "Describes the original sequence in annotations that describe natural or artifical sequence variations.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "variation", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + metadata={ + "comment": "Describes the variant sequence in annotations that describe natural or artifical sequence variations." + }, + ), + pa.field( + "location", + pa.struct( + [ + pa.field( + "_sequence", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "begin", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "position", + pa.struct( + [ + pa.field( + "_position", + pa.uint64(), + nullable=True, + metadata={"xsd_type": "xs:unsignedLong"}, + ), + pa.field( + "_status", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Describes the sequence coordinates of the annotation."}, + ), + pa.field( + "ligand", + pa.struct( + [ + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + pa.field( + "label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "note", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the chemical entity that is bound in annotations that describe binding sites." + }, + ), + pa.field( + "ligandPart", + pa.struct( + [ + pa.field( + "name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + pa.field( + "label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "note", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the specific part of a molecule that is bound in annotations that describe binding sites." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={ + "comment": "Describes different types of sequence annotations. Equivalent to the flat file FT-line." + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the type of an evidence using the Evidence Code Ontology (http://www.obofoundry.org/cgi-bin/detail.cgi?id=evidence_code).", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_key", + pa.int64(), + nullable=False, + metadata={ + "comment": "A unique key to link annotations (via 'evidence' attributes) to evidences.", + "xsd_type": "xs:integer", + }, + ), + pa.field( + "source", + pa.struct( + [ + pa.field( + "_ref", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"} + ), + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the data using a database cross-reference (or a 'ref' attribute when the source cannot be found in a public data source, such as PubMed, and is cited only within the UniProtKB entry)." + }, + ), + pa.field( + "importedFrom", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes the name of the database.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={ + "comment": "Describes a unique database identifier.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_evidence", + pa.list_( + pa.field( + "item", + pa.int32(), + nullable=True, + metadata={"xsd_type": "xs:int"}, + ), + ), + nullable=True, + ), + pa.field( + "molecule", + pa.struct( + [ + pa.field( + "_id", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes a molecule by name or unique identifier." + }, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Describes a database cross-reference. Equivalent to the flat file DR-line." + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "Describes the source of the evidence, when it is not assigned by UniProt, but imported from an external database." + }, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "Describes the evidence for an annotation. No flat file equivalent."}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field("_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_mass", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_modified", pa.date32(), nullable=False, metadata={"xsd_type": "xs:date"}), + pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_precursor", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + pa.field("_fragment", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + ] +) diff --git a/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd b/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd new file mode 100644 index 00000000..e79ba5ea --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd @@ -0,0 +1,1125 @@ + + + + + + + + + + Describes a collection of UniProtKB entries. + + + + + + + + + + + + + + Describes a UniProtKB entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes the names for the protein and parts thereof. + Equivalent to the flat file DE-line. + + + + + + Describes names of "domains". + Equivalent to the flat file DE-line Includes: section. + + + + + + + + Describes names of processed products. + Equivalent to the flat file DE-line Contains: section. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a gene. + Equivalent to the flat file GN-line. + + + + + + + + Describes different types of gene designations. + Equivalent to the flat file GN-line. + + + + + + + + + + + + + + + + + + + + + + + Describes the source organism. + + + + + Describes the names of the source organism. + Equivalent to the flat file OS-line. + + + + + Describes a cross-reference to the NCBI taxonomy database. + Equivalent to the flat file OX-line. + + + + + Describes the lineage of the source organism. + Equivalent to the flat file OC-line. + + + + + + + + + + + + + Describes different types of source organism names. + + + + + + + + + + + + + + + + + + + + + + + Describes non-nuclear gene locations (organelles and plasmids). + Equivalent to the flat file OG-line. + + + + + + + + + + + + + + + + + + + + + + + + + Indicates whether the name of a plasmid is known or unknown. + + + + + + + + + + + + + + + + + + + + Describes a citation and a summary of its content. + Equivalent to the flat file RN-, RP-, RC-, RX-, RG-, RA-, RT- and RL-lines. + + + + + + + + + + + + + + Describes different types of citations. + Equivalent to the flat file RX-, RG-, RA-, RT- and RL-lines. + + + + + Describes the title of a citation. + Equivalent to the flat file RT-line. + + + + + Describes the editors of a book (only used for books). + Equivalent to part of the flat file RL-line of books. + + + + + Describes the authors of a citation. + Equivalent to the flat file RA-line. + + + + + Describes the location (URL) of an online journal article. + No flat file equivalent. + + + + + Describes cross-references to bibliography databases (MEDLINE, PubMed, AGRICOLA) or other online resources (DOI). + Equivalent to the flat file RX-line. + + + + + + Describes the type of a citation. + + + + + + + + + + + + + + + + + + + + + Describes the name of an (online) journal or book. + + + + + Describes the volume of a journal or book. + + + + + Describes the first page of an article. + + + + + Describes the last page of an article. + + + + + Describes the publisher of a book. + + + + + Describes the city where a book was published. + + + + + Describes the database name of submissions. + + + + + Describes a patent number. + + + + + Describes the institute where a thesis was made. + + + + + Describes the country where a thesis was made. + + + + + + Describes the authors of a citation when these are represented by a consortium. + Equivalent to the flat file RG-line. + + + + + + + + + + + + + + + + + + Groups a citation's scope and source descriptions. + + + + + Describes the scope of a citation. + Equivalent to the flat file RP-line. + + + + + + + + Describes the source of the sequence according to the citation. + Equivalent to the flat file RC-line. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes different types of general annotations. + Equivalent to the flat file CC-line. + + + + + + + + + Used in 'catalytic activity' annotations. + + + + + + + + Used in 'cofactor' annotations. + + + + + + + Used in 'subcellular location' annotations. + + + + + + + Used in 'sequence caution' annotations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Refers to the 'key' attribute of a 'reference' element. + + + + + + + + + Used in 'online information' annotations. + + + + + + + + + + Used in 'alternative products' annotations. + + + + + + + + Used in 'interaction' annotations. + + + + + + + + + Used in 'disease' annotations. + + + + + + + + + + + + + + + + + Used in 'mass spectrometry' and 'sequence caution' annotations. + + + + + + + + + + Describes the type of a general annotation. + Equivalent to the flat file CC comment topics (except for "DATABASE" which is translated to "online information"). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes the type of sequence location in 'RNA editing' annotations. Common values are "Not_applicable" and "Undetermined". + + + + + + Describes an optional name for an 'online information'. + + + + + + Describes the molecular mass in 'mass spectrometry' annotations. + + + + + Describes the error of the mass measurement in 'mass spectrometry' annotations. + + + + + Describes the experimental method in 'mass spectrometry' annotations. + + + + + + + + + Describes different types of biophysicochemical properties. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a chemical reaction. + + + + + + + + + + + Describes a physiological reaction. + + + + + + + + + + + + + + + + + + Describes a cofactor. + + + + + + + + + + + Describes the subcellular location and optionally the topology and orientation of a molecule. + + + + + + + + + + + Describes the type of events that cause alternative products. + + + + + + + + + + + + + + + + Describes isoforms in 'alternative products' annotations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a database cross-reference. + Equivalent to the flat file DR-line. + + + + + + + + + Describes the name of the database. + + + + + Describes a unique database identifier. + + + + + + + + + + + + + + + + Describes the evidence for the protein's existence. + Equivalent to the flat file PE-line. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes different types of sequence annotations. + Equivalent to the flat file FT-line. + + + + + Describes the original sequence in annotations that describe natural or artifical sequence variations. + + + + + Describes the variant sequence in annotations that describe natural or artifical sequence variations. + + + + + Describes the sequence coordinates of the annotation. + + + + + Describes the chemical entity that is bound in annotations that describe binding sites. + + + + + Describes the specific part of a molecule that is bound in annotations that describe binding sites. + + + + + + Describes the type of a sequence annotation. + Equivalent to the flat file FT feature keys, but using full terms instead of acronyms. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a sequence location as either a range with a begin and end or as a position. The 'sequence' attribute is only used when the location is not on the canonical sequence displayed in the current entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a ligand. + + + + + + + + + + + + Describes a ligand part. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Describes a molecule by name or unique identifier. + + + + + + + + + + + + + Describes the evidence for an annotation. + No flat file equivalent. + + + + + + + + Describes the type of an evidence using the Evidence Code Ontology (http://www.obofoundry.org/cgi-bin/detail.cgi?id=evidence_code). + + + + + A unique key to link annotations (via 'evidence' attributes) to evidences. + + + + + + Describes the source of the data using a database cross-reference (or a 'ref' attribute when the source cannot be found in a public data source, such as PubMed, and is cited only within the UniProtKB entry). + + + + + + + + + Describes the source of the evidence, when it is not assigned by UniProt, but imported from an external database. + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/cdm_data_loaders/schemas/uniref/__init__.py b/src/cdm_data_loaders/schemas/uniref/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/schemas/uniref/schema.py b/src/cdm_data_loaders/schemas/uniref/schema.py new file mode 100644 index 00000000..f834378c --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniref/schema.py @@ -0,0 +1,186 @@ +"""PyArrow schema for uniref (xml_tag='entry').""" + +import pyarrow as pa + +SCHEMA = pa.schema( + [ + pa.field( + "_id", pa.string(), nullable=False, metadata={"comment": "The entry identifier.", "xsd_type": "xs:ID"} + ), + pa.field( + "_updated", + pa.date32(), + nullable=False, + metadata={"comment": "The update date for the entry.", "xsd_type": "xs:date"}, + ), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_value", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "representativeMember", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "The name of the database this cross-reference is referring to.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"comment": "The ID referred to.", "xsd_type": "xs:string"}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Database cross-references"}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field("_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), + pa.field("_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "UniRef50/90/100 cluster members"}, + ), + pa.field( + "member", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "dbReference", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={ + "comment": "The name of the database this cross-reference is referring to.", + "xsd_type": "xs:string", + }, + ), + pa.field( + "_id", + pa.string(), + nullable=False, + metadata={"comment": "The ID referred to.", "xsd_type": "xs:string"}, + ), + pa.field( + "property", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_type", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + metadata={"comment": "Database cross-references"}, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "_length", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"} + ), + pa.field( + "_checksum", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + metadata={"comment": "UniRef50/90/100 cluster members"}, + ), + ] +) diff --git a/src/cdm_data_loaders/schemas/uniref/uniref.xsd b/src/cdm_data_loaders/schemas/uniref/uniref.xsd new file mode 100644 index 00000000..c6d87954 --- /dev/null +++ b/src/cdm_data_loaders/schemas/uniref/uniref.xsd @@ -0,0 +1,116 @@ + + + + + + + + + + Database cross-references + + + + + + + The name of the database this cross-reference is referring to. + + + + + The ID referred to. + + + + + + UniRef50/90/100 cluster members + + + + + + + + + + + + + + + + + + UniRef50/90/100 entry + + + + + + + + + + The entry identifier. + + + + + The update date for the entry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/cdm_data_loaders/pipelines/framework.py b/tests/cdm_data_loaders/pipelines/framework.py index 66d11659..5f26debe 100644 --- a/tests/cdm_data_loaders/pipelines/framework.py +++ b/tests/cdm_data_loaders/pipelines/framework.py @@ -74,7 +74,9 @@ "ncbi_ftp_sync": ("ncbi_ftp_sync", "cdm_data_loaders.pipelines.ncbi_ftp_download"), "ncbi_rest_api": ("ncbi_rest_api", "cdm_data_loaders.pipelines.ncbi_rest_api"), "pdb_manifest": ("pdb_manifest", "cdm_data_loaders.pipelines.pdb_manifest"), + "uniprot": ("uniprot", "cdm_data_loaders.pipelines.xmltopyarrow.uniprot"), "uniprot_cdm": ("uniprot_cdm", "cdm_data_loaders.pipelines.uniprot_kb_cdm"), + "uniref": ("uniref", "cdm_data_loaders.pipelines.xmltopyarrow.uniref"), "uniref_cdm": ("uniref_cdm", "cdm_data_loaders.pipelines.uniref_cdm"), "xmltodict": ("xmltodict", "cdm_data_loaders.pipelines.xmltodict.pipeline"), "xmltopyarrow": ("xmltopyarrow", "cdm_data_loaders.pipelines.xmltopyarrow.pipeline"), diff --git a/tests/cdm_data_loaders/pipelines/helpers.py b/tests/cdm_data_loaders/pipelines/helpers.py index 9708d6a0..80ff6aa6 100644 --- a/tests/cdm_data_loaders/pipelines/helpers.py +++ b/tests/cdm_data_loaders/pipelines/helpers.py @@ -3,7 +3,7 @@ import gzip import json from collections import defaultdict -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from contextlib import contextmanager from datetime import datetime from logging import Logger, getLogger @@ -182,21 +182,23 @@ def sorted_json(entries: list[dict[str, Any]]) -> list[str]: return sorted(json.dumps(entry, sort_keys=True, default=str) for entry in entries) -def settings_to_argv(settings: Any) -> list[str]: +def settings_to_argv(settings: Any, *, exclude: Iterable[str] = ()) -> list[str]: # noqa: ANN401 """Derive CLI arguments from a settings instance. Every serialized field becomes a ``--kebab-case`` flag, with bools rendered as their ``true``/``false`` CLI spelling, sequence fields expanded after the - flag, and None-valued or computed fields skipped. + flag, and None-valued, excluded, or computed fields skipped. :param settings: a CtsSettings (or derivative) instance :type settings: Any + :param exclude: field names to omit from the resulting argument list + :type exclude: Iterable[str] :return: command-line argument list reproducing the settings :rtype: list[str] """ argv: list[str] = [] - computed_fields: set[str] = set(getattr(type(settings), "model_computed_fields", {}).keys()) - for name, value in settings.model_dump(exclude=computed_fields).items(): + excluded_fields: set[str] = set(getattr(type(settings), "model_computed_fields", {}).keys()) | set(exclude) + for name, value in settings.model_dump(exclude=excluded_fields).items(): flag = f"--{name.replace('_', '-')}" if isinstance(value, bool): argv.extend([flag, "true" if value else "false"]) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py index fc225dc4..67deda9d 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/conftest.py @@ -7,6 +7,7 @@ import pytest +from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings from tests.cdm_data_loaders.pipelines.helpers import make_cts_settings from tests.xml_samples import LIBRARY_XSD @@ -21,20 +22,29 @@ def library_xsd(tmp_path: Path) -> Path: @pytest.fixture -def settings_factory(tmp_path: Path, library_xsd: Path) -> Callable[..., XmlToPyArrowSettings]: - """Return a factory that builds a valid XmlToPyArrowSettings, with fields open to override.""" +def settings_factory(tmp_path: Path, library_xsd: Path) -> Callable[..., Any]: + """Return a factory that builds a valid XmlToPyArrowSettings (or CtsSettings), with fields open to override.""" - def _factory(**overrides: Any) -> XmlToPyArrowSettings: # noqa: ANN401 + def _factory( + settings_cls: type[CtsSettings] = XmlToPyArrowSettings, + **overrides: Any, # noqa: ANN401 + ) -> Any: # noqa: ANN401 defaults: dict[str, Any] = { "dlt_dev_mode": True, - "dataset_name": "xml_test_dataset", - "table_name": "book", - "xml_tag": "book", - "xsd_file": str(library_xsd), - "file_glob": "*.xml*", } + if issubclass(settings_cls, XmlToPyArrowSettings): + defaults["file_glob"] = "*.xml*" + if settings_cls is XmlToPyArrowSettings: + defaults.update( + { + "dataset_name": "xml_test_dataset", + "table_name": "book", + "xml_tag": "book", + "xsd_file": str(library_xsd), + } + ) defaults.update(overrides) - return make_cts_settings(XmlToPyArrowSettings, tmp_path, **defaults) + return make_cts_settings(settings_cls, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py index 7e295bd7..c92fc94f 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/conftest.py @@ -50,19 +50,27 @@ def settings_factory( dlt_destination_config: str, library_xsd: Path, ) -> Callable[..., XmlToPyArrowSettings]: - """Return a factory that builds a valid XmlToPyArrowSettings, with fields open to override.""" + """Return a factory that builds a valid XmlToPyArrowSettings (or subclass), with fields open to override.""" - def _factory(**overrides: Any) -> XmlToPyArrowSettings: # noqa: ANN401 + def _factory( + settings_cls: type[XmlToPyArrowSettings] = XmlToPyArrowSettings, + **overrides: Any, # noqa: ANN401 + ) -> XmlToPyArrowSettings: defaults: dict[str, Any] = { **DEFAULT_XMLTOPYARROW_SETTINGS, - "dataset_name": "xml_test_dataset", - "table_name": "book", "use_destination": dlt_destination_config, - "xml_tag": "book", - "xsd_file": str(library_xsd), } + if settings_cls is XmlToPyArrowSettings: + defaults.update( + { + "dataset_name": "xml_test_dataset", + "table_name": "book", + "xml_tag": "book", + "xsd_file": str(library_xsd), + } + ) defaults.update(overrides) - return make_cts_settings(XmlToPyArrowSettings, tmp_path, **defaults) + return make_cts_settings(settings_cls, tmp_path, **defaults) return _factory diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py index 8c7724cb..f33c5635 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/integration/test_xmltopyarrow_end_to_end.py @@ -8,8 +8,15 @@ from dlt.common.pipeline import LoadInfo from pandas import DataFrame -from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings -from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + PIPELINE_NAME, + UNIPROT_PIPELINE_NAME, + UNIREF_PIPELINE_NAME, + UniProtXmlToPyArrowSettings, + UniRefXmlToPyArrowSettings, + XmlToPyArrowSettings, +) +from tests.cdm_data_loaders.pipelines.conftest import DEFAULT_DLT_TABLES, copy_test_files from tests.cdm_data_loaders.pipelines.framework import PipelineTestRunner from tests.cdm_data_loaders.pipelines.helpers import ( LOADER_FILE_FORMATS, @@ -142,3 +149,31 @@ def test_run_xmltopyarrow_pipeline_pass_uniref_chunk_5_el( assert len(loaded_rows) == N_REFERENCE_XML_ENTRIES assert sorted_json(loaded_rows) == sorted_json(expected_rows) assert_pipeline_report_written(Path(settings.output_dir), load_info) + + +@pytest.mark.parametrize( + ("target_name", "settings_cls", "sample_xml_rel_dir"), + [ + (UNIPROT_PIPELINE_NAME, UniProtXmlToPyArrowSettings, "xsd/examples/uniprot"), + (UNIREF_PIPELINE_NAME, UniRefXmlToPyArrowSettings, "xsd/examples/uniref"), + ], +) +def test_run_subclass_pipeline_pass_end_to_end( + settings_factory: Callable[..., XmlToPyArrowSettings], + test_data_dir: Path, + pipeline_runner: PipelineTestRunner, + *, + target_name: str, + settings_cls: type[XmlToPyArrowSettings], + sample_xml_rel_dir: str, +) -> None: + """UniProt and UniRef pipelines load XML entries into the 'entry' table using precomputed PyArrow schemas.""" + settings = settings_factory(settings_cls) + copy_test_files(test_data_dir / sample_xml_rel_dir, ["minimal.xml.gz"], Path(settings.input_dir)) + + load_info = assert_run_ok(pipeline_runner.run(target_name, settings=settings)) + dataset = load_info.pipeline.dataset() + assert set(dataset.tables) == {"entry", *DEFAULT_DLT_TABLES} + entry_df = dataset.table("entry").df() + assert len(entry_df) == 1 + assert_pipeline_report_written(Path(settings.output_dir), load_info) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py index e6053cd9..dbd5d649 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py @@ -14,12 +14,25 @@ import cdm_data_loaders.pipelines.xmltopyarrow.pipeline as xmltopyarrow_module from cdm_data_loaders.core.fields import LoaderFileFormatEnum +from cdm_data_loaders.core.settings import CtsSettings from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import ( cli, run_xmltopyarrow_pipeline, ) -from cdm_data_loaders.pipelines.xmltopyarrow.settings import PIPELINE_NAME, XmlToPyArrowSettings -from tests.cdm_data_loaders.pipelines.conftest import duckdb_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + PIPELINE_NAME, + UNIPROT_PIPELINE_NAME, + UNIREF_PIPELINE_NAME, + UniProtXmlToPyArrowSettings, + UniRefXmlToPyArrowSettings, + XmlToPyArrowSettings, +) +from cdm_data_loaders.pipelines.xmltopyarrow.uniprot import cli as uniprot_cli +from cdm_data_loaders.pipelines.xmltopyarrow.uniprot import run_uniprot_pipeline +from cdm_data_loaders.pipelines.xmltopyarrow.uniref import cli as uniref_cli +from cdm_data_loaders.pipelines.xmltopyarrow.uniref import run_uniref_pipeline +from tests.cdm_data_loaders.pipelines.conftest import copy_test_files, duckdb_pipeline +from tests.cdm_data_loaders.pipelines.helpers import assert_run_ok, fetch_all_duckdb, settings_to_argv from tests.xml_samples import TWO_BOOK_LIBRARY_XML @@ -64,7 +77,7 @@ def test_run_xmltopyarrow_pipeline_pass_binds_reader_before_run_pipeline( def test_cli_pass_runs_end_to_end_from_command_line_arguments( tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() reads command-line arguments and runs the pipeline via run_xmltopyarrow_pipeline. @@ -73,38 +86,9 @@ def test_cli_pass_runs_end_to_end_from_command_line_arguments( config isolation fixture), and core.run_pipeline is redirected to a real DuckDB pipeline so the full flow is validated. """ - input_dir = tmp_path / "cli_input" - input_dir.mkdir() - (input_dir / "library.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--xsd-file", - str(library_xsd), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory(dataset_name="cli_dataset", dlt_dev_mode=False) + (Path(settings.input_dir) / "library.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") + monkeypatch.setattr(sys, "argv", [PIPELINE_NAME, *settings_to_argv(settings)]) # unique pipeline name: the duckdb database file is derived from it, so # repeated runs of the suite do not see rows from previous runs @@ -121,58 +105,20 @@ def fake_run_pipeline( return pipeline.run(resource) with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): - load_info = cli.run() - - assert load_info is not None - assert not load_info.has_failed_jobs + load_info = assert_run_ok(cli.run()) - with ( - load_info.pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM book") as cur, - ): - (row_count,) = cur.fetchone() - assert row_count == 2 # noqa: PLR2004 + assert fetch_all_duckdb(load_info.pipeline, "SELECT COUNT(*) FROM book") == [(2,)] def test_cli_pass_writes_rows_for_matching_xml_files( tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], monkeypatch: pytest.MonkeyPatch, ) -> None: """cli() with xml files in the input dir loads one row per matching element.""" - input_dir = tmp_path / "xml_input" - input_dir.mkdir() - (input_dir / "data.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") - - output_dir = tmp_path / "cli_output" - output_dir.mkdir() - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - argv = [ - PIPELINE_NAME, - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--xsd-file", - str(library_xsd), - "--use-output-dir-for-pipeline-metadata", - "false", - "--dlt-dev-mode", - "false", - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory(dataset_name="cli_dataset", dlt_dev_mode=False) + (Path(settings.input_dir) / "data.xml").write_text(TWO_BOOK_LIBRARY_XML, encoding="utf-8") + monkeypatch.setattr(sys, "argv", [PIPELINE_NAME, *settings_to_argv(settings)]) pipeline_name = f"test_xmltopyarrow_cli_rows_pipeline_{uuid4().hex}" @@ -185,56 +131,95 @@ def fake_run_pipeline( return pipeline.run(resource) with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): - load_info = cli.run() + load_info = assert_run_ok(cli.run()) - assert load_info is not None - assert not load_info.has_failed_jobs - pipeline = load_info.pipeline - with ( - pipeline.sql_client() as client, - client.execute_query("SELECT COUNT(*) FROM book") as cur, - ): - (row_count,) = cur.fetchone() - assert row_count == 2 # noqa: PLR2004 + assert fetch_all_duckdb(load_info.pipeline, "SELECT COUNT(*) FROM book") == [(2,)] -@pytest.mark.parametrize( - ("missing_args", "expected_fragment"), - [ - (["--table-name", "book", "--xml-tag", "book", "--xsd-file", "{xsd}"], "dataset_name"), - (["--dataset-name", "cli_dataset", "--xml-tag", "book", "--xsd-file", "{xsd}"], "table_name"), - (["--dataset-name", "cli_dataset", "--table-name", "book", "--xsd-file", "{xsd}"], "xml_tag"), - (["--dataset-name", "cli_dataset", "--table-name", "book", "--xml-tag", "book"], "xsd_file"), - ], -) +@pytest.mark.parametrize("missing_field", ["dataset_name", "table_name", "xml_tag", "xsd_file"]) def test_cli_fail_missing_required_args_raise( - tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], monkeypatch: pytest.MonkeyPatch, - missing_args: list[str], - expected_fragment: str, + missing_field: str, ) -> None: """cli() without required pipeline-specific args raises before running the pipeline.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - - resolved_args = [arg.format(xsd=library_xsd) for arg in missing_args] - argv = [ - PIPELINE_NAME, - "--input-dir", - str(tmp_path / "input"), - "--output-dir", - str(tmp_path / "output"), - "--log-config-file", - str(log_config_file), - *resolved_args, - ] - monkeypatch.setattr(sys, "argv", argv) + settings = settings_factory() + monkeypatch.setattr(sys, "argv", [PIPELINE_NAME, *settings_to_argv(settings, exclude=[missing_field])]) with ( patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline, - pytest.raises(Exception, match=expected_fragment), + pytest.raises(Exception, match=missing_field), ): cli() mock_run_pipeline.assert_not_called() + + +@pytest.mark.parametrize( + ("settings_cls", "run_fn", "expected_pipeline_name"), + [ + (UniProtXmlToPyArrowSettings, run_uniprot_pipeline, UNIPROT_PIPELINE_NAME), + (UniRefXmlToPyArrowSettings, run_uniref_pipeline, UNIREF_PIPELINE_NAME), + ], +) +def test_run_subclass_pipeline_pass_sets_core_run_pipeline_args_correctly( + settings_factory: Callable[..., Any], + *, + settings_cls: type[XmlToPyArrowSettings], + run_fn: Callable[[Any], LoadInfo | None], + expected_pipeline_name: str, +) -> None: + """run_uniprot_pipeline and run_uniref_pipeline delegate to run_pipeline with the subclass pipeline_name.""" + settings = settings_factory(settings_cls) + + with patch.object(xmltopyarrow_module, "run_pipeline") as mock_run_pipeline: + run_fn(settings) + + mock_run_pipeline.assert_called_once() + _, kwargs = mock_run_pipeline.call_args + assert kwargs["settings"] == settings + assert kwargs["pipeline_kwargs"] == { + "pipeline_name": expected_pipeline_name, + "dataset_name": expected_pipeline_name, + } + assert isinstance(kwargs["resource"], DltResource) + + +@pytest.mark.parametrize( + ("prog_name", "cli_obj", "sample_xml_rel_dir"), + [ + (UNIPROT_PIPELINE_NAME, uniprot_cli, "xsd/examples/uniprot"), + (UNIREF_PIPELINE_NAME, uniref_cli, "xsd/examples/uniref"), + ], +) +def test_subclass_cli_pass_runs_end_to_end_with_default_settings( + tmp_path: Path, + test_data_dir: Path, + settings_factory: Callable[..., Any], + monkeypatch: pytest.MonkeyPatch, + *, + prog_name: str, + cli_obj: Any, # noqa: ANN401 + sample_xml_rel_dir: str, +) -> None: + """UniProt and UniRef CLIs load entry rows end-to-end using their default settings.""" + base_settings = settings_factory(CtsSettings, dlt_dev_mode=False) + copy_test_files(test_data_dir / sample_xml_rel_dir, ["minimal.xml.gz"], Path(base_settings.input_dir)) + monkeypatch.setattr(sys, "argv", [prog_name, *settings_to_argv(base_settings)]) + + pipeline_name = f"test_{prog_name}_cli_pipeline_{uuid4().hex}" + + def fake_run_pipeline( + *, + resource: DltResource, + pipeline_kwargs: dict[str, Any], + **_: dict[str, Any], + ) -> LoadInfo | None: + assert pipeline_kwargs == {"pipeline_name": prog_name, "dataset_name": prog_name} + pipeline = duckdb_pipeline(tmp_path, pipeline_name, prog_name) + return pipeline.run(resource) + + with patch.object(xmltopyarrow_module, "run_pipeline", fake_run_pipeline): + load_info = assert_run_ok(cli_obj.run()) + + assert fetch_all_duckdb(load_info.pipeline, "SELECT COUNT(*) FROM entry") == [(1,)] diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py index df4ae695..3dfd03f5 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py @@ -2,6 +2,8 @@ from collections.abc import Callable from pathlib import Path +from typing import Any +from unittest.mock import patch import pyarrow as pa import pytest @@ -9,9 +11,22 @@ from pydantic import ValidationError from pydantic_settings import CliApp +import cdm_data_loaders.pipelines.xmltopyarrow.settings as xmltopyarrow_settings_module from cdm_data_loaders.core.fields import DEFAULTS -from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings +from cdm_data_loaders.core.settings import CtsSettings +from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( + UNIPROT_PIPELINE_NAME, + UNIPROT_XSD_FILE, + UNIREF_PIPELINE_NAME, + UNIREF_XSD_FILE, + UniProtXmlToPyArrowSettings, + UniRefXmlToPyArrowSettings, + XmlToPyArrowSettings, +) from cdm_data_loaders.readers.xsd import DEFAULT_MAX_RECURSION_DEPTH, generate_pyarrow_schema, load_schema +from cdm_data_loaders.schemas.uniprot.schema import SCHEMA as UNIPROT_SCHEMA +from cdm_data_loaders.schemas.uniref.schema import SCHEMA as UNIREF_SCHEMA +from tests.cdm_data_loaders.pipelines.helpers import settings_to_argv from tests.conftest import S3 UNIREF_LIKE_XSD: str = "uniref_like.xsd" @@ -175,39 +190,70 @@ def test_xmltopyarrow_settings_fail_xml_tag_not_in_xsd_raises_validation_error( @pytest.mark.parametrize("file_glob_arg", ["--file-glob", "-g"]) def test_xmltopyarrow_settings_pass_xml_tag_shortcut( - tmp_path: Path, - library_xsd: Path, + settings_factory: Callable[..., Any], file_glob_arg: str, ) -> None: """The file_glob field is exposed as both --file-glob and -g on the command line.""" - log_config_file = tmp_path / "logging.json" - log_config_file.write_text('{"version": 1}') - input_dir = tmp_path / "input" - input_dir.mkdir() - output_dir = tmp_path / "output" - output_dir.mkdir() - - argv = [ - "--input-dir", - str(input_dir), - "--output-dir", - str(output_dir), - "--use-destination", - "local_fs", - "--log-config-file", - str(log_config_file), - file_glob_arg, - "*.xml.gz", - "--dataset-name", - "cli_dataset", - "--table-name", - "book", - "--xml-tag", - "book", - "--xsd-file", - str(library_xsd), - ] + base_settings = settings_factory() + argv = [*settings_to_argv(base_settings, exclude=["file_glob"]), file_glob_arg, "*.xml.gz"] settings = CliApp.run(XmlToPyArrowSettings, cli_args=argv) assert settings.file_glob == "*.xml.gz" assert settings.xml_tag == "book" + + +def test_uniprot_xmltopyarrow_settings_pass_defaults( + settings_factory: Callable[..., Any], +) -> None: + """UniProtXmlToPyArrowSettings defaults dataset_name, table_name, xml_tag, xsd_file and uses precomputed SCHEMA.""" + with patch.object(xmltopyarrow_settings_module, "generate_pyarrow_schema") as mock_gen_schema: + settings = settings_factory(UniProtXmlToPyArrowSettings) + + mock_gen_schema.assert_not_called() + assert settings.dataset_name == UNIPROT_PIPELINE_NAME + assert settings.table_name == "entry" + assert settings.xml_tag == "entry" + assert settings.xsd_file == UNIPROT_XSD_FILE + assert isinstance(settings.xsd_schema, xmlschema.XMLSchema) + assert settings.pyarrow_schema is UNIPROT_SCHEMA + + +def test_uniref_xmltopyarrow_settings_pass_defaults( + settings_factory: Callable[..., Any], +) -> None: + """UniRefXmlToPyArrowSettings defaults dataset_name, table_name, xml_tag, xsd_file and uses precomputed SCHEMA.""" + with patch.object(xmltopyarrow_settings_module, "generate_pyarrow_schema") as mock_gen_schema: + settings = settings_factory(UniRefXmlToPyArrowSettings) + + mock_gen_schema.assert_not_called() + assert settings.dataset_name == UNIREF_PIPELINE_NAME + assert settings.table_name == "entry" + assert settings.xml_tag == "entry" + assert settings.xsd_file == UNIREF_XSD_FILE + assert isinstance(settings.xsd_schema, xmlschema.XMLSchema) + assert settings.pyarrow_schema is UNIREF_SCHEMA + + +@pytest.mark.parametrize( + ("settings_cls", "expected_dataset_name", "expected_xsd_file", "expected_schema"), + [ + (UniProtXmlToPyArrowSettings, UNIPROT_PIPELINE_NAME, UNIPROT_XSD_FILE, UNIPROT_SCHEMA), + (UniRefXmlToPyArrowSettings, UNIREF_PIPELINE_NAME, UNIREF_XSD_FILE, UNIREF_SCHEMA), + ], +) +def test_subclass_xmltopyarrow_settings_pass_cli_defaults( + settings_factory: Callable[..., Any], + *, + settings_cls: type[XmlToPyArrowSettings], + expected_dataset_name: str, + expected_xsd_file: Path, + expected_schema: pa.Schema, +) -> None: + """UniProt and UniRef settings subclasses can be parsed via CLI without specifying defaulted arguments.""" + base_settings = settings_factory(CtsSettings) + settings = CliApp.run(settings_cls, cli_args=settings_to_argv(base_settings)) + assert settings.dataset_name == expected_dataset_name + assert settings.table_name == "entry" + assert settings.xml_tag == "entry" + assert settings.xsd_file == expected_xsd_file + assert settings.pyarrow_schema is expected_schema diff --git a/uv.lock b/uv.lock index 26009a94..f244ce77 100644 --- a/uv.lock +++ b/uv.lock @@ -3,12 +3,12 @@ revision = 5 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.15' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version >= '3.15' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'emscripten'", "python_full_version < '3.14' and sys_platform == 'emscripten'", ] @@ -470,7 +470,7 @@ wheels = [ [[package]] name = "cdm-data-loaders" -version = "0.1.11" +version = "0.1.12" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },