From 9b4cc6de52d541d580baa2ea73e5d1bbe6083423 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 27 Oct 2026 11:44:53 -0700 Subject: [PATCH 1/2] Corrections to JSONSchema to pyarrow converter, plus addition of test cases. Add test and example cases for xsd to pyarrow. Autogenerate specs document from examples/tests. --- Dockerfile | 1 + docs/jsonschema_to_pyarrow_specification.md | 3231 +++++++++ docs/xsd_to_pyarrow_specification.md | 2855 +++++++- pyproject.toml | 7 +- scripts/gen_converter_spec.py | 352 + .../converters/jsonschema_to_pyarrow.py | 886 ++- src/cdm_data_loaders/pipelines/core.py | 6 +- src/cdm_data_loaders/readers/xsd.py | 121 +- .../test_jsonschema_to_pyarrow_spec.py | 225 + .../cdm_data_loaders/readers/test_xsd_spec.py | 598 +- tests/converter_spec.py | 188 + .../jsonschema/examples/cts_job/case.yaml | 17 + .../examples/cts_job/expected_schema.py | 635 ++ .../jsonschema/examples/cts_job/maximal.json | 748 +++ .../examples/cts_job/maximal_expected.json | 790 +++ .../cts_job/maximal_expected_table.py | 706 ++ .../jsonschema/examples/cts_job/minimal.json | 35 + .../examples/cts_job/minimal_expected.json | 35 + .../cts_job/minimal_expected_table.py | 38 + .../jsonschema/examples/cts_job/schema.json | 1013 +++ .../jsonschema/examples/cts_jobs/case.yaml | 17 + .../examples/cts_jobs/expected_schema.py | 554 ++ .../jsonschema/examples/cts_jobs/maximal.json | 847 +++ .../examples/cts_jobs/maximal_expected.json | 1127 ++++ .../cts_jobs/maximal_expected_table.py | 627 ++ .../jsonschema/examples/cts_jobs/minimal.json | 33 + .../examples/cts_jobs/minimal_expected.json | 33 + .../cts_jobs/minimal_expected_table.py | 36 + .../jsonschema/examples/cts_jobs/schema.json | 872 +++ .../examples/ncbi_annotation_report/case.yaml | 17 + .../ncbi_annotation_report/expected_schema.py | 234 + .../ncbi_annotation_report/maximal.json | 45 + .../maximal_expected.json | 45 + .../maximal_expected_table.py | 54 + .../ncbi_annotation_report/minimal.json | 6 + .../minimal_expected.json | 6 + .../minimal_expected_table.py | 13 + .../ncbi_annotation_report/schema.json | 153 + .../examples/uniprot_search/case.yaml | 18 + .../uniprot_search/expected_schema.py | 5905 +++++++++++++++++ .../examples/uniprot_search/maximal.json | 5704 ++++++++++++++++ .../uniprot_search/maximal_expected.json | 5704 ++++++++++++++++ .../uniprot_search/maximal_expected_table.py | 3503 ++++++++++ .../examples/uniprot_search/minimal.json | 10 + .../uniprot_search/minimal_expected.json | 10 + .../uniprot_search/minimal_expected_table.py | 13 + .../examples/uniprot_search/schema.json | 2384 +++++++ tests/data/jsonschema/rules.yaml | 175 + .../spec/01_scalars_and_formats/case.yaml | 11 + .../spec/01_scalars_and_formats/data.json | 56 + .../spec/01_scalars_and_formats/expected.json | 56 + .../01_scalars_and_formats/expected_schema.py | 57 + .../01_scalars_and_formats/expected_table.py | 63 + .../spec/01_scalars_and_formats/schema.json | 125 + .../02_nullability_and_required/case.yaml | 10 + .../02_nullability_and_required/data.json | 39 + .../02_nullability_and_required/expected.json | 39 + .../expected_schema.py | 49 + .../expected_table.py | 35 + .../02_nullability_and_required/schema.json | 84 + .../spec/03_arrays_and_tuples/case.yaml | 9 + .../spec/03_arrays_and_tuples/data.json | 54 + .../spec/03_arrays_and_tuples/expected.json | 54 + .../03_arrays_and_tuples/expected_schema.py | 93 + .../03_arrays_and_tuples/expected_table.py | 24 + .../spec/03_arrays_and_tuples/schema.json | 94 + .../spec/04_objects_and_maps/case.yaml | 10 + .../spec/04_objects_and_maps/data.json | 40 + .../spec/04_objects_and_maps/expected.json | 40 + .../04_objects_and_maps/expected_schema.py | 74 + .../04_objects_and_maps/expected_table.py | 22 + .../spec/04_objects_and_maps/schema.json | 84 + .../spec/05_defs_and_refs/case.yaml | 9 + .../spec/05_defs_and_refs/data.json | 26 + .../spec/05_defs_and_refs/expected.json | 26 + .../spec/05_defs_and_refs/expected_schema.py | 48 + .../spec/05_defs_and_refs/expected_table.py | 22 + .../spec/05_defs_and_refs/schema.json | 57 + .../spec/06_recursion_depth/case.yaml | 16 + .../spec/06_recursion_depth/data.json | 48 + .../spec/06_recursion_depth/expected.json | 36 + .../06_recursion_depth/expected_depth_1.json | 22 + .../06_recursion_depth/expected_schema.py | 171 + .../spec/06_recursion_depth/expected_table.py | 68 + .../spec/06_recursion_depth/schema.json | 71 + .../case.yaml | 11 + .../data.json | 36 + .../expected.json | 38 + .../expected_schema.py | 55 + .../expected_table.py | 29 + .../schema.json | 143 + .../case.yaml | 9 + .../data.json | 16 + .../expected.json | 16 + .../expected_schema.py | 32 + .../expected_table.py | 21 + .../schema.json | 73 + .../case.yaml | 10 + .../data.json | 41 + .../expected.json | 41 + .../expected_schema.py | 61 + .../expected_table.py | 21 + .../schema.json | 62 + .../jsonschema/spec/10_combinations/case.yaml | 10 + .../jsonschema/spec/10_combinations/data.json | 65 + .../spec/10_combinations/expected.json | 62 + .../spec/10_combinations/expected_schema.py | 105 + .../spec/10_combinations/expected_table.py | 52 + .../spec/10_combinations/schema.json | 165 + .../spec/errors/array_top_level.json | 8 + .../spec/errors/boolean_property_schema.json | 10 + .../spec/errors/const_null_field.json | 12 + .../jsonschema/spec/errors/dynamic_ref.json | 9 + .../spec/errors/empty_object_nested.json | 17 + .../spec/errors/empty_object_top_level.json | 6 + .../spec/errors/enum_null_field.json | 14 + .../errors/incompatible_branch_shapes.json | 25 + .../data/jsonschema/spec/errors/manifest.yaml | 89 + .../jsonschema/spec/errors/map_top_level.json | 8 + .../spec/errors/missing_schema_dialect.json | 10 + .../spec/errors/missing_schema_keyword.json | 8 + .../spec/errors/mixed_properties_and_map.json | 18 + .../spec/errors/negation_not_keyword.json | 12 + .../spec/errors/null_only_field.json | 13 + .../spec/errors/open_tuple_prefix_items.json | 21 + .../spec/errors/recursive_binary_type.json | 23 + .../spec/errors/recursive_date_type.json | 24 + .../spec/errors/scalar_top_level.json | 5 + .../spec/errors/unconstrained_property.json | 14 + .../spec/errors/unresolvable_ref.json | 10 + .../spec/errors/value_field_collision.json | 24 + .../xsd/examples/pubmed_article/case.yaml | 18 + .../pubmed_article/maximal_expected_table.py | 583 ++ .../pubmed_article/minimal_expected_table.py | 31 + .../examples/pubmed_book_article/case.yaml | 18 + .../maximal_expected_table.py | 705 ++ .../minimal_expected_table.py | 23 + tests/data/xsd/examples/uniprot/case.yaml | 18 + .../uniprot/maximal_expected_table.py | 1269 ++++ .../uniprot/minimal_expected_table.py | 43 + tests/data/xsd/examples/uniref/case.yaml | 18 + .../examples/uniref/maximal_expected_table.py | 67 + .../examples/uniref/minimal_expected_table.py | 20 + tests/data/xsd/rules.yaml | 169 + .../01_scalars_and_simple_types/case.yaml | 11 + .../expected_table.py | 67 + .../spec/02_namespaces_and_nillable/case.yaml | 16 + .../expected_table.py | 18 + .../spec/03_mixed_content_raw_text/case.yaml | 11 + .../expected_table.py | 22 + .../xsd/spec/04_recursion_depth/case.yaml | 17 + .../spec/04_recursion_depth/expected_table.py | 53 + .../spec/05_groups_and_compositors/case.yaml | 12 + .../expected_table.py | 21 + .../spec/06_context_dependent_paths/case.yaml | 10 + .../expected_table.py | 16 + .../spec/07_multi_root_shared_type/case.yaml | 10 + .../expected_table.py | 14 + .../08_nullability_and_empty_tags/case.yaml | 12 + .../expected_table.py | 13 + .../spec/09_empty_marker_elements/case.yaml | 10 + .../expected_table.py | 14 + tests/data/xsd/spec/10_combinations/case.yaml | 12 + .../spec/10_combinations/expected_table.py | 101 + tests/data/xsd/spec/errors/manifest.yaml | 84 + .../errors/nested_binary_under_recursion.xsd | 16 + .../errors/nested_date_under_recursion.xsd | 16 + tests/pyarrow_spec_helpers.py | 132 + uv.lock | 15 +- 169 files changed, 46897 insertions(+), 836 deletions(-) create mode 100644 docs/jsonschema_to_pyarrow_specification.md create mode 100644 scripts/gen_converter_spec.py create mode 100644 tests/cdm_data_loaders/converters/test_jsonschema_to_pyarrow_spec.py create mode 100644 tests/converter_spec.py create mode 100644 tests/data/jsonschema/examples/cts_job/case.yaml create mode 100644 tests/data/jsonschema/examples/cts_job/expected_schema.py create mode 100644 tests/data/jsonschema/examples/cts_job/maximal.json create mode 100644 tests/data/jsonschema/examples/cts_job/maximal_expected.json create mode 100644 tests/data/jsonschema/examples/cts_job/maximal_expected_table.py create mode 100644 tests/data/jsonschema/examples/cts_job/minimal.json create mode 100644 tests/data/jsonschema/examples/cts_job/minimal_expected.json create mode 100644 tests/data/jsonschema/examples/cts_job/minimal_expected_table.py create mode 100644 tests/data/jsonschema/examples/cts_job/schema.json create mode 100644 tests/data/jsonschema/examples/cts_jobs/case.yaml create mode 100644 tests/data/jsonschema/examples/cts_jobs/expected_schema.py create mode 100644 tests/data/jsonschema/examples/cts_jobs/maximal.json create mode 100644 tests/data/jsonschema/examples/cts_jobs/maximal_expected.json create mode 100644 tests/data/jsonschema/examples/cts_jobs/maximal_expected_table.py create mode 100644 tests/data/jsonschema/examples/cts_jobs/minimal.json create mode 100644 tests/data/jsonschema/examples/cts_jobs/minimal_expected.json create mode 100644 tests/data/jsonschema/examples/cts_jobs/minimal_expected_table.py create mode 100644 tests/data/jsonschema/examples/cts_jobs/schema.json create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/case.yaml create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/expected_schema.py create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/maximal.json create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected.json create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected_table.py create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/minimal.json create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected.json create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected_table.py create mode 100644 tests/data/jsonschema/examples/ncbi_annotation_report/schema.json create mode 100644 tests/data/jsonschema/examples/uniprot_search/case.yaml create mode 100644 tests/data/jsonschema/examples/uniprot_search/expected_schema.py create mode 100644 tests/data/jsonschema/examples/uniprot_search/maximal.json create mode 100644 tests/data/jsonschema/examples/uniprot_search/maximal_expected.json create mode 100644 tests/data/jsonschema/examples/uniprot_search/maximal_expected_table.py create mode 100644 tests/data/jsonschema/examples/uniprot_search/minimal.json create mode 100644 tests/data/jsonschema/examples/uniprot_search/minimal_expected.json create mode 100644 tests/data/jsonschema/examples/uniprot_search/minimal_expected_table.py create mode 100644 tests/data/jsonschema/examples/uniprot_search/schema.json create mode 100644 tests/data/jsonschema/rules.yaml create mode 100644 tests/data/jsonschema/spec/01_scalars_and_formats/case.yaml create mode 100644 tests/data/jsonschema/spec/01_scalars_and_formats/data.json create mode 100644 tests/data/jsonschema/spec/01_scalars_and_formats/expected.json create mode 100644 tests/data/jsonschema/spec/01_scalars_and_formats/expected_schema.py create mode 100644 tests/data/jsonschema/spec/01_scalars_and_formats/expected_table.py create mode 100644 tests/data/jsonschema/spec/01_scalars_and_formats/schema.json create mode 100644 tests/data/jsonschema/spec/02_nullability_and_required/case.yaml create mode 100644 tests/data/jsonschema/spec/02_nullability_and_required/data.json create mode 100644 tests/data/jsonschema/spec/02_nullability_and_required/expected.json create mode 100644 tests/data/jsonschema/spec/02_nullability_and_required/expected_schema.py create mode 100644 tests/data/jsonschema/spec/02_nullability_and_required/expected_table.py create mode 100644 tests/data/jsonschema/spec/02_nullability_and_required/schema.json create mode 100644 tests/data/jsonschema/spec/03_arrays_and_tuples/case.yaml create mode 100644 tests/data/jsonschema/spec/03_arrays_and_tuples/data.json create mode 100644 tests/data/jsonschema/spec/03_arrays_and_tuples/expected.json create mode 100644 tests/data/jsonschema/spec/03_arrays_and_tuples/expected_schema.py create mode 100644 tests/data/jsonschema/spec/03_arrays_and_tuples/expected_table.py create mode 100644 tests/data/jsonschema/spec/03_arrays_and_tuples/schema.json create mode 100644 tests/data/jsonschema/spec/04_objects_and_maps/case.yaml create mode 100644 tests/data/jsonschema/spec/04_objects_and_maps/data.json create mode 100644 tests/data/jsonschema/spec/04_objects_and_maps/expected.json create mode 100644 tests/data/jsonschema/spec/04_objects_and_maps/expected_schema.py create mode 100644 tests/data/jsonschema/spec/04_objects_and_maps/expected_table.py create mode 100644 tests/data/jsonschema/spec/04_objects_and_maps/schema.json create mode 100644 tests/data/jsonschema/spec/05_defs_and_refs/case.yaml create mode 100644 tests/data/jsonschema/spec/05_defs_and_refs/data.json create mode 100644 tests/data/jsonschema/spec/05_defs_and_refs/expected.json create mode 100644 tests/data/jsonschema/spec/05_defs_and_refs/expected_schema.py create mode 100644 tests/data/jsonschema/spec/05_defs_and_refs/expected_table.py create mode 100644 tests/data/jsonschema/spec/05_defs_and_refs/schema.json create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/case.yaml create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/data.json create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/expected.json create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/expected_depth_1.json create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/expected_schema.py create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/expected_table.py create mode 100644 tests/data/jsonschema/spec/06_recursion_depth/schema.json create mode 100644 tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/case.yaml create mode 100644 tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/data.json create mode 100644 tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected.json create mode 100644 tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_schema.py create mode 100644 tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_table.py create mode 100644 tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/schema.json create mode 100644 tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/case.yaml create mode 100644 tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/data.json create mode 100644 tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected.json create mode 100644 tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_schema.py create mode 100644 tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_table.py create mode 100644 tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/schema.json create mode 100644 tests/data/jsonschema/spec/09_context_dependent_shared_defs/case.yaml create mode 100644 tests/data/jsonschema/spec/09_context_dependent_shared_defs/data.json create mode 100644 tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected.json create mode 100644 tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_schema.py create mode 100644 tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_table.py create mode 100644 tests/data/jsonschema/spec/09_context_dependent_shared_defs/schema.json create mode 100644 tests/data/jsonschema/spec/10_combinations/case.yaml create mode 100644 tests/data/jsonschema/spec/10_combinations/data.json create mode 100644 tests/data/jsonschema/spec/10_combinations/expected.json create mode 100644 tests/data/jsonschema/spec/10_combinations/expected_schema.py create mode 100644 tests/data/jsonschema/spec/10_combinations/expected_table.py create mode 100644 tests/data/jsonschema/spec/10_combinations/schema.json create mode 100644 tests/data/jsonschema/spec/errors/array_top_level.json create mode 100644 tests/data/jsonschema/spec/errors/boolean_property_schema.json create mode 100644 tests/data/jsonschema/spec/errors/const_null_field.json create mode 100644 tests/data/jsonschema/spec/errors/dynamic_ref.json create mode 100644 tests/data/jsonschema/spec/errors/empty_object_nested.json create mode 100644 tests/data/jsonschema/spec/errors/empty_object_top_level.json create mode 100644 tests/data/jsonschema/spec/errors/enum_null_field.json create mode 100644 tests/data/jsonschema/spec/errors/incompatible_branch_shapes.json create mode 100644 tests/data/jsonschema/spec/errors/manifest.yaml create mode 100644 tests/data/jsonschema/spec/errors/map_top_level.json create mode 100644 tests/data/jsonschema/spec/errors/missing_schema_dialect.json create mode 100644 tests/data/jsonschema/spec/errors/missing_schema_keyword.json create mode 100644 tests/data/jsonschema/spec/errors/mixed_properties_and_map.json create mode 100644 tests/data/jsonschema/spec/errors/negation_not_keyword.json create mode 100644 tests/data/jsonschema/spec/errors/null_only_field.json create mode 100644 tests/data/jsonschema/spec/errors/open_tuple_prefix_items.json create mode 100644 tests/data/jsonschema/spec/errors/recursive_binary_type.json create mode 100644 tests/data/jsonschema/spec/errors/recursive_date_type.json create mode 100644 tests/data/jsonschema/spec/errors/scalar_top_level.json create mode 100644 tests/data/jsonschema/spec/errors/unconstrained_property.json create mode 100644 tests/data/jsonschema/spec/errors/unresolvable_ref.json create mode 100644 tests/data/jsonschema/spec/errors/value_field_collision.json create mode 100644 tests/data/xsd/examples/pubmed_article/case.yaml create mode 100644 tests/data/xsd/examples/pubmed_article/maximal_expected_table.py create mode 100644 tests/data/xsd/examples/pubmed_article/minimal_expected_table.py create mode 100644 tests/data/xsd/examples/pubmed_book_article/case.yaml create mode 100644 tests/data/xsd/examples/pubmed_book_article/maximal_expected_table.py create mode 100644 tests/data/xsd/examples/pubmed_book_article/minimal_expected_table.py create mode 100644 tests/data/xsd/examples/uniprot/case.yaml create mode 100644 tests/data/xsd/examples/uniprot/maximal_expected_table.py create mode 100644 tests/data/xsd/examples/uniprot/minimal_expected_table.py create mode 100644 tests/data/xsd/examples/uniref/case.yaml create mode 100644 tests/data/xsd/examples/uniref/maximal_expected_table.py create mode 100644 tests/data/xsd/examples/uniref/minimal_expected_table.py create mode 100644 tests/data/xsd/rules.yaml create mode 100644 tests/data/xsd/spec/01_scalars_and_simple_types/case.yaml create mode 100644 tests/data/xsd/spec/01_scalars_and_simple_types/expected_table.py create mode 100644 tests/data/xsd/spec/02_namespaces_and_nillable/case.yaml create mode 100644 tests/data/xsd/spec/02_namespaces_and_nillable/expected_table.py create mode 100644 tests/data/xsd/spec/03_mixed_content_raw_text/case.yaml create mode 100644 tests/data/xsd/spec/03_mixed_content_raw_text/expected_table.py create mode 100644 tests/data/xsd/spec/04_recursion_depth/case.yaml create mode 100644 tests/data/xsd/spec/04_recursion_depth/expected_table.py create mode 100644 tests/data/xsd/spec/05_groups_and_compositors/case.yaml create mode 100644 tests/data/xsd/spec/05_groups_and_compositors/expected_table.py create mode 100644 tests/data/xsd/spec/06_context_dependent_paths/case.yaml create mode 100644 tests/data/xsd/spec/06_context_dependent_paths/expected_table.py create mode 100644 tests/data/xsd/spec/07_multi_root_shared_type/case.yaml create mode 100644 tests/data/xsd/spec/07_multi_root_shared_type/expected_table.py create mode 100644 tests/data/xsd/spec/08_nullability_and_empty_tags/case.yaml create mode 100644 tests/data/xsd/spec/08_nullability_and_empty_tags/expected_table.py create mode 100644 tests/data/xsd/spec/09_empty_marker_elements/case.yaml create mode 100644 tests/data/xsd/spec/09_empty_marker_elements/expected_table.py create mode 100644 tests/data/xsd/spec/10_combinations/case.yaml create mode 100644 tests/data/xsd/spec/10_combinations/expected_table.py create mode 100644 tests/data/xsd/spec/errors/manifest.yaml create mode 100644 tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd create mode 100644 tests/data/xsd/spec/errors/nested_date_under_recursion.xsd create mode 100644 tests/pyarrow_spec_helpers.py diff --git a/Dockerfile b/Dockerfile index 65fccab5..aaf75304 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,6 +94,7 @@ ENV HOME="/home/nonroot" # Place executables in the environment at the front of the path ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONUNBUFFERED=1 USER nonroot ENTRYPOINT ["./scripts/entrypoint.sh"] diff --git a/docs/jsonschema_to_pyarrow_specification.md b/docs/jsonschema_to_pyarrow_specification.md new file mode 100644 index 00000000..9fd7e50d --- /dev/null +++ b/docs/jsonschema_to_pyarrow_specification.md @@ -0,0 +1,3231 @@ + + +# JSON Schema (Draft 2020-12) to PyArrow Specification + +This document specifies how a **JSON Schema Draft 2020-12** document determines: + +1. The **PyArrow `pa.Schema`** produced by `generate_pyarrow_schema`. +2. The **Python row dictionaries** produced by `coerce_record_for_pyarrow` from schema-valid JSON records, + ready for `pa.Table.from_pylist(rows, schema=pa_schema)` and Parquet serialization. +3. The **errors** raised for constructs that cannot be represented as a static columnar schema. + +**Contents:** [1. Conversion Rules](#1-conversion-rules) · [2. Cases](#2-cases) · [3. Errors](#3-errors-unsupported--invalid-input) · [4. Fixture Index](#4-fixture-index) + +## 1. Conversion Rules + +### 1.1 Top-Level Resolution and Unwrapping + + + +- **R-DIALECT**: The root must declare `"$schema": "https://json-schema.org/draft/2020-12/schema"` or the KBase + XSV metaschema `"https://json-schema.kbase.us/draft/2020-12-xsv/schema"`; anything else raises **E1**. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats) + + + +- **R-ROOT-REF**: A root consisting of `"$ref": "#/$defs/"` (without its own `properties`) is replaced by the + referenced definition before classification. That definition counts as one expansion for recursion. + + *Cases:* [05_defs_and_refs](#case-spec-05-defs-and-refs), [ncbi_annotation_report](#case-examples-ncbi-annotation-report) + + + +- **R-UNWRAP**: The resolved root must be a fixed-property object producing at least one field. Its fields become + the top-level columns. A scalar, array or pure-map root raises **E2**; an object with no fields raises + **E3**. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats) + +### 1.2 Naming and Field Ordering + + + +- **R-NAME-PROPERTY**: Property keys are used verbatim as field names. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats) + + + +- **R-ORDER**: Struct fields are emitted in this order: properties from each `allOf` subschema in list order, then + direct `properties` in declaration order, then properties contributed by `anyOf`, `oneOf`, + `dependentSchemas`, `then` and `else`. A property seen more than once keeps the position of its + first occurrence. + + *Cases:* [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof), [08_conditional_and_dependent_schemas](#case-spec-08-conditional-and-dependent-schemas), [10_combinations](#case-spec-10-combinations) + + + +- **R-MERGE-DUPLICATE**: When a property is declared more than once (across `allOf`, `anyOf`/`oneOf`, `dependentSchemas` or + `then`/`else`), identical declarations collapse to one; differing declarations are combined as an + `anyOf` of the declarations and resolved with the union rules ([R-UNION-SCALAR](#r-union-scalar), + [R-UNION-OBJECT](#r-union-object)). The first declared `description` is kept. + + *Cases:* [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof) + +### 1.3 Construct Classification (Shape) + + + +- **R-SHAPE-OBJECT**: An object with `properties` (or object combinators) maps to `pa.struct([...])`. Boolean + `additionalProperties` (`true`, `false` or omitted) on such an object is ignored. + + *Cases:* [04_objects_and_maps](#case-spec-04-objects-and-maps) + + + +- **R-SHAPE-ARRAY**: `type: "array"` with `items: ` maps to `pa.list_(pa.field("item", ))`; nested + arrays nest lists. + + *Cases:* [03_arrays_and_tuples](#case-spec-03-arrays-and-tuples), [cts_jobs](#case-examples-cts-jobs), [ncbi_annotation_report](#case-examples-ncbi-annotation-report) + + + +- **R-SHAPE-TUPLE**: A closed tuple (`prefixItems: [S_0, ..., S_k-1]` with `items` `false` or omitted) maps to + `pa.struct` with fields `item_0` ... `item_k-1`. Any other `items` value raises **E5**. + + *Cases:* [03_arrays_and_tuples](#case-spec-03-arrays-and-tuples), [10_combinations](#case-spec-10-combinations) + + + +- **R-SHAPE-MAP**: An object with schema-valued `additionalProperties` and/or `patternProperties` and no fixed + properties maps to `pa.map_(pa.string(), pa.field("value", ))`. Several value schemas are + combined with the union rules. Mixing fixed properties with a map schema raises **E6**. + + *Cases:* [04_objects_and_maps](#case-spec-04-objects-and-maps), [10_combinations](#case-spec-10-combinations), [cts_job](#case-examples-cts-job), [uniprot_search](#case-examples-uniprot-search) + + + +- **R-SHAPE-REF**: `"$ref": "#/$defs/"` is replaced by the definition; sibling keywords on the referencing node + are overlaid on the definition. Only local `#/$defs/` pointers are supported (**E9**). + + *Cases:* [05_defs_and_refs](#case-spec-05-defs-and-refs), [09_context_dependent_shared_defs](#case-spec-09-context-dependent-shared-defs), [cts_job](#case-examples-cts-job), [cts_jobs](#case-examples-cts-jobs), [ncbi_annotation_report](#case-examples-ncbi-annotation-report) + + + +- **R-RECURSION**: Each `$defs` target may be expanded at most `max_recursion_depth` times (default `2`) along a path. + A deeper reference becomes `pa.string()` (or `pa.list_(pa.string())` inside an array), carries no + `jsonschema_type`, and its runtime value is serialized with `json.dumps`. Applies to self-recursion + and mutual recursion. + + *Cases:* [06_recursion_depth](#case-spec-06-recursion-depth), [10_combinations](#case-spec-10-combinations), [uniprot_search](#case-examples-uniprot-search) + +### 1.4 Field Nullability + + + +- **R-NULL-REQUIRED**: A property field is `nullable=False` only when it is in the enclosing object's effective `required` + set and its schema does not allow `null`. The effective `required` set is the union of `required` + across the object and its `allOf` subschemas, plus the intersection of `required` across + `anyOf`/`oneOf` object branches. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [02_nullability_and_required](#case-spec-02-nullability-and-required), [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof), [09_context_dependent_shared_defs](#case-spec-09-context-dependent-shared-defs) + + + +- **R-NULL-ALLOWS-NULL**: A schema allows `null` when `"null"` is in `type`, `null` is in `enum` or is the `const`, or any + `anyOf`/`oneOf`/`allOf` branch (or `$ref` target) allows `null`. Such fields are `nullable=True` even + when required. + + *Cases:* [02_nullability_and_required](#case-spec-02-nullability-and-required), [09_context_dependent_shared_defs](#case-spec-09-context-dependent-shared-defs), [cts_job](#case-examples-cts-job) + + + +- **R-NULL-CONDITIONAL**: Properties introduced only by `dependentSchemas`, `then` or `else` are `nullable=True`; + `dependentRequired` does not affect nullability. + + *Cases:* [08_conditional_and_dependent_schemas](#case-spec-08-conditional-and-dependent-schemas), [10_combinations](#case-spec-10-combinations) + + + +- **R-NULL-DEFAULT**: `default` is an annotation only: it neither changes nullability nor fills in missing values. + + *Cases:* [02_nullability_and_required](#case-spec-02-nullability-and-required) + + + +- **R-NULL-ITEMS**: A list `item` field and a map `value` field are `nullable=False` unless the item/value schema allows + `null`. A tuple field `item_i` is `nullable=False` only when `minItems >= i + 1` and `prefixItems[i]` + does not allow `null`. + + *Cases:* [02_nullability_and_required](#case-spec-02-nullability-and-required), [03_arrays_and_tuples](#case-spec-03-arrays-and-tuples), [04_objects_and_maps](#case-spec-04-objects-and-maps), [09_context_dependent_shared_defs](#case-spec-09-context-dependent-shared-defs) + +### 1.5 Scalar Types and Unions + + + +- **R-SCALAR-TYPE**: Scalar `type`, `format` and `contentEncoding` map to Arrow types per the table below. Numeric + validation keywords (`minimum`, `maximum`, `exclusiveMinimum`, `exclusiveMaximum`, `multipleOf`) do + not change the type. Only `format: "date"` becomes a temporal type; other formats stay `pa.string()`. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [10_combinations](#case-spec-10-combinations), [uniprot_search](#case-examples-uniprot-search) + + + +- **R-SCALAR-ENUM-CONST**: With an explicit `type`, `enum`/`const` use that type. Without one, each non-null literal's type is + inferred (`bool` -> boolean, `int` -> integer, `float` -> number, `str` -> string) and combined with + the union rules. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [02_nullability_and_required](#case-spec-02-nullability-and-required), [10_combinations](#case-spec-10-combinations), [ncbi_annotation_report](#case-examples-ncbi-annotation-report) + + + +- **R-UNION-SCALAR**: For `type: [...]` and scalar-only `anyOf`/`oneOf`, `null` members are dropped (making the field + nullable). If the rest share one Arrow type it is used; if they are only `integer` and `number` the + type is `pa.float64()`; otherwise `pa.string()`. `jsonschema_type` joins the distinct member labels + with `" | "` in declaration order. `object` or `array` in a `type` list raises **E8**. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof), [10_combinations](#case-spec-10-combinations), [cts_job](#case-examples-cts-job) + + + +- **R-UNION-OBJECT**: `anyOf`/`oneOf` whose non-null branches are all objects merge their properties into one struct. + Branches of other structural shapes (array, map) raise **E8**. + + *Cases:* [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof), [10_combinations](#case-spec-10-combinations), [uniprot_search](#case-examples-uniprot-search) + + + +- **R-UNION-SCALAR-OBJECT**: `anyOf`/`oneOf` mixing object and scalar branches yields a struct of the merged object properties, + all `nullable=True`, followed by a synthesized `_value` field holding the unified scalar type. A + declared `_value` property raises **E12**. + + *Cases:* [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof), [uniprot_search](#case-examples-uniprot-search) + +| JSON Schema | Python type (coerced) | PyArrow `DataType` | `metadata["jsonschema_type"]` | +| :--- | :--- | :--- | :--- | +| `{"type": "boolean"}` | `bool` | `pa.bool_()` | `boolean` | +| `{"type": "integer"}` | `int` | `pa.int64()` | `integer` | +| `{"type": "number"}` | `float` | `pa.float64()` | `number` | +| `{"type": "string"}` | `str` | `pa.string()` | `string` | +| `{"type": "string", "format": "date"}` | `datetime.date` | `pa.date32()` | `string (date)` | +| `{"type": "string", "format": "date-time"}` | `str` | `pa.string()` | `string (date-time)` | +| `{"type": "string", "format": "time"}` | `str` | `pa.string()` | `string (time)` | +| `{"type": "string", "format": "duration"}` | `str` | `pa.string()` | `string (duration)` | +| `{"type": "string", "format": "uuid"}` | `str` | `pa.string()` | `string (uuid)` | +| `{"type": "string", "format": "uri"}` | `str` | `pa.string()` | `string (uri)` | +| `{"type": "string", "format": "email"}` | `str` | `pa.string()` | `string (email)` | +| `{"type": "string", "contentEncoding": "base64"}` | `bytes` | `pa.binary()` | `string (base64)` | +| `{"type": "string", "contentEncoding": "base16"}` | `bytes` | `pa.binary()` | `string (base16)` | +| `{"enum": [1, 2]}` | `int` | `pa.int64()` | `integer` | +| `{"enum": [1, 2.5]}` | `float` | `pa.float64()` | `integer \| number` | +| `{"enum": [1, "two", true]}` | `str` | `pa.string()` | `integer \| string \| boolean` | +| `{"type": ["integer", "number"]}` | `float` | `pa.float64()` | `integer \| number` | +| `{"type": ["integer", "string"]}` | `str` | `pa.string()` | `integer \| string` | +| `{"type": ["string", "null"]}` | `str` | `pa.string()` | `string` | + +### 1.6 Field Metadata + + + +- **R-META-JSONSCHEMA-TYPE**: Every scalar field (including scalar list `item`, map `value` and tuple `item_i` fields) carries + `metadata["jsonschema_type"]`. Struct, list, map and recursion-truncated fields do not. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats) + + + +- **R-META-COMMENT**: A non-empty `description` is stored in `metadata["comment"]`. If the node has none, it is inherited + from its `$ref` target, or from the only non-null `$ref` branch of an `anyOf`/`oneOf`. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [03_arrays_and_tuples](#case-spec-03-arrays-and-tuples), [05_defs_and_refs](#case-spec-05-defs-and-refs), [09_context_dependent_shared_defs](#case-spec-09-context-dependent-shared-defs) + +### 1.7 Record Coercion + + + +- **R-COERCE-SCALAR**: Strings for `pa.date32()` are parsed with `datetime.date.fromisoformat`; `pa.binary()` strings are + decoded as Base16 (`bytes.fromhex`) when `jsonschema_type` mentions `base16`, otherwise Base64. Ints + become floats for float fields, and integral floats become ints for integer fields. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [10_combinations](#case-spec-10-combinations) + + + +- **R-COERCE-STRING-FALLBACK**: Non-string values written to a `pa.string()` field are stringified: booleans, objects and arrays with + `json.dumps` (`true`/`false`), numbers with `str()`. This covers heterogeneous scalar unions and + recursion-truncated subtrees. + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats), [06_recursion_depth](#case-spec-06-recursion-depth) + + + +- **R-COERCE-TUPLE**: A JSON array for a closed tuple becomes `{"item_0": v_0, ...}`; positions that are missing or `null` + are omitted. + + *Cases:* [03_arrays_and_tuples](#case-spec-03-arrays-and-tuples) + + + +- **R-COERCE-VALUE**: A scalar written to a scalar + object union struct becomes `{"_value": }`. + + *Cases:* [07_combinators_allof_anyof_oneof](#case-spec-07-combinators-allof-anyof-oneof) + + + +- **R-COERCE-SPARSE**: Only keys declared in the schema are kept; absent keys stay absent (they become null in the table). + + *Cases:* [01_scalars_and_formats](#case-spec-01-scalars-and-formats) + +## 2. Cases + + + +### Scalars, Formats, Binary Encodings, Enums and Scalar Unions (`spec/01_scalars_and_formats`) + +**Rules:** [R-DIALECT](#r-dialect), [R-UNWRAP](#r-unwrap), [R-NAME-PROPERTY](#r-name-property), [R-SCALAR-TYPE](#r-scalar-type), [R-SCALAR-ENUM-CONST](#r-scalar-enum-const), [R-UNION-SCALAR](#r-union-scalar), [R-META-JSONSCHEMA-TYPE](#r-meta-jsonschema-type), [R-META-COMMENT](#r-meta-comment), [R-NULL-REQUIRED](#r-null-required), [R-COERCE-SCALAR](#r-coerce-scalar), [R-COERCE-STRING-FALLBACK](#r-coerce-string-fallback), [R-COERCE-SPARSE](#r-coerce-sparse) + +- `untyped_mixed_enum` (`enum: [1, "two", true]`) falls back to `pa.string()`; the value `1` is coerced to `"1"`. +- `numeric_union` (`["integer", "number"]`) promotes to `pa.float64()`; `scalar_union` (`["integer", "string"]`) falls back to `pa.string()`. +- `date_or_year_union` mixes `string (date)` (`pa.date32()`) and `string`, so it falls back to `pa.string()`. +- The second record omits every optional key; they stay absent in the coerced row. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/01_scalars_and_formats.schema.json", + "title": "ScalarsAndFormatsRecord", + "type": "object", + "required": [ + "id", + "status", + "version", + "flag" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique record identifier." + }, + "status": { + "type": "string", + "enum": [ + "active", + "archived", + "pending" + ], + "description": "Record lifecycle status." + }, + "version": { + "type": "integer", + "minimum": 1, + "maximum": 32767 + }, + "flag": { + "type": "boolean", + "description": "Primary boolean flag." + }, + "bounded_int": { + "type": "integer", + "exclusiveMinimum": -1000, + "exclusiveMaximum": 1000 + }, + "big_int": { + "type": "integer" + }, + "float_val": { + "type": "number" + }, + "decimal_multiple": { + "type": "number", + "multipleOf": 0.01 + }, + "created_date": { + "type": "string", + "format": "date" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "event_time": { + "type": "string", + "format": "time" + }, + "elapsed": { + "type": "string", + "format": "duration" + }, + "record_uuid": { + "type": "string", + "format": "uuid" + }, + "homepage": { + "type": "string", + "format": "uri" + }, + "const_tag": { + "const": "v1" + }, + "untyped_int_enum": { + "enum": [ + 10, + 20, + 30 + ] + }, + "untyped_mixed_enum": { + "enum": [ + 1, + "two", + true + ] + }, + "numeric_union": { + "type": [ + "integer", + "number" + ] + }, + "scalar_union": { + "type": [ + "integer", + "string" + ] + }, + "date_or_year_union": { + "anyOf": [ + { + "type": "string", + "format": "date" + }, + { + "type": "string", + "pattern": "^[0-9]{4}$" + } + ] + }, + "raw_b64": { + "type": "string", + "contentEncoding": "base64" + }, + "raw_hex": { + "type": "string", + "contentEncoding": "base16" + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "id": "rec1", + "status": "active", + "version": 2, + "flag": true, + "bounded_int": -42, + "big_int": 999999999999, + "float_val": 2.718281828, + "decimal_multiple": 3.14, + "created_date": "2025-06-15", + "updated_at": "2025-06-15T12:30:00Z", + "event_time": "12:30:00Z", + "elapsed": "P3DT4H", + "record_uuid": "550e8400-e29b-41d4-a716-446655440000", + "homepage": "https://example.org/rec1", + "const_tag": "v1", + "untyped_int_enum": 20, + "untyped_mixed_enum": 1, + "numeric_union": 42, + "scalar_union": 100, + "date_or_year_union": "2025-06-15", + "raw_b64": "SGVsbG8=", + "raw_hex": "48656c6c6f" + }, + { + "id": "rec2", + "status": "archived", + "version": 1, + "flag": false + }, + { + "id": "rec3", + "status": "pending", + "version": 32767, + "flag": true, + "bounded_int": 999, + "big_int": -999999999999, + "float_val": -0.5, + "decimal_multiple": 42.0, + "created_date": "1999-12-31", + "updated_at": "2025-06-15T12:30:00.123+02:00", + "event_time": "23:59:59+02:00", + "elapsed": "PT15M", + "record_uuid": "123e4567-e89b-12d3-a456-426614174000", + "homepage": "https://example.org/rec3", + "const_tag": "v1", + "untyped_int_enum": 30, + "untyped_mixed_enum": true, + "numeric_union": 3.5, + "scalar_union": "hundred", + "date_or_year_union": "1999", + "raw_b64": "V29ybGQ=", + "raw_hex": "576f726c64" + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "id": "rec1", + "status": "active", + "version": 2, + "flag": true, + "bounded_int": -42, + "big_int": 999999999999, + "float_val": 2.718281828, + "decimal_multiple": 3.14, + "created_date": "2025-06-15", + "updated_at": "2025-06-15T12:30:00Z", + "event_time": "12:30:00Z", + "elapsed": "P3DT4H", + "record_uuid": "550e8400-e29b-41d4-a716-446655440000", + "homepage": "https://example.org/rec1", + "const_tag": "v1", + "untyped_int_enum": 20, + "untyped_mixed_enum": "1", + "numeric_union": 42.0, + "scalar_union": "100", + "date_or_year_union": "2025-06-15", + "raw_b64": "Hello", + "raw_hex": "Hello" + }, + { + "id": "rec2", + "status": "archived", + "version": 1, + "flag": false + }, + { + "id": "rec3", + "status": "pending", + "version": 32767, + "flag": true, + "bounded_int": 999, + "big_int": -999999999999, + "float_val": -0.5, + "decimal_multiple": 42.0, + "created_date": "1999-12-31", + "updated_at": "2025-06-15T12:30:00.123+02:00", + "event_time": "23:59:59+02:00", + "elapsed": "PT15M", + "record_uuid": "123e4567-e89b-12d3-a456-426614174000", + "homepage": "https://example.org/rec3", + "const_tag": "v1", + "untyped_int_enum": 30, + "untyped_mixed_enum": "true", + "numeric_union": 3.5, + "scalar_union": "hundred", + "date_or_year_union": "1999", + "raw_b64": "World", + "raw_hex": "World" + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/01_scalars_and_formats/expected_schema.py)) + +```python +"""Expected PyArrow schema for 01_scalars_and_formats.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "id", + pa.string(), + nullable=False, + metadata={"comment": "Unique record identifier.", "jsonschema_type": "string"}, + ), + pa.field( + "status", + pa.string(), + nullable=False, + metadata={"comment": "Record lifecycle status.", "jsonschema_type": "string"}, + ), + pa.field("version", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field( + "flag", + pa.bool_(), + nullable=False, + metadata={"comment": "Primary boolean flag.", "jsonschema_type": "boolean"}, + ), + pa.field("bounded_int", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("big_int", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("float_val", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field("decimal_multiple", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field("created_date", pa.date32(), nullable=True, metadata={"jsonschema_type": "string (date)"}), + pa.field("updated_at", pa.string(), nullable=True, metadata={"jsonschema_type": "string (date-time)"}), + pa.field("event_time", pa.string(), nullable=True, metadata={"jsonschema_type": "string (time)"}), + pa.field("elapsed", pa.string(), nullable=True, metadata={"jsonschema_type": "string (duration)"}), + pa.field("record_uuid", pa.string(), nullable=True, metadata={"jsonschema_type": "string (uuid)"}), + pa.field("homepage", pa.string(), nullable=True, metadata={"jsonschema_type": "string (uri)"}), + pa.field("const_tag", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("untyped_int_enum", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field( + "untyped_mixed_enum", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "integer | string | boolean"}, + ), + pa.field("numeric_union", pa.float64(), nullable=True, metadata={"jsonschema_type": "integer | number"}), + pa.field("scalar_union", pa.string(), nullable=True, metadata={"jsonschema_type": "integer | string"}), + pa.field( + "date_or_year_union", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string (date) | string"}, + ), + pa.field("raw_b64", pa.binary(), nullable=True, metadata={"jsonschema_type": "string (base64)"}), + pa.field("raw_hex", pa.binary(), nullable=True, metadata={"jsonschema_type": "string (base16)"}), + ] +) +``` + +--- + + + +### Nullability, Required Fields and Array Item Nullability (`spec/02_nullability_and_required`) + +**Rules:** [R-NULL-REQUIRED](#r-null-required), [R-NULL-ALLOWS-NULL](#r-null-allows-null), [R-NULL-DEFAULT](#r-null-default), [R-NULL-ITEMS](#r-null-items), [R-SCALAR-ENUM-CONST](#r-scalar-enum-const) + +- `required_nullable_str`, `required_anyof_nullable` and `required_enum_nullable` are required but allow `null`, so they are nullable. +- `optional_with_default` has a `default` and stays nullable. +- `strict_int_list` items are non-nullable; `nullable_int_list` items (`["integer", "null"]`) are nullable. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/02_nullability_and_required.schema.json", + "title": "NullabilityAndRequiredRecord", + "type": "object", + "required": [ + "required_str", + "required_int", + "required_nullable_str", + "required_anyof_nullable", + "required_enum_nullable", + "strict_int_list", + "nullable_int_list" + ], + "properties": { + "required_str": { + "type": "string", + "description": "Strictly required non-null string." + }, + "required_int": { + "type": "integer" + }, + "required_nullable_str": { + "type": [ + "string", + "null" + ], + "description": "Required key whose value may be null." + }, + "required_anyof_nullable": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "required_enum_nullable": { + "enum": [ + "low", + "medium", + "high", + null + ] + }, + "optional_str": { + "type": "string" + }, + "optional_with_default": { + "type": "string", + "default": "fallback" + }, + "optional_nullable_number": { + "type": [ + "number", + "null" + ] + }, + "strict_int_list": { + "type": "array", + "items": { + "type": "integer" + } + }, + "nullable_int_list": { + "type": "array", + "items": { + "type": [ + "integer", + "null" + ] + } + }, + "optional_str_list": { + "type": "array", + "items": { + "type": "string" + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "required_str": "alpha", + "required_int": 10, + "required_nullable_str": "present", + "required_anyof_nullable": 99, + "required_enum_nullable": "high", + "optional_str": "opt_value", + "optional_with_default": "explicit", + "optional_nullable_number": 1.25, + "strict_int_list": [ + 1, + 2, + 3 + ], + "nullable_int_list": [ + 10, + null, + 30 + ], + "optional_str_list": [ + "a", + "b" + ] + }, + { + "required_str": "beta", + "required_int": 0, + "required_nullable_str": null, + "required_anyof_nullable": null, + "required_enum_nullable": null, + "optional_nullable_number": null, + "strict_int_list": [], + "nullable_int_list": [ + null + ] + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "required_str": "alpha", + "required_int": 10, + "required_nullable_str": "present", + "required_anyof_nullable": 99, + "required_enum_nullable": "high", + "optional_str": "opt_value", + "optional_with_default": "explicit", + "optional_nullable_number": 1.25, + "strict_int_list": [ + 1, + 2, + 3 + ], + "nullable_int_list": [ + 10, + null, + 30 + ], + "optional_str_list": [ + "a", + "b" + ] + }, + { + "required_str": "beta", + "required_int": 0, + "required_nullable_str": null, + "required_anyof_nullable": null, + "required_enum_nullable": null, + "optional_nullable_number": null, + "strict_int_list": [], + "nullable_int_list": [ + null + ] + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/02_nullability_and_required/expected_schema.py)) + +```python +"""Expected PyArrow schema for 02_nullability_and_required.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "required_str", + pa.string(), + nullable=False, + metadata={"comment": "Strictly required non-null string.", "jsonschema_type": "string"}, + ), + pa.field("required_int", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field( + "required_nullable_str", + pa.string(), + nullable=True, + metadata={"comment": "Required key whose value may be null.", "jsonschema_type": "string"}, + ), + pa.field("required_anyof_nullable", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("required_enum_nullable", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("optional_str", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("optional_with_default", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("optional_nullable_number", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field( + "strict_int_list", + pa.list_( + pa.field("item", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + ), + nullable=False, + ), + pa.field( + "nullable_int_list", + pa.list_( + pa.field("item", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + ), + nullable=False, + ), + pa.field( + "optional_str_list", + pa.list_( + pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Homogeneous Arrays, 2D Arrays and Closed Tuples (`spec/03_arrays_and_tuples`) + +**Rules:** [R-SHAPE-ARRAY](#r-shape-array), [R-SHAPE-TUPLE](#r-shape-tuple), [R-NULL-ITEMS](#r-null-items), [R-COERCE-TUPLE](#r-coerce-tuple), [R-META-COMMENT](#r-meta-comment) + +`coord_3d` has `minItems: 2`, so `item_0` and `item_1` are non-nullable and `item_2` (`["number", "null"]`) is +nullable. `[10.5, 20.25, 30.0]` becomes `{"item_0": 10.5, "item_1": 20.25, "item_2": 30.0}`. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/03_arrays_and_tuples.schema.json", + "title": "ArraysAndTuplesRecord", + "type": "object", + "required": [ + "id", + "tags", + "coord_3d" + ], + "properties": { + "id": { + "type": "string" + }, + "tags": { + "type": "array", + "description": "Ordered list of string tags.", + "items": { + "type": "string" + } + }, + "measurements": { + "type": "array", + "items": { + "type": "object", + "required": [ + "label", + "value" + ], + "properties": { + "label": { + "type": "string" + }, + "value": { + "type": "number" + }, + "unit": { + "type": "string" + } + } + } + }, + "matrix_2d": { + "type": "array", + "description": "Two-dimensional numeric matrix.", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "coord_3d": { + "type": "array", + "description": "Closed 3D coordinate tuple (x, y, optional z).", + "minItems": 2, + "prefixItems": [ + { + "type": "number", + "description": "X coordinate." + }, + { + "type": "number", + "description": "Y coordinate." + }, + { + "type": [ + "number", + "null" + ], + "description": "Optional Z coordinate." + } + ], + "items": false + }, + "pair_tuple_list": { + "type": "array", + "items": { + "type": "array", + "minItems": 2, + "prefixItems": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "items": false + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "id": "arr1", + "tags": [ + "genomics", + "assembly" + ], + "measurements": [ + { + "label": "gc_content", + "value": 0.512, + "unit": "ratio" + }, + { + "label": "n50", + "value": 4500000.0 + } + ], + "matrix_2d": [ + [ + 1.0, + 2.0 + ], + [ + 3.5, + 4.5 + ] + ], + "coord_3d": [ + 10.5, + 20.25, + 30.0 + ], + "pair_tuple_list": [ + [ + "chr1", + 248956422 + ], + [ + "chr2", + 242193529 + ] + ] + }, + { + "id": "arr2", + "tags": [], + "coord_3d": [ + -1.0, + 2.0 + ] + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "id": "arr1", + "tags": [ + "genomics", + "assembly" + ], + "measurements": [ + { + "label": "gc_content", + "value": 0.512, + "unit": "ratio" + }, + { + "label": "n50", + "value": 4500000.0 + } + ], + "matrix_2d": [ + [ + 1.0, + 2.0 + ], + [ + 3.5, + 4.5 + ] + ], + "coord_3d": { + "item_0": 10.5, + "item_1": 20.25, + "item_2": 30.0 + }, + "pair_tuple_list": [ + { + "item_0": "chr1", + "item_1": 248956422 + }, + { + "item_0": "chr2", + "item_1": 242193529 + } + ] + }, + { + "id": "arr2", + "tags": [], + "coord_3d": { + "item_0": -1.0, + "item_1": 2.0 + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/03_arrays_and_tuples/expected_schema.py)) + +```python +"""Expected PyArrow schema for 03_arrays_and_tuples.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "tags", + pa.list_( + pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=False, + metadata={"comment": "Ordered list of string tags."}, + ), + pa.field( + "measurements", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("label", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("value", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + pa.field("unit", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + pa.field( + "matrix_2d", + pa.list_( + pa.field( + "item", + pa.list_( + pa.field("item", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + ), + nullable=False, + ), + ), + nullable=True, + metadata={"comment": "Two-dimensional numeric matrix."}, + ), + pa.field( + "coord_3d", + pa.struct( + [ + pa.field( + "item_0", + pa.float64(), + nullable=False, + metadata={"comment": "X coordinate.", "jsonschema_type": "number"}, + ), + pa.field( + "item_1", + pa.float64(), + nullable=False, + metadata={"comment": "Y coordinate.", "jsonschema_type": "number"}, + ), + pa.field( + "item_2", + pa.float64(), + nullable=True, + metadata={"comment": "Optional Z coordinate.", "jsonschema_type": "number"}, + ), + ] + ), + nullable=False, + metadata={"comment": "Closed 3D coordinate tuple (x, y, optional z)."}, + ), + pa.field( + "pair_tuple_list", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("item_0", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("item_1", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Nested Fixed Objects and Dynamic Maps (`spec/04_objects_and_maps`) + +**Rules:** [R-SHAPE-OBJECT](#r-shape-object), [R-SHAPE-MAP](#r-shape-map), [R-NULL-ITEMS](#r-null-items) + +`closed_meta` (`additionalProperties: false`) and `open_meta` (`additionalProperties: true`) are both structs. +`string_to_int_map`, `nullable_score_map` and `struct_map` use `additionalProperties`; `pattern_metric_map` +uses `patternProperties`. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/04_objects_and_maps.schema.json", + "title": "ObjectsAndMapsRecord", + "type": "object", + "required": [ + "id", + "closed_meta" + ], + "properties": { + "id": { + "type": "string" + }, + "closed_meta": { + "type": "object", + "description": "Strict object with additionalProperties set to false.", + "required": [ + "owner" + ], + "additionalProperties": false, + "properties": { + "owner": { + "type": "string" + }, + "priority": { + "type": "integer" + } + } + }, + "open_meta": { + "type": "object", + "description": "Fixed-property object with boolean additionalProperties true.", + "additionalProperties": true, + "properties": { + "source": { + "type": "string" + } + } + }, + "string_to_int_map": { + "type": "object", + "description": "Dynamic map from string keys to non-null integer counts.", + "additionalProperties": { + "type": "integer" + } + }, + "nullable_score_map": { + "type": "object", + "additionalProperties": { + "type": [ + "number", + "null" + ] + } + }, + "pattern_metric_map": { + "type": "object", + "patternProperties": { + "^metric_[0-9]+$": { + "type": "number" + } + }, + "additionalProperties": false + }, + "struct_map": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "count" + ], + "properties": { + "count": { + "type": "integer" + }, + "note": { + "type": "string" + } + } + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "id": "obj1", + "closed_meta": { + "owner": "alice", + "priority": 1 + }, + "open_meta": { + "source": "ncbi" + }, + "string_to_int_map": { + "reads": 1500, + "contigs": 12 + }, + "nullable_score_map": { + "completeness": 99.4, + "contamination": null + }, + "pattern_metric_map": { + "metric_1": 0.95, + "metric_2": 0.88 + }, + "struct_map": { + "kmer_31": { + "count": 42000, + "note": "canonical" + }, + "kmer_21": { + "count": 38000 + } + } + }, + { + "id": "obj2", + "closed_meta": { + "owner": "bob" + } + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "id": "obj1", + "closed_meta": { + "owner": "alice", + "priority": 1 + }, + "open_meta": { + "source": "ncbi" + }, + "string_to_int_map": { + "reads": 1500, + "contigs": 12 + }, + "nullable_score_map": { + "completeness": 99.4, + "contamination": null + }, + "pattern_metric_map": { + "metric_1": 0.95, + "metric_2": 0.88 + }, + "struct_map": { + "kmer_31": { + "count": 42000, + "note": "canonical" + }, + "kmer_21": { + "count": 38000 + } + } + }, + { + "id": "obj2", + "closed_meta": { + "owner": "bob" + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/04_objects_and_maps/expected_schema.py)) + +```python +"""Expected PyArrow schema for 04_objects_and_maps.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "closed_meta", + pa.struct( + [ + pa.field("owner", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("priority", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + ] + ), + nullable=False, + metadata={"comment": "Strict object with additionalProperties set to false."}, + ), + pa.field( + "open_meta", + pa.struct( + [ + pa.field("source", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + metadata={"comment": "Fixed-property object with boolean additionalProperties true."}, + ), + pa.field( + "string_to_int_map", + pa.map_( + pa.string(), + pa.field("value", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + ), + nullable=True, + metadata={"comment": "Dynamic map from string keys to non-null integer counts."}, + ), + pa.field( + "nullable_score_map", + pa.map_( + pa.string(), + pa.field("value", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + ), + nullable=True, + ), + pa.field( + "pattern_metric_map", + pa.map_( + pa.string(), + pa.field("value", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + ), + nullable=True, + ), + pa.field( + "struct_map", + pa.map_( + pa.string(), + pa.field( + "value", + pa.struct( + [ + pa.field("count", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("note", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Shared Definitions, `$ref` Reuse and Root `$ref` (`spec/05_defs_and_refs`) + +**Rules:** [R-ROOT-REF](#r-root-ref), [R-SHAPE-REF](#r-shape-ref), [R-META-COMMENT](#r-meta-comment) + +The root is `"$ref": "#/$defs/Document"`. `lead_author` declares its own `description`, overriding the +`Author` definition's; `reviewer` and `co_authors` items inherit it. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/05_defs_and_refs.schema.json", + "title": "DefsAndRefsRoot", + "$ref": "#/$defs/Document", + "$defs": { + "IsoDate": { + "type": "string", + "format": "date", + "description": "ISO-8601 calendar date." + }, + "Author": { + "type": "object", + "description": "Author contact record.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "orcid": { + "type": "string" + } + } + }, + "Document": { + "type": "object", + "required": [ + "doc_id", + "lead_author" + ], + "properties": { + "doc_id": { + "type": "string" + }, + "published_on": { + "$ref": "#/$defs/IsoDate" + }, + "lead_author": { + "$ref": "#/$defs/Author", + "description": "Primary corresponding author." + }, + "reviewer": { + "$ref": "#/$defs/Author" + }, + "co_authors": { + "type": "array", + "items": { + "$ref": "#/$defs/Author" + } + } + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "doc_id": "doc1", + "published_on": "2025-07-01", + "lead_author": { + "name": "Ada Lovelace", + "orcid": "0000-0001-2345-6789" + }, + "reviewer": { + "name": "Grace Hopper" + }, + "co_authors": [ + { + "name": "Alan Turing", + "orcid": "0000-0002-3456-7890" + } + ] + }, + { + "doc_id": "doc2", + "lead_author": { + "name": "Barbara Liskov" + } + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "doc_id": "doc1", + "published_on": "2025-07-01", + "lead_author": { + "name": "Ada Lovelace", + "orcid": "0000-0001-2345-6789" + }, + "reviewer": { + "name": "Grace Hopper" + }, + "co_authors": [ + { + "name": "Alan Turing", + "orcid": "0000-0002-3456-7890" + } + ] + }, + { + "doc_id": "doc2", + "lead_author": { + "name": "Barbara Liskov" + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/05_defs_and_refs/expected_schema.py)) + +```python +"""Expected PyArrow schema for 05_defs_and_refs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +_AUTHOR_STRUCT = pa.struct( + [ + pa.field("name", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("orcid", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] +) + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("doc_id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "published_on", + pa.date32(), + nullable=True, + metadata={"comment": "ISO-8601 calendar date.", "jsonschema_type": "string (date)"}, + ), + pa.field( + "lead_author", + _AUTHOR_STRUCT, + nullable=False, + metadata={"comment": "Primary corresponding author."}, + ), + pa.field( + "reviewer", + _AUTHOR_STRUCT, + nullable=True, + metadata={"comment": "Author contact record."}, + ), + pa.field( + "co_authors", + pa.list_( + pa.field( + "item", + _AUTHOR_STRUCT, + nullable=False, + metadata={"comment": "Author contact record."}, + ), + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Structural Recursion and `max_recursion_depth` (`spec/06_recursion_depth`) + +**Rules:** [R-RECURSION](#r-recursion), [R-COERCE-STRING-FALLBACK](#r-coerce-string-fallback) + +`TreeNode` is self-recursive (`branch: array of TreeNode`); `Section` and `Subsection` are mutually recursive. +At depth 2 the third `TreeNode` level becomes JSON strings; at depth 1 the second level already does. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/06_recursion_depth.schema.json", + "title": "RecursionDepthRecord", + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "tree": { + "$ref": "#/$defs/TreeNode" + }, + "section": { + "$ref": "#/$defs/Section" + } + }, + "$defs": { + "TreeNode": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "weight": { + "type": "integer" + }, + "branch": { + "type": "array", + "items": { + "$ref": "#/$defs/TreeNode" + } + } + } + }, + "Section": { + "type": "object", + "required": [ + "section_title" + ], + "properties": { + "section_title": { + "type": "string" + }, + "subsection": { + "$ref": "#/$defs/Subsection" + } + } + }, + "Subsection": { + "type": "object", + "required": [ + "subsection_title" + ], + "properties": { + "subsection_title": { + "type": "string" + }, + "nested_sections": { + "type": "array", + "items": { + "$ref": "#/$defs/Section" + } + } + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + { + "label": "L2", + "branch": [ + { + "label": "L3", + "weight": 30, + "branch": [ + { + "label": "L4" + } + ] + }, + { + "label": "L3_sibling" + } + ] + } + ] + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + { + "section_title": "Sec 1.1", + "subsection": { + "subsection_title": "Sub 1.1.a", + "nested_sections": [ + { + "section_title": "Subsec 1.1.1" + } + ] + } + } + ] + } + } + } +] +``` + +#### Converted rows: run `depth-2` (`max_recursion_depth=2`) + +```json +[ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + { + "label": "L2", + "branch": [ + "{\"label\": \"L3\", \"weight\": 30, \"branch\": [{\"label\": \"L4\"}]}", + "{\"label\": \"L3_sibling\"}" + ] + } + ] + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + { + "section_title": "Sec 1.1", + "subsection": { + "subsection_title": "Sub 1.1.a", + "nested_sections": [ + "{\"section_title\": \"Subsec 1.1.1\"}" + ] + } + } + ] + } + } + } +] +``` + +#### Converted rows: run `depth-1` (`max_recursion_depth=1`) + +```json +[ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + "{\"label\": \"L2\", \"branch\": [{\"label\": \"L3\", \"weight\": 30, \"branch\": [{\"label\": \"L4\"}]}, {\"label\": \"L3_sibling\"}]}" + ] + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + "{\"section_title\": \"Sec 1.1\", \"subsection\": {\"subsection_title\": \"Sub 1.1.a\", \"nested_sections\": [{\"section_title\": \"Subsec 1.1.1\"}]}}" + ] + } + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/06_recursion_depth/expected_schema.py)) + +```python +"""Expected PyArrow schema for 06_recursion_depth.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("label", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("weight", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field( + "branch", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "weight", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "section", + pa.struct( + [ + pa.field("section_title", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "subsection", + pa.struct( + [ + pa.field( + "subsection_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "nested_sections", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "section_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "subsection", + pa.struct( + [ + pa.field( + "subsection_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "nested_sections", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) + +EXPECTED_SCHEMA_DEPTH_1 = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("label", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("weight", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "section", + pa.struct( + [ + pa.field("section_title", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "subsection", + pa.struct( + [ + pa.field( + "subsection_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "nested_sections", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Combinators — `allOf`, `anyOf` and `oneOf` (`spec/07_combinators_allof_anyof_oneof`) + +**Rules:** [R-ORDER](#r-order), [R-MERGE-DUPLICATE](#r-merge-duplicate), [R-UNION-OBJECT](#r-union-object), [R-UNION-SCALAR](#r-union-scalar), [R-UNION-SCALAR-OBJECT](#r-union-scalar-object), [R-NULL-REQUIRED](#r-null-required), [R-COERCE-VALUE](#r-coerce-value) + +- `allOf: [BaseEntity]` puts `entity_id` and `created_by` before the direct properties. +- `location` merges two `oneOf` object branches. `storage_kind` is required in both, so it is non-nullable; it is + declared with different `const` values, which merge to `string`. +- `note` is `string | object`; the plain string `"plain text note"` becomes `{"_value": "plain text note"}`. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/07_combinators_allof_anyof_oneof.schema.json", + "title": "CombinatorsRecord", + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/BaseEntity" + } + ], + "required": [ + "sample_name", + "location" + ], + "properties": { + "sample_name": { + "type": "string" + }, + "location": { + "description": "Storage location choice across S3 or HTTP.", + "oneOf": [ + { + "type": "object", + "required": [ + "storage_kind", + "bucket", + "key" + ], + "properties": { + "storage_kind": { + "type": "string", + "const": "s3" + }, + "bucket": { + "type": "string" + }, + "key": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "storage_kind", + "url" + ], + "properties": { + "storage_kind": { + "type": "string", + "const": "http" + }, + "url": { + "type": "string", + "format": "uri" + }, + "checksum": { + "type": "string" + } + } + } + ] + }, + "pagination": { + "anyOf": [ + { + "type": "object", + "required": [ + "start_page" + ], + "properties": { + "start_page": { + "type": "string" + }, + "end_page": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "medline_pgn" + ], + "properties": { + "medline_pgn": { + "type": "string" + } + } + } + ] + }, + "identifier_union": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [ + "text" + ], + "properties": { + "text": { + "type": "string" + }, + "author": { + "type": "string" + } + } + } + ] + } + }, + "$defs": { + "BaseEntity": { + "type": "object", + "required": [ + "entity_id" + ], + "properties": { + "entity_id": { + "type": "string", + "description": "Inherited base entity identifier." + }, + "created_by": { + "type": "string" + } + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "entity_id": "ent1", + "created_by": "pipeline", + "sample_name": "E. coli K-12", + "location": { + "storage_kind": "s3", + "bucket": "cdm-lake", + "key": "genomes/ecoli.fna.gz" + }, + "pagination": { + "start_page": "101", + "end_page": "115" + }, + "identifier_union": 12345, + "note": { + "text": "Structured note", + "author": "curator" + } + }, + { + "entity_id": "ent2", + "sample_name": "B. subtilis", + "location": { + "storage_kind": "http", + "url": "https://example.org/bsub.fna.gz", + "checksum": "sha256:abc123" + }, + "pagination": { + "medline_pgn": "e10042" + }, + "identifier_union": "GCF_000009045.1", + "note": "Plain text note" + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "entity_id": "ent1", + "created_by": "pipeline", + "sample_name": "E. coli K-12", + "location": { + "storage_kind": "s3", + "bucket": "cdm-lake", + "key": "genomes/ecoli.fna.gz" + }, + "pagination": { + "start_page": "101", + "end_page": "115" + }, + "identifier_union": "12345", + "note": { + "text": "Structured note", + "author": "curator" + } + }, + { + "entity_id": "ent2", + "sample_name": "B. subtilis", + "location": { + "storage_kind": "http", + "url": "https://example.org/bsub.fna.gz", + "checksum": "sha256:abc123" + }, + "pagination": { + "medline_pgn": "e10042" + }, + "identifier_union": "GCF_000009045.1", + "note": { + "_value": "Plain text note" + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_schema.py)) + +```python +"""Expected PyArrow schema for 07_combinators_allof_anyof_oneof.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "entity_id", + pa.string(), + nullable=False, + metadata={"comment": "Inherited base entity identifier.", "jsonschema_type": "string"}, + ), + pa.field("created_by", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("sample_name", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "location", + pa.struct( + [ + pa.field("storage_kind", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("bucket", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("key", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("url", pa.string(), nullable=True, metadata={"jsonschema_type": "string (uri)"}), + pa.field("checksum", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + metadata={"comment": "Storage location choice across S3 or HTTP."}, + ), + pa.field( + "pagination", + pa.struct( + [ + pa.field("start_page", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("end_page", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("medline_pgn", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + ), + pa.field("identifier_union", pa.string(), nullable=True, metadata={"jsonschema_type": "integer | string"}), + pa.field( + "note", + pa.struct( + [ + pa.field("text", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("author", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("_value", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Conditional (`if`/`then`/`else`) and Dependent Schemas (`spec/08_conditional_and_dependent_schemas`) + +**Rules:** [R-NULL-CONDITIONAL](#r-null-conditional), [R-ORDER](#r-order) + +`strain_name` (from `then`), `biome` (from `else`) and `platform` (from `dependentSchemas`) are nullable. +`depth_unit` appears only in `dependentRequired` and stays nullable. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/08_conditional_and_dependent_schemas.schema.json", + "title": "ConditionalAndDependentRecord", + "type": "object", + "required": [ + "id", + "specimen_type" + ], + "properties": { + "id": { + "type": "string" + }, + "specimen_type": { + "type": "string", + "enum": [ + "isolate", + "metagenome" + ] + }, + "sequencing_depth": { + "type": "number" + }, + "depth_unit": { + "type": "string" + } + }, + "dependentRequired": { + "sequencing_depth": [ + "depth_unit" + ] + }, + "dependentSchemas": { + "sequencing_depth": { + "properties": { + "platform": { + "type": "string", + "description": "Sequencing platform when depth is reported." + } + } + } + }, + "if": { + "properties": { + "specimen_type": { + "const": "isolate" + } + } + }, + "then": { + "required": [ + "strain_name" + ], + "properties": { + "strain_name": { + "type": "string", + "description": "Cultured isolate strain identifier." + } + } + }, + "else": { + "required": [ + "biome" + ], + "properties": { + "biome": { + "type": "string", + "description": "Environmental biome for metagenome samples." + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "id": "cond1", + "specimen_type": "isolate", + "sequencing_depth": 120.5, + "depth_unit": "x", + "platform": "Illumina NovaSeq", + "strain_name": "K-12 MG1655" + }, + { + "id": "cond2", + "specimen_type": "metagenome", + "biome": "marine pelagic" + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "id": "cond1", + "specimen_type": "isolate", + "sequencing_depth": 120.5, + "depth_unit": "x", + "platform": "Illumina NovaSeq", + "strain_name": "K-12 MG1655" + }, + { + "id": "cond2", + "specimen_type": "metagenome", + "biome": "marine pelagic" + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_schema.py)) + +```python +"""Expected PyArrow schema for 08_conditional_and_dependent_schemas.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("specimen_type", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("sequencing_depth", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field("depth_unit", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "platform", + pa.string(), + nullable=True, + metadata={"comment": "Sequencing platform when depth is reported.", "jsonschema_type": "string"}, + ), + pa.field( + "strain_name", + pa.string(), + nullable=True, + metadata={"comment": "Cultured isolate strain identifier.", "jsonschema_type": "string"}, + ), + pa.field( + "biome", + pa.string(), + nullable=True, + metadata={"comment": "Environmental biome for metagenome samples.", "jsonschema_type": "string"}, + ), + ] +) +``` + +--- + + + +### Context-Dependent Shared Definitions (`spec/09_context_dependent_shared_defs`) + +**Rules:** [R-SHAPE-REF](#r-shape-ref), [R-NULL-REQUIRED](#r-null-required), [R-NULL-ALLOWS-NULL](#r-null-allows-null), [R-NULL-ITEMS](#r-null-items), [R-META-COMMENT](#r-meta-comment) + +One `#/$defs/Tag` used as a required property (`primary_tag`), an optional property (`secondary_tag`), a +required-but-nullable `anyOf` property (`fallback_tag`), an array item (`tag_list`) and a map value +(`tag_lookup`). + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/09_context_dependent_shared_defs.schema.json", + "title": "ContextDependentSharedDefsRecord", + "type": "object", + "required": [ + "id", + "primary_tag", + "fallback_tag" + ], + "properties": { + "id": { + "type": "string" + }, + "primary_tag": { + "$ref": "#/$defs/Tag" + }, + "secondary_tag": { + "$ref": "#/$defs/Tag" + }, + "fallback_tag": { + "anyOf": [ + { + "$ref": "#/$defs/Tag" + }, + { + "type": "null" + } + ] + }, + "tag_list": { + "type": "array", + "items": { + "$ref": "#/$defs/Tag" + } + }, + "tag_lookup": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/Tag" + } + } + }, + "$defs": { + "Tag": { + "type": "object", + "description": "Reusable key-value tag structure.", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "id": "ctx1", + "primary_tag": { + "key": "domain", + "value": "Bacteria" + }, + "secondary_tag": { + "key": "phylum", + "value": "Proteobacteria" + }, + "fallback_tag": { + "key": "status", + "value": "curated" + }, + "tag_list": [ + { + "key": "gram", + "value": "negative" + }, + { + "key": "motile" + } + ], + "tag_lookup": { + "ncbi": { + "key": "taxid", + "value": "562" + } + } + }, + { + "id": "ctx2", + "primary_tag": { + "key": "domain", + "value": "Archaea" + }, + "fallback_tag": null + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "id": "ctx1", + "primary_tag": { + "key": "domain", + "value": "Bacteria" + }, + "secondary_tag": { + "key": "phylum", + "value": "Proteobacteria" + }, + "fallback_tag": { + "key": "status", + "value": "curated" + }, + "tag_list": [ + { + "key": "gram", + "value": "negative" + }, + { + "key": "motile" + } + ], + "tag_lookup": { + "ncbi": { + "key": "taxid", + "value": "562" + } + } + }, + { + "id": "ctx2", + "primary_tag": { + "key": "domain", + "value": "Archaea" + }, + "fallback_tag": null + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_schema.py)) + +```python +"""Expected PyArrow schema for 09_context_dependent_shared_defs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +_TAG_STRUCT = pa.struct( + [ + pa.field("key", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("value", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] +) + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "primary_tag", + _TAG_STRUCT, + nullable=False, + metadata={"comment": "Reusable key-value tag structure."}, + ), + pa.field( + "secondary_tag", + _TAG_STRUCT, + nullable=True, + metadata={"comment": "Reusable key-value tag structure."}, + ), + pa.field( + "fallback_tag", + _TAG_STRUCT, + nullable=True, + metadata={"comment": "Reusable key-value tag structure."}, + ), + pa.field( + "tag_list", + pa.list_( + pa.field( + "item", + _TAG_STRUCT, + nullable=False, + metadata={"comment": "Reusable key-value tag structure."}, + ), + ), + nullable=True, + ), + pa.field( + "tag_lookup", + pa.map_( + pa.string(), + pa.field( + "value", + _TAG_STRUCT, + nullable=False, + metadata={"comment": "Reusable key-value tag structure."}, + ), + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Combinations (`spec/10_combinations`) + +**Rules:** [R-ORDER](#r-order), [R-SCALAR-TYPE](#r-scalar-type), [R-UNION-SCALAR](#r-union-scalar), [R-UNION-OBJECT](#r-union-object), [R-SHAPE-TUPLE](#r-shape-tuple), [R-SHAPE-MAP](#r-shape-map), [R-RECURSION](#r-recursion), [R-NULL-CONDITIONAL](#r-null-conditional), [R-SCALAR-ENUM-CONST](#r-scalar-enum-const), [R-COERCE-SCALAR](#r-coerce-scalar) + +Stress test combining `allOf` inheritance, `const`, date / date-time / Base64 scalars, a nullable +`["number", "string", "null"]` union, a closed tuple, a string map, a `oneOf` object choice, a recursive +`TaxNode` and a `dependentSchemas` property. + +#### Schema (`schema.json`) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/10_combinations.schema.json", + "title": "ComprehensiveCombinationsRecord", + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/AuditHeader" + } + ], + "required": [ + "record_id", + "active", + "count", + "ratio", + "release_date", + "range_tuple" + ], + "properties": { + "record_id": { + "type": "string", + "description": "Primary record identifier." + }, + "active": { + "type": "boolean" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "ratio": { + "type": "number", + "multipleOf": 0.001 + }, + "release_date": { + "type": "string", + "format": "date" + }, + "timestamp_str": { + "type": "string", + "format": "date-time" + }, + "payload_b64": { + "type": "string", + "contentEncoding": "base64" + }, + "score_or_label": { + "type": [ + "number", + "string", + "null" + ] + }, + "range_tuple": { + "type": "array", + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "items": false + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "source_choice": { + "oneOf": [ + { + "type": "object", + "required": [ + "kind", + "accession" + ], + "properties": { + "kind": { + "type": "string", + "const": "genbank" + }, + "accession": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "doi" + ], + "properties": { + "kind": { + "type": "string", + "const": "publication" + }, + "doi": { + "type": "string" + } + } + } + ] + }, + "lineage_tree": { + "$ref": "#/$defs/TaxNode" + } + }, + "dependentSchemas": { + "payload_b64": { + "properties": { + "payload_sha256": { + "type": "string" + } + } + } + }, + "$defs": { + "AuditHeader": { + "type": "object", + "required": [ + "schema_version" + ], + "properties": { + "schema_version": { + "const": 1 + } + } + }, + "TaxNode": { + "type": "object", + "required": [ + "taxon_name" + ], + "properties": { + "taxon_name": { + "type": "string" + }, + "rank": { + "type": "string" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/TaxNode" + } + } + } + } + } +} +``` + +#### Input (`data.json`) + +```json +[ + { + "schema_version": 1, + "record_id": "comb1", + "active": true, + "count": 42, + "ratio": 0.875, + "release_date": "2025-08-15", + "timestamp_str": "2025-08-15T09:00:00Z", + "payload_b64": "Q0RN", + "payload_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "score_or_label": 98.5, + "range_tuple": [ + 100, + 500, + "+" + ], + "tags": [ + "complete", + "reference" + ], + "annotations": { + "bioproject": "PRJNA225", + "biosample": "SAMN02604091" + }, + "source_choice": { + "kind": "genbank", + "accession": "GCF_000005845.2" + }, + "lineage_tree": { + "taxon_name": "Bacteria", + "rank": "domain", + "children": [ + { + "taxon_name": "Proteobacteria", + "rank": "phylum", + "children": [ + { + "taxon_name": "Gammaproteobacteria", + "rank": "class" + } + ] + } + ] + } + }, + { + "schema_version": 1, + "record_id": "comb2", + "active": false, + "count": 0, + "ratio": 0.0, + "release_date": "2025-01-01", + "score_or_label": null, + "range_tuple": [ + 1, + 10 + ], + "source_choice": { + "kind": "publication", + "doi": "10.1038/s41586-025-00001-x" + } + } +] +``` + +#### Converted rows: run `default` + +```json +[ + { + "schema_version": 1, + "record_id": "comb1", + "active": true, + "count": 42, + "ratio": 0.875, + "release_date": "2025-08-15", + "timestamp_str": "2025-08-15T09:00:00Z", + "payload_b64": "CDM", + "payload_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "score_or_label": "98.5", + "range_tuple": { + "item_0": 100, + "item_1": 500, + "item_2": "+" + }, + "tags": [ + "complete", + "reference" + ], + "annotations": { + "bioproject": "PRJNA225", + "biosample": "SAMN02604091" + }, + "source_choice": { + "kind": "genbank", + "accession": "GCF_000005845.2" + }, + "lineage_tree": { + "taxon_name": "Bacteria", + "rank": "domain", + "children": [ + { + "taxon_name": "Proteobacteria", + "rank": "phylum", + "children": [ + "{\"taxon_name\": \"Gammaproteobacteria\", \"rank\": \"class\"}" + ] + } + ] + } + }, + { + "schema_version": 1, + "record_id": "comb2", + "active": false, + "count": 0, + "ratio": 0.0, + "release_date": "2025-01-01", + "score_or_label": null, + "range_tuple": { + "item_0": 1, + "item_1": 10 + }, + "source_choice": { + "kind": "publication", + "doi": "10.1038/s41586-025-00001-x" + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/jsonschema/spec/10_combinations/expected_schema.py)) + +```python +"""Expected PyArrow schema for 10_combinations.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("schema_version", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field( + "record_id", + pa.string(), + nullable=False, + metadata={"comment": "Primary record identifier.", "jsonschema_type": "string"}, + ), + pa.field("active", pa.bool_(), nullable=False, metadata={"jsonschema_type": "boolean"}), + pa.field("count", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("ratio", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + pa.field("release_date", pa.date32(), nullable=False, metadata={"jsonschema_type": "string (date)"}), + pa.field("timestamp_str", pa.string(), nullable=True, metadata={"jsonschema_type": "string (date-time)"}), + pa.field("payload_b64", pa.binary(), nullable=True, metadata={"jsonschema_type": "string (base64)"}), + pa.field("score_or_label", pa.string(), nullable=True, metadata={"jsonschema_type": "number | string"}), + pa.field( + "range_tuple", + pa.struct( + [ + pa.field("item_0", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("item_1", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("item_2", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + pa.field( + "tags", + pa.list_( + pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=True, + ), + pa.field( + "annotations", + pa.map_( + pa.string(), + pa.field("value", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=True, + ), + pa.field( + "source_choice", + pa.struct( + [ + pa.field("kind", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("accession", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("doi", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + ), + pa.field( + "lineage_tree", + pa.struct( + [ + pa.field("taxon_name", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("rank", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "children", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "taxon_name", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "rank", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "children", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field("payload_sha256", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] +) +``` + +--- + + + +### KBase CTS `Job` (`examples/cts_job`) + +**Rules:** [R-UNION-SCALAR](#r-union-scalar), [R-NULL-ALLOWS-NULL](#r-null-allows-null), [R-SHAPE-MAP](#r-shape-map), [R-SHAPE-REF](#r-shape-ref) + +`/jobs/{job_id}` response schema from `not_for_github/cts/openapi.json` (`#/components/schemas/Job`), with nested +`JobInput`, `JobImage`, `RefData` / `S3FileWithDataID`, a dynamic `params` map and `anyOf` nullability. + +**Schema:** [`tests/data/jsonschema/examples/cts_job/schema.json`](../tests/data/jsonschema/examples/cts_job/schema.json) + +- Run `minimal`: [`minimal.json`](../tests/data/jsonschema/examples/cts_job/minimal.json), [`minimal_expected.json`](../tests/data/jsonschema/examples/cts_job/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/jsonschema/examples/cts_job/minimal_expected_table.py) +- Run `maximal`: [`maximal.json`](../tests/data/jsonschema/examples/cts_job/maximal.json), [`maximal_expected.json`](../tests/data/jsonschema/examples/cts_job/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/jsonschema/examples/cts_job/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/jsonschema/examples/cts_job/expected_schema.py) + +--- + + + +### KBase CTS `ListJobsResponse` (`examples/cts_jobs`) + +**Rules:** [R-SHAPE-ARRAY](#r-shape-array), [R-SHAPE-REF](#r-shape-ref) + +`/jobs` response schema from `not_for_github/cts/openapi.json` (`#/components/schemas/ListJobsResponse`): an +array of `JobPreview` structs with `JobInputPreview`, `JobImage` and state transition timestamps. + +**Schema:** [`tests/data/jsonschema/examples/cts_jobs/schema.json`](../tests/data/jsonschema/examples/cts_jobs/schema.json) + +- Run `minimal`: [`minimal.json`](../tests/data/jsonschema/examples/cts_jobs/minimal.json), [`minimal_expected.json`](../tests/data/jsonschema/examples/cts_jobs/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/jsonschema/examples/cts_jobs/minimal_expected_table.py) +- Run `maximal`: [`maximal.json`](../tests/data/jsonschema/examples/cts_jobs/maximal.json), [`maximal_expected.json`](../tests/data/jsonschema/examples/cts_jobs/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/jsonschema/examples/cts_jobs/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/jsonschema/examples/cts_jobs/expected_schema.py) + +--- + + + +### NCBI Datasets `AnnotationReportResponse` (`examples/ncbi_annotation_report`) + +**Rules:** [R-ROOT-REF](#r-root-ref), [R-SHAPE-REF](#r-shape-ref), [R-SHAPE-ARRAY](#r-shape-array), [R-SCALAR-ENUM-CONST](#r-scalar-enum-const) + +Deeply nested NCBI Datasets v2 schema (`parsers/ncbi/datasets_api/AnnotationReportResponse.schema.json`) with a +root `$ref`, many `$defs`, arrays of structs, enums, integers and booleans. + +**Schema:** [`tests/data/jsonschema/examples/ncbi_annotation_report/schema.json`](../tests/data/jsonschema/examples/ncbi_annotation_report/schema.json) + +- Run `minimal`: [`minimal.json`](../tests/data/jsonschema/examples/ncbi_annotation_report/minimal.json), [`minimal_expected.json`](../tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected_table.py) +- Run `maximal`: [`maximal.json`](../tests/data/jsonschema/examples/ncbi_annotation_report/maximal.json), [`maximal_expected.json`](../tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/jsonschema/examples/ncbi_annotation_report/expected_schema.py) + +--- + + + +### UniProtKB `SearchResult` (`examples/uniprot_search`) + +**Rules:** [R-UNION-OBJECT](#r-union-object), [R-UNION-SCALAR-OBJECT](#r-union-scalar-object), [R-RECURSION](#r-recursion), [R-SHAPE-MAP](#r-shape-map), [R-SCALAR-TYPE](#r-scalar-type) + +`/uniprotkb/search` response schema from `not_for_github/uniprot/api/openapi-3.1.json` +(`#/components/schemas/SearchResult`): a 24-branch polymorphic `Comment` `anyOf` (including a string + object +`note` with a synthesized `_value`), recursive `ProteinSection`, `extraAttributes` maps and `pa.date32()` dates. + +**Schema:** [`tests/data/jsonschema/examples/uniprot_search/schema.json`](../tests/data/jsonschema/examples/uniprot_search/schema.json) + +- Run `minimal`: [`minimal.json`](../tests/data/jsonschema/examples/uniprot_search/minimal.json), [`minimal_expected.json`](../tests/data/jsonschema/examples/uniprot_search/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/jsonschema/examples/uniprot_search/minimal_expected_table.py) +- Run `maximal`: [`maximal.json`](../tests/data/jsonschema/examples/uniprot_search/maximal.json), [`maximal_expected.json`](../tests/data/jsonschema/examples/uniprot_search/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/jsonschema/examples/uniprot_search/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/jsonschema/examples/uniprot_search/expected_schema.py) + +--- + +## 3. Errors (Unsupported / Invalid Input) + +Schema generation raises a subclass of `JSONSchemaToPyArrowError` (itself a `ValueError`). + +| Code | Exception | Condition | Reason | Fixtures | +| :--- | :--- | :--- | :--- | :--- | +| **E1** | `UnsupportedDialectError` | Missing or unsupported `$schema` dialect | Only JSON Schema Draft 2020-12 and the KBase Draft 2020-12 XSV metaschema are supported. | [`missing_schema_dialect.json`](../tests/data/jsonschema/spec/errors/missing_schema_dialect.json): Declares the Draft-07 dialect URI.
[`missing_schema_keyword.json`](../tests/data/jsonschema/spec/errors/missing_schema_keyword.json): No `$schema` keyword. | +| **E2** | `NonStructRootError` | Non-struct top-level schema | A `pa.Schema` needs top-level columns, so the root must resolve to a fixed-property object. | [`scalar_top_level.json`](../tests/data/jsonschema/spec/errors/scalar_top_level.json): Root is `{"type": "string"}`.
[`array_top_level.json`](../tests/data/jsonschema/spec/errors/array_top_level.json): Root is `{"type": "array", "items": {"type": "integer"}}`.
[`map_top_level.json`](../tests/data/jsonschema/spec/errors/map_top_level.json): Root is a pure map (`additionalProperties: {"type": "string"}`). | +| **E3** | `EmptyObjectError` | Empty object or tuple (zero fields) | Parquet cannot store a `pa.struct([])` with no child fields. | [`empty_object_top_level.json`](../tests/data/jsonschema/spec/errors/empty_object_top_level.json): Root object has `properties: {}`.
[`empty_object_nested.json`](../tests/data/jsonschema/spec/errors/empty_object_nested.json): Nested property has `properties: {}`. | +| **E4** | `UnconstrainedSchemaError` | Unconstrained or boolean schema | An untyped JSON value cannot be assigned a concrete Arrow `DataType`. | [`unconstrained_property.json`](../tests/data/jsonschema/spec/errors/unconstrained_property.json): Property schema is `{}`.
[`boolean_property_schema.json`](../tests/data/jsonschema/spec/errors/boolean_property_schema.json): Property schema is `true`. | +| **E5** | `OpenTupleError` | Open tuple (`prefixItems` with schema-valued `items`) | A field cannot be both a positional `pa.struct` and an unbounded `pa.list_`. | [`open_tuple_prefix_items.json`](../tests/data/jsonschema/spec/errors/open_tuple_prefix_items.json): `prefixItems` with `items: {"type": "boolean"}`. | +| **E6** | `MixedPropertiesAndMapError` | Fixed `properties` mixed with a dynamic map | An object cannot be both a fixed-field `pa.struct` and a dynamic `pa.map_`. | [`mixed_properties_and_map.json`](../tests/data/jsonschema/spec/errors/mixed_properties_and_map.json): `properties` together with `additionalProperties: {"type": "integer"}`. | +| **E7** | `NegationError` | Negation (`not`) | A complement type does not define a positive storage `DataType`. | [`negation_not_keyword.json`](../tests/data/jsonschema/spec/errors/negation_not_keyword.json): Property uses `"not": {"type": "string"}`. | +| **E8** | `IncompatibleBranchShapesError` | Incompatible structural union branches | Branches with different structural shapes (object vs array, array vs scalar, object vs map) cannot share one column. | [`incompatible_branch_shapes.json`](../tests/data/jsonschema/spec/errors/incompatible_branch_shapes.json): `oneOf` with an object branch and an array branch. | +| **E9** | `UnsupportedRefError` | Unresolvable or unsupported reference | Every `$ref` must resolve to a local `#/$defs/...` definition; `$dynamicRef` and `$recursiveRef` are not supported. | [`unresolvable_ref.json`](../tests/data/jsonschema/spec/errors/unresolvable_ref.json): `$ref` to the missing `#/$defs/DoesNotExist`.
[`dynamic_ref.json`](../tests/data/jsonschema/spec/errors/dynamic_ref.json): Property uses `$dynamicRef`. | +| **E10** | `RecursiveNonJsonTypeError` | Non-JSON-serializable type in a recursive `$ref` cycle | Truncated recursive subtrees are serialized with `json.dumps`, which cannot encode `datetime.date` or `bytes`. | [`recursive_date_type.json`](../tests/data/jsonschema/spec/errors/recursive_date_type.json): Recursive `DatedNode` has a `format: "date"` property.
[`recursive_binary_type.json`](../tests/data/jsonschema/spec/errors/recursive_binary_type.json): Recursive `BlobNode` has a `contentEncoding: "base64"` property. | +| **E11** | `NullOnlySchemaError` | Null-only schema | Parquet has no physical storage type for a `pa.null()` column. | [`null_only_field.json`](../tests/data/jsonschema/spec/errors/null_only_field.json): Property is `{"type": "null"}`.
[`const_null_field.json`](../tests/data/jsonschema/spec/errors/const_null_field.json): Property is `{"const": null}`.
[`enum_null_field.json`](../tests/data/jsonschema/spec/errors/enum_null_field.json): Property is `{"enum": [null]}`. | +| **E12** | `ValueFieldCollisionError` | `_value` collision in a scalar + object union | The synthesized `_value` field of a scalar + object union would collide with a declared `_value` property. | [`value_field_collision.json`](../tests/data/jsonschema/spec/errors/value_field_collision.json): Object branch of a string + object `anyOf` declares `_value`. | + +## 4. Fixture Index + +Every case directory contains a `case.yaml` manifest; the test suite and this document are both driven by it. + +| Case | Schema | Runs | +| :--- | :--- | :--- | +| [`spec/01_scalars_and_formats`](../tests/data/jsonschema/spec/01_scalars_and_formats/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/01_scalars_and_formats/schema.json) | default | +| [`spec/02_nullability_and_required`](../tests/data/jsonschema/spec/02_nullability_and_required/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/02_nullability_and_required/schema.json) | default | +| [`spec/03_arrays_and_tuples`](../tests/data/jsonschema/spec/03_arrays_and_tuples/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/03_arrays_and_tuples/schema.json) | default | +| [`spec/04_objects_and_maps`](../tests/data/jsonschema/spec/04_objects_and_maps/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/04_objects_and_maps/schema.json) | default | +| [`spec/05_defs_and_refs`](../tests/data/jsonschema/spec/05_defs_and_refs/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/05_defs_and_refs/schema.json) | default | +| [`spec/06_recursion_depth`](../tests/data/jsonschema/spec/06_recursion_depth/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/06_recursion_depth/schema.json) | depth-2, depth-1 | +| [`spec/07_combinators_allof_anyof_oneof`](../tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/schema.json) | default | +| [`spec/08_conditional_and_dependent_schemas`](../tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/schema.json) | default | +| [`spec/09_context_dependent_shared_defs`](../tests/data/jsonschema/spec/09_context_dependent_shared_defs/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/09_context_dependent_shared_defs/schema.json) | default | +| [`spec/10_combinations`](../tests/data/jsonschema/spec/10_combinations/case.yaml) | [`schema.json`](../tests/data/jsonschema/spec/10_combinations/schema.json) | default | +| [`examples/cts_job`](../tests/data/jsonschema/examples/cts_job/case.yaml) | [`schema.json`](../tests/data/jsonschema/examples/cts_job/schema.json) | minimal, maximal | +| [`examples/cts_jobs`](../tests/data/jsonschema/examples/cts_jobs/case.yaml) | [`schema.json`](../tests/data/jsonschema/examples/cts_jobs/schema.json) | minimal, maximal | +| [`examples/ncbi_annotation_report`](../tests/data/jsonschema/examples/ncbi_annotation_report/case.yaml) | [`schema.json`](../tests/data/jsonschema/examples/ncbi_annotation_report/schema.json) | minimal, maximal | +| [`examples/uniprot_search`](../tests/data/jsonschema/examples/uniprot_search/case.yaml) | [`schema.json`](../tests/data/jsonschema/examples/uniprot_search/schema.json) | minimal, maximal | diff --git a/docs/xsd_to_pyarrow_specification.md b/docs/xsd_to_pyarrow_specification.md index 2744c0a9..7081241b 100644 --- a/docs/xsd_to_pyarrow_specification.md +++ b/docs/xsd_to_pyarrow_specification.md @@ -1,69 +1,250 @@ -# XSD-to-PyArrow Specification + -This document specifies how an XML Schema (`.xsd`) and target `xml_tag` determine: -1. The **PyArrow `pa.Schema`** for the table yielded by the pipeline. -2. The **deterministic Python / JSON row dictionaries** produced by schema-driven XML element conversion (`convert_xml_element`) before constructing `pa.Table.from_pylist(rows, schema=pa_schema)`. -3. The **error conditions** raised when an XSD or `xml_tag` uses unsupported or ambiguous constructs. +# XSD to PyArrow Specification ---- +This document specifies how an XML Schema (`.xsd`) and a target `xml_tag` determine: + +1. The **PyArrow `pa.Schema`** of the table yielded by the pipeline (`generate_pyarrow_schema`). +2. The **Python / JSON row dictionaries** produced by `convert_xml_element` before + `pa.Table.from_pylist(rows, schema=pa_schema)`. +3. The **errors** raised when an XSD or `xml_tag` uses unsupported or ambiguous constructs. + +**Contents:** [1. Conversion Rules](#1-conversion-rules) · [2. Cases](#2-cases) · [3. Errors](#3-errors-unsupported--invalid-input) · [4. Fixture Index](#4-fixture-index) + +## 1. Conversion Rules + +### 1.1 Top-Level `xml_tag` Resolution and Unwrapping + + + +- **R-RESOLVE-DEPTH**: `xml_tag` is searched for among global elements (depth 0), then breadth-first through the content + models of their types (depth 1, 2, ...). Matches at the shallowest depth are deduplicated by type + identity; one type resolves, zero matches raise **E1**, and more than one distinct type raises **E2**. + + *Cases:* [07_multi_root_shared_type](#case-spec-07-multi-root-shared-type), [uniref](#case-examples-uniref) + + + +- **R-RESOLVE-CLARK**: `xml_tag` may be a local name (`entry`) or Clark notation (`{http://example.org/ns}entry`); both + resolve to the same declaration. + + *Cases:* [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable) + + + +- **R-UNWRAP**: Each `` element becomes one row. Its type must map to a `pa.struct` (see **E3**), and the + struct's fields become the top-level columns of the `pa.Schema`. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types) + +### 1.2 Naming and Field Ordering + + -## 1. Summary of Conversion Rules +- **R-NAME-ATTR**: Attributes become fields prefixed with `_` (`id` -> `_id`). Namespace declarations (`xmlns`, + `xmlns:*`) and `xsi:*` attributes are stripped. -### 1.1 Top-Level `xml_tag` Resolution & Unwrapping -- **Resolution:** Search the XSD for `xml_tag` starting at depth 0 (global root elements) and depth 1 (direct children of global root elements), deduplicating matches by `id(element.type)`. - - If **0** elements match (even after falling back to deeper elements), raise an error. - - If **>1** distinct `XsdType` definitions match at the shallowest matching depth, raise an ambiguity error. -- **Unwrapping:** Each streamed `<{xml_tag}>` element becomes one row in the output `pa.Table`. - - `xml_tag` must have a `complexType` that maps to a `pa.StructType` (at least one attribute, `#text`, or child element field). The outer `xml_tag` key is unwrapped so that the top-level columns of `pa.Schema` are the struct fields of `xml_tag`. - - If `xml_tag` maps to a scalar, list, or empty marker type with no attributes and no struct fields, schema generation raises a `ValueError` (see Error E3 in Section 3). + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types), [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable) -### 1.2 Naming & Field Ordering -- **Attributes:** Prefixed with `_` (e.g., `id` $\rightarrow$ `_id`). Namespace declarations (`xmlns`, `xmlns:*`) and `xsi:*` attributes (`xsi:nil`, `xsi:schemaLocation`, `xsi:noNamespaceSchemaLocation`) are stripped. -- **Text content (`#text`):** When a complex type has both attributes and text (`simpleContent` with attributes, or `mixed="true"` with attributes), text is stored under `#text`. -- **Struct Field Order:** - 1. Declared attributes (`_attr`), in XSD declaration order (base type attributes before extension attributes). - 2. `#text` (if applicable). - 3. Child elements, in XSD content-model traversal order (deduplicated on first appearance if the same element name appears in multiple branches of a `` or ``). + + +- **R-NAME-TEXT**: When a type has attributes and text (`simpleContent` with attributes, or `mixed="true"` with + attributes), the text is stored in a field named `#text`. + + *Cases:* [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable), [03_mixed_content_raw_text](#case-spec-03-mixed-content-raw-text), [10_combinations](#case-spec-10-combinations) + + + +- **R-ORDER**: Struct fields are ordered: declared attributes in declaration order (base type before extension), + then `#text`, then child elements in content-model traversal order. A child name appearing in + several branches or positions is emitted once, at its first appearance. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types), [05_groups_and_compositors](#case-spec-05-groups-and-compositors), [10_combinations](#case-spec-10-combinations) ### 1.3 Element Classification (Shape) -| XSD Element Type | Cardinality (`overall_max_occurs`) | Python / JSON Value | PyArrow DataType | -| :--- | :--- | :--- | :--- | -| **`xs:simpleType`** (atomic) | `1` | Scalar (`str`, `int`, `float`, `bool`, `date`, `bytes`) or `None` | Scalar `pa.DataType` | -| **`xs:simpleType`** (atomic) | `> 1` or `unbounded` | `list[Scalar \| None]` or `None` (when absent) | `pa.list_(Scalar)` | -| **`xs:simpleType`** (`xs:list`) | `1` | `list[ItemScalar]` or `None` | `pa.list_(ItemScalar)` | -| **`xs:simpleType`** (`xs:list`) | `> 1` or `unbounded` | `list[list[ItemScalar] \| None]` or `None` | `pa.list_(pa.list_(ItemScalar))` | -| **`complexType` (`simpleContent`)** without attributes | `1` / `> 1` | Same as underlying `simpleType` | Same as underlying `simpleType` | -| **`complexType` (`simpleContent`)** with attributes | `1` | `{"_attr": ..., "#text": Scalar}` or `None` | `pa.struct([("_attr", ...), ("#text", Scalar)])` | -| **`complexType` (`mixed="true"`)** without attributes | `1` | Raw inner XML string `str` or `None` | `pa.string()` | -| **`complexType` (`mixed="true"`)** with attributes | `1` | `{"_attr": ..., "#text": str}` or `None` | `pa.struct([("_attr", ...), ("#text", pa.string())])` | -| **`complexType` (attribute-only)** | `1` | `{"_attr": ...}` or `None` | `pa.struct([("_attr", ...)])` | -| **`complexType` (element-only)** | `1` | `{"_attr": ..., "child": ...}` or `None` | `pa.struct([...])` | -| **`complexType` (empty: no attrs, no children, not mixed)** | `1` / `> 1` | `""` (when present) or `None` (when absent) / `["", ...]` | `pa.string()` / `pa.list_(pa.string())` | - -*(For any `complexType` row above with `overall_max_occurs > 1` or `unbounded`, the Python value is `list[...]` and the PyArrow type is `pa.list_(...)`. When structural recursion exceeds `max_recursion_depth`, a single recursive element becomes `pa.string()` and a repeated recursive element (`overall_max_occurs > 1`) becomes `pa.list_(pa.string())`, where each truncated item is serialized as a JSON string.)* - -### 1.4 Field Nullability (`nullable=False` vs `nullable=True`) & Empty Tag Coercion -- **Attributes (`_attr`):** - - `nullable=False` when `use="required"` (and the enclosing element is not a `nillable="true"` element that can be nilled without attributes). - - `nullable=True` when `use="optional"` (default). -- **Child Elements:** - - `nullable=False` when **all** of the following hold: - 1. The element particle has `minOccurs >= 1`. - 2. The element declaration has `nillable="false"` (default). - 3. Every enclosing compositor/group (``, ``, ``) up to the parent `complexType` has `minOccurs >= 1`, and the element is not conditional inside an `` branch (unless required in every branch of the choice). - 4. The parent element is not `nillable="true"` (since a parent with `xsi:nil="true"` and attributes, e.g. ``, produces a struct `{"_type": "function"}` whose child elements are `None`). - - `nullable=True` in all other cases. -- **`#text` Field Inside a Struct with Attributes:** - - `nullable=True` for `mixed="true"` types, `nillable="true"` elements, and string-based `simpleContent` types (where `` has no `#text` key). - - `nullable=False` for non-nillable `simpleContent` types with non-string atomic base types (e.g., `xs:int`, `xs:decimal`), where valid XML always provides lexical text. -- **Empty Standalone Tag Coercion (`` $\rightarrow$ `""`):** - - When a standalone element mapped to `pa.string()` (atomic string `simpleType`, string `simpleContent` without attributes, `mixed="true"` without attributes, or empty `complexType` marker) is **present** in the XML as `` or `` (without `xsi:nil="true"`), its value is coerced to `""` so that present empty elements satisfy `nullable=False` and are distinguishable from omitted optional elements (`None`). - -### 1.5 Built-In XSD Scalar Type Mapping & Field Metadata -| XSD Built-In Type(s) | Python Type (Coerced) | PyArrow `DataType` | + +| XSD element type | Single (`overall_max_occurs == 1`) | Repeated (`> 1` or `unbounded`) | +| :--- | :--- | :--- | +| atomic `simpleType` | scalar | `pa.list_(scalar)` | +| `xs:list` `simpleType` | `pa.list_(item)` | `pa.list_(pa.list_(item))` | +| `simpleContent`, no attributes | as underlying `simpleType` | as underlying `simpleType`, listed | +| `simpleContent` with attributes | `pa.struct([_attr..., #text])` | `pa.list_(pa.struct(...))` | +| `mixed="true"`, no attributes | `pa.string()` (raw inner XML) | `pa.list_(pa.string())` | +| `mixed="true"` with attributes | `pa.struct([_attr..., #text: string])` | `pa.list_(pa.struct(...))` | +| attribute-only / element-only `complexType` | `pa.struct([...])` | `pa.list_(pa.struct(...))` | +| empty `complexType` (no attrs, no children, not mixed) | `pa.string()` | `pa.list_(pa.string())` | + + + +- **R-SHAPE-CARDINALITY**: A child's `overall_max_occurs` is computed across the whole content model: element `maxOccurs` + multiplied through enclosing compositors, summed over `xs:sequence` / `xs:all` positions and + maximised over `xs:choice` branches. A value greater than 1 (including a child of a repeated + compositor that itself has `maxOccurs="1"`) makes the field a `pa.list_`. `maxOccurs="0"` children + are omitted. + + *Cases:* [04_recursion_depth](#case-spec-04-recursion-depth), [05_groups_and_compositors](#case-spec-05-groups-and-compositors), [09_empty_marker_elements](#case-spec-09-empty-marker-elements), [10_combinations](#case-spec-10-combinations) + + + +- **R-SHAPE-GROUPS**: `xs:group` and `xs:attributeGroup` references are expanded in place; `xs:sequence`, `xs:choice` + and `xs:all` contribute their elements to the enclosing struct. + + *Cases:* [03_mixed_content_raw_text](#case-spec-03-mixed-content-raw-text), [05_groups_and_compositors](#case-spec-05-groups-and-compositors) + + + +- **R-SHAPE-EXTENSION**: `complexContent` / `xs:extension` merges base-type attributes and children with the extension's, + base first. + + *Cases:* [10_combinations](#case-spec-10-combinations) + + + +- **R-SHAPE-XSLIST**: `xs:list` simple types, and the built-in list types `xs:NMTOKENS`, `xs:IDREFS`, `xs:ENTITIES`, map to + `pa.list_(item)` and are coerced by splitting on whitespace and coercing each token. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types), [uniprot](#case-examples-uniprot) + + + +- **R-SHAPE-SIMPLECONTENT**: `simpleContent` without attributes maps exactly like its base `simpleType`. With attributes it is a + struct of the attribute fields followed by `#text`. + + *Cases:* [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable), [06_context_dependent_paths](#case-spec-06-context-dependent-paths), [10_combinations](#case-spec-10-combinations) + + + +- **R-SHAPE-MIXED**: `mixed="true"` content is captured as the raw inner XML string; declared child elements are not + emitted as fields. Inline tags (``, ``, ...) are kept, text entities stay escaped (`&`, + `<`, `>`), and namespace declarations are dropped from inline tags. + + *Cases:* [03_mixed_content_raw_text](#case-spec-03-mixed-content-raw-text), [10_combinations](#case-spec-10-combinations), [pubmed_article](#case-examples-pubmed-article), [pubmed_book_article](#case-examples-pubmed-book-article) + + + +- **R-SHAPE-ATTR-ONLY**: A `complexType` with attributes but no content maps to a struct of its attribute fields. + + *Cases:* [10_combinations](#case-spec-10-combinations) + + + +- **R-SHAPE-EMPTY-MARKER**: A `complexType` with no attributes, no children and no mixed content is an empty marker: it maps + to `pa.string()`, with value `""` when present and `None` when absent. + + *Cases:* [09_empty_marker_elements](#case-spec-09-empty-marker-elements), [10_combinations](#case-spec-10-combinations), [pubmed_article](#case-examples-pubmed-article), [pubmed_book_article](#case-examples-pubmed-book-article) + + + +- **R-SHAPE-CONTEXT-PATH**: Types are resolved per ancestor path from `xml_tag`, not per `(parent, child)` name pair, so the + same child name under differently-typed parents gets each parent's exact type. + + *Cases:* [06_context_dependent_paths](#case-spec-06-context-dependent-paths), [10_combinations](#case-spec-10-combinations) + + + +- **R-RECURSION**: Each complex type may be expanded at most `max_recursion_depth` times (default `2`) along a path. + A deeper occurrence becomes `pa.string()` (or `pa.list_(pa.string())` when repeated) and its + subtree is serialized with `json.dumps`. Applies to type self-recursion, element-`ref` + self-recursion and mutual recursion. Truncated `pa.string()` fields carry no `xsd_type`. + + *Cases:* [04_recursion_depth](#case-spec-04-recursion-depth), [10_combinations](#case-spec-10-combinations), [pubmed_article](#case-examples-pubmed-article), [pubmed_book_article](#case-examples-pubmed-book-article) + +### 1.4 Field Nullability and Empty Tag Coercion + + + +- **R-NULL-ATTR**: Attribute fields are `nullable=False` when `use="required"` and the enclosing element is not + `nillable="true"`; otherwise `nullable=True`. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types), [08_nullability_and_empty_tags](#case-spec-08-nullability-and-empty-tags) + + + +- **R-NULL-CHILD**: A child field is `nullable=False` only when its overall `minOccurs` (multiplied through all + enclosing compositors) is at least 1, it is not `nillable="true"`, and its parent element is not + `nillable="true"`. Otherwise it is `nullable=True`. + + *Cases:* [05_groups_and_compositors](#case-spec-05-groups-and-compositors), [06_context_dependent_paths](#case-spec-06-context-dependent-paths), [08_nullability_and_empty_tags](#case-spec-08-nullability-and-empty-tags) + + + +- **R-NULL-CHOICE**: Elements that appear in only some branches of an `xs:choice` have overall `minOccurs == 0` and are + therefore `nullable=True`. + + *Cases:* [05_groups_and_compositors](#case-spec-05-groups-and-compositors), [08_nullability_and_empty_tags](#case-spec-08-nullability-and-empty-tags), [pubmed_article](#case-examples-pubmed-article) + + + +- **R-NULL-NILLABLE-PARENT**: All attribute and child fields inside a `nillable="true"` complex element are `nullable=True`, since + `` produces `{"_type": "x"}` with no children. + + *Cases:* [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable), [08_nullability_and_empty_tags](#case-spec-08-nullability-and-empty-tags), [uniprot](#case-examples-uniprot) + + + +- **R-NULL-TEXT**: `#text` fields are always `nullable=True` (`` has no text), and list `item` fields + are always `nullable=True`. + + *Cases:* [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable), [03_mixed_content_raw_text](#case-spec-03-mixed-content-raw-text) + + + +- **R-NIL**: `xsi:nil="true"` yields `None` for an element with no attributes, and a struct holding only its + attributes otherwise. + + *Cases:* [02_namespaces_and_nillable](#case-spec-02-namespaces-and-nillable), [08_nullability_and_empty_tags](#case-spec-08-nullability-and-empty-tags), [uniprot](#case-examples-uniprot) + + + +- **R-EMPTY-COERCE**: A present empty element (``, no `xsi:nil`) that maps to a non-nullable `pa.string()` field is + coerced to `""`, distinguishing it from an absent optional element (`None`). Empty markers are + always `""` when present. + + *Cases:* [03_mixed_content_raw_text](#case-spec-03-mixed-content-raw-text), [08_nullability_and_empty_tags](#case-spec-08-nullability-and-empty-tags), [09_empty_marker_elements](#case-spec-09-empty-marker-elements) + +### 1.5 Scalar Type Mapping and Field Metadata + + + +- **R-SCALAR-BUILTIN**: Built-in XSD types map to Arrow types per the table below. Unlisted built-ins fall back to + `pa.string()`. `xs:boolean` accepts `true`/`1` and `false`/`0`; `xs:date` is coerced with + `datetime.date.fromisoformat`; `xs:hexBinary` with `bytes.fromhex`; `xs:base64Binary` with + `base64.b64decode`. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types) + + + +- **R-SCALAR-RESTRICTION**: `xs:restriction` types (including enumerations) inherit the mapping of their built-in base type. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types) + + + +- **R-SCALAR-UNION**: An `xs:union` uses the shared Arrow type when every member maps to the same type; otherwise it + falls back to `pa.string()`. Its `xsd_type` joins the member names with `" | "`. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types), [uniprot](#case-examples-uniprot) + + + +- **R-META-XSD-TYPE**: Every leaf scalar field (including `#text` and scalar list `item` fields) carries + `metadata["xsd_type"]` with the built-in `xs:` name. Struct, outer list, empty-marker and + recursion-truncated fields do not. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types) + + + +- **R-META-COMMENT**: `` on an element or attribute (or, failing that, on its `ref` + target or type) is whitespace-normalized and stored in `metadata["comment"]`. + + *Cases:* [01_scalars_and_simple_types](#case-spec-01-scalars-and-simple-types), [uniprot](#case-examples-uniprot), [uniref](#case-examples-uniref) + +| XSD built-in type(s) | Python type (coerced) | PyArrow `DataType` | | :--- | :--- | :--- | -| `xs:string`, `xs:normalizedString`, `xs:token`, `xs:language`, `xs:Name`, `xs:NCName`, `xs:NMTOKEN`, `xs:ID`, `xs:IDREF`, `xs:ENTITY`, `xs:anyURI`, `xs:QName`, `xs:NOTATION` | `str` | `pa.string()` | -| `xs:boolean` (`"true"`/`"1"` $\rightarrow$ `True`, `"false"`/`"0"` $\rightarrow$ `False`) | `bool` | `pa.bool_()` | +| `xs:string`, `xs:normalizedString`, `xs:token`, `xs:language`, `xs:Name`, `xs:NCName`, `xs:NMTOKEN`, `xs:ID`, `xs:IDREF`, `xs:ENTITY`, `xs:anyURI`, `xs:QName`, `xs:NOTATION`, `xs:dateTime`, `xs:time`, `xs:duration`, `xs:gYear`, `xs:gYearMonth`, `xs:gMonthDay`, `xs:gDay`, `xs:gMonth` | `str` | `pa.string()` | +| `xs:boolean` | `bool` | `pa.bool_()` | | `xs:byte` | `int` | `pa.int8()` | | `xs:short` | `int` | `pa.int16()` | | `xs:int` | `int` | `pa.int32()` | @@ -74,26 +255,25 @@ This document specifies how an XML Schema (`.xsd`) and target `xml_tag` determin | `xs:unsignedLong` | `int` | `pa.uint64()` | | `xs:float`, `xs:double`, `xs:decimal` | `float` | `pa.float64()` | | `xs:date` | `datetime.date` | `pa.date32()` | -| `xs:dateTime`, `xs:time`, `xs:duration`, `xs:gYear`, `xs:gYearMonth`, `xs:gMonthDay`, `xs:gDay`, `xs:gMonth` | `str` | `pa.string()` | -| `xs:base64Binary` | `bytes` (`base64.b64decode`) | `pa.binary()` | -| `xs:hexBinary` | `bytes` (`bytes.fromhex`) | `pa.binary()` | -| `xs:NMTOKENS`, `xs:IDREFS`, `xs:ENTITIES` | `list[str]` (`.split()`) | `pa.list_(pa.string())` | - -- **`xs:restriction`:** Inherits the primitive/built-in base type mapping above. -- **`xs:list`:** Maps to `pa.list_(item_type)` and coerces by splitting whitespace (`value.split()`) and coercing each item. -- **`xs:union`:** If all `memberTypes` map to the same `pa.DataType`, uses that type; otherwise falls back to `pa.string()` (`str`). -- **`pa.Field.metadata`:** - - **`"xsd_type"`:** Recorded on every leaf scalar `pa.Field` (including `#text` and list `"item"` fields when list elements are scalars) with the prefixed built-in XSD type name (e.g., `"xs:string"`, `"xs:int"`, `"xs:float"`, `"xs:double"`, `"xs:decimal"`, or `"xs:date | xs:gYearMonth | xs:gYear"` for unions). Omitted on `pa.struct(...)`, outer `pa.list_(...)`, empty-marker `pa.string()`, and recursion-truncated JSON `pa.string()` fields. - - **`"comment"`:** Extracted from `` on the element or attribute declaration (falling back to its referenced `type`'s `` if the declaration itself has none), whitespace-normalized with `" ".join(doc.split())`. +| `xs:base64Binary`, `xs:hexBinary` | `bytes` | `pa.binary()` | +| `xs:NMTOKENS` (items `xs:NMTOKEN`), `xs:IDREFS` (items `xs:IDREF`), `xs:ENTITIES` (items `xs:ENTITY`) | `list[str]` | `pa.list_(pa.string())` | ---- +## 2. Cases + + + +### Scalar Types, Restrictions, Lists and Unions (`spec/01_scalars_and_simple_types`) -## 2. Specification Examples +**Rules:** [R-UNWRAP](#r-unwrap), [R-NAME-ATTR](#r-name-attr), [R-ORDER](#r-order), [R-SHAPE-XSLIST](#r-shape-xslist), [R-SCALAR-BUILTIN](#r-scalar-builtin), [R-SCALAR-RESTRICTION](#r-scalar-restriction), [R-SCALAR-UNION](#r-scalar-union), [R-META-XSD-TYPE](#r-meta-xsd-type), [R-META-COMMENT](#r-meta-comment), [R-NULL-ATTR](#r-null-attr) -### Example 1: Scalar Types, Restrictions, Lists (`xs:list`), and Unions (`xs:union`) -Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumerations, `xs:list` (`intListType` from [`uniprot.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd#L1122-L1124)), and `xs:union` (`date | gYearMonth | gYear` from [`uniprot.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd#L341-L345)). +**Target:** `xml_tag='record'` + +- `code` is `xs:int | xs:short`: members map to different Arrow types (`int32` vs `int16`), so it falls back to `pa.string()`. +- `published` is `xs:date | xs:gYearMonth | xs:gYear` and also resolves to `pa.string()`. +- `scores` and `_evidence` use the `xs:list` type `intListType` and become `pa.list_(pa.int32())`. + +#### Schema (`schema.xsd`) -#### XSD ```xml @@ -105,11 +285,14 @@ Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumeratio - - + + + + Record lifecycle status. + @@ -122,23 +305,36 @@ Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumeratio - + + + Primary boolean flag. + + - + + + + + - + + - + + + Unique record identifier. + + @@ -150,8 +346,10 @@ Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumeratio ``` -#### Sample XML (`xml_tag="record"`) +#### Input (`data.xml`) + ```xml + true @@ -159,83 +357,194 @@ Covers built-in numeric/boolean/date/binary scalars, `xs:restriction` enumeratio 1024 100000 999999999999 - 18446744073709551615 + 255 + 65535 + 4000000000 + 18446744073709551615 1.5 2.718281828 3.14 2025-06-15 + 2025-06-15T12:30:00Z 2025-06 - 42 + 42000000000 10 20 30 + alpha beta gamma 48656c6c6f SGVsbG8= 0 + + 1 + -128 + -32768 + -2147483648 + -999999999999 + 0 + 0 + 0 + 0 + -0.5 + 1.25e-4 + 42 + 2025-06-15Z + 2025-06-15T12:30:00.123+02:00 + 1999 + -42 + 100 200 + one two + 48656C6C6F + SGVsbG8= + ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "_id": "rec1", "_status": "active", - "_evidence": [1, 4, 9], + "_evidence": [ + 1, + 4, + 9 + ], "_version": 2, - "flag": True, + "flag": true, "smallInt": -5, "medInt": 1024, "stdInt": 100000, "bigInt": 999999999999, - "ulong": 18446744073709551615, + "uByte": 255, + "uShort": 65535, + "uInt": 4000000000, + "uLong": 18446744073709551615, "f32": 1.5, "f64": 2.718281828, "dec": 3.14, - "created": datetime.date(2025, 6, 15), + "created": "2025-06-15", + "updatedAt": "2025-06-15T12:30:00Z", "published": "2025-06", - "code": "42", # int + short -> different Arrow types (int32 vs int16) -> falls back to str - "scores": [10, 20, 30], - "rawHex": b"Hello", - "rawB64": b"Hello", + "homogeneousUnion": 42000000000, + "scores": [ + 10, + 20, + 30 + ], + "tokens": [ + "alpha", + "beta", + "gamma" + ], + "rawHex": "Hello", + "rawB64": "Hello" }, { "_id": "rec2", "_status": "archived", "_version": 1, - "flag": False, + "flag": false }, + { + "_id": "rec3", + "_status": "active", + "_evidence": [ + 10, + 20 + ], + "_version": -42, + "flag": true, + "smallInt": -128, + "medInt": -32768, + "stdInt": -2147483648, + "bigInt": -999999999999, + "uByte": 0, + "uShort": 0, + "uInt": 0, + "uLong": 0, + "f32": -0.5, + "f64": 0.000125, + "dec": 42.0, + "created": "2025-06-15", + "updatedAt": "2025-06-15T12:30:00.123+02:00", + "published": "1999", + "homogeneousUnion": -42, + "scores": [ + 100, + 200 + ], + "tokens": [ + "one", + "two" + ], + "rawHex": "Hello", + "rawB64": "Hello" + } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 01_scalars_and_simple_types (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:ID"}), - pa.field("_status", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "_id", pa.string(), nullable=False, metadata={"comment": "Unique record identifier.", "xsd_type": "xs:ID"} + ), + pa.field( + "_status", + pa.string(), + nullable=False, + metadata={"comment": "Record lifecycle status.", "xsd_type": "xs:string"}, + ), pa.field( "_evidence", - pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), nullable=True, ), pa.field("_version", pa.int32(), nullable=False, metadata={"xsd_type": "xs:int"}), - pa.field("flag", pa.bool_(), nullable=False, metadata={"xsd_type": "xs:boolean"}), + pa.field( + "flag", pa.bool_(), nullable=False, metadata={"comment": "Primary boolean flag.", "xsd_type": "xs:boolean"} + ), pa.field("smallInt", pa.int8(), nullable=True, metadata={"xsd_type": "xs:byte"}), pa.field("medInt", pa.int16(), nullable=True, metadata={"xsd_type": "xs:short"}), pa.field("stdInt", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), pa.field("bigInt", pa.int64(), nullable=True, metadata={"xsd_type": "xs:integer"}), - pa.field("ulong", pa.uint64(), nullable=True, metadata={"xsd_type": "xs:unsignedLong"}), + pa.field("uByte", pa.uint8(), nullable=True, metadata={"xsd_type": "xs:unsignedByte"}), + pa.field("uShort", pa.uint16(), nullable=True, metadata={"xsd_type": "xs:unsignedShort"}), + pa.field("uInt", pa.uint32(), nullable=True, metadata={"xsd_type": "xs:unsignedInt"}), + pa.field("uLong", pa.uint64(), nullable=True, metadata={"xsd_type": "xs:unsignedLong"}), pa.field("f32", pa.float64(), nullable=True, metadata={"xsd_type": "xs:float"}), pa.field("f64", pa.float64(), nullable=True, metadata={"xsd_type": "xs:double"}), pa.field("dec", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), pa.field("created", pa.date32(), nullable=True, metadata={"xsd_type": "xs:date"}), + pa.field("updatedAt", pa.string(), nullable=True, metadata={"xsd_type": "xs:dateTime"}), pa.field("published", pa.string(), nullable=True, metadata={"xsd_type": "xs:date | xs:gYearMonth | xs:gYear"}), - pa.field("code", pa.string(), nullable=True, metadata={"xsd_type": "xs:int | xs:short"}), + pa.field("homogeneousUnion", pa.int64(), nullable=True, metadata={"xsd_type": "xs:long | xs:integer"}), pa.field( "scores", - pa.list_(pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"})), + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=True, + ), + pa.field( + "tokens", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:NMTOKEN"}), + ), nullable=True, ), pa.field("rawHex", pa.binary(), nullable=True, metadata={"xsd_type": "xs:hexBinary"}), @@ -243,17 +552,25 @@ pa.schema( ] ) ``` -*(In Examples 2–9 below, `metadata={"xsd_type": ...}` on scalar fields is omitted for brevity.)* --- -### Example 2: Namespaces & `nillable="true"` (`xsi:nil="true"`) -Covers `targetNamespace`, `elementFormDefault="qualified"`, stripping `xmlns` / `xmlns:xsi`, and the 3 cases of `xsi:nil="true"`: -1. Nillable element with **no other attributes** $\rightarrow$ `None`. -2. Nillable complex element **with attributes** (like `` in [`uniprot.xsd` L47](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd#L47)) $\rightarrow$ `{"_type": "function"}` (with `_xsi:nil` stripped and child fields `null`). -3. Nillable `simpleContent` element **with attributes** $\rightarrow$ `{"_lang": "en"}` (`#text` is `null`). + + +### Namespaces and `nillable="true"` (`spec/02_namespaces_and_nillable`) + +**Rules:** [R-RESOLVE-CLARK](#r-resolve-clark), [R-NAME-ATTR](#r-name-attr), [R-NAME-TEXT](#r-name-text), [R-SHAPE-SIMPLECONTENT](#r-shape-simplecontent), [R-NIL](#r-nil), [R-NULL-NILLABLE-PARENT](#r-null-nillable-parent), [R-NULL-TEXT](#r-null-text) + +**Target:** `xml_tag='entry'` + +Covers `targetNamespace` with `elementFormDefault="qualified"` and the three `xsi:nil="true"` cases: + +1. Nillable element with no attributes (``) -> `None`. +2. Nillable complex element with attributes (`<comment type="function" xsi:nil="true"/>`) -> `{"_type": "function"}`. +3. Nillable `simpleContent` element with attributes (`<label lang="en" xsi:nil="true"/>`) -> `{"_lang": "en"}`. + +#### Schema (`schema.xsd`) -#### XSD ```xml <?xml version="1.0" encoding="UTF-8"?> <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" @@ -294,8 +611,10 @@ Covers `targetNamespace`, `elementFormDefault="qualified"`, stripping `xmlns` / </xs:schema> ``` -#### Sample XML (`xml_tag="entry"`) +#### Input (`data.xml`) + ```xml +<?xml version="1.0" encoding="UTF-8"?> <root xmlns="http://example.org/ns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <entry id="e1"> <title xsi:nil="true"/> @@ -312,38 +631,77 @@ Covers `targetNamespace`, `elementFormDefault="qualified"`, stripping `xmlns` / </root> ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "_id": "e1", - "title": None, - "label": {"_lang": "en"}, + "label": { + "_lang": "en" + }, "comment": [ - {"_type": "function"}, - {"_type": "subunit", "text": "Homodimer"}, - ], + { + "_type": "function" + }, + { + "_type": "subunit", + "text": "Homodimer" + } + ] }, { "_id": "e2", - "title": "Normal Title", - "label": None, + "title": "Normal Title" + } +] +``` + +#### Converted rows: run `clark-notation` (`xml_tag='{http://example.org/ns}entry'`) + +```json +[ + { + "_id": "e1", + "label": { + "_lang": "en" + }, + "comment": [ + { + "_type": "function" + }, + { + "_type": "subunit", + "text": "Homodimer" + } + ] }, + { + "_id": "e2", + "title": "Normal Title" + } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 02_namespaces_and_nillable (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("_id", pa.string(), nullable=False), - pa.field("title", pa.string(), nullable=True), + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("title", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), pa.field( "label", pa.struct( [ - pa.field("_lang", pa.string(), nullable=True), - pa.field("#text", pa.string(), nullable=True), + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), ] ), nullable=True, @@ -351,12 +709,16 @@ pa.schema( pa.field( "comment", pa.list_( - pa.struct( - [ - pa.field("_type", pa.string(), nullable=False), - pa.field("text", pa.string(), nullable=True), - ] - ) + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), ), nullable=True, ), @@ -366,15 +728,20 @@ pa.schema( --- -### Example 3: Mixed Content Kept as Raw Text -Covers `mixed="true"` elements with inline HTML/formatting tags (`<b>`, `<i>`, `<sup>`, `<sub>`, `<u>`), both with and without attributes on the mixed element itself (as in [`pubmed.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd) `ArticleTitle`, `AbstractText`, and [`10_combinations/schema.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd) `ParaType`, `NoteType`, `RemarkType`). +<a id="case-spec-03-mixed-content-raw-text"></a> + +### Mixed Content Kept as Raw Text (`spec/03_mixed_content_raw_text`) -- Child elements declared inside a `mixed="true"` complexType are **not** emitted as struct fields; instead, the mixed element's entire inner XML is captured as a raw text string. -- Character entities in text nodes (`&`, `<`, `>`) remain XML-escaped in the captured raw text string so that literal `<` or `>` is distinguishable from inline HTML tags like `<b>`. -- If the `mixed="true"` type has **no attributes**, its PyArrow type is `pa.string()` (and an empty tag `<title/>` without `xsi:nil="true"` coerces to `""`). -- If the `mixed="true"` type **has attributes**, its PyArrow type is `pa.struct([("_attr", ...), ("#text", pa.string())])`. +**Rules:** [R-SHAPE-MIXED](#r-shape-mixed), [R-NAME-TEXT](#r-name-text), [R-SHAPE-GROUPS](#r-shape-groups), [R-EMPTY-COERCE](#r-empty-coerce), [R-NULL-TEXT](#r-null-text) + +**Target:** `xml_tag='article'` + +- `title` is `mixed="true"` without attributes -> `pa.string()`; the empty `<title/>` coerces to `""`. +- `abstractText` is `mixed="true"` with a `Label` attribute -> `pa.struct([_Label, #text])`. +- `&` in text remains escaped so literal `<`/`>` are distinguishable from inline tags. + +#### Schema (`schema.xsd`) -#### XSD ```xml <?xml version="1.0" encoding="UTF-8"?> <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> @@ -383,6 +750,7 @@ Covers `mixed="true"` elements with inline HTML/formatting tags (`<b>`, `<i>`, ` <xs:element name="b" type="formattedText"/> <xs:element name="i" type="formattedText"/> <xs:element name="sup" type="formattedText"/> + <xs:element name="sub" type="formattedText"/> </xs:choice> </xs:group> @@ -412,13 +780,15 @@ Covers `mixed="true"` elements with inline HTML/formatting tags (`<b>`, `<i>`, ` </xs:schema> ``` -#### Sample XML (`xml_tag="article"`) +#### Input (`data.xml`) + ```xml +<?xml version="1.0" encoding="UTF-8"?> <articles> <article> - <title> Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells - Studied bold & italic text. - Unlabelled paragraph with tags. + Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells + Studied bold & italic text where x < y. + Unlabelled paragraph with 2O and empty tag. @@ -428,38 +798,57 @@ Covers `mixed="true"` elements with inline HTML/formatting tags (``, ``, ` ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "title": "Ca2+ signaling in E. coli cells", "abstractText": [ - {"_Label": "BACKGROUND", "#text": "Studied bold & italic text."}, - {"#text": "Unlabelled paragraph with tags."}, - {"_Label": "EMPTY"}, - None, - ], + { + "_Label": "BACKGROUND", + "#text": "Studied bold & italic text where x < y." + }, + { + "#text": "Unlabelled paragraph with 2O and empty tag." + }, + { + "_Label": "EMPTY" + }, + null + ] }, { - "title": "", - }, + "title": "" + } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 03_mixed_content_raw_text (xml_tag='article').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("title", pa.string(), nullable=False), + pa.field("title", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), pa.field( "abstractText", pa.list_( - pa.struct( - [ - pa.field("_Label", pa.string(), nullable=True), - pa.field("#text", pa.string(), nullable=True), - ] - ) + pa.field( + "item", + pa.struct( + [ + pa.field("_Label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), ), nullable=True, ), @@ -469,18 +858,19 @@ pa.schema( --- -### Example 4: Structural Recursion with Configurable `max_recursion_depth` -Covers both: -1. **Type self-recursion** (`TreeType` $\rightarrow$ `branch: TreeType`) -2. **Element-ref self-recursion** (`Section` $\rightarrow$ `` as in [`pubmed.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd)) -3. **Mutual recursion** (`NodeA` $\rightarrow$ `NodeB` $\rightarrow$ `NodeA`) + -With `max_recursion_depth = 2`: -- Occurrence 1 (top-level `Section` or `tree`) = depth 1 (`pa.struct`) -- Occurrence 2 (child `Section` or `branch`) = depth 2 (`pa.struct`) -- Occurrence 3 (grandchild `Section` or `branch` beyond `max_recursion_depth=2`) = flattened to JSON string (`pa.string()` when `overall_max_occurs == 1`, or `pa.list_(pa.string())` when `overall_max_occurs > 1`). +### Structural Recursion with Configurable `max_recursion_depth` (`spec/04_recursion_depth`) + +**Rules:** [R-RECURSION](#r-recursion), [R-SHAPE-CARDINALITY](#r-shape-cardinality) + +**Target:** `xml_tag='record'` + +With `max_recursion_depth=2`, `tree` (depth 1) and its `branch` children (depth 2) are structs; grandchild +`branch` elements are serialized to JSON strings. With `max_recursion_depth=1`, `branch` is already truncated. + +#### Schema (`schema.xsd`) -#### XSD ```xml @@ -489,8 +879,18 @@ With `max_recursion_depth = 2`: + + + + + + + + + + @@ -498,6 +898,7 @@ With `max_recursion_depth = 2`: + @@ -508,64 +909,166 @@ With `max_recursion_depth = 2`: ``` -#### Sample XML (`xml_tag="record"`, `max_recursion_depth=2`) +#### Input (`data.xml`) + ```xml + - + - + +
+ Ch 1 +
+ Sec 1.1 +
+ Subsec 1.1.1 +
+
+
``` -#### Python / JSON Translation (`max_recursion_depth=2`) -```python +#### Converted rows: run `depth-2` (`max_recursion_depth=2`) + +```json [ { "_id": "r1", "tree": { "_label": "L1", + "_weight": 10, "branch": [ { "_label": "L2", "branch": [ - '{"_label": "L3", "branch": [{"_label": "L4"}]}', - '{"_label": "L3_sibling"}', - ], + "{\"_label\": \"L3\", \"_weight\": 30, \"branch\": [{\"_label\": \"L4\"}]}", + "{\"_label\": \"L3_sibling\"}" + ] } - ], + ] + }, + "Section": { + "SectionTitle": "Ch 1", + "Section": [ + { + "SectionTitle": "Sec 1.1", + "Section": [ + "{\"SectionTitle\": \"Subsec 1.1.1\"}" + ] + } + ] + } + } +] +``` + +#### Converted rows: run `depth-1` (`max_recursion_depth=1`) + +```json +[ + { + "_id": "r1", + "tree": { + "_label": "L1", + "_weight": 10, + "branch": [ + "{\"_label\": \"L2\", \"branch\": [{\"_label\": \"L3\", \"_weight\": 30, \"branch\": [{\"_label\": \"L4\"}]}, {\"_label\": \"L3_sibling\"}]}" + ] }, + "Section": { + "SectionTitle": "Ch 1", + "Section": [ + "{\"SectionTitle\": \"Sec 1.1\", \"Section\": [{\"SectionTitle\": \"Subsec 1.1.1\"}]}" + ] + } } ] ``` -#### PyArrow Schema (`max_recursion_depth=2`) +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/04_recursion_depth/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 04_recursion_depth (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("_id", pa.string(), nullable=False), + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), pa.field( "tree", pa.struct( - [ # depth 1 of TreeType - pa.field("_label", pa.string(), nullable=True), + [ + pa.field("_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_weight", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), pa.field( "branch", pa.list_( - pa.struct( - [ # depth 2 of TreeType - pa.field("_label", pa.string(), nullable=True), - pa.field( - "branch", pa.list_(pa.string()), nullable=True - ), # depth > 2 -> JSON string per item - ] - ) + pa.field( + "item", + pa.struct( + [ + pa.field( + "_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field("_weight", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "Section", + pa.struct( + [ + pa.field("SectionTitle", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "Section", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "SectionTitle", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "Section", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), ), nullable=True, ), @@ -575,36 +1078,81 @@ pa.schema( ), ] ) -``` - ---- - -### Example 5: Model Groups (`xs:group`), Attribute Groups (`xs:attributeGroup`), ``, and Compositors with Duplicate Element Names -Covers: -- `` (including with `maxOccurs="unbounded"`) -- `` -- Unrepeated `` where the **same element name** appears in multiple branches (e.g., `(StartPage, EndPage?, MedlinePgn?) | MedlinePgn` from [`pubmed.xsd` L672–679](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd#L672-L679)) $\rightarrow$ `MedlinePgn` has `overall_max_occurs == 1`, so it is a **single** `pa.string()` field emitted once. -- `` where the **same element name** appears twice (``) $\rightarrow$ `overall_max_occurs == 2`, so it is a **list** field emitted once. -- `` compositor. - -#### XSD -```xml - - - - - - - - - - - - - - - +EXPECTED_SCHEMA_DEPTH_1 = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_weight", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "Section", + pa.struct( + [ + pa.field("SectionTitle", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "Section", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### Model Groups, Attribute Groups, `xs:all` and Duplicate Element Names (`spec/05_groups_and_compositors`) + +**Rules:** [R-SHAPE-GROUPS](#r-shape-groups), [R-SHAPE-CARDINALITY](#r-shape-cardinality), [R-ORDER](#r-order), [R-NULL-CHOICE](#r-null-choice), [R-NULL-CHILD](#r-null-child) + +**Target:** `xml_tag='item'` + +- `pagination` is `(StartPage, EndPage?, MedlinePgn?) | MedlinePgn`: `MedlinePgn` appears in both choice + branches but has `overall_max_occurs == 1`, so it is a single field emitted once. +- `pairs` repeats ``, so `k` and `v` are lists. +- `meta` uses ``; `auditAttrs` is an `xs:attributeGroup`. + +#### Schema (`schema.xsd`) + +```xml + + + + + + + + + + + + + + + + @@ -614,6 +1162,13 @@ Covers: + + + + + + + @@ -628,6 +1183,7 @@ Covers: + @@ -644,8 +1200,10 @@ Covers: ``` -#### Sample XML (`xml_tag="item"`) +#### Input (`data.xml`) + ```xml + @@ -653,6 +1211,10 @@ Covers: 20 10-20 + + P12345 + Q67890 + 2025 Bob @@ -672,8 +1234,9 @@ Covers: ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "_createdBy": "alice", @@ -681,38 +1244,72 @@ Covers: "pagination": { "StartPage": "10", "EndPage": "20", - "MedlinePgn": "10-20", + "MedlinePgn": "10-20" + }, + "interaction": { + "interactant": [ + "P12345", + "Q67890" + ] }, "meta": { "author": "Bob", - "year": 2025, + "year": 2025 }, "pairs": { - "k": ["a", "b"], - "v": [1, 2], - }, + "k": [ + "a", + "b" + ], + "v": [ + 1, + 2 + ] + } }, { "pagination": { - "MedlinePgn": "e12345", - }, - }, + "MedlinePgn": "e12345" + } + } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 05_groups_and_compositors (xml_tag='item').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("_createdBy", pa.string(), nullable=True), - pa.field("_version", pa.int32(), nullable=True), + pa.field("_createdBy", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_version", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), pa.field( "pagination", pa.struct( [ - pa.field("StartPage", pa.string(), nullable=True), - pa.field("EndPage", pa.string(), nullable=True), - pa.field("MedlinePgn", pa.string(), nullable=True), + pa.field("StartPage", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("EndPage", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("MedlinePgn", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "interaction", + pa.struct( + [ + pa.field( + "interactant", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), ] ), nullable=True, @@ -721,8 +1318,8 @@ pa.schema( "meta", pa.struct( [ - pa.field("author", pa.string(), nullable=False), - pa.field("year", pa.int32(), nullable=True), + pa.field("author", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("year", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), ] ), nullable=True, @@ -731,8 +1328,20 @@ pa.schema( "pairs", pa.struct( [ - pa.field("k", pa.list_(pa.string()), nullable=False), - pa.field("v", pa.list_(pa.int32()), nullable=False), + pa.field( + "k", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "v", + pa.list_( + pa.field("item", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + ), + nullable=False, + ), ] ), nullable=True, @@ -743,11 +1352,19 @@ pa.schema( --- -### Example 6: Context-Dependent Same `(parent, child)` Names with Different Types -In [`06_context_dependent_paths/schema.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd) and [`10_combinations/schema.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd#L195-L226), `left/node/leaf` is a single `xs:string` while `right/node/leaf` is an unbounded `LabelType` (complex `simpleContent` with attribute `lang`). -Because the full ancestor path from `xml_tag` (`("root", "left", "node")` vs `("root", "right", "node")`) distinguishes them, each gets its exact PyArrow type instead of merging `("node", "leaf")` globally. + + +### Context-Dependent Same `(parent, child)` Names with Different Types (`spec/06_context_dependent_paths`) + +**Rules:** [R-SHAPE-CONTEXT-PATH](#r-shape-context-path), [R-SHAPE-SIMPLECONTENT](#r-shape-simplecontent), [R-NULL-CHILD](#r-null-child) + +**Target:** `xml_tag='root'` + +`left/node/leaf` is a single `xs:string`; `right/node/leaf` is an unbounded `simpleContent` type with a `lang` +attribute. The ancestor paths differ, so each `leaf` gets its own type. + +#### Schema (`schema.xsd`) -#### XSD ```xml @@ -794,8 +1411,10 @@ Because the full ancestor path from `xml_tag` (`("root", "left", "node")` vs `(" ``` -#### Sample XML (`xml_tag="root"`) +#### Input (`data.xml`) + ```xml + @@ -811,30 +1430,43 @@ Because the full ancestor path from `xml_tag` (`("root", "left", "node")` vs `(" ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "left": { "node": { - "leaf": "l1", + "leaf": "l1" } }, "right": { "node": { "leaf": [ - {"#text": "r1"}, - {"_lang": "en", "#text": "r2"}, + { + "#text": "r1" + }, + { + "_lang": "en", + "#text": "r2" + } ] } - }, + } } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 06_context_dependent_paths (xml_tag='root').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ pa.field( "left", @@ -844,7 +1476,7 @@ pa.schema( "node", pa.struct( [ - pa.field("leaf", pa.string(), nullable=False), + pa.field("leaf", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), ] ), nullable=False, @@ -864,12 +1496,26 @@ pa.schema( pa.field( "leaf", pa.list_( - pa.struct( - [ - pa.field("_lang", pa.string(), nullable=True), - pa.field("#text", pa.string(), nullable=True), - ] - ) + pa.field( + "item", + pa.struct( + [ + pa.field( + "_lang", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), ), nullable=False, ), @@ -887,19 +1533,104 @@ pa.schema( --- -### Example 7: Multiple Root Elements Sharing the Same Child `xml_tag` (`uniref.xsd` Pattern) -In [`uniref.xsd`](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniref/uniref.xsd), four root elements (`UniRef100`, `UniRef90`, `UniRef50`, `UniRef`) each declare a child ``. Because all four `entry` declarations share the same `entryType`, resolving `xml_tag="entry"` succeeds unambiguously. + + +### Multiple Root Elements Sharing the Same Child `xml_tag` (`spec/07_multi_root_shared_type`) + +**Rules:** [R-RESOLVE-DEPTH](#r-resolve-depth) + +**Target:** `xml_tag='entry'` + +Several root elements each declare a child `entry` of the same named type (the `uniref.xsd` pattern), so +resolving `xml_tag="entry"` is unambiguous. + +#### Schema (`schema.xsd`) + +```xml + + + + + + + + + + + + + + + + + + + +``` + +#### Input (`data.xml`) + +```xml + + + + First Cluster + + + Second Cluster + + +``` + +#### Converted rows: run `default` + +```json +[ + { + "_id": "C1", + "name": "First Cluster" + }, + { + "_id": "C2", + "name": "Second Cluster" + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py)) + +```python +"""Expected PyArrow schema for 07_multi_root_shared_type (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] +) +``` --- -### Example 8: Field Nullability & Empty String Tag Coercion -Covers: -1. Required attributes (`use="required"` $\rightarrow$ `nullable=False`) vs optional attributes (`use="optional"` $\rightarrow$ `nullable=True`). -2. Required non-nillable elements (`minOccurs="1"`, `nillable="false"` $\rightarrow$ `nullable=False`), with present empty string tags `` coerced to `""`. -3. Elements inside `` branches (`nullable=True`). -4. Children of `nillable="true"` complex elements (`nullable=True` so `` can omit child `text`). + + +### Field Nullability and Empty String Tag Coercion (`spec/08_nullability_and_empty_tags`) + +**Rules:** [R-NULL-ATTR](#r-null-attr), [R-NULL-CHILD](#r-null-child), [R-NULL-CHOICE](#r-null-choice), [R-NULL-NILLABLE-PARENT](#r-null-nillable-parent), [R-EMPTY-COERCE](#r-empty-coerce), [R-NIL](#r-nil) + +**Target:** `xml_tag='entry'` + +- `_id` (`use="required"`) is non-nullable; `_source` is nullable. +- `summary` is required and non-nillable, so the present `` coerces to `""`. +- `geneId` / `proteinId` sit in an `xs:choice` and are nullable. +- `comment` is `nillable="true"`, so its required child `text` is nullable. + +#### Schema (`schema.xsd`) -#### XSD ```xml @@ -935,8 +1666,10 @@ Covers: ``` -#### Sample XML (`xml_tag="entry"`) +#### Input (`data.xml`) + ```xml + BRCA1 @@ -947,52 +1680,69 @@ Covers: ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "_id": "e1", "name": "BRCA1", - "summary": "", # coerced from None -> "" to satisfy nullable=False + "summary": "", "geneId": 672, - "comment": {"_type": "function"}, + "comment": { + "_type": "function" + } } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 08_nullability_and_empty_tags (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("_id", pa.string(), nullable=False), # use="required" - pa.field("_source", pa.string(), nullable=True), # use="optional" - pa.field("name", pa.string(), nullable=False), # minOccurs=1, non-nillable - pa.field("summary", pa.string(), nullable=False), # minOccurs=1, non-nillable - pa.field("optionalNote", pa.string(), nullable=True), # minOccurs=0 - pa.field("geneId", pa.int32(), nullable=True), # inside xs:choice -> nullable=True - pa.field("proteinId", pa.string(), nullable=True), # inside xs:choice -> nullable=True + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("summary", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("optionalNote", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("geneId", pa.int32(), nullable=True, metadata={"xsd_type": "xs:int"}), + pa.field("proteinId", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), pa.field( "comment", pa.struct( [ - pa.field("_type", pa.string(), nullable=False), # use="required" - pa.field("text", pa.string(), nullable=True), # parent is nillable="true" -> nullable=True + pa.field("_type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), ] ), nullable=True, - ), # nillable="true" -> nullable=True + ), ] ) ``` --- -### Example 9: Purely Empty `complexType` Marker Elements -Covers DTD-derived empty marker elements like [`pubmed.xsd` L1312](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd#L1312) (``) and [`10_combinations/schema.xsd` L11–12](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd#L11-L12) (`EmptyType`): -- Mapped to `pa.string()` (or `pa.list_(pa.string())` when `overall_max_occurs > 1`). -- Coerced to `""` when present in the XML, and `None` when omitted. + + +### Purely Empty `complexType` Marker Elements (`spec/09_empty_marker_elements`) + +**Rules:** [R-SHAPE-EMPTY-MARKER](#r-shape-empty-marker), [R-EMPTY-COERCE](#r-empty-coerce), [R-SHAPE-CARDINALITY](#r-shape-cardinality) + +**Target:** `xml_tag='book'` + +DTD-derived markers such as PubMed's `` map to `pa.string()` (`""` when present) and repeated +markers to `pa.list_(pa.string())`. + +#### Schema (`schema.xsd`) -#### XSD ```xml @@ -1016,8 +1766,10 @@ Covers DTD-derived empty marker elements like [`pubmed.xsd` L1312](file:///Users ``` -#### Sample XML (`xml_tag="book"`) +#### Input (`data.xml`) + ```xml + In-Process Book @@ -1031,92 +1783,1471 @@ Covers DTD-derived empty marker elements like [`pubmed.xsd` L1312](file:///Users ``` -#### Python / JSON Translation -```python +#### Converted rows: run `default` + +```json [ { "title": "In-Process Book", "BookInProcess": "", - "flag": ["", ""], + "flag": [ + "", + "" + ] }, { - "title": "Completed Book", - }, + "title": "Completed Book" + } ] ``` -#### PyArrow Schema +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py)) + ```python -pa.schema( +"""Expected PyArrow schema for 09_empty_marker_elements (xml_tag='book').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( [ - pa.field("title", pa.string(), nullable=False), + pa.field("title", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), pa.field("BookInProcess", pa.string(), nullable=True), - pa.field("flag", pa.list_(pa.string()), nullable=True), + pa.field( + "flag", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), ] ) ``` --- -## 3. Error Specification (Unsupported / Invalid XSD Features) + -The converter raises a descriptive `ValueError` during schema generation when encountering any of the following: +### Combinations (`spec/10_combinations`) -| # | Error Case | Example XSD Pattern | Reason | -| :--- | :--- | :--- | :--- | -| **E1** | **`xml_tag` Not Found** | `` or `xml_tag="nonexistent"` | Target element does not exist in the XSD. | -| **E2** | **Ambiguous `xml_tag` Type** | Root `A` has `` and root `B` has `` | Cannot determine a single `pa.Schema` for `xml_tag="entry"` when declarations at the same resolution depth have different types. | -| **E3** | **Non-Struct Top-Level `xml_tag`** | `` or empty `` with no attributes/children | A `pa.Table` schema requires top-level columns (at least one attribute, child element, or `#text` field). | -| **E4** | **Element Wildcard (``)** | `` | Arbitrary undeclared child elements cannot be represented as fixed fields in a `pa.struct`. | -| **E5** | **Attribute Wildcard (``)** | `` | Arbitrary undeclared attributes cannot be represented as fixed fields in a `pa.struct`. | -| **E6** | **Abstract Element or Abstract Type (`abstract="true"`)** | `` used as an element's type, or `` | Concrete runtime instances require `xsi:type` or `substitutionGroup` polymorphism with varying schemas. | -| **E7** | **Substitution Group with Heterogeneous Types** | `` | Substituted elements introduce different element names/types not declared in the parent's content model. | -| **E8** | **Prefixed Attribute / Child Name Collision** | Element has both `` (mapped to `_id`) and `` | Both map to field name `"_id"` in the same `pa.struct`. | -| **E9** | **Non-JSON-Serializable Scalar Type Under Recursion** | Recursive element subtree contains `xs:date`, `xs:dateTime`, `xs:hexBinary`, or `xs:base64Binary` | Elements truncated beyond `max_recursion_depth` are serialized to JSON (`json.dumps`), which does not natively serialize `datetime.date` or `bytes` without lossy/ambiguous string encoding. | +**Rules:** [R-SHAPE-CARDINALITY](#r-shape-cardinality), [R-SHAPE-EXTENSION](#r-shape-extension), [R-SHAPE-ATTR-ONLY](#r-shape-attr-only), [R-SHAPE-SIMPLECONTENT](#r-shape-simplecontent), [R-SHAPE-MIXED](#r-shape-mixed), [R-SHAPE-EMPTY-MARKER](#r-shape-empty-marker), [R-SHAPE-CONTEXT-PATH](#r-shape-context-path), [R-RECURSION](#r-recursion), [R-ORDER](#r-order), [R-NAME-TEXT](#r-name-text) ---- +**Target:** `xml_tag='record'` + +Stress test combining the constructs above in one type: bounded and unbounded repeats, `maxOccurs="0"` +children, children of repeated `xs:choice` / `xs:sequence` compositors, `simpleContent` with string and +decimal bases, attribute-only types, mixed types with and without attributes, element `ref`, a +self-recursive type and `complexContent` extension. + +#### Schema (`schema.xsd`) + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` -## 4. Generated Specification Fixtures, Real-World Examples & Tests - -### 4.1 Specification Fixtures (`tests/data/xsd/spec/`) - -| Suite | `xml_tag` | XSD | Expected PyArrow Schema | Sample XML | Expected JSON | -| :--- | :--- | :--- | :--- | :--- | :--- | -| **`01_scalars_and_simple_types`** | `record` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/01_scalars_and_simple_types/expected.json) | -| **`02_namespaces_and_nillable`** | `entry` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/02_namespaces_and_nillable/expected.json) | -| **`03_mixed_content_raw_text`** | `article` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/03_mixed_content_raw_text/expected.json) | -| **`04_recursion_depth`** | `record` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/expected.json), [expected_depth_1.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/04_recursion_depth/expected_depth_1.json) | -| **`05_groups_and_compositors`** | `item` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/05_groups_and_compositors/expected.json) | -| **`06_context_dependent_paths`** | `root` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/06_context_dependent_paths/expected.json) | -| **`07_multi_root_shared_type`** | `entry` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/07_multi_root_shared_type/expected.json) | -| **`08_nullability_and_empty_tags`** | `entry` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/08_nullability_and_empty_tags/expected.json) | -| **`09_empty_marker_elements`** | `book` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/09_empty_marker_elements/expected.json) | -| **`10_combinations`** | `record` | [schema.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/schema.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/expected_schema.py) | [data.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/data.xml) | [expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/10_combinations/expected.json) | - -### 4.2 Error Specification Fixtures (`tests/data/xsd/spec/errors/`) - -- [ambiguous_xml_tag.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd) (E2) -- [scalar_top_level.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/scalar_top_level.xsd) (E3) -- [empty_marker_top_level.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/empty_marker_top_level.xsd) (E3) -- [wildcard_any_element.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/wildcard_any_element.xsd) (E4) -- [wildcard_any_attribute.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/wildcard_any_attribute.xsd) (E5) -- [abstract_element.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/abstract_element.xsd) (E6) -- [abstract_type.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/abstract_type.xsd) (E6) -- [substitution_group.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/substitution_group.xsd) (E7) -- [attr_child_name_collision.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/attr_child_name_collision.xsd) (E8) -- [nested_date_under_recursion.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/nested_date_under_recursion.xsd) (E9) -- [nested_binary_under_recursion.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd) (E9) - -### 4.3 Real-World Minimal & Maximal Examples (`tests/data/xsd/examples/`) - -| Dataset | `xml_tag` | XSD | Expected PyArrow Schema | Minimal XML & Expected JSON | Maximal XML & Expected JSON | -| :--- | :--- | :--- | :--- | :--- | :--- | -| **UniRef** | `entry` | [uniref.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniref/uniref.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniref/maximal_expected.json) | -| **UniProtKB** | `entry` | [uniprot.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/uniprot/uniprot_kb/uniprot.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/uniprot/maximal_expected.json) | -| **PubMed (`PubmedArticle`)** | `PubmedArticle` | [pubmed.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_article/maximal_expected.json) | -| **PubMed (`PubmedBookArticle`)** | `PubmedBookArticle` | [pubmed.xsd](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/pubmed/pubmed.xsd) | [expected_schema.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/expected_schema.py) | [minimal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/minimal.xml), [minimal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/minimal_expected.json) | [maximal.xml](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/maximal.xml), [maximal_expected.json](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/data/xsd/examples/pubmed_book_article/maximal_expected.json) | - -### 4.4 Specification Test Suite - -- [test_xsd_spec.py](file:///Users/gwg/code/kbase/cdm-data-loaders-no-batcher/tests/cdm_data_loaders/readers/test_xsd_spec.py): Validates all fixtures against their XSDs, tests `generate_pyarrow_schema`, tests error cases E1–E9, tests `convert_xml_element` + PyArrow Table / Parquet round-trips, and tests `process_xml_file_to_dict` yielding `pa.Table` batches. +#### Input (`data.xml`) +```xml + + + + + + + + a + x + only + + + + + x + + + y + b1 + b2 + + + + + + + padded + Tom & Jerry + + + + + one + two + + + + + + + solo + + + + + + + + + + + + + + + + + + + + + + + + + g1 + a1 + a2 + + + + child-name + + + + + + + + + g2 + + + 1 + + + + + + + + plain text + Hello + world again + + + x + + + A + 1B + 2C + 3 + + Intro + bold + + side text + just text + + + + + only italic + + + bare note + + + + + + a1 + b1 + a2 + b2 + + + + solo + + a + 1 + + + + + + + a + 1 + b + 2 + + + + + + Book title + + Ch1 + + + Ch2 + Ch2 alt + + + + + + + Lonely + + + + + + + l1 + + + + + r1 + r2 + + + + + + + + + + + + + r + + + + + + + + + + + + + + + + + + + + + + + + + + + + s1 + + base + x1 + + + + + s1 + s2 + + x1 + x2 + + + +``` + +#### Converted rows: run `default` + +```json +[ + { + "_id": "r01" + }, + { + "_id": "r02", + "plain": "a", + "plainList": [ + "x" + ], + "bounded": [ + "only" + ] + }, + { + "_id": "r03", + "plainList": [ + "x", + null, + null, + "y" + ], + "bounded": [ + "b1", + "b2" + ] + }, + { + "_id": "r04", + "plain": "a < b & c", + "plainList": [ + "padded", + "Tom & Jerry" + ] + }, + { + "_id": "r05", + "label": { + "_lang": "en", + "#text": "Hello" + }, + "labelList": [ + { + "_lang": "en", + "_source": "curated", + "#text": "one" + }, + { + "#text": "two" + }, + { + "_lang": "fr" + }, + null + ] + }, + { + "_id": "r06", + "label": { + "#text": "Hello" + }, + "labelList": [ + { + "#text": "solo" + } + ] + }, + { + "_id": "r07", + "label": { + "_lang": "de" + }, + "textNoAttr": "t", + "measure": { + "_unit": "kg", + "#text": 3.5 + } + }, + { + "_id": "r08", + "measure": { + "#text": 7.0 + } + }, + { + "_id": "r09", + "flag": { + "_on": true + }, + "flagList": [ + { + "_on": false + }, + null + ], + "marker": [ + "", + "" + ] + }, + { + "_id": "r10", + "flagList": [ + { + "_on": true + } + ], + "marker": [ + "" + ] + }, + { + "_id": "r11", + "group": { + "name": "g1", + "alias": [ + "a1", + "a2" + ] + }, + "entity": [ + { + "_type": "person", + "_name": "attr-name", + "name": "child-name", + "prop": [ + { + "_key": "k1", + "_value": "v1" + }, + { + "_key": "k2" + } + ] + }, + { + "_type": "org" + } + ] + }, + { + "_id": "r12", + "group": { + "name": "g2" + }, + "box": { + "item": [ + "1" + ] + }, + "entity": [ + { + "prop": [ + { + "_key": "only" + } + ] + } + ] + }, + { + "_id": "r13", + "para": [ + { + "#text": "plain text" + }, + { + "#text": "Hello\n world again" + }, + { + "#text": "x" + }, + null, + { + "#text": "A\n 1B\n 2C\n 3" + }, + { + "_style": "em", + "#text": "Intro\n bold" + } + ], + "note": { + "_kind": "aside", + "#text": "side text" + }, + "remark": "just text" + }, + { + "_id": "r14", + "para": [ + { + "#text": "only italic" + }, + { + "_style": "x" + } + ], + "note": { + "#text": "bare note" + } + }, + { + "_id": "r15", + "note": { + "_kind": "empty" + }, + "optA": [ + "a1", + "a2" + ], + "optB": [ + { + "_lang": "en", + "#text": "b1" + }, + { + "#text": "b2" + } + ] + }, + { + "_id": "r16", + "optA": [ + "solo" + ], + "pairs": { + "k": [ + "a" + ], + "v": [ + "1" + ] + } + }, + { + "_id": "r17", + "optB": [ + null + ], + "pairs": { + "k": [ + "a", + "b" + ], + "v": [ + "1", + "2" + ] + } + }, + { + "_id": "r18", + "book": { + "title": "Book title", + "chapter": [ + { + "title": [ + { + "#text": "Ch1" + } + ] + }, + { + "title": [ + { + "_lang": "en", + "#text": "Ch2" + }, + { + "#text": "Ch2 alt" + } + ] + } + ] + } + }, + { + "_id": "r19", + "book": { + "title": "Lonely" + } + }, + { + "_id": "r20", + "left": { + "node": { + "leaf": "l1" + } + }, + "right": { + "node": { + "leaf": [ + { + "#text": "r1" + }, + { + "_lang": "en", + "#text": "r2" + } + ] + } + } + }, + { + "_id": "r21", + "left": { + "node": { + "leaf": "" + } + }, + "right": { + "node": { + "leaf": [ + { + "#text": "r" + } + ] + } + } + }, + { + "_id": "r22", + "tree": { + "_label": "root", + "branch": [ + { + "_label": "a", + "branch": [ + "{\"_label\": \"a1\"}", + "{\"_label\": \"a2\", \"branch\": [{\"_label\": \"a2x\"}]}" + ] + }, + null + ] + } + }, + { + "_id": "r23", + "tree": { + "_label": "t", + "branch": [ + { + "branch": [ + "{\"_label\": \"deep\"}" + ] + } + ] + } + }, + { + "_id": "r24", + "shared": [ + "s1" + ], + "derived": { + "_baseAttr": "b", + "_extraAttr": "e", + "baseField": "base", + "extraField": [ + "x1" + ] + } + }, + { + "_id": "r25", + "shared": [ + "s1", + "s2" + ], + "derived": { + "extraField": [ + "x1", + "x2" + ] + } + } +] +``` + +#### PyArrow schema ([`expected_schema.py`](../tests/data/xsd/spec/10_combinations/expected_schema.py)) + +```python +"""Expected PyArrow schema for 10_combinations (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("_id", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field("plain", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "plainList", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "bounded", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "label", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field( + "labelList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field("textNoAttr", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "measure", + pa.struct( + [ + pa.field("_unit", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.float64(), nullable=True, metadata={"xsd_type": "xs:decimal"}), + ] + ), + nullable=True, + ), + pa.field( + "flag", + pa.struct( + [ + pa.field("_on", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + ] + ), + nullable=True, + ), + pa.field( + "flagList", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_on", pa.bool_(), nullable=True, metadata={"xsd_type": "xs:boolean"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "marker", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + pa.field( + "group", + pa.struct( + [ + pa.field("name", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "alias", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "box", + pa.struct( + [ + pa.field( + "item", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "entity", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_type", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_name", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("name", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "prop", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_key", + pa.string(), + nullable=False, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "para", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_style", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "note", + pa.struct( + [ + pa.field("_kind", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + pa.field("remark", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "optA", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "optB", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("_lang", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_source", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("#text", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + pa.field( + "pairs", + pa.struct( + [ + pa.field( + "k", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + pa.field( + "v", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "book", + pa.struct( + [ + pa.field("title", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + pa.field( + "chapter", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "title", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_lang", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_source", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "left", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field("leaf", pa.string(), nullable=False, metadata={"xsd_type": "xs:string"}), + ] + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "right", + pa.struct( + [ + pa.field( + "node", + pa.struct( + [ + pa.field( + "leaf", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_lang", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "_source", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + pa.field( + "#text", + pa.string(), + nullable=True, + metadata={"xsd_type": "xs:string"}, + ), + ] + ), + nullable=True, + ), + ), + nullable=False, + ), + ] + ), + nullable=False, + ), + ] + ), + nullable=True, + ), + pa.field( + "tree", + pa.struct( + [ + pa.field("_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "branch", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "_label", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"} + ), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=True), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "shared", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + pa.field( + "derived", + pa.struct( + [ + pa.field("_baseAttr", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("_extraAttr", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field("baseField", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + pa.field( + "extraField", + pa.list_( + pa.field("item", pa.string(), nullable=True, metadata={"xsd_type": "xs:string"}), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) +``` + +--- + + + +### PubMed (`PubmedArticle`) (`examples/pubmed_article`) + +**Rules:** [R-SHAPE-MIXED](#r-shape-mixed), [R-RECURSION](#r-recursion), [R-SHAPE-EMPTY-MARKER](#r-shape-empty-marker), [R-NULL-CHOICE](#r-null-choice) + +**Target:** `xml_tag='PubmedArticle'` + +DTD-derived schema with mixed-content titles and abstracts, the `(StartPage, EndPage?, MedlinePgn?) | MedlinePgn` +pagination choice and element-`ref` self-recursion. + +**Schema:** [`tests/data/pubmed/pubmed.xsd`](../tests/data/pubmed/pubmed.xsd) + +- Run `minimal`: [`minimal.xml`](../tests/data/xsd/examples/pubmed_article/minimal.xml), [`minimal_expected.json`](../tests/data/xsd/examples/pubmed_article/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/xsd/examples/pubmed_article/minimal_expected_table.py) +- Run `maximal`: [`maximal.xml`](../tests/data/xsd/examples/pubmed_article/maximal.xml), [`maximal_expected.json`](../tests/data/xsd/examples/pubmed_article/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/xsd/examples/pubmed_article/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/xsd/examples/pubmed_article/expected_schema.py) + +--- + + + +### PubMed (`PubmedBookArticle`) (`examples/pubmed_book_article`) + +**Rules:** [R-SHAPE-MIXED](#r-shape-mixed), [R-RECURSION](#r-recursion), [R-SHAPE-EMPTY-MARKER](#r-shape-empty-marker) + +**Target:** `xml_tag='PubmedBookArticle'` + +Book article variant of the PubMed schema, including recursive `Section` elements and the empty +`BookInProcess` marker. + +**Schema:** [`tests/data/pubmed/pubmed.xsd`](../tests/data/pubmed/pubmed.xsd) + +- Run `minimal`: [`minimal.xml`](../tests/data/xsd/examples/pubmed_book_article/minimal.xml), [`minimal_expected.json`](../tests/data/xsd/examples/pubmed_book_article/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/xsd/examples/pubmed_book_article/minimal_expected_table.py) +- Run `maximal`: [`maximal.xml`](../tests/data/xsd/examples/pubmed_book_article/maximal.xml), [`maximal_expected.json`](../tests/data/xsd/examples/pubmed_book_article/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/xsd/examples/pubmed_book_article/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/xsd/examples/pubmed_book_article/expected_schema.py) + +--- + + + +### UniProtKB (`examples/uniprot`) + +**Rules:** [R-SHAPE-XSLIST](#r-shape-xslist), [R-SCALAR-UNION](#r-scalar-union), [R-NIL](#r-nil), [R-NULL-NILLABLE-PARENT](#r-null-nillable-parent), [R-META-COMMENT](#r-meta-comment) + +**Target:** `xml_tag='entry'` + +Exercises `intListType` (`xs:list`), the `date | gYearMonth | gYear` union and nillable `comment` elements +with attributes. + +**Schema:** [`tests/data/uniprot/uniprot_kb/uniprot.xsd`](../tests/data/uniprot/uniprot_kb/uniprot.xsd) + +- Run `minimal`: [`minimal.xml`](../tests/data/xsd/examples/uniprot/minimal.xml), [`minimal_expected.json`](../tests/data/xsd/examples/uniprot/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/xsd/examples/uniprot/minimal_expected_table.py) +- Run `maximal`: [`maximal.xml`](../tests/data/xsd/examples/uniprot/maximal.xml), [`maximal_expected.json`](../tests/data/xsd/examples/uniprot/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/xsd/examples/uniprot/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/xsd/examples/uniprot/expected_schema.py) + +--- + + + +### UniRef (`examples/uniref`) + +**Rules:** [R-RESOLVE-DEPTH](#r-resolve-depth), [R-META-COMMENT](#r-meta-comment) + +**Target:** `xml_tag='entry'` + +Four root elements (`UniRef100`, `UniRef90`, `UniRef50`, `UniRef`) declare `entry` with the shared +`entryType`, so resolution succeeds. + +**Schema:** [`tests/data/uniprot/uniref/uniref.xsd`](../tests/data/uniprot/uniref/uniref.xsd) + +- Run `minimal`: [`minimal.xml`](../tests/data/xsd/examples/uniref/minimal.xml), [`minimal_expected.json`](../tests/data/xsd/examples/uniref/minimal_expected.json), [`minimal_expected_table.py`](../tests/data/xsd/examples/uniref/minimal_expected_table.py) +- Run `maximal`: [`maximal.xml`](../tests/data/xsd/examples/uniref/maximal.xml), [`maximal_expected.json`](../tests/data/xsd/examples/uniref/maximal_expected.json), [`maximal_expected_table.py`](../tests/data/xsd/examples/uniref/maximal_expected_table.py) + +**Expected PyArrow schema:** [`expected_schema.py`](../tests/data/xsd/examples/uniref/expected_schema.py) + +--- + +## 3. Errors (Unsupported / Invalid Input) + +Schema generation raises a subclass of `XsdToPyArrowError` (itself a `ValueError`). + +| Code | Exception | Condition | Reason | Fixtures | +| :--- | :--- | :--- | :--- | :--- | +| **E1** | `XmlTagNotFoundError` | `xml_tag` not found | The target element is not declared anywhere in the XSD. | [`empty_schema.xsd`](../tests/data/xsd/empty_schema.xsd): Schema declares no elements.
[`uniref.xsd`](../tests/data/uniprot/uniref/uniref.xsd): `xml_tag` is not declared in an otherwise valid schema. | +| **E2** | `AmbiguousXmlTagError` | Ambiguous `xml_tag` type | Declarations of `xml_tag` at the shallowest matching depth have different types, so no single `pa.Schema` can be chosen. | [`ambiguous_xml_tag.xsd`](../tests/data/xsd/spec/errors/ambiguous_xml_tag.xsd): Root `A` declares `` and root `B` declares ``. | +| **E3** | `NonStructXmlTagError` | Non-struct top-level `xml_tag` | A `pa.Table` schema needs top-level columns: the type of `xml_tag` must yield at least one attribute, `#text`, or child element field. | [`scalar_top_level.xsd`](../tests/data/xsd/spec/errors/scalar_top_level.xsd): `xml_tag` has a simple type (`xs:string`).
[`empty_marker_top_level.xsd`](../tests/data/xsd/spec/errors/empty_marker_top_level.xsd): `xml_tag` has an empty `complexType` with no attributes or children.
[`no_children.xsd`](../tests/data/xsd/no_children.xsd): `xml_tag` is a scalar root element. | +| **E4** | `WildcardElementError` | Element wildcard (`xs:any`) | Arbitrary undeclared child elements cannot be represented as fixed `pa.struct` fields. | [`wildcard_any_element.xsd`](../tests/data/xsd/spec/errors/wildcard_any_element.xsd): `` | +| **E5** | `WildcardAttributeError` | Attribute wildcard (`xs:anyAttribute`) | Arbitrary undeclared attributes cannot be represented as fixed `pa.struct` fields. | [`wildcard_any_attribute.xsd`](../tests/data/xsd/spec/errors/wildcard_any_attribute.xsd): ``
[`cdata_variety.xsd`](../tests/data/xsd/cdata_variety.xsd): Legacy fixture containing `xs:anyAttribute`.
[`combinations.xsd`](../tests/data/xsd/combinations.xsd): Legacy fixture containing `xs:anyAttribute`. | +| **E6** | `AbstractComponentError` | Abstract element or type (`abstract="true"`) | Concrete instances of an abstract element or type require `xsi:type` or substitution polymorphism, which has no single schema. | [`abstract_element.xsd`](../tests/data/xsd/spec/errors/abstract_element.xsd): Child element declared with `abstract="true"`.
[`abstract_type.xsd`](../tests/data/xsd/spec/errors/abstract_type.xsd): Child element typed by a `complexType` with `abstract="true"`. | +| **E7** | `SubstitutionGroupError` | Substitution group | Substituted elements introduce element names and types that are not declared in the parent content model. | [`substitution_group.xsd`](../tests/data/xsd/spec/errors/substitution_group.xsd): `` | +| **E8** | `FieldNameCollisionError` | Attribute / child field name collision | An attribute (prefixed `_`) and a child element map to the same field name in one `pa.struct`. | [`attr_child_name_collision.xsd`](../tests/data/xsd/spec/errors/attr_child_name_collision.xsd): Attribute `id` (field `_id`) and child element `_id` in the same type. | +| **E9** | `RecursiveNonJsonTypeError` | Non-JSON-serializable scalar under recursion | Subtrees truncated beyond `max_recursion_depth` are serialized with `json.dumps`, which cannot encode `datetime.date` or `bytes` without lossy conversion. | [`nested_date_under_recursion.xsd`](../tests/data/xsd/spec/errors/nested_date_under_recursion.xsd): Recursive `NodeType` has an `xs:date` child element.
[`nested_binary_under_recursion.xsd`](../tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd): Recursive `NodeType` has an `xs:hexBinary` attribute. | + +## 4. Fixture Index + +Every case directory contains a `case.yaml` manifest; the test suite and this document are both driven by it. + +| Case | `xml_tag` | Schema | Runs | +| :--- | :--- | :--- | :--- | +| [`spec/01_scalars_and_simple_types`](../tests/data/xsd/spec/01_scalars_and_simple_types/case.yaml) | `record` | [`schema.xsd`](../tests/data/xsd/spec/01_scalars_and_simple_types/schema.xsd) | default | +| [`spec/02_namespaces_and_nillable`](../tests/data/xsd/spec/02_namespaces_and_nillable/case.yaml) | `entry` | [`schema.xsd`](../tests/data/xsd/spec/02_namespaces_and_nillable/schema.xsd) | default, clark-notation | +| [`spec/03_mixed_content_raw_text`](../tests/data/xsd/spec/03_mixed_content_raw_text/case.yaml) | `article` | [`schema.xsd`](../tests/data/xsd/spec/03_mixed_content_raw_text/schema.xsd) | default | +| [`spec/04_recursion_depth`](../tests/data/xsd/spec/04_recursion_depth/case.yaml) | `record` | [`schema.xsd`](../tests/data/xsd/spec/04_recursion_depth/schema.xsd) | depth-2, depth-1 | +| [`spec/05_groups_and_compositors`](../tests/data/xsd/spec/05_groups_and_compositors/case.yaml) | `item` | [`schema.xsd`](../tests/data/xsd/spec/05_groups_and_compositors/schema.xsd) | default | +| [`spec/06_context_dependent_paths`](../tests/data/xsd/spec/06_context_dependent_paths/case.yaml) | `root` | [`schema.xsd`](../tests/data/xsd/spec/06_context_dependent_paths/schema.xsd) | default | +| [`spec/07_multi_root_shared_type`](../tests/data/xsd/spec/07_multi_root_shared_type/case.yaml) | `entry` | [`schema.xsd`](../tests/data/xsd/spec/07_multi_root_shared_type/schema.xsd) | default | +| [`spec/08_nullability_and_empty_tags`](../tests/data/xsd/spec/08_nullability_and_empty_tags/case.yaml) | `entry` | [`schema.xsd`](../tests/data/xsd/spec/08_nullability_and_empty_tags/schema.xsd) | default | +| [`spec/09_empty_marker_elements`](../tests/data/xsd/spec/09_empty_marker_elements/case.yaml) | `book` | [`schema.xsd`](../tests/data/xsd/spec/09_empty_marker_elements/schema.xsd) | default | +| [`spec/10_combinations`](../tests/data/xsd/spec/10_combinations/case.yaml) | `record` | [`schema.xsd`](../tests/data/xsd/spec/10_combinations/schema.xsd) | default | +| [`examples/pubmed_article`](../tests/data/xsd/examples/pubmed_article/case.yaml) | `PubmedArticle` | [`pubmed.xsd`](../tests/data/pubmed/pubmed.xsd) | minimal, maximal | +| [`examples/pubmed_book_article`](../tests/data/xsd/examples/pubmed_book_article/case.yaml) | `PubmedBookArticle` | [`pubmed.xsd`](../tests/data/pubmed/pubmed.xsd) | minimal, maximal | +| [`examples/uniprot`](../tests/data/xsd/examples/uniprot/case.yaml) | `entry` | [`uniprot.xsd`](../tests/data/uniprot/uniprot_kb/uniprot.xsd) | minimal, maximal | +| [`examples/uniref`](../tests/data/xsd/examples/uniref/case.yaml) | `entry` | [`uniref.xsd`](../tests/data/uniprot/uniref/uniref.xsd) | minimal, maximal | diff --git a/pyproject.toml b/pyproject.toml index ab5a4093..58432d6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ hello_world = "cdm_data_loaders.utils.hello_world:cli" dev = [ "linkml>=1.9.5", "moto[s3]>=5.2.1", - "platformdirs>=4.11.15", "pytest>=9.0.2", "pytest-asyncio>=1.3.0", "pytest-cov>=7.1.0", @@ -70,7 +69,9 @@ dev = [ "ruff>=0.15.14", "types-boto3[s3]>=1.43.62", ] -lakehouse = [] +lakehouse = [ + "cdm-task-service-client", +] models = [ "genson>=1.3.0", "json2python-models>=0.3.1", @@ -274,5 +275,5 @@ DLT_PROJECT_DIR = "/tmp/dlt" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } dlt = { git = "https://github.com/dlt-hub/dlt", branch = "devel" } +cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" } # berdl-notebook-utils = { git = "https://github.com/KBaseDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" } -# cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" } diff --git a/scripts/gen_converter_spec.py b/scripts/gen_converter_spec.py new file mode 100644 index 00000000..fbf3355e --- /dev/null +++ b/scripts/gen_converter_spec.py @@ -0,0 +1,352 @@ +"""Generate converter specification documents from rule catalogues, case manifests and converter code. + +Usage:: + + uv run python -m scripts.gen_converter_spec # write every docs/*_specification.md in CONFIGS + uv run python -m scripts.gen_converter_spec --check # exit 1 if a committed document is stale +""" + +import argparse +import json +import os +import sys +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pyarrow as pa + +from cdm_data_loaders.converters.jsonschema_to_pyarrow import ( + JSONSCHEMA_TO_PYARROW_ERRORS, + JSONSchemaToPyArrowError, + resolve_scalar_type, +) +from cdm_data_loaders.readers.xsd import ( + BUILTIN_LIST_ITEM_TYPES, + SCALAR_ARROW_TYPES, + XSD_TO_PYARROW_ERRORS, + XsdToPyArrowError, +) +from tests.converter_spec import Case, ErrorCase, RuleCatalogue, load_cases, load_error_cases, load_rules + +REPO_ROOT = Path(__file__).resolve().parent.parent + +GENERATED_BANNER = "" + +_PA_CONSTRUCTORS: dict[str, str] = { + "string": "pa.string()", + "bool": "pa.bool_()", + "int8": "pa.int8()", + "int16": "pa.int16()", + "int32": "pa.int32()", + "int64": "pa.int64()", + "uint8": "pa.uint8()", + "uint16": "pa.uint16()", + "uint32": "pa.uint32()", + "uint64": "pa.uint64()", + "double": "pa.float64()", + "date32[day]": "pa.date32()", + "binary": "pa.binary()", +} + +_PY_TYPES: dict[str, str] = { + "string": "`str`", + "bool": "`bool`", + "double": "`float`", + "date32[day]": "`datetime.date`", + "binary": "`bytes`", +} + + +@dataclass(frozen=True, slots=True) +class ConverterSpecConfig: + """Everything needed to render one converter's specification.""" + + data_dir: Path + output: Path + suites: Sequence[str] + error_manifest: Path + error_base: type[ValueError] + error_classes: Sequence[type[ValueError]] + schema_lang: str + data_lang: str + generated_tables: dict[str, Callable[[], str]] + target_label: str | None = None + + +def _rel(path: Path, start: Path) -> str: + return Path(os.path.relpath(path, start)).as_posix() + + +def _code_block(path: Path, lang: str) -> str: + return f"```{lang}\n{path.read_text(encoding='utf-8').rstrip()}\n```" + + +def _pa_ctor(dtype: pa.DataType) -> str: + return _PA_CONSTRUCTORS.get(str(dtype), f"`{dtype}`") + + +def _py_type(dtype: pa.DataType) -> str: + if pa.types.is_integer(dtype): + return "`int`" + return _PY_TYPES.get(str(dtype), f"`{dtype}`") + + +def render_xsd_scalar_table() -> str: + """Render the built-in XSD scalar mapping table from ``SCALAR_ARROW_TYPES`` and ``BUILTIN_LIST_ITEM_TYPES``.""" + grouped: dict[str, list[str]] = {} + dtypes: dict[str, pa.DataType] = {} + for xsd_name, dtype in SCALAR_ARROW_TYPES.items(): + key = str(dtype) + grouped.setdefault(key, []).append(xsd_name) + dtypes[key] = dtype + lines = [ + "| XSD built-in type(s) | Python type (coerced) | PyArrow `DataType` |", + "| :--- | :--- | :--- |", + ] + for key, names in grouped.items(): + dtype = dtypes[key] + lines.append(f"| {', '.join(f'`{n}`' for n in names)} | {_py_type(dtype)} | `{_pa_ctor(dtype)}` |") + list_names = ", ".join(f"`{n}` (items `{item}`)" for n, item in BUILTIN_LIST_ITEM_TYPES.items()) + lines.append(f"| {list_names} | `list[str]` | `pa.list_(pa.string())` |") + return "\n".join(lines) + + +JSONSCHEMA_SCALAR_PROBES: tuple[Mapping[str, Any], ...] = ( + {"type": "boolean"}, + {"type": "integer"}, + {"type": "number"}, + {"type": "string"}, + {"type": "string", "format": "date"}, + {"type": "string", "format": "date-time"}, + {"type": "string", "format": "time"}, + {"type": "string", "format": "duration"}, + {"type": "string", "format": "uuid"}, + {"type": "string", "format": "uri"}, + {"type": "string", "format": "email"}, + {"type": "string", "contentEncoding": "base64"}, + {"type": "string", "contentEncoding": "base16"}, + {"enum": [1, 2]}, + {"enum": [1, 2.5]}, + {"enum": [1, "two", True]}, + {"type": ["integer", "number"]}, + {"type": ["integer", "string"]}, + {"type": ["string", "null"]}, +) + + +def render_jsonschema_scalar_table() -> str: + """Render the JSON Schema scalar mapping table by resolving probe schemas through the converter.""" + lines = [ + '| JSON Schema | Python type (coerced) | PyArrow `DataType` | `metadata["jsonschema_type"]` |', + "| :--- | :--- | :--- | :--- |", + ] + for probe in JSONSCHEMA_SCALAR_PROBES: + dtype, label = resolve_scalar_type(probe) + cell_label = label.replace("|", "\\|") + lines.append(f"| `{json.dumps(probe)}` | {_py_type(dtype)} | `{_pa_ctor(dtype)}` | `{cell_label}` |") + return "\n".join(lines) + + +def _render_rules(catalogue: RuleCatalogue, cases: Sequence[Case], config: ConverterSpecConfig) -> list[str]: + covering: dict[str, list[Case]] = {rule_id: [] for rule_id in catalogue.rules} + for case in cases: + for rule_id in case.rules: + covering.setdefault(rule_id, []).append(case) + + out = ["## 1. Conversion Rules", ""] + for number, section in enumerate(catalogue.sections, start=1): + out += [f"### 1.{number} {section.title}", ""] + if section.intro: + out += [section.intro, ""] + for rule in section.rules: + links = ", ".join(f"[{c.name}](#{c.anchor})" for c in covering[rule.id]) + statement = rule.statement.replace("\n", "\n ") + out += [ + f'', + "", + f"- **{rule.id}**: {statement}", + "", + f" *Cases:* {links}", + "", + ] + if section.generated: + out += [config.generated_tables[section.generated](), ""] + return out + + +def _render_case(case: Case, catalogue: RuleCatalogue, config: ConverterSpecConfig) -> list[str]: + doc_dir = config.output.parent + out = [f'', "", f"### {case.title} (`{case.suite}/{case.name}`)", ""] + rules = ", ".join(f"[{r}](#{r.lower()})" for r in case.rules if r in catalogue.rules) + out += [f"**Rules:** {rules}", ""] + if config.target_label and case.xml_tag: + out += [f"**Target:** `{config.target_label}={case.xml_tag!r}`", ""] + if case.notes: + out += [case.notes, ""] + + embed = case.suite == "spec" + if embed: + out += [f"#### Schema (`{case.schema.name}`)", "", _code_block(case.schema, config.schema_lang), ""] + else: + out += [f"**Schema:** [`{_rel(case.schema, REPO_ROOT)}`]({_rel(case.schema, doc_dir)})", ""] + + seen_data: set[Path] = set() + for run in case.runs: + params = ", ".join(f"`{k}={v}`" for k, v in run.params.items()) + suffix = f" ({params})" if params else "" + if config.target_label and run.xml_tag != case.xml_tag: + suffix += f" (`{config.target_label}={run.xml_tag!r}`)" + if embed: + if run.data not in seen_data: + out += [f"#### Input (`{run.data.name}`)", "", _code_block(run.data, config.data_lang), ""] + seen_data.add(run.data) + out += [f"#### Converted rows: run `{run.id}`{suffix}", "", _code_block(run.expected_json, "json"), ""] + else: + files = [run.data, run.expected_json, run.expected_table] + listed = ", ".join(f"[`{f.name}`]({_rel(f, doc_dir)})" for f in files) + out += [f"- Run `{run.id}`{suffix}: {listed}"] + if not embed: + out.append("") + + schema_link = f"[`{case.expected_schema.name}`]({_rel(case.expected_schema, doc_dir)})" + if embed: + out += [f"#### PyArrow schema ({schema_link})", "", _code_block(case.expected_schema, "python"), ""] + else: + out += [f"**Expected PyArrow schema:** {schema_link}", ""] + out += ["---", ""] + return out + + +def _render_errors(errors: Sequence[ErrorCase], config: ConverterSpecConfig) -> list[str]: + doc_dir = config.output.parent + by_code: dict[str, list[ErrorCase]] = {} + for err in errors: + by_code.setdefault(err.code, []).append(err) + + out = [ + "## 3. Errors (Unsupported / Invalid Input)", + "", + f"Schema generation raises a subclass of `{config.error_base.__name__}` (itself a `ValueError`).", + "", + "| Code | Exception | Condition | Reason | Fixtures |", + "| :--- | :--- | :--- | :--- | :--- |", + ] + for cls in config.error_classes: + code: str = getattr(cls, "code", "") + title: str = getattr(cls, "title", "") + reason = " ".join((cls.__doc__ or "").split()) + fixtures = "
".join( + f"[`{e.file.name}`]({_rel(e.file, doc_dir)}): {e.description}" for e in by_code.get(code, []) + ) + out.append(f"| **{code}** | `{cls.__name__}` | {title} | {reason} | {fixtures} |") + out.append("") + return out + + +def _render_index(cases: Sequence[Case], config: ConverterSpecConfig) -> list[str]: + doc_dir = config.output.parent + out = [ + "## 4. Fixture Index", + "", + "Every case directory contains a `case.yaml` manifest; the test suite and this document are both driven by it.", + "", + ] + target = config.target_label + out += ( + [f"| Case | `{target}` | Schema | Runs |", "| :--- | :--- | :--- | :--- |"] + if target + else ["| Case | Schema | Runs |", "| :--- | :--- | :--- |"] + ) + for case in cases: + manifest = case.directory / "case.yaml" + runs = ", ".join(r.id for r in case.runs) + target_cell = f" `{case.xml_tag}` |" if target else "" + out.append( + f"| [`{case.suite}/{case.name}`]({_rel(manifest, doc_dir)}) |{target_cell} " + f"[`{case.schema.name}`]({_rel(case.schema, doc_dir)}) | {runs} |" + ) + out.append("") + return out + + +def render(config: ConverterSpecConfig) -> str: + """Render the full specification document as Markdown.""" + catalogue = load_rules(config.data_dir / "rules.yaml") + cases = [case for suite in config.suites for case in load_cases(config.data_dir / suite)] + errors = load_error_cases(config.error_manifest) + + sources = ", ".join( + f"`{_rel(p, REPO_ROOT)}`" + for p in (config.data_dir / "rules.yaml", *(config.data_dir / s for s in config.suites)) + ) + out = [GENERATED_BANNER.format(sources=sources), "", f"# {catalogue.title}", "", catalogue.intro, ""] + contents = ( + "**Contents:** [1. Conversion Rules](#1-conversion-rules) · [2. Cases](#2-cases) · " + "[3. Errors](#3-errors-unsupported--invalid-input) · [4. Fixture Index](#4-fixture-index)" + ) + out += [contents, ""] + out += _render_rules(catalogue, cases, config) + out += ["## 2. Cases", ""] + for case in cases: + out += _render_case(case, catalogue, config) + out += _render_errors(errors, config) + out += _render_index(cases, config) + text = "\n".join(out) + while "\n\n\n" in text: + text = text.replace("\n\n\n", "\n\n") + return text.rstrip() + "\n" + + +XSD_TO_PYARROW = ConverterSpecConfig( + data_dir=REPO_ROOT / "tests" / "data" / "xsd", + output=REPO_ROOT / "docs" / "xsd_to_pyarrow_specification.md", + suites=("spec", "examples"), + error_manifest=REPO_ROOT / "tests" / "data" / "xsd" / "spec" / "errors" / "manifest.yaml", + error_base=XsdToPyArrowError, + error_classes=XSD_TO_PYARROW_ERRORS, + schema_lang="xml", + data_lang="xml", + generated_tables={"scalar_table": render_xsd_scalar_table}, + target_label="xml_tag", +) + +JSONSCHEMA_TO_PYARROW = ConverterSpecConfig( + data_dir=REPO_ROOT / "tests" / "data" / "jsonschema", + output=REPO_ROOT / "docs" / "jsonschema_to_pyarrow_specification.md", + suites=("spec", "examples"), + error_manifest=REPO_ROOT / "tests" / "data" / "jsonschema" / "spec" / "errors" / "manifest.yaml", + error_base=JSONSchemaToPyArrowError, + error_classes=JSONSCHEMA_TO_PYARROW_ERRORS, + schema_lang="json", + data_lang="json", + generated_tables={"scalar_table": render_jsonschema_scalar_table}, +) + +CONFIGS: tuple[ConverterSpecConfig, ...] = (XSD_TO_PYARROW, JSONSCHEMA_TO_PYARROW) + + +def main(argv: Sequence[str] | None = None) -> int: + """Write or check every configured specification document.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if a committed document is out of date") + args = parser.parse_args(argv) + + stale: list[Path] = [] + for config in CONFIGS: + text = render(config) + current = config.output.read_text(encoding="utf-8") if config.output.exists() else None + if current == text: + continue + if args.check: + stale.append(config.output) + else: + config.output.write_text(text, encoding="utf-8") + for path in stale: + sys.stderr.write(f"stale: {_rel(path, REPO_ROOT)}\n") + return 1 if stale else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cdm_data_loaders/converters/jsonschema_to_pyarrow.py b/src/cdm_data_loaders/converters/jsonschema_to_pyarrow.py index bf3184f5..d33d41d5 100644 --- a/src/cdm_data_loaders/converters/jsonschema_to_pyarrow.py +++ b/src/cdm_data_loaders/converters/jsonschema_to_pyarrow.py @@ -1,6 +1,13 @@ """Convert JSON Schema through the shared reader and PyArrow emitter.""" -from typing import Any +import base64 +import copy +import json +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, date, datetime, time +from decimal import Decimal +from typing import Any, ClassVar, Final import pyarrow as pa from frozendict import frozendict @@ -13,15 +20,157 @@ from cdm_data_loaders.converters.emitters.pyarrow import DEFAULT_FORMAT_MAP, PyArrowEmitter, merge_format_map from cdm_data_loaders.converters.readers.json_schema import JsonSchemaReader +DEFAULT_MAX_RECURSION_DEPTH: Final[int] = 2 + +SUPPORTED_SCHEMA_DIALECTS: Final[frozenset[str]] = frozenset( + { + "https://json-schema.org/draft/2020-12/schema", + "https://json-schema.kbase.us/draft/2020-12-xsv/schema", + } +) + +_CONSTRAINING_KEYWORDS: Final[frozenset[str]] = frozenset( + { + "type", + "properties", + "items", + "prefixItems", + "patternProperties", + "enum", + "const", + "$ref", + "allOf", + "anyOf", + "oneOf", + "dependentSchemas", + "then", + "else", + } +) + +_NON_EXTENDABLE_SCALAR_MAP: Final[dict[str, tuple[pa.DataType, str]]] = { + "boolean": (pa.bool_(), "boolean"), + "integer": (pa.int64(), "integer"), + "number": (pa.float64(), "number"), +} + class JSONSchemaToPyArrowError(ConversionError): - """Raised when a JSON Schema construct cannot be converted.""" + """Raised when a JSON Schema construct cannot be converted. + + Subclasses raised by :func:`generate_pyarrow_schema` carry a stable specification ``code`` and a short + ``title``; their docstrings state why the construct is rejected. The converter specification document is + generated from these attributes. + """ + + code: ClassVar[str] = "" + title: ClassVar[str] = "" class InvalidJSONSchemaError(JSONSchemaToPyArrowError): """Raised when the input document is not a valid JSON Schema.""" +class UnsupportedDialectError(InvalidJSONSchemaError): + """Only JSON Schema Draft 2020-12 and the KBase Draft 2020-12 XSV metaschema are supported.""" + + code = "E1" + title = "Missing or unsupported `$schema` dialect" + + +class NonStructRootError(JSONSchemaToPyArrowError): + """A `pa.Schema` needs top-level columns, so the root must resolve to a fixed-property object.""" + + code = "E2" + title = "Non-struct top-level schema" + + +class EmptyObjectError(JSONSchemaToPyArrowError): + """Parquet cannot store a `pa.struct([])` with no child fields.""" + + code = "E3" + title = "Empty object or tuple (zero fields)" + + +class UnconstrainedSchemaError(JSONSchemaToPyArrowError): + """An untyped JSON value cannot be assigned a concrete Arrow `DataType`.""" + + code = "E4" + title = "Unconstrained or boolean schema" + + +class OpenTupleError(JSONSchemaToPyArrowError): + """A field cannot be both a positional `pa.struct` and an unbounded `pa.list_`.""" + + code = "E5" + title = "Open tuple (`prefixItems` with schema-valued `items`)" + + +class MixedPropertiesAndMapError(JSONSchemaToPyArrowError): + """An object cannot be both a fixed-field `pa.struct` and a dynamic `pa.map_`.""" + + code = "E6" + title = "Fixed `properties` mixed with a dynamic map" + + +class NegationError(JSONSchemaToPyArrowError): + """A complement type does not define a positive storage `DataType`.""" + + code = "E7" + title = "Negation (`not`)" + + +class IncompatibleBranchShapesError(JSONSchemaToPyArrowError): + """Branches with different structural shapes (object vs array, array vs scalar, object vs map) cannot share one column.""" + + code = "E8" + title = "Incompatible structural union branches" + + +class UnsupportedRefError(JSONSchemaToPyArrowError): + """Every `$ref` must resolve to a local `#/$defs/...` definition; `$dynamicRef` and `$recursiveRef` are not supported.""" + + code = "E9" + title = "Unresolvable or unsupported reference" + + +class RecursiveNonJsonTypeError(JSONSchemaToPyArrowError): + """Truncated recursive subtrees are serialized with `json.dumps`, which cannot encode `datetime.date` or `bytes`.""" + + code = "E10" + title = "Non-JSON-serializable type in a recursive `$ref` cycle" + + +class NullOnlySchemaError(JSONSchemaToPyArrowError): + """Parquet has no physical storage type for a `pa.null()` column.""" + + code = "E11" + title = "Null-only schema" + + +class ValueFieldCollisionError(JSONSchemaToPyArrowError): + """The synthesized `_value` field of a scalar + object union would collide with a declared `_value` property.""" + + code = "E12" + title = "`_value` collision in a scalar + object union" + + +JSONSCHEMA_TO_PYARROW_ERRORS: Final[tuple[type[JSONSchemaToPyArrowError], ...]] = ( + UnsupportedDialectError, + NonStructRootError, + EmptyObjectError, + UnconstrainedSchemaError, + OpenTupleError, + MixedPropertiesAndMapError, + NegationError, + IncompatibleBranchShapesError, + UnsupportedRefError, + RecursiveNonJsonTypeError, + NullOnlySchemaError, + ValueFieldCollisionError, +) + + class JSONSchemaToPyArrow(BaseModel): """Read dereferenced JSON Schema and emit PyArrow schemas and empty tables.""" @@ -113,3 +262,736 @@ def convert_from_file(self, path: str) -> pa.Schema: :rtype: pa.Schema """ return self.convert(load_schema_file(path)) + + +@dataclass(frozen=True, slots=True) +class _SchemaBuildContext: + """Immutable traversal context for JSON Schema to PyArrow schema generation.""" + + defs: Mapping[str, Any] + ref_counts: Mapping[str, int] + max_recursion_depth: int + + def with_ref(self, ref_name: str) -> "_SchemaBuildContext": + """Return a new context with the recursion counter for ``ref_name`` incremented by 1.""" + return _SchemaBuildContext( + defs=self.defs, + ref_counts={**self.ref_counts, ref_name: self.ref_counts.get(ref_name, 0) + 1}, + max_recursion_depth=self.max_recursion_depth, + ) + + +def _resolve_ref(ref: object, defs: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: + """Resolve a local ``#/$defs/`` reference pointer against ``defs``.""" + prefix = "#/$defs/" + if not isinstance(ref, str) or not ref.startswith(prefix): + msg = f"Unresolvable or unsupported non-local $ref: {ref!r}" + raise UnsupportedRefError(msg) + name = ref[len(prefix) :] + if name not in defs or not isinstance(defs[name], dict): + msg = f"Unresolvable $ref {ref!r}: definition {name!r} does not exist in $defs" + raise UnsupportedRefError(msg) + return name, defs[name] + + +def _validate_node_constraints(node: object) -> dict[str, Any]: + """Validate that ``node`` is a constrained JSON Schema object without unsupported keywords.""" + if isinstance(node, bool) or not isinstance(node, dict): + msg = f"Unconstrained or boolean schema node is not supported: {node!r}" + raise UnconstrainedSchemaError(msg) + if "not" in node: + msg = "Unsupported JSON Schema keyword 'not': negation schemas cannot be converted to PyArrow" + raise NegationError(msg) + if "$dynamicRef" in node or "$recursiveRef" in node: + msg = "Unsupported dynamic reference keyword '$dynamicRef' in JSON Schema" + raise UnsupportedRefError(msg) + + has_constraint = any(k in node for k in _CONSTRAINING_KEYWORDS) or isinstance( + node.get("additionalProperties"), dict + ) + if not has_constraint: + msg = f"Unconstrained schema without type or structural keywords is not supported: {node!r}" + raise UnconstrainedSchemaError(msg) + + has_fixed_props = bool(node.get("properties")) or any( + k in node for k in ("allOf", "anyOf", "oneOf", "dependentSchemas", "then", "else") + ) + has_map_schema = isinstance(node.get("additionalProperties"), dict) or bool(node.get("patternProperties")) + if has_fixed_props and has_map_schema: + msg = "Mixed fixed properties and dynamic map (additionalProperties/patternProperties) is not supported" + raise MixedPropertiesAndMapError(msg) + return node + + +def _check_no_forbidden_types_in_recursive_cycle( + ref_name: str, + target: Mapping[str, Any], + defs: Mapping[str, Any], +) -> None: + """Raise ``JSONSchemaToPyArrowError`` if a recursive ``$defs`` cycle contains non-JSON-serializable types.""" + visited_refs: set[str] = {ref_name} + stack: list[object] = [target] + while stack: + current = stack.pop() + if isinstance(current, Mapping): + if current.get("format") == "date" or current.get("contentEncoding") in {"base64", "base16"}: + msg = ( + f"Non-JSON-serializable type (format={current.get('format')!r}, " + f"contentEncoding={current.get('contentEncoding')!r}) is not supported " + f"inside recursive $defs cycle {ref_name!r}" + ) + raise RecursiveNonJsonTypeError(msg) + if "$ref" in current and isinstance(current["$ref"], str): + child_ref_name, child_target = _resolve_ref(current["$ref"], defs) + if child_ref_name not in visited_refs: + visited_refs.add(child_ref_name) + stack.append(child_target) + stack.extend(v for k, v in current.items() if k != "$defs") + elif isinstance(current, list): + stack.extend(current) + + +def _is_null_branch(node: Mapping[str, Any]) -> bool: + """Return True if ``node`` represents a null-only schema.""" + node_type = node.get("type") + if node_type in ("null", ["null"]): + return True + if "const" in node and node["const"] is None and "type" not in node: + return True + return bool( + "enum" in node + and isinstance(node["enum"], list) + and all(v is None for v in node["enum"]) + and "type" not in node + ) + + +def _has_null_possibility(node: Mapping[str, Any], defs: Mapping[str, Any]) -> bool: + """Return True if ``node`` or any of its referenced/combinator branches allows ``null``.""" + if "$ref" in node: + _, target = _resolve_ref(node["$ref"], defs) + if _has_null_possibility(target, defs): + return True + node_type = node.get("type") + if node_type == "null" or (isinstance(node_type, list) and "null" in node_type): + return True + if ("const" in node and node["const"] is None) or ( + "enum" in node and isinstance(node["enum"], list) and None in node["enum"] + ): + return True + for kw in ("anyOf", "oneOf", "allOf"): + if kw in node and isinstance(node[kw], list): + for branch in node[kw]: + if isinstance(branch, Mapping) and (_is_null_branch(branch) or _has_null_possibility(branch, defs)): + return True + return False + + +def _resolve_string_scalar(node: Mapping[str, Any]) -> tuple[pa.DataType, str]: + """Resolve a ``type: 'string'`` node with optional ``format`` or ``contentEncoding``.""" + content_enc = node.get("contentEncoding") + if content_enc in {"base64", "base16"}: + return pa.binary(), f"string ({content_enc})" + fmt = node.get("format") + if fmt == "date": + return pa.date32(), "string (date)" + if isinstance(fmt, str) and fmt: + return pa.string(), f"string ({fmt})" + return pa.string(), "string" + + +def _scalar_for_type_name(type_name: str, node: Mapping[str, Any]) -> tuple[pa.DataType, str]: + """Map a single JSON Schema scalar type name and format/encoding facets to ``(pa.DataType, label)``.""" + if type_name in _NON_EXTENDABLE_SCALAR_MAP: + return _NON_EXTENDABLE_SCALAR_MAP[type_name] + if type_name == "string": + return _resolve_string_scalar(node) + msg = f"Unsupported or non-scalar type in scalar context: {type_name!r}" + raise JSONSchemaToPyArrowError(msg) + + +def _unify_scalar_types(branches: list[tuple[pa.DataType, str]]) -> tuple[pa.DataType, str]: + """Unify multiple scalar ``(pa.DataType, jsonschema_type)`` pairs according to the scalar union rules.""" + seen_labels: list[str] = [] + for _, label in branches: + if label not in seen_labels: + seen_labels.append(label) + joined_label = " | ".join(seen_labels) if len(seen_labels) > 1 else seen_labels[0] + + first_type = branches[0][0] + if all(dt == first_type for dt, _ in branches[1:]): + return first_type, joined_label + base_types = {label.split(" ", maxsplit=1)[0] for _, label in branches} + if base_types <= {"integer", "number"}: + return pa.float64(), joined_label + return pa.string(), joined_label + + +def _infer_literal_type_name(value: object) -> str: + """Infer the JSON Schema scalar type name of a Python literal inside ``const`` or ``enum``.""" + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "integer" + if isinstance(value, float): + return "number" + if isinstance(value, str): + return "string" + msg = f"Unsupported non-scalar literal in const/enum: {value!r}" + raise JSONSchemaToPyArrowError(msg) + + +def _resolve_scalar(node: Mapping[str, Any]) -> tuple[pa.DataType, str]: + """Resolve a scalar JSON Schema node to ``(pa.DataType, jsonschema_type)``.""" + if _is_null_branch(node): + msg = "Standalone null-only schema cannot be converted to a PyArrow field" + raise NullOnlySchemaError(msg) + if "const" in node and "type" not in node: + return _scalar_for_type_name(_infer_literal_type_name(node["const"]), {}) + if "enum" in node and "type" not in node: + non_null_vals = [v for v in node["enum"] if v is not None] + types_seen: list[str] = [] + for val in non_null_vals: + t_name = _infer_literal_type_name(val) + if t_name not in types_seen: + types_seen.append(t_name) + return _unify_scalar_types([_scalar_for_type_name(t_name, {}) for t_name in types_seen]) + + node_type = node.get("type") + if isinstance(node_type, list): + non_null = [t for t in node_type if t != "null"] + if len(non_null) == 1: + single_node = {**node, "type": non_null[0]} + return _resolve_scalar(single_node) + if any(t in {"object", "array"} for t in non_null): + msg = f"Incompatible branch types in multi-type union: {node_type!r}" + raise IncompatibleBranchShapesError(msg) + return _unify_scalar_types([_scalar_for_type_name(t, node) for t in non_null]) + if isinstance(node_type, str): + return _scalar_for_type_name(node_type, node) + msg = f"Cannot resolve scalar type for schema node: {node!r}" + raise JSONSchemaToPyArrowError(msg) + + +def resolve_scalar_type(node: Mapping[str, Any]) -> tuple[pa.DataType, str]: + """Map a scalar JSON Schema node to its PyArrow type and ``jsonschema_type`` label. + + :param node: scalar schema node (``type``, ``format``, ``contentEncoding``, ``enum`` or ``const``) + :type node: Mapping[str, Any] + :return: ``(pa.DataType, jsonschema_type)`` + :rtype: tuple[pa.DataType, str] + :raises JSONSchemaToPyArrowError: if ``node`` is not a convertible scalar schema + """ + return _resolve_scalar(node) + + +def _classify_resolved_node_kind(node: Mapping[str, Any]) -> str: + """Classify a validated, non-``$ref`` schema node into its structural shape.""" + if any(kw in node for kw in ("anyOf", "oneOf")) and "properties" not in node and "type" not in node: + return "union" + if "allOf" in node and "properties" not in node and "type" not in node: + return "object" + node_type = node.get("type") + if isinstance(node_type, list): + non_null = [t for t in node_type if t != "null"] + if len(non_null) == 1: + node_type = non_null[0] + elif any(t in {"object", "array"} for t in non_null): + msg = f"Incompatible branch types in multi-type union: {node_type!r}" + raise IncompatibleBranchShapesError(msg) + else: + return "scalar" + if node_type == "object" or any(k in node for k in ("properties", "additionalProperties", "patternProperties")): + has_props = bool(node.get("properties")) or any( + k in node for k in ("allOf", "anyOf", "oneOf", "dependentSchemas", "then", "else") + ) + has_map = isinstance(node.get("additionalProperties"), dict) or bool(node.get("patternProperties")) + return "map" if (has_map and not has_props) else "object" + if node_type == "array" or "items" in node or "prefixItems" in node: + return "array" + return "scalar" + + +def _classify_node_kind(node: object, defs: Mapping[str, Any]) -> str: + """Validate ``node`` and return ``'null'``, ``'object'``, ``'array'``, ``'map'``, ``'scalar'``, or ``'union'``.""" + validated = _validate_node_constraints(node) + if _is_null_branch(validated): + return "null" + if "$ref" in validated: + _, target = _resolve_ref(validated["$ref"], defs) + if len(validated) == 1: + return _classify_node_kind(target, defs) + merged = {**target, **{k: v for k, v in validated.items() if k != "$ref"}} + return _classify_node_kind(merged, defs) + return _classify_resolved_node_kind(validated) + + +def _get_inherited_description(node: Mapping[str, Any], defs: Mapping[str, Any]) -> str | None: + """Extract the local or ``$ref``-inherited ``description`` string for a schema node.""" + if isinstance(node.get("description"), str) and node["description"]: + return node["description"] + if "$ref" in node: + _, target = _resolve_ref(node["$ref"], defs) + return _get_inherited_description(target, defs) + for kw in ("anyOf", "oneOf"): + if kw in node and isinstance(node[kw], list): + non_null = [b for b in node[kw] if isinstance(b, Mapping) and not _is_null_branch(b)] + if len(non_null) == 1 and "$ref" in non_null[0]: + return _get_inherited_description(non_null[0], defs) + return None + + +def _merge_property_definitions(prop_defs: list[dict[str, Any]]) -> dict[str, Any]: + """Merge multiple schema definitions for the same property name.""" + if len(prop_defs) == 1: + return prop_defs[0] + first_json = json.dumps(prop_defs[0], sort_keys=True) + if all(json.dumps(p, sort_keys=True) == first_json for p in prop_defs[1:]): + return prop_defs[0] + merged: dict[str, Any] = {"anyOf": prop_defs} + for prop_schema in prop_defs: + if "description" in prop_schema: + merged["description"] = prop_schema["description"] + break + return merged + + +def _extract_choice_branches_into_object( + branches: list[Any], + kw: str, + ctx: _SchemaBuildContext, + raw_props: dict[str, list[tuple[dict[str, Any], _SchemaBuildContext]]], + scalar_branches: list[dict[str, Any]], +) -> set[str]: + """Extract properties, common required fields, and scalar branches from an ``anyOf``/``oneOf`` list.""" + non_null = [b for b in branches if not (isinstance(b, Mapping) and _is_null_branch(b))] + if not non_null: + msg = f"Standalone null-only {kw} combinator cannot be converted to a PyArrow type" + raise NullOnlySchemaError(msg) + branch_req_sets: list[set[str]] = [] + has_scalar = False + for branch in non_null: + branch_kind = _classify_node_kind(branch, ctx.defs) + if branch_kind == "scalar": + has_scalar = True + scalar_branches.append(branch) + elif branch_kind == "object": + b_props, b_req, b_scalars = _extract_object_parts_with_refs(branch, ctx) + for prop_name, (prop_schema, prop_ctx) in b_props.items(): + raw_props.setdefault(prop_name, []).append((prop_schema, prop_ctx)) + branch_req_sets.append(b_req) + scalar_branches.extend(b_scalars) + else: + msg = f"Incompatible branch shape {branch_kind!r} in {kw} combinator" + raise IncompatibleBranchShapesError(msg) + if branch_req_sets and not has_scalar: + return set.intersection(*branch_req_sets) + return set() + + +def _extract_object_parts_with_refs( + node: dict[str, Any], + ctx: _SchemaBuildContext, +) -> tuple[dict[str, tuple[dict[str, Any], _SchemaBuildContext]], set[str], list[dict[str, Any]]]: + """Extract ordered merged property definitions, required set, and scalar union branches for an object node.""" + validated = _validate_node_constraints(node) + if "$ref" in validated: + ref_name, target = _resolve_ref(validated["$ref"], ctx.defs) + if ctx.ref_counts.get(ref_name, 0) >= 1: + _check_no_forbidden_types_in_recursive_cycle(ref_name, target, ctx.defs) + next_ctx = ctx.with_ref(ref_name) + merged_node = {**copy.deepcopy(target), **{k: v for k, v in validated.items() if k != "$ref"}} + return _extract_object_parts_with_refs(merged_node, next_ctx) + + raw_props: dict[str, list[tuple[dict[str, Any], _SchemaBuildContext]]] = {} + required: set[str] = set(validated.get("required", [])) + scalar_branches: list[dict[str, Any]] = [] + + for sub in validated.get("allOf", []): + sub_props, sub_req, sub_scalars = _extract_object_parts_with_refs(sub, ctx) + for prop_name, item in sub_props.items(): + raw_props.setdefault(prop_name, []).append(item) + required |= sub_req + scalar_branches.extend(sub_scalars) + + for prop_name, prop_schema in validated.get("properties", {}).items(): + raw_props.setdefault(prop_name, []).append((_validate_node_constraints(prop_schema), ctx)) + + for kw in ("anyOf", "oneOf"): + if kw in validated: + required |= _extract_choice_branches_into_object(validated[kw], kw, ctx, raw_props, scalar_branches) + + for dep_schema in validated.get("dependentSchemas", {}).values(): + d_props, _, _ = _extract_object_parts_with_refs(dep_schema, ctx) + for prop_name, item in d_props.items(): + raw_props.setdefault(prop_name, []).append(item) + + for kw in ("then", "else"): + if kw in validated: + c_props, _, _ = _extract_object_parts_with_refs(validated[kw], ctx) + for prop_name, item in c_props.items(): + raw_props.setdefault(prop_name, []).append(item) + + merged_props: dict[str, tuple[dict[str, Any], _SchemaBuildContext]] = { + prop_name: (_merge_property_definitions([item[0] for item in items]), items[0][1]) + for prop_name, items in raw_props.items() + } + return merged_props, required, scalar_branches + + +def _build_merged_object( + node: dict[str, Any], + ctx: _SchemaBuildContext, + *, + force_all_nullable: bool = False, + scalar_branches: list[dict[str, Any]] | None = None, +) -> pa.StructType: + """Build a ``pa.StructType`` from an object schema and its combinator branches.""" + props_by_name, required, extracted_scalars = _extract_object_parts_with_refs(node, ctx) + all_scalars = list(scalar_branches or []) + for scalar_node in extracted_scalars: + if scalar_node not in all_scalars: + all_scalars.append(scalar_node) + + if all_scalars: + force_all_nullable = True + if "_value" in props_by_name: + msg = "Property name '_value' in object branch collides with synthesized '_value' field in scalar+object union" + raise ValueFieldCollisionError(msg) + + fields: list[pa.Field] = [] + for prop_name, (merged_prop, prop_ctx) in props_by_name.items(): + is_req = (prop_name in required) and not force_all_nullable + fields.append(_build_field(prop_name, merged_prop, prop_ctx, required_in_parent=is_req)) + + if all_scalars: + resolved_scalars: list[tuple[pa.DataType, str]] = [] + for scalar_branch in all_scalars: + s_type, s_label = _build_type(scalar_branch, ctx) + if s_label is None: + msg = f"Expected scalar branch with jsonschema_type label, got {scalar_branch!r}" + raise JSONSchemaToPyArrowError(msg) + resolved_scalars.append((s_type, s_label)) + val_type, val_label = _unify_scalar_types(resolved_scalars) + fields.append(pa.field("_value", val_type, nullable=True, metadata={"jsonschema_type": val_label})) + + if not fields: + msg = "Empty object schema with no properties cannot be converted to a PyArrow struct" + raise EmptyObjectError(msg) + return pa.struct(fields) + + +def _build_map(node: dict[str, Any], ctx: _SchemaBuildContext) -> pa.MapType: + """Build a ``pa.MapType`` from a dynamic map object schema (``additionalProperties`` / ``patternProperties``).""" + val_schemas: list[dict[str, Any]] = [] + if isinstance(node.get("additionalProperties"), dict): + val_schemas.append(_validate_node_constraints(node["additionalProperties"])) + val_schemas.extend( + _validate_node_constraints(pat_schema) for pat_schema in node.get("patternProperties", {}).values() + ) + val_schema = _merge_property_definitions(val_schemas) + val_field = _build_field("value", val_schema, ctx, required_in_parent=True) + return pa.map_(pa.string(), val_field) + + +def _build_array(node: dict[str, Any], ctx: _SchemaBuildContext) -> pa.DataType: + """Build a ``pa.ListType`` (homogeneous array) or ``pa.StructType`` (closed tuple ``prefixItems``).""" + if "prefixItems" in node: + if node.get("items") not in (False, None): + msg = "Open tuple combining prefixItems with schema-valued items is not supported" + raise OpenTupleError(msg) + min_items = node.get("minItems", 0) + fields = [ + _build_field(f"item_{idx}", item_schema, ctx, required_in_parent=idx < min_items) + for idx, item_schema in enumerate(node["prefixItems"]) + ] + if not fields: + msg = "Empty prefixItems tuple cannot be converted to a PyArrow struct" + raise EmptyObjectError(msg) + return pa.struct(fields) + + items_schema = node.get("items") + validated_items = _validate_node_constraints(items_schema) + item_field = _build_field("item", validated_items, ctx, required_in_parent=True) + return pa.list_(item_field) + + +def _build_union_type(node: dict[str, Any], ctx: _SchemaBuildContext) -> tuple[pa.DataType, str | None]: + """Resolve an ``anyOf`` or ``oneOf`` combinator node to ``(pa.DataType, jsonschema_type)``.""" + kw = "anyOf" if "anyOf" in node else "oneOf" + non_null = [_validate_node_constraints(b) for b in node[kw] if not (isinstance(b, Mapping) and _is_null_branch(b))] + if not non_null: + msg = f"Standalone null-only {kw} combinator cannot be converted to a PyArrow field" + raise NullOnlySchemaError(msg) + if len(non_null) == 1: + return _build_type(non_null[0], ctx) + + obj_branches: list[dict[str, Any]] = [] + scalar_branches: list[dict[str, Any]] = [] + for branch in non_null: + kind = _classify_node_kind(branch, ctx.defs) + if kind == "object": + obj_branches.append(branch) + elif kind == "scalar": + scalar_branches.append(branch) + else: + msg = f"Incompatible branch shape {kind!r} in {kw} combinator" + raise IncompatibleBranchShapesError(msg) + + if scalar_branches and not obj_branches: + resolved_scalars: list[tuple[pa.DataType, str]] = [] + for scalar_branch in scalar_branches: + s_type, s_label = _build_type(scalar_branch, ctx) + if s_label is None: + msg = f"Expected scalar branch with jsonschema_type label, got {scalar_branch!r}" + raise JSONSchemaToPyArrowError(msg) + resolved_scalars.append((s_type, s_label)) + return _unify_scalar_types(resolved_scalars) + + if obj_branches and not scalar_branches: + return _build_merged_object(node, ctx), None + + return ( + _build_merged_object( + node, + ctx, + force_all_nullable=True, + scalar_branches=scalar_branches, + ), + None, + ) + + +def _build_ref_type(validated: dict[str, Any], ctx: _SchemaBuildContext) -> tuple[pa.DataType, str | None]: + """Resolve and build a ``$ref`` schema node with recursion depth tracking.""" + ref_name, target = _resolve_ref(validated["$ref"], ctx.defs) + count = ctx.ref_counts.get(ref_name, 0) + if count >= 1: + _check_no_forbidden_types_in_recursive_cycle(ref_name, target, ctx.defs) + if count >= ctx.max_recursion_depth: + return pa.string(), None + next_ctx = ctx.with_ref(ref_name) + if len(validated) == 1: + return _build_type(target, next_ctx) + merged = {**copy.deepcopy(target), **{k: v for k, v in validated.items() if k not in {"$ref", "description"}}} + return _build_type(merged, next_ctx) + + +def _build_type(node: object, ctx: _SchemaBuildContext) -> tuple[pa.DataType, str | None]: + """Convert any JSON Schema node into ``(pa.DataType, jsonschema_type | None)``.""" + validated = _validate_node_constraints(node) + if _is_null_branch(validated): + msg = "Standalone null-only schema cannot be converted to a PyArrow field" + raise NullOnlySchemaError(msg) + + if "$ref" in validated: + return _build_ref_type(validated, ctx) + + if ("anyOf" in validated or "oneOf" in validated) and "properties" not in validated and "allOf" not in validated: + return _build_union_type(validated, ctx) + + kind = _classify_node_kind(validated, ctx.defs) + if kind == "object": + return _build_merged_object(validated, ctx), None + if kind == "map": + return _build_map(validated, ctx), None + if kind == "array": + return _build_array(validated, ctx), None + return _resolve_scalar(validated) + + +def _build_field( + name: str, + node: object, + ctx: _SchemaBuildContext, + *, + required_in_parent: bool, +) -> pa.Field: + """Build a ``pa.Field`` with nullability and metadata from a JSON Schema property/item node.""" + validated = _validate_node_constraints(node) + nullable = (not required_in_parent) or _has_null_possibility(validated, ctx.defs) + desc = _get_inherited_description(validated, ctx.defs) + data_type, js_type = _build_type(validated, ctx) + metadata: dict[str, str] = {} + if desc: + metadata["comment"] = desc + if js_type: + metadata["jsonschema_type"] = js_type + return pa.field(name, data_type, nullable=nullable, metadata=metadata or None) + + +def generate_pyarrow_schema( + schema: dict[str, Any], + *, + max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH, +) -> pa.Schema: + """Generate a deterministic PyArrow Schema from a Draft 2020-12 JSON Schema. + + :param schema: JSON Schema Draft 2020-12 dictionary + :type schema: dict[str, Any] + :param max_recursion_depth: maximum expansion depth for recursive ``$defs`` references + :type max_recursion_depth: int + :return: PyArrow schema with top-level fields corresponding to root object properties + :rtype: pa.Schema + :raises JSONSchemaToPyArrowError: if ``schema`` uses an unsupported dialect or construct + """ + if not isinstance(schema, dict) or schema.get("$schema") not in SUPPORTED_SCHEMA_DIALECTS: + msg = f"Missing or unsupported $schema dialect: {schema.get('$schema') if isinstance(schema, dict) else None!r}; expected JSON Schema Draft 2020-12" + raise UnsupportedDialectError(msg) + + defs = schema.get("$defs", {}) + if not isinstance(defs, Mapping): + msg = "$defs must be a mapping of definition names to schema objects" + raise JSONSchemaToPyArrowError(msg) + + root: dict[str, Any] = _validate_node_constraints(schema) + ref_counts: dict[str, int] = {} + if "$ref" in root and "properties" not in root: + ref_name, target = _resolve_ref(root["$ref"], defs) + ref_counts[ref_name] = 1 + root = _validate_node_constraints(target) + + root_kind = _classify_node_kind(root, defs) + if root_kind == "map": + msg = "Top-level root schema cannot be a dynamic map without fixed properties" + raise NonStructRootError(msg) + if root_kind != "object": + msg = f"Top-level root schema must resolve to a fixed-property object, got {root_kind!r}" + raise NonStructRootError(msg) + + ctx = _SchemaBuildContext( + defs=defs, + ref_counts=ref_counts, + max_recursion_depth=max_recursion_depth, + ) + struct_type = _build_merged_object(root, ctx) + return pa.schema(list(struct_type)) + + +def _parse_iso_timestamp(value: str, data_type: pa.TimestampType) -> datetime: + """Parse an ISO-8601 timestamp string, attaching UTC when target type is tz-aware.""" + parsed_dt = datetime.fromisoformat(value) + if parsed_dt.tzinfo is None and data_type.tz is not None: + return parsed_dt.replace(tzinfo=UTC) + return parsed_dt + + +def _decode_binary_scalar(value: str, metadata: dict[bytes, bytes] | None) -> bytes: + """Decode a Base64 or Base16 (hex) string into raw bytes based on field metadata.""" + if metadata and b"jsonschema_type" in metadata and b"base16" in metadata[b"jsonschema_type"]: + return bytes.fromhex(value) + return base64.b64decode(value) + + +def _coerce_string_input_scalar( + value: str, + data_type: pa.DataType, + metadata: dict[bytes, bytes] | None, +) -> Any: # noqa: ANN401 + """Coerce a string value when the target PyArrow type is binary, temporal, or decimal.""" + if pa.types.is_binary(data_type): + return _decode_binary_scalar(value, metadata) + if pa.types.is_date(data_type): + return date.fromisoformat(value) + if pa.types.is_timestamp(data_type): + return _parse_iso_timestamp(value, data_type) + if pa.types.is_time(data_type): + return time.fromisoformat(value) + if pa.types.is_decimal(data_type): + return Decimal(value) + return value + + +def _coerce_scalar_for_pyarrow( + value: Any, # noqa: ANN401 + data_type: pa.DataType, + metadata: dict[bytes, bytes] | None = None, +) -> Any: # noqa: ANN401 + """Coerce a JSON-parsed value to match a PyArrow scalar type.""" + if isinstance(value, str): + return _coerce_string_input_scalar(value, data_type, metadata) + if pa.types.is_decimal(data_type) and isinstance(value, int | float) and not isinstance(value, bool): + return Decimal(str(value)) + if pa.types.is_floating(data_type) and isinstance(value, int) and not isinstance(value, bool): + return float(value) + if pa.types.is_integer(data_type) and isinstance(value, float) and value.is_integer(): + return int(value) + if pa.types.is_string(data_type) or pa.types.is_large_string(data_type): + return json.dumps(value) if isinstance(value, Mapping | list | bool) else str(value) + return value + + +def _coerce_struct_for_pyarrow(value: Any, data_type: pa.StructType) -> Any: # noqa: ANN401 + """Coerce a mapping, closed tuple list, or scalar union value to match a PyArrow StructType.""" + if isinstance(value, Mapping): + return { + field.name: _coerce_value_for_pyarrow(value[field.name], field.type, field.metadata) + for field in data_type + if field.name in value + } + if isinstance(value, list) and len(data_type) > 0 and data_type[0].name == "item_0": + return { + field.name: _coerce_value_for_pyarrow(value[idx], field.type, field.metadata) + for idx, field in enumerate(data_type) + if idx < len(value) and value[idx] is not None + } + value_field = next((field for field in data_type if field.name == "_value"), None) + if value_field is not None: + return {"_value": _coerce_value_for_pyarrow(value, value_field.type, value_field.metadata)} + return value + + +def _coerce_value_for_pyarrow( + value: Any, # noqa: ANN401 + data_type: pa.DataType, + metadata: dict[bytes, bytes] | None = None, +) -> Any: # noqa: ANN401 + """Recursively coerce a JSON-parsed value to match the target PyArrow DataType.""" + if value is None: + return None + if pa.types.is_struct(data_type): + return _coerce_struct_for_pyarrow(value, data_type) + if (pa.types.is_list(data_type) or pa.types.is_large_list(data_type)) and isinstance(value, list): + item_field = data_type.value_field + return [_coerce_value_for_pyarrow(item, item_field.type, item_field.metadata) for item in value] + if pa.types.is_map(data_type) and isinstance(value, Mapping): + key_field = data_type.key_field + item_field = data_type.item_field + return { + _coerce_value_for_pyarrow(k, key_field.type, key_field.metadata): _coerce_value_for_pyarrow( + v, item_field.type, item_field.metadata + ) + for k, v in value.items() + } + return _coerce_scalar_for_pyarrow(value, data_type, metadata) + + +def coerce_record_for_pyarrow( + record: dict[str, Any], + schema: pa.Schema, + *, + max_recursion_depth: int | None = DEFAULT_MAX_RECURSION_DEPTH, +) -> dict[str, Any]: + """Coerce a JSON-validated record dictionary to match a PyArrow schema. + + Only fields declared in ``schema`` are retained and coerced recursively so + ``pyarrow.Table.from_pylist`` can construct a table matching ``schema``. + + :param record: JSON-parsed and schema-validated record dictionary + :type record: dict[str, Any] + :param schema: target PyArrow schema for the record's table + :type schema: pa.Schema + :param max_recursion_depth: optional recursion depth parameter accepted for caller compatibility + :type max_recursion_depth: int | None + :return: coerced record dictionary ready for ``ArrowListBuffer`` or ``pa.Table.from_pylist`` + :rtype: dict[str, Any] + """ + _ = max_recursion_depth + return { + field.name: _coerce_value_for_pyarrow(record[field.name], field.type, field.metadata) + for field in schema + if field.name in record + } + + +convert_json_record = coerce_record_for_pyarrow diff --git a/src/cdm_data_loaders/pipelines/core.py b/src/cdm_data_loaders/pipelines/core.py index faa35c11..a1aa1a68 100644 --- a/src/cdm_data_loaders/pipelines/core.py +++ b/src/cdm_data_loaders/pipelines/core.py @@ -206,7 +206,7 @@ def save_pipeline_metadata( "environment", "staging_type", "staging_name", - "staging_display_credentials", + "staging_displayable_credentials", "dataset_name", ] }, @@ -290,6 +290,8 @@ def run_pipeline( pipeline = dlt.pipeline(**pipeline_kwargs) + logger.info("Prepping pipeline...") + try: pipeline.destination = dlt.destination(settings.use_destination, **(destination_kwargs or {})) except Exception as e: @@ -305,7 +307,9 @@ def run_pipeline( load_info: LoadInfo | None = None err_msg: str | None = None try: + logger.info("Running pipeline!") load_info = pipeline.run(resource, **(pipeline_run_kwargs or {})) + logger.info("Pipeline complete!") except PipelineStepFailed as e: # errors raise PipelineStepFailed err_msg = f"Pipeline failed: {e!s}" diff --git a/src/cdm_data_loaders/readers/xsd.py b/src/cdm_data_loaders/readers/xsd.py index ae8311d9..846cce80 100644 --- a/src/cdm_data_loaders/readers/xsd.py +++ b/src/cdm_data_loaders/readers/xsd.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any, Final, Literal +from typing import Any, ClassVar, Final, Literal import pyarrow as pa import xmlschema @@ -90,6 +90,93 @@ } +class XsdToPyArrowError(ValueError): + """Base class for XSD constructs that cannot be converted to a PyArrow schema. + + Each subclass carries a stable specification ``code`` and a short ``title``; its docstring states the reason + the construct is rejected. The converter specification document is generated from these attributes. + """ + + code: ClassVar[str] = "" + title: ClassVar[str] = "" + + +class XmlTagNotFoundError(XsdToPyArrowError): + """The target element is not declared anywhere in the XSD.""" + + code = "E1" + title = "`xml_tag` not found" + + +class AmbiguousXmlTagError(XsdToPyArrowError): + """Declarations of `xml_tag` at the shallowest matching depth have different types, so no single `pa.Schema` can be chosen.""" + + code = "E2" + title = "Ambiguous `xml_tag` type" + + +class NonStructXmlTagError(XsdToPyArrowError): + """A `pa.Table` schema needs top-level columns: the type of `xml_tag` must yield at least one attribute, `#text`, or child element field.""" + + code = "E3" + title = "Non-struct top-level `xml_tag`" + + +class WildcardElementError(XsdToPyArrowError): + """Arbitrary undeclared child elements cannot be represented as fixed `pa.struct` fields.""" + + code = "E4" + title = "Element wildcard (`xs:any`)" + + +class WildcardAttributeError(XsdToPyArrowError): + """Arbitrary undeclared attributes cannot be represented as fixed `pa.struct` fields.""" + + code = "E5" + title = "Attribute wildcard (`xs:anyAttribute`)" + + +class AbstractComponentError(XsdToPyArrowError): + """Concrete instances of an abstract element or type require `xsi:type` or substitution polymorphism, which has no single schema.""" + + code = "E6" + title = 'Abstract element or type (`abstract="true"`)' + + +class SubstitutionGroupError(XsdToPyArrowError): + """Substituted elements introduce element names and types that are not declared in the parent content model.""" + + code = "E7" + title = "Substitution group" + + +class FieldNameCollisionError(XsdToPyArrowError): + """An attribute (prefixed `_`) and a child element map to the same field name in one `pa.struct`.""" + + code = "E8" + title = "Attribute / child field name collision" + + +class RecursiveNonJsonTypeError(XsdToPyArrowError): + """Subtrees truncated beyond `max_recursion_depth` are serialized with `json.dumps`, which cannot encode `datetime.date` or `bytes` without lossy conversion.""" + + code = "E9" + title = "Non-JSON-serializable scalar under recursion" + + +XSD_TO_PYARROW_ERRORS: Final[tuple[type[XsdToPyArrowError], ...]] = ( + XmlTagNotFoundError, + AmbiguousXmlTagError, + NonStructXmlTagError, + WildcardElementError, + WildcardAttributeError, + AbstractComponentError, + SubstitutionGroupError, + FieldNameCollisionError, + RecursiveNonJsonTypeError, +) + + def load_schema(schema_path: str | Path) -> xmlschema.XMLSchema: """Load an XML schema document into an XMLSchema object. @@ -246,7 +333,7 @@ def _check_complex_type_forbidden_recursion(root_type: XsdType, current: XsdComp f"Unsupported XSD feature: recursive type {root_type.local_name or root_type.name!r} " f"contains attribute @{attr.local_name} of type {xsd_name!r}" ) - raise ValueError(msg) + raise RecursiveNonJsonTypeError(msg) if current.has_simple_content() and isinstance(current.content, XsdType): return [current.content] @@ -277,7 +364,7 @@ def _check_no_forbidden_types_under_recursion(root_type: XsdType) -> None: f"Unsupported XSD feature: recursive type {root_type.local_name or root_type.name!r} " f"contains {xsd_name!r}, which cannot be serialized to JSON when recursion is truncated" ) - raise ValueError(msg) + raise RecursiveNonJsonTypeError(msg) continue stack.extend(_check_complex_type_forbidden_recursion(root_type, current)) @@ -348,14 +435,14 @@ def _validate_group_particles(schema: xmlschema.XMLSchema, group: XsdGroup) -> N for item in target_group.content: if _is_any_element(item): msg = "Unsupported XSD feature: xs:any wildcard element is not supported" - raise ValueError(msg) + raise WildcardElementError(msg) if isinstance(item, XsdGroup): _validate_group_particles(schema, item) elif isinstance(item, XsdElement): target_el = item.ref if item.ref is not None else item if getattr(item, "abstract", False) or getattr(target_el, "abstract", False): msg = f"Unsupported XSD feature: abstract element {item.local_name!r} is not supported" - raise ValueError(msg) + raise AbstractComponentError(msg) if ( item.name in schema.substitution_groups or target_el.name in schema.substitution_groups @@ -363,10 +450,10 @@ def _validate_group_particles(schema: xmlschema.XMLSchema, group: XsdGroup) -> N or getattr(target_el, "substitution_group", None) is not None ): msg = f"Unsupported XSD feature: substitutionGroup on element {item.local_name!r} is not supported" - raise ValueError(msg) + raise SubstitutionGroupError(msg) if item.type is not None and getattr(item.type, "abstract", False): msg = f"Unsupported XSD feature: abstract type {item.type.local_name!r} is not supported" - raise ValueError(msg) + raise AbstractComponentError(msg) def _collect_unique_children(schema: xmlschema.XMLSchema, complex_type: XsdComplexType) -> list[XsdElement]: @@ -394,7 +481,7 @@ def _collect_attributes(complex_type: XsdComplexType) -> list[XsdAttribute]: for attr in complex_type.attributes.values(): if _is_any_attribute(attr): msg = "Unsupported XSD feature: xs:anyAttribute wildcard is not supported" - raise ValueError(msg) + raise WildcardAttributeError(msg) if isinstance(attr, XsdAttribute) and getattr(attr, "use", None) != "prohibited": if attr.local_name is None: continue @@ -501,7 +588,7 @@ def _build_complex_struct_fields( f"Field name collision in {complex_type.local_name or 'complexType'!r}: " f"attribute and child element both map to {child_name!r}" ) - raise ValueError(msg) + raise FieldNameCollisionError(msg) is_list, nullable = _resolve_child_cardinality(model_group, child, parent_nillable=parent_nillable) @@ -674,7 +761,7 @@ def _matches(el: XsdElement) -> bool: current_level = next_level msg = f"Element {xml_tag!r} not found in XSD schema" - raise ValueError(msg) + raise XmlTagNotFoundError(msg) def _deduplicate_target_elements(matches: list[XsdElement], xml_tag: str) -> XsdElement: @@ -682,15 +769,15 @@ def _deduplicate_target_elements(matches: list[XsdElement], xml_tag: str) -> Xsd by_type: dict[int, XsdElement] = {id(el.type): el for el in matches} if len(by_type) > 1: msg = f"Ambiguous xml_tag {xml_tag!r}: found {len(by_type)} declarations with different types" - raise ValueError(msg) + raise AmbiguousXmlTagError(msg) target = next(iter(by_type.values())) target_ref = target.ref if target.ref is not None else target if getattr(target, "abstract", False) or getattr(target_ref, "abstract", False): msg = f"Unsupported XSD feature: abstract element {xml_tag!r} is not supported" - raise ValueError(msg) + raise AbstractComponentError(msg) if target.type is not None and getattr(target.type, "abstract", False): msg = f"Unsupported XSD feature: abstract type on {xml_tag!r} is not supported" - raise ValueError(msg) + raise AbstractComponentError(msg) return target @@ -698,7 +785,7 @@ def _require_complex_type(elem_type: XsdType | None, xml_tag: str) -> XsdComplex """Ensure that the resolved target element has an ``XsdComplexType``.""" if elem_type is None or not elem_type.is_complex(): msg = f"Top-level xml_tag {xml_tag!r} must have a complex type with attributes or child elements" - raise ValueError(msg) + raise NonStructXmlTagError(msg) assert isinstance(elem_type, XsdComplexType) # noqa: S101 return elem_type @@ -745,12 +832,12 @@ def generate_pyarrow_schema( if _is_empty_marker_complex_type(elem_type): msg = f"Top-level xml_tag {xml_tag!r} has an empty complex type with no attributes or child elements" - raise ValueError(msg) + raise NonStructXmlTagError(msg) attrs = _collect_attributes(elem_type) if (elem_type.has_simple_content() or elem_type.mixed) and not attrs: msg = f"Top-level xml_tag {xml_tag!r} must have a complex type that resolves to a struct" - raise ValueError(msg) + raise NonStructXmlTagError(msg) fields = _build_complex_struct_fields( ctx, @@ -759,7 +846,7 @@ def generate_pyarrow_schema( ) if not fields: msg = f"Top-level xml_tag {xml_tag!r} must have a complex type with at least one field" - raise ValueError(msg) + raise NonStructXmlTagError(msg) return pa.schema(fields) diff --git a/tests/cdm_data_loaders/converters/test_jsonschema_to_pyarrow_spec.py b/tests/cdm_data_loaders/converters/test_jsonschema_to_pyarrow_spec.py new file mode 100644 index 00000000..2a2dc1d1 --- /dev/null +++ b/tests/cdm_data_loaders/converters/test_jsonschema_to_pyarrow_spec.py @@ -0,0 +1,225 @@ +"""Specification tests for JSON Schema-to-PyArrow schema generation and JSON-to-Arrow record coercion. + +Cases, rules and error fixtures are declared in manifests under ``tests/data/jsonschema`` (``rules.yaml``, +``*/case.yaml`` and ``spec/errors/manifest.yaml``). The same manifests generate +``docs/jsonschema_to_pyarrow_specification.md`` via ``scripts/gen_converter_spec.py``. +""" + +import json +from pathlib import Path +from typing import Any + +import pytest +from _pytest.mark.structures import ParameterSet +from jsonschema import Draft202012Validator + +from cdm_data_loaders.converters.jsonschema_to_pyarrow import ( + JSONSCHEMA_TO_PYARROW_ERRORS, + JSONSchemaToPyArrowError, + RecursiveNonJsonTypeError, + coerce_record_for_pyarrow, + generate_pyarrow_schema, +) +from scripts.gen_converter_spec import JSONSCHEMA_TO_PYARROW, render +from tests.converter_spec import Case, ErrorCase, Run, load_cases, load_error_cases, load_rules +from tests.pyarrow_spec_helpers import ( + assert_expected_table_valid, + assert_rows_and_table_match_expected, + assert_schema_matches_expected, +) + +JSONSCHEMA_DATA_DIR = Path(__file__).resolve().parents[2] / "data" / "jsonschema" +RULES = load_rules(JSONSCHEMA_DATA_DIR / "rules.yaml") +CASES: list[Case] = load_cases(JSONSCHEMA_DATA_DIR / "spec") + load_cases(JSONSCHEMA_DATA_DIR / "examples") +ERROR_CASES: list[ErrorCase] = load_error_cases(JSONSCHEMA_DATA_DIR / "spec" / "errors" / "manifest.yaml") +ERRORS_BY_CODE: dict[str, type[JSONSchemaToPyArrowError]] = {cls.code: cls for cls in JSONSCHEMA_TO_PYARROW_ERRORS} + +RUN_PARAMS: list[ParameterSet] = [ + pytest.param(case, run, id=f"{case.suite}-{case.name}-{run.id}") for case in CASES for run in case.runs +] +MAXIMAL_RUN_PARAMS: list[ParameterSet] = [ + pytest.param(run.data, id=f"{case.name}-{run.id}") for case in CASES for run in case.runs if run.id == "maximal" +] +ERROR_PARAMS: list[ParameterSet] = [pytest.param(err, id=f"{err.code}-{err.file.stem}") for err in ERROR_CASES] + + +def _load_json_object(path: Path) -> dict[str, Any]: + """Load a JSON object from disk.""" + loaded = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(loaded, dict): + msg = f"Expected a JSON object in {path}, got {type(loaded)}" + raise TypeError(msg) + return loaded + + +def _load_json_records(path: Path) -> list[dict[str, Any]]: + """Load a list of JSON record objects from disk.""" + loaded = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(loaded, list): + msg = f"Expected a list of JSON records in {path}, got {type(loaded)}" + raise TypeError(msg) + return loaded + + +def _max_recursion_depth(run: Run) -> int: + return int(run.params.get("max_recursion_depth", 2)) + + +def _collect_multi_item_lists(value: object, path: str = "$") -> dict[str, list[str]]: + """Collect canonical JSON element signatures for every list with more than one element.""" + lists_by_path: dict[str, list[str]] = {} + if isinstance(value, list): + if len(value) > 1: + lists_by_path[path] = [json.dumps(item, sort_keys=True) for item in value] + for idx, item in enumerate(value): + lists_by_path.update(_collect_multi_item_lists(item, f"{path}[{idx}]")) + elif isinstance(value, dict): + for key, child in value.items(): + lists_by_path.update(_collect_multi_item_lists(child, f"{path}.{key}")) + return lists_by_path + + +# Manifest consistency + + +def test_manifests_pass_every_case_rule_exists() -> None: + """Every rule referenced by a case manifest is defined in rules.yaml.""" + unknown = {f"{case.suite}/{case.name}": sorted(set(case.rules) - RULES.rules.keys()) for case in CASES} + assert {name: ids for name, ids in unknown.items() if ids} == {} + + +def test_manifests_pass_every_rule_is_exercised() -> None: + """Every rule in rules.yaml is exercised by at least one case.""" + used = {rule_id for case in CASES for rule_id in case.rules} + assert sorted(RULES.rules.keys() - used) == [] + + +def test_manifests_pass_every_case_file_exists() -> None: + """Every file referenced by a case or error manifest exists.""" + paths = [p for case in CASES for p in (case.schema, case.expected_schema)] + paths += [p for case in CASES for run in case.runs for p in (run.data, run.expected_json, run.expected_table)] + paths += [err.file for err in ERROR_CASES] + assert [str(p) for p in paths if not p.is_file()] == [] + + +def test_manifests_pass_every_error_fixture_listed() -> None: + """Every JSON file in spec/errors is listed in the error manifest.""" + listed = {err.file for err in ERROR_CASES} + on_disk = set((JSONSCHEMA_DATA_DIR / "spec" / "errors").glob("*.json")) + assert sorted(str(p) for p in on_disk - listed) == [] + + +def test_manifests_pass_error_codes_unique_and_covered() -> None: + """Error codes are unique, every manifest entry uses a known code, and every code has a fixture.""" + assert len(ERRORS_BY_CODE) == len(JSONSCHEMA_TO_PYARROW_ERRORS) + assert all(cls.code and cls.title and cls.__doc__ for cls in JSONSCHEMA_TO_PYARROW_ERRORS) + manifest_codes = {err.code for err in ERROR_CASES} + assert sorted(manifest_codes - ERRORS_BY_CODE.keys()) == [] + assert sorted(ERRORS_BY_CODE.keys() - manifest_codes) == [] + + +def test_specification_document_pass_up_to_date() -> None: + """The committed specification document matches the output of scripts/gen_converter_spec.py.""" + assert JSONSCHEMA_TO_PYARROW.output.read_text(encoding="utf-8") == render(JSONSCHEMA_TO_PYARROW), ( + "docs are stale: run `uv run python -m scripts.gen_converter_spec`" + ) + + +# Fixture validity + + +@pytest.mark.parametrize(("case", "run"), RUN_PARAMS) +def test_fixtures_pass_records_and_expected_table_validate(case: Case, run: Run) -> None: + """Input records validate against the schema and the expected table matches the expected JSON and schema.""" + schema = _load_json_object(case.schema) + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + records = _load_json_records(run.data) + for record in records: + validator.validate(record) + assert_expected_table_valid( + records, + run.expected_json, + run.expected_table, + expected_count=run.expected_count, + schema_attr=run.schema_attr, + table_attr=run.table_attr, + ) + + +@pytest.mark.parametrize("data_path", MAXIMAL_RUN_PARAMS) +def test_maximal_examples_pass_repeated_items_have_distinct_content(data_path: Path) -> None: + """Every multi-element array in a maximal JSON example contains distinct elements.""" + lists_by_path = _collect_multi_item_lists(_load_json_records(data_path)) + duplicates = {path: sigs for path, sigs in lists_by_path.items() if len(set(sigs)) < len(sigs)} + assert not duplicates + + +# Schema generation and coercion + + +@pytest.mark.parametrize(("case", "run"), RUN_PARAMS) +def test_generate_pyarrow_schema_pass_cases(case: Case, run: Run) -> None: + """generate_pyarrow_schema produces the exact expected PyArrow schema.""" + pa_schema = generate_pyarrow_schema(_load_json_object(case.schema), max_recursion_depth=_max_recursion_depth(run)) + assert_schema_matches_expected(pa_schema, case.expected_schema, run.schema_attr) + + +@pytest.mark.parametrize(("case", "run"), RUN_PARAMS) +def test_coerce_record_for_pyarrow_pass_cases(case: Case, run: Run) -> None: + """Records coerce to the expected rows, build the expected table and round-trip through Parquet.""" + depth = _max_recursion_depth(run) + pa_schema = generate_pyarrow_schema(_load_json_object(case.schema), max_recursion_depth=depth) + rows = [ + coerce_record_for_pyarrow(record, pa_schema, max_recursion_depth=depth) + for record in _load_json_records(run.data) + ] + assert_rows_and_table_match_expected( + rows, pa_schema, run.expected_json, run.expected_table, table_attr=run.table_attr + ) + + +def test_generate_pyarrow_schema_pass_kbase_xsv_metaschema_dialect() -> None: + """The KBase Draft 2020-12 XSV metaschema URI is accepted identically to standard Draft 2020-12.""" + fixture_dir = JSONSCHEMA_DATA_DIR / "spec" / "01_scalars_and_formats" + schema = _load_json_object(fixture_dir / "schema.json") + schema["$schema"] = "https://json-schema.kbase.us/draft/2020-12-xsv/schema" + assert_schema_matches_expected(generate_pyarrow_schema(schema), fixture_dir / "expected_schema.py") + + +# Errors + + +@pytest.mark.parametrize("err", ERROR_PARAMS) +def test_generate_pyarrow_schema_fail_error_manifest(err: ErrorCase) -> None: + """Each error fixture raises the JSONSchemaToPyArrowError subclass for its code.""" + schema = _load_json_object(err.file) + with pytest.raises(ERRORS_BY_CODE[err.code], match=err.match): + generate_pyarrow_schema(schema) + + +@pytest.mark.parametrize( + "forbidden_leaf_schema", + [ + pytest.param({"type": "string", "format": "date"}, id="format-date"), + pytest.param({"type": "string", "contentEncoding": "base64"}, id="contentEncoding-base64"), + pytest.param({"type": "string", "contentEncoding": "base16"}, id="contentEncoding-base16"), + ], +) +def test_generate_pyarrow_schema_fail_non_json_serializable_type_under_recursion( + forbidden_leaf_schema: dict[str, str], +) -> None: + """Date and Base64/Base16 types inside a recursive $defs cycle raise RecursiveNonJsonTypeError (E10).""" + schema: dict[str, Any] = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "$defs": { + "Node": { + "type": "object", + "properties": {"payload": forbidden_leaf_schema, "child": {"$ref": "#/$defs/Node"}}, + }, + }, + } + with pytest.raises(RecursiveNonJsonTypeError, match="recursive"): + generate_pyarrow_schema(schema) diff --git a/tests/cdm_data_loaders/readers/test_xsd_spec.py b/tests/cdm_data_loaders/readers/test_xsd_spec.py index ca4358b8..386706a3 100644 --- a/tests/cdm_data_loaders/readers/test_xsd_spec.py +++ b/tests/cdm_data_loaders/readers/test_xsd_spec.py @@ -1,241 +1,133 @@ -"""Specification tests for XSD-to-PyArrow schema generation and deterministic XML-to-Arrow conversion.""" +"""Specification tests for XSD-to-PyArrow schema generation and XML-to-Arrow conversion. + +Cases, rules and error fixtures are declared in manifests under ``tests/data/xsd`` (``rules.yaml``, +``*/case.yaml`` and ``spec/errors/manifest.yaml``). The same manifests generate +``docs/xsd_to_pyarrow_specification.md`` via ``scripts/gen_converter_spec.py``. +""" import copy -import datetime as dt -import importlib.util -import io -import json from collections import defaultdict from pathlib import Path -from typing import Any import pyarrow as pa -import pyarrow.parquet as pq import pytest import xmlschema from _pytest.mark.structures import ParameterSet -from lxml import etree +from lxml.etree import Element, parse, tostring from cdm_data_loaders.readers.xsd import ( + XSD_TO_PYARROW_ERRORS, + RecursiveNonJsonTypeError, + XsdToPyArrowError, convert_xml_element, generate_pyarrow_schema, load_schema, ) +from scripts.gen_converter_spec import XSD_TO_PYARROW, render +from tests.converter_spec import Case, ErrorCase, Run, load_cases, load_error_cases, load_rules +from tests.pyarrow_spec_helpers import ( + assert_expected_table_valid, + assert_rows_and_table_match_expected, + assert_schema_matches_expected, + list_field, +) + +XSD_DATA_DIR = Path(__file__).resolve().parents[2] / "data" / "xsd" +RULES = load_rules(XSD_DATA_DIR / "rules.yaml") +CASES: list[Case] = load_cases(XSD_DATA_DIR / "spec") + load_cases(XSD_DATA_DIR / "examples") +ERROR_CASES: list[ErrorCase] = load_error_cases(XSD_DATA_DIR / "spec" / "errors" / "manifest.yaml") +ERRORS_BY_CODE: dict[str, type[XsdToPyArrowError]] = {cls.code: cls for cls in XSD_TO_PYARROW_ERRORS} + +RUN_PARAMS: list[ParameterSet] = [ + pytest.param(case, run, id=f"{case.suite}-{case.name}-{run.id}") for case in CASES for run in case.runs +] +MAXIMAL_RUN_PARAMS: list[ParameterSet] = [ + pytest.param(run.data, id=f"{case.name}-{run.id}") for case in CASES for run in case.runs if run.id == "maximal" +] +ERROR_PARAMS: list[ParameterSet] = [pytest.param(err, id=f"{err.code}-{err.file.stem}") for err in ERROR_CASES] -def _list_field(name: str, item_type: pa.DataType, *, nullable: bool = True) -> pa.Field: - """Build a PyArrow list field with standard nullable item field.""" - return pa.field(name, pa.list_(pa.field("item", item_type, nullable=True)), nullable=nullable) - - -def _load_expected_schema(schema_file: Path, attr_name: str = "EXPECTED_SCHEMA") -> pa.Schema: - """Load a pa.Schema constant from an expected_schema.py fixture file.""" - spec = importlib.util.spec_from_file_location(schema_file.stem, schema_file) - if spec is None or spec.loader is None: - msg = f"Could not load schema module from {schema_file}" - raise RuntimeError(msg) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - schema = getattr(module, attr_name) - if not isinstance(schema, pa.Schema): - msg = f"Expected {attr_name} in {schema_file} to be a pa.Schema, got {type(schema)}" - raise TypeError(msg) - return schema - - -def _coerce_json_rows_for_arrow(value: Any, dtype: pa.DataType) -> Any: # noqa: ANN401 - """Coerce ISO date strings and binary strings from expected JSON into Python objects for pa.Table.from_pylist.""" - if value is None: - return None - if pa.types.is_date32(dtype) and isinstance(value, str): - return dt.date.fromisoformat(value) - if pa.types.is_binary(dtype) and isinstance(value, str): - return value.encode("utf-8") - if pa.types.is_list(dtype) and isinstance(value, list): - return [_coerce_json_rows_for_arrow(item, dtype.value_type) for item in value] - if pa.types.is_struct(dtype) and isinstance(value, dict): - return { - field.name: _coerce_json_rows_for_arrow(value[field.name], field.type) - for field in dtype - if field.name in value - } - return value - - -def _coerce_rows_to_schema(rows: list[dict[str, Any]], schema: pa.Schema) -> list[dict[str, Any]]: - """Coerce JSON-loaded rows using field types in schema so pa.Table.from_pylist can materialize them.""" - struct_type = pa.struct(list(schema)) - return [_coerce_json_rows_for_arrow(row, struct_type) for row in rows] - - -def _json_default(obj: object) -> str: - """Serialize Python date and bytes values from Arrow tables for comparison with expected JSON.""" - if isinstance(obj, dt.date): - return obj.isoformat() - if isinstance(obj, bytes): - return obj.decode("utf-8") - msg = f"Object of type {type(obj)} is not JSON serializable" - raise TypeError(msg) - - -def _strip_none(value: object) -> object: - """Recursively remove keys with None values from dicts to compare Arrow-materialized rows with sparse expected.json.""" - if isinstance(value, dict): - return {k: _strip_none(v) for k, v in value.items() if v is not None} - if isinstance(value, list): - return [_strip_none(item) for item in value] - return value - - -def _extract_matching_elements(xml_path: Path, xml_tag: str) -> list[etree._Element]: +def _local_tag(tag: str) -> str: + """Strip an optional Clark-notation '{namespace}' prefix from an XML tag.""" + return tag.rsplit("}", maxsplit=1)[-1] if "}" in tag else tag + + +def _extract_matching_elements(xml_path: Path, xml_tag: str) -> list[Element]: """Parse xml_path and return all elements matching xml_tag at the root or as a direct child of root.""" - local_tag = xml_tag.rsplit("}", maxsplit=1)[-1] if "}" in xml_tag else xml_tag - tree = etree.parse(str(xml_path)) - root = tree.getroot() - root_local = root.tag.rsplit("}", maxsplit=1)[-1] if "}" in root.tag else root.tag - if root_local == local_tag: + local_tag = _local_tag(xml_tag) + root = parse(str(xml_path)).getroot() + if _local_tag(root.tag) == local_tag: return [root] - return [ - child - for child in root - if isinstance(child.tag, str) - and (child.tag.rsplit("}", maxsplit=1)[-1] if "}" in child.tag else child.tag) == local_tag - ] + return [child for child in root if isinstance(child.tag, str) and _local_tag(child.tag) == local_tag] -SPEC_CASES: list[ParameterSet] = [ - pytest.param("01_scalars_and_simple_types", "record", id="01-scalars-and-simple-types"), - pytest.param("02_namespaces_and_nillable", "entry", id="02-namespaces-and-nillable"), - pytest.param("03_mixed_content_raw_text", "article", id="03-mixed-content-raw-text"), - pytest.param("04_recursion_depth", "record", id="04-recursion-depth"), - pytest.param("05_groups_and_compositors", "item", id="05-groups-and-compositors"), - pytest.param("06_context_dependent_paths", "root", id="06-context-dependent-paths"), - pytest.param("07_multi_root_shared_type", "entry", id="07-multi-root-shared-type"), - pytest.param("08_nullability_and_empty_tags", "entry", id="08-nullability-and-empty-tags"), - pytest.param("09_empty_marker_elements", "book", id="09-empty-marker-elements"), - pytest.param("10_combinations", "record", id="10-combinations"), -] +def _max_recursion_depth(run: Run) -> int: + return int(run.params.get("max_recursion_depth", 2)) -REAL_WORLD_SCHEMAS: list[ParameterSet] = [ - pytest.param("uniref", "uniprot/uniref/uniref.xsd", "entry", id="uniref"), - pytest.param("uniprot", "uniprot/uniprot_kb/uniprot.xsd", "entry", id="uniprot"), - pytest.param("pubmed_article", "pubmed/pubmed.xsd", "PubmedArticle", id="pubmed-article"), - pytest.param("pubmed_book_article", "pubmed/pubmed.xsd", "PubmedBookArticle", id="pubmed-book-article"), -] -REAL_WORLD_EXAMPLES: list[ParameterSet] = [ - pytest.param("uniref", "uniprot/uniref/uniref.xsd", "entry", "minimal", id="uniref-minimal"), - pytest.param("uniref", "uniprot/uniref/uniref.xsd", "entry", "maximal", id="uniref-maximal"), - pytest.param("uniprot", "uniprot/uniprot_kb/uniprot.xsd", "entry", "minimal", id="uniprot-minimal"), - pytest.param("uniprot", "uniprot/uniprot_kb/uniprot.xsd", "entry", "maximal", id="uniprot-maximal"), - pytest.param("pubmed_article", "pubmed/pubmed.xsd", "PubmedArticle", "minimal", id="pubmed-article-minimal"), - pytest.param("pubmed_article", "pubmed/pubmed.xsd", "PubmedArticle", "maximal", id="pubmed-article-maximal"), - pytest.param( - "pubmed_book_article", - "pubmed/pubmed.xsd", - "PubmedBookArticle", - "minimal", - id="pubmed-book-article-minimal", - ), - pytest.param( - "pubmed_book_article", - "pubmed/pubmed.xsd", - "PubmedBookArticle", - "maximal", - id="pubmed-book-article-maximal", - ), -] +def _xml_tag(run: Run) -> str: + assert run.xml_tag is not None + return run.xml_tag -@pytest.mark.parametrize(("case_dir", "xml_tag"), SPEC_CASES) -def test_spec_fixtures_pass_xml_and_expected_schema_validate( - test_data_dir: Path, - case_dir: str, - xml_tag: str, -) -> None: - """Every specification data.xml validates against schema.xsd and expected.json materializes with expected_schema.py.""" - fixture_dir = test_data_dir / "xsd" / "spec" / case_dir - schema = load_schema(fixture_dir / "schema.xsd") - schema.validate(str(fixture_dir / "data.xml")) +# Manifest consistency - elements = _extract_matching_elements(fixture_dir / "data.xml", xml_tag) - expected = json.loads((fixture_dir / "expected.json").read_text(encoding="utf-8")) - assert len(elements) == len(expected) - expected_pa_schema = _load_expected_schema(fixture_dir / "expected_schema.py") - coerced_rows = _coerce_rows_to_schema(expected, expected_pa_schema) - table = pa.Table.from_pylist(coerced_rows, schema=expected_pa_schema) - table.validate(full=True) - assert table.num_rows == len(expected) +def test_manifests_pass_every_case_rule_exists() -> None: + """Every rule referenced by a case manifest is defined in rules.yaml.""" + unknown = {f"{case.suite}/{case.name}": sorted(set(case.rules) - RULES.rules.keys()) for case in CASES} + assert {name: ids for name, ids in unknown.items() if ids} == {} - dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) - assert dumped_rows == expected - buf = io.BytesIO() - pq.write_table(table, buf) - buf.seek(0) - roundtrip_table = pq.read_table(buf) - roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) - assert roundtrip_rows == expected +def test_manifests_pass_every_rule_is_exercised() -> None: + """Every rule in rules.yaml is exercised by at least one case.""" + used = {rule_id for case in CASES for rule_id in case.rules} + assert sorted(RULES.rules.keys() - used) == [] -@pytest.mark.parametrize( - ("example_dir", "xsd_rel_path", "xml_tag", "variant"), - REAL_WORLD_EXAMPLES, -) -def test_real_world_examples_pass_xml_and_expected_schema_validate( - test_data_dir: Path, - example_dir: str, - xsd_rel_path: str, - xml_tag: str, - variant: str, -) -> None: - """Every minimal and maximal real-world XML validates against its XSD and materializes with expected_schema.py.""" - schema = load_schema(test_data_dir / xsd_rel_path) - example_base = test_data_dir / "xsd" / "examples" / example_dir - xml_path = example_base / f"{variant}.xml" - expected_path = example_base / f"{variant}_expected.json" - - schema.validate(str(xml_path)) - elements = _extract_matching_elements(xml_path, xml_tag) - expected = json.loads(expected_path.read_text(encoding="utf-8")) - assert len(elements) == 1 - assert len(expected) == 1 - - expected_pa_schema = _load_expected_schema(example_base / "expected_schema.py") - coerced_rows = _coerce_rows_to_schema(expected, expected_pa_schema) - table = pa.Table.from_pylist(coerced_rows, schema=expected_pa_schema) - table.validate(full=True) - assert table.num_rows == 1 - - dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) - assert dumped_rows == expected - - buf = io.BytesIO() - pq.write_table(table, buf) - buf.seek(0) - roundtrip_table = pq.read_table(buf) - roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) - assert roundtrip_rows == expected +def test_manifests_pass_every_case_file_exists() -> None: + """Every file referenced by a case manifest exists.""" + paths = [p for case in CASES for p in (case.schema, case.expected_schema)] + paths += [p for case in CASES for run in case.runs for p in (run.data, run.expected_json, run.expected_table)] + paths += [err.file for err in ERROR_CASES] + assert [str(p) for p in paths if not p.is_file()] == [] -@pytest.mark.parametrize( - "example_dir", - [ - pytest.param("uniref", id="uniref-maximal"), - pytest.param("uniprot", id="uniprot-maximal"), - pytest.param("pubmed_article", id="pubmed-article-maximal"), - pytest.param("pubmed_book_article", id="pubmed-book-article-maximal"), - ], -) -def test_real_world_maximal_examples_pass_repeated_elements_have_distinct_content( - test_data_dir: Path, - example_dir: str, -) -> None: - """Every repeated element in a maximal XML example contains distinct data.""" - xml_path = test_data_dir / "xsd" / "examples" / example_dir / "maximal.xml" - root = etree.parse(str(xml_path)).getroot() +def test_manifests_pass_error_codes_unique_and_covered() -> None: + """Error codes are unique, every error manifest entry uses a known code, and every code has a fixture.""" + assert len(ERRORS_BY_CODE) == len(XSD_TO_PYARROW_ERRORS) + assert all(cls.code and cls.title and cls.__doc__ for cls in XSD_TO_PYARROW_ERRORS) + manifest_codes = {err.code for err in ERROR_CASES} + assert sorted(manifest_codes - ERRORS_BY_CODE.keys()) == [] + assert sorted(ERRORS_BY_CODE.keys() - manifest_codes) == [] + +def test_specification_document_pass_up_to_date() -> None: + """The committed specification document matches the output of scripts/gen_converter_spec.py.""" + assert XSD_TO_PYARROW.output.read_text(encoding="utf-8") == render(XSD_TO_PYARROW), ( + "docs are stale: run `uv run python -m scripts.gen_converter_spec`" + ) + + +# Fixture validity + + +@pytest.mark.parametrize(("case", "run"), RUN_PARAMS) +def test_fixtures_pass_xml_and_expected_table_validate(case: Case, run: Run) -> None: + """Input XML validates against the XSD and the expected table matches the expected JSON and schema.""" + load_schema(case.schema).validate(str(run.data)) + assert_expected_table_valid( + _extract_matching_elements(run.data, _xml_tag(run)), + run.expected_json, + run.expected_table, + expected_count=run.expected_count, + ) + + +@pytest.mark.parametrize("xml_path", MAXIMAL_RUN_PARAMS) +def test_maximal_examples_pass_repeated_elements_have_distinct_content(xml_path: Path) -> None: + """Every repeated element in a maximal XML example contains distinct data.""" + root = parse(str(xml_path)).getroot() by_tag: dict[str, list[str]] = defaultdict(list) for elem in root.iter(): if elem is root or not isinstance(elem.tag, str): @@ -246,151 +138,69 @@ def test_real_world_maximal_examples_pass_repeated_elements_have_distinct_conten node.text = node.text.strip() if node.tail is not None: node.tail = node.tail.strip() - local = elem.tag.rsplit("}", maxsplit=1)[-1] if "}" in elem.tag else elem.tag - by_tag[local].append(etree.tostring(clone, encoding="unicode")) + by_tag[_local_tag(elem.tag)].append(tostring(clone, encoding="unicode")) duplicates = {tag: sigs for tag, sigs in by_tag.items() if len(sigs) > 1 and len(set(sigs)) < len(sigs)} assert not duplicates -@pytest.mark.parametrize(("case_dir", "xml_tag"), SPEC_CASES) -def test_generate_pyarrow_schema_pass_spec_cases( - test_data_dir: Path, - case_dir: str, - xml_tag: str, -) -> None: - """Verify XSD specification schemas generate the exact expected PyArrow schema in expected_schema.py.""" - fixture_dir = test_data_dir / "xsd" / "spec" / case_dir - schema = load_schema(fixture_dir / "schema.xsd") - pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) - expected_schema = _load_expected_schema(fixture_dir / "expected_schema.py") - assert pa_schema.equals(expected_schema, check_metadata=True) - - -def test_generate_pyarrow_schema_pass_clark_notation_tag(test_data_dir: Path) -> None: - """Verify Clark-notation {namespace}local_name resolves identically to local_name.""" - fixture_dir = test_data_dir / "xsd" / "spec" / "02_namespaces_and_nillable" - schema = load_schema(fixture_dir / "schema.xsd") - pa_clark = generate_pyarrow_schema(schema, "{http://example.org/ns}entry") - expected_schema = _load_expected_schema(fixture_dir / "expected_schema.py") - assert pa_clark.equals(expected_schema, check_metadata=True) +# Schema generation and conversion -def test_generate_pyarrow_schema_pass_04_recursion_depth_1(test_data_dir: Path) -> None: - """Verify structural recursion with max_recursion_depth=1 matches EXPECTED_SCHEMA_DEPTH_1.""" - fixture_dir = test_data_dir / "xsd" / "spec" / "04_recursion_depth" - schema = load_schema(fixture_dir / "schema.xsd") - schema_d1 = generate_pyarrow_schema(schema, "record", max_recursion_depth=1) - expected_d1 = _load_expected_schema(fixture_dir / "expected_schema.py", "EXPECTED_SCHEMA_DEPTH_1") - assert schema_d1.equals(expected_d1, check_metadata=True) +@pytest.mark.parametrize(("case", "run"), RUN_PARAMS) +def test_generate_pyarrow_schema_pass_cases(case: Case, run: Run) -> None: + """generate_pyarrow_schema produces the exact expected PyArrow schema.""" + pa_schema = generate_pyarrow_schema( + load_schema(case.schema), _xml_tag(run), max_recursion_depth=_max_recursion_depth(run) + ) + assert_schema_matches_expected(pa_schema, case.expected_schema, run.schema_attr) + + +@pytest.mark.parametrize(("case", "run"), RUN_PARAMS) +def test_convert_xml_element_pass_cases(case: Case, run: Run) -> None: + """XML elements convert to the expected rows, build the expected table and round-trip through Parquet.""" + schema = load_schema(case.schema) + depth = _max_recursion_depth(run) + xml_tag = _xml_tag(run) + pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=depth) + rows = [ + convert_xml_element(elem, schema, xml_tag, max_recursion_depth=depth) + for elem in _extract_matching_elements(run.data, xml_tag) + ] + assert_rows_and_table_match_expected( + rows, pa_schema, run.expected_json, run.expected_table, table_attr=run.table_attr + ) def test_generate_pyarrow_schema_pass_mutual_and_self_referential_xsds(test_data_dir: Path) -> None: - """Verify existing self_referential.xsd and mutual_recursion.xsd schemas terminate at max_recursion_depth.""" - self_ref_schema = load_schema(test_data_dir / "xsd" / "self_referential.xsd") - pa_self = generate_pyarrow_schema(self_ref_schema, "node", max_recursion_depth=2) + """self_referential.xsd and mutual_recursion.xsd terminate at max_recursion_depth.""" + pa_self = generate_pyarrow_schema( + load_schema(test_data_dir / "xsd" / "self_referential.xsd"), "node", max_recursion_depth=2 + ) expected_self = pa.schema( - [ - _list_field( - "child", - pa.struct( - [ - _list_field("child", pa.string(), nullable=True), - ] - ), - nullable=True, - ), - ] + [list_field("child", pa.struct([list_field("child", pa.string(), nullable=True)]), nullable=True)] ) assert pa_self.equals(expected_self, check_metadata=True) - mutual_schema = load_schema(test_data_dir / "xsd" / "mutual_recursion.xsd") - pa_mutual = generate_pyarrow_schema(mutual_schema, "a", max_recursion_depth=1) + pa_mutual = generate_pyarrow_schema( + load_schema(test_data_dir / "xsd" / "mutual_recursion.xsd"), "a", max_recursion_depth=1 + ) expected_mutual = pa.schema( - [ - _list_field( - "b", - pa.struct( - [ - pa.field("a", pa.string(), nullable=True), - ] - ), - nullable=True, - ), - ] + [list_field("b", pa.struct([pa.field("a", pa.string(), nullable=True)]), nullable=True)] ) assert pa_mutual.equals(expected_mutual, check_metadata=True) -@pytest.mark.parametrize( - ("example_dir", "xsd_rel_path", "xml_tag"), - REAL_WORLD_SCHEMAS, -) -def test_generate_pyarrow_schema_pass_real_world_schemas( - test_data_dir: Path, - example_dir: str, - xsd_rel_path: str, - xml_tag: str, -) -> None: - """Verify PyArrow schema generation on UniRef, UniProtKB, and PubMed XSDs matches expected_schema.py.""" - schema = load_schema(test_data_dir / xsd_rel_path) - pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) - expected_schema = _load_expected_schema(test_data_dir / "xsd" / "examples" / example_dir / "expected_schema.py") - assert pa_schema.equals(expected_schema, check_metadata=True) +# Errors -@pytest.mark.parametrize( - ("error_xsd_filename", "xml_tag", "expected_match"), - [ - pytest.param("ambiguous_xml_tag.xsd", "entry", "Ambiguous", id="E2-ambiguous-xml-tag"), - pytest.param("scalar_top_level.xsd", "leaf", "complex type", id="E3-scalar-top-level"), - pytest.param("empty_marker_top_level.xsd", "marker", "complex type", id="E3-empty-marker-top-level"), - pytest.param("wildcard_any_element.xsd", "entry", "xs:any", id="E4-wildcard-any-element"), - pytest.param("wildcard_any_attribute.xsd", "entry", "xs:anyAttribute", id="E5-wildcard-any-attribute"), - pytest.param("abstract_element.xsd", "entry", "abstract", id="E6-abstract-element"), - pytest.param("abstract_type.xsd", "entry", "abstract", id="E6-abstract-type"), - pytest.param("substitution_group.xsd", "entry", "substitutionGroup", id="E7-substitution-group"), - pytest.param("attr_child_name_collision.xsd", "entry", "collision", id="E8-attr-child-name-collision"), - ], -) -def test_generate_pyarrow_schema_fail_unsupported_or_invalid_xsd( - test_data_dir: Path, - error_xsd_filename: str, - xml_tag: str, - expected_match: str, -) -> None: - """Verify unsupported or ambiguous XSD features raise a descriptive ValueError.""" - schema = load_schema(test_data_dir / "xsd" / "spec" / "errors" / error_xsd_filename) - with pytest.raises(ValueError, match=expected_match): - generate_pyarrow_schema(schema, xml_tag) - - -@pytest.mark.parametrize( - ("xsd_filename", "xml_tag", "expected_match"), - [ - pytest.param("empty_schema.xsd", "root", "not found", id="E1-empty-schema"), - pytest.param("no_children.xsd", "leaf", "complex type", id="E3-no-children-scalar-root"), - pytest.param("cdata_variety.xsd", "root", "xs:anyAttribute", id="E5-cdata-variety-has-anyAttribute"), - pytest.param("combinations.xsd", "record", "xs:anyAttribute", id="E5-legacy-combinations-has-anyAttribute"), - ], -) -def test_generate_pyarrow_schema_fail_existing_invalid_schemas( - test_data_dir: Path, - xsd_filename: str, - xml_tag: str, - expected_match: str, -) -> None: - """Verify existing XSDs with scalar roots, empty schemas, or xs:anyAttribute raise ValueError.""" - schema = load_schema(test_data_dir / "xsd" / xsd_filename) - with pytest.raises(ValueError, match=expected_match): - generate_pyarrow_schema(schema, xml_tag) - - -def test_generate_pyarrow_schema_fail_nonexistent_xml_tag(test_data_dir: Path) -> None: - """Verify requesting an xml_tag that does not exist in the XSD raises ValueError.""" - schema = load_schema(test_data_dir / "uniprot" / "uniref" / "uniref.xsd") - with pytest.raises(ValueError, match="not found"): - generate_pyarrow_schema(schema, "nonExistentElement") +@pytest.mark.parametrize("err", ERROR_PARAMS) +def test_generate_pyarrow_schema_fail_error_manifest(err: ErrorCase) -> None: + """Each error fixture raises the XsdToPyArrowError subclass for its code.""" + assert err.xml_tag is not None + schema = load_schema(err.file) + with pytest.raises(ERRORS_BY_CODE[err.code], match=err.match): + generate_pyarrow_schema(schema, err.xml_tag) @pytest.mark.parametrize( @@ -402,135 +212,27 @@ def test_generate_pyarrow_schema_fail_nonexistent_xml_tag(test_data_dir: Path) - pytest.param("xs:base64Binary", id="xs-base64Binary"), ], ) +@pytest.mark.parametrize("position", ["child", "attribute"]) def test_generate_pyarrow_schema_fail_non_json_serializable_type_under_recursion( forbidden_xsd_type: str, + position: str, ) -> None: - """Verify xs:date, xs:dateTime, xs:hexBinary, and xs:base64Binary under a recursive element raise ValueError.""" - xsd_with_child = f""" - - - - - - - - - - """ - schema_child = xmlschema.XMLSchema(xsd_with_child) - with pytest.raises(ValueError, match="recursive type"): - generate_pyarrow_schema(schema_child, "root") - - xsd_with_attr = f""" + """Date, dateTime and binary types under a recursive element raise RecursiveNonJsonTypeError (E9).""" + extra_child = ( + f'' if position == "child" else "" + ) + extra_attr = f'' if position == "attribute" else "" + xsd_text = f""" + {extra_child} - + {extra_attr} """ - schema_attr = xmlschema.XMLSchema(xsd_with_attr) - with pytest.raises(ValueError, match="recursive type"): - generate_pyarrow_schema(schema_attr, "root") - - -@pytest.mark.parametrize(("case_dir", "xml_tag"), SPEC_CASES) -def test_convert_xml_element_and_arrow_table_pass_spec_fixtures( - test_data_dir: Path, - case_dir: str, - xml_tag: str, -) -> None: - """Verify XML elements convert to expected JSON and build a valid PyArrow Table and Parquet file.""" - fixture_dir = test_data_dir / "xsd" / "spec" / case_dir - schema = load_schema(fixture_dir / "schema.xsd") - pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) - - elements = _extract_matching_elements(fixture_dir / "data.xml", xml_tag) - rows = [convert_xml_element(elem, schema, xml_tag, max_recursion_depth=2) for elem in elements] - - expected = json.loads((fixture_dir / "expected.json").read_text(encoding="utf-8")) - serialized_rows = json.loads(json.dumps(rows, default=_json_default)) - assert serialized_rows == expected - - table = pa.Table.from_pylist(rows, schema=pa_schema) - table.validate(full=True) - assert table.num_rows == len(expected) - - dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) - assert dumped_rows == expected - - buf = io.BytesIO() - pq.write_table(table, buf) - buf.seek(0) - roundtrip_table = pq.read_table(buf) - roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) - assert roundtrip_rows == expected - - -def test_convert_xml_element_and_arrow_table_pass_recursion_depth_1(test_data_dir: Path) -> None: - """Verify max_recursion_depth=1 truncates recursive structures one level earlier than depth=2.""" - fixture_dir = test_data_dir / "xsd" / "spec" / "04_recursion_depth" - schema = load_schema(fixture_dir / "schema.xsd") - pa_schema = generate_pyarrow_schema(schema, "record", max_recursion_depth=1) - - elements = _extract_matching_elements(fixture_dir / "data.xml", "record") - rows = [convert_xml_element(elem, schema, "record", max_recursion_depth=1) for elem in elements] - - expected = json.loads((fixture_dir / "expected_depth_1.json").read_text(encoding="utf-8")) - serialized_rows = json.loads(json.dumps(rows, default=_json_default)) - assert serialized_rows == expected - - table = pa.Table.from_pylist(rows, schema=pa_schema) - table.validate(full=True) - dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) - assert dumped_rows == expected - - buf = io.BytesIO() - pq.write_table(table, buf) - buf.seek(0) - roundtrip_table = pq.read_table(buf) - roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) - assert roundtrip_rows == expected - - -@pytest.mark.parametrize( - ("example_dir", "xsd_rel_path", "xml_tag", "variant"), - REAL_WORLD_EXAMPLES, -) -def test_convert_xml_element_and_arrow_table_pass_real_world_examples( - test_data_dir: Path, - example_dir: str, - xsd_rel_path: str, - xml_tag: str, - variant: str, -) -> None: - """Verify minimal and maximal UniRef, UniProt, and PubMed entries convert to expected JSON and Arrow tables.""" - schema = load_schema(test_data_dir / xsd_rel_path) - xml_path = test_data_dir / "xsd" / "examples" / example_dir / f"{variant}.xml" - expected_path = test_data_dir / "xsd" / "examples" / example_dir / f"{variant}_expected.json" - - pa_schema = generate_pyarrow_schema(schema, xml_tag, max_recursion_depth=2) - elements = _extract_matching_elements(xml_path, xml_tag) - rows = [convert_xml_element(elem, schema, xml_tag, max_recursion_depth=2) for elem in elements] - - expected = json.loads(expected_path.read_text(encoding="utf-8")) - serialized_rows = json.loads(json.dumps(rows, default=_json_default)) - assert serialized_rows == expected - - table = pa.Table.from_pylist(rows, schema=pa_schema) - table.validate(full=True) - assert table.num_rows == 1 - - dumped_rows = json.loads(json.dumps(_strip_none(table.to_pylist()), default=_json_default)) - assert dumped_rows == expected - - buf = io.BytesIO() - pq.write_table(table, buf) - buf.seek(0) - roundtrip_table = pq.read_table(buf) - assert roundtrip_table.num_rows == 1 - roundtrip_rows = json.loads(json.dumps(_strip_none(roundtrip_table.to_pylist()), default=_json_default)) - assert roundtrip_rows == expected + with pytest.raises(RecursiveNonJsonTypeError, match="recursive type"): + generate_pyarrow_schema(xmlschema.XMLSchema(xsd_text), "root") diff --git a/tests/converter_spec.py b/tests/converter_spec.py new file mode 100644 index 00000000..44772b29 --- /dev/null +++ b/tests/converter_spec.py @@ -0,0 +1,188 @@ +"""Load converter specification manifests: rule catalogues, per-case manifests and error manifests. + +A converter specification lives next to its test fixtures: + +- ``rules.yaml``: rule catalogue (sections of rule IDs and statements). +- ``//case.yaml``: one manifest per fixture directory listing the rules it exercises and its runs. +- ``/errors/manifest.yaml``: error fixtures mapped to error codes. + +The same manifests drive the specification tests and the generated specification document. +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +CASE_MANIFEST = "case.yaml" +ERROR_MANIFEST = "manifest.yaml" + + +@dataclass(frozen=True, slots=True) +class Rule: + """A single conversion rule.""" + + id: str + statement: str + section_id: str + + +@dataclass(frozen=True, slots=True) +class Section: + """A group of rules rendered under one heading.""" + + id: str + title: str + intro: str + generated: str | None + rules: tuple[Rule, ...] + + +@dataclass(frozen=True, slots=True) +class RuleCatalogue: + """All rules for one converter.""" + + title: str + intro: str + sections: tuple[Section, ...] + + @property + def rules(self) -> dict[str, Rule]: + """Map rule ID to rule, in catalogue order.""" + return {rule.id: rule for section in self.sections for rule in section.rules} + + +@dataclass(frozen=True, slots=True) +class Run: + """One execution of a case: input data, parameters and expected outputs.""" + + id: str + data: Path + expected_json: Path + expected_table: Path + xml_tag: str | None + params: dict[str, Any] = field(default_factory=dict) + schema_attr: str = "EXPECTED_SCHEMA" + table_attr: str = "EXPECTED_TABLE" + expected_count: int | None = None + + +@dataclass(frozen=True, slots=True) +class Case: + """A fixture directory with its manifest.""" + + suite: str + name: str + directory: Path + title: str + schema: Path + expected_schema: Path + xml_tag: str | None + rules: tuple[str, ...] + notes: str + runs: tuple[Run, ...] + + @property + def anchor(self) -> str: + """Stable HTML anchor for the case.""" + return f"case-{self.suite}-{self.name}".replace("_", "-").lower() + + +@dataclass(frozen=True, slots=True) +class ErrorCase: + """An input that must be rejected with a specific error code.""" + + file: Path + code: str + match: str + description: str + xml_tag: str | None + + +def _load_yaml(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as fh: + loaded = yaml.safe_load(fh) + if not isinstance(loaded, dict): + msg = f"{path}: expected a mapping at the top level" + raise TypeError(msg) + return loaded + + +def load_rules(path: Path) -> RuleCatalogue: + """Load a ``rules.yaml`` catalogue, rejecting duplicate rule IDs.""" + raw = _load_yaml(path) + seen: set[str] = set() + sections: list[Section] = [] + for raw_section in raw["sections"]: + rules: list[Rule] = [] + for raw_rule in raw_section.get("rules", []): + rule_id = raw_rule["id"] + if rule_id in seen: + msg = f"{path}: duplicate rule id {rule_id!r}" + raise ValueError(msg) + seen.add(rule_id) + rules.append(Rule(id=rule_id, statement=raw_rule["statement"].strip(), section_id=raw_section["id"])) + sections.append( + Section( + id=raw_section["id"], + title=raw_section["title"], + intro=raw_section.get("intro", "").strip(), + generated=raw_section.get("generated"), + rules=tuple(rules), + ) + ) + return RuleCatalogue(title=raw["title"], intro=raw.get("intro", "").strip(), sections=tuple(sections)) + + +def _load_run(directory: Path, raw: dict[str, Any], case_xml_tag: str | None) -> Run: + return Run( + id=raw["id"], + data=(directory / raw["data"]).resolve(), + expected_json=(directory / raw.get("expected_json", "expected.json")).resolve(), + expected_table=(directory / raw.get("expected_table", "expected_table.py")).resolve(), + xml_tag=raw.get("xml_tag", case_xml_tag), + params=dict(raw.get("params", {})), + schema_attr=raw.get("schema_attr", "EXPECTED_SCHEMA"), + table_attr=raw.get("table_attr", "EXPECTED_TABLE"), + expected_count=raw.get("expected_count"), + ) + + +def load_case(manifest: Path, suite: str) -> Case: + """Load one ``case.yaml`` manifest.""" + raw = _load_yaml(manifest) + directory = manifest.parent + xml_tag = raw.get("xml_tag") + return Case( + suite=suite, + name=directory.name, + directory=directory, + title=raw["title"], + schema=(directory / raw["schema"]).resolve(), + expected_schema=(directory / raw.get("expected_schema", "expected_schema.py")).resolve(), + xml_tag=xml_tag, + rules=tuple(raw.get("rules", [])), + notes=raw.get("notes", "").strip(), + runs=tuple(_load_run(directory, raw_run, xml_tag) for raw_run in raw["runs"]), + ) + + +def load_cases(suite_dir: Path) -> list[Case]: + """Load every ``/*/case.yaml`` manifest, sorted by directory name.""" + return [load_case(manifest, suite_dir.name) for manifest in sorted(suite_dir.glob(f"*/{CASE_MANIFEST}"))] + + +def load_error_cases(manifest: Path) -> list[ErrorCase]: + """Load an error ``manifest.yaml``.""" + raw = _load_yaml(manifest) + return [ + ErrorCase( + file=(manifest.parent / entry["file"]).resolve(), + code=entry["code"], + match=entry["match"], + description=entry.get("description", "").strip(), + xml_tag=entry.get("xml_tag"), + ) + for entry in raw["errors"] + ] diff --git a/tests/data/jsonschema/examples/cts_job/case.yaml b/tests/data/jsonschema/examples/cts_job/case.yaml new file mode 100644 index 00000000..9efcf072 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/case.yaml @@ -0,0 +1,17 @@ +title: KBase CTS `Job` +schema: schema.json +rules: [R-UNION-SCALAR, R-NULL-ALLOWS-NULL, R-SHAPE-MAP, R-SHAPE-REF] +notes: | + `/jobs/{job_id}` response schema from `not_for_github/cts/openapi.json` (`#/components/schemas/Job`), with nested + `JobInput`, `JobImage`, `RefData` / `S3FileWithDataID`, a dynamic `params` map and `anyOf` nullability. +runs: + - id: minimal + data: minimal.json + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.json + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 2 diff --git a/tests/data/jsonschema/examples/cts_job/expected_schema.py b/tests/data/jsonschema/examples/cts_job/expected_schema.py new file mode 100644 index 00000000..91603803 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/expected_schema.py @@ -0,0 +1,635 @@ +"""Expected PyArrow schema for real-world example cts_job.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "outputs", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "file", + pa.string(), + nullable=False, + metadata={ + "comment": "A path to an object in an S3 instance, starting with the bucket. Please note that spaces are valid S3 object characters, so be careful about trailing spaces in the input.", + "jsonschema_type": "string", + }, + ), + pa.field( + "crc64nvme", + pa.string(), + nullable=True, + metadata={ + "comment": "The base64 encoded CRC64/NVME checksum of the file. If provided on input it is checked against the target file checksum before proceeding. Always provided with output.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=False, + metadata={"comment": "A file in an S3 instance. "}, + ) + ), + nullable=True, + ), + pa.field( + "trans_history", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "state", + pa.string(), + nullable=False, + metadata={"comment": "The state of a job.", "jsonschema_type": "string"}, + ), + pa.field( + "time", + pa.string(), + nullable=False, + metadata={ + "comment": "The time at which the new state was entered.", + "jsonschema_type": "string (date-time)", + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Denotes the new state and the entry time when a state change occurs."}, + ) + ), + nullable=True, + metadata={ + "comment": "Historical state transition records from previous recovery attempts. Populated when the job is reset during job recovery." + }, + ), + pa.field( + "error", + pa.string(), + nullable=True, + metadata={"comment": "A description of the error that occurred.", "jsonschema_type": "string"}, + ), + pa.field( + "output_file_count", + pa.int64(), + nullable=True, + metadata={"comment": "The number of output files, if available.", "jsonschema_type": "integer"}, + ), + pa.field( + "id", + pa.string(), + nullable=False, + metadata={"comment": "An opaque, unique string that serves as the job's ID.", "jsonschema_type": "string"}, + ), + pa.field( + "state", + pa.string(), + nullable=False, + metadata={"comment": "The state of a job.", "jsonschema_type": "string"}, + ), + pa.field( + "transition_times", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "state", + pa.string(), + nullable=False, + metadata={"comment": "The state of a job.", "jsonschema_type": "string"}, + ), + pa.field( + "time", + pa.string(), + nullable=False, + metadata={ + "comment": "The time at which the new state was entered.", + "jsonschema_type": "string (date-time)", + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Denotes the new state and the entry time when a state change occurs."}, + ) + ), + nullable=False, + metadata={"comment": "A list of job state transitions."}, + ), + pa.field( + "user", + pa.string(), + nullable=False, + metadata={"comment": "The user that ran the job.", "jsonschema_type": "string"}, + ), + pa.field( + "admin_meta", + pa.map_( + pa.string(), + pa.field( + "value", pa.string(), nullable=False, metadata={"jsonschema_type": "string | integer | number"} + ), + ), + nullable=True, + metadata={ + "comment": "Metadata for the job set by service administrators. This metadata is mutable and arbitrary from the point of view of the service." + }, + ), + pa.field( + "job_input", + pa.struct( + [ + pa.field( + "cluster", + pa.string(), + nullable=False, + metadata={ + "comment": "All clusters that have ever been registered in the system, including those that are no longer\nregistered but may have associated job records.\n\nperlmutter-jaws: The Perlmutter cluster at NERSC run via JAWS. \nlawrencium-jaws: The Lawrencium cluster at LBNL run via JAWS. \nkbase: KBase compute nodes.", + "jsonschema_type": "string", + }, + ), + pa.field( + "image", + pa.string(), + nullable=False, + metadata={ + "comment": "The Docker image to run for the job. Include the SHA to ensure the exact code requested is run.", + "jsonschema_type": "string", + }, + ), + pa.field( + "params", + pa.struct( + [ + pa.field( + "input_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "Where to place input files in the container. Must start from the container root and include at least one directory when resolved.", + "jsonschema_type": "string", + }, + ), + pa.field( + "output_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "Where output files should be written in the container. Must start from the container root and include at least one directory when resolved. Note that the mount point is not included in the file paths thatare eventually transferred to S3.", + "jsonschema_type": "string", + }, + ), + pa.field( + "declobber", + pa.bool_(), + nullable=True, + metadata={ + "comment": "If true, prepend the container number to all output file paths as a directory. This prevents containers that write to the same file path from clobbering each other and losing data. If the output files of containers are guaranteed to be unique (e.g. output file names are based on input file names and input file names are unique) then this setting is not necessary. The same effect can be created by use of a container number parameter when specifying the output directory in the arguments or environment.", + "jsonschema_type": "boolean", + }, + ), + pa.field( + "refdata_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "Where reference data files should be pleased in the container. Must start from the container root and include at least one directory when resolved. Required if the job image requires reference data and a default reference data mount point isn't specified for the image. If a default is provided, this setting overrides it.", + "jsonschema_type": "string", + }, + ), + pa.field( + "args", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "type", + pa.string(), + nullable=True, + metadata={ + "comment": "The type of a parameter if not a string literal.\n\ninput_files will cause the list of input files for the container to be inserted\ninto the entrypoint command line or an environment variable.\n\nmanifest_file will cause the list of input files for the container to be placed in a\nmanifest file and will insert the file location into the entrypoint command line\nor an environment variable.\n\ncontainer_number will insert the integer number of the container into the\nentrypoint command line or an environment variable. This can be used so prevent file path\ncollisions if multiple containers write to the same file paths.", + "jsonschema_type": "string", + }, + ), + pa.field( + "input_files_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of an input files type parameter.\n\nComma and space separated lists format the list of files as one would expect.\n\nRepeat parameter causes the input files to be inserted as a space separated list,\nwith each file preceded by the parameter flag. Only valid for CLI flag based arguments.\nIf the flag ends with an equals sign, there is no space left between the flag and the\nparameter.\n\nExamples: \n`-i`: `-i file1 -i file2 ... -i fileN` \n`--input_file=`: `--input_file=file1 --input_file=file2 ... --input_file=fileN` ", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of a manifest file.\n\nfiles will cause the manifest file to consist of a list of the input files, one per line.\n\ndata_ids will cause the manifest file to consist of a list of the data IDs associated with\nthe input files, one per line.", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_header", + pa.string(), + nullable=True, + metadata={ + "comment": "The header for the manifest file, if any.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_prefix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be prepended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_suffix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be appended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "flag", + pa.string(), + nullable=True, + metadata={ + "comment": "A command line argument flag to precede the parameter. If the flag ends with '=', no space is left between the flag and its value.", + "jsonschema_type": "string", + }, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + metadata={ + "comment": "A list of command line arguments to be inserted after the container entrypoint command. Strings are treated as literals." + }, + ), + pa.field( + "environment", + pa.map_( + pa.string(), + pa.field( + "value", + pa.struct( + [ + pa.field( + "type", + pa.string(), + nullable=True, + metadata={ + "comment": "The type of a parameter if not a string literal.\n\ninput_files will cause the list of input files for the container to be inserted\ninto the entrypoint command line or an environment variable.\n\nmanifest_file will cause the list of input files for the container to be placed in a\nmanifest file and will insert the file location into the entrypoint command line\nor an environment variable.\n\ncontainer_number will insert the integer number of the container into the\nentrypoint command line or an environment variable. This can be used so prevent file path\ncollisions if multiple containers write to the same file paths.", + "jsonschema_type": "string", + }, + ), + pa.field( + "input_files_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of an input files type parameter.\n\nComma and space separated lists format the list of files as one would expect.\n\nRepeat parameter causes the input files to be inserted as a space separated list,\nwith each file preceded by the parameter flag. Only valid for CLI flag based arguments.\nIf the flag ends with an equals sign, there is no space left between the flag and the\nparameter.\n\nExamples: \n`-i`: `-i file1 -i file2 ... -i fileN` \n`--input_file=`: `--input_file=file1 --input_file=file2 ... --input_file=fileN` ", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of a manifest file.\n\nfiles will cause the manifest file to consist of a list of the input files, one per line.\n\ndata_ids will cause the manifest file to consist of a list of the data IDs associated with\nthe input files, one per line.", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_header", + pa.string(), + nullable=True, + metadata={ + "comment": "The header for the manifest file, if any.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_prefix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be prepended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_suffix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be appended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + metadata={ + "comment": "A dictionary of environment variables to be inserted into the container. String values are treated as literals." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "A set of parameters for a container in a job.\n\nSpecifying files as part of arguments to a job can be done in three ways:\n\n* If the entrypoint command takes a directory as an argument, that\n directory can be specified literally in an argument. All files will be available in the\n input mount point, with paths resulting from the path in S3 as well as the input_roots\n parameter from the job input.\n* If individual filenames need to be specified, use a Parameter instance in\n the positional or environmental arguments. At most one Parameter can\n be specified per parameter set.\n* A Parameter instance can also be used to specify a manifest of file names or\n file IDs to be passed to the container." + }, + ), + pa.field( + "num_containers", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of containers to run in parallel. Input files will be split between the containers.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "cpus", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of CPUs to allocate per container.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "gpus", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of GPUs to allocate per container. Jobs requesting 0 GPUs are never scheduled on GPU-capable nodes.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "memory", + pa.int64(), + nullable=True, + metadata={ + "comment": "The amount of memory to allocate per container either as the number of bytes or a specification string such as 100MB, 2GB, etc.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "runtime", + pa.string(), + nullable=True, + metadata={ + "comment": "The runtime required for each container as the number of seconds or an ISO8601 duration string.", + "jsonschema_type": "string (duration)", + }, + ), + pa.field( + "input_roots", + pa.list_(pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"})), + nullable=True, + metadata={ + "comment": 'If specified, preserves file hierarchies for S3 files below the given root paths, starting from the bucket. Any files that are not prefixed by a root path are placed in the root directory of the job input dir. If any input files have the same path in the input dir, the job fails. For example, given a input_roots entry of ["mybucket/foo/bar"] and the input files ["otherbucket/foo", "mybucket/bar", "mybucket/foo/bar/baz/bat"] the job input directory would include the files foo, bar, and baz/bat. To preserve hierarchies for all files, set input_roots to [""]. If an input file matches more than one root, the longest root that isn\'t identical to the file path is used. Whitespace on the left side of the path is is ignored. Duplicate roots are ignored.' + }, + ), + pa.field( + "output_dir", + pa.string(), + nullable=False, + metadata={ + "comment": "The S3 folder, starting with the bucket, in which to place results.", + "jsonschema_type": "string", + }, + ), + pa.field( + "input_files", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "file", + pa.string(), + nullable=False, + metadata={ + "comment": "A path to an object in an S3 instance, starting with the bucket. Please note that spaces are valid S3 object characters, so be careful about trailing spaces in the input.", + "jsonschema_type": "string", + }, + ), + pa.field( + "crc64nvme", + pa.string(), + nullable=True, + metadata={ + "comment": "The base64 encoded CRC64/NVME checksum of the file. If provided on input it is checked against the target file checksum before proceeding. Always provided with output.", + "jsonschema_type": "string", + }, + ), + pa.field( + "data_id", + pa.string(), + nullable=True, + metadata={ + "comment": "An arbitrary string representing the ID of the data in the file.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=False, + metadata={"comment": "A file in an S3 instance, optionally with a data ID. "}, + ) + ), + nullable=False, + metadata={ + "comment": "The S3 input files for the job, either a list of file paths as strings or a list of data structures including the file path and optionally a data ID and / or CRC64/NVME checksum. The file paths always start with the bucket. All S3 input files are expected to have a CRC64/NVME checksum available, even if it is not provided in the input data structure. Either all or no files must have data IDs associated with them. When returned from the service, the checksum is always included." + }, + ), + pa.field( + "script", + pa.struct( + [ + pa.field( + "file", + pa.string(), + nullable=False, + metadata={ + "comment": "A path to an object in an S3 instance, starting with the bucket. Please note that spaces are valid S3 object characters, so be careful about trailing spaces in the input.", + "jsonschema_type": "string", + }, + ), + pa.field( + "crc64nvme", + pa.string(), + nullable=True, + metadata={ + "comment": "The base64 encoded CRC64/NVME checksum of the file. If provided on input it is checked against the target file checksum before proceeding. Always provided with output.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=True, + metadata={ + "comment": "A user-provided script, either as a file path string or a data structure including the file path and optionally a CRC64/NVME checksum. The file path always starts with the bucket. The script is expected to have a CRC64/NVME checksum available, even if it is not provided in the input data structure. When returned from the service, the checksum is always included. The script is placed at the root of the input mount point for every container as `.__script__` - as such, the input mount point chosen for the job must be compatible with the image entrypoint. It is the responsibility of the image entrypoint to run the script. Only supported at the `kbase` site, and only for images that allow script execution." + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Input to a Job."}, + ), + pa.field( + "image", + pa.struct( + [ + pa.field( + "registered_by", + pa.string(), + nullable=False, + metadata={ + "comment": "The username of the user that performed the registration.", + "jsonschema_type": "string", + }, + ), + pa.field( + "registered_on", + pa.string(), + nullable=False, + metadata={"comment": "The time of registration.", "jsonschema_type": "string (date-time)"}, + ), + pa.field( + "name", + pa.string(), + nullable=False, + metadata={ + "comment": "The normalized name of the a docker image, consisting of the host and path.", + "jsonschema_type": "string", + }, + ), + pa.field( + "digest", + pa.string(), + nullable=False, + metadata={"comment": "The image digest.", "jsonschema_type": "string"}, + ), + pa.field( + "entrypoint", + pa.list_(pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"})), + nullable=False, + metadata={"comment": "The entrypoint command extracted from the image."}, + ), + pa.field( + "tag", + pa.string(), + nullable=True, + metadata={ + "comment": "The image tag at registration time. The tag may no longer point to the same image.", + "jsonschema_type": "string", + }, + ), + pa.field( + "refdata_id", + pa.string(), + nullable=True, + metadata={ + "comment": "The ID of reference data associated with the image. The image requires this reference data to run, and it will be mounted into the image container at the refdata mount point.", + "jsonschema_type": "string", + }, + ), + pa.field( + "default_refdata_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "The default mount point for the image refdata in the container. Overridden by the refdata mount point in a job submission if provided there. If a default mount point is provided, then providing a mount point in the job submission is optional. Must be an absolute path.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=False, + metadata={"comment": "Information about a Docker image as stored in a Job data structure."}, + ), + pa.field( + "input_file_count", + pa.int64(), + nullable=False, + metadata={"comment": "The number of input files.", "jsonschema_type": "integer"}, + ), + pa.field( + "cpu_hours", + pa.float64(), + nullable=True, + metadata={"comment": "The total CPU hours used by the job, if available.", "jsonschema_type": "number"}, + ), + pa.field( + "cpu_factor", + pa.float64(), + nullable=True, + metadata={ + "comment": "The ratio of the cpu time used to the number of cpus requested times the job's actual time, if available. Perfect usage is 1, < 1 means requested CPUs are partially idle, possibly due to memory or IO bounded work, > 1 means more CPU usage than requested.", + "jsonschema_type": "number", + }, + ), + pa.field( + "max_memory", + pa.int64(), + nullable=True, + metadata={ + "comment": "The maximum memory in bytes used by a single container in the job, if available.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "logpath", + pa.string(), + nullable=True, + metadata={ + "comment": "A location in S3 where the logfiles for the job containers can be viewed.", + "jsonschema_type": "string", + }, + ), + ] +) diff --git a/tests/data/jsonschema/examples/cts_job/maximal.json b/tests/data/jsonschema/examples/cts_job/maximal.json new file mode 100644 index 00000000..a5ef3cd0 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/maximal.json @@ -0,0 +1,748 @@ +[ + { + "outputs": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/log_file.txt", + "crc64nvme": "YetpCoji89E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/report_1792766652.4976728__2026-10-23 15:41:54.557359+00:00.txt", + "crc64nvme": "oFlF2ps4FRA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/init", + "crc64nvme": "Bb62uECKpvw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_pipeline_state/uniref__1792766652.4976728__219847d34c216fd49eaa0a29b5685af1527b4cb7961bd786a5e26b3d2a3948ee.jsonl", + "crc64nvme": "Oc/EWEbBI24=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_loads/uniref__1792766652.4976728.jsonl", + "crc64nvme": "H1xMKkQHOFY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_loads/uniref__1792770114.583901.jsonl", + "crc64nvme": "QK7ttbxhQqU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.813434744e.parquet", + "crc64nvme": "z9Cvo3R6sX4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.51a72d392c.parquet", + "crc64nvme": "B8ha5S1/xXE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.d83e2c6f87.parquet", + "crc64nvme": "Rfy09JuYVeU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.8e924d979f.parquet", + "crc64nvme": "Ptx4ZtzfrI4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.92504fbf37.parquet", + "crc64nvme": "QY5cXWS8Kuc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_version/uniref__1792770114.5197887__219847d34c216fd49eaa0a29b5685af1527b4cb7961bd786a5e26b3d2a3948ee.jsonl", + "crc64nvme": "lxnoFj56GaU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_version/uniref__1792770114.6481373__eb434791363e8c934022b6faf778c49b4125d350419c4f9dd2e04791906a5393.jsonl", + "crc64nvme": "6M+qilyqvxc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_load_info/1792770114.583901.443a8320dd.jsonl.gz", + "crc64nvme": "D80ZjtzgUK0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/state.json", + "crc64nvme": "Iwe4P8J1Af0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/trace.pickle", + "crc64nvme": "hN5641iL3Hs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/schemas/uniref.schema.json", + "crc64nvme": "8H8HIYpB1Qg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/normalize/.version", + "crc64nvme": "zOeiWdY8BCE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/.version", + "crc64nvme": "TVSzAYZin6o=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/load_package_state.json", + "crc64nvme": "KTHoLjwSTAM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/schema.json", + "crc64nvme": "5bTldziTvNc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/applied_schema_updates.json", + "crc64nvme": "4XFncxoZtjY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/package_completed.json", + "crc64nvme": "QRhACZr0J8A=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/load_package_state.json", + "crc64nvme": "Zv61rVvcMbs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/schema.json", + "crc64nvme": "X8cM63TM3kg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/applied_schema_updates.json", + "crc64nvme": "JhddfQZvrCU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/package_completed.json", + "crc64nvme": "QRhACZr0J8A=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.813434744e.0.parquet", + "crc64nvme": "z9Cvo3R6sX4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/_dlt_pipeline_state.b1dbfcdc2a.0.jsonl.gz", + "crc64nvme": "jb/PpRfGC3I=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.51a72d392c.0.parquet", + "crc64nvme": "B8ha5S1/xXE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.d83e2c6f87.0.parquet", + "crc64nvme": "Rfy09JuYVeU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.8e924d979f.0.parquet", + "crc64nvme": "Ptx4ZtzfrI4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.92504fbf37.0.parquet", + "crc64nvme": "QY5cXWS8Kuc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/.restore/state.json", + "crc64nvme": "h+ukr8cdheA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/completed_jobs/_load_info.443a8320dd.0.jsonl.gz", + "crc64nvme": "D80ZjtzgUK0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/.restore/state.json", + "crc64nvme": "Iwe4P8J1Af0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/.restore/schemas/uniref.schema.json", + "crc64nvme": "nCdltW/rRYY=" + } + ], + "output_file_count": 37, + "id": "86844795-67ed-4d4b-b981-5eb482f499c6", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:50.339000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:50.367000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:44:10.612000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:44:11.279000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:41:55.483000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:41:55.551000Z" + }, + { + "state": "complete", + "time": "2026-10-23T15:42:57.859000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:43:04.780730+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:43:04.943741+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00901.xml.gz", + "crc64nvme": "I7ZvxE4/2Cs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00902.xml.gz", + "crc64nvme": "PShwZ1YTS1M=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00903.xml.gz", + "crc64nvme": "uhlsDw2jnyI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00904.xml.gz", + "crc64nvme": "LFnossLcgko=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00905.xml.gz", + "crc64nvme": "kyZyDfn1yBM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00906.xml.gz", + "crc64nvme": "pRe+89Ajp6w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00907.xml.gz", + "crc64nvme": "MDygV6n+IUg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00908.xml.gz", + "crc64nvme": "gOE6kyBqsOA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00909.xml.gz", + "crc64nvme": "4DqQGjF2uN4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00910.xml.gz", + "crc64nvme": "0NDUxs3D/KA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00911.xml.gz", + "crc64nvme": "ErPRtYHG0qs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00912.xml.gz", + "crc64nvme": "QIkR+wNHjFE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00913.xml.gz", + "crc64nvme": "yskjnjIP9ZE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00914.xml.gz", + "crc64nvme": "xiFhSIJmzkM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00915.xml.gz", + "crc64nvme": "LqcSEL2Flgw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00916.xml.gz", + "crc64nvme": "toI46/7cP6w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00917.xml.gz", + "crc64nvme": "f+i+lTDBR1U=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00918.xml.gz", + "crc64nvme": "9F8hdpcZrFs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00919.xml.gz", + "crc64nvme": "DeE6y166rZw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00920.xml.gz", + "crc64nvme": "v/XLAIPhlPk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00921.xml.gz", + "crc64nvme": "4mwGQwn35WY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00922.xml.gz", + "crc64nvme": "RDXOBgsURrQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00923.xml.gz", + "crc64nvme": "s0Xa33+zjU0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00924.xml.gz", + "crc64nvme": "ZX76o6BbG+k=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00925.xml.gz", + "crc64nvme": "fv+e0Ap5xq4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00926.xml.gz", + "crc64nvme": "nMvPKcQsgG8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00927.xml.gz", + "crc64nvme": "xtGxOC71tYc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00928.xml.gz", + "crc64nvme": "FwyVDrJ7UJE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00929.xml.gz", + "crc64nvme": "XKjQf39LCDc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00930.xml.gz", + "crc64nvme": "YVBpJMjTF0Q=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00931.xml.gz", + "crc64nvme": "t99Jn6AdHzc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00932.xml.gz", + "crc64nvme": "AZ+5HCuHvxA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00933.xml.gz", + "crc64nvme": "JzSAx/2bLWk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00934.xml.gz", + "crc64nvme": "hsb116dQ6rI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00935.xml.gz", + "crc64nvme": "dIMvpUazcyw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00936.xml.gz", + "crc64nvme": "Vb4NZPsUSws=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00937.xml.gz", + "crc64nvme": "zyX6aKkq7jk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00938.xml.gz", + "crc64nvme": "O0UomXy622c=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00939.xml.gz", + "crc64nvme": "/znK28KJTjk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00940.xml.gz", + "crc64nvme": "Ul7txpVB4cQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00941.xml.gz", + "crc64nvme": "mgxRdhEvuOg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00942.xml.gz", + "crc64nvme": "SVWekta8l0c=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00943.xml.gz", + "crc64nvme": "2sd+HrUemIs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00944.xml.gz", + "crc64nvme": "3Di/rXN8NWA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00945.xml.gz", + "crc64nvme": "DeP+sg5WtTk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00946.xml.gz", + "crc64nvme": "afn9IyJ7rWg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00947.xml.gz", + "crc64nvme": "wOFqkr18rE8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00948.xml.gz", + "crc64nvme": "/YoD8rlxIDs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00949.xml.gz", + "crc64nvme": "rrfQfh8sViQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00950.xml.gz", + "crc64nvme": "cKgbZTEvoEY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00951.xml.gz", + "crc64nvme": "fRIDwpT8ZY4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00952.xml.gz", + "crc64nvme": "VXxAVmcmIUY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00953.xml.gz", + "crc64nvme": "rXgY1zN0LBg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00954.xml.gz", + "crc64nvme": "2OWmJ0BtuMI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00955.xml.gz", + "crc64nvme": "rHT2/UmnK3w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00956.xml.gz", + "crc64nvme": "VaUSc6zE3tg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00957.xml.gz", + "crc64nvme": "GBcwCd5LByI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00958.xml.gz", + "crc64nvme": "Zyre71e/Vd8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00959.xml.gz", + "crc64nvme": "P7emDdBJ6S8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00960.xml.gz", + "crc64nvme": "pXxHRBlVN8E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00961.xml.gz", + "crc64nvme": "IvDue8zTBaA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00962.xml.gz", + "crc64nvme": "oq8cuWxCpWs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00963.xml.gz", + "crc64nvme": "s3MU8QHgzdo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00964.xml.gz", + "crc64nvme": "dNyIkdkJD68=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00965.xml.gz", + "crc64nvme": "Qmi310kWfrQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00966.xml.gz", + "crc64nvme": "waf6BC/NyNU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00967.xml.gz", + "crc64nvme": "P68CJd7KmGo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00968.xml.gz", + "crc64nvme": "OsmvTygQlKw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00969.xml.gz", + "crc64nvme": "7iyD21gjMsw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00970.xml.gz", + "crc64nvme": "CMUz+fM55Bs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00971.xml.gz", + "crc64nvme": "U4z1MVCayWo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00972.xml.gz", + "crc64nvme": "fR68wEfg+hU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00973.xml.gz", + "crc64nvme": "jl3JpfnijRc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00974.xml.gz", + "crc64nvme": "A51PgS8iUIU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00975.xml.gz", + "crc64nvme": "NAjQmGkKXPs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00976.xml.gz", + "crc64nvme": "FePD5NBKgCg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00977.xml.gz", + "crc64nvme": "OFs7PoZG/Q0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00978.xml.gz", + "crc64nvme": "seSyambXUdI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00979.xml.gz", + "crc64nvme": "xGBgEYfGGm4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00980.xml.gz", + "crc64nvme": "AJapve6eeUA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00981.xml.gz", + "crc64nvme": "Ut3dINq172I=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00982.xml.gz", + "crc64nvme": "eztQlsjZdu8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00983.xml.gz", + "crc64nvme": "fmMOg/cYAYo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00984.xml.gz", + "crc64nvme": "lz7dNy4S+jo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00985.xml.gz", + "crc64nvme": "VTSosmfTvJU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00986.xml.gz", + "crc64nvme": "8NgCDRCDd0E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00987.xml.gz", + "crc64nvme": "veAwqrRhzbM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00988.xml.gz", + "crc64nvme": "iCJnOeDZ8Lc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00989.xml.gz", + "crc64nvme": "SstUmtV9x8s=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00990.xml.gz", + "crc64nvme": "hvdb09Q3R1E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00991.xml.gz", + "crc64nvme": "u1eoNuvR2bk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00992.xml.gz", + "crc64nvme": "mfyfoapFs4A=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00993.xml.gz", + "crc64nvme": "GNBAMAfiQHs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00994.xml.gz", + "crc64nvme": "YFwQTzM/WnA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00995.xml.gz", + "crc64nvme": "NU7s7gDnsyI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00996.xml.gz", + "crc64nvme": "emyRjNDoDcw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00997.xml.gz", + "crc64nvme": "oF6O3t+xBmo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00998.xml.gz", + "crc64nvme": "thNC/o2Qq+U=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00999.xml.gz", + "crc64nvme": "Ve0WPqO300w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_01000.xml.gz", + "crc64nvme": "AmKF7K3jDe4=" + }, + { + "file": "cts/io/ialarmedalien/misc/logger_config.json", + "crc64nvme": "ImY6G9xmYs4=" + } + ] + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9685900804811111, + "cpu_factor": 0.09150890887030012, + "max_memory": 4710187008 + }, + { + "error": "The job produced no output files", + "id": "253089e2-7ad1-4e32-a9fe-11aabda31c62", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-22T04:27:42.903000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-22T04:27:42.988000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-22T04:27:52.802000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-22T04:28:31.509000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-22T04:28:33.503000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-22T04:28:33.560000Z" + }, + { + "state": "error", + "time": "2026-10-22T04:29:28.225000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "uniprot", + "-i", + "/input_dir", + "-o", + "/output_dir", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/batch_001/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00001.xml.gz", + "crc64nvme": "AFAZfDzbbfc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00002.xml.gz", + "crc64nvme": "ZpZSu7BcMGs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00003.xml.gz", + "crc64nvme": "q8rI7iaPk50=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00004.xml.gz", + "crc64nvme": "ea4sVxZ8JFI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00005.xml.gz", + "crc64nvme": "KPrvcCVDlwM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00006.xml.gz", + "crc64nvme": "XWV3eNCm6hY=" + }, + { + "file": "cts/io/ialarmedalien/misc/logger_config.json", + "crc64nvme": "ImY6G9xmYs4=" + } + ] + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-04-26T15:46:11.342000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.8" + }, + "input_file_count": 7, + "cpu_hours": 0.0023639620611111112, + "cpu_factor": 0.4487487295293731, + "max_memory": 116162560 + } +] diff --git a/tests/data/jsonschema/examples/cts_job/maximal_expected.json b/tests/data/jsonschema/examples/cts_job/maximal_expected.json new file mode 100644 index 00000000..b4211d71 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/maximal_expected.json @@ -0,0 +1,790 @@ +[ + { + "outputs": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/log_file.txt", + "crc64nvme": "YetpCoji89E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/report_1792766652.4976728__2026-10-23 15:41:54.557359+00:00.txt", + "crc64nvme": "oFlF2ps4FRA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/init", + "crc64nvme": "Bb62uECKpvw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_pipeline_state/uniref__1792766652.4976728__219847d34c216fd49eaa0a29b5685af1527b4cb7961bd786a5e26b3d2a3948ee.jsonl", + "crc64nvme": "Oc/EWEbBI24=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_loads/uniref__1792766652.4976728.jsonl", + "crc64nvme": "H1xMKkQHOFY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_loads/uniref__1792770114.583901.jsonl", + "crc64nvme": "QK7ttbxhQqU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.813434744e.parquet", + "crc64nvme": "z9Cvo3R6sX4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.51a72d392c.parquet", + "crc64nvme": "B8ha5S1/xXE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.d83e2c6f87.parquet", + "crc64nvme": "Rfy09JuYVeU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.8e924d979f.parquet", + "crc64nvme": "Ptx4ZtzfrI4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.92504fbf37.parquet", + "crc64nvme": "QY5cXWS8Kuc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_version/uniref__1792770114.5197887__219847d34c216fd49eaa0a29b5685af1527b4cb7961bd786a5e26b3d2a3948ee.jsonl", + "crc64nvme": "lxnoFj56GaU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_version/uniref__1792770114.6481373__eb434791363e8c934022b6faf778c49b4125d350419c4f9dd2e04791906a5393.jsonl", + "crc64nvme": "6M+qilyqvxc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_load_info/1792770114.583901.443a8320dd.jsonl.gz", + "crc64nvme": "D80ZjtzgUK0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/state.json", + "crc64nvme": "Iwe4P8J1Af0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/trace.pickle", + "crc64nvme": "hN5641iL3Hs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/schemas/uniref.schema.json", + "crc64nvme": "8H8HIYpB1Qg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/normalize/.version", + "crc64nvme": "zOeiWdY8BCE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/.version", + "crc64nvme": "TVSzAYZin6o=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/load_package_state.json", + "crc64nvme": "KTHoLjwSTAM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/schema.json", + "crc64nvme": "5bTldziTvNc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/applied_schema_updates.json", + "crc64nvme": "4XFncxoZtjY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/package_completed.json", + "crc64nvme": "QRhACZr0J8A=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/load_package_state.json", + "crc64nvme": "Zv61rVvcMbs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/schema.json", + "crc64nvme": "X8cM63TM3kg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/applied_schema_updates.json", + "crc64nvme": "JhddfQZvrCU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/package_completed.json", + "crc64nvme": "QRhACZr0J8A=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.813434744e.0.parquet", + "crc64nvme": "z9Cvo3R6sX4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/_dlt_pipeline_state.b1dbfcdc2a.0.jsonl.gz", + "crc64nvme": "jb/PpRfGC3I=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.51a72d392c.0.parquet", + "crc64nvme": "B8ha5S1/xXE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.d83e2c6f87.0.parquet", + "crc64nvme": "Rfy09JuYVeU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.8e924d979f.0.parquet", + "crc64nvme": "Ptx4ZtzfrI4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.92504fbf37.0.parquet", + "crc64nvme": "QY5cXWS8Kuc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/.restore/state.json", + "crc64nvme": "h+ukr8cdheA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/completed_jobs/_load_info.443a8320dd.0.jsonl.gz", + "crc64nvme": "D80ZjtzgUK0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/.restore/state.json", + "crc64nvme": "Iwe4P8J1Af0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/.restore/schemas/uniref.schema.json", + "crc64nvme": "nCdltW/rRYY=" + } + ], + "output_file_count": 37, + "id": "86844795-67ed-4d4b-b981-5eb482f499c6", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:50.339000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:50.367000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:44:10.612000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:44:11.279000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:41:55.483000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:41:55.551000Z" + }, + { + "state": "complete", + "time": "2026-10-23T15:42:57.859000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:43:04.780730+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:43:04.943741+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00901.xml.gz", + "crc64nvme": "I7ZvxE4/2Cs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00902.xml.gz", + "crc64nvme": "PShwZ1YTS1M=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00903.xml.gz", + "crc64nvme": "uhlsDw2jnyI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00904.xml.gz", + "crc64nvme": "LFnossLcgko=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00905.xml.gz", + "crc64nvme": "kyZyDfn1yBM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00906.xml.gz", + "crc64nvme": "pRe+89Ajp6w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00907.xml.gz", + "crc64nvme": "MDygV6n+IUg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00908.xml.gz", + "crc64nvme": "gOE6kyBqsOA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00909.xml.gz", + "crc64nvme": "4DqQGjF2uN4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00910.xml.gz", + "crc64nvme": "0NDUxs3D/KA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00911.xml.gz", + "crc64nvme": "ErPRtYHG0qs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00912.xml.gz", + "crc64nvme": "QIkR+wNHjFE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00913.xml.gz", + "crc64nvme": "yskjnjIP9ZE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00914.xml.gz", + "crc64nvme": "xiFhSIJmzkM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00915.xml.gz", + "crc64nvme": "LqcSEL2Flgw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00916.xml.gz", + "crc64nvme": "toI46/7cP6w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00917.xml.gz", + "crc64nvme": "f+i+lTDBR1U=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00918.xml.gz", + "crc64nvme": "9F8hdpcZrFs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00919.xml.gz", + "crc64nvme": "DeE6y166rZw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00920.xml.gz", + "crc64nvme": "v/XLAIPhlPk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00921.xml.gz", + "crc64nvme": "4mwGQwn35WY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00922.xml.gz", + "crc64nvme": "RDXOBgsURrQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00923.xml.gz", + "crc64nvme": "s0Xa33+zjU0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00924.xml.gz", + "crc64nvme": "ZX76o6BbG+k=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00925.xml.gz", + "crc64nvme": "fv+e0Ap5xq4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00926.xml.gz", + "crc64nvme": "nMvPKcQsgG8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00927.xml.gz", + "crc64nvme": "xtGxOC71tYc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00928.xml.gz", + "crc64nvme": "FwyVDrJ7UJE=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00929.xml.gz", + "crc64nvme": "XKjQf39LCDc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00930.xml.gz", + "crc64nvme": "YVBpJMjTF0Q=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00931.xml.gz", + "crc64nvme": "t99Jn6AdHzc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00932.xml.gz", + "crc64nvme": "AZ+5HCuHvxA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00933.xml.gz", + "crc64nvme": "JzSAx/2bLWk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00934.xml.gz", + "crc64nvme": "hsb116dQ6rI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00935.xml.gz", + "crc64nvme": "dIMvpUazcyw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00936.xml.gz", + "crc64nvme": "Vb4NZPsUSws=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00937.xml.gz", + "crc64nvme": "zyX6aKkq7jk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00938.xml.gz", + "crc64nvme": "O0UomXy622c=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00939.xml.gz", + "crc64nvme": "/znK28KJTjk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00940.xml.gz", + "crc64nvme": "Ul7txpVB4cQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00941.xml.gz", + "crc64nvme": "mgxRdhEvuOg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00942.xml.gz", + "crc64nvme": "SVWekta8l0c=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00943.xml.gz", + "crc64nvme": "2sd+HrUemIs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00944.xml.gz", + "crc64nvme": "3Di/rXN8NWA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00945.xml.gz", + "crc64nvme": "DeP+sg5WtTk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00946.xml.gz", + "crc64nvme": "afn9IyJ7rWg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00947.xml.gz", + "crc64nvme": "wOFqkr18rE8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00948.xml.gz", + "crc64nvme": "/YoD8rlxIDs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00949.xml.gz", + "crc64nvme": "rrfQfh8sViQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00950.xml.gz", + "crc64nvme": "cKgbZTEvoEY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00951.xml.gz", + "crc64nvme": "fRIDwpT8ZY4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00952.xml.gz", + "crc64nvme": "VXxAVmcmIUY=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00953.xml.gz", + "crc64nvme": "rXgY1zN0LBg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00954.xml.gz", + "crc64nvme": "2OWmJ0BtuMI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00955.xml.gz", + "crc64nvme": "rHT2/UmnK3w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00956.xml.gz", + "crc64nvme": "VaUSc6zE3tg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00957.xml.gz", + "crc64nvme": "GBcwCd5LByI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00958.xml.gz", + "crc64nvme": "Zyre71e/Vd8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00959.xml.gz", + "crc64nvme": "P7emDdBJ6S8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00960.xml.gz", + "crc64nvme": "pXxHRBlVN8E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00961.xml.gz", + "crc64nvme": "IvDue8zTBaA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00962.xml.gz", + "crc64nvme": "oq8cuWxCpWs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00963.xml.gz", + "crc64nvme": "s3MU8QHgzdo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00964.xml.gz", + "crc64nvme": "dNyIkdkJD68=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00965.xml.gz", + "crc64nvme": "Qmi310kWfrQ=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00966.xml.gz", + "crc64nvme": "waf6BC/NyNU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00967.xml.gz", + "crc64nvme": "P68CJd7KmGo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00968.xml.gz", + "crc64nvme": "OsmvTygQlKw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00969.xml.gz", + "crc64nvme": "7iyD21gjMsw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00970.xml.gz", + "crc64nvme": "CMUz+fM55Bs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00971.xml.gz", + "crc64nvme": "U4z1MVCayWo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00972.xml.gz", + "crc64nvme": "fR68wEfg+hU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00973.xml.gz", + "crc64nvme": "jl3JpfnijRc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00974.xml.gz", + "crc64nvme": "A51PgS8iUIU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00975.xml.gz", + "crc64nvme": "NAjQmGkKXPs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00976.xml.gz", + "crc64nvme": "FePD5NBKgCg=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00977.xml.gz", + "crc64nvme": "OFs7PoZG/Q0=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00978.xml.gz", + "crc64nvme": "seSyambXUdI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00979.xml.gz", + "crc64nvme": "xGBgEYfGGm4=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00980.xml.gz", + "crc64nvme": "AJapve6eeUA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00981.xml.gz", + "crc64nvme": "Ut3dINq172I=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00982.xml.gz", + "crc64nvme": "eztQlsjZdu8=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00983.xml.gz", + "crc64nvme": "fmMOg/cYAYo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00984.xml.gz", + "crc64nvme": "lz7dNy4S+jo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00985.xml.gz", + "crc64nvme": "VTSosmfTvJU=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00986.xml.gz", + "crc64nvme": "8NgCDRCDd0E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00987.xml.gz", + "crc64nvme": "veAwqrRhzbM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00988.xml.gz", + "crc64nvme": "iCJnOeDZ8Lc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00989.xml.gz", + "crc64nvme": "SstUmtV9x8s=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00990.xml.gz", + "crc64nvme": "hvdb09Q3R1E=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00991.xml.gz", + "crc64nvme": "u1eoNuvR2bk=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00992.xml.gz", + "crc64nvme": "mfyfoapFs4A=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00993.xml.gz", + "crc64nvme": "GNBAMAfiQHs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00994.xml.gz", + "crc64nvme": "YFwQTzM/WnA=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00995.xml.gz", + "crc64nvme": "NU7s7gDnsyI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00996.xml.gz", + "crc64nvme": "emyRjNDoDcw=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00997.xml.gz", + "crc64nvme": "oF6O3t+xBmo=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00998.xml.gz", + "crc64nvme": "thNC/o2Qq+U=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00999.xml.gz", + "crc64nvme": "Ve0WPqO300w=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_01000.xml.gz", + "crc64nvme": "AmKF7K3jDe4=" + }, + { + "file": "cts/io/ialarmedalien/misc/logger_config.json", + "crc64nvme": "ImY6G9xmYs4=" + } + ] + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9685900804811111, + "cpu_factor": 0.09150890887030012, + "max_memory": 4710187008 + }, + { + "error": "The job produced no output files", + "id": "253089e2-7ad1-4e32-a9fe-11aabda31c62", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-22T04:27:42.903000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-22T04:27:42.988000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-22T04:27:52.802000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-22T04:28:31.509000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-22T04:28:33.503000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-22T04:28:33.560000Z" + }, + { + "state": "error", + "time": "2026-10-22T04:29:28.225000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "uniprot" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/batch_001/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00001.xml.gz", + "crc64nvme": "AFAZfDzbbfc=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00002.xml.gz", + "crc64nvme": "ZpZSu7BcMGs=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00003.xml.gz", + "crc64nvme": "q8rI7iaPk50=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00004.xml.gz", + "crc64nvme": "ea4sVxZ8JFI=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00005.xml.gz", + "crc64nvme": "KPrvcCVDlwM=" + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00006.xml.gz", + "crc64nvme": "XWV3eNCm6hY=" + }, + { + "file": "cts/io/ialarmedalien/misc/logger_config.json", + "crc64nvme": "ImY6G9xmYs4=" + } + ] + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-04-26T15:46:11.342000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.8" + }, + "input_file_count": 7, + "cpu_hours": 0.0023639620611111112, + "cpu_factor": 0.4487487295293731, + "max_memory": 116162560 + } +] diff --git a/tests/data/jsonschema/examples/cts_job/maximal_expected_table.py b/tests/data/jsonschema/examples/cts_job/maximal_expected_table.py new file mode 100644 index 00000000..0e140064 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/maximal_expected_table.py @@ -0,0 +1,706 @@ +"""Expected PyArrow table for maximal real-world example cts_job.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "outputs": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/log_file.txt", + "crc64nvme": "YetpCoji89E=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/report_1792766652.4976728__2026-10-23 15:41:54.557359+00:00.txt", + "crc64nvme": "oFlF2ps4FRA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/init", + "crc64nvme": "Bb62uECKpvw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_pipeline_state/uniref__1792766652.4976728__219847d34c216fd49eaa0a29b5685af1527b4cb7961bd786a5e26b3d2a3948ee.jsonl", + "crc64nvme": "Oc/EWEbBI24=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_loads/uniref__1792766652.4976728.jsonl", + "crc64nvme": "H1xMKkQHOFY=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_loads/uniref__1792770114.583901.jsonl", + "crc64nvme": "QK7ttbxhQqU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.813434744e.parquet", + "crc64nvme": "z9Cvo3R6sX4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.51a72d392c.parquet", + "crc64nvme": "B8ha5S1/xXE=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.d83e2c6f87.parquet", + "crc64nvme": "Rfy09JuYVeU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.8e924d979f.parquet", + "crc64nvme": "Ptx4ZtzfrI4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/entry/1792766652.4976728.92504fbf37.parquet", + "crc64nvme": "QY5cXWS8Kuc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_version/uniref__1792770114.5197887__219847d34c216fd49eaa0a29b5685af1527b4cb7961bd786a5e26b3d2a3948ee.jsonl", + "crc64nvme": "lxnoFj56GaU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_dlt_version/uniref__1792770114.6481373__eb434791363e8c934022b6faf778c49b4125d350419c4f9dd2e04791906a5393.jsonl", + "crc64nvme": "6M+qilyqvxc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/uniref_100/_load_info/1792770114.583901.443a8320dd.jsonl.gz", + "crc64nvme": "D80ZjtzgUK0=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/state.json", + "crc64nvme": "Iwe4P8J1Af0=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/trace.pickle", + "crc64nvme": "hN5641iL3Hs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/schemas/uniref.schema.json", + "crc64nvme": "8H8HIYpB1Qg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/normalize/.version", + "crc64nvme": "zOeiWdY8BCE=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/.version", + "crc64nvme": "TVSzAYZin6o=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/load_package_state.json", + "crc64nvme": "KTHoLjwSTAM=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/schema.json", + "crc64nvme": "5bTldziTvNc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/applied_schema_updates.json", + "crc64nvme": "4XFncxoZtjY=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/package_completed.json", + "crc64nvme": "QRhACZr0J8A=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/load_package_state.json", + "crc64nvme": "Zv61rVvcMbs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/schema.json", + "crc64nvme": "X8cM63TM3kg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/applied_schema_updates.json", + "crc64nvme": "JhddfQZvrCU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/package_completed.json", + "crc64nvme": "QRhACZr0J8A=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.813434744e.0.parquet", + "crc64nvme": "z9Cvo3R6sX4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/_dlt_pipeline_state.b1dbfcdc2a.0.jsonl.gz", + "crc64nvme": "jb/PpRfGC3I=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.51a72d392c.0.parquet", + "crc64nvme": "B8ha5S1/xXE=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.d83e2c6f87.0.parquet", + "crc64nvme": "Rfy09JuYVeU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.8e924d979f.0.parquet", + "crc64nvme": "Ptx4ZtzfrI4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/completed_jobs/entry.92504fbf37.0.parquet", + "crc64nvme": "QY5cXWS8Kuc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792766652.4976728/.restore/state.json", + "crc64nvme": "h+ukr8cdheA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/completed_jobs/_load_info.443a8320dd.0.jsonl.gz", + "crc64nvme": "D80ZjtzgUK0=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/.restore/state.json", + "crc64nvme": "Iwe4P8J1Af0=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/1792766652.4976728__2026-10-23 15:41:54.557359+00:00/uniref/load/loaded/1792770114.583901/.restore/schemas/uniref.schema.json", + "crc64nvme": "nCdltW/rRYY=", + }, + ], + "output_file_count": 37, + "id": "86844795-67ed-4d4b-b981-5eb482f499c6", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:43:50.339000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:43:50.367000Z"}, + {"state": "job_submitting", "time": "2026-10-23T14:44:10.612000Z"}, + {"state": "job_submitted", "time": "2026-10-23T14:44:11.279000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T15:41:55.483000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T15:41:55.551000Z"}, + {"state": "complete", "time": "2026-10-23T15:42:57.859000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:43:04.780730+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:43:04.943741+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00901.xml.gz", + "crc64nvme": "I7ZvxE4/2Cs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00902.xml.gz", + "crc64nvme": "PShwZ1YTS1M=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00903.xml.gz", + "crc64nvme": "uhlsDw2jnyI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00904.xml.gz", + "crc64nvme": "LFnossLcgko=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00905.xml.gz", + "crc64nvme": "kyZyDfn1yBM=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00906.xml.gz", + "crc64nvme": "pRe+89Ajp6w=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00907.xml.gz", + "crc64nvme": "MDygV6n+IUg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00908.xml.gz", + "crc64nvme": "gOE6kyBqsOA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00909.xml.gz", + "crc64nvme": "4DqQGjF2uN4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00910.xml.gz", + "crc64nvme": "0NDUxs3D/KA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00911.xml.gz", + "crc64nvme": "ErPRtYHG0qs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00912.xml.gz", + "crc64nvme": "QIkR+wNHjFE=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00913.xml.gz", + "crc64nvme": "yskjnjIP9ZE=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00914.xml.gz", + "crc64nvme": "xiFhSIJmzkM=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00915.xml.gz", + "crc64nvme": "LqcSEL2Flgw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00916.xml.gz", + "crc64nvme": "toI46/7cP6w=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00917.xml.gz", + "crc64nvme": "f+i+lTDBR1U=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00918.xml.gz", + "crc64nvme": "9F8hdpcZrFs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00919.xml.gz", + "crc64nvme": "DeE6y166rZw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00920.xml.gz", + "crc64nvme": "v/XLAIPhlPk=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00921.xml.gz", + "crc64nvme": "4mwGQwn35WY=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00922.xml.gz", + "crc64nvme": "RDXOBgsURrQ=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00923.xml.gz", + "crc64nvme": "s0Xa33+zjU0=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00924.xml.gz", + "crc64nvme": "ZX76o6BbG+k=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00925.xml.gz", + "crc64nvme": "fv+e0Ap5xq4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00926.xml.gz", + "crc64nvme": "nMvPKcQsgG8=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00927.xml.gz", + "crc64nvme": "xtGxOC71tYc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00928.xml.gz", + "crc64nvme": "FwyVDrJ7UJE=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00929.xml.gz", + "crc64nvme": "XKjQf39LCDc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00930.xml.gz", + "crc64nvme": "YVBpJMjTF0Q=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00931.xml.gz", + "crc64nvme": "t99Jn6AdHzc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00932.xml.gz", + "crc64nvme": "AZ+5HCuHvxA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00933.xml.gz", + "crc64nvme": "JzSAx/2bLWk=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00934.xml.gz", + "crc64nvme": "hsb116dQ6rI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00935.xml.gz", + "crc64nvme": "dIMvpUazcyw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00936.xml.gz", + "crc64nvme": "Vb4NZPsUSws=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00937.xml.gz", + "crc64nvme": "zyX6aKkq7jk=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00938.xml.gz", + "crc64nvme": "O0UomXy622c=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00939.xml.gz", + "crc64nvme": "/znK28KJTjk=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00940.xml.gz", + "crc64nvme": "Ul7txpVB4cQ=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00941.xml.gz", + "crc64nvme": "mgxRdhEvuOg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00942.xml.gz", + "crc64nvme": "SVWekta8l0c=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00943.xml.gz", + "crc64nvme": "2sd+HrUemIs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00944.xml.gz", + "crc64nvme": "3Di/rXN8NWA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00945.xml.gz", + "crc64nvme": "DeP+sg5WtTk=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00946.xml.gz", + "crc64nvme": "afn9IyJ7rWg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00947.xml.gz", + "crc64nvme": "wOFqkr18rE8=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00948.xml.gz", + "crc64nvme": "/YoD8rlxIDs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00949.xml.gz", + "crc64nvme": "rrfQfh8sViQ=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00950.xml.gz", + "crc64nvme": "cKgbZTEvoEY=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00951.xml.gz", + "crc64nvme": "fRIDwpT8ZY4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00952.xml.gz", + "crc64nvme": "VXxAVmcmIUY=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00953.xml.gz", + "crc64nvme": "rXgY1zN0LBg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00954.xml.gz", + "crc64nvme": "2OWmJ0BtuMI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00955.xml.gz", + "crc64nvme": "rHT2/UmnK3w=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00956.xml.gz", + "crc64nvme": "VaUSc6zE3tg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00957.xml.gz", + "crc64nvme": "GBcwCd5LByI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00958.xml.gz", + "crc64nvme": "Zyre71e/Vd8=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00959.xml.gz", + "crc64nvme": "P7emDdBJ6S8=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00960.xml.gz", + "crc64nvme": "pXxHRBlVN8E=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00961.xml.gz", + "crc64nvme": "IvDue8zTBaA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00962.xml.gz", + "crc64nvme": "oq8cuWxCpWs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00963.xml.gz", + "crc64nvme": "s3MU8QHgzdo=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00964.xml.gz", + "crc64nvme": "dNyIkdkJD68=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00965.xml.gz", + "crc64nvme": "Qmi310kWfrQ=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00966.xml.gz", + "crc64nvme": "waf6BC/NyNU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00967.xml.gz", + "crc64nvme": "P68CJd7KmGo=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00968.xml.gz", + "crc64nvme": "OsmvTygQlKw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00969.xml.gz", + "crc64nvme": "7iyD21gjMsw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00970.xml.gz", + "crc64nvme": "CMUz+fM55Bs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00971.xml.gz", + "crc64nvme": "U4z1MVCayWo=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00972.xml.gz", + "crc64nvme": "fR68wEfg+hU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00973.xml.gz", + "crc64nvme": "jl3JpfnijRc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00974.xml.gz", + "crc64nvme": "A51PgS8iUIU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00975.xml.gz", + "crc64nvme": "NAjQmGkKXPs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00976.xml.gz", + "crc64nvme": "FePD5NBKgCg=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00977.xml.gz", + "crc64nvme": "OFs7PoZG/Q0=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00978.xml.gz", + "crc64nvme": "seSyambXUdI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00979.xml.gz", + "crc64nvme": "xGBgEYfGGm4=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00980.xml.gz", + "crc64nvme": "AJapve6eeUA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00981.xml.gz", + "crc64nvme": "Ut3dINq172I=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00982.xml.gz", + "crc64nvme": "eztQlsjZdu8=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00983.xml.gz", + "crc64nvme": "fmMOg/cYAYo=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00984.xml.gz", + "crc64nvme": "lz7dNy4S+jo=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00985.xml.gz", + "crc64nvme": "VTSosmfTvJU=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00986.xml.gz", + "crc64nvme": "8NgCDRCDd0E=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00987.xml.gz", + "crc64nvme": "veAwqrRhzbM=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00988.xml.gz", + "crc64nvme": "iCJnOeDZ8Lc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00989.xml.gz", + "crc64nvme": "SstUmtV9x8s=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00990.xml.gz", + "crc64nvme": "hvdb09Q3R1E=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00991.xml.gz", + "crc64nvme": "u1eoNuvR2bk=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00992.xml.gz", + "crc64nvme": "mfyfoapFs4A=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00993.xml.gz", + "crc64nvme": "GNBAMAfiQHs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00994.xml.gz", + "crc64nvme": "YFwQTzM/WnA=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00995.xml.gz", + "crc64nvme": "NU7s7gDnsyI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00996.xml.gz", + "crc64nvme": "emyRjNDoDcw=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00997.xml.gz", + "crc64nvme": "oF6O3t+xBmo=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00998.xml.gz", + "crc64nvme": "thNC/o2Qq+U=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_00999.xml.gz", + "crc64nvme": "Ve0WPqO300w=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/uniref/uniref100/uniref100.xml.gz/part_01000.xml.gz", + "crc64nvme": "AmKF7K3jDe4=", + }, + {"file": "cts/io/ialarmedalien/misc/logger_config.json", "crc64nvme": "ImY6G9xmYs4="}, + ], + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9685900804811111, + "cpu_factor": 0.09150890887030012, + "max_memory": 4710187008, + }, + { + "error": "The job produced no output files", + "id": "253089e2-7ad1-4e32-a9fe-11aabda31c62", + "state": "error", + "transition_times": [ + {"state": "created", "time": "2026-10-22T04:27:42.903000Z"}, + {"state": "download_submitted", "time": "2026-10-22T04:27:42.988000Z"}, + {"state": "job_submitting", "time": "2026-10-22T04:27:52.802000Z"}, + {"state": "job_submitted", "time": "2026-10-22T04:28:31.509000Z"}, + {"state": "upload_submitting", "time": "2026-10-22T04:28:33.503000Z"}, + {"state": "upload_submitted", "time": "2026-10-22T04:28:33.560000Z"}, + {"state": "error", "time": "2026-10-22T04:29:28.225000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "uniprot"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/batch_001/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00001.xml.gz", + "crc64nvme": "AFAZfDzbbfc=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00002.xml.gz", + "crc64nvme": "ZpZSu7BcMGs=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00003.xml.gz", + "crc64nvme": "q8rI7iaPk50=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00004.xml.gz", + "crc64nvme": "ea4sVxZ8JFI=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00005.xml.gz", + "crc64nvme": "KPrvcCVDlwM=", + }, + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00006.xml.gz", + "crc64nvme": "XWV3eNCm6hY=", + }, + {"file": "cts/io/ialarmedalien/misc/logger_config.json", "crc64nvme": "ImY6G9xmYs4="}, + ], + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-04-26T15:46:11.342000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.8", + }, + "input_file_count": 7, + "cpu_hours": 0.0023639620611111112, + "cpu_factor": 0.4487487295293731, + "max_memory": 116162560, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/cts_job/minimal.json b/tests/data/jsonschema/examples/cts_job/minimal.json new file mode 100644 index 00000000..357eb698 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/minimal.json @@ -0,0 +1,35 @@ +[ + { + "id": "253089e2-7ad1-4e32-a9fe-11aabda31c62", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-22T04:27:42.903000Z" + } + ], + "user": "ialarmedalien", + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "params": {}, + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/batch_001/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00001.xml.gz", + "crc64nvme": "AFAZfDzbbfc=" + } + ] + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-04-26T15:46:11.342000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "entrypoint": [ + "./scripts/entrypoint.sh" + ] + }, + "input_file_count": 1 + } +] diff --git a/tests/data/jsonschema/examples/cts_job/minimal_expected.json b/tests/data/jsonschema/examples/cts_job/minimal_expected.json new file mode 100644 index 00000000..357eb698 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/minimal_expected.json @@ -0,0 +1,35 @@ +[ + { + "id": "253089e2-7ad1-4e32-a9fe-11aabda31c62", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-22T04:27:42.903000Z" + } + ], + "user": "ialarmedalien", + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "params": {}, + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/batch_001/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00001.xml.gz", + "crc64nvme": "AFAZfDzbbfc=" + } + ] + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-04-26T15:46:11.342000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "entrypoint": [ + "./scripts/entrypoint.sh" + ] + }, + "input_file_count": 1 + } +] diff --git a/tests/data/jsonschema/examples/cts_job/minimal_expected_table.py b/tests/data/jsonschema/examples/cts_job/minimal_expected_table.py new file mode 100644 index 00000000..090404a4 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/minimal_expected_table.py @@ -0,0 +1,38 @@ +"""Expected PyArrow table for minimal real-world example cts_job.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "253089e2-7ad1-4e32-a9fe-11aabda31c62", + "state": "error", + "transition_times": [{"state": "created", "time": "2026-10-22T04:27:42.903000Z"}], + "user": "ialarmedalien", + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "params": {}, + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/batch_001/", + "input_files": [ + { + "file": "cts/io/ialarmedalien/datasets/uniprot/derived/2026_03/knowledgebase/complete/uniprot_sprot.xml.gz/part_00001.xml.gz", + "crc64nvme": "AFAZfDzbbfc=", + } + ], + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-04-26T15:46:11.342000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:029f2893e1109f1d9961233f439cb411de8d7623203eb8f17410dca0bd516088", + "entrypoint": ["./scripts/entrypoint.sh"], + }, + "input_file_count": 1, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/cts_job/schema.json b/tests/data/jsonschema/examples/cts_job/schema.json new file mode 100644 index 00000000..e8fd1f4c --- /dev/null +++ b/tests/data/jsonschema/examples/cts_job/schema.json @@ -0,0 +1,1013 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jsonschema:job.json", + "title": "Job", + "$ref": "#/$defs/Job", + "$defs": { + "Job": { + "properties": { + "outputs": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/S3File" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "trans_history": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/JobStateTransition" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Trans History", + "description": "Historical state transition records from previous recovery attempts. Populated when the job is reset during job recovery." + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "A description of the error that occurred.", + "examples": [ + "The front fell off" + ] + }, + "output_file_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Output File Count", + "description": "The number of output files, if available.", + "examples": [ + 24 + ] + }, + "id": { + "type": "string", + "title": "Id", + "description": "An opaque, unique string that serves as the job's ID." + }, + "state": { + "$ref": "#/$defs/JobState", + "examples": [ + "complete" + ] + }, + "transition_times": { + "items": { + "$ref": "#/$defs/JobStateTransition" + }, + "type": "array", + "title": "Transition Times", + "description": "A list of job state transitions.", + "examples": [ + [ + { + "state": "created", + "time": "2024-10-24T22:35:40Z" + }, + { + "state": "upload_submitted", + "time": "2024-10-24T22:35:41Z" + }, + { + "state": "job_submitting", + "time": "2024-10-24T22:47:67Z" + } + ] + ] + }, + "user": { + "type": "string", + "title": "User", + "description": "The user that ran the job.", + "examples": [ + "myuserid" + ] + }, + "admin_meta": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "object", + "title": "Admin Meta", + "description": "Metadata for the job set by service administrators. This metadata is mutable and arbitrary from the point of view of the service.", + "examples": [ + { + "children": 31619, + "name": "Getrude" + } + ] + }, + "job_input": { + "$ref": "#/$defs/JobInput" + }, + "image": { + "$ref": "#/$defs/JobImage" + }, + "input_file_count": { + "type": "integer", + "title": "Input File Count", + "description": "The number of input files.", + "examples": [ + 42 + ] + }, + "cpu_hours": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cpu Hours", + "description": "The total CPU hours used by the job, if available.", + "examples": [ + 52.4 + ] + }, + "cpu_factor": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cpu Factor", + "description": "The ratio of the cpu time used to the number of cpus requested times the job's actual time, if available. Perfect usage is 1, < 1 means requested CPUs are partially idle, possibly due to memory or IO bounded work, > 1 means more CPU usage than requested.", + "examples": [ + 0.8 + ] + }, + "max_memory": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Memory", + "description": "The maximum memory in bytes used by a single container in the job, if available.", + "examples": [ + 1032000000 + ] + }, + "logpath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Logpath", + "description": "A location in S3 where the logfiles for the job containers can be viewed.", + "examples": [ + "cts-logs/container_logs/e14a21ba-032d-42f2-b235-d82606675b17" + ] + } + }, + "type": "object", + "required": [ + "id", + "state", + "transition_times", + "user", + "job_input", + "image", + "input_file_count" + ], + "title": "Job", + "description": "Information about a job." + }, + "S3File": { + "properties": { + "file": { + "type": "string", + "maxLength": 1088, + "minLength": 5, + "title": "File", + "description": "A path to an object in an S3 instance, starting with the bucket. Please note that spaces are valid S3 object characters, so be careful about trailing spaces in the input.", + "examples": [ + "mybucket/foo/bar/baz.jpg" + ] + }, + "crc64nvme": { + "anyOf": [ + { + "type": "string", + "maxLength": 12, + "minLength": 12 + }, + { + "type": "null" + } + ], + "title": "Crc64Nvme", + "description": "The base64 encoded CRC64/NVME checksum of the file. If provided on input it is checked against the target file checksum before proceeding. Always provided with output.", + "examples": [ + "4ekt2WB1KO4=" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "file" + ], + "title": "S3File", + "description": "A file in an S3 instance. " + }, + "JobStateTransition": { + "properties": { + "state": { + "$ref": "#/$defs/JobState" + }, + "time": { + "type": "string", + "format": "date-time", + "title": "Time", + "description": "The time at which the new state was entered.", + "examples": [ + "2024-10-24T22:35:40Z" + ] + } + }, + "type": "object", + "required": [ + "state", + "time" + ], + "title": "JobStateTransition", + "description": "Denotes the new state and the entry time when a state change occurs." + }, + "JobState": { + "type": "string", + "enum": [ + "created", + "download_submitted", + "job_submitting", + "job_submitted", + "upload_submitting", + "upload_submitted", + "complete", + "recovering", + "canceling", + "canceled", + "error_processing_submitting", + "error_processing_submitted", + "error" + ], + "title": "JobState", + "description": "The state of a job." + }, + "JobInput": { + "properties": { + "cluster": { + "$ref": "#/$defs/Cluster", + "examples": [ + "perlmutter-jaws" + ] + }, + "image": { + "type": "string", + "maxLength": 1000, + "minLength": 1, + "title": "Image", + "description": "The Docker image to run for the job. Include the SHA to ensure the exact code requested is run.", + "examples": [ + "ghcr.io/kbase/collections:checkm2_0.1.6@sha256:c9291c94c382b88975184203100d119cba865c1be91b1c5891749ee02193d380" + ] + }, + "params": { + "$ref": "#/$defs/Parameters-Output" + }, + "num_containers": { + "type": "integer", + "maximum": 1000.0, + "minimum": 1.0, + "title": "Num Containers", + "description": "The number of containers to run in parallel. Input files will be split between the containers.", + "default": 1, + "examples": [ + 1 + ] + }, + "cpus": { + "type": "integer", + "maximum": 256.0, + "minimum": 1.0, + "title": "Cpus", + "description": "The number of CPUs to allocate per container.", + "default": 1, + "examples": [ + 1 + ] + }, + "gpus": { + "type": "integer", + "maximum": 4.0, + "minimum": 0.0, + "title": "Gpus", + "description": "The number of GPUs to allocate per container. Jobs requesting 0 GPUs are never scheduled on GPU-capable nodes.", + "default": 0, + "examples": [ + 1 + ] + }, + "memory": { + "type": "integer", + "minimum": 0.0, + "title": "Memory", + "description": "The amount of memory to allocate per container either as the number of bytes or a specification string such as 100MB, 2GB, etc.", + "default": 10000000, + "examples": [ + "10MB" + ] + }, + "runtime": { + "type": "string", + "format": "duration", + "title": "Runtime", + "description": "The runtime required for each container as the number of seconds or an ISO8601 duration string.", + "default": "PT1M", + "examples": [ + "PT12H30M5S", + 45005 + ] + }, + "input_roots": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Input Roots", + "description": "If specified, preserves file hierarchies for S3 files below the given root paths, starting from the bucket. Any files that are not prefixed by a root path are placed in the root directory of the job input dir. If any input files have the same path in the input dir, the job fails. For example, given a input_roots entry of [\"mybucket/foo/bar\"] and the input files [\"otherbucket/foo\", \"mybucket/bar\", \"mybucket/foo/bar/baz/bat\"] the job input directory would include the files foo, bar, and baz/bat. To preserve hierarchies for all files, set input_roots to [\"\"]. If an input file matches more than one root, the longest root that isn't identical to the file path is used. Whitespace on the left side of the path is is ignored. Duplicate roots are ignored.", + "examples": [ + [ + "mybucket/foo/bar" + ] + ] + }, + "output_dir": { + "type": "string", + "maxLength": 1088, + "minLength": 5, + "title": "Output Dir", + "description": "The S3 folder, starting with the bucket, in which to place results.", + "examples": [ + "mybucket/out" + ] + }, + "input_files": { + "items": { + "$ref": "#/$defs/S3FileWithDataID" + }, + "type": "array", + "maxItems": 10000, + "minItems": 1, + "title": "Input Files", + "description": "The S3 input files for the job, either a list of file paths as strings or a list of data structures including the file path and optionally a data ID and / or CRC64/NVME checksum. The file paths always start with the bucket. All S3 input files are expected to have a CRC64/NVME checksum available, even if it is not provided in the input data structure. Either all or no files must have data IDs associated with them. When returned from the service, the checksum is always included.", + "examples": [ + [ + { + "crc64name": "4ekt2WB1KO4=", + "data_id": "GCA_000146795.3", + "file": "mybucket/foo/bat" + } + ], + [ + "mybucket/foo/bat" + ] + ] + }, + "script": { + "anyOf": [ + { + "$ref": "#/$defs/S3File" + }, + { + "type": "null" + } + ], + "description": "A user-provided script, either as a file path string or a data structure including the file path and optionally a CRC64/NVME checksum. The file path always starts with the bucket. The script is expected to have a CRC64/NVME checksum available, even if it is not provided in the input data structure. When returned from the service, the checksum is always included. The script is placed at the root of the input mount point for every container as `.__script__` - as such, the input mount point chosen for the job must be compatible with the image entrypoint. It is the responsibility of the image entrypoint to run the script. Only supported at the `kbase` site, and only for images that allow script execution.", + "examples": [ + "mybucket/foo/script.sh" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "cluster", + "image", + "params", + "output_dir", + "input_files" + ], + "title": "JobInput", + "description": "Input to a Job." + }, + "Cluster": { + "type": "string", + "enum": [ + "perlmutter-jaws", + "lawrencium-jaws", + "kbase" + ], + "title": "Cluster", + "description": "All clusters that have ever been registered in the system, including those that are no longer\nregistered but may have associated job records.\n\nperlmutter-jaws: The Perlmutter cluster at NERSC run via JAWS. \nlawrencium-jaws: The Lawrencium cluster at LBNL run via JAWS. \nkbase: KBase compute nodes." + }, + "Parameters-Output": { + "properties": { + "input_mount_point": { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^/(?:[\\w.-]+/)*[\\w.-]+/?$", + "title": "Input Mount Point", + "description": "Where to place input files in the container. Must start from the container root and include at least one directory when resolved.", + "default": "/input_files", + "examples": [ + "/input_files" + ] + }, + "output_mount_point": { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^/(?:[\\w.-]+/)*[\\w.-]+/?$", + "title": "Output Mount Point", + "description": "Where output files should be written in the container. Must start from the container root and include at least one directory when resolved. Note that the mount point is not included in the file paths thatare eventually transferred to S3.", + "default": "/output_files", + "examples": [ + "/output_files" + ] + }, + "declobber": { + "type": "boolean", + "title": "Declobber", + "description": "If true, prepend the container number to all output file paths as a directory. This prevents containers that write to the same file path from clobbering each other and losing data. If the output files of containers are guaranteed to be unique (e.g. output file names are based on input file names and input file names are unique) then this setting is not necessary. The same effect can be created by use of a container number parameter when specifying the output directory in the arguments or environment.", + "default": false + }, + "refdata_mount_point": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^/(?:[\\w.-]+/)*[\\w.-]+/?$" + }, + { + "type": "null" + } + ], + "title": "Refdata Mount Point", + "description": "Where reference data files should be pleased in the container. Must start from the container root and include at least one directory when resolved. Required if the job image requires reference data and a default reference data mount point isn't specified for the image. If a default is provided, this setting overrides it.", + "examples": [ + "/reference_data" + ] + }, + "args": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "string", + "maxLength": 256, + "minLength": 2, + "pattern": "^--?[a-zA-Z0-9][\\w\\.-]*=?$" + }, + { + "$ref": "#/$defs/ParameterWithFlag" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Args", + "description": "A list of command line arguments to be inserted after the container entrypoint command. Strings are treated as literals.", + "examples": [ + [ + "process", + "--output-dir", + "/output_files", + { + "flag": "--input-file=", + "input_files_format": "repeat_parameter", + "type": "input_files" + }, + "--jobid", + { + "type": "container_number" + } + ] + ] + }, + "environment": { + "anyOf": [ + { + "patternProperties": { + "^[A-Z_][A-Z0-9_]*$": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "$ref": "#/$defs/Parameter" + } + ] + } + }, + "propertyNames": { + "maxLength": 256, + "minLength": 1 + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Environment", + "description": "A dictionary of environment variables to be inserted into the container. String values are treated as literals.", + "examples": [ + { + "DIAMOND_DB_PATH": "/reference_data", + "FILE_MANIFEST": { + "manifest_file_format": "files", + "manifest_file_header": "infiles", + "type": "manifest_file" + } + } + ] + } + }, + "additionalProperties": false, + "type": "object", + "title": "Parameters", + "description": "A set of parameters for a container in a job.\n\nSpecifying files as part of arguments to a job can be done in three ways:\n\n* If the entrypoint command takes a directory as an argument, that\n directory can be specified literally in an argument. All files will be available in the\n input mount point, with paths resulting from the path in S3 as well as the input_roots\n parameter from the job input.\n* If individual filenames need to be specified, use a Parameter instance in\n the positional or environmental arguments. At most one Parameter can\n be specified per parameter set.\n* A Parameter instance can also be used to specify a manifest of file names or\n file IDs to be passed to the container." + }, + "ParameterWithFlag": { + "properties": { + "type": { + "$ref": "#/$defs/ParameterType", + "examples": [ + "input_files" + ] + }, + "input_files_format": { + "anyOf": [ + { + "$ref": "#/$defs/InputFilesFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "comma_separated_list" + ] + }, + "manifest_file_format": { + "anyOf": [ + { + "$ref": "#/$defs/ManifestFileFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "files" + ] + }, + "manifest_file_header": { + "anyOf": [ + { + "type": "string", + "maxLength": 1000, + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Manifest File Header", + "description": "The header for the manifest file, if any.", + "examples": [ + "genome_id" + ] + }, + "container_num_prefix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Prefix", + "description": "A string that will be prepended to the container number.", + "examples": [ + "/output/job-" + ] + }, + "container_num_suffix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Suffix", + "description": "A string that will be appended to the container number.", + "examples": [ + "_output" + ] + }, + "flag": { + "anyOf": [ + { + "type": "string", + "maxLength": 256, + "minLength": 2, + "pattern": "^--?[a-zA-Z0-9][\\w\\.-]*=?$" + }, + { + "type": "null" + } + ], + "title": "Flag", + "description": "A command line argument flag to precede the parameter. If the flag ends with '=', no space is left between the flag and its value.", + "examples": [ + "--file" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "title": "ParameterWithFlag", + "description": "Represents a special parameter passed to a container, optionally with a\npreceding flag argument.\n\nCan represent a set of input file names formatted in various ways, a manifest file\ncontaining file paths or data IDs, or a container number.\n\ninput_files_format is required when the type is input_files, but ignored otherwise. \nmanifest_file_format is required when the type is manifest_file, but ignored otherwise. \nmanifest_file_header is optional and only taken into account when the type is manifest_file. " + }, + "ParameterType": { + "type": "string", + "enum": [ + "input_files", + "manifest_file", + "container_number" + ], + "title": "ParameterType", + "description": "The type of a parameter if not a string literal.\n\ninput_files will cause the list of input files for the container to be inserted\ninto the entrypoint command line or an environment variable.\n\nmanifest_file will cause the list of input files for the container to be placed in a\nmanifest file and will insert the file location into the entrypoint command line\nor an environment variable.\n\ncontainer_number will insert the integer number of the container into the\nentrypoint command line or an environment variable. This can be used so prevent file path\ncollisions if multiple containers write to the same file paths." + }, + "InputFilesFormat": { + "type": "string", + "enum": [ + "comma_separated_list", + "space_separated_list", + "repeat_parameter" + ], + "title": "InputFilesFormat", + "description": "The format of an input files type parameter.\n\nComma and space separated lists format the list of files as one would expect.\n\nRepeat parameter causes the input files to be inserted as a space separated list,\nwith each file preceded by the parameter flag. Only valid for CLI flag based arguments.\nIf the flag ends with an equals sign, there is no space left between the flag and the\nparameter.\n\nExamples: \n`-i`: `-i file1 -i file2 ... -i fileN` \n`--input_file=`: `--input_file=file1 --input_file=file2 ... --input_file=fileN` " + }, + "ManifestFileFormat": { + "type": "string", + "enum": [ + "files", + "data_ids" + ], + "title": "ManifestFileFormat", + "description": "The format of a manifest file.\n\nfiles will cause the manifest file to consist of a list of the input files, one per line.\n\ndata_ids will cause the manifest file to consist of a list of the data IDs associated with\nthe input files, one per line." + }, + "Parameter": { + "properties": { + "type": { + "$ref": "#/$defs/ParameterType", + "examples": [ + "input_files" + ] + }, + "input_files_format": { + "anyOf": [ + { + "$ref": "#/$defs/InputFilesFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "comma_separated_list" + ] + }, + "manifest_file_format": { + "anyOf": [ + { + "$ref": "#/$defs/ManifestFileFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "files" + ] + }, + "manifest_file_header": { + "anyOf": [ + { + "type": "string", + "maxLength": 1000, + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Manifest File Header", + "description": "The header for the manifest file, if any.", + "examples": [ + "genome_id" + ] + }, + "container_num_prefix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Prefix", + "description": "A string that will be prepended to the container number.", + "examples": [ + "/output/job-" + ] + }, + "container_num_suffix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Suffix", + "description": "A string that will be appended to the container number.", + "examples": [ + "_output" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "title": "Parameter", + "description": "Represents a special parameter passed to a container.\n\nCan represent a set of input file names formatted in various ways, a manifest file\ncontaining file paths or data IDs, or a container number.\n\ninput_files_format is required when the type is input_files, but ignored otherwise. \nmanifest_file_format is required when the type is manifest_file, but ignored otherwise. \nmanifest_file_header is optional and only taken into account when the type is manifest_file. " + }, + "S3FileWithDataID": { + "properties": { + "file": { + "type": "string", + "maxLength": 1088, + "minLength": 5, + "title": "File", + "description": "A path to an object in an S3 instance, starting with the bucket. Please note that spaces are valid S3 object characters, so be careful about trailing spaces in the input.", + "examples": [ + "mybucket/foo/bar/baz.jpg" + ] + }, + "crc64nvme": { + "anyOf": [ + { + "type": "string", + "maxLength": 12, + "minLength": 12 + }, + { + "type": "null" + } + ], + "title": "Crc64Nvme", + "description": "The base64 encoded CRC64/NVME checksum of the file. If provided on input it is checked against the target file checksum before proceeding. Always provided with output.", + "examples": [ + "4ekt2WB1KO4=" + ] + }, + "data_id": { + "anyOf": [ + { + "type": "string", + "maxLength": 255, + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Data Id", + "description": "An arbitrary string representing the ID of the data in the file.", + "examples": [ + "GCA_000146795.3" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "file" + ], + "title": "S3FileWithDataID", + "description": "A file in an S3 instance, optionally with a data ID. " + }, + "JobImage": { + "properties": { + "registered_by": { + "type": "string", + "title": "Registered By", + "description": "The username of the user that performed the registration.", + "examples": [ + "aparkin" + ] + }, + "registered_on": { + "type": "string", + "format": "date-time", + "title": "Registered On", + "description": "The time of registration.", + "examples": [ + "2024-10-24T22:35:40Z" + ] + }, + "name": { + "type": "string", + "title": "Name", + "description": "The normalized name of the a docker image, consisting of the host and path.", + "examples": [ + "ghcr.io/kbase/collections" + ] + }, + "digest": { + "type": "string", + "title": "Digest", + "description": "The image digest.", + "examples": [ + "sha256:c9291c94c382b88975184203100d119cba865c1be91b1c5891749ee02193d380" + ] + }, + "entrypoint": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Entrypoint", + "description": "The entrypoint command extracted from the image.", + "examples": [ + [ + "checkm2", + "predict" + ] + ] + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag", + "description": "The image tag at registration time. The tag may no longer point to the same image.", + "examples": [ + "checkm2_0.1.6" + ] + }, + "refdata_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Refdata Id", + "description": "The ID of reference data associated with the image. The image requires this reference data to run, and it will be mounted into the image container at the refdata mount point.", + "examples": [ + "3a28c155-ea8b-4e1b-baef-242d991a8200" + ] + }, + "default_refdata_mount_point": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Default Refdata Mount Point", + "description": "The default mount point for the image refdata in the container. Overridden by the refdata mount point in a job submission if provided there. If a default mount point is provided, then providing a mount point in the job submission is optional. Must be an absolute path.", + "examples": [ + "/reference_data" + ] + } + }, + "type": "object", + "required": [ + "registered_by", + "registered_on", + "name", + "digest", + "entrypoint" + ], + "title": "JobImage", + "description": "Information about a Docker image as stored in a Job data structure." + } + } +} diff --git a/tests/data/jsonschema/examples/cts_jobs/case.yaml b/tests/data/jsonschema/examples/cts_jobs/case.yaml new file mode 100644 index 00000000..c7f0adf9 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/case.yaml @@ -0,0 +1,17 @@ +title: KBase CTS `ListJobsResponse` +schema: schema.json +rules: [R-SHAPE-ARRAY, R-SHAPE-REF] +notes: | + `/jobs` response schema from `not_for_github/cts/openapi.json` (`#/components/schemas/ListJobsResponse`): an + array of `JobPreview` structs with `JobInputPreview`, `JobImage` and state transition timestamps. +runs: + - id: minimal + data: minimal.json + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.json + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/jsonschema/examples/cts_jobs/expected_schema.py b/tests/data/jsonschema/examples/cts_jobs/expected_schema.py new file mode 100644 index 00000000..e8f4150a --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/expected_schema.py @@ -0,0 +1,554 @@ +"""Expected PyArrow schema for real-world example cts_jobs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "jobs", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "error", + pa.string(), + nullable=True, + metadata={ + "comment": "A description of the error that occurred.", + "jsonschema_type": "string", + }, + ), + pa.field( + "output_file_count", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of output files, if available.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "id", + pa.string(), + nullable=False, + metadata={ + "comment": "An opaque, unique string that serves as the job's ID.", + "jsonschema_type": "string", + }, + ), + pa.field( + "state", + pa.string(), + nullable=False, + metadata={"comment": "The state of a job.", "jsonschema_type": "string"}, + ), + pa.field( + "transition_times", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "state", + pa.string(), + nullable=False, + metadata={ + "comment": "The state of a job.", + "jsonschema_type": "string", + }, + ), + pa.field( + "time", + pa.string(), + nullable=False, + metadata={ + "comment": "The time at which the new state was entered.", + "jsonschema_type": "string (date-time)", + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Denotes the new state and the entry time when a state change occurs." + }, + ) + ), + nullable=False, + metadata={"comment": "A list of job state transitions."}, + ), + pa.field( + "user", + pa.string(), + nullable=False, + metadata={"comment": "The user that ran the job.", "jsonschema_type": "string"}, + ), + pa.field( + "admin_meta", + pa.map_( + pa.string(), + pa.field( + "value", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string | integer | number"}, + ), + ), + nullable=True, + metadata={ + "comment": "Metadata for the job set by service administrators. This metadata is mutable and arbitrary from the point of view of the service." + }, + ), + pa.field( + "job_input", + pa.struct( + [ + pa.field( + "cluster", + pa.string(), + nullable=False, + metadata={ + "comment": "All clusters that have ever been registered in the system, including those that are no longer\nregistered but may have associated job records.\n\nperlmutter-jaws: The Perlmutter cluster at NERSC run via JAWS. \nlawrencium-jaws: The Lawrencium cluster at LBNL run via JAWS. \nkbase: KBase compute nodes.", + "jsonschema_type": "string", + }, + ), + pa.field( + "image", + pa.string(), + nullable=False, + metadata={ + "comment": "The Docker image to run for the job. Include the SHA to ensure the exact code requested is run.", + "jsonschema_type": "string", + }, + ), + pa.field( + "params", + pa.struct( + [ + pa.field( + "input_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "Where to place input files in the container. Must start from the container root and include at least one directory when resolved.", + "jsonschema_type": "string", + }, + ), + pa.field( + "output_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "Where output files should be written in the container. Must start from the container root and include at least one directory when resolved. Note that the mount point is not included in the file paths thatare eventually transferred to S3.", + "jsonschema_type": "string", + }, + ), + pa.field( + "declobber", + pa.bool_(), + nullable=True, + metadata={ + "comment": "If true, prepend the container number to all output file paths as a directory. This prevents containers that write to the same file path from clobbering each other and losing data. If the output files of containers are guaranteed to be unique (e.g. output file names are based on input file names and input file names are unique) then this setting is not necessary. The same effect can be created by use of a container number parameter when specifying the output directory in the arguments or environment.", + "jsonschema_type": "boolean", + }, + ), + pa.field( + "refdata_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "Where reference data files should be pleased in the container. Must start from the container root and include at least one directory when resolved. Required if the job image requires reference data and a default reference data mount point isn't specified for the image. If a default is provided, this setting overrides it.", + "jsonschema_type": "string", + }, + ), + pa.field( + "args", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "type", + pa.string(), + nullable=True, + metadata={ + "comment": "The type of a parameter if not a string literal.\n\ninput_files will cause the list of input files for the container to be inserted\ninto the entrypoint command line or an environment variable.\n\nmanifest_file will cause the list of input files for the container to be placed in a\nmanifest file and will insert the file location into the entrypoint command line\nor an environment variable.\n\ncontainer_number will insert the integer number of the container into the\nentrypoint command line or an environment variable. This can be used so prevent file path\ncollisions if multiple containers write to the same file paths.", + "jsonschema_type": "string", + }, + ), + pa.field( + "input_files_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of an input files type parameter.\n\nComma and space separated lists format the list of files as one would expect.\n\nRepeat parameter causes the input files to be inserted as a space separated list,\nwith each file preceded by the parameter flag. Only valid for CLI flag based arguments.\nIf the flag ends with an equals sign, there is no space left between the flag and the\nparameter.\n\nExamples: \n`-i`: `-i file1 -i file2 ... -i fileN` \n`--input_file=`: `--input_file=file1 --input_file=file2 ... --input_file=fileN` ", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of a manifest file.\n\nfiles will cause the manifest file to consist of a list of the input files, one per line.\n\ndata_ids will cause the manifest file to consist of a list of the data IDs associated with\nthe input files, one per line.", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_header", + pa.string(), + nullable=True, + metadata={ + "comment": "The header for the manifest file, if any.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_prefix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be prepended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_suffix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be appended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "flag", + pa.string(), + nullable=True, + metadata={ + "comment": "A command line argument flag to precede the parameter. If the flag ends with '=', no space is left between the flag and its value.", + "jsonschema_type": "string", + }, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + metadata={ + "comment": "A list of command line arguments to be inserted after the container entrypoint command. Strings are treated as literals." + }, + ), + pa.field( + "environment", + pa.map_( + pa.string(), + pa.field( + "value", + pa.struct( + [ + pa.field( + "type", + pa.string(), + nullable=True, + metadata={ + "comment": "The type of a parameter if not a string literal.\n\ninput_files will cause the list of input files for the container to be inserted\ninto the entrypoint command line or an environment variable.\n\nmanifest_file will cause the list of input files for the container to be placed in a\nmanifest file and will insert the file location into the entrypoint command line\nor an environment variable.\n\ncontainer_number will insert the integer number of the container into the\nentrypoint command line or an environment variable. This can be used so prevent file path\ncollisions if multiple containers write to the same file paths.", + "jsonschema_type": "string", + }, + ), + pa.field( + "input_files_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of an input files type parameter.\n\nComma and space separated lists format the list of files as one would expect.\n\nRepeat parameter causes the input files to be inserted as a space separated list,\nwith each file preceded by the parameter flag. Only valid for CLI flag based arguments.\nIf the flag ends with an equals sign, there is no space left between the flag and the\nparameter.\n\nExamples: \n`-i`: `-i file1 -i file2 ... -i fileN` \n`--input_file=`: `--input_file=file1 --input_file=file2 ... --input_file=fileN` ", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_format", + pa.string(), + nullable=True, + metadata={ + "comment": "The format of a manifest file.\n\nfiles will cause the manifest file to consist of a list of the input files, one per line.\n\ndata_ids will cause the manifest file to consist of a list of the data IDs associated with\nthe input files, one per line.", + "jsonschema_type": "string", + }, + ), + pa.field( + "manifest_file_header", + pa.string(), + nullable=True, + metadata={ + "comment": "The header for the manifest file, if any.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_prefix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be prepended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "container_num_suffix", + pa.string(), + nullable=True, + metadata={ + "comment": "A string that will be appended to the container number.", + "jsonschema_type": "string", + }, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + metadata={ + "comment": "A dictionary of environment variables to be inserted into the container. String values are treated as literals." + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "A set of parameters for a container in a job.\n\nSpecifying files as part of arguments to a job can be done in three ways:\n\n* If the entrypoint command takes a directory as an argument, that\n directory can be specified literally in an argument. All files will be available in the\n input mount point, with paths resulting from the path in S3 as well as the input_roots\n parameter from the job input.\n* If individual filenames need to be specified, use a Parameter instance in\n the positional or environmental arguments. At most one Parameter can\n be specified per parameter set.\n* A Parameter instance can also be used to specify a manifest of file names or\n file IDs to be passed to the container." + }, + ), + pa.field( + "num_containers", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of containers to run in parallel. Input files will be split between the containers.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "cpus", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of CPUs to allocate per container.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "gpus", + pa.int64(), + nullable=True, + metadata={ + "comment": "The number of GPUs to allocate per container. Jobs requesting 0 GPUs are never scheduled on GPU-capable nodes.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "memory", + pa.int64(), + nullable=True, + metadata={ + "comment": "The amount of memory to allocate per container either as the number of bytes or a specification string such as 100MB, 2GB, etc.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "runtime", + pa.string(), + nullable=True, + metadata={ + "comment": "The runtime required for each container as the number of seconds or an ISO8601 duration string.", + "jsonschema_type": "string (duration)", + }, + ), + pa.field( + "input_roots", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + metadata={ + "comment": 'If specified, preserves file hierarchies for S3 files below the given root paths, starting from the bucket. Any files that are not prefixed by a root path are placed in the root directory of the job input dir. If any input files have the same path in the input dir, the job fails. For example, given a input_roots entry of ["mybucket/foo/bar"] and the input files ["otherbucket/foo", "mybucket/bar", "mybucket/foo/bar/baz/bat"] the job input directory would include the files foo, bar, and baz/bat. To preserve hierarchies for all files, set input_roots to [""]. If an input file matches more than one root, the longest root that isn\'t identical to the file path is used. Whitespace on the left side of the path is is ignored. Duplicate roots are ignored.' + }, + ), + pa.field( + "output_dir", + pa.string(), + nullable=False, + metadata={ + "comment": "The S3 folder, starting with the bucket, in which to place results.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Input to a Job, consisting of fields containing small amounts of data.\nSuitable for a list of jobs." + }, + ), + pa.field( + "image", + pa.struct( + [ + pa.field( + "registered_by", + pa.string(), + nullable=False, + metadata={ + "comment": "The username of the user that performed the registration.", + "jsonschema_type": "string", + }, + ), + pa.field( + "registered_on", + pa.string(), + nullable=False, + metadata={ + "comment": "The time of registration.", + "jsonschema_type": "string (date-time)", + }, + ), + pa.field( + "name", + pa.string(), + nullable=False, + metadata={ + "comment": "The normalized name of the a docker image, consisting of the host and path.", + "jsonschema_type": "string", + }, + ), + pa.field( + "digest", + pa.string(), + nullable=False, + metadata={"comment": "The image digest.", "jsonschema_type": "string"}, + ), + pa.field( + "entrypoint", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=False, + metadata={"comment": "The entrypoint command extracted from the image."}, + ), + pa.field( + "tag", + pa.string(), + nullable=True, + metadata={ + "comment": "The image tag at registration time. The tag may no longer point to the same image.", + "jsonschema_type": "string", + }, + ), + pa.field( + "refdata_id", + pa.string(), + nullable=True, + metadata={ + "comment": "The ID of reference data associated with the image. The image requires this reference data to run, and it will be mounted into the image container at the refdata mount point.", + "jsonschema_type": "string", + }, + ), + pa.field( + "default_refdata_mount_point", + pa.string(), + nullable=True, + metadata={ + "comment": "The default mount point for the image refdata in the container. Overridden by the refdata mount point in a job submission if provided there. If a default mount point is provided, then providing a mount point in the job submission is optional. Must be an absolute path.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Information about a Docker image as stored in a Job data structure." + }, + ), + pa.field( + "input_file_count", + pa.int64(), + nullable=False, + metadata={"comment": "The number of input files.", "jsonschema_type": "integer"}, + ), + pa.field( + "cpu_hours", + pa.float64(), + nullable=True, + metadata={ + "comment": "The total CPU hours used by the job, if available.", + "jsonschema_type": "number", + }, + ), + pa.field( + "cpu_factor", + pa.float64(), + nullable=True, + metadata={ + "comment": "The ratio of the cpu time used to the number of cpus requested times the job's actual time, if available. Perfect usage is 1, < 1 means requested CPUs are partially idle, possibly due to memory or IO bounded work, > 1 means more CPU usage than requested.", + "jsonschema_type": "number", + }, + ), + pa.field( + "max_memory", + pa.int64(), + nullable=True, + metadata={ + "comment": "The maximum memory in bytes used by a single container in the job, if available.", + "jsonschema_type": "integer", + }, + ), + pa.field( + "logpath", + pa.string(), + nullable=True, + metadata={ + "comment": "A location in S3 where the logfiles for the job containers can be viewed.", + "jsonschema_type": "string", + }, + ), + ] + ), + nullable=False, + metadata={ + "comment": "Information about a job, consisting of fields containing small amounts of data.\nSuitable for a list of jobs." + }, + ) + ), + nullable=False, + metadata={"comment": "The jobs"}, + ) + ] +) diff --git a/tests/data/jsonschema/examples/cts_jobs/maximal.json b/tests/data/jsonschema/examples/cts_jobs/maximal.json new file mode 100644 index 00000000..7705942d --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/maximal.json @@ -0,0 +1,847 @@ +[ + { + "jobs": [ + { + "output_file_count": 6, + "id": "f4f0a1d9-74e9-4c68-ba42-fa86ca904c99", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T21:01:25.546000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T21:01:25.596000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T21:02:19.450000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T21:02:27.295000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T21:11:50.641000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T21:11:50.821000Z" + }, + { + "state": "complete", + "time": "2026-10-23T21:12:00.259000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T21:12:08.031833+00:00", + "cse_event_processing_no_operation": "2026-10-23T21:12:08.183346+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 601, + "cpu_hours": 0.25747902255694444, + "cpu_factor": 0.1496170812720058, + "max_memory": 708870144 + }, + { + "output_file_count": 37, + "id": "10c04d8e-3c9c-48db-b206-c02e5bb50b8b", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T17:32:06.815000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T17:32:06.843000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T17:32:29.827000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T17:32:31.370000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T18:28:50.554000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T18:28:50.863000Z" + }, + { + "state": "complete", + "time": "2026-10-23T18:29:54.333000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T18:30:01.039121+00:00", + "cse_event_processing_no_operation": "2026-10-23T18:30:01.331790+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9455135589769444, + "cpu_factor": 0.09157269718005094, + "max_memory": 4781117440 + }, + { + "output_file_count": 39, + "id": "cef77650-ca60-4eec-b7f5-c033bde45bf0", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:44:03.561000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:44:03.605000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T15:37:30.176000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T15:37:32.261000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:33:15.496000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:33:15.576000Z" + }, + { + "state": "complete", + "time": "2026-10-23T16:34:50.275000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:34:58.311663+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:34:58.482302+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9353148957144445, + "cpu_factor": 0.09156829683263108, + "max_memory": 5655007232 + }, + { + "error": "An unexpected error occurred", + "id": "633f29d3-ca3f-400c-a8bb-57fabe167051", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:54.692000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:54.734000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T15:36:46.901000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T15:36:47.823000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:32:59.429000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:32:59.514000Z" + }, + { + "state": "error", + "time": "2026-10-23T16:34:33.904000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9432991146552778, + "cpu_factor": 0.09158218888054424, + "max_memory": 5393035264 + }, + { + "error": "An unexpected error occurred", + "id": "1df36a15-549d-45ab-a7f5-e12568d317f6", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:44:01.493000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:44:01.523000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T15:37:10.807000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T15:37:13.096000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:32:24.166000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:32:24.325000Z" + }, + { + "state": "error", + "time": "2026-10-23T16:34:07.132000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9264953317444444, + "cpu_factor": 0.09158733241531918, + "max_memory": 5874806784 + }, + { + "output_file_count": 45, + "id": "a03cd127-7e68-43a9-8989-44bc80923ecd", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:20.775000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:20.816000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:43:52.510000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:43:55.608000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:15:44.432000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:15:44.537000Z" + }, + { + "state": "complete", + "time": "2026-10-23T16:17:43.064000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:17:50.576373+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:17:50.711466+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_90", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 1.5411014602744444, + "cpu_factor": 0.09156035278542289, + "max_memory": 9653383168 + }, + { + "output_file_count": 37, + "id": "7b9e8329-1411-4c82-8d37-f9ca98632033", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:42:57.719000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:42:57.759000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:43:24.637000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:43:27.657000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:04:18.896000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:04:19.050000Z" + }, + { + "state": "complete", + "time": "2026-10-23T16:05:20.327000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:05:27.346896+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:05:27.583906+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_90", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 1.356260786318889, + "cpu_factor": 0.09149990403758473, + "max_memory": 5356208128 + }, + { + "output_file_count": 35, + "id": "1122c90f-ee4d-4232-b613-ccb813db3d8c", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:14.631000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:14.659000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:43:38.356000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:43:41.026000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:58:29.847000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:58:29.903000Z" + }, + { + "state": "complete", + "time": "2026-10-23T15:59:38.462000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:59:45.396630+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:59:45.577656+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_90", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 1.2549099128680556, + "cpu_factor": 0.0915027900246276, + "max_memory": 4348801024 + }, + { + "error": "An unexpected error occurred", + "id": "f6c56d4a-d9bd-4246-a673-91f7c70ed4a3", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:36.781000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:36.831000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:44:00.966000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:44:02.561000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:41:44.311000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:41:44.396000Z" + }, + { + "state": "error", + "time": "2026-10-23T15:43:04.681000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9680559440797222, + "cpu_factor": 0.09152350511957622, + "max_memory": 5595000832 + }, + { + "output_file_count": 37, + "id": "86844795-67ed-4d4b-b981-5eb482f499c6", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:50.339000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:50.367000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:44:10.612000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:44:11.279000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:41:55.483000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:41:55.551000Z" + }, + { + "state": "complete", + "time": "2026-10-23T15:42:57.859000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:43:04.780730+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:43:04.943741+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + "pipeline", + "uniref", + "--dataset-name", + "uniref_100", + "-i", + "/input_dir", + "-o", + "/output_dir", + "-d", + "cts", + "-p", + "True", + "--log-config-file", + "/input_dir/logger_config.json" + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9685900804811111, + "cpu_factor": 0.09150890887030012, + "max_memory": 4710187008 + } + ] + } +] diff --git a/tests/data/jsonschema/examples/cts_jobs/maximal_expected.json b/tests/data/jsonschema/examples/cts_jobs/maximal_expected.json new file mode 100644 index 00000000..fc1ca6a2 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/maximal_expected.json @@ -0,0 +1,1127 @@ +[ + { + "jobs": [ + { + "output_file_count": 6, + "id": "f4f0a1d9-74e9-4c68-ba42-fa86ca904c99", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T21:01:25.546000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T21:01:25.596000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T21:02:19.450000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T21:02:27.295000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T21:11:50.641000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T21:11:50.821000Z" + }, + { + "state": "complete", + "time": "2026-10-23T21:12:00.259000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T21:12:08.031833+00:00", + "cse_event_processing_no_operation": "2026-10-23T21:12:08.183346+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 601, + "cpu_hours": 0.25747902255694444, + "cpu_factor": 0.1496170812720058, + "max_memory": 708870144 + }, + { + "output_file_count": 37, + "id": "10c04d8e-3c9c-48db-b206-c02e5bb50b8b", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T17:32:06.815000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T17:32:06.843000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T17:32:29.827000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T17:32:31.370000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T18:28:50.554000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T18:28:50.863000Z" + }, + { + "state": "complete", + "time": "2026-10-23T18:29:54.333000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T18:30:01.039121+00:00", + "cse_event_processing_no_operation": "2026-10-23T18:30:01.331790+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9455135589769444, + "cpu_factor": 0.09157269718005094, + "max_memory": 4781117440 + }, + { + "output_file_count": 39, + "id": "cef77650-ca60-4eec-b7f5-c033bde45bf0", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:44:03.561000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:44:03.605000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T15:37:30.176000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T15:37:32.261000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:33:15.496000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:33:15.576000Z" + }, + { + "state": "complete", + "time": "2026-10-23T16:34:50.275000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:34:58.311663+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:34:58.482302+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9353148957144445, + "cpu_factor": 0.09156829683263108, + "max_memory": 5655007232 + }, + { + "error": "An unexpected error occurred", + "id": "633f29d3-ca3f-400c-a8bb-57fabe167051", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:54.692000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:54.734000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T15:36:46.901000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T15:36:47.823000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:32:59.429000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:32:59.514000Z" + }, + { + "state": "error", + "time": "2026-10-23T16:34:33.904000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9432991146552778, + "cpu_factor": 0.09158218888054424, + "max_memory": 5393035264 + }, + { + "error": "An unexpected error occurred", + "id": "1df36a15-549d-45ab-a7f5-e12568d317f6", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:44:01.493000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:44:01.523000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T15:37:10.807000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T15:37:13.096000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:32:24.166000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:32:24.325000Z" + }, + { + "state": "error", + "time": "2026-10-23T16:34:07.132000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9264953317444444, + "cpu_factor": 0.09158733241531918, + "max_memory": 5874806784 + }, + { + "output_file_count": 45, + "id": "a03cd127-7e68-43a9-8989-44bc80923ecd", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:20.775000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:20.816000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:43:52.510000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:43:55.608000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:15:44.432000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:15:44.537000Z" + }, + { + "state": "complete", + "time": "2026-10-23T16:17:43.064000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:17:50.576373+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:17:50.711466+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_90" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 1.5411014602744444, + "cpu_factor": 0.09156035278542289, + "max_memory": 9653383168 + }, + { + "output_file_count": 37, + "id": "7b9e8329-1411-4c82-8d37-f9ca98632033", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:42:57.719000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:42:57.759000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:43:24.637000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:43:27.657000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T16:04:18.896000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T16:04:19.050000Z" + }, + { + "state": "complete", + "time": "2026-10-23T16:05:20.327000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:05:27.346896+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:05:27.583906+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_90" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 1.356260786318889, + "cpu_factor": 0.09149990403758473, + "max_memory": 5356208128 + }, + { + "output_file_count": 35, + "id": "1122c90f-ee4d-4232-b613-ccb813db3d8c", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:14.631000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:14.659000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:43:38.356000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:43:41.026000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:58:29.847000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:58:29.903000Z" + }, + { + "state": "complete", + "time": "2026-10-23T15:59:38.462000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:59:45.396630+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:59:45.577656+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_90" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 1.2549099128680556, + "cpu_factor": 0.0915027900246276, + "max_memory": 4348801024 + }, + { + "error": "An unexpected error occurred", + "id": "f6c56d4a-d9bd-4246-a673-91f7c70ed4a3", + "state": "error", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:36.781000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:36.831000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:44:00.966000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:44:02.561000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:41:44.311000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:41:44.396000Z" + }, + { + "state": "error", + "time": "2026-10-23T15:43:04.681000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9680559440797222, + "cpu_factor": 0.09152350511957622, + "max_memory": 5595000832 + }, + { + "output_file_count": 37, + "id": "86844795-67ed-4d4b-b981-5eb482f499c6", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T14:43:50.339000Z" + }, + { + "state": "download_submitted", + "time": "2026-10-23T14:43:50.367000Z" + }, + { + "state": "job_submitting", + "time": "2026-10-23T14:44:10.612000Z" + }, + { + "state": "job_submitted", + "time": "2026-10-23T14:44:11.279000Z" + }, + { + "state": "upload_submitting", + "time": "2026-10-23T15:41:55.483000Z" + }, + { + "state": "upload_submitted", + "time": "2026-10-23T15:41:55.551000Z" + }, + { + "state": "complete", + "time": "2026-10-23T15:42:57.859000Z" + } + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:43:04.780730+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:43:04.943741+00:00" + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": false, + "args": [ + { + "_value": "pipeline" + }, + { + "_value": "uniref" + }, + { + "_value": "--dataset-name" + }, + { + "_value": "uniref_100" + }, + { + "_value": "-i" + }, + { + "_value": "/input_dir" + }, + { + "_value": "-o" + }, + { + "_value": "/output_dir" + }, + { + "_value": "-d" + }, + { + "_value": "cts" + }, + { + "_value": "-p" + }, + { + "_value": "True" + }, + { + "_value": "--log-config-file" + }, + { + "_value": "/input_dir/logger_config.json" + } + ] + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ], + "tag": "v0.1.12" + }, + "input_file_count": 101, + "cpu_hours": 0.9685900804811111, + "cpu_factor": 0.09150890887030012, + "max_memory": 4710187008 + } + ] + } +] diff --git a/tests/data/jsonschema/examples/cts_jobs/maximal_expected_table.py b/tests/data/jsonschema/examples/cts_jobs/maximal_expected_table.py new file mode 100644 index 00000000..3305912b --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/maximal_expected_table.py @@ -0,0 +1,627 @@ +"""Expected PyArrow table for maximal real-world example cts_jobs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "jobs": [ + { + "output_file_count": 6, + "id": "f4f0a1d9-74e9-4c68-ba42-fa86ca904c99", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T21:01:25.546000Z"}, + {"state": "download_submitted", "time": "2026-10-23T21:01:25.596000Z"}, + {"state": "job_submitting", "time": "2026-10-23T21:02:19.450000Z"}, + {"state": "job_submitted", "time": "2026-10-23T21:02:27.295000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T21:11:50.641000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T21:11:50.821000Z"}, + {"state": "complete", "time": "2026-10-23T21:12:00.259000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T21:12:08.031833+00:00", + "cse_event_processing_no_operation": "2026-10-23T21:12:08.183346+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 601, + "cpu_hours": 0.25747902255694444, + "cpu_factor": 0.1496170812720058, + "max_memory": 708870144, + }, + { + "output_file_count": 37, + "id": "10c04d8e-3c9c-48db-b206-c02e5bb50b8b", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T17:32:06.815000Z"}, + {"state": "download_submitted", "time": "2026-10-23T17:32:06.843000Z"}, + {"state": "job_submitting", "time": "2026-10-23T17:32:29.827000Z"}, + {"state": "job_submitted", "time": "2026-10-23T17:32:31.370000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T18:28:50.554000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T18:28:50.863000Z"}, + {"state": "complete", "time": "2026-10-23T18:29:54.333000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T18:30:01.039121+00:00", + "cse_event_processing_no_operation": "2026-10-23T18:30:01.331790+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9455135589769444, + "cpu_factor": 0.09157269718005094, + "max_memory": 4781117440, + }, + { + "output_file_count": 39, + "id": "cef77650-ca60-4eec-b7f5-c033bde45bf0", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:44:03.561000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:44:03.605000Z"}, + {"state": "job_submitting", "time": "2026-10-23T15:37:30.176000Z"}, + {"state": "job_submitted", "time": "2026-10-23T15:37:32.261000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T16:33:15.496000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T16:33:15.576000Z"}, + {"state": "complete", "time": "2026-10-23T16:34:50.275000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:34:58.311663+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:34:58.482302+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9353148957144445, + "cpu_factor": 0.09156829683263108, + "max_memory": 5655007232, + }, + { + "error": "An unexpected error occurred", + "id": "633f29d3-ca3f-400c-a8bb-57fabe167051", + "state": "error", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:43:54.692000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:43:54.734000Z"}, + {"state": "job_submitting", "time": "2026-10-23T15:36:46.901000Z"}, + {"state": "job_submitted", "time": "2026-10-23T15:36:47.823000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T16:32:59.429000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T16:32:59.514000Z"}, + {"state": "error", "time": "2026-10-23T16:34:33.904000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9432991146552778, + "cpu_factor": 0.09158218888054424, + "max_memory": 5393035264, + }, + { + "error": "An unexpected error occurred", + "id": "1df36a15-549d-45ab-a7f5-e12568d317f6", + "state": "error", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:44:01.493000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:44:01.523000Z"}, + {"state": "job_submitting", "time": "2026-10-23T15:37:10.807000Z"}, + {"state": "job_submitted", "time": "2026-10-23T15:37:13.096000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T16:32:24.166000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T16:32:24.325000Z"}, + {"state": "error", "time": "2026-10-23T16:34:07.132000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9264953317444444, + "cpu_factor": 0.09158733241531918, + "max_memory": 5874806784, + }, + { + "output_file_count": 45, + "id": "a03cd127-7e68-43a9-8989-44bc80923ecd", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:43:20.775000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:43:20.816000Z"}, + {"state": "job_submitting", "time": "2026-10-23T14:43:52.510000Z"}, + {"state": "job_submitted", "time": "2026-10-23T14:43:55.608000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T16:15:44.432000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T16:15:44.537000Z"}, + {"state": "complete", "time": "2026-10-23T16:17:43.064000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:17:50.576373+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:17:50.711466+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_90"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 1.5411014602744444, + "cpu_factor": 0.09156035278542289, + "max_memory": 9653383168, + }, + { + "output_file_count": 37, + "id": "7b9e8329-1411-4c82-8d37-f9ca98632033", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:42:57.719000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:42:57.759000Z"}, + {"state": "job_submitting", "time": "2026-10-23T14:43:24.637000Z"}, + {"state": "job_submitted", "time": "2026-10-23T14:43:27.657000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T16:04:18.896000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T16:04:19.050000Z"}, + {"state": "complete", "time": "2026-10-23T16:05:20.327000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T16:05:27.346896+00:00", + "cse_event_processing_no_operation": "2026-10-23T16:05:27.583906+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_90"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 1.356260786318889, + "cpu_factor": 0.09149990403758473, + "max_memory": 5356208128, + }, + { + "output_file_count": 35, + "id": "1122c90f-ee4d-4232-b613-ccb813db3d8c", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:43:14.631000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:43:14.659000Z"}, + {"state": "job_submitting", "time": "2026-10-23T14:43:38.356000Z"}, + {"state": "job_submitted", "time": "2026-10-23T14:43:41.026000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T15:58:29.847000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T15:58:29.903000Z"}, + {"state": "complete", "time": "2026-10-23T15:59:38.462000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:59:45.396630+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:59:45.577656+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_90"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref90/uniref90.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 1.2549099128680556, + "cpu_factor": 0.0915027900246276, + "max_memory": 4348801024, + }, + { + "error": "An unexpected error occurred", + "id": "f6c56d4a-d9bd-4246-a673-91f7c70ed4a3", + "state": "error", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:43:36.781000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:43:36.831000Z"}, + {"state": "job_submitting", "time": "2026-10-23T14:44:00.966000Z"}, + {"state": "job_submitted", "time": "2026-10-23T14:44:02.561000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T15:41:44.311000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T15:41:44.396000Z"}, + {"state": "error", "time": "2026-10-23T15:43:04.681000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": {}, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9680559440797222, + "cpu_factor": 0.09152350511957622, + "max_memory": 5595000832, + }, + { + "output_file_count": 37, + "id": "86844795-67ed-4d4b-b981-5eb482f499c6", + "state": "complete", + "transition_times": [ + {"state": "created", "time": "2026-10-23T14:43:50.339000Z"}, + {"state": "download_submitted", "time": "2026-10-23T14:43:50.367000Z"}, + {"state": "job_submitting", "time": "2026-10-23T14:44:10.612000Z"}, + {"state": "job_submitted", "time": "2026-10-23T14:44:11.279000Z"}, + {"state": "upload_submitting", "time": "2026-10-23T15:41:55.483000Z"}, + {"state": "upload_submitted", "time": "2026-10-23T15:41:55.551000Z"}, + {"state": "complete", "time": "2026-10-23T15:42:57.859000Z"}, + ], + "user": "ialarmedalien", + "admin_meta": { + "cse_event_processing_start": "2026-10-23T15:43:04.780730+00:00", + "cse_event_processing_no_operation": "2026-10-23T15:43:04.943741+00:00", + }, + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": { + "input_mount_point": "/input_dir", + "output_mount_point": "/output_dir", + "declobber": False, + "args": [ + {"_value": "pipeline"}, + {"_value": "uniref"}, + {"_value": "--dataset-name"}, + {"_value": "uniref_100"}, + {"_value": "-i"}, + {"_value": "/input_dir"}, + {"_value": "-o"}, + {"_value": "/output_dir"}, + {"_value": "-d"}, + {"_value": "cts"}, + {"_value": "-p"}, + {"_value": "True"}, + {"_value": "--log-config-file"}, + {"_value": "/input_dir/logger_config.json"}, + ], + }, + "num_containers": 1, + "cpus": 11, + "gpus": 0, + "memory": 100000000000, + "runtime": "PT10M", + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + "tag": "v0.1.12", + }, + "input_file_count": 101, + "cpu_hours": 0.9685900804811111, + "cpu_factor": 0.09150890887030012, + "max_memory": 4710187008, + }, + ] + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/cts_jobs/minimal.json b/tests/data/jsonschema/examples/cts_jobs/minimal.json new file mode 100644 index 00000000..ef947e71 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/minimal.json @@ -0,0 +1,33 @@ +[ + { + "jobs": [ + { + "id": "f4f0a1d9-74e9-4c68-ba42-fa86ca904c99", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T21:01:25.546000Z" + } + ], + "user": "ialarmedalien", + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": {}, + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ] + }, + "input_file_count": 601 + } + ] + } +] diff --git a/tests/data/jsonschema/examples/cts_jobs/minimal_expected.json b/tests/data/jsonschema/examples/cts_jobs/minimal_expected.json new file mode 100644 index 00000000..ef947e71 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/minimal_expected.json @@ -0,0 +1,33 @@ +[ + { + "jobs": [ + { + "id": "f4f0a1d9-74e9-4c68-ba42-fa86ca904c99", + "state": "complete", + "transition_times": [ + { + "state": "created", + "time": "2026-10-23T21:01:25.546000Z" + } + ], + "user": "ialarmedalien", + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": {}, + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/" + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": [ + "./scripts/entrypoint.sh" + ] + }, + "input_file_count": 601 + } + ] + } +] diff --git a/tests/data/jsonschema/examples/cts_jobs/minimal_expected_table.py b/tests/data/jsonschema/examples/cts_jobs/minimal_expected_table.py new file mode 100644 index 00000000..5da98bbb --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/minimal_expected_table.py @@ -0,0 +1,36 @@ +"""Expected PyArrow table for minimal real-world example cts_jobs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "jobs": [ + { + "id": "f4f0a1d9-74e9-4c68-ba42-fa86ca904c99", + "state": "complete", + "transition_times": [{"state": "created", "time": "2026-10-23T21:01:25.546000Z"}], + "user": "ialarmedalien", + "job_input": { + "cluster": "kbase", + "image": "ghcr.io/kbase/cdm-data-loaders@sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "params": {}, + "output_dir": "cts/io/ialarmedalien/datasets/uniprot/cts_output/2026_03/uniref/uniref100/uniref100.xml.gz/", + }, + "image": { + "registered_by": "gavinlocaladmin", + "registered_on": "2026-10-22T02:34:49.514000Z", + "name": "ghcr.io/kbase/cdm-data-loaders", + "digest": "sha256:747f6a055fb3a5dc811373565f61e94df4a9d08d09bf09ecd7be5467f82aebbb", + "entrypoint": ["./scripts/entrypoint.sh"], + }, + "input_file_count": 601, + } + ] + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/cts_jobs/schema.json b/tests/data/jsonschema/examples/cts_jobs/schema.json new file mode 100644 index 00000000..11f29c59 --- /dev/null +++ b/tests/data/jsonschema/examples/cts_jobs/schema.json @@ -0,0 +1,872 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jsonschema:listjobsresponse.json", + "title": "ListJobsResponse", + "$ref": "#/$defs/ListJobsResponse", + "$defs": { + "ListJobsResponse": { + "properties": { + "jobs": { + "items": { + "$ref": "#/$defs/JobPreview" + }, + "type": "array", + "title": "Jobs", + "description": "The jobs" + } + }, + "type": "object", + "required": [ + "jobs" + ], + "title": "ListJobsResponse", + "description": "The response to a successful job listing request. " + }, + "JobPreview": { + "properties": { + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "A description of the error that occurred.", + "examples": [ + "The front fell off" + ] + }, + "output_file_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Output File Count", + "description": "The number of output files, if available.", + "examples": [ + 24 + ] + }, + "id": { + "type": "string", + "title": "Id", + "description": "An opaque, unique string that serves as the job's ID." + }, + "state": { + "$ref": "#/$defs/JobState", + "examples": [ + "complete" + ] + }, + "transition_times": { + "items": { + "$ref": "#/$defs/JobStateTransition" + }, + "type": "array", + "title": "Transition Times", + "description": "A list of job state transitions.", + "examples": [ + [ + { + "state": "created", + "time": "2024-10-24T22:35:40Z" + }, + { + "state": "upload_submitted", + "time": "2024-10-24T22:35:41Z" + }, + { + "state": "job_submitting", + "time": "2024-10-24T22:47:67Z" + } + ] + ] + }, + "user": { + "type": "string", + "title": "User", + "description": "The user that ran the job.", + "examples": [ + "myuserid" + ] + }, + "admin_meta": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "object", + "title": "Admin Meta", + "description": "Metadata for the job set by service administrators. This metadata is mutable and arbitrary from the point of view of the service.", + "examples": [ + { + "children": 31619, + "name": "Getrude" + } + ] + }, + "job_input": { + "$ref": "#/$defs/JobInputPreview" + }, + "image": { + "$ref": "#/$defs/JobImage" + }, + "input_file_count": { + "type": "integer", + "title": "Input File Count", + "description": "The number of input files.", + "examples": [ + 42 + ] + }, + "cpu_hours": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cpu Hours", + "description": "The total CPU hours used by the job, if available.", + "examples": [ + 52.4 + ] + }, + "cpu_factor": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cpu Factor", + "description": "The ratio of the cpu time used to the number of cpus requested times the job's actual time, if available. Perfect usage is 1, < 1 means requested CPUs are partially idle, possibly due to memory or IO bounded work, > 1 means more CPU usage than requested.", + "examples": [ + 0.8 + ] + }, + "max_memory": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Memory", + "description": "The maximum memory in bytes used by a single container in the job, if available.", + "examples": [ + 1032000000 + ] + }, + "logpath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Logpath", + "description": "A location in S3 where the logfiles for the job containers can be viewed.", + "examples": [ + "cts-logs/container_logs/e14a21ba-032d-42f2-b235-d82606675b17" + ] + } + }, + "type": "object", + "required": [ + "id", + "state", + "transition_times", + "user", + "job_input", + "image", + "input_file_count" + ], + "title": "JobPreview", + "description": "Information about a job, consisting of fields containing small amounts of data.\nSuitable for a list of jobs." + }, + "JobState": { + "type": "string", + "enum": [ + "created", + "download_submitted", + "job_submitting", + "job_submitted", + "upload_submitting", + "upload_submitted", + "complete", + "recovering", + "canceling", + "canceled", + "error_processing_submitting", + "error_processing_submitted", + "error" + ], + "title": "JobState", + "description": "The state of a job." + }, + "JobStateTransition": { + "properties": { + "state": { + "$ref": "#/$defs/JobState" + }, + "time": { + "type": "string", + "format": "date-time", + "title": "Time", + "description": "The time at which the new state was entered.", + "examples": [ + "2024-10-24T22:35:40Z" + ] + } + }, + "type": "object", + "required": [ + "state", + "time" + ], + "title": "JobStateTransition", + "description": "Denotes the new state and the entry time when a state change occurs." + }, + "JobInputPreview": { + "properties": { + "cluster": { + "$ref": "#/$defs/Cluster", + "examples": [ + "perlmutter-jaws" + ] + }, + "image": { + "type": "string", + "maxLength": 1000, + "minLength": 1, + "title": "Image", + "description": "The Docker image to run for the job. Include the SHA to ensure the exact code requested is run.", + "examples": [ + "ghcr.io/kbase/collections:checkm2_0.1.6@sha256:c9291c94c382b88975184203100d119cba865c1be91b1c5891749ee02193d380" + ] + }, + "params": { + "$ref": "#/$defs/Parameters-Output" + }, + "num_containers": { + "type": "integer", + "maximum": 1000.0, + "minimum": 1.0, + "title": "Num Containers", + "description": "The number of containers to run in parallel. Input files will be split between the containers.", + "default": 1, + "examples": [ + 1 + ] + }, + "cpus": { + "type": "integer", + "maximum": 256.0, + "minimum": 1.0, + "title": "Cpus", + "description": "The number of CPUs to allocate per container.", + "default": 1, + "examples": [ + 1 + ] + }, + "gpus": { + "type": "integer", + "maximum": 4.0, + "minimum": 0.0, + "title": "Gpus", + "description": "The number of GPUs to allocate per container. Jobs requesting 0 GPUs are never scheduled on GPU-capable nodes.", + "default": 0, + "examples": [ + 1 + ] + }, + "memory": { + "type": "integer", + "minimum": 0.0, + "title": "Memory", + "description": "The amount of memory to allocate per container either as the number of bytes or a specification string such as 100MB, 2GB, etc.", + "default": 10000000, + "examples": [ + "10MB" + ] + }, + "runtime": { + "type": "string", + "format": "duration", + "title": "Runtime", + "description": "The runtime required for each container as the number of seconds or an ISO8601 duration string.", + "default": "PT1M", + "examples": [ + "PT12H30M5S", + 45005 + ] + }, + "input_roots": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Input Roots", + "description": "If specified, preserves file hierarchies for S3 files below the given root paths, starting from the bucket. Any files that are not prefixed by a root path are placed in the root directory of the job input dir. If any input files have the same path in the input dir, the job fails. For example, given a input_roots entry of [\"mybucket/foo/bar\"] and the input files [\"otherbucket/foo\", \"mybucket/bar\", \"mybucket/foo/bar/baz/bat\"] the job input directory would include the files foo, bar, and baz/bat. To preserve hierarchies for all files, set input_roots to [\"\"]. If an input file matches more than one root, the longest root that isn't identical to the file path is used. Whitespace on the left side of the path is is ignored. Duplicate roots are ignored.", + "examples": [ + [ + "mybucket/foo/bar" + ] + ] + }, + "output_dir": { + "type": "string", + "maxLength": 1088, + "minLength": 5, + "title": "Output Dir", + "description": "The S3 folder, starting with the bucket, in which to place results.", + "examples": [ + "mybucket/out" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "cluster", + "image", + "params", + "output_dir" + ], + "title": "JobInputPreview", + "description": "Input to a Job, consisting of fields containing small amounts of data.\nSuitable for a list of jobs." + }, + "Cluster": { + "type": "string", + "enum": [ + "perlmutter-jaws", + "lawrencium-jaws", + "kbase" + ], + "title": "Cluster", + "description": "All clusters that have ever been registered in the system, including those that are no longer\nregistered but may have associated job records.\n\nperlmutter-jaws: The Perlmutter cluster at NERSC run via JAWS. \nlawrencium-jaws: The Lawrencium cluster at LBNL run via JAWS. \nkbase: KBase compute nodes." + }, + "Parameters-Output": { + "properties": { + "input_mount_point": { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^/(?:[\\w.-]+/)*[\\w.-]+/?$", + "title": "Input Mount Point", + "description": "Where to place input files in the container. Must start from the container root and include at least one directory when resolved.", + "default": "/input_files", + "examples": [ + "/input_files" + ] + }, + "output_mount_point": { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^/(?:[\\w.-]+/)*[\\w.-]+/?$", + "title": "Output Mount Point", + "description": "Where output files should be written in the container. Must start from the container root and include at least one directory when resolved. Note that the mount point is not included in the file paths thatare eventually transferred to S3.", + "default": "/output_files", + "examples": [ + "/output_files" + ] + }, + "declobber": { + "type": "boolean", + "title": "Declobber", + "description": "If true, prepend the container number to all output file paths as a directory. This prevents containers that write to the same file path from clobbering each other and losing data. If the output files of containers are guaranteed to be unique (e.g. output file names are based on input file names and input file names are unique) then this setting is not necessary. The same effect can be created by use of a container number parameter when specifying the output directory in the arguments or environment.", + "default": false + }, + "refdata_mount_point": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^/(?:[\\w.-]+/)*[\\w.-]+/?$" + }, + { + "type": "null" + } + ], + "title": "Refdata Mount Point", + "description": "Where reference data files should be pleased in the container. Must start from the container root and include at least one directory when resolved. Required if the job image requires reference data and a default reference data mount point isn't specified for the image. If a default is provided, this setting overrides it.", + "examples": [ + "/reference_data" + ] + }, + "args": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "string", + "maxLength": 256, + "minLength": 2, + "pattern": "^--?[a-zA-Z0-9][\\w\\.-]*=?$" + }, + { + "$ref": "#/$defs/ParameterWithFlag" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Args", + "description": "A list of command line arguments to be inserted after the container entrypoint command. Strings are treated as literals.", + "examples": [ + [ + "process", + "--output-dir", + "/output_files", + { + "flag": "--input-file=", + "input_files_format": "repeat_parameter", + "type": "input_files" + }, + "--jobid", + { + "type": "container_number" + } + ] + ] + }, + "environment": { + "anyOf": [ + { + "patternProperties": { + "^[A-Z_][A-Z0-9_]*$": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "$ref": "#/$defs/Parameter" + } + ] + } + }, + "propertyNames": { + "maxLength": 256, + "minLength": 1 + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Environment", + "description": "A dictionary of environment variables to be inserted into the container. String values are treated as literals.", + "examples": [ + { + "DIAMOND_DB_PATH": "/reference_data", + "FILE_MANIFEST": { + "manifest_file_format": "files", + "manifest_file_header": "infiles", + "type": "manifest_file" + } + } + ] + } + }, + "additionalProperties": false, + "type": "object", + "title": "Parameters", + "description": "A set of parameters for a container in a job.\n\nSpecifying files as part of arguments to a job can be done in three ways:\n\n* If the entrypoint command takes a directory as an argument, that\n directory can be specified literally in an argument. All files will be available in the\n input mount point, with paths resulting from the path in S3 as well as the input_roots\n parameter from the job input.\n* If individual filenames need to be specified, use a Parameter instance in\n the positional or environmental arguments. At most one Parameter can\n be specified per parameter set.\n* A Parameter instance can also be used to specify a manifest of file names or\n file IDs to be passed to the container." + }, + "ParameterWithFlag": { + "properties": { + "type": { + "$ref": "#/$defs/ParameterType", + "examples": [ + "input_files" + ] + }, + "input_files_format": { + "anyOf": [ + { + "$ref": "#/$defs/InputFilesFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "comma_separated_list" + ] + }, + "manifest_file_format": { + "anyOf": [ + { + "$ref": "#/$defs/ManifestFileFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "files" + ] + }, + "manifest_file_header": { + "anyOf": [ + { + "type": "string", + "maxLength": 1000, + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Manifest File Header", + "description": "The header for the manifest file, if any.", + "examples": [ + "genome_id" + ] + }, + "container_num_prefix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Prefix", + "description": "A string that will be prepended to the container number.", + "examples": [ + "/output/job-" + ] + }, + "container_num_suffix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Suffix", + "description": "A string that will be appended to the container number.", + "examples": [ + "_output" + ] + }, + "flag": { + "anyOf": [ + { + "type": "string", + "maxLength": 256, + "minLength": 2, + "pattern": "^--?[a-zA-Z0-9][\\w\\.-]*=?$" + }, + { + "type": "null" + } + ], + "title": "Flag", + "description": "A command line argument flag to precede the parameter. If the flag ends with '=', no space is left between the flag and its value.", + "examples": [ + "--file" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "title": "ParameterWithFlag", + "description": "Represents a special parameter passed to a container, optionally with a\npreceding flag argument.\n\nCan represent a set of input file names formatted in various ways, a manifest file\ncontaining file paths or data IDs, or a container number.\n\ninput_files_format is required when the type is input_files, but ignored otherwise. \nmanifest_file_format is required when the type is manifest_file, but ignored otherwise. \nmanifest_file_header is optional and only taken into account when the type is manifest_file. " + }, + "ParameterType": { + "type": "string", + "enum": [ + "input_files", + "manifest_file", + "container_number" + ], + "title": "ParameterType", + "description": "The type of a parameter if not a string literal.\n\ninput_files will cause the list of input files for the container to be inserted\ninto the entrypoint command line or an environment variable.\n\nmanifest_file will cause the list of input files for the container to be placed in a\nmanifest file and will insert the file location into the entrypoint command line\nor an environment variable.\n\ncontainer_number will insert the integer number of the container into the\nentrypoint command line or an environment variable. This can be used so prevent file path\ncollisions if multiple containers write to the same file paths." + }, + "InputFilesFormat": { + "type": "string", + "enum": [ + "comma_separated_list", + "space_separated_list", + "repeat_parameter" + ], + "title": "InputFilesFormat", + "description": "The format of an input files type parameter.\n\nComma and space separated lists format the list of files as one would expect.\n\nRepeat parameter causes the input files to be inserted as a space separated list,\nwith each file preceded by the parameter flag. Only valid for CLI flag based arguments.\nIf the flag ends with an equals sign, there is no space left between the flag and the\nparameter.\n\nExamples: \n`-i`: `-i file1 -i file2 ... -i fileN` \n`--input_file=`: `--input_file=file1 --input_file=file2 ... --input_file=fileN` " + }, + "ManifestFileFormat": { + "type": "string", + "enum": [ + "files", + "data_ids" + ], + "title": "ManifestFileFormat", + "description": "The format of a manifest file.\n\nfiles will cause the manifest file to consist of a list of the input files, one per line.\n\ndata_ids will cause the manifest file to consist of a list of the data IDs associated with\nthe input files, one per line." + }, + "Parameter": { + "properties": { + "type": { + "$ref": "#/$defs/ParameterType", + "examples": [ + "input_files" + ] + }, + "input_files_format": { + "anyOf": [ + { + "$ref": "#/$defs/InputFilesFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "comma_separated_list" + ] + }, + "manifest_file_format": { + "anyOf": [ + { + "$ref": "#/$defs/ManifestFileFormat" + }, + { + "type": "null" + } + ], + "examples": [ + "files" + ] + }, + "manifest_file_header": { + "anyOf": [ + { + "type": "string", + "maxLength": 1000, + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Manifest File Header", + "description": "The header for the manifest file, if any.", + "examples": [ + "genome_id" + ] + }, + "container_num_prefix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Prefix", + "description": "A string that will be prepended to the container number.", + "examples": [ + "/output/job-" + ] + }, + "container_num_suffix": { + "anyOf": [ + { + "type": "string", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[\\w\\./][\\w\\.\\,+/-]*$" + }, + { + "type": "null" + } + ], + "title": "Container Num Suffix", + "description": "A string that will be appended to the container number.", + "examples": [ + "_output" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "title": "Parameter", + "description": "Represents a special parameter passed to a container.\n\nCan represent a set of input file names formatted in various ways, a manifest file\ncontaining file paths or data IDs, or a container number.\n\ninput_files_format is required when the type is input_files, but ignored otherwise. \nmanifest_file_format is required when the type is manifest_file, but ignored otherwise. \nmanifest_file_header is optional and only taken into account when the type is manifest_file. " + }, + "JobImage": { + "properties": { + "registered_by": { + "type": "string", + "title": "Registered By", + "description": "The username of the user that performed the registration.", + "examples": [ + "aparkin" + ] + }, + "registered_on": { + "type": "string", + "format": "date-time", + "title": "Registered On", + "description": "The time of registration.", + "examples": [ + "2024-10-24T22:35:40Z" + ] + }, + "name": { + "type": "string", + "title": "Name", + "description": "The normalized name of the a docker image, consisting of the host and path.", + "examples": [ + "ghcr.io/kbase/collections" + ] + }, + "digest": { + "type": "string", + "title": "Digest", + "description": "The image digest.", + "examples": [ + "sha256:c9291c94c382b88975184203100d119cba865c1be91b1c5891749ee02193d380" + ] + }, + "entrypoint": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Entrypoint", + "description": "The entrypoint command extracted from the image.", + "examples": [ + [ + "checkm2", + "predict" + ] + ] + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag", + "description": "The image tag at registration time. The tag may no longer point to the same image.", + "examples": [ + "checkm2_0.1.6" + ] + }, + "refdata_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Refdata Id", + "description": "The ID of reference data associated with the image. The image requires this reference data to run, and it will be mounted into the image container at the refdata mount point.", + "examples": [ + "3a28c155-ea8b-4e1b-baef-242d991a8200" + ] + }, + "default_refdata_mount_point": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Default Refdata Mount Point", + "description": "The default mount point for the image refdata in the container. Overridden by the refdata mount point in a job submission if provided there. If a default mount point is provided, then providing a mount point in the job submission is optional. Must be an absolute path.", + "examples": [ + "/reference_data" + ] + } + }, + "type": "object", + "required": [ + "registered_by", + "registered_on", + "name", + "digest", + "entrypoint" + ], + "title": "JobImage", + "description": "Information about a Docker image as stored in a Job data structure." + } + } +} diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/case.yaml b/tests/data/jsonschema/examples/ncbi_annotation_report/case.yaml new file mode 100644 index 00000000..9cdba902 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/case.yaml @@ -0,0 +1,17 @@ +title: NCBI Datasets `AnnotationReportResponse` +schema: schema.json +rules: [R-ROOT-REF, R-SHAPE-REF, R-SHAPE-ARRAY, R-SCALAR-ENUM-CONST] +notes: | + Deeply nested NCBI Datasets v2 schema (`parsers/ncbi/datasets_api/AnnotationReportResponse.schema.json`) with a + root `$ref`, many `$defs`, arrays of structs, enums, integers and booleans. +runs: + - id: minimal + data: minimal.json + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.json + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/expected_schema.py b/tests/data/jsonschema/examples/ncbi_annotation_report/expected_schema.py new file mode 100644 index 00000000..5cc59aa4 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/expected_schema.py @@ -0,0 +1,234 @@ +"""Expected PyArrow schema for real-world example ncbi_annotation_report.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "reports", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "annotation", + pa.struct( + [ + pa.field( + "gene_id", + pa.string(), + nullable=True, + metadata={ + "comment": "NCBI GeneID", + "jsonschema_type": "string (uint64)", + }, + ), + pa.field( + "symbol", + pa.string(), + nullable=True, + metadata={ + "comment": "Gene symbol", + "jsonschema_type": "string", + }, + ), + pa.field( + "description", + pa.string(), + nullable=True, + metadata={ + "comment": "Gene name", + "jsonschema_type": "string", + }, + ), + pa.field( + "tax_id", + pa.string(), + nullable=True, + metadata={ + "comment": "NCBI Taxonomy ID for the organism", + "jsonschema_type": "string (uint64)", + }, + ), + pa.field( + "taxname", + pa.string(), + nullable=True, + metadata={ + "comment": "Taxonomic name of the organism", + "jsonschema_type": "string", + }, + ), + pa.field( + "gene_type", + pa.string(), + nullable=True, + metadata={ + "comment": "Gene locus type", + "jsonschema_type": "string", + }, + ), + pa.field( + "orientation", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "locus_tag", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "annotations", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "assembly_accession", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "assembly_name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "annotation_name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "annotation_release_date", + pa.date32(), + nullable=True, + metadata={"jsonschema_type": "string (date)"}, + ), + pa.field( + "genomic_locations", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "genomic_accession_version", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "sequence_name", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "genomic_range", + pa.struct( + [ + pa.field( + "begin", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string (int64)" + }, + ), + pa.field( + "end", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string (int64)" + }, + ), + pa.field( + "orientation", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "order", + pa.int64(), + nullable=True, + metadata={ + "jsonschema_type": "integer" + }, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "query", + pa.list_( + pa.field( + "item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"} + ), + ), + nullable=True, + ), + pa.field("row_id", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + pa.field( + "total_count", + pa.int64(), + nullable=True, + metadata={ + "comment": "The total count of available genes (ignoring the page_size parameter).", + "jsonschema_type": "integer", + }, + ), + pa.field( + "next_page_token", + pa.string(), + nullable=True, + metadata={ + "comment": "A token that can be sent as page_token to retrieve the next page.", + "jsonschema_type": "string", + }, + ), + ] +) diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/maximal.json b/tests/data/jsonschema/examples/ncbi_annotation_report/maximal.json new file mode 100644 index 00000000..eede8a90 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/maximal.json @@ -0,0 +1,45 @@ +[ + { + "reports": [ + { + "annotation": { + "gene_id": "944742", + "symbol": "thrL", + "description": "thr operon leader peptide", + "tax_id": "511145", + "taxname": "Escherichia coli str. K-12 substr. MG1655", + "gene_type": "PROTEIN_CODING", + "orientation": "plus", + "locus_tag": "b0001", + "annotations": [ + { + "assembly_accession": "GCF_000005845.2", + "assembly_name": "ASM584v2", + "annotation_name": "NCBI Escherichia coli str. K-12 substr. MG1655 Annotation Release 102", + "annotation_release_date": "2013-09-26", + "genomic_locations": [ + { + "genomic_accession_version": "NC_000913.3", + "sequence_name": "Chromosome", + "genomic_range": { + "begin": "190", + "end": "255", + "orientation": "plus", + "order": 1 + } + } + ] + } + ] + }, + "query": [ + "b0001" + ], + "row_id": "1" + } + ], + "total_count": 4639, + "next_page_token": "tok_next_page_002" + } +] + diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected.json b/tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected.json new file mode 100644 index 00000000..eede8a90 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected.json @@ -0,0 +1,45 @@ +[ + { + "reports": [ + { + "annotation": { + "gene_id": "944742", + "symbol": "thrL", + "description": "thr operon leader peptide", + "tax_id": "511145", + "taxname": "Escherichia coli str. K-12 substr. MG1655", + "gene_type": "PROTEIN_CODING", + "orientation": "plus", + "locus_tag": "b0001", + "annotations": [ + { + "assembly_accession": "GCF_000005845.2", + "assembly_name": "ASM584v2", + "annotation_name": "NCBI Escherichia coli str. K-12 substr. MG1655 Annotation Release 102", + "annotation_release_date": "2013-09-26", + "genomic_locations": [ + { + "genomic_accession_version": "NC_000913.3", + "sequence_name": "Chromosome", + "genomic_range": { + "begin": "190", + "end": "255", + "orientation": "plus", + "order": 1 + } + } + ] + } + ] + }, + "query": [ + "b0001" + ], + "row_id": "1" + } + ], + "total_count": 4639, + "next_page_token": "tok_next_page_002" + } +] + diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected_table.py b/tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected_table.py new file mode 100644 index 00000000..4cc3cb32 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/maximal_expected_table.py @@ -0,0 +1,54 @@ +"""Expected PyArrow table for maximal real-world example ncbi_annotation_report.""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "reports": [ + { + "annotation": { + "gene_id": "944742", + "symbol": "thrL", + "description": "thr operon leader peptide", + "tax_id": "511145", + "taxname": "Escherichia coli str. K-12 substr. MG1655", + "gene_type": "PROTEIN_CODING", + "orientation": "plus", + "locus_tag": "b0001", + "annotations": [ + { + "assembly_accession": "GCF_000005845.2", + "assembly_name": "ASM584v2", + "annotation_name": "NCBI Escherichia coli str. K-12 substr. MG1655 Annotation Release 102", + "annotation_release_date": dt.date(2013, 9, 26), + "genomic_locations": [ + { + "genomic_accession_version": "NC_000913.3", + "sequence_name": "Chromosome", + "genomic_range": { + "begin": "190", + "end": "255", + "orientation": "plus", + "order": 1, + }, + } + ], + } + ], + }, + "query": ["b0001"], + "row_id": "1", + } + ], + "total_count": 4639, + "next_page_token": "tok_next_page_002", + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/minimal.json b/tests/data/jsonschema/examples/ncbi_annotation_report/minimal.json new file mode 100644 index 00000000..24a43464 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/minimal.json @@ -0,0 +1,6 @@ +[ + { + "total_count": 1 + } +] + diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected.json b/tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected.json new file mode 100644 index 00000000..24a43464 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected.json @@ -0,0 +1,6 @@ +[ + { + "total_count": 1 + } +] + diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected_table.py b/tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected_table.py new file mode 100644 index 00000000..a184a0af --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/minimal_expected_table.py @@ -0,0 +1,13 @@ +"""Expected PyArrow table for minimal real-world example ncbi_annotation_report.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + {"total_count": 1}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/ncbi_annotation_report/schema.json b/tests/data/jsonschema/examples/ncbi_annotation_report/schema.json new file mode 100644 index 00000000..12cb17e8 --- /dev/null +++ b/tests/data/jsonschema/examples/ncbi_annotation_report/schema.json @@ -0,0 +1,153 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jsonschema:annotationreportresponse.json", + "title": "AnnotationReportResponse", + "$ref": "#/$defs/v2reportsGenomeAnnotationReportPage", + "$defs": { + "v2reportsGenomeAnnotationReportPage": { + "type": "object", + "properties": { + "reports": { + "type": "array", + "items": { + "$ref": "#/$defs/v2reportsGenomeAnnotationReportMatch" + } + }, + "total_count": { + "type": "integer", + "description": "The total count of available genes (ignoring the page_size parameter)." + }, + "next_page_token": { + "type": "string", + "description": "A token that can be sent as page_token to retrieve the next page." + } + } + }, + "v2reportsGenomeAnnotationReportMatch": { + "type": "object", + "properties": { + "annotation": { + "$ref": "#/$defs/v2reportsGenomeAnnotation" + }, + "query": { + "type": "array", + "items": { + "type": "string" + } + }, + "row_id": { + "type": "string" + } + } + }, + "v2reportsGenomeAnnotation": { + "type": "object", + "properties": { + "gene_id": { + "type": "string", + "format": "uint64", + "description": "NCBI GeneID" + }, + "symbol": { + "type": "string", + "description": "Gene symbol" + }, + "description": { + "type": "string", + "description": "Gene name" + }, + "tax_id": { + "type": "string", + "format": "uint64", + "description": "NCBI Taxonomy ID for the organism" + }, + "taxname": { + "type": "string", + "description": "Taxonomic name of the organism" + }, + "gene_type": { + "type": "string", + "description": "Gene locus type" + }, + "orientation": { + "$ref": "#/$defs/v2reportsOrientation" + }, + "locus_tag": { + "type": "string" + }, + "annotations": { + "type": "array", + "items": { + "$ref": "#/$defs/v2reportsAnnotation" + } + } + } + }, + "v2reportsOrientation": { + "type": "string", + "enum": [ + "none", + "plus", + "minus" + ], + "default": "none" + }, + "v2reportsAnnotation": { + "type": "object", + "properties": { + "assembly_accession": { + "type": "string" + }, + "assembly_name": { + "type": "string" + }, + "annotation_name": { + "type": "string" + }, + "annotation_release_date": { + "type": "string", + "format": "date" + }, + "genomic_locations": { + "type": "array", + "items": { + "$ref": "#/$defs/v2reportsGenomicLocation" + } + } + } + }, + "v2reportsGenomicLocation": { + "type": "object", + "properties": { + "genomic_accession_version": { + "type": "string" + }, + "sequence_name": { + "type": "string" + }, + "genomic_range": { + "$ref": "#/$defs/v2reportsRange" + } + } + }, + "v2reportsRange": { + "type": "object", + "properties": { + "begin": { + "type": "string", + "format": "int64" + }, + "end": { + "type": "string", + "format": "int64" + }, + "orientation": { + "$ref": "#/$defs/v2reportsOrientation" + }, + "order": { + "type": "integer" + } + } + } + } +} diff --git a/tests/data/jsonschema/examples/uniprot_search/case.yaml b/tests/data/jsonschema/examples/uniprot_search/case.yaml new file mode 100644 index 00000000..50386d9d --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/case.yaml @@ -0,0 +1,18 @@ +title: UniProtKB `SearchResult` +schema: schema.json +rules: [R-UNION-OBJECT, R-UNION-SCALAR-OBJECT, R-RECURSION, R-SHAPE-MAP, R-SCALAR-TYPE] +notes: | + `/uniprotkb/search` response schema from `not_for_github/uniprot/api/openapi-3.1.json` + (`#/components/schemas/SearchResult`): a 24-branch polymorphic `Comment` `anyOf` (including a string + object + `note` with a synthesized `_value`), recursive `ProteinSection`, `extraAttributes` maps and `pa.date32()` dates. +runs: + - id: minimal + data: minimal.json + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.json + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/jsonschema/examples/uniprot_search/expected_schema.py b/tests/data/jsonschema/examples/uniprot_search/expected_schema.py new file mode 100644 index 00000000..4cb744c9 --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/expected_schema.py @@ -0,0 +1,5905 @@ +"""Expected PyArrow schema for real-world example uniprot_search.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "results", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "entryAudit", + pa.struct( + [ + pa.field( + "sequenceVersion", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "firstPublicDate", + pa.date32(), + nullable=True, + metadata={"jsonschema_type": "string (date)"}, + ), + pa.field( + "lastAnnotationUpdateDate", + pa.date32(), + nullable=True, + metadata={"jsonschema_type": "string (date)"}, + ), + pa.field( + "lastSequenceUpdateDate", + pa.date32(), + nullable=True, + metadata={"jsonschema_type": "string (date)"}, + ), + pa.field( + "entryVersion", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "organismHosts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "taxonId", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "commonName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "synonyms", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "scientificName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field("entryType", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "annotationScore", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"} + ), + pa.field( + "secondaryAccessions", + pa.list_( + pa.field( + "item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"} + ) + ), + nullable=True, + ), + pa.field( + "proteinDescription", + pa.struct( + [ + pa.field( + "alternativeNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "shortNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "shortNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "submissionNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "includes", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "alternativeNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "shortNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "shortNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "innNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "innNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "contains", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "alternativeNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "shortNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "recommendedName", + pa.struct( + [ + pa.field( + "shortNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumbers", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "fullName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "allergenName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "biotechName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "cdAntigenNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "innNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidence", + pa.list_( + pa.field( + "item", + pa.int64(), + nullable=False, + metadata={ + "jsonschema_type": "integer" + }, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "flag", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + ] + ), + nullable=True, + ), + pa.field( + "uniProtKBCrossReferences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "isoformId", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "database", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "genes", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "orderedLocusNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "geneName", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "orfNames", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "synonyms", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "comments", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "commentType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "isoforms", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "isoformIds", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "sequenceIds", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "isoformSequenceStatus", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "note", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={ + "jsonschema_type": "boolean" + }, + ), + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "synonyms", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "name", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "events", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "note", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "_value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "kineticParameters", + pa.struct( + [ + pa.field( + "maximumVelocities", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "unit", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "enzyme", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "velocity", + pa.float64(), + nullable=True, + metadata={ + "jsonschema_type": "number" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "michaelisConstants", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "constant", + pa.float64(), + nullable=True, + metadata={ + "jsonschema_type": "number" + }, + ), + pa.field( + "unit", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "substrate", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "note", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "absorption", + pa.struct( + [ + pa.field( + "approximate", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "note", + pa.struct( + [ + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "max", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "phDependence", + pa.struct( + [ + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ) + ] + ), + nullable=True, + ), + pa.field( + "redoxPotential", + pa.struct( + [ + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ) + ] + ), + nullable=True, + ), + pa.field( + "temperatureDependence", + pa.struct( + [ + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ) + ] + ), + nullable=True, + ), + pa.field( + "molecule", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "physiologicalReactions", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "reactionCrossReference", + pa.struct( + [ + pa.field( + "database", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=True, + ), + pa.field( + "directionType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "reaction", + pa.struct( + [ + pa.field( + "reactionCrossReferences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "database", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "ecNumber", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "cofactors", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "cofactorCrossReference", + pa.struct( + [ + pa.field( + "database", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=True, + ), + pa.field( + "name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "valid", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "disease", + pa.struct( + [ + pa.field( + "diseaseAccession", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "diseaseCrossReference", + pa.struct( + [ + pa.field( + "database", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "diseaseId", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "acronym", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "description", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "texts", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "interactions", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "numberOfExperiments", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "interactantOne", + pa.struct( + [ + pa.field( + "geneName", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "chainId", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "intActId", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "uniProtKBAccession", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=True, + ), + pa.field( + "organismDiffer", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "interactantTwo", + pa.struct( + [ + pa.field( + "geneName", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "chainId", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "intActId", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "uniProtKBAccession", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "molWeight", + pa.float64(), + nullable=True, + metadata={"jsonschema_type": "number"}, + ), + pa.field( + "molWeightError", + pa.float64(), + nullable=True, + metadata={"jsonschema_type": "number"}, + ), + pa.field( + "method", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "positions", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "position", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "locationType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "sequenceCautionType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "sequence", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "subcellularLocations", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "topology", + pa.struct( + [ + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "orientation", + pa.struct( + [ + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "location", + pa.struct( + [ + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "resourceUrl", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "ftp", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "resourceName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "geneLocations", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "geneEncodingType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "keywords", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "category", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "sequence", + pa.struct( + [ + pa.field( + "type", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + pa.field( + "ref", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + pa.field( + "value", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + pa.field( + "length", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"} + ), + pa.field( + "molWeight", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "crc64", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + pa.field( + "md5", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + ] + ), + nullable=True, + ), + pa.field( + "organism", + pa.struct( + [ + pa.field( + "lineages", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "taxonId", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "commonName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "synonyms", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "scientificName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "lineage", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "lineages", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "rank", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "taxonId", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "hidden", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "commonName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "synonyms", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "scientificName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "proteinExistence", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + pa.field("uniProtkbId", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "primaryAccession", pa.string(), nullable=True, metadata={"jsonschema_type": "string"} + ), + pa.field( + "inactiveReason", + pa.struct( + [ + pa.field( + "deletedReason", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "inactiveReasonType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "mergeDemergeTos", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "references", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "referenceComments", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "type", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "referenceNumber", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "referencePositions", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "citation", + pa.struct( + [ + pa.field( + "firstPage", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "lastPage", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "volume", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "publisher", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "bookName", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "editors", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "address", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "citationCrossReferences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "database", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "authors", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "publicationDate", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "authoringGroups", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "title", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "authoringGroup", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + pa.field( + "journal", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "locator", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "literatureAbstract", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "completeAuthorList", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "pubmedId", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "doiId", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "patentNumber", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "submissionDatabase", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "institute", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "citationType", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "internalSection", + pa.struct( + [ + pa.field( + "internalLines", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "type", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "sourceLines", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ) + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "evidenceLines", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidence", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "createDate", + pa.date32(), + nullable=True, + metadata={"jsonschema_type": "string (date)"}, + ), + pa.field( + "curator", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field("fragment", pa.bool_(), nullable=True, metadata={"jsonschema_type": "boolean"}), + pa.field( + "features", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "alternativeSequence", + pa.struct( + [ + pa.field( + "originalSequence", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "alternativeSequences", + pa.list_( + pa.field( + "item", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ) + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "featureId", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "ligand", + pa.struct( + [ + pa.field( + "note", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "ligandPart", + pa.struct( + [ + pa.field( + "note", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "name", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "featureCrossReferences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "database", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "properties", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "key", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={ + "jsonschema_type": "string" + }, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "location", + pa.struct( + [ + pa.field( + "start", + pa.struct( + [ + pa.field( + "value", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "modifier", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "end", + pa.struct( + [ + pa.field( + "value", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "modifier", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "sequence", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + pa.field( + "type", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "description", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "evidences", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "evidenceCode", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "id", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "source", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field("active", pa.bool_(), nullable=True, metadata={"jsonschema_type": "boolean"}), + pa.field( + "extraAttributes", + pa.struct( + [ + pa.field( + "countByCommentType", + pa.map_( + pa.string(), + pa.field( + "value", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ), + ), + nullable=True, + ), + pa.field( + "countByFeatureType", + pa.map_( + pa.string(), + pa.field( + "value", + pa.int64(), + nullable=False, + metadata={"jsonschema_type": "integer"}, + ), + ), + nullable=True, + ), + pa.field( + "uniParcId", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "facets", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("label", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("name", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "allowMultipleSelection", + pa.bool_(), + nullable=True, + metadata={"jsonschema_type": "boolean"}, + ), + pa.field( + "values", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "value", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "count", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "matchedFields", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("name", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("hits", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "suggestions", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("query", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("hits", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + pa.field( + "warnings", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("code", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("message", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ) + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/examples/uniprot_search/maximal.json b/tests/data/jsonschema/examples/uniprot_search/maximal.json new file mode 100644 index 00000000..b5d3e82a --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/maximal.json @@ -0,0 +1,5704 @@ +[ + { + "results": [ + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "X5I2V6", + "uniProtkbId": "X5I2V6_9PARA", + "entryAudit": { + "firstPublicDate": "2014-06-11", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2014-06-11", + "entryVersion": 59, + "sequenceVersion": 1 + }, + "annotationScore": 4, + "organism": { + "scientificName": "Feline morbillivirus", + "taxonId": 1170234, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAO58314.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Negarnaviricota", + "Haploviricotina", + "Monjiviricetes", + "Mononegavirales", + "Paramyxoviridae", + "Orthoparamyxovirinae", + "Morbillivirus", + "Morbillivirus felis" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "RNA-directed RNA polymerase L" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "Protein L" + } + ] + }, + "alternativeNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "Large structural protein" + } + }, + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "Replicase" + } + }, + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "Transcriptase" + } + } + ], + "includes": [ + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "RNA-directed RNA polymerase" + }, + "ecNumbers": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "2.7.7.48" + } + ] + } + }, + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "GTP phosphohydrolase" + }, + "ecNumbers": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "3.6.1.-" + } + ] + } + }, + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "GDP polyribonucleotidyltransferase" + }, + "ecNumbers": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "2.7.7.88" + } + ] + }, + "alternativeNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "PRNTase" + } + } + ] + }, + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "mRNA (nucleoside-2'-O-)-methyltransferase" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "N1-2'-O-MTase" + } + ] + } + }, + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "mRNA (guanine-N(7)-)-methyltransferase" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "G-N7-MTase" + } + ] + } + } + ] + }, + "genes": [ + { + "geneName": { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAO58314.1" + } + ], + "value": "L" + } + } + ], + "comments": [ + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "name": "RNA(n) + a ribonucleoside 5'-triphosphate = RNA(n+1) + diphosphate", + "reactionCrossReferences": [ + { + "database": "Rhea", + "id": "RHEA:21248" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:14527" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:17342" + }, + { + "database": "ChEBI", + "id": "CHEBI:33019" + }, + { + "database": "ChEBI", + "id": "CHEBI:61557" + }, + { + "database": "ChEBI", + "id": "CHEBI:140395" + } + ], + "ecNumber": "2.7.7.48", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PROSITE-ProRule", + "id": "PRU00539" + } + ] + } + }, + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "Virion", + "id": "SL-0274" + } + }, + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "Host cytoplasm", + "id": "SL-0381" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00003132" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "value": "RNA-directed RNA polymerase that catalyzes the replication of viral genomic RNA. The template is composed of the viral RNA tightly encapsidated by the nucleoprotein (N). The replicase mode is dependent on intracellular N protein concentration. In this mode, the polymerase replicates the whole viral genome without recognizing transcriptional signals, and the replicated genome is not caped or polyadenylated" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Domain", + "location": { + "start": { + "value": 672, + "modifier": "EXACT" + }, + "end": { + "value": 856, + "modifier": "EXACT" + } + }, + "description": "RdRp catalytic", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "source": "PROSITE", + "id": "PS50526" + } + ] + }, + { + "type": "Domain", + "location": { + "start": { + "value": 1771, + "modifier": "EXACT" + }, + "end": { + "value": 1976, + "modifier": "EXACT" + } + }, + "description": "Mononegavirus-type SAM-dependent 2'-O-MTase", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "source": "PROSITE", + "id": "PS51590" + } + ] + }, + { + "type": "Region", + "location": { + "start": { + "value": 620, + "modifier": "EXACT" + }, + "end": { + "value": 658, + "modifier": "EXACT" + } + }, + "description": "Disordered", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "MobiDB-lite" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022840" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0067", + "category": "Ligand", + "name": "ATP-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023200" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-1035", + "category": "Cellular component", + "name": "Host cytoplasm" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022801" + } + ], + "id": "KW-0378", + "category": "Molecular function", + "name": "Hydrolase" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022842" + } + ], + "id": "KW-0460", + "category": "Ligand", + "name": "Magnesium" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022723" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022603" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0489", + "category": "Molecular function", + "name": "Methyltransferase" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023042" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0506", + "category": "Biological process", + "name": "mRNA capping" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022664" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0507", + "category": "Biological process", + "name": "mRNA processing" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023268" + } + ], + "id": "KW-0511", + "category": "Molecular function", + "name": "Multifunctional enzyme" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022741" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0547", + "category": "Ligand", + "name": "Nucleotide-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022695" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0548", + "category": "Molecular function", + "name": "Nucleotidyltransferase" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022484" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0696", + "category": "Molecular function", + "name": "RNA-directed RNA polymerase" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022691" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0949", + "category": "Ligand", + "name": "S-adenosyl-L-methionine" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022679" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0808", + "category": "Molecular function", + "name": "Transferase" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022953" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0693", + "category": "Biological process", + "name": "Viral RNA replication" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022844" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR000830" + } + ], + "id": "KW-0946", + "category": "Cellular component", + "name": "Virion" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022833" + } + ], + "id": "KW-0862", + "category": "Ligand", + "name": "Zinc" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24728711", + "citationType": "journal article", + "authors": [ + "Sakaguchi S.", + "Nakagawa S.", + "Yoshikawa R.", + "Kuwahara C.", + "Hagiwara H.", + "Asai K.", + "Kawakami K.", + "Yamamoto Y.", + "Ogawa M.", + "Miyazawa T." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24728711" + }, + { + "database": "DOI", + "id": "10.1099/vir.0.065029-0" + } + ], + "title": "Genetic diversity of feline morbilliviruses isolated in Japan.", + "publicationDate": "2014", + "journal": "J. Gen. Virol.", + "firstPage": "1464", + "lastPage": "1468", + "volume": "95" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAO58314.1" + } + ], + "value": "SS1", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAO58314.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "AB910309", + "properties": [ + { + "key": "ProteinId", + "value": "BAO58314.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Viral_cRNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0030430", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell cytoplasm" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ] + } + ], + "sequence": { + "value": "MEQSDYQDILYPEVHLNSPIVISKLVGILEYAQVIHNQLLTDHTIIKNIQFRLRNGFNSPRIQTLSTMGEIINKIKNKYSNYFHIPYPECNQKLFRIVDPELTSKLESLLNKGDTLYLKIRSDIIKCFDKLKMKMNISNDLLNDNSQLILDLPLIIKGSQWFFPFLFWFSIKTETRNCIRQNQKARVRSQYRPHLSETKRITLVVTSDLITIFDHINKCIFYLTFEMLLMYCDVIEGRLMTETAMSLDCRFTNLLPRVQYMWDLLDGMFESLGNQIYSVIALLEPLSLAYLQLIDADPQIRGTFLHHCFSELEEIIFDKSPFDPFVYENLINGLDYIYLTDDIHLTAEVFSFFRSFGHPYLEAQNAANNVRKYMNQPKVISYQTLMQGHAIFCGIIINGFRDRHGGTWPPVELPSHASAVIRNAQLSGEGLTSEQCAQHWRSFCGFKFKCFMPLSLDSDLTMYLRDKALSPVRSEWDSVYAKEYLRYNPGLPTSSRRLVNVFLEDDKFDPYEMIMYVINGDYLIDDEFNLSYSLKEKEIKEVGRLFAKMTYKMRACQVIAENLIANGVGKFFKDNGMAKDEHKLTKTLHKLAISGVPKDNSQLYLDECWEQAVRHCSSDTHTKKQTMGSQSKRIIESKSSRSPLNNRDTLKGKRDPNTQLKYPSNTEYYETISSFITTDLKKYCLNWRYESSSVFAERLNEIYGLPGFFQWLHKILEKSVLYVSDPSSPPDFDQHIDIESVPNDHIFIKYPMGGIEGFCQKLWTISTIPFLYLAAFDTGVRIASLVQGDNQAIAVTKRVPSSWSYSKKKEESTKITTQYFLNLRQRLHDIGHELKANETIISSHFFVYSKGIYYDGILLSQALKSMARCVFWSETIVDETRSACSNISTTLAKAIERGYDKFVAYAINIYKTIHQVLIALSFTINPTMTPDITEPFYKSLDLLKNLVLIPAPLGGMNYMNMSRLFVRNIGDPITASFADIKRMIECGLLGCSILSQIMYQKCGSSKYLDWASDPYSINLPYSQSMTKVLKNVTARYVLMHSPNPMLKDLFHEKSQEEDEILAEFLLDRHLIIPRAAHEILSNSVTGARESIAGMLDTTKGLIRASMSRGGLTSSLVLKLSTYDYQQFRTCLEWLYAPTTGIAVSVDSCSVFLAKTIRKRMWVHLTKGREIYGLEVPDILECMQNNIIIDHEDCYSCIQGSRYYTWFFVPSNCQLDQINKSTNSLRVPYVGSTTEERSDMRLSYVRSPSRPLKAAVRIAAVYTWAYGDDDLSWHEAWYLARTRANITFDELKLITPIATSTNLAHRLRDRSTQVKYSGTSLVRVARYTTISNDNMSFIINNKKVDTNFVYQQGMLLGLSILEYIFRYCTSTGQSNTVIHLHADINCCIVQMTDQPYTPSLTKKLPDIKPINNKLIYDPAPIIDTDAARLYSQKYLSHLIDFPSWSTTQLNTVLAKVVAVSIVELITKASKDHLNEIIAVVGDDDINSFITEFLLVDPRLFTLYLGQYTSLQWAYEVHYHRPVGKYQMAEVLHNLLSRASRGIFSVLTNAFSHPRVYRRFWECGLLEPIYGPYIGSQNLHTAVIDYIYNAYITYLDAYLSDQVDDTDIIICETEETCLANRIDNYQSRHLAVLIDMYCDSTRCPNIKGSDTIMRNSILRSFIDNERRTNPLGLTWNLDPLLVDHFSCSITYLRRGIIKQMRLRFDPNISLELAKMIKPDVIYQAPKVPSSWALIDINPEVNDLNTIFGELNSKWKDIPIGQIKIQNYEIHAYRRVGVNSTACYKALEILSVLNRFMSNPLGALFLGEGAGSMLVTYRAFIPFKTIYYNSGISVQHVQGQRELSLYPSEVALVDNKNRLANDPNIKVLFNGKPESTWVGNIDCFAYILSHIETSSLTLIHSDIESSLSKTKNKILEELCHILSMALILGKLGSVLVIKLLPRVGDYTYSFCKYASEFYQQNFLILPRFSNMSSSEIYYIGVHLNTNRLIDPDRIVQYIIRNLQPTPVTFLSYIFETKYRNNMVTNYGLCLSDGHKSDYLSSITKIENVLLSCGLELNGPKIIQQLSGHDYANGETSLESSIMILVREYLNATIQGRETLGLFSPYPVLHESQLREINKCIALKYVVYLLFYLNSISSSKQIMSNLRKGILMYDLRDEFFISRLSANYKKKVMLQEVKTTWIFNIDTPTRKALYKLVGYSLIINHI", + "length": 2202, + "molWeight": 252830, + "crc64": "0B4B3846D2B0EF16", + "md5": "E98D71C0EAA4317745F8AEA2695302FD" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "CATALYTIC ACTIVITY": 5, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Domain": 2, + "Region": 1 + }, + "uniParcId": "UPI00043ABFDE" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "X2JFE9", + "uniProtkbId": "X2JFE9_HELLU", + "entryAudit": { + "firstPublicDate": "2014-06-11", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2014-06-11", + "entryVersion": 38, + "sequenceVersion": 1 + }, + "annotationScore": 3, + "organism": { + "scientificName": "Helix lucorum", + "commonName": "Snail", + "taxonId": 31229, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64827.1" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Spiralia", + "Lophotrochozoa", + "Mollusca", + "Gastropoda", + "Heterobranchia", + "Euthyneura", + "Panpulmonata", + "Eupulmonata", + "Stylommatophora", + "Helicina", + "Helicoidea", + "Helicidae", + "Helix" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00015947" + }, + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "value": "Cytochrome c oxidase subunit 1" + }, + "ecNumbers": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "value": "7.1.1.9" + } + ] + }, + "flag": "Fragment" + }, + "genes": [ + { + "geneName": { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AXB54992.1" + } + ], + "value": "COX1" + }, + "synonyms": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AKZ31582.1" + } + ], + "value": "cox1" + } + ] + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "value": "Mitochondrion inner membrane", + "id": "SL-0168" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "value": "Multi-pass membrane protein", + "id": "SL-9909" + } + } + ] + }, + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "name": "4 Fe(II)-[cytochrome c] + O2 + 8 H(+)(in) = 4 Fe(III)-[cytochrome c] + 2 H2O + 4 H(+)(out)", + "reactionCrossReferences": [ + { + "database": "Rhea", + "id": "RHEA:11436" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:10350" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:14399" + }, + { + "database": "ChEBI", + "id": "CHEBI:15377" + }, + { + "database": "ChEBI", + "id": "CHEBI:15378" + }, + { + "database": "ChEBI", + "id": "CHEBI:15379" + }, + { + "database": "ChEBI", + "id": "CHEBI:29033" + }, + { + "database": "ChEBI", + "id": "CHEBI:29034" + } + ], + "ecNumber": "7.1.1.9", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ] + } + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "name": "heme", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00001971" + } + ], + "cofactorCrossReference": { + "database": "ChEBI", + "id": "CHEBI:30413" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "value": "Component of the cytochrome c oxidase, the last enzyme in the mitochondrial electron transport chain which drives oxidative phosphorylation. The respiratory chain contains 3 multisubunit complexes succinate dehydrogenase (complex II, CII), ubiquinol-cytochrome c oxidoreductase (cytochrome b-c1 complex, complex III, CIII) and cytochrome c oxidase (complex IV, CIV), that cooperate to transfer electrons derived from NADH and succinate to molecular oxygen, creating an electrochemical gradient over the inner membrane that drives transmembrane transport and the ATP synthase. Cytochrome c oxidase is the component of the respiratory chain that catalyzes the reduction of oxygen to water. Electrons originating from reduced cytochrome c in the intermembrane space (IMS) are transferred via the dinuclear copper A center (CU(A)) of subunit 2 and heme A of subunit 1 to the active site in subunit 1, a binuclear center (BNC) formed by heme A3 and copper B (CU(B)). The BNC reduces molecular oxygen to 2 water molecules using 4 electrons from cytochrome c in the IMS and 4 protons from the mitochondrial matrix" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Transmembrane", + "location": { + "start": { + "value": 6, + "modifier": "EXACT" + }, + "end": { + "value": 25, + "modifier": "EXACT" + } + }, + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ] + }, + { + "type": "Transmembrane", + "location": { + "start": { + "value": 37, + "modifier": "EXACT" + }, + "end": { + "value": 66, + "modifier": "EXACT" + } + }, + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ] + } + ], + "geneLocations": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64827.1" + } + ], + "geneEncodingType": "Mitochondrion" + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0186", + "category": "Ligand", + "name": "Copper" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0249", + "category": "Biological process", + "name": "Electron transport" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0349", + "category": "Ligand", + "name": "Heme" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0408", + "category": "Ligand", + "name": "Iron" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + }, + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64827.1" + } + ], + "id": "KW-0496", + "category": "Cellular component", + "name": "Mitochondrion" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0999", + "category": "Cellular component", + "name": "Mitochondrion inner membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0679", + "category": "Biological process", + "name": "Respiratory chain" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0812", + "category": "Domain", + "name": "Transmembrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-1133", + "category": "Domain", + "name": "Transmembrane helix" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU000369" + } + ], + "id": "KW-0813", + "category": "Biological process", + "name": "Transport" + } + ], + "references": [ + { + "referenceNumber": 5, + "citation": { + "id": "32650821", + "citationType": "journal article", + "authors": [ + "Dimzas D.", + "Morelli S.", + "Traversa D.", + "Di Cesare A.", + "Van Bourgonie Y.R.", + "Breugelmans K.", + "Backeljau T.", + "di Regalbono A.F.", + "Diakou A." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "32650821" + } + ], + "title": "Intermediate gastropod hosts of major feline cardiopulmonary nematodes in an area of wildcat and domestic cat sympatry in Greece.", + "publicationDate": "2020", + "journal": "Parasit. Vectors", + "firstPage": "0", + "lastPage": "345", + "volume": "13" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "QIU80631.1" + } + ], + "value": "Sn-14", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "QIU80631.1" + } + ] + }, + { + "referenceNumber": 1, + "citation": { + "id": "CI-22T6FP2MKNDTE", + "citationType": "journal article", + "authors": [ + "Korabek O.", + "Jurickova L.", + "Petrusek A." + ], + "title": "Resurrecting Helix straminea, a forgotten escargot with trans-Adriatic distribution: first insights into the genetic variation within the genus Helix (Gastropoda: Pulmonata).", + "publicationDate": "2014", + "journal": "Zool. J. Linn. Soc.", + "firstPage": "72", + "lastPage": "91", + "volume": "171" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64827.1" + } + ], + "value": "LC18", + "type": "STRAIN" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64824.1" + } + ], + "value": "LC3", + "type": "STRAIN" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64826.1" + } + ], + "value": "LC37", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHN64827.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KF823081", + "properties": [ + { + "key": "ProteinId", + "value": "AHN64824.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_DNA" + } + ] + }, + { + "database": "EMBL", + "id": "KF823083", + "properties": [ + { + "key": "ProteinId", + "value": "AHN64826.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_DNA" + } + ] + } + ], + "sequence": { + "value": "TLYMLFGVWCGMVGTGLSLLIRWELGTAGVFSDDHFYNVIVTAHAFVMIFFMVMPIMIGGFGNWMVPLLIGAPDMSFPRMNNMSFWLLPPSFLLLISSSLVEGGAGTGWTVYPPLSGLIGHSGASVDLAIFSLHLAGMSSILGAINFITTIFNMRSPGISLELMSLFVWSILVTVFLLLLSLPVLAGAITMLLTDRNFSTSFFDPAGGGDPILYQHLF", + "length": 218, + "molWeight": 23642, + "crc64": "33CDF182AB84C18E", + "md5": "25AB05A09758DF77890BC20192EFAE87" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "CATALYTIC ACTIVITY": 1, + "COFACTOR": 1, + "PATHWAY": 1, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Transmembrane": 5, + "Domain": 1, + "Non-terminal residue": 2 + }, + "uniParcId": "UPI000438205C" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W8Q8A4", + "uniProtkbId": "W8Q8A4_9REOV", + "entryAudit": { + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 49, + "sequenceVersion": 1 + }, + "annotationScore": 4, + "organism": { + "scientificName": "Human rotavirus A", + "taxonId": 10941, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44736.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Outer capsid protein VP4" + } + }, + "alternativeNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Hemagglutinin" + } + } + ], + "contains": [ + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Outer capsid protein VP8*" + } + } + }, + { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Outer capsid protein VP5*" + } + } + } + ] + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "molecule": "Outer capsid protein VP5*", + "note": { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Outer capsid protein" + } + ] + }, + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Virion", + "id": "SL-0274" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "value": "Outer capsid protein VP4: Spike-forming protein that mediates virion attachment to the host epithelial cell receptors and plays a major role in cell penetration, determination of host range restriction and virulence. Rotavirus attachment and entry into the host cell probably involves multiple sequential contacts between the outer capsid proteins VP4 and VP7, and the cell receptors. It is subsequently lost, together with VP7, following virus entry into the host cell. Following entry into the host cell, low intracellular or intravesicular Ca(2+) concentration probably causes the calcium-stabilized VP7 trimers to dissociate from the virion. This step is probably necessary for the membrane-disrupting entry step and the release of VP4, which is locked onto the virion by VP7. During the virus exit from the host cell, VP4 seems to be required to target the newly formed virions to the host cell lipid rafts" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Chain", + "location": { + "start": { + "value": 1, + "modifier": "EXACT" + }, + "end": { + "value": 775, + "modifier": "EXACT" + } + }, + "description": "Outer capsid protein VP4", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "featureId": "PRO_5023367224" + }, + { + "type": "Chain", + "location": { + "start": { + "value": 1, + "modifier": "EXACT" + }, + "end": { + "value": 231, + "modifier": "EXACT" + } + }, + "description": "Outer capsid protein VP8*", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "featureId": "PRO_5029066481" + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022561" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-0167", + "category": "Cellular component", + "name": "Capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023054" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-0175", + "category": "Domain", + "name": "Coiled coil" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1015", + "category": "PTM", + "name": "Disulfide bond" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022546" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-0348", + "category": "Molecular function", + "name": "Hemagglutinin" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022511" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1032", + "category": "Cellular component", + "name": "Host cell membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023200" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1035", + "category": "Cellular component", + "name": "Host cytoplasm" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023111" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1037", + "category": "Cellular component", + "name": "Host cytoskeleton" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023184" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1038", + "category": "Cellular component", + "name": "Host endoplasmic reticulum" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022870" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1043", + "category": "Cellular component", + "name": "Host membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022581" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-0945", + "category": "Biological process", + "name": "Host-virus interaction" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023136" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022770" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1152", + "category": "Cellular component", + "name": "Outer capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022804" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1161", + "category": "Biological process", + "name": "Viral attachment to host cell" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022595" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1162", + "category": "Biological process", + "name": "Viral penetration into host cytoplasm" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022648" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1173", + "category": "Biological process", + "name": "Viral penetration via permeabilization of host membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022844" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-0946", + "category": "Cellular component", + "name": "Virion" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023296" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04132" + } + ], + "id": "KW-1160", + "category": "Biological process", + "name": "Virus entry into host cell" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24818762", + "citationType": "journal article", + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "publicationDate": "2014", + "journal": "PLoS ONE", + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44736.1" + } + ], + "value": "CAU12-2-51", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44736.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KJ187602", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44736.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0044172", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell endoplasmic reticulum-Golgi intermediate compartment" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ] + } + ], + "sequence": { + "value": "MASLIYRQLLSNSYVTNISDEVNEIGNKKTTNVTVNPGPFAQTGYAPVNWGHGELPDSTLVQPTLDGPYQPTSLNLPVDYWMLIAPTREGKVAEGTNTTDRWFACVLVEPNVQNTQRQYVLDGQNVQLNVSNDSSTSWKFILFIKLTPDGTYTQYSTLSTPHKLCAWMKRDNRVYWYQGATPNASESYYLTINNDNSNVSSDAEFYLIPQSQTAMCTQYINNGLPPIQNTRNIVPVNITSRQIKDIRAQMNEDIVISKTSLWKEMQYNRDIIIRFKFANSIIKSGGLGYKWSEISFKPMNYQYTYTRDEEEVTAHTTCSVNGVNDFNYNGGTLPTDFAISRFEVIKENSYVYVDYWDDSQAFRNMVYVRSLAANLNDVVCSGGSYSFALPVGNHPVMSGGAVTLTSAGVTLSTQYTDYVSLNSLRFRFRLAVSEPSFSISRTRMSGIYGLPAVNPNNNAEYYEVAGRFSLISLVPTNDDYQTPIANSVTVRQDLERQLGELREEFNSLSQEIAVSQLIDLATLPLDMFSMFSGIKSTVEAVKSMTTNVMKRFKTSSLANAISDLTSNMSEAASSVRLTSVRSVGAITLPRARVSLQVGDDLRSMQDVSTQVSNVSRNLRLKEFTTQTDTLSFDDISAAVLKTKLDKSTQISQQTMPDIITESSEKFIPKRSYRIVDEDTAFETGIDGTFYAYKVDTFNEIPFDMERFNKLITDPPVLSAIIDFKTLKNLNDNYGITKKQAMELLHSNPKTLKEFINDNNPIIRNRIENLISQCRL", + "length": 775, + "molWeight": 87160, + "crc64": "E8FC8398CFF982A6", + "md5": "65E31E3EC001110F2507BA47DB99CEDF" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 3, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 4, + "DOMAIN": 1, + "PTM": 1, + "MISCELLANEOUS": 2, + "SIMILARITY": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Chain": 3, + "Domain": 3, + "Region": 3, + "Coiled coil": 1, + "Motif": 1, + "Site": 3, + "Disulfide bond": 1 + }, + "uniParcId": "UPI00042F3651" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W8QKB0", + "uniProtkbId": "W8QKB0_9REOV", + "entryAudit": { + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 30, + "sequenceVersion": 1 + }, + "annotationScore": 3, + "organism": { + "scientificName": "Human rotavirus A", + "taxonId": 10941, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44728.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "Non-structural protein 2" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "NSP2" + } + ], + "ecNumbers": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "3.6.4.-" + } + ] + }, + "alternativeNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "NCVP3" + } + }, + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "Non-structural RNA-binding protein 35" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "NS35" + } + ] + } + ] + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "Found in spherical cytoplasmic structures, called viral factories, that appear early after infection and are the site of viral replication and packaging" + } + ] + }, + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "Host cytoplasm", + "id": "SL-0381" + } + } + ] + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "name": "Mg(2+)", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + }, + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU364009" + } + ], + "cofactorCrossReference": { + "database": "ChEBI", + "id": "CHEBI:18420" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "value": "Participates in replication and packaging of the viral genome. Plays a crucial role, together with NSP5, in the formation of virus factories (viroplasms) which are large inclusions in the host cytoplasm where replication intermediates are assembled and viral RNA replication takes place. Displays ssRNA binding, NTPase, RNA triphosphatase (RTPase) and ATP-independent helix-unwinding activities. The unwinding activity may prepare and organize plus-strand RNAs for packaging and replication by removing interfering secondary structures. The RTPase activity plays a role in the removal of the gamma-phosphate from the rotavirus RNA minus strands of dsRNA genome segments. Participates in the selective exclusion of host proteins from stress granules (SG) and P bodies (PB). Participates also in the sequestration of these remodeled organelles in viral factories" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Binding site", + "location": { + "start": { + "value": 107, + "modifier": "EXACT" + }, + "end": { + "value": 109, + "modifier": "EXACT" + } + }, + "description": "", + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:30616" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "ligand": { + "name": "ATP", + "id": "ChEBI:CHEBI:30616" + } + }, + { + "type": "Domain", + "location": { + "start": { + "value": 2, + "modifier": "EXACT" + }, + "end": { + "value": 142, + "modifier": "EXACT" + } + }, + "description": "Rotavirus non-structural protein 35 N-terminal", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "source": "Pfam", + "id": "PF21067" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022840" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-0067", + "category": "Ligand", + "name": "ATP-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023200" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-1035", + "category": "Cellular component", + "name": "Host cytoplasm" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022801" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-0378", + "category": "Molecular function", + "name": "Hydrolase" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022842" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-0460", + "category": "Ligand", + "name": "Magnesium" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022723" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022741" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-0547", + "category": "Ligand", + "name": "Nucleotide-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022884" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04089" + } + ], + "id": "KW-0694", + "category": "Molecular function", + "name": "RNA-binding" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24818762", + "citationType": "journal article", + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "publicationDate": "2014", + "journal": "PLoS ONE", + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44728.1" + } + ], + "value": "CAU12-2-51", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44728.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KJ187595", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44728.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0030430", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell cytoplasm" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ] + } + ], + "sequence": { + "value": "MAELACFCYPHLENDSYKFIPFNSLAIKCMLTAKVDKKDQDKFYNSIVYGITPPPQFKKRYNTNDNSRGMNYETPMFNKVAVLICEALNSIKVTQSDVANVLSKVVSVRHLENLVLRRENHQDVLFHSKELLLKSVLIAIGHSKEIETTATAEGGEIVFQNAAFTMWKLTYLEHKLMPILDQNFIEYKITMNEDKPISESHVKELIAELRWQYNKFAVITHGKGHYRVVKYSSVANHADRVYATFKSNNKNGSMTEFNLLDQRIIWQNWYAFTSSMKQGNTLEICKKLLFQKMKRESNPFKGLSTDRKMDEVSQIGI", + "length": 317, + "molWeight": 36673, + "crc64": "81A32DF875978D2E", + "md5": "451FD69051210DF8F2D857E25AEB19CC" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "COFACTOR": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Domain": 2, + "Region": 1, + "Active site": 1, + "Binding site": 4 + }, + "uniParcId": "UPI00042E6D38" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W6U8K5", + "uniProtkbId": "W6U8K5_ECHGR", + "entryAudit": { + "firstPublicDate": "2014-04-16", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2014-04-16", + "entryVersion": 45, + "sequenceVersion": 1 + }, + "annotationScore": 2, + "organism": { + "scientificName": "Echinococcus granulosus", + "commonName": "Hydatid tapeworm", + "taxonId": 6210, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "EUB57593.1" + }, + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000019149" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Spiralia", + "Lophotrochozoa", + "Platyhelminthes", + "Rhabditophora", + "Neodermata", + "Cestoda", + "Eucestoda", + "Cyclophyllidea", + "Taeniidae", + "Echinococcus", + "Echinococcus granulosus group" + ] + }, + "proteinExistence": "4: Predicted", + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "EUB57593.1" + } + ], + "value": "Feline leukemia virus subgroup C receptor-related protein" + } + } + ] + }, + "genes": [ + { + "orfNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "EUB57593.1" + } + ], + "value": "EGR_07535" + } + ] + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004141" + } + ], + "value": "Membrane", + "id": "SL-0162" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004141" + } + ], + "value": "Multi-pass membrane protein", + "id": "SL-9909" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "EUB57593.1" + } + ], + "value": "The sequence shown here is derived from an EMBL/GenBank/DDBJ whole genome shotgun (WGS) entry which is preliminary data" + } + ], + "commentType": "CAUTION" + } + ], + "features": [ + { + "type": "Transmembrane", + "location": { + "start": { + "value": 435, + "modifier": "EXACT" + }, + "end": { + "value": 455, + "modifier": "EXACT" + } + }, + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ] + }, + { + "type": "Transmembrane", + "location": { + "start": { + "value": 461, + "modifier": "EXACT" + }, + "end": { + "value": 480, + "modifier": "EXACT" + } + }, + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023136" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "EUB57593.1" + } + ], + "id": "KW-0675", + "category": "Molecular function", + "name": "Receptor" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000019149" + } + ], + "id": "KW-1185", + "category": "Technical term", + "name": "Reference proteome" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022692" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0812", + "category": "Domain", + "name": "Transmembrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022989" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-1133", + "category": "Domain", + "name": "Transmembrane helix" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24013640", + "citationType": "journal article", + "authors": [ + "Zheng H.", + "Zhang W.", + "Zhang L.", + "Zhang Z.", + "Li J.", + "Lu G.", + "Zhu Y.", + "Wang Y.", + "Huang Y.", + "Liu J.", + "Kang H.", + "Chen J.", + "Wang L.", + "Chen A.", + "Yu S.", + "Gao Z.", + "Jin L.", + "Gu W.", + "Wang Z.", + "Zhao L.", + "Shi B.", + "Wen H.", + "Lin R.", + "Jones M.K.", + "Brejova B.", + "Vinar T.", + "Zhao G.", + "McManus D.P.", + "Chen Z.", + "Zhou Y.", + "Wang S." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24013640" + }, + { + "database": "DOI", + "id": "10.1038/ng.2757" + } + ], + "title": "The genome of the hydatid tapeworm Echinococcus granulosus.", + "publicationDate": "2013", + "journal": "Nat. Genet.", + "firstPage": "1168", + "lastPage": "1175", + "volume": "45" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "EUB57593.1" + }, + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000019149" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "APAU02000080", + "properties": [ + { + "key": "ProteinId", + "value": "EUB57593.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_DNA" + } + ] + }, + { + "database": "RefSeq", + "id": "XP_024348789.1", + "properties": [ + { + "key": "NucleotideSequenceId", + "value": "XM_024496784.1" + } + ] + } + ], + "sequence": { + "value": "MPTPLTHTAYRSPPQEEKINISLYRHRKPKMKSATIATTFIFPNKTYLTCIFAKTQPMRAARGQSKNKGPQQNGSLVPPLDANANPALDVNLGRFSEEEREAILTVVKRDLEIRCMEKYRLKKIRSGLLQEDSQAPLSLPGSANACTICRGRFIIFLNPRVTCSLCRREVCRRCTKAYNGEMLCRICSHDIFYRAVKCPWFYDSSPDKLCGHASDTIVKSLFKDNLPVLNGLTEEQLHKALSNYLKPDRETYVNGVVITLEVMLRIENEQIRTFTKRYQEVRSTAVNALNKVRGTAMKNGQEDSEKEVDRITSQFLQQTREALRKLLQMLCITEEKYKLSVHNSRQVNTATRVLEITQKQVEQLVGHKIQIPDEYTNVEVEEDLESFVASVASGPEEFFANVLLEDLTKAMRNSISHGRNSRAPLSSDQQGKDVAIAWLSMLYMLVYIPMIFPATWLLNHYGLRVSILIGATLNMLGAWIKCLSMELSQPTSSPASNASFPLLMVAQLVCALGQVFLLGVPAQLAATWFSKSELAMATAIGVGCALGFGLPPLMVPAVTSSTGVADFEDFRKGFRIMFYGGAAIMSFDLLLVAIFFKEEPEIAPSRAQYKRILQRRAQLAGDKPDEADFAPAFDHESSVLTAEESVLVNQSYFHQVAHSFKCVSFIYLNICYGVNTGVYYEIGTLLNSIVAEFFPTAQGARMELPNLERCVKKSPSGVKQINVVKDGRIEAISTGDQFCTSPGSEDVEGSDELSLNLSSETEQVAIGWIGFSMVIAGLVGSIVAGVVLKKTGLYRLVLIIFYFLSVISMGAFMGSLYSNLIGTVFLTMILLGFFQSGFLPLGFEYAAEITYPIDEGLTSGILNTSAHIFGIILTQVATAMIGNYGALPTNIFILVCMVIAGVPACFMKDDLKRQRAHEKMRVEVEANSGDEFKYSSISKTKDV", + "length": 943, + "molWeight": 104642, + "crc64": "EBEC2FBABAA7A2A2", + "md5": "52E715C9787B9CD778FA5533664DC417" + }, + "extraAttributes": { + "countByCommentType": { + "SUBCELLULAR LOCATION": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Transmembrane": 10, + "Domain": 1, + "Region": 1, + "Compositional bias": 1 + }, + "uniParcId": "UPI0003E7E985" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W5L4Z9", + "uniProtkbId": "W5L4Z9_ASTMX", + "entryAudit": { + "firstPublicDate": "2014-04-16", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2018-12-05", + "entryVersion": 58, + "sequenceVersion": 2 + }, + "annotationScore": 2, + "organism": { + "scientificName": "Astyanax mexicanus", + "commonName": "Blind cave fish", + "synonyms": [ + "Astyanax fasciatus mexicanus" + ], + "taxonId": 7994, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Ensembl", + "id": "ENSAMXP00000014911.2" + }, + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000018467" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Chordata", + "Craniata", + "Vertebrata", + "Euteleostomi", + "Actinopterygii", + "Neopterygii", + "Teleostei", + "Ostariophysi", + "Characiformes", + "Characoidei", + "Acestrorhamphidae", + "Acestrorhamphinae", + "Astyanax" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Ensembl", + "id": "ENSAMXP00000014911.2" + } + ], + "value": "FLVCR choline and putative heme transporter 2" + } + } + ] + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004651" + } + ], + "value": "Cell membrane", + "id": "SL-0039" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004651" + } + ], + "value": "Multi-pass membrane protein", + "id": "SL-9909" + } + }, + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004477" + } + ], + "value": "Endoplasmic reticulum membrane", + "id": "SL-0097" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004477" + } + ], + "value": "Multi-pass membrane protein", + "id": "SL-9909" + } + }, + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004225" + } + ], + "value": "Mitochondrion membrane", + "id": "SL-0171" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004225" + } + ], + "value": "Multi-pass membrane protein", + "id": "SL-9909" + } + } + ] + }, + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "name": "choline(out) = choline(in)", + "reactionCrossReferences": [ + { + "database": "Rhea", + "id": "RHEA:32751" + }, + { + "database": "ChEBI", + "id": "CHEBI:15354" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00036811" + } + ] + } + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00053231" + } + ], + "value": "Choline uniporter that specifically mediates choline uptake at the blood-brain-barrier. Responsible for the majority of choline uptake across the blood-brain-barrier from the circulation into the brain. Choline, a nutrient critical for brain development, is a precursor of phosphatidylcholine, as well as betaine. Also mediates transport of ethanolamine. Choline and ethanolamine transport is not coupled with proton transport and is exclusively driven by the choline gradient across the plasma membrane. Also acts as a heme b transporter" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Domain", + "location": { + "start": { + "value": 58, + "modifier": "EXACT" + }, + "end": { + "value": 463, + "modifier": "EXACT" + } + }, + "description": "Major facilitator superfamily (MFS) profile", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "source": "PROSITE", + "id": "PS50850" + } + ] + }, + { + "type": "Region", + "location": { + "start": { + "value": 473, + "modifier": "EXACT" + }, + "end": { + "value": 494, + "modifier": "EXACT" + } + }, + "description": "Disordered", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "MobiDB-lite" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022475" + } + ], + "id": "KW-1003", + "category": "Cellular component", + "name": "Cell membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022824" + } + ], + "id": "KW-0256", + "category": "Cellular component", + "name": "Endoplasmic reticulum" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023136" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023128" + } + ], + "id": "KW-0496", + "category": "Cellular component", + "name": "Mitochondrion" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000018467" + } + ], + "id": "KW-1185", + "category": "Technical term", + "name": "Reference proteome" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022692" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0812", + "category": "Domain", + "name": "Transmembrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022989" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-1133", + "category": "Domain", + "name": "Transmembrane helix" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022448" + } + ], + "id": "KW-0813", + "category": "Biological process", + "name": "Transport" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "25329095", + "citationType": "journal article", + "authors": [ + "McGaugh S.E.", + "Gross J.B.", + "Aken B.", + "Blin M.", + "Borowsky R.", + "Chalopin D.", + "Hinaux H.", + "Jeffery W.R.", + "Keene A.", + "Ma L.", + "Minx P.", + "Murphy D.", + "O'Quin K.E.", + "Retaux S.", + "Rohner N.", + "Searle S.M.", + "Stahl B.A.", + "Tabin C.", + "Volff J.N.", + "Yoshizawa M.", + "Warren W.C." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "25329095" + }, + { + "database": "DOI", + "id": "10.1038/ncomms6307" + } + ], + "title": "The cavefish genome reveals candidate genes for eye loss.", + "publicationDate": "2014", + "journal": "Nat. Commun.", + "firstPage": "5307", + "lastPage": "5307", + "volume": "5" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000018467" + } + ], + "value": "female", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Proteomes", + "id": "UP000018467" + } + ] + }, + { + "referenceNumber": 2, + "citation": { + "id": "CI-20IM145UGQVF7", + "citationType": "submission", + "authoringGroup": [ + "Ensembl" + ], + "publicationDate": "JAN-2026", + "submissionDatabase": "UniProtKB" + }, + "referencePositions": [ + "IDENTIFICATION" + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "Ensembl", + "id": "ENSAMXP00000014911.2" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "AlphaFoldDB", + "id": "W5L4Z9", + "properties": [ + { + "key": "Description", + "value": "-" + } + ] + }, + { + "database": "Ensembl", + "id": "ENSAMXT00000014911.2", + "properties": [ + { + "key": "ProteinId", + "value": "ENSAMXP00000014911.2" + }, + { + "key": "GeneId", + "value": "ENSAMXG00000014488.2" + } + ] + } + ], + "sequence": { + "value": "MSACENSEDLKGQTNGPALVKSNISTSEPAAEQIECKVFSPEAPQECTTRLYKRRWAIVFLFSSYSLCNAFQWIQYGIINNIFMHFYSVSSFAIDWMSMIYMLTYIPFIFPVTWLLDKKGLRVVALVASSINCLGTWIKVASVKPHLFWVTMLGQFACSFAQVFILGMPSRIASVWFGSGEVSTACSIGVFGNQLGIAIGFLVPPILVPNVEDLDKLAHHISIMFYITAGVATLIFFLVIIVFQEKPEIPPSQAQAAMRNMSPQEYSYLASIKRLIHNKSFILLIITYGLNVGSFYAVSTLLNRMIIDHYPGEEVNAGRIGLTIVIAGMVGSLICGIWLDKSKTYKQTTLAVYFLTLVGMLIYAFTLDLGHLWVVFLTAGALGFFMTGYLPLGFEFAVELTYPESEGTSSGLLNCSAQVFGIIFTICQGKIIDYYGTLAGNIFLCIFLLIGTIMTGFIKADLRRQRANQVAETNAKKPGCQEEGLSSPLKEDKV", + "length": 494, + "molWeight": 54676, + "crc64": "9D5DBF17CAD5C77E", + "md5": "E646D99157273186F09C65F70AD4B392" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "CATALYTIC ACTIVITY": 3, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Domain": 1, + "Region": 1 + }, + "uniParcId": "UPI000BBDE7D1" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W8Q983", + "uniProtkbId": "W8Q983_9REOV", + "entryAudit": { + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 37, + "sequenceVersion": 1 + }, + "annotationScore": 4, + "organism": { + "scientificName": "Human rotavirus A", + "taxonId": 10941, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44730.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Non-structural glycoprotein 4" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "NSP4" + } + ] + }, + "alternativeNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "NCVP5" + } + }, + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "NS28" + } + } + ] + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "NSP4 localizes also in vesicular structures which contain autophagosomal markers and associate with viroplasms in virus-infected cells. Additionally, a soluble form of glycosylated NSP4 is secreted despite retention of its transmembrane domain" + } + ] + }, + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Host rough endoplasmic reticulum membrane", + "id": "SL-0423" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Single-pass type III membrane protein", + "id": "SL-9907" + } + }, + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Host membrane, host caveola", + "id": "SL-0428" + }, + "topology": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Single-pass type III membrane protein", + "id": "SL-9907" + } + }, + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Secreted", + "id": "SL-0243" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "value": "Plays an essential role in the virus replication cycle by acting as a viroporin. Creates a pore in the host endoplasmic reticulum and as a consequence releases Ca(2+) in the cytoplasm of infected cell. In turn, high levels of cytoplasmic calcium trigger membrane trafficking and transport of viral ER-associated proteins to viroplasms, sites of viral genome replication and immature particle assembly" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Binding site", + "location": { + "start": { + "value": 120, + "modifier": "EXACT" + }, + "end": { + "value": 120, + "modifier": "EXACT" + } + }, + "description": "", + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:29108" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "ligand": { + "name": "Ca(2+)", + "id": "ChEBI:CHEBI:29108" + } + }, + { + "type": "Glycosylation", + "location": { + "start": { + "value": 8, + "modifier": "EXACT" + }, + "end": { + "value": 8, + "modifier": "EXACT" + } + }, + "description": "N-linked (GlcNAc...) asparagine; by host", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "featureId": "" + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023050" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-1072", + "category": "Biological process", + "name": "Activation of host autophagy by virus" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022837" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0106", + "category": "Ligand", + "name": "Calcium" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Coils" + } + ], + "id": "KW-0175", + "category": "Domain", + "name": "Coiled coil" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022861" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0260", + "category": "Molecular function", + "name": "Enterotoxin" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023180" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0325", + "category": "PTM", + "name": "Glycoprotein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023184" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-1038", + "category": "Cellular component", + "name": "Host endoplasmic reticulum" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022870" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-1043", + "category": "Cellular component", + "name": "Host membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022581" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0945", + "category": "Biological process", + "name": "Host-virus interaction" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023303" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0407", + "category": "Molecular function", + "name": "Ion channel" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023065" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0406", + "category": "Biological process", + "name": "Ion transport" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023136" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022723" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022525" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0964", + "category": "Cellular component", + "name": "Secreted" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022968" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0735", + "category": "Domain", + "name": "Signal-anchor" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022656" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0800", + "category": "Molecular function", + "name": "Toxin" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022692" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0812", + "category": "Domain", + "name": "Transmembrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022989" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-1133", + "category": "Domain", + "name": "Transmembrane helix" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022448" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0813", + "category": "Biological process", + "name": "Transport" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023039" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-1182", + "category": "Molecular function", + "name": "Viral ion channel" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023026" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04091" + } + ], + "id": "KW-0843", + "category": "Biological process", + "name": "Virulence" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24818762", + "citationType": "journal article", + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "publicationDate": "2014", + "journal": "PLoS ONE", + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44730.1" + } + ], + "value": "CAU12-2-51", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44730.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KJ187597", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44730.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0005576", + "properties": [ + { + "key": "GoTerm", + "value": "C:extracellular region" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ] + } + ], + "sequence": { + "value": "MEKLTDLNYTLSVITLMNDTLHTIMEDPGMAYFPYIASVLTVLFTLHKASIPTMKIALRTSRCSYKVIKYCIVSIFNTLLKLAGYKEQITTKDEIEKQMDRVVKEMRRQLEMIDKLTTREIEQVELLKRIHDMLVIKPVDKVDMSQEFNQIQFKTLTEWVEGENPYEPKEVTASL", + "length": 175, + "molWeight": 20457, + "crc64": "C336CE8F0EE54E0C", + "md5": "5FEED6986EFE388FAC3C63734DF453FD" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "DOMAIN": 1, + "PTM": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Topological domain": 2, + "Transmembrane": 1, + "Coiled coil": 1, + "Binding site": 2, + "Glycosylation": 2 + }, + "uniParcId": "UPI00042F2EA0" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W8R3F3", + "uniProtkbId": "W8R3F3_9REOV", + "entryAudit": { + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 31, + "sequenceVersion": 1 + }, + "annotationScore": 2, + "organism": { + "scientificName": "Human rotavirus A", + "taxonId": 10941, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44737.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "value": "Intermediate capsid protein VP6" + } + } + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "value": "Component of the intermediate capsid. Also found in spherical cytoplasmic structures, called virus factories, that appear early after infection and are the site of viral replication and packaging" + } + ] + }, + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "value": "Virion", + "id": "SL-0274" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00057631" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "value": "Intermediate capsid protein that self assembles to form an icosahedral capsid with a T=13 symmetry, which consists of 230 trimers of VP6, with channels at each of its five-fold vertices. This capsid constitutes the middle concentric layer of the viral mature particle. The innermost VP2 capsid and the intermediate VP6 capsid remain intact following cell entry to protect the dsRNA from degradation and to prevent unfavorable antiviral responses in the host cell during all the replication cycle of the virus. Nascent transcripts are transcribed within the structural confines of this double-layered particle (DLP) and are extruded through the channels at the five-fold axes. VP6 is required for the transcription activity of the DLP" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Binding site", + "location": { + "start": { + "value": 153, + "modifier": "EXACT" + }, + "end": { + "value": 153, + "modifier": "EXACT" + } + }, + "description": "", + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:29105" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "ligand": { + "name": "Zn(2+)", + "id": "ChEBI:CHEBI:29105", + "note": "ligand shared between all trimeric partners" + } + }, + { + "type": "Region", + "location": { + "start": { + "value": 62, + "modifier": "EXACT" + }, + "end": { + "value": 73, + "modifier": "EXACT" + } + }, + "description": "Interaction with the inner capsid protein VP2", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022837" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-0106", + "category": "Ligand", + "name": "Calcium" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022561" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-0167", + "category": "Cellular component", + "name": "Capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022493" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-1154", + "category": "Cellular component", + "name": "Intermediate capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022723" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022843" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-0832", + "category": "PTM", + "name": "Ubl conjugation" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022844" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-0946", + "category": "Cellular component", + "name": "Virion" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022833" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04129" + } + ], + "id": "KW-0862", + "category": "Ligand", + "name": "Zinc" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24818762", + "citationType": "journal article", + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "publicationDate": "2014", + "journal": "PLoS ONE", + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44737.1" + } + ], + "value": "CAU12-2-51", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44737.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KJ187603", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44737.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0039621", + "properties": [ + { + "key": "GoTerm", + "value": "C:T=13 icosahedral viral capsid" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-ARBA" + } + ] + } + ], + "sequence": { + "value": "MDVLYSLSKTLKDARDKIVEGTLYSNVSDLIQQFNQMIITMNGNEFQTGGIGNLPVRNWNFDFGLLGTTLLNLDANYVETARNTIDYFVDFVDNVCMDEMVRESQRNGIAPQSDSLRKLSGIKFKRINFDNSSEYIENWNLQNRRQRTGFTFHKPNIFPYSASFTLNRSQPAHDNLMGTMWLNAGSEIQVAGFDYSCAINAPANIQQFEHIVQLRRVLTTATITLLPDAERFSFPRVINSADGATTWYFNPVILRPNNVEVEFLLNGQIINTYQARFGTIIARNFDTIRLSFQLMRPPNMTPAVAALFPNAQPFEHHATVGLTLRIESAVCESVLADASETMLANVTSVRQEYAIPVGPVFPPGMNWTDLITNYSPSREDNLQRVFTVASIRSMLIK", + "length": 397, + "molWeight": 44897, + "crc64": "1AC9A23DA62FBEC0", + "md5": "74A8486A6D109C92A854B58EFBEB03A4" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "MISCELLANEOUS": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Region": 1, + "Binding site": 3 + }, + "uniParcId": "UPI00042F181F" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W8QKB9", + "uniProtkbId": "W8QKB9_9REOV", + "entryAudit": { + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 50, + "sequenceVersion": 1 + }, + "annotationScore": 3, + "organism": { + "scientificName": "Human rotavirus A", + "taxonId": 10941, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44738.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + }, + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU363022" + } + ], + "value": "Outer capsid glycoprotein VP7" + } + } + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "value": "The outer layer contains 780 copies of VP7, grouped as 260 trimers. Immature double-layered particles assembled in the cytoplasm bud across the membrane of the endoplasmic reticulum, acquiring during this process a transient lipid membrane that is modified with the ER resident viral glycoproteins NSP4 and VP7; these enveloped particles also contain VP4. As the particles move towards the interior of the ER cisternae, the transient lipid membrane and the non-structural protein NSP4 are lost, while the virus surface proteins VP4 and VP7 rearrange to form the outermost virus protein layer, yielding mature infectious triple-layered particles" + } + ] + }, + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + }, + { + "evidenceCode": "ECO:0000256", + "source": "RuleBase", + "id": "RU363022" + } + ], + "value": "Host endoplasmic reticulum lumen", + "id": "SL-0389" + } + }, + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "value": "Virion", + "id": "SL-0274" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00059356" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "value": "Calcium-binding protein that interacts with rotavirus cell receptors once the initial attachment by VP4 has been achieved. Rotavirus attachment and entry into the host cell probably involves multiple sequential contacts between the outer capsid proteins VP4 and VP7, and the cell receptors. Following entry into the host cell, low intracellular or intravesicular Ca(2+) concentration probably causes the calcium-stabilized VP7 trimers to dissociate from the virion. This step is probably necessary for the membrane-disrupting entry step and the release of VP4, which is locked onto the virion by VP7" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Binding site", + "location": { + "start": { + "value": 95, + "modifier": "EXACT" + }, + "end": { + "value": 95, + "modifier": "EXACT" + } + }, + "description": "", + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:29108" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04131" + } + ], + "ligand": { + "name": "Ca(2+)", + "id": "ChEBI:CHEBI:29108", + "label": "1" + } + }, + { + "type": "Chain", + "location": { + "start": { + "value": 51, + "modifier": "EXACT" + }, + "end": { + "value": 326, + "modifier": "EXACT" + } + }, + "description": "Outer capsid glycoprotein VP7", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04131" + } + ], + "featureId": "PRO_5023428870" + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022837" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-0106", + "category": "Ligand", + "name": "Calcium" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022561" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-0167", + "category": "Cellular component", + "name": "Capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023157" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-1015", + "category": "PTM", + "name": "Disulfide bond" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023180" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-0325", + "category": "PTM", + "name": "Glycoprotein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023184" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-1038", + "category": "Cellular component", + "name": "Host endoplasmic reticulum" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022581" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-0945", + "category": "Biological process", + "name": "Host-virus interaction" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022723" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022770" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-1152", + "category": "Cellular component", + "name": "Outer capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022729" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04131" + } + ], + "id": "KW-0732", + "category": "Domain", + "name": "Signal" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022708" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-1146", + "category": "Cellular component", + "name": "T=13 icosahedral capsid protein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0812", + "category": "Domain", + "name": "Transmembrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-1133", + "category": "Domain", + "name": "Transmembrane helix" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022844" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04130" + } + ], + "id": "KW-0946", + "category": "Cellular component", + "name": "Virion" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24818762", + "citationType": "journal article", + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "publicationDate": "2014", + "journal": "PLoS ONE", + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44738.1" + } + ], + "value": "CAU12-2-51", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44738.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KJ187604", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44738.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0044166", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell endoplasmic reticulum lumen" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ] + } + ], + "sequence": { + "value": "MYGIEYTTVLTFLISVILLNYALKSLTRIMDFIIYRFLLIVVVLSPFLNAQNYGINLPITGSIDTPYTNSTQEEMFLTSTLCLYYPTEAATEINDNSWKDTLSQLFLTKGWPTGSVYFKDYADIASFSIDPQLYCDYNLVLMKYDAALQLDMSELADLILNEWLCNPMDITLYYYQQTDEANKWISMGSSCTIKVCPLNVQTLGIGCLTTDTSTFEEIATAEKLVITDVVDGVNHKLNVTTNTCTIRNCKKLGPRENVAVIQVGGSNVLDITADPTTAPQTERMMRVNWKKWWQVFYTIVDYVNQIVQAMSKRSRSLNSAAFYYRV", + "length": 326, + "molWeight": 37073, + "crc64": "BE3B85417F66327D", + "md5": "D5F596F60A5D582F6C2D226E70983AB9" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "SUBUNIT": 2, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "MISCELLANEOUS": 2, + "SIMILARITY": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Chain": 1, + "Transmembrane": 2, + "Region": 2, + "Binding site": 9, + "Disulfide bond": 4 + }, + "uniParcId": "UPI00042E6A3F" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W8Q8A1", + "uniProtkbId": "W8Q8A1_9REOV", + "entryAudit": { + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 33, + "sequenceVersion": 1 + }, + "annotationScore": 3, + "organism": { + "scientificName": "Human rotavirus A", + "taxonId": 10941, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44731.1" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "proteinDescription": { + "recommendedName": { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR004006" + } + ], + "value": "Non-structural protein 5" + }, + "shortNames": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR004006" + } + ], + "value": "NSP5" + } + ] + }, + "alternativeNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "value": "NS26" + } + } + ] + }, + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR004006" + } + ], + "value": "Found in spherical cytoplasmic structures, called virus factories, that appear early after infection and are the site of viral replication and packaging" + } + ] + }, + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + }, + { + "evidenceCode": "ECO:0000256", + "source": "PIRNR", + "id": "PIRNR004006" + } + ], + "value": "Host cytoplasm", + "id": "SL-0381" + } + } + ] + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "name": "Mg(2+)", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "cofactorCrossReference": { + "database": "ChEBI", + "id": "CHEBI:18420" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "value": "Plays an essential role in the viral genome replication. Participates, together with NSP2, in the formation of viral factories (viroplasms) which are large inclusions in the host cytoplasm where replication intermediates are assembled and viral RNA replication takes place. Orchestrates the recruitment of viroplasmic proteins such as capsid proteins to these factories. Participates in the selective exclusion of host proteins from stress granules (SG) and P bodies (PB). Participates also in the sequestration of these remodeled organelles in viral factories" + } + ], + "commentType": "FUNCTION" + } + ], + "features": [ + { + "type": "Binding site", + "location": { + "start": { + "value": 92, + "modifier": "EXACT" + }, + "end": { + "value": 92, + "modifier": "EXACT" + } + }, + "description": "", + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:18420" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "ligand": { + "name": "Mg(2+)", + "id": "ChEBI:CHEBI:18420" + } + }, + { + "type": "Region", + "location": { + "start": { + "value": 16, + "modifier": "EXACT" + }, + "end": { + "value": 37, + "modifier": "EXACT" + } + }, + "description": "Disordered", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "MobiDB-lite" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023180" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-0325", + "category": "PTM", + "name": "Glycoprotein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023200" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-1035", + "category": "Cellular component", + "name": "Host cytoplasm" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022842" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-0460", + "category": "Ligand", + "name": "Magnesium" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-0479", + "category": "Ligand", + "name": "Metal-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022741" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-0547", + "category": "Ligand", + "name": "Nucleotide-binding" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-0597", + "category": "PTM", + "name": "Phosphoprotein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022884" + }, + { + "evidenceCode": "ECO:0000256", + "source": "HAMAP-Rule", + "id": "MF_04092" + } + ], + "id": "KW-0694", + "category": "Molecular function", + "name": "RNA-binding" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "24818762", + "citationType": "journal article", + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "publicationDate": "2014", + "journal": "PLoS ONE", + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44731.1" + } + ], + "value": "CAU12-2-51", + "type": "STRAIN" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "AHL44731.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "KJ187598", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44731.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ] + }, + { + "database": "GO", + "id": "GO:0030430", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell cytoplasm" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ] + } + ], + "sequence": { + "value": "MSLSIDVTSLPSISSSIYKNESSSTTSTLSGKSIGRSEQYISPDAEAFGKYMLSKSPEDIGPSDSASNDPLTSFSIRSNAVKTNADASVSMDSSTQSRPSSNVGCDQVDFSFNKGIKMNTNLDSSISIATFSKKEKSKNDHKSRKHYPKVEAESDSDDYVLDDSDSDDGKCKNCKYKRKYFALRVRMKQVAMQLIEDL", + "length": 198, + "molWeight": 21726, + "crc64": "27C0104400E9CE41", + "md5": "F11A74C9F145ED70EB6CD8BCAE6D3989" + }, + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "COFACTOR": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Region": 4, + "Compositional bias": 2, + "Binding site": 1, + "Modified residue": 5 + }, + "uniParcId": "UPI00042E2A3D" + } + }, + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "W6JD38", + "uniProtkbId": "W6JD38_FELCA", + "entryAudit": { + "firstPublicDate": "2018-10-10", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2018-10-10", + "entryVersion": 27, + "sequenceVersion": 1 + }, + "annotationScore": 2, + "organism": { + "scientificName": "Felis catus", + "commonName": "Cat", + "synonyms": [ + "Felis silvestris catus" + ], + "taxonId": 9685, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAN42597.1" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Chordata", + "Craniata", + "Vertebrata", + "Euteleostomi", + "Mammalia", + "Eutheria", + "Laurasiatheria", + "Carnivora", + "Feliformia", + "Felidae", + "Felinae", + "Felis" + ] + }, + "proteinExistence": "2: Evidence at transcript level", + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAN42597.1" + } + ], + "value": "Signaling lymphocyte activation molecule" + } + } + ] + }, + "genes": [ + { + "geneName": { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAN42597.1" + } + ], + "value": "SLAM" + } + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00004370" + } + ], + "value": "Membrane", + "id": "SL-0162" + } + } + ] + }, + { + "texts": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PROSITE-ProRule", + "id": "PRU00114" + } + ], + "value": "Lacks conserved residue(s) required for the propagation of feature annotation" + } + ], + "commentType": "CAUTION" + } + ], + "features": [ + { + "type": "Chain", + "location": { + "start": { + "value": 19, + "modifier": "EXACT" + }, + "end": { + "value": 337, + "modifier": "EXACT" + } + }, + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "SignalP" + } + ], + "featureId": "PRO_5016357255" + }, + { + "type": "Signal", + "location": { + "start": { + "value": 1, + "modifier": "EXACT" + }, + "end": { + "value": 18, + "modifier": "EXACT" + } + }, + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "SignalP" + } + ] + } + ], + "keywords": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023180" + } + ], + "id": "KW-0325", + "category": "PTM", + "name": "Glycoprotein" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "PROSITE-ProRule", + "id": "PRU00114" + } + ], + "id": "KW-0393", + "category": "Domain", + "name": "Immunoglobulin domain" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00023136" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0472", + "category": "Cellular component", + "name": "Membrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "ARBA", + "id": "ARBA00022729" + }, + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "SignalP" + } + ], + "id": "KW-0732", + "category": "Domain", + "name": "Signal" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-0812", + "category": "Domain", + "name": "Transmembrane" + }, + { + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "source": "SAM", + "id": "Phobius" + } + ], + "id": "KW-1133", + "category": "Domain", + "name": "Transmembrane helix" + } + ], + "references": [ + { + "referenceNumber": 1, + "citation": { + "id": "23535266", + "citationType": "journal article", + "authors": [ + "Hara Y.", + "Suzuki J.", + "Noguchi K.", + "Terada Y.", + "Shimoda H.", + "Mizuno T.", + "Maeda K." + ], + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "23535266" + }, + { + "database": "DOI", + "id": "10.1292/jvms.13-0003" + } + ], + "title": "Function of feline signaling lymphocyte activation molecule as a receptor of canine distemper virus.", + "publicationDate": "2013", + "journal": "J. Vet. Med. Sci.", + "firstPage": "1085", + "lastPage": "1089", + "volume": "75" + }, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE" + ], + "referenceComments": [ + { + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAN42597.1" + } + ], + "value": "Thymus", + "type": "TISSUE" + } + ], + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "source": "EMBL", + "id": "BAN42597.1" + } + ] + } + ], + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "id": "AB771742", + "properties": [ + { + "key": "ProteinId", + "value": "BAN42597.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "mRNA" + } + ] + }, + { + "database": "RefSeq", + "id": "NP_001265755.1", + "properties": [ + { + "key": "NucleotideSequenceId", + "value": "NM_001278826.1" + } + ] + } + ], + "sequence": { + "value": "MDLKGLLSLNFLLFLSLAFELSCGTGESLMTCQEVTEQLGRNLLLPLASEGISKSMNKSIHILVTKSESPQKNVKTKIVSLSLPERASPRFLENGYKFHPENLKLEILQSKKENEGWYFTTLEENFSVRHFCLQLKLFEQVSTPEIKVLNWTQENGNCSLTLACEVEKGDHVAYSWREETGADPPISANSSHLHLSLGPQHVHNVYVCTVSNPISNHSQTFTPGSVCMPDPPELRPWGLYVGLSSGAVIGVILTLKVAVLLLRRRGKANHYQPTTEAKSLTIYAQVQKSGSIQKKPDAPSAQDPCTTIYVAATEPVPEPVQEPKSITVYASVTLPES", + "length": 337, + "molWeight": 37201, + "crc64": "F24FDE8F6BDA985B", + "md5": "246583275688D762B2F35467EAB3FE5C" + }, + "extraAttributes": { + "countByCommentType": { + "SUBCELLULAR LOCATION": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Signal": 1, + "Chain": 1, + "Transmembrane": 1, + "Domain": 1 + }, + "uniParcId": "UPI000298A0A1" + } + } + ] + } +] diff --git a/tests/data/jsonschema/examples/uniprot_search/maximal_expected.json b/tests/data/jsonschema/examples/uniprot_search/maximal_expected.json new file mode 100644 index 00000000..584a72bc --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/maximal_expected.json @@ -0,0 +1,5704 @@ +[ + { + "results": [ + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-06-11", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2014-06-11", + "entryVersion": 59 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 4.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "Large structural protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + }, + { + "fullName": { + "value": "Replicase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + }, + { + "fullName": { + "value": "Transcriptase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + ], + "recommendedName": { + "shortNames": [ + { + "value": "Protein L", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "RNA-directed RNA polymerase L", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + }, + "includes": [ + { + "recommendedName": { + "ecNumbers": [ + { + "value": "2.7.7.48", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "RNA-directed RNA polymerase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + }, + { + "recommendedName": { + "ecNumbers": [ + { + "value": "3.6.1.-", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "GTP phosphohydrolase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + }, + { + "alternativeNames": [ + { + "fullName": { + "value": "PRNTase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + ], + "recommendedName": { + "ecNumbers": [ + { + "value": "2.7.7.88", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "GDP polyribonucleotidyltransferase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + }, + { + "recommendedName": { + "shortNames": [ + { + "value": "N1-2'-O-MTase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "mRNA (nucleoside-2'-O-)-methyltransferase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + }, + { + "recommendedName": { + "shortNames": [ + { + "value": "G-N7-MTase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "mRNA (guanine-N(7)-)-methyltransferase", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "BAO58314.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Viral_cRNA" + } + ], + "id": "AB910309" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell cytoplasm" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ], + "id": "GO:0030430" + } + ], + "genes": [ + { + "geneName": { + "value": "L", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAO58314.1", + "source": "EMBL" + } + ] + } + } + ], + "comments": [ + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "reactionCrossReferences": [ + { + "database": "Rhea", + "id": "RHEA:21248" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:14527" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:17342" + }, + { + "database": "ChEBI", + "id": "CHEBI:33019" + }, + { + "database": "ChEBI", + "id": "CHEBI:61557" + }, + { + "database": "ChEBI", + "id": "CHEBI:140395" + } + ], + "ecNumber": "2.7.7.48", + "name": "RNA(n) + a ribonucleoside 5'-triphosphate = RNA(n+1) + diphosphate", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PRU00539", + "source": "PROSITE-ProRule" + } + ] + } + }, + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + }, + { + "location": { + "id": "SL-0381", + "value": "Host cytoplasm", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "RNA-directed RNA polymerase that catalyzes the replication of viral genomic RNA. The template is composed of the viral RNA tightly encapsidated by the nucleoprotein (N). The replicase mode is dependent on intracellular N protein concentration. In this mode, the polymerase replicates the whole viral genome without recognizing transcriptional signals, and the replicated genome is not caped or polyadenylated", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00003132", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Ligand", + "name": "ATP-binding", + "id": "KW-0067", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022840", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023200", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Molecular function", + "name": "Hydrolase", + "id": "KW-0378", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022801", + "source": "ARBA" + } + ] + }, + { + "category": "Ligand", + "name": "Magnesium", + "id": "KW-0460", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022842", + "source": "ARBA" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022723", + "source": "ARBA" + } + ] + }, + { + "category": "Molecular function", + "name": "Methyltransferase", + "id": "KW-0489", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022603", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Biological process", + "name": "mRNA capping", + "id": "KW-0506", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023042", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Biological process", + "name": "mRNA processing", + "id": "KW-0507", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022664", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Molecular function", + "name": "Multifunctional enzyme", + "id": "KW-0511", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023268", + "source": "ARBA" + } + ] + }, + { + "category": "Ligand", + "name": "Nucleotide-binding", + "id": "KW-0547", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022741", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Molecular function", + "name": "Nucleotidyltransferase", + "id": "KW-0548", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022695", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Molecular function", + "name": "RNA-directed RNA polymerase", + "id": "KW-0696", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022484", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Ligand", + "name": "S-adenosyl-L-methionine", + "id": "KW-0949", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022691", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Molecular function", + "name": "Transferase", + "id": "KW-0808", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022679", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Biological process", + "name": "Viral RNA replication", + "id": "KW-0693", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022953", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022844", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR000830", + "source": "PIRNR" + } + ] + }, + { + "category": "Ligand", + "name": "Zinc", + "id": "KW-0862", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022833", + "source": "ARBA" + } + ] + } + ], + "sequence": { + "value": "MEQSDYQDILYPEVHLNSPIVISKLVGILEYAQVIHNQLLTDHTIIKNIQFRLRNGFNSPRIQTLSTMGEIINKIKNKYSNYFHIPYPECNQKLFRIVDPELTSKLESLLNKGDTLYLKIRSDIIKCFDKLKMKMNISNDLLNDNSQLILDLPLIIKGSQWFFPFLFWFSIKTETRNCIRQNQKARVRSQYRPHLSETKRITLVVTSDLITIFDHINKCIFYLTFEMLLMYCDVIEGRLMTETAMSLDCRFTNLLPRVQYMWDLLDGMFESLGNQIYSVIALLEPLSLAYLQLIDADPQIRGTFLHHCFSELEEIIFDKSPFDPFVYENLINGLDYIYLTDDIHLTAEVFSFFRSFGHPYLEAQNAANNVRKYMNQPKVISYQTLMQGHAIFCGIIINGFRDRHGGTWPPVELPSHASAVIRNAQLSGEGLTSEQCAQHWRSFCGFKFKCFMPLSLDSDLTMYLRDKALSPVRSEWDSVYAKEYLRYNPGLPTSSRRLVNVFLEDDKFDPYEMIMYVINGDYLIDDEFNLSYSLKEKEIKEVGRLFAKMTYKMRACQVIAENLIANGVGKFFKDNGMAKDEHKLTKTLHKLAISGVPKDNSQLYLDECWEQAVRHCSSDTHTKKQTMGSQSKRIIESKSSRSPLNNRDTLKGKRDPNTQLKYPSNTEYYETISSFITTDLKKYCLNWRYESSSVFAERLNEIYGLPGFFQWLHKILEKSVLYVSDPSSPPDFDQHIDIESVPNDHIFIKYPMGGIEGFCQKLWTISTIPFLYLAAFDTGVRIASLVQGDNQAIAVTKRVPSSWSYSKKKEESTKITTQYFLNLRQRLHDIGHELKANETIISSHFFVYSKGIYYDGILLSQALKSMARCVFWSETIVDETRSACSNISTTLAKAIERGYDKFVAYAINIYKTIHQVLIALSFTINPTMTPDITEPFYKSLDLLKNLVLIPAPLGGMNYMNMSRLFVRNIGDPITASFADIKRMIECGLLGCSILSQIMYQKCGSSKYLDWASDPYSINLPYSQSMTKVLKNVTARYVLMHSPNPMLKDLFHEKSQEEDEILAEFLLDRHLIIPRAAHEILSNSVTGARESIAGMLDTTKGLIRASMSRGGLTSSLVLKLSTYDYQQFRTCLEWLYAPTTGIAVSVDSCSVFLAKTIRKRMWVHLTKGREIYGLEVPDILECMQNNIIIDHEDCYSCIQGSRYYTWFFVPSNCQLDQINKSTNSLRVPYVGSTTEERSDMRLSYVRSPSRPLKAAVRIAAVYTWAYGDDDLSWHEAWYLARTRANITFDELKLITPIATSTNLAHRLRDRSTQVKYSGTSLVRVARYTTISNDNMSFIINNKKVDTNFVYQQGMLLGLSILEYIFRYCTSTGQSNTVIHLHADINCCIVQMTDQPYTPSLTKKLPDIKPINNKLIYDPAPIIDTDAARLYSQKYLSHLIDFPSWSTTQLNTVLAKVVAVSIVELITKASKDHLNEIIAVVGDDDINSFITEFLLVDPRLFTLYLGQYTSLQWAYEVHYHRPVGKYQMAEVLHNLLSRASRGIFSVLTNAFSHPRVYRRFWECGLLEPIYGPYIGSQNLHTAVIDYIYNAYITYLDAYLSDQVDDTDIIICETEETCLANRIDNYQSRHLAVLIDMYCDSTRCPNIKGSDTIMRNSILRSFIDNERRTNPLGLTWNLDPLLVDHFSCSITYLRRGIIKQMRLRFDPNISLELAKMIKPDVIYQAPKVPSSWALIDINPEVNDLNTIFGELNSKWKDIPIGQIKIQNYEIHAYRRVGVNSTACYKALEILSVLNRFMSNPLGALFLGEGAGSMLVTYRAFIPFKTIYYNSGISVQHVQGQRELSLYPSEVALVDNKNRLANDPNIKVLFNGKPESTWVGNIDCFAYILSHIETSSLTLIHSDIESSLSKTKNKILEELCHILSMALILGKLGSVLVIKLLPRVGDYTYSFCKYASEFYQQNFLILPRFSNMSSSEIYYIGVHLNTNRLIDPDRIVQYIIRNLQPTPVTFLSYIFETKYRNNMVTNYGLCLSDGHKSDYLSSITKIENVLLSCGLELNGPKIIQQLSGHDYANGETSLESSIMILVREYLNATIQGRETLGLFSPYPVLHESQLREINKCIALKYVVYLLFYLNSISSSKQIMSNLRKGILMYDLRDEFFISRLSANYKKKVMLQEVKTTWIFNIDTPTRKALYKLVGYSLIINHI", + "length": 2202, + "molWeight": 252830, + "crc64": "0B4B3846D2B0EF16", + "md5": "E98D71C0EAA4317745F8AEA2695302FD" + }, + "organism": { + "taxonId": 1170234, + "scientificName": "Feline morbillivirus", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAO58314.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Negarnaviricota", + "Haploviricotina", + "Monjiviricetes", + "Mononegavirales", + "Paramyxoviridae", + "Orthoparamyxovirinae", + "Morbillivirus", + "Morbillivirus felis" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "X5I2V6_9PARA", + "primaryAccession": "X5I2V6", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "SS1", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAO58314.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "1464", + "lastPage": "1468", + "volume": "95", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24728711" + }, + { + "database": "DOI", + "id": "10.1099/vir.0.065029-0" + } + ], + "authors": [ + "Sakaguchi S.", + "Nakagawa S.", + "Yoshikawa R.", + "Kuwahara C.", + "Hagiwara H.", + "Asai K.", + "Kawakami K.", + "Yamamoto Y.", + "Ogawa M.", + "Miyazawa T." + ], + "publicationDate": "2014", + "title": "Genetic diversity of feline morbilliviruses isolated in Japan.", + "id": "24728711", + "journal": "J. Gen. Virol.", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAO58314.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "location": { + "start": { + "value": 672, + "modifier": "EXACT" + }, + "end": { + "value": 856, + "modifier": "EXACT" + } + }, + "type": "Domain", + "description": "RdRp catalytic", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "id": "PS50526", + "source": "PROSITE" + } + ] + }, + { + "location": { + "start": { + "value": 1771, + "modifier": "EXACT" + }, + "end": { + "value": 1976, + "modifier": "EXACT" + } + }, + "type": "Domain", + "description": "Mononegavirus-type SAM-dependent 2'-O-MTase", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "id": "PS51590", + "source": "PROSITE" + } + ] + }, + { + "location": { + "start": { + "value": 620, + "modifier": "EXACT" + }, + "end": { + "value": 658, + "modifier": "EXACT" + } + }, + "type": "Region", + "description": "Disordered", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MobiDB-lite", + "source": "SAM" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "CATALYTIC ACTIVITY": 5, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Domain": 2, + "Region": 1 + }, + "uniParcId": "UPI00043ABFDE" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-06-11", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2014-06-11", + "entryVersion": 38 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "recommendedName": { + "ecNumbers": [ + { + "value": "7.1.1.9", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + } + ], + "fullName": { + "value": "Cytochrome c oxidase subunit 1", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00015947", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + } + }, + "flag": "Fragment" + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHN64824.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_DNA" + } + ], + "id": "KF823081" + }, + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHN64826.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_DNA" + } + ], + "id": "KF823083" + } + ], + "genes": [ + { + "geneName": { + "value": "COX1", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AXB54992.1", + "source": "EMBL" + } + ] + }, + "synonyms": [ + { + "value": "cox1", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AKZ31582.1", + "source": "EMBL" + } + ] + } + ] + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + "location": { + "id": "SL-0168", + "value": "Mitochondrion inner membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + } + } + ] + }, + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "reactionCrossReferences": [ + { + "database": "Rhea", + "id": "RHEA:11436" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:10350" + }, + { + "database": "Rhea", + "id": "RHEA-COMP:14399" + }, + { + "database": "ChEBI", + "id": "CHEBI:15377" + }, + { + "database": "ChEBI", + "id": "CHEBI:15378" + }, + { + "database": "ChEBI", + "id": "CHEBI:15379" + }, + { + "database": "ChEBI", + "id": "CHEBI:29033" + }, + { + "database": "ChEBI", + "id": "CHEBI:29034" + } + ], + "ecNumber": "7.1.1.9", + "name": "4 Fe(II)-[cytochrome c] + O2 + 8 H(+)(in) = 4 Fe(III)-[cytochrome c] + 2 H2O + 4 H(+)(out)", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + } + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "cofactorCrossReference": { + "database": "ChEBI", + "id": "CHEBI:30413" + }, + "name": "heme", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00001971", + "source": "ARBA" + } + ] + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Component of the cytochrome c oxidase, the last enzyme in the mitochondrial electron transport chain which drives oxidative phosphorylation. The respiratory chain contains 3 multisubunit complexes succinate dehydrogenase (complex II, CII), ubiquinol-cytochrome c oxidoreductase (cytochrome b-c1 complex, complex III, CIII) and cytochrome c oxidase (complex IV, CIV), that cooperate to transfer electrons derived from NADH and succinate to molecular oxygen, creating an electrochemical gradient over the inner membrane that drives transmembrane transport and the ATP synthase. Cytochrome c oxidase is the component of the respiratory chain that catalyzes the reduction of oxygen to water. Electrons originating from reduced cytochrome c in the intermembrane space (IMS) are transferred via the dinuclear copper A center (CU(A)) of subunit 2 and heme A of subunit 1 to the active site in subunit 1, a binuclear center (BNC) formed by heme A3 and copper B (CU(B)). The BNC reduces molecular oxygen to 2 water molecules using 4 electrons from cytochrome c in the IMS and 4 protons from the mitochondrial matrix", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + } + ] + } + ], + "geneLocations": [ + { + "geneEncodingType": "Mitochondrion", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHN64827.1", + "source": "EMBL" + } + ] + } + ], + "keywords": [ + { + "category": "Ligand", + "name": "Copper", + "id": "KW-0186", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Biological process", + "name": "Electron transport", + "id": "KW-0249", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Ligand", + "name": "Heme", + "id": "KW-0349", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Ligand", + "name": "Iron", + "id": "KW-0408", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Cellular component", + "name": "Mitochondrion", + "id": "KW-0496", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + }, + { + "evidenceCode": "ECO:0000313", + "id": "AHN64827.1", + "source": "EMBL" + } + ] + }, + { + "category": "Cellular component", + "name": "Mitochondrion inner membrane", + "id": "KW-0999", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Biological process", + "name": "Respiratory chain", + "id": "KW-0679", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Biological process", + "name": "Transport", + "id": "KW-0813", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "RU000369", + "source": "RuleBase" + } + ] + } + ], + "sequence": { + "value": "TLYMLFGVWCGMVGTGLSLLIRWELGTAGVFSDDHFYNVIVTAHAFVMIFFMVMPIMIGGFGNWMVPLLIGAPDMSFPRMNNMSFWLLPPSFLLLISSSLVEGGAGTGWTVYPPLSGLIGHSGASVDLAIFSLHLAGMSSILGAINFITTIFNMRSPGISLELMSLFVWSILVTVFLLLLSLPVLAGAITMLLTDRNFSTSFFDPAGGGDPILYQHLF", + "length": 218, + "molWeight": 23642, + "crc64": "33CDF182AB84C18E", + "md5": "25AB05A09758DF77890BC20192EFAE87" + }, + "organism": { + "taxonId": 31229, + "commonName": "Snail", + "scientificName": "Helix lucorum", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHN64827.1", + "source": "EMBL" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Spiralia", + "Lophotrochozoa", + "Mollusca", + "Gastropoda", + "Heterobranchia", + "Euthyneura", + "Panpulmonata", + "Eupulmonata", + "Stylommatophora", + "Helicina", + "Helicoidea", + "Helicidae", + "Helix" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "X2JFE9_HELLU", + "primaryAccession": "X2JFE9", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "Sn-14", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "QIU80631.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 5, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE" + ], + "citation": { + "firstPage": "0", + "lastPage": "345", + "volume": "13", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "32650821" + } + ], + "authors": [ + "Dimzas D.", + "Morelli S.", + "Traversa D.", + "Di Cesare A.", + "Van Bourgonie Y.R.", + "Breugelmans K.", + "Backeljau T.", + "di Regalbono A.F.", + "Diakou A." + ], + "publicationDate": "2020", + "title": "Intermediate gastropod hosts of major feline cardiopulmonary nematodes in an area of wildcat and domestic cat sympatry in Greece.", + "id": "32650821", + "journal": "Parasit. Vectors", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "QIU80631.1", + "source": "EMBL" + } + ] + }, + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "LC18", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHN64827.1", + "source": "EMBL" + } + ] + }, + { + "type": "STRAIN", + "value": "LC3", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHN64824.1", + "source": "EMBL" + } + ] + }, + { + "type": "STRAIN", + "value": "LC37", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHN64826.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE" + ], + "citation": { + "firstPage": "72", + "lastPage": "91", + "volume": "171", + "authors": [ + "Korabek O.", + "Jurickova L.", + "Petrusek A." + ], + "publicationDate": "2014", + "title": "Resurrecting Helix straminea, a forgotten escargot with trans-Adriatic distribution: first insights into the genetic variation within the genus Helix (Gastropoda: Pulmonata).", + "id": "CI-22T6FP2MKNDTE", + "journal": "Zool. J. Linn. Soc.", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHN64827.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "location": { + "start": { + "value": 6, + "modifier": "EXACT" + }, + "end": { + "value": 25, + "modifier": "EXACT" + } + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "location": { + "start": { + "value": 37, + "modifier": "EXACT" + }, + "end": { + "value": 66, + "modifier": "EXACT" + } + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "CATALYTIC ACTIVITY": 1, + "COFACTOR": 1, + "PATHWAY": 1, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Transmembrane": 5, + "Domain": 1, + "Non-terminal residue": 2 + }, + "uniParcId": "UPI000438205C" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 49 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 4.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "Hemagglutinin", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + } + ], + "recommendedName": { + "fullName": { + "value": "Outer capsid protein VP4", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + }, + "contains": [ + { + "recommendedName": { + "fullName": { + "value": "Outer capsid protein VP8*", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + } + }, + { + "recommendedName": { + "fullName": { + "value": "Outer capsid protein VP5*", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44736.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ], + "id": "KJ187602" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell endoplasmic reticulum-Golgi intermediate compartment" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ], + "id": "GO:0044172" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Outer capsid protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + ] + }, + "molecule": "Outer capsid protein VP5*", + "subcellularLocations": [ + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Outer capsid protein VP4: Spike-forming protein that mediates virion attachment to the host epithelial cell receptors and plays a major role in cell penetration, determination of host range restriction and virulence. Rotavirus attachment and entry into the host cell probably involves multiple sequential contacts between the outer capsid proteins VP4 and VP7, and the cell receptors. It is subsequently lost, together with VP7, following virus entry into the host cell. Following entry into the host cell, low intracellular or intravesicular Ca(2+) concentration probably causes the calcium-stabilized VP7 trimers to dissociate from the virion. This step is probably necessary for the membrane-disrupting entry step and the release of VP4, which is locked onto the virion by VP7. During the virus exit from the host cell, VP4 seems to be required to target the newly formed virions to the host cell lipid rafts", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Cellular component", + "name": "Capsid protein", + "id": "KW-0167", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022561", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Coiled coil", + "id": "KW-0175", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023054", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "PTM", + "name": "Disulfide bond", + "id": "KW-1015", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "Hemagglutinin", + "id": "KW-0348", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022546", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host cell membrane", + "id": "KW-1032", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022511", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023200", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host cytoskeleton", + "id": "KW-1037", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023111", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host endoplasmic reticulum", + "id": "KW-1038", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023184", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host membrane", + "id": "KW-1043", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022870", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Host-virus interaction", + "id": "KW-0945", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022581", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023136", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Outer capsid protein", + "id": "KW-1152", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022770", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Viral attachment to host cell", + "id": "KW-1161", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022804", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Viral penetration into host cytoplasm", + "id": "KW-1162", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022595", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Viral penetration via permeabilization of host membrane", + "id": "KW-1173", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022648", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022844", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Virus entry into host cell", + "id": "KW-1160", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023296", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + ], + "sequence": { + "value": "MASLIYRQLLSNSYVTNISDEVNEIGNKKTTNVTVNPGPFAQTGYAPVNWGHGELPDSTLVQPTLDGPYQPTSLNLPVDYWMLIAPTREGKVAEGTNTTDRWFACVLVEPNVQNTQRQYVLDGQNVQLNVSNDSSTSWKFILFIKLTPDGTYTQYSTLSTPHKLCAWMKRDNRVYWYQGATPNASESYYLTINNDNSNVSSDAEFYLIPQSQTAMCTQYINNGLPPIQNTRNIVPVNITSRQIKDIRAQMNEDIVISKTSLWKEMQYNRDIIIRFKFANSIIKSGGLGYKWSEISFKPMNYQYTYTRDEEEVTAHTTCSVNGVNDFNYNGGTLPTDFAISRFEVIKENSYVYVDYWDDSQAFRNMVYVRSLAANLNDVVCSGGSYSFALPVGNHPVMSGGAVTLTSAGVTLSTQYTDYVSLNSLRFRFRLAVSEPSFSISRTRMSGIYGLPAVNPNNNAEYYEVAGRFSLISLVPTNDDYQTPIANSVTVRQDLERQLGELREEFNSLSQEIAVSQLIDLATLPLDMFSMFSGIKSTVEAVKSMTTNVMKRFKTSSLANAISDLTSNMSEAASSVRLTSVRSVGAITLPRARVSLQVGDDLRSMQDVSTQVSNVSRNLRLKEFTTQTDTLSFDDISAAVLKTKLDKSTQISQQTMPDIITESSEKFIPKRSYRIVDEDTAFETGIDGTFYAYKVDTFNEIPFDMERFNKLITDPPVLSAIIDFKTLKNLNDNYGITKKQAMELLHSNPKTLKEFINDNNPIIRNRIENLISQCRL", + "length": 775, + "molWeight": 87160, + "crc64": "E8FC8398CFF982A6", + "md5": "65E31E3EC001110F2507BA47DB99CEDF" + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44736.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8Q8A4_9REOV", + "primaryAccession": "W8Q8A4", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44736.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44736.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "featureId": "PRO_5023367224", + "location": { + "start": { + "value": 1, + "modifier": "EXACT" + }, + "end": { + "value": 775, + "modifier": "EXACT" + } + }, + "type": "Chain", + "description": "Outer capsid protein VP4", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + }, + { + "featureId": "PRO_5029066481", + "location": { + "start": { + "value": 1, + "modifier": "EXACT" + }, + "end": { + "value": 231, + "modifier": "EXACT" + } + }, + "type": "Chain", + "description": "Outer capsid protein VP8*", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04132", + "source": "HAMAP-Rule" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 3, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 4, + "DOMAIN": 1, + "PTM": 1, + "MISCELLANEOUS": 2, + "SIMILARITY": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Chain": 3, + "Domain": 3, + "Region": 3, + "Coiled coil": 1, + "Motif": 1, + "Site": 3, + "Disulfide bond": 1 + }, + "uniParcId": "UPI00042F3651" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 30 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "NCVP3", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + }, + { + "shortNames": [ + { + "value": "NS35", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + ], + "fullName": { + "value": "Non-structural RNA-binding protein 35", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + } + ], + "recommendedName": { + "shortNames": [ + { + "value": "NSP2", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + ], + "ecNumbers": [ + { + "value": "3.6.4.-", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + ], + "fullName": { + "value": "Non-structural protein 2", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44728.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ], + "id": "KJ187595" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell cytoplasm" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ], + "id": "GO:0030430" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Found in spherical cytoplasmic structures, called viral factories, that appear early after infection and are the site of viral replication and packaging", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0381", + "value": "Host cytoplasm", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + } + ] + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "cofactorCrossReference": { + "database": "ChEBI", + "id": "CHEBI:18420" + }, + "name": "Mg(2+)", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "RU364009", + "source": "RuleBase" + } + ] + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Participates in replication and packaging of the viral genome. Plays a crucial role, together with NSP5, in the formation of virus factories (viroplasms) which are large inclusions in the host cytoplasm where replication intermediates are assembled and viral RNA replication takes place. Displays ssRNA binding, NTPase, RNA triphosphatase (RTPase) and ATP-independent helix-unwinding activities. The unwinding activity may prepare and organize plus-strand RNAs for packaging and replication by removing interfering secondary structures. The RTPase activity plays a role in the removal of the gamma-phosphate from the rotavirus RNA minus strands of dsRNA genome segments. Participates in the selective exclusion of host proteins from stress granules (SG) and P bodies (PB). Participates also in the sequestration of these remodeled organelles in viral factories", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Ligand", + "name": "ATP-binding", + "id": "KW-0067", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022840", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023200", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "Hydrolase", + "id": "KW-0378", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022801", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Magnesium", + "id": "KW-0460", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022842", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022723", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Nucleotide-binding", + "id": "KW-0547", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022741", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "RNA-binding", + "id": "KW-0694", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022884", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + } + ], + "sequence": { + "value": "MAELACFCYPHLENDSYKFIPFNSLAIKCMLTAKVDKKDQDKFYNSIVYGITPPPQFKKRYNTNDNSRGMNYETPMFNKVAVLICEALNSIKVTQSDVANVLSKVVSVRHLENLVLRRENHQDVLFHSKELLLKSVLIAIGHSKEIETTATAEGGEIVFQNAAFTMWKLTYLEHKLMPILDQNFIEYKITMNEDKPISESHVKELIAELRWQYNKFAVITHGKGHYRVVKYSSVANHADRVYATFKSNNKNGSMTEFNLLDQRIIWQNWYAFTSSMKQGNTLEICKKLLFQKMKRESNPFKGLSTDRKMDEVSQIGI", + "length": 317, + "molWeight": 36673, + "crc64": "81A32DF875978D2E", + "md5": "451FD69051210DF8F2D857E25AEB19CC" + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44728.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8QKB0_9REOV", + "primaryAccession": "W8QKB0", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44728.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44728.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "ligand": { + "name": "ATP", + "id": "ChEBI:CHEBI:30616" + }, + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:30616" + } + ], + "location": { + "start": { + "value": 107, + "modifier": "EXACT" + }, + "end": { + "value": 109, + "modifier": "EXACT" + } + }, + "type": "Binding site", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04089", + "source": "HAMAP-Rule" + } + ] + }, + { + "location": { + "start": { + "value": 2, + "modifier": "EXACT" + }, + "end": { + "value": 142, + "modifier": "EXACT" + } + }, + "type": "Domain", + "description": "Rotavirus non-structural protein 35 N-terminal", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "id": "PF21067", + "source": "Pfam" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "COFACTOR": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Domain": 2, + "Region": 1, + "Active site": 1, + "Binding site": 4 + }, + "uniParcId": "UPI00042E6D38" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-04-16", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2014-04-16", + "entryVersion": 45 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "value": "Feline leukemia virus subgroup C receptor-related protein", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "EUB57593.1", + "source": "EMBL" + } + ] + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "EUB57593.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_DNA" + } + ], + "id": "APAU02000080" + }, + { + "database": "RefSeq", + "properties": [ + { + "key": "NucleotideSequenceId", + "value": "XM_024496784.1" + } + ], + "id": "XP_024348789.1" + } + ], + "genes": [ + { + "orfNames": [ + { + "value": "EGR_07535", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "EUB57593.1", + "source": "EMBL" + } + ] + } + ] + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004141", + "source": "ARBA" + } + ] + }, + "location": { + "id": "SL-0162", + "value": "Membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004141", + "source": "ARBA" + } + ] + } + } + ] + }, + { + "commentType": "CAUTION", + "texts": [ + { + "value": "The sequence shown here is derived from an EMBL/GenBank/DDBJ whole genome shotgun (WGS) entry which is preliminary data", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "EUB57593.1", + "source": "EMBL" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023136", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Molecular function", + "name": "Receptor", + "id": "KW-0675", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "EUB57593.1", + "source": "EMBL" + } + ] + }, + { + "category": "Technical term", + "name": "Reference proteome", + "id": "KW-1185", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "UP000019149", + "source": "Proteomes" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022692", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022989", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + } + ], + "sequence": { + "value": "MPTPLTHTAYRSPPQEEKINISLYRHRKPKMKSATIATTFIFPNKTYLTCIFAKTQPMRAARGQSKNKGPQQNGSLVPPLDANANPALDVNLGRFSEEEREAILTVVKRDLEIRCMEKYRLKKIRSGLLQEDSQAPLSLPGSANACTICRGRFIIFLNPRVTCSLCRREVCRRCTKAYNGEMLCRICSHDIFYRAVKCPWFYDSSPDKLCGHASDTIVKSLFKDNLPVLNGLTEEQLHKALSNYLKPDRETYVNGVVITLEVMLRIENEQIRTFTKRYQEVRSTAVNALNKVRGTAMKNGQEDSEKEVDRITSQFLQQTREALRKLLQMLCITEEKYKLSVHNSRQVNTATRVLEITQKQVEQLVGHKIQIPDEYTNVEVEEDLESFVASVASGPEEFFANVLLEDLTKAMRNSISHGRNSRAPLSSDQQGKDVAIAWLSMLYMLVYIPMIFPATWLLNHYGLRVSILIGATLNMLGAWIKCLSMELSQPTSSPASNASFPLLMVAQLVCALGQVFLLGVPAQLAATWFSKSELAMATAIGVGCALGFGLPPLMVPAVTSSTGVADFEDFRKGFRIMFYGGAAIMSFDLLLVAIFFKEEPEIAPSRAQYKRILQRRAQLAGDKPDEADFAPAFDHESSVLTAEESVLVNQSYFHQVAHSFKCVSFIYLNICYGVNTGVYYEIGTLLNSIVAEFFPTAQGARMELPNLERCVKKSPSGVKQINVVKDGRIEAISTGDQFCTSPGSEDVEGSDELSLNLSSETEQVAIGWIGFSMVIAGLVGSIVAGVVLKKTGLYRLVLIIFYFLSVISMGAFMGSLYSNLIGTVFLTMILLGFFQSGFLPLGFEYAAEITYPIDEGLTSGILNTSAHIFGIILTQVATAMIGNYGALPTNIFILVCMVIAGVPACFMKDDLKRQRAHEKMRVEVEANSGDEFKYSSISKTKDV", + "length": 943, + "molWeight": 104642, + "crc64": "EBEC2FBABAA7A2A2", + "md5": "52E715C9787B9CD778FA5533664DC417" + }, + "organism": { + "taxonId": 6210, + "commonName": "Hydatid tapeworm", + "scientificName": "Echinococcus granulosus", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "EUB57593.1", + "source": "EMBL" + }, + { + "evidenceCode": "ECO:0000313", + "id": "UP000019149", + "source": "Proteomes" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Spiralia", + "Lophotrochozoa", + "Platyhelminthes", + "Rhabditophora", + "Neodermata", + "Cestoda", + "Eucestoda", + "Cyclophyllidea", + "Taeniidae", + "Echinococcus", + "Echinococcus granulosus group" + ] + }, + "proteinExistence": "4: Predicted", + "uniProtkbId": "W6U8K5_ECHGR", + "primaryAccession": "W6U8K5", + "references": [ + { + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "1168", + "lastPage": "1175", + "volume": "45", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24013640" + }, + { + "database": "DOI", + "id": "10.1038/ng.2757" + } + ], + "authors": [ + "Zheng H.", + "Zhang W.", + "Zhang L.", + "Zhang Z.", + "Li J.", + "Lu G.", + "Zhu Y.", + "Wang Y.", + "Huang Y.", + "Liu J.", + "Kang H.", + "Chen J.", + "Wang L.", + "Chen A.", + "Yu S.", + "Gao Z.", + "Jin L.", + "Gu W.", + "Wang Z.", + "Zhao L.", + "Shi B.", + "Wen H.", + "Lin R.", + "Jones M.K.", + "Brejova B.", + "Vinar T.", + "Zhao G.", + "McManus D.P.", + "Chen Z.", + "Zhou Y.", + "Wang S." + ], + "publicationDate": "2013", + "title": "The genome of the hydatid tapeworm Echinococcus granulosus.", + "id": "24013640", + "journal": "Nat. Genet.", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "EUB57593.1", + "source": "EMBL" + }, + { + "evidenceCode": "ECO:0000313", + "id": "UP000019149", + "source": "Proteomes" + } + ] + } + ], + "features": [ + { + "location": { + "start": { + "value": 435, + "modifier": "EXACT" + }, + "end": { + "value": 455, + "modifier": "EXACT" + } + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "location": { + "start": { + "value": 461, + "modifier": "EXACT" + }, + "end": { + "value": 480, + "modifier": "EXACT" + } + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "SUBCELLULAR LOCATION": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Transmembrane": 10, + "Domain": 1, + "Region": 1, + "Compositional bias": 1 + }, + "uniParcId": "UPI0003E7E985" + } + }, + { + "entryAudit": { + "sequenceVersion": 2, + "firstPublicDate": "2014-04-16", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2018-12-05", + "entryVersion": 58 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "value": "FLVCR choline and putative heme transporter 2", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "ENSAMXP00000014911.2", + "source": "Ensembl" + } + ] + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "AlphaFoldDB", + "properties": [ + { + "key": "Description", + "value": "-" + } + ], + "id": "W5L4Z9" + }, + { + "database": "Ensembl", + "properties": [ + { + "key": "ProteinId", + "value": "ENSAMXP00000014911.2" + }, + { + "key": "GeneId", + "value": "ENSAMXG00000014488.2" + } + ], + "id": "ENSAMXT00000014911.2" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004651", + "source": "ARBA" + } + ] + }, + "location": { + "id": "SL-0039", + "value": "Cell membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004651", + "source": "ARBA" + } + ] + } + }, + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004477", + "source": "ARBA" + } + ] + }, + "location": { + "id": "SL-0097", + "value": "Endoplasmic reticulum membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004477", + "source": "ARBA" + } + ] + } + }, + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004225", + "source": "ARBA" + } + ] + }, + "location": { + "id": "SL-0171", + "value": "Mitochondrion membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004225", + "source": "ARBA" + } + ] + } + } + ] + }, + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "reactionCrossReferences": [ + { + "database": "Rhea", + "id": "RHEA:32751" + }, + { + "database": "ChEBI", + "id": "CHEBI:15354" + } + ], + "name": "choline(out) = choline(in)", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00036811", + "source": "ARBA" + } + ] + } + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Choline uniporter that specifically mediates choline uptake at the blood-brain-barrier. Responsible for the majority of choline uptake across the blood-brain-barrier from the circulation into the brain. Choline, a nutrient critical for brain development, is a precursor of phosphatidylcholine, as well as betaine. Also mediates transport of ethanolamine. Choline and ethanolamine transport is not coupled with proton transport and is exclusively driven by the choline gradient across the plasma membrane. Also acts as a heme b transporter", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00053231", + "source": "ARBA" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Cellular component", + "name": "Cell membrane", + "id": "KW-1003", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022475", + "source": "ARBA" + } + ] + }, + { + "category": "Cellular component", + "name": "Endoplasmic reticulum", + "id": "KW-0256", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022824", + "source": "ARBA" + } + ] + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023136", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Cellular component", + "name": "Mitochondrion", + "id": "KW-0496", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023128", + "source": "ARBA" + } + ] + }, + { + "category": "Technical term", + "name": "Reference proteome", + "id": "KW-1185", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "UP000018467", + "source": "Proteomes" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022692", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022989", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Biological process", + "name": "Transport", + "id": "KW-0813", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022448", + "source": "ARBA" + } + ] + } + ], + "sequence": { + "value": "MSACENSEDLKGQTNGPALVKSNISTSEPAAEQIECKVFSPEAPQECTTRLYKRRWAIVFLFSSYSLCNAFQWIQYGIINNIFMHFYSVSSFAIDWMSMIYMLTYIPFIFPVTWLLDKKGLRVVALVASSINCLGTWIKVASVKPHLFWVTMLGQFACSFAQVFILGMPSRIASVWFGSGEVSTACSIGVFGNQLGIAIGFLVPPILVPNVEDLDKLAHHISIMFYITAGVATLIFFLVIIVFQEKPEIPPSQAQAAMRNMSPQEYSYLASIKRLIHNKSFILLIITYGLNVGSFYAVSTLLNRMIIDHYPGEEVNAGRIGLTIVIAGMVGSLICGIWLDKSKTYKQTTLAVYFLTLVGMLIYAFTLDLGHLWVVFLTAGALGFFMTGYLPLGFEFAVELTYPESEGTSSGLLNCSAQVFGIIFTICQGKIIDYYGTLAGNIFLCIFLLIGTIMTGFIKADLRRQRANQVAETNAKKPGCQEEGLSSPLKEDKV", + "length": 494, + "molWeight": 54676, + "crc64": "9D5DBF17CAD5C77E", + "md5": "E646D99157273186F09C65F70AD4B392" + }, + "organism": { + "taxonId": 7994, + "commonName": "Blind cave fish", + "synonyms": [ + "Astyanax fasciatus mexicanus" + ], + "scientificName": "Astyanax mexicanus", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "ENSAMXP00000014911.2", + "source": "Ensembl" + }, + { + "evidenceCode": "ECO:0000313", + "id": "UP000018467", + "source": "Proteomes" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Chordata", + "Craniata", + "Vertebrata", + "Euteleostomi", + "Actinopterygii", + "Neopterygii", + "Teleostei", + "Ostariophysi", + "Characiformes", + "Characoidei", + "Acestrorhamphidae", + "Acestrorhamphinae", + "Astyanax" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W5L4Z9_ASTMX", + "primaryAccession": "W5L4Z9", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "female", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "UP000018467", + "source": "Proteomes" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "5307", + "lastPage": "5307", + "volume": "5", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "25329095" + }, + { + "database": "DOI", + "id": "10.1038/ncomms6307" + } + ], + "authors": [ + "McGaugh S.E.", + "Gross J.B.", + "Aken B.", + "Blin M.", + "Borowsky R.", + "Chalopin D.", + "Hinaux H.", + "Jeffery W.R.", + "Keene A.", + "Ma L.", + "Minx P.", + "Murphy D.", + "O'Quin K.E.", + "Retaux S.", + "Rohner N.", + "Searle S.M.", + "Stahl B.A.", + "Tabin C.", + "Volff J.N.", + "Yoshizawa M.", + "Warren W.C." + ], + "publicationDate": "2014", + "title": "The cavefish genome reveals candidate genes for eye loss.", + "id": "25329095", + "journal": "Nat. Commun.", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "UP000018467", + "source": "Proteomes" + } + ] + }, + { + "referenceNumber": 2, + "referencePositions": [ + "IDENTIFICATION" + ], + "citation": { + "publicationDate": "JAN-2026", + "id": "CI-20IM145UGQVF7", + "authoringGroup": [ + "Ensembl" + ], + "submissionDatabase": "UniProtKB", + "citationType": "submission" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "ENSAMXP00000014911.2", + "source": "Ensembl" + } + ] + } + ], + "features": [ + { + "location": { + "start": { + "value": 58, + "modifier": "EXACT" + }, + "end": { + "value": 463, + "modifier": "EXACT" + } + }, + "type": "Domain", + "description": "Major facilitator superfamily (MFS) profile", + "evidences": [ + { + "evidenceCode": "ECO:0000259", + "id": "PS50850", + "source": "PROSITE" + } + ] + }, + { + "location": { + "start": { + "value": 473, + "modifier": "EXACT" + }, + "end": { + "value": 494, + "modifier": "EXACT" + } + }, + "type": "Region", + "description": "Disordered", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MobiDB-lite", + "source": "SAM" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "CATALYTIC ACTIVITY": 3, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Domain": 1, + "Region": 1 + }, + "uniParcId": "UPI000BBDE7D1" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 37 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 4.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "NCVP5", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + }, + { + "fullName": { + "value": "NS28", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + } + ], + "recommendedName": { + "shortNames": [ + { + "value": "NSP4", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + ], + "fullName": { + "value": "Non-structural glycoprotein 4", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44730.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ], + "id": "KJ187597" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:extracellular region" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ], + "id": "GO:0005576" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "NSP4 localizes also in vesicular structures which contain autophagosomal markers and associate with viroplasms in virus-infected cells. Additionally, a soluble form of glycosylated NSP4 is secreted despite retention of its transmembrane domain", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + ] + }, + "subcellularLocations": [ + { + "topology": { + "id": "SL-9907", + "value": "Single-pass type III membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + "location": { + "id": "SL-0423", + "value": "Host rough endoplasmic reticulum membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + }, + { + "topology": { + "id": "SL-9907", + "value": "Single-pass type III membrane protein", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + "location": { + "id": "SL-0428", + "value": "Host membrane, host caveola", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + }, + { + "location": { + "id": "SL-0243", + "value": "Secreted", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Plays an essential role in the virus replication cycle by acting as a viroporin. Creates a pore in the host endoplasmic reticulum and as a consequence releases Ca(2+) in the cytoplasm of infected cell. In turn, high levels of cytoplasmic calcium trigger membrane trafficking and transport of viral ER-associated proteins to viroplasms, sites of viral genome replication and immature particle assembly", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Biological process", + "name": "Activation of host autophagy by virus", + "id": "KW-1072", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023050", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Calcium", + "id": "KW-0106", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022837", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Coiled coil", + "id": "KW-0175", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Coils", + "source": "SAM" + } + ] + }, + { + "category": "Molecular function", + "name": "Enterotoxin", + "id": "KW-0260", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022861", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023180", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host endoplasmic reticulum", + "id": "KW-1038", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023184", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host membrane", + "id": "KW-1043", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022870", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Host-virus interaction", + "id": "KW-0945", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022581", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "Ion channel", + "id": "KW-0407", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023303", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Ion transport", + "id": "KW-0406", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023065", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023136", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022723", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Secreted", + "id": "KW-0964", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022525", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Signal-anchor", + "id": "KW-0735", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022968", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "Toxin", + "id": "KW-0800", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022656", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022692", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022989", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Transport", + "id": "KW-0813", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022448", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "Viral ion channel", + "id": "KW-1182", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023039", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Virulence", + "id": "KW-0843", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023026", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + ], + "sequence": { + "value": "MEKLTDLNYTLSVITLMNDTLHTIMEDPGMAYFPYIASVLTVLFTLHKASIPTMKIALRTSRCSYKVIKYCIVSIFNTLLKLAGYKEQITTKDEIEKQMDRVVKEMRRQLEMIDKLTTREIEQVELLKRIHDMLVIKPVDKVDMSQEFNQIQFKTLTEWVEGENPYEPKEVTASL", + "length": 175, + "molWeight": 20457, + "crc64": "C336CE8F0EE54E0C", + "md5": "5FEED6986EFE388FAC3C63734DF453FD" + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44730.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8Q983_9REOV", + "primaryAccession": "W8Q983", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44730.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44730.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "ligand": { + "name": "Ca(2+)", + "id": "ChEBI:CHEBI:29108" + }, + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:29108" + } + ], + "location": { + "start": { + "value": 120, + "modifier": "EXACT" + }, + "end": { + "value": 120, + "modifier": "EXACT" + } + }, + "type": "Binding site", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + }, + { + "featureId": "", + "location": { + "start": { + "value": 8, + "modifier": "EXACT" + }, + "end": { + "value": 8, + "modifier": "EXACT" + } + }, + "type": "Glycosylation", + "description": "N-linked (GlcNAc...) asparagine; by host", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04091", + "source": "HAMAP-Rule" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "DOMAIN": 1, + "PTM": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Topological domain": 2, + "Transmembrane": 1, + "Coiled coil": 1, + "Binding site": 2, + "Glycosylation": 2 + }, + "uniParcId": "UPI00042F2EA0" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 31 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "recommendedName": { + "fullName": { + "value": "Intermediate capsid protein VP6", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44737.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ], + "id": "KJ187603" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:T=13 icosahedral viral capsid" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-ARBA" + } + ], + "id": "GO:0039621" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Component of the intermediate capsid. Also found in spherical cytoplasmic structures, called virus factories, that appear early after infection and are the site of viral replication and packaging", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + } + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Intermediate capsid protein that self assembles to form an icosahedral capsid with a T=13 symmetry, which consists of 230 trimers of VP6, with channels at each of its five-fold vertices. This capsid constitutes the middle concentric layer of the viral mature particle. The innermost VP2 capsid and the intermediate VP6 capsid remain intact following cell entry to protect the dsRNA from degradation and to prevent unfavorable antiviral responses in the host cell during all the replication cycle of the virus. Nascent transcripts are transcribed within the structural confines of this double-layered particle (DLP) and are extruded through the channels at the five-fold axes. VP6 is required for the transcription activity of the DLP", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00057631", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Ligand", + "name": "Calcium", + "id": "KW-0106", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022837", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Capsid protein", + "id": "KW-0167", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022561", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Intermediate capsid protein", + "id": "KW-1154", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022493", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022723", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "PTM", + "name": "Ubl conjugation", + "id": "KW-0832", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022843", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022844", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Zinc", + "id": "KW-0862", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022833", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + } + ], + "sequence": { + "value": "MDVLYSLSKTLKDARDKIVEGTLYSNVSDLIQQFNQMIITMNGNEFQTGGIGNLPVRNWNFDFGLLGTTLLNLDANYVETARNTIDYFVDFVDNVCMDEMVRESQRNGIAPQSDSLRKLSGIKFKRINFDNSSEYIENWNLQNRRQRTGFTFHKPNIFPYSASFTLNRSQPAHDNLMGTMWLNAGSEIQVAGFDYSCAINAPANIQQFEHIVQLRRVLTTATITLLPDAERFSFPRVINSADGATTWYFNPVILRPNNVEVEFLLNGQIINTYQARFGTIIARNFDTIRLSFQLMRPPNMTPAVAALFPNAQPFEHHATVGLTLRIESAVCESVLADASETMLANVTSVRQEYAIPVGPVFPPGMNWTDLITNYSPSREDNLQRVFTVASIRSMLIK", + "length": 397, + "molWeight": 44897, + "crc64": "1AC9A23DA62FBEC0", + "md5": "74A8486A6D109C92A854B58EFBEB03A4" + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44737.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8R3F3_9REOV", + "primaryAccession": "W8R3F3", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44737.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44737.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "ligand": { + "note": "ligand shared between all trimeric partners", + "name": "Zn(2+)", + "id": "ChEBI:CHEBI:29105" + }, + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:29105" + } + ], + "location": { + "start": { + "value": 153, + "modifier": "EXACT" + }, + "end": { + "value": 153, + "modifier": "EXACT" + } + }, + "type": "Binding site", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + }, + { + "location": { + "start": { + "value": 62, + "modifier": "EXACT" + }, + "end": { + "value": 73, + "modifier": "EXACT" + } + }, + "type": "Region", + "description": "Interaction with the inner capsid protein VP2", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04129", + "source": "HAMAP-Rule" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "MISCELLANEOUS": 1, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Region": 1, + "Binding site": 3 + }, + "uniParcId": "UPI00042F181F" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 50 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "recommendedName": { + "fullName": { + "value": "Outer capsid glycoprotein VP7", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "RU363022", + "source": "RuleBase" + } + ] + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44738.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ], + "id": "KJ187604" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell endoplasmic reticulum lumen" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ], + "id": "GO:0044166" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "The outer layer contains 780 copies of VP7, grouped as 260 trimers. Immature double-layered particles assembled in the cytoplasm bud across the membrane of the endoplasmic reticulum, acquiring during this process a transient lipid membrane that is modified with the ER resident viral glycoproteins NSP4 and VP7; these enveloped particles also contain VP4. As the particles move towards the interior of the ER cisternae, the transient lipid membrane and the non-structural protein NSP4 are lost, while the virus surface proteins VP4 and VP7 rearrange to form the outermost virus protein layer, yielding mature infectious triple-layered particles", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0389", + "value": "Host endoplasmic reticulum lumen", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "RU363022", + "source": "RuleBase" + } + ] + } + }, + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + } + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Calcium-binding protein that interacts with rotavirus cell receptors once the initial attachment by VP4 has been achieved. Rotavirus attachment and entry into the host cell probably involves multiple sequential contacts between the outer capsid proteins VP4 and VP7, and the cell receptors. Following entry into the host cell, low intracellular or intravesicular Ca(2+) concentration probably causes the calcium-stabilized VP7 trimers to dissociate from the virion. This step is probably necessary for the membrane-disrupting entry step and the release of VP4, which is locked onto the virion by VP7", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00059356", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "Ligand", + "name": "Calcium", + "id": "KW-0106", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022837", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Capsid protein", + "id": "KW-0167", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022561", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "PTM", + "name": "Disulfide bond", + "id": "KW-1015", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023157", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023180", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host endoplasmic reticulum", + "id": "KW-1038", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023184", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Biological process", + "name": "Host-virus interaction", + "id": "KW-0945", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022581", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022723", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Outer capsid protein", + "id": "KW-1152", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022770", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Signal", + "id": "KW-0732", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022729", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04131", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "T=13 icosahedral capsid protein", + "id": "KW-1146", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022708", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022844", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04130", + "source": "HAMAP-Rule" + } + ] + } + ], + "sequence": { + "value": "MYGIEYTTVLTFLISVILLNYALKSLTRIMDFIIYRFLLIVVVLSPFLNAQNYGINLPITGSIDTPYTNSTQEEMFLTSTLCLYYPTEAATEINDNSWKDTLSQLFLTKGWPTGSVYFKDYADIASFSIDPQLYCDYNLVLMKYDAALQLDMSELADLILNEWLCNPMDITLYYYQQTDEANKWISMGSSCTIKVCPLNVQTLGIGCLTTDTSTFEEIATAEKLVITDVVDGVNHKLNVTTNTCTIRNCKKLGPRENVAVIQVGGSNVLDITADPTTAPQTERMMRVNWKKWWQVFYTIVDYVNQIVQAMSKRSRSLNSAAFYYRV", + "length": 326, + "molWeight": 37073, + "crc64": "BE3B85417F66327D", + "md5": "D5F596F60A5D582F6C2D226E70983AB9" + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44738.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8QKB9_9REOV", + "primaryAccession": "W8QKB9", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44738.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44738.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "ligand": { + "name": "Ca(2+)", + "id": "ChEBI:CHEBI:29108", + "label": "1" + }, + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:29108" + } + ], + "location": { + "start": { + "value": 95, + "modifier": "EXACT" + }, + "end": { + "value": 95, + "modifier": "EXACT" + } + }, + "type": "Binding site", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04131", + "source": "HAMAP-Rule" + } + ] + }, + { + "featureId": "PRO_5023428870", + "location": { + "start": { + "value": 51, + "modifier": "EXACT" + }, + "end": { + "value": 326, + "modifier": "EXACT" + } + }, + "type": "Chain", + "description": "Outer capsid glycoprotein VP7", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04131", + "source": "HAMAP-Rule" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "SUBUNIT": 2, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "MISCELLANEOUS": 2, + "SIMILARITY": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Chain": 1, + "Transmembrane": 2, + "Region": 2, + "Binding site": 9, + "Disulfide bond": 4 + }, + "uniParcId": "UPI00042E6A3F" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2014-05-14", + "lastAnnotationUpdateDate": "2026-06-10", + "lastSequenceUpdateDate": "2014-05-14", + "entryVersion": 33 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "NS26", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + } + } + ], + "recommendedName": { + "shortNames": [ + { + "value": "NSP5", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR004006", + "source": "PIRNR" + } + ] + } + ], + "fullName": { + "value": "Non-structural protein 5", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR004006", + "source": "PIRNR" + } + ] + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "AHL44731.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "Genomic_RNA" + } + ], + "id": "KJ187598" + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell cytoplasm" + }, + { + "key": "GoEvidenceType", + "value": "IEA:UniProtKB-SubCell" + } + ], + "id": "GO:0030430" + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Found in spherical cytoplasmic structures, called virus factories, that appear early after infection and are the site of viral replication and packaging", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR004006", + "source": "PIRNR" + } + ] + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0381", + "value": "Host cytoplasm", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + }, + { + "evidenceCode": "ECO:0000256", + "id": "PIRNR004006", + "source": "PIRNR" + } + ] + } + } + ] + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "cofactorCrossReference": { + "database": "ChEBI", + "id": "CHEBI:18420" + }, + "name": "Mg(2+)", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + } + ] + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Plays an essential role in the viral genome replication. Participates, together with NSP2, in the formation of viral factories (viroplasms) which are large inclusions in the host cytoplasm where replication intermediates are assembled and viral RNA replication takes place. Orchestrates the recruitment of viroplasmic proteins such as capsid proteins to these factories. Participates in the selective exclusion of host proteins from stress granules (SG) and P bodies (PB). Participates also in the sequestration of these remodeled organelles in viral factories", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023180", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023200", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Magnesium", + "id": "KW-0460", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022842", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Ligand", + "name": "Nucleotide-binding", + "id": "KW-0547", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022741", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "PTM", + "name": "Phosphoprotein", + "id": "KW-0597", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "category": "Molecular function", + "name": "RNA-binding", + "id": "KW-0694", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022884", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + } + ], + "sequence": { + "value": "MSLSIDVTSLPSISSSIYKNESSSTTSTLSGKSIGRSEQYISPDAEAFGKYMLSKSPEDIGPSDSASNDPLTSFSIRSNAVKTNADASVSMDSSTQSRPSSNVGCDQVDFSFNKGIKMNTNLDSSISIATFSKKEKSKNDHKSRKHYPKVEAESDSDDYVLDDSDSDDGKCKNCKYKRKYFALRVRMKQVAMQLIEDL", + "length": 198, + "molWeight": 21726, + "crc64": "27C0104400E9CE41", + "md5": "F11A74C9F145ED70EB6CD8BCAE6D3989" + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44731.1", + "source": "EMBL" + } + ], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A" + ] + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8Q8A1_9REOV", + "primaryAccession": "W8Q8A1", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44731.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]" + ], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "24818762" + } + ], + "authors": [ + "Jeong S.", + "Than V.T.", + "Lim I.", + "Kim W." + ], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "AHL44731.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "ligand": { + "name": "Mg(2+)", + "id": "ChEBI:CHEBI:18420" + }, + "featureCrossReferences": [ + { + "database": "ChEBI", + "id": "CHEBI:18420" + } + ], + "location": { + "start": { + "value": 92, + "modifier": "EXACT" + }, + "end": { + "value": 92, + "modifier": "EXACT" + } + }, + "type": "Binding site", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MF_04092", + "source": "HAMAP-Rule" + } + ] + }, + { + "location": { + "start": { + "value": 16, + "modifier": "EXACT" + }, + "end": { + "value": 37, + "modifier": "EXACT" + } + }, + "type": "Region", + "description": "Disordered", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "MobiDB-lite", + "source": "SAM" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "COFACTOR": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "SIMILARITY": 1 + }, + "countByFeatureType": { + "Region": 4, + "Compositional bias": 2, + "Binding site": 1, + "Modified residue": 5 + }, + "uniParcId": "UPI00042E2A3D" + } + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": "2018-10-10", + "lastAnnotationUpdateDate": "2026-09-02", + "lastSequenceUpdateDate": "2018-10-10", + "entryVersion": 27 + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "value": "Signaling lymphocyte activation molecule", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAN42597.1", + "source": "EMBL" + } + ] + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + { + "key": "ProteinId", + "value": "BAN42597.1" + }, + { + "key": "Status", + "value": "-" + }, + { + "key": "MoleculeType", + "value": "mRNA" + } + ], + "id": "AB771742" + }, + { + "database": "RefSeq", + "properties": [ + { + "key": "NucleotideSequenceId", + "value": "NM_001278826.1" + } + ], + "id": "NP_001265755.1" + } + ], + "genes": [ + { + "geneName": { + "value": "SLAM", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAN42597.1", + "source": "EMBL" + } + ] + } + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "id": "SL-0162", + "value": "Membrane", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00004370", + "source": "ARBA" + } + ] + } + } + ] + }, + { + "commentType": "CAUTION", + "texts": [ + { + "value": "Lacks conserved residue(s) required for the propagation of feature annotation", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PRU00114", + "source": "PROSITE-ProRule" + } + ] + } + ] + } + ], + "keywords": [ + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023180", + "source": "ARBA" + } + ] + }, + { + "category": "Domain", + "name": "Immunoglobulin domain", + "id": "KW-0393", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "PRU00114", + "source": "PROSITE-ProRule" + } + ] + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00023136", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Signal", + "id": "KW-0732", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "ARBA00022729", + "source": "ARBA" + }, + { + "evidenceCode": "ECO:0000256", + "id": "SignalP", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "Phobius", + "source": "SAM" + } + ] + } + ], + "sequence": { + "value": "MDLKGLLSLNFLLFLSLAFELSCGTGESLMTCQEVTEQLGRNLLLPLASEGISKSMNKSIHILVTKSESPQKNVKTKIVSLSLPERASPRFLENGYKFHPENLKLEILQSKKENEGWYFTTLEENFSVRHFCLQLKLFEQVSTPEIKVLNWTQENGNCSLTLACEVEKGDHVAYSWREETGADPPISANSSHLHLSLGPQHVHNVYVCTVSNPISNHSQTFTPGSVCMPDPPELRPWGLYVGLSSGAVIGVILTLKVAVLLLRRRGKANHYQPTTEAKSLTIYAQVQKSGSIQKKPDAPSAQDPCTTIYVAATEPVPEPVQEPKSITVYASVTLPES", + "length": 337, + "molWeight": 37201, + "crc64": "F24FDE8F6BDA985B", + "md5": "246583275688D762B2F35467EAB3FE5C" + }, + "organism": { + "taxonId": 9685, + "commonName": "Cat", + "synonyms": [ + "Felis silvestris catus" + ], + "scientificName": "Felis catus", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAN42597.1", + "source": "EMBL" + } + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Chordata", + "Craniata", + "Vertebrata", + "Euteleostomi", + "Mammalia", + "Eutheria", + "Laurasiatheria", + "Carnivora", + "Feliformia", + "Felidae", + "Felinae", + "Felis" + ] + }, + "proteinExistence": "2: Evidence at transcript level", + "uniProtkbId": "W6JD38_FELCA", + "primaryAccession": "W6JD38", + "references": [ + { + "referenceComments": [ + { + "type": "TISSUE", + "value": "Thymus", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAN42597.1", + "source": "EMBL" + } + ] + } + ], + "referenceNumber": 1, + "referencePositions": [ + "NUCLEOTIDE SEQUENCE" + ], + "citation": { + "firstPage": "1085", + "lastPage": "1089", + "volume": "75", + "citationCrossReferences": [ + { + "database": "PubMed", + "id": "23535266" + }, + { + "database": "DOI", + "id": "10.1292/jvms.13-0003" + } + ], + "authors": [ + "Hara Y.", + "Suzuki J.", + "Noguchi K.", + "Terada Y.", + "Shimoda H.", + "Mizuno T.", + "Maeda K." + ], + "publicationDate": "2013", + "title": "Function of feline signaling lymphocyte activation molecule as a receptor of canine distemper virus.", + "id": "23535266", + "journal": "J. Vet. Med. Sci.", + "citationType": "journal article" + }, + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "BAN42597.1", + "source": "EMBL" + } + ] + } + ], + "features": [ + { + "featureId": "PRO_5016357255", + "location": { + "start": { + "value": 19, + "modifier": "EXACT" + }, + "end": { + "value": 337, + "modifier": "EXACT" + } + }, + "type": "Chain", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "SignalP", + "source": "SAM" + } + ] + }, + { + "location": { + "start": { + "value": 1, + "modifier": "EXACT" + }, + "end": { + "value": 18, + "modifier": "EXACT" + } + }, + "type": "Signal", + "description": "", + "evidences": [ + { + "evidenceCode": "ECO:0000256", + "id": "SignalP", + "source": "SAM" + } + ] + } + ], + "extraAttributes": { + "countByCommentType": { + "SUBCELLULAR LOCATION": 1, + "CAUTION": 1 + }, + "countByFeatureType": { + "Signal": 1, + "Chain": 1, + "Transmembrane": 1, + "Domain": 1 + }, + "uniParcId": "UPI000298A0A1" + } + } + ] + } +] diff --git a/tests/data/jsonschema/examples/uniprot_search/maximal_expected_table.py b/tests/data/jsonschema/examples/uniprot_search/maximal_expected_table.py new file mode 100644 index 00000000..72915135 --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/maximal_expected_table.py @@ -0,0 +1,3503 @@ +"""Expected PyArrow table for maximal real-world example uniprot_search.""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "results": [ + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 6, 11), + "lastAnnotationUpdateDate": dt.date(2026, 9, 2), + "lastSequenceUpdateDate": dt.date(2014, 6, 11), + "entryVersion": 59, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 4.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "Large structural protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + }, + { + "fullName": { + "value": "Replicase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + }, + { + "fullName": { + "value": "Transcriptase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + }, + ], + "recommendedName": { + "shortNames": [ + { + "value": "Protein L", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + ], + "fullName": { + "value": "RNA-directed RNA polymerase L", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}], + }, + }, + "includes": [ + { + "recommendedName": { + "ecNumbers": [ + { + "value": "2.7.7.48", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + ], + "fullName": { + "value": "RNA-directed RNA polymerase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + }, + } + }, + { + "recommendedName": { + "ecNumbers": [ + { + "value": "3.6.1.-", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + ], + "fullName": { + "value": "GTP phosphohydrolase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + }, + } + }, + { + "alternativeNames": [ + { + "fullName": { + "value": "PRNTase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + } + ], + "recommendedName": { + "ecNumbers": [ + { + "value": "2.7.7.88", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + ], + "fullName": { + "value": "GDP polyribonucleotidyltransferase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + }, + }, + }, + { + "recommendedName": { + "shortNames": [ + { + "value": "N1-2'-O-MTase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + ], + "fullName": { + "value": "mRNA (nucleoside-2'-O-)-methyltransferase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + }, + } + }, + { + "recommendedName": { + "shortNames": [ + { + "value": "G-N7-MTase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + ], + "fullName": { + "value": "mRNA (guanine-N(7)-)-methyltransferase", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + }, + } + }, + ], + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "BAO58314.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Viral_cRNA"}, + ], + "id": "AB910309", + }, + { + "database": "GO", + "properties": [ + {"key": "GoTerm", "value": "C:host cell cytoplasm"}, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-SubCell"}, + ], + "id": "GO:0030430", + }, + ], + "genes": [ + { + "geneName": { + "value": "L", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "BAO58314.1", "source": "EMBL"}], + } + } + ], + "comments": [ + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "reactionCrossReferences": [ + {"database": "Rhea", "id": "RHEA:21248"}, + {"database": "Rhea", "id": "RHEA-COMP:14527"}, + {"database": "Rhea", "id": "RHEA-COMP:17342"}, + {"database": "ChEBI", "id": "CHEBI:33019"}, + {"database": "ChEBI", "id": "CHEBI:61557"}, + {"database": "ChEBI", "id": "CHEBI:140395"}, + ], + "ecNumber": "2.7.7.48", + "name": "RNA(n) + a ribonucleoside 5'-triphosphate = RNA(n+1) + diphosphate", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + {"evidenceCode": "ECO:0000256", "id": "PRU00539", "source": "PROSITE-ProRule"}, + ], + }, + }, + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + }, + { + "location": { + "id": "SL-0381", + "value": "Host cytoplasm", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"} + ], + } + }, + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "RNA-directed RNA polymerase that catalyzes the replication of viral genomic RNA. The template is composed of the viral RNA tightly encapsidated by the nucleoprotein (N). The replicase mode is dependent on intracellular N protein concentration. In this mode, the polymerase replicates the whole viral genome without recognizing transcriptional signals, and the replicated genome is not caped or polyadenylated", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00003132", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Ligand", + "name": "ATP-binding", + "id": "KW-0067", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022840", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023200", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Molecular function", + "name": "Hydrolase", + "id": "KW-0378", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022801", "source": "ARBA"}], + }, + { + "category": "Ligand", + "name": "Magnesium", + "id": "KW-0460", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022842", "source": "ARBA"}], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022723", "source": "ARBA"}], + }, + { + "category": "Molecular function", + "name": "Methyltransferase", + "id": "KW-0489", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022603", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Biological process", + "name": "mRNA capping", + "id": "KW-0506", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023042", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Biological process", + "name": "mRNA processing", + "id": "KW-0507", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022664", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Molecular function", + "name": "Multifunctional enzyme", + "id": "KW-0511", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00023268", "source": "ARBA"}], + }, + { + "category": "Ligand", + "name": "Nucleotide-binding", + "id": "KW-0547", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022741", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Molecular function", + "name": "Nucleotidyltransferase", + "id": "KW-0548", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022695", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Molecular function", + "name": "RNA-directed RNA polymerase", + "id": "KW-0696", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022484", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Ligand", + "name": "S-adenosyl-L-methionine", + "id": "KW-0949", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022691", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Molecular function", + "name": "Transferase", + "id": "KW-0808", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022679", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Biological process", + "name": "Viral RNA replication", + "id": "KW-0693", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022953", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022844", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR000830", "source": "PIRNR"}, + ], + }, + { + "category": "Ligand", + "name": "Zinc", + "id": "KW-0862", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022833", "source": "ARBA"}], + }, + ], + "sequence": { + "value": "MEQSDYQDILYPEVHLNSPIVISKLVGILEYAQVIHNQLLTDHTIIKNIQFRLRNGFNSPRIQTLSTMGEIINKIKNKYSNYFHIPYPECNQKLFRIVDPELTSKLESLLNKGDTLYLKIRSDIIKCFDKLKMKMNISNDLLNDNSQLILDLPLIIKGSQWFFPFLFWFSIKTETRNCIRQNQKARVRSQYRPHLSETKRITLVVTSDLITIFDHINKCIFYLTFEMLLMYCDVIEGRLMTETAMSLDCRFTNLLPRVQYMWDLLDGMFESLGNQIYSVIALLEPLSLAYLQLIDADPQIRGTFLHHCFSELEEIIFDKSPFDPFVYENLINGLDYIYLTDDIHLTAEVFSFFRSFGHPYLEAQNAANNVRKYMNQPKVISYQTLMQGHAIFCGIIINGFRDRHGGTWPPVELPSHASAVIRNAQLSGEGLTSEQCAQHWRSFCGFKFKCFMPLSLDSDLTMYLRDKALSPVRSEWDSVYAKEYLRYNPGLPTSSRRLVNVFLEDDKFDPYEMIMYVINGDYLIDDEFNLSYSLKEKEIKEVGRLFAKMTYKMRACQVIAENLIANGVGKFFKDNGMAKDEHKLTKTLHKLAISGVPKDNSQLYLDECWEQAVRHCSSDTHTKKQTMGSQSKRIIESKSSRSPLNNRDTLKGKRDPNTQLKYPSNTEYYETISSFITTDLKKYCLNWRYESSSVFAERLNEIYGLPGFFQWLHKILEKSVLYVSDPSSPPDFDQHIDIESVPNDHIFIKYPMGGIEGFCQKLWTISTIPFLYLAAFDTGVRIASLVQGDNQAIAVTKRVPSSWSYSKKKEESTKITTQYFLNLRQRLHDIGHELKANETIISSHFFVYSKGIYYDGILLSQALKSMARCVFWSETIVDETRSACSNISTTLAKAIERGYDKFVAYAINIYKTIHQVLIALSFTINPTMTPDITEPFYKSLDLLKNLVLIPAPLGGMNYMNMSRLFVRNIGDPITASFADIKRMIECGLLGCSILSQIMYQKCGSSKYLDWASDPYSINLPYSQSMTKVLKNVTARYVLMHSPNPMLKDLFHEKSQEEDEILAEFLLDRHLIIPRAAHEILSNSVTGARESIAGMLDTTKGLIRASMSRGGLTSSLVLKLSTYDYQQFRTCLEWLYAPTTGIAVSVDSCSVFLAKTIRKRMWVHLTKGREIYGLEVPDILECMQNNIIIDHEDCYSCIQGSRYYTWFFVPSNCQLDQINKSTNSLRVPYVGSTTEERSDMRLSYVRSPSRPLKAAVRIAAVYTWAYGDDDLSWHEAWYLARTRANITFDELKLITPIATSTNLAHRLRDRSTQVKYSGTSLVRVARYTTISNDNMSFIINNKKVDTNFVYQQGMLLGLSILEYIFRYCTSTGQSNTVIHLHADINCCIVQMTDQPYTPSLTKKLPDIKPINNKLIYDPAPIIDTDAARLYSQKYLSHLIDFPSWSTTQLNTVLAKVVAVSIVELITKASKDHLNEIIAVVGDDDINSFITEFLLVDPRLFTLYLGQYTSLQWAYEVHYHRPVGKYQMAEVLHNLLSRASRGIFSVLTNAFSHPRVYRRFWECGLLEPIYGPYIGSQNLHTAVIDYIYNAYITYLDAYLSDQVDDTDIIICETEETCLANRIDNYQSRHLAVLIDMYCDSTRCPNIKGSDTIMRNSILRSFIDNERRTNPLGLTWNLDPLLVDHFSCSITYLRRGIIKQMRLRFDPNISLELAKMIKPDVIYQAPKVPSSWALIDINPEVNDLNTIFGELNSKWKDIPIGQIKIQNYEIHAYRRVGVNSTACYKALEILSVLNRFMSNPLGALFLGEGAGSMLVTYRAFIPFKTIYYNSGISVQHVQGQRELSLYPSEVALVDNKNRLANDPNIKVLFNGKPESTWVGNIDCFAYILSHIETSSLTLIHSDIESSLSKTKNKILEELCHILSMALILGKLGSVLVIKLLPRVGDYTYSFCKYASEFYQQNFLILPRFSNMSSSEIYYIGVHLNTNRLIDPDRIVQYIIRNLQPTPVTFLSYIFETKYRNNMVTNYGLCLSDGHKSDYLSSITKIENVLLSCGLELNGPKIIQQLSGHDYANGETSLESSIMILVREYLNATIQGRETLGLFSPYPVLHESQLREINKCIALKYVVYLLFYLNSISSSKQIMSNLRKGILMYDLRDEFFISRLSANYKKKVMLQEVKTTWIFNIDTPTRKALYKLVGYSLIINHI", + "length": 2202, + "molWeight": 252830, + "crc64": "0B4B3846D2B0EF16", + "md5": "E98D71C0EAA4317745F8AEA2695302FD", + }, + "organism": { + "taxonId": 1170234, + "scientificName": "Feline morbillivirus", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "BAO58314.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Negarnaviricota", + "Haploviricotina", + "Monjiviricetes", + "Mononegavirales", + "Paramyxoviridae", + "Orthoparamyxovirinae", + "Morbillivirus", + "Morbillivirus felis", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "X5I2V6_9PARA", + "primaryAccession": "X5I2V6", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "SS1", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "BAO58314.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "1464", + "lastPage": "1468", + "volume": "95", + "citationCrossReferences": [ + {"database": "PubMed", "id": "24728711"}, + {"database": "DOI", "id": "10.1099/vir.0.065029-0"}, + ], + "authors": [ + "Sakaguchi S.", + "Nakagawa S.", + "Yoshikawa R.", + "Kuwahara C.", + "Hagiwara H.", + "Asai K.", + "Kawakami K.", + "Yamamoto Y.", + "Ogawa M.", + "Miyazawa T.", + ], + "publicationDate": "2014", + "title": "Genetic diversity of feline morbilliviruses isolated in Japan.", + "id": "24728711", + "journal": "J. Gen. Virol.", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "BAO58314.1", "source": "EMBL"}], + } + ], + "features": [ + { + "location": { + "start": {"value": 672, "modifier": "EXACT"}, + "end": {"value": 856, "modifier": "EXACT"}, + }, + "type": "Domain", + "description": "RdRp catalytic", + "evidences": [{"evidenceCode": "ECO:0000259", "id": "PS50526", "source": "PROSITE"}], + }, + { + "location": { + "start": {"value": 1771, "modifier": "EXACT"}, + "end": {"value": 1976, "modifier": "EXACT"}, + }, + "type": "Domain", + "description": "Mononegavirus-type SAM-dependent 2'-O-MTase", + "evidences": [{"evidenceCode": "ECO:0000259", "id": "PS51590", "source": "PROSITE"}], + }, + { + "location": { + "start": {"value": 620, "modifier": "EXACT"}, + "end": {"value": 658, "modifier": "EXACT"}, + }, + "type": "Region", + "description": "Disordered", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MobiDB-lite", "source": "SAM"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "CATALYTIC ACTIVITY": 5, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1, + }, + "countByFeatureType": {"Domain": 2, "Region": 1}, + "uniParcId": "UPI00043ABFDE", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 6, 11), + "lastAnnotationUpdateDate": dt.date(2026, 9, 2), + "lastSequenceUpdateDate": dt.date(2014, 6, 11), + "entryVersion": 38, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "recommendedName": { + "ecNumbers": [ + { + "value": "7.1.1.9", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"} + ], + } + ], + "fullName": { + "value": "Cytochrome c oxidase subunit 1", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00015947", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}, + ], + }, + }, + "flag": "Fragment", + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHN64824.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_DNA"}, + ], + "id": "KF823081", + }, + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHN64826.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_DNA"}, + ], + "id": "KF823083", + }, + ], + "genes": [ + { + "geneName": { + "value": "COX1", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AXB54992.1", "source": "EMBL"}], + }, + "synonyms": [ + { + "value": "cox1", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AKZ31582.1", "source": "EMBL"} + ], + } + ], + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"} + ], + }, + "location": { + "id": "SL-0168", + "value": "Mitochondrion inner membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"} + ], + }, + } + ], + }, + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "reactionCrossReferences": [ + {"database": "Rhea", "id": "RHEA:11436"}, + {"database": "Rhea", "id": "RHEA-COMP:10350"}, + {"database": "Rhea", "id": "RHEA-COMP:14399"}, + {"database": "ChEBI", "id": "CHEBI:15377"}, + {"database": "ChEBI", "id": "CHEBI:15378"}, + {"database": "ChEBI", "id": "CHEBI:15379"}, + {"database": "ChEBI", "id": "CHEBI:29033"}, + {"database": "ChEBI", "id": "CHEBI:29034"}, + ], + "ecNumber": "7.1.1.9", + "name": "4 Fe(II)-[cytochrome c] + O2 + 8 H(+)(in) = 4 Fe(III)-[cytochrome c] + 2 H2O + 4 H(+)(out)", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "cofactorCrossReference": {"database": "ChEBI", "id": "CHEBI:30413"}, + "name": "heme", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00001971", "source": "ARBA"} + ], + } + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Component of the cytochrome c oxidase, the last enzyme in the mitochondrial electron transport chain which drives oxidative phosphorylation. The respiratory chain contains 3 multisubunit complexes succinate dehydrogenase (complex II, CII), ubiquinol-cytochrome c oxidoreductase (cytochrome b-c1 complex, complex III, CIII) and cytochrome c oxidase (complex IV, CIV), that cooperate to transfer electrons derived from NADH and succinate to molecular oxygen, creating an electrochemical gradient over the inner membrane that drives transmembrane transport and the ATP synthase. Cytochrome c oxidase is the component of the respiratory chain that catalyzes the reduction of oxygen to water. Electrons originating from reduced cytochrome c in the intermembrane space (IMS) are transferred via the dinuclear copper A center (CU(A)) of subunit 2 and heme A of subunit 1 to the active site in subunit 1, a binuclear center (BNC) formed by heme A3 and copper B (CU(B)). The BNC reduces molecular oxygen to 2 water molecules using 4 electrons from cytochrome c in the IMS and 4 protons from the mitochondrial matrix", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"} + ], + } + ], + }, + ], + "geneLocations": [ + { + "geneEncodingType": "Mitochondrion", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHN64827.1", "source": "EMBL"}], + } + ], + "keywords": [ + { + "category": "Ligand", + "name": "Copper", + "id": "KW-0186", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Biological process", + "name": "Electron transport", + "id": "KW-0249", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Ligand", + "name": "Heme", + "id": "KW-0349", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Ligand", + "name": "Iron", + "id": "KW-0408", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Cellular component", + "name": "Mitochondrion", + "id": "KW-0496", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}, + {"evidenceCode": "ECO:0000313", "id": "AHN64827.1", "source": "EMBL"}, + ], + }, + { + "category": "Cellular component", + "name": "Mitochondrion inner membrane", + "id": "KW-0999", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Biological process", + "name": "Respiratory chain", + "id": "KW-0679", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "category": "Biological process", + "name": "Transport", + "id": "KW-0813", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "RU000369", "source": "RuleBase"}], + }, + ], + "sequence": { + "value": "TLYMLFGVWCGMVGTGLSLLIRWELGTAGVFSDDHFYNVIVTAHAFVMIFFMVMPIMIGGFGNWMVPLLIGAPDMSFPRMNNMSFWLLPPSFLLLISSSLVEGGAGTGWTVYPPLSGLIGHSGASVDLAIFSLHLAGMSSILGAINFITTIFNMRSPGISLELMSLFVWSILVTVFLLLLSLPVLAGAITMLLTDRNFSTSFFDPAGGGDPILYQHLF", + "length": 218, + "molWeight": 23642, + "crc64": "33CDF182AB84C18E", + "md5": "25AB05A09758DF77890BC20192EFAE87", + }, + "organism": { + "taxonId": 31229, + "commonName": "Snail", + "scientificName": "Helix lucorum", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHN64827.1", "source": "EMBL"}], + "lineage": [ + "Eukaryota", + "Metazoa", + "Spiralia", + "Lophotrochozoa", + "Mollusca", + "Gastropoda", + "Heterobranchia", + "Euthyneura", + "Panpulmonata", + "Eupulmonata", + "Stylommatophora", + "Helicina", + "Helicoidea", + "Helicidae", + "Helix", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "X2JFE9_HELLU", + "primaryAccession": "X2JFE9", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "Sn-14", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "QIU80631.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 5, + "referencePositions": ["NUCLEOTIDE SEQUENCE"], + "citation": { + "firstPage": "0", + "lastPage": "345", + "volume": "13", + "citationCrossReferences": [{"database": "PubMed", "id": "32650821"}], + "authors": [ + "Dimzas D.", + "Morelli S.", + "Traversa D.", + "Di Cesare A.", + "Van Bourgonie Y.R.", + "Breugelmans K.", + "Backeljau T.", + "di Regalbono A.F.", + "Diakou A.", + ], + "publicationDate": "2020", + "title": "Intermediate gastropod hosts of major feline cardiopulmonary nematodes in an area of wildcat and domestic cat sympatry in Greece.", + "id": "32650821", + "journal": "Parasit. Vectors", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "QIU80631.1", "source": "EMBL"}], + }, + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "LC18", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHN64827.1", "source": "EMBL"} + ], + }, + { + "type": "STRAIN", + "value": "LC3", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHN64824.1", "source": "EMBL"} + ], + }, + { + "type": "STRAIN", + "value": "LC37", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHN64826.1", "source": "EMBL"} + ], + }, + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE"], + "citation": { + "firstPage": "72", + "lastPage": "91", + "volume": "171", + "authors": ["Korabek O.", "Jurickova L.", "Petrusek A."], + "publicationDate": "2014", + "title": "Resurrecting Helix straminea, a forgotten escargot with trans-Adriatic distribution: first insights into the genetic variation within the genus Helix (Gastropoda: Pulmonata).", + "id": "CI-22T6FP2MKNDTE", + "journal": "Zool. J. Linn. Soc.", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHN64827.1", "source": "EMBL"}], + }, + ], + "features": [ + { + "location": { + "start": {"value": 6, "modifier": "EXACT"}, + "end": {"value": 25, "modifier": "EXACT"}, + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "location": { + "start": {"value": 37, "modifier": "EXACT"}, + "end": {"value": 66, "modifier": "EXACT"}, + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "CATALYTIC ACTIVITY": 1, + "COFACTOR": 1, + "PATHWAY": 1, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1, + }, + "countByFeatureType": {"Transmembrane": 5, "Domain": 1, "Non-terminal residue": 2}, + "uniParcId": "UPI000438205C", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 5, 14), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2014, 5, 14), + "entryVersion": 49, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 4.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "Hemagglutinin", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + } + ], + "recommendedName": { + "fullName": { + "value": "Outer capsid protein VP4", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + }, + "contains": [ + { + "recommendedName": { + "fullName": { + "value": "Outer capsid protein VP8*", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + } + }, + { + "recommendedName": { + "fullName": { + "value": "Outer capsid protein VP5*", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + } + }, + ], + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHL44736.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_RNA"}, + ], + "id": "KJ187602", + }, + { + "database": "GO", + "properties": [ + { + "key": "GoTerm", + "value": "C:host cell endoplasmic reticulum-Golgi intermediate compartment", + }, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-SubCell"}, + ], + "id": "GO:0044172", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Outer capsid protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + ] + }, + "molecule": "Outer capsid protein VP5*", + "subcellularLocations": [ + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + } + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Outer capsid protein VP4: Spike-forming protein that mediates virion attachment to the host epithelial cell receptors and plays a major role in cell penetration, determination of host range restriction and virulence. Rotavirus attachment and entry into the host cell probably involves multiple sequential contacts between the outer capsid proteins VP4 and VP7, and the cell receptors. It is subsequently lost, together with VP7, following virus entry into the host cell. Following entry into the host cell, low intracellular or intravesicular Ca(2+) concentration probably causes the calcium-stabilized VP7 trimers to dissociate from the virion. This step is probably necessary for the membrane-disrupting entry step and the release of VP4, which is locked onto the virion by VP7. During the virus exit from the host cell, VP4 seems to be required to target the newly formed virions to the host cell lipid rafts", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Cellular component", + "name": "Capsid protein", + "id": "KW-0167", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022561", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Coiled coil", + "id": "KW-0175", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023054", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "PTM", + "name": "Disulfide bond", + "id": "KW-1015", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}], + }, + { + "category": "Molecular function", + "name": "Hemagglutinin", + "id": "KW-0348", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022546", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host cell membrane", + "id": "KW-1032", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022511", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023200", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host cytoskeleton", + "id": "KW-1037", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023111", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host endoplasmic reticulum", + "id": "KW-1038", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023184", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host membrane", + "id": "KW-1043", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022870", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Host-virus interaction", + "id": "KW-0945", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022581", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023136", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Outer capsid protein", + "id": "KW-1152", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022770", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Viral attachment to host cell", + "id": "KW-1161", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022804", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Viral penetration into host cytoplasm", + "id": "KW-1162", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022595", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Viral penetration via permeabilization of host membrane", + "id": "KW-1173", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022648", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022844", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Virus entry into host cell", + "id": "KW-1160", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023296", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}, + ], + }, + ], + "sequence": { + "value": "MASLIYRQLLSNSYVTNISDEVNEIGNKKTTNVTVNPGPFAQTGYAPVNWGHGELPDSTLVQPTLDGPYQPTSLNLPVDYWMLIAPTREGKVAEGTNTTDRWFACVLVEPNVQNTQRQYVLDGQNVQLNVSNDSSTSWKFILFIKLTPDGTYTQYSTLSTPHKLCAWMKRDNRVYWYQGATPNASESYYLTINNDNSNVSSDAEFYLIPQSQTAMCTQYINNGLPPIQNTRNIVPVNITSRQIKDIRAQMNEDIVISKTSLWKEMQYNRDIIIRFKFANSIIKSGGLGYKWSEISFKPMNYQYTYTRDEEEVTAHTTCSVNGVNDFNYNGGTLPTDFAISRFEVIKENSYVYVDYWDDSQAFRNMVYVRSLAANLNDVVCSGGSYSFALPVGNHPVMSGGAVTLTSAGVTLSTQYTDYVSLNSLRFRFRLAVSEPSFSISRTRMSGIYGLPAVNPNNNAEYYEVAGRFSLISLVPTNDDYQTPIANSVTVRQDLERQLGELREEFNSLSQEIAVSQLIDLATLPLDMFSMFSGIKSTVEAVKSMTTNVMKRFKTSSLANAISDLTSNMSEAASSVRLTSVRSVGAITLPRARVSLQVGDDLRSMQDVSTQVSNVSRNLRLKEFTTQTDTLSFDDISAAVLKTKLDKSTQISQQTMPDIITESSEKFIPKRSYRIVDEDTAFETGIDGTFYAYKVDTFNEIPFDMERFNKLITDPPVLSAIIDFKTLKNLNDNYGITKKQAMELLHSNPKTLKEFINDNNPIIRNRIENLISQCRL", + "length": 775, + "molWeight": 87160, + "crc64": "E8FC8398CFF982A6", + "md5": "65E31E3EC001110F2507BA47DB99CEDF", + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44736.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8Q8A4_9REOV", + "primaryAccession": "W8Q8A4", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHL44736.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [{"database": "PubMed", "id": "24818762"}], + "authors": ["Jeong S.", "Than V.T.", "Lim I.", "Kim W."], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44736.1", "source": "EMBL"}], + } + ], + "features": [ + { + "featureId": "PRO_5023367224", + "location": { + "start": {"value": 1, "modifier": "EXACT"}, + "end": {"value": 775, "modifier": "EXACT"}, + }, + "type": "Chain", + "description": "Outer capsid protein VP4", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}], + }, + { + "featureId": "PRO_5029066481", + "location": { + "start": {"value": 1, "modifier": "EXACT"}, + "end": {"value": 231, "modifier": "EXACT"}, + }, + "type": "Chain", + "description": "Outer capsid protein VP8*", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04132", "source": "HAMAP-Rule"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 3, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 4, + "DOMAIN": 1, + "PTM": 1, + "MISCELLANEOUS": 2, + "SIMILARITY": 1, + "CAUTION": 1, + }, + "countByFeatureType": { + "Chain": 3, + "Domain": 3, + "Region": 3, + "Coiled coil": 1, + "Motif": 1, + "Site": 3, + "Disulfide bond": 1, + }, + "uniParcId": "UPI00042F3651", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 5, 14), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2014, 5, 14), + "entryVersion": 30, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "NCVP3", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + }, + { + "shortNames": [ + { + "value": "NS35", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + ], + "fullName": { + "value": "Non-structural RNA-binding protein 35", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + }, + }, + ], + "recommendedName": { + "shortNames": [ + { + "value": "NSP2", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + ], + "ecNumbers": [ + { + "value": "3.6.4.-", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + ], + "fullName": { + "value": "Non-structural protein 2", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + }, + }, + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHL44728.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_RNA"}, + ], + "id": "KJ187595", + }, + { + "database": "GO", + "properties": [ + {"key": "GoTerm", "value": "C:host cell cytoplasm"}, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-SubCell"}, + ], + "id": "GO:0030430", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Found in spherical cytoplasmic structures, called viral factories, that appear early after infection and are the site of viral replication and packaging", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0381", + "value": "Host cytoplasm", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + } + ], + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "cofactorCrossReference": {"database": "ChEBI", "id": "CHEBI:18420"}, + "name": "Mg(2+)", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "RU364009", "source": "RuleBase"}, + ], + } + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Participates in replication and packaging of the viral genome. Plays a crucial role, together with NSP5, in the formation of virus factories (viroplasms) which are large inclusions in the host cytoplasm where replication intermediates are assembled and viral RNA replication takes place. Displays ssRNA binding, NTPase, RNA triphosphatase (RTPase) and ATP-independent helix-unwinding activities. The unwinding activity may prepare and organize plus-strand RNAs for packaging and replication by removing interfering secondary structures. The RTPase activity plays a role in the removal of the gamma-phosphate from the rotavirus RNA minus strands of dsRNA genome segments. Participates in the selective exclusion of host proteins from stress granules (SG) and P bodies (PB). Participates also in the sequestration of these remodeled organelles in viral factories", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Ligand", + "name": "ATP-binding", + "id": "KW-0067", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022840", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023200", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Molecular function", + "name": "Hydrolase", + "id": "KW-0378", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022801", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Magnesium", + "id": "KW-0460", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022842", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022723", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Nucleotide-binding", + "id": "KW-0547", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022741", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Molecular function", + "name": "RNA-binding", + "id": "KW-0694", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022884", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}, + ], + }, + ], + "sequence": { + "value": "MAELACFCYPHLENDSYKFIPFNSLAIKCMLTAKVDKKDQDKFYNSIVYGITPPPQFKKRYNTNDNSRGMNYETPMFNKVAVLICEALNSIKVTQSDVANVLSKVVSVRHLENLVLRRENHQDVLFHSKELLLKSVLIAIGHSKEIETTATAEGGEIVFQNAAFTMWKLTYLEHKLMPILDQNFIEYKITMNEDKPISESHVKELIAELRWQYNKFAVITHGKGHYRVVKYSSVANHADRVYATFKSNNKNGSMTEFNLLDQRIIWQNWYAFTSSMKQGNTLEICKKLLFQKMKRESNPFKGLSTDRKMDEVSQIGI", + "length": 317, + "molWeight": 36673, + "crc64": "81A32DF875978D2E", + "md5": "451FD69051210DF8F2D857E25AEB19CC", + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44728.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8QKB0_9REOV", + "primaryAccession": "W8QKB0", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHL44728.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [{"database": "PubMed", "id": "24818762"}], + "authors": ["Jeong S.", "Than V.T.", "Lim I.", "Kim W."], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44728.1", "source": "EMBL"}], + } + ], + "features": [ + { + "ligand": {"name": "ATP", "id": "ChEBI:CHEBI:30616"}, + "featureCrossReferences": [{"database": "ChEBI", "id": "CHEBI:30616"}], + "location": { + "start": {"value": 107, "modifier": "EXACT"}, + "end": {"value": 109, "modifier": "EXACT"}, + }, + "type": "Binding site", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04089", "source": "HAMAP-Rule"}], + }, + { + "location": { + "start": {"value": 2, "modifier": "EXACT"}, + "end": {"value": 142, "modifier": "EXACT"}, + }, + "type": "Domain", + "description": "Rotavirus non-structural protein 35 N-terminal", + "evidences": [{"evidenceCode": "ECO:0000259", "id": "PF21067", "source": "Pfam"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "COFACTOR": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1, + }, + "countByFeatureType": {"Domain": 2, "Region": 1, "Active site": 1, "Binding site": 4}, + "uniParcId": "UPI00042E6D38", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 4, 16), + "lastAnnotationUpdateDate": dt.date(2026, 9, 2), + "lastSequenceUpdateDate": dt.date(2014, 4, 16), + "entryVersion": 45, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "value": "Feline leukemia virus subgroup C receptor-related protein", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "EUB57593.1", "source": "EMBL"} + ], + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "EUB57593.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_DNA"}, + ], + "id": "APAU02000080", + }, + { + "database": "RefSeq", + "properties": [{"key": "NucleotideSequenceId", "value": "XM_024496784.1"}], + "id": "XP_024348789.1", + }, + ], + "genes": [ + { + "orfNames": [ + { + "value": "EGR_07535", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "EUB57593.1", "source": "EMBL"} + ], + } + ] + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004141", "source": "ARBA"} + ], + }, + "location": { + "id": "SL-0162", + "value": "Membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004141", "source": "ARBA"} + ], + }, + } + ], + }, + { + "commentType": "CAUTION", + "texts": [ + { + "value": "The sequence shown here is derived from an EMBL/GenBank/DDBJ whole genome shotgun (WGS) entry which is preliminary data", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "EUB57593.1", "source": "EMBL"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023136", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Molecular function", + "name": "Receptor", + "id": "KW-0675", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "EUB57593.1", "source": "EMBL"}], + }, + { + "category": "Technical term", + "name": "Reference proteome", + "id": "KW-1185", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "UP000019149", "source": "Proteomes"}], + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022692", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022989", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + ], + "sequence": { + "value": "MPTPLTHTAYRSPPQEEKINISLYRHRKPKMKSATIATTFIFPNKTYLTCIFAKTQPMRAARGQSKNKGPQQNGSLVPPLDANANPALDVNLGRFSEEEREAILTVVKRDLEIRCMEKYRLKKIRSGLLQEDSQAPLSLPGSANACTICRGRFIIFLNPRVTCSLCRREVCRRCTKAYNGEMLCRICSHDIFYRAVKCPWFYDSSPDKLCGHASDTIVKSLFKDNLPVLNGLTEEQLHKALSNYLKPDRETYVNGVVITLEVMLRIENEQIRTFTKRYQEVRSTAVNALNKVRGTAMKNGQEDSEKEVDRITSQFLQQTREALRKLLQMLCITEEKYKLSVHNSRQVNTATRVLEITQKQVEQLVGHKIQIPDEYTNVEVEEDLESFVASVASGPEEFFANVLLEDLTKAMRNSISHGRNSRAPLSSDQQGKDVAIAWLSMLYMLVYIPMIFPATWLLNHYGLRVSILIGATLNMLGAWIKCLSMELSQPTSSPASNASFPLLMVAQLVCALGQVFLLGVPAQLAATWFSKSELAMATAIGVGCALGFGLPPLMVPAVTSSTGVADFEDFRKGFRIMFYGGAAIMSFDLLLVAIFFKEEPEIAPSRAQYKRILQRRAQLAGDKPDEADFAPAFDHESSVLTAEESVLVNQSYFHQVAHSFKCVSFIYLNICYGVNTGVYYEIGTLLNSIVAEFFPTAQGARMELPNLERCVKKSPSGVKQINVVKDGRIEAISTGDQFCTSPGSEDVEGSDELSLNLSSETEQVAIGWIGFSMVIAGLVGSIVAGVVLKKTGLYRLVLIIFYFLSVISMGAFMGSLYSNLIGTVFLTMILLGFFQSGFLPLGFEYAAEITYPIDEGLTSGILNTSAHIFGIILTQVATAMIGNYGALPTNIFILVCMVIAGVPACFMKDDLKRQRAHEKMRVEVEANSGDEFKYSSISKTKDV", + "length": 943, + "molWeight": 104642, + "crc64": "EBEC2FBABAA7A2A2", + "md5": "52E715C9787B9CD778FA5533664DC417", + }, + "organism": { + "taxonId": 6210, + "commonName": "Hydatid tapeworm", + "scientificName": "Echinococcus granulosus", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "EUB57593.1", "source": "EMBL"}, + {"evidenceCode": "ECO:0000313", "id": "UP000019149", "source": "Proteomes"}, + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Spiralia", + "Lophotrochozoa", + "Platyhelminthes", + "Rhabditophora", + "Neodermata", + "Cestoda", + "Eucestoda", + "Cyclophyllidea", + "Taeniidae", + "Echinococcus", + "Echinococcus granulosus group", + ], + }, + "proteinExistence": "4: Predicted", + "uniProtkbId": "W6U8K5_ECHGR", + "primaryAccession": "W6U8K5", + "references": [ + { + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "1168", + "lastPage": "1175", + "volume": "45", + "citationCrossReferences": [ + {"database": "PubMed", "id": "24013640"}, + {"database": "DOI", "id": "10.1038/ng.2757"}, + ], + "authors": [ + "Zheng H.", + "Zhang W.", + "Zhang L.", + "Zhang Z.", + "Li J.", + "Lu G.", + "Zhu Y.", + "Wang Y.", + "Huang Y.", + "Liu J.", + "Kang H.", + "Chen J.", + "Wang L.", + "Chen A.", + "Yu S.", + "Gao Z.", + "Jin L.", + "Gu W.", + "Wang Z.", + "Zhao L.", + "Shi B.", + "Wen H.", + "Lin R.", + "Jones M.K.", + "Brejova B.", + "Vinar T.", + "Zhao G.", + "McManus D.P.", + "Chen Z.", + "Zhou Y.", + "Wang S.", + ], + "publicationDate": "2013", + "title": "The genome of the hydatid tapeworm Echinococcus granulosus.", + "id": "24013640", + "journal": "Nat. Genet.", + "citationType": "journal article", + }, + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "EUB57593.1", "source": "EMBL"}, + {"evidenceCode": "ECO:0000313", "id": "UP000019149", "source": "Proteomes"}, + ], + } + ], + "features": [ + { + "location": { + "start": {"value": 435, "modifier": "EXACT"}, + "end": {"value": 455, "modifier": "EXACT"}, + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "location": { + "start": {"value": 461, "modifier": "EXACT"}, + "end": {"value": 480, "modifier": "EXACT"}, + }, + "type": "Transmembrane", + "description": "Helical", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + ], + "extraAttributes": { + "countByCommentType": {"SUBCELLULAR LOCATION": 1, "CAUTION": 1}, + "countByFeatureType": {"Transmembrane": 10, "Domain": 1, "Region": 1, "Compositional bias": 1}, + "uniParcId": "UPI0003E7E985", + }, + }, + { + "entryAudit": { + "sequenceVersion": 2, + "firstPublicDate": dt.date(2014, 4, 16), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2018, 12, 5), + "entryVersion": 58, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "value": "FLVCR choline and putative heme transporter 2", + "evidences": [ + { + "evidenceCode": "ECO:0000313", + "id": "ENSAMXP00000014911.2", + "source": "Ensembl", + } + ], + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "AlphaFoldDB", + "properties": [{"key": "Description", "value": "-"}], + "id": "W5L4Z9", + }, + { + "database": "Ensembl", + "properties": [ + {"key": "ProteinId", "value": "ENSAMXP00000014911.2"}, + {"key": "GeneId", "value": "ENSAMXG00000014488.2"}, + ], + "id": "ENSAMXT00000014911.2", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004651", "source": "ARBA"} + ], + }, + "location": { + "id": "SL-0039", + "value": "Cell membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004651", "source": "ARBA"} + ], + }, + }, + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004477", "source": "ARBA"} + ], + }, + "location": { + "id": "SL-0097", + "value": "Endoplasmic reticulum membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004477", "source": "ARBA"} + ], + }, + }, + { + "topology": { + "id": "SL-9909", + "value": "Multi-pass membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004225", "source": "ARBA"} + ], + }, + "location": { + "id": "SL-0171", + "value": "Mitochondrion membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004225", "source": "ARBA"} + ], + }, + }, + ], + }, + { + "commentType": "CATALYTIC ACTIVITY", + "reaction": { + "reactionCrossReferences": [ + {"database": "Rhea", "id": "RHEA:32751"}, + {"database": "ChEBI", "id": "CHEBI:15354"}, + ], + "name": "choline(out) = choline(in)", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00036811", "source": "ARBA"}], + }, + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Choline uniporter that specifically mediates choline uptake at the blood-brain-barrier. Responsible for the majority of choline uptake across the blood-brain-barrier from the circulation into the brain. Choline, a nutrient critical for brain development, is a precursor of phosphatidylcholine, as well as betaine. Also mediates transport of ethanolamine. Choline and ethanolamine transport is not coupled with proton transport and is exclusively driven by the choline gradient across the plasma membrane. Also acts as a heme b transporter", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00053231", "source": "ARBA"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Cellular component", + "name": "Cell membrane", + "id": "KW-1003", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022475", "source": "ARBA"}], + }, + { + "category": "Cellular component", + "name": "Endoplasmic reticulum", + "id": "KW-0256", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022824", "source": "ARBA"}], + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023136", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Cellular component", + "name": "Mitochondrion", + "id": "KW-0496", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00023128", "source": "ARBA"}], + }, + { + "category": "Technical term", + "name": "Reference proteome", + "id": "KW-1185", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "UP000018467", "source": "Proteomes"}], + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022692", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022989", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Biological process", + "name": "Transport", + "id": "KW-0813", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00022448", "source": "ARBA"}], + }, + ], + "sequence": { + "value": "MSACENSEDLKGQTNGPALVKSNISTSEPAAEQIECKVFSPEAPQECTTRLYKRRWAIVFLFSSYSLCNAFQWIQYGIINNIFMHFYSVSSFAIDWMSMIYMLTYIPFIFPVTWLLDKKGLRVVALVASSINCLGTWIKVASVKPHLFWVTMLGQFACSFAQVFILGMPSRIASVWFGSGEVSTACSIGVFGNQLGIAIGFLVPPILVPNVEDLDKLAHHISIMFYITAGVATLIFFLVIIVFQEKPEIPPSQAQAAMRNMSPQEYSYLASIKRLIHNKSFILLIITYGLNVGSFYAVSTLLNRMIIDHYPGEEVNAGRIGLTIVIAGMVGSLICGIWLDKSKTYKQTTLAVYFLTLVGMLIYAFTLDLGHLWVVFLTAGALGFFMTGYLPLGFEFAVELTYPESEGTSSGLLNCSAQVFGIIFTICQGKIIDYYGTLAGNIFLCIFLLIGTIMTGFIKADLRRQRANQVAETNAKKPGCQEEGLSSPLKEDKV", + "length": 494, + "molWeight": 54676, + "crc64": "9D5DBF17CAD5C77E", + "md5": "E646D99157273186F09C65F70AD4B392", + }, + "organism": { + "taxonId": 7994, + "commonName": "Blind cave fish", + "synonyms": ["Astyanax fasciatus mexicanus"], + "scientificName": "Astyanax mexicanus", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "ENSAMXP00000014911.2", "source": "Ensembl"}, + {"evidenceCode": "ECO:0000313", "id": "UP000018467", "source": "Proteomes"}, + ], + "lineage": [ + "Eukaryota", + "Metazoa", + "Chordata", + "Craniata", + "Vertebrata", + "Euteleostomi", + "Actinopterygii", + "Neopterygii", + "Teleostei", + "Ostariophysi", + "Characiformes", + "Characoidei", + "Acestrorhamphidae", + "Acestrorhamphinae", + "Astyanax", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W5L4Z9_ASTMX", + "primaryAccession": "W5L4Z9", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "female", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "UP000018467", "source": "Proteomes"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "5307", + "lastPage": "5307", + "volume": "5", + "citationCrossReferences": [ + {"database": "PubMed", "id": "25329095"}, + {"database": "DOI", "id": "10.1038/ncomms6307"}, + ], + "authors": [ + "McGaugh S.E.", + "Gross J.B.", + "Aken B.", + "Blin M.", + "Borowsky R.", + "Chalopin D.", + "Hinaux H.", + "Jeffery W.R.", + "Keene A.", + "Ma L.", + "Minx P.", + "Murphy D.", + "O'Quin K.E.", + "Retaux S.", + "Rohner N.", + "Searle S.M.", + "Stahl B.A.", + "Tabin C.", + "Volff J.N.", + "Yoshizawa M.", + "Warren W.C.", + ], + "publicationDate": "2014", + "title": "The cavefish genome reveals candidate genes for eye loss.", + "id": "25329095", + "journal": "Nat. Commun.", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "UP000018467", "source": "Proteomes"}], + }, + { + "referenceNumber": 2, + "referencePositions": ["IDENTIFICATION"], + "citation": { + "publicationDate": "JAN-2026", + "id": "CI-20IM145UGQVF7", + "authoringGroup": ["Ensembl"], + "submissionDatabase": "UniProtKB", + "citationType": "submission", + }, + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "ENSAMXP00000014911.2", "source": "Ensembl"} + ], + }, + ], + "features": [ + { + "location": { + "start": {"value": 58, "modifier": "EXACT"}, + "end": {"value": 463, "modifier": "EXACT"}, + }, + "type": "Domain", + "description": "Major facilitator superfamily (MFS) profile", + "evidences": [{"evidenceCode": "ECO:0000259", "id": "PS50850", "source": "PROSITE"}], + }, + { + "location": { + "start": {"value": 473, "modifier": "EXACT"}, + "end": {"value": 494, "modifier": "EXACT"}, + }, + "type": "Region", + "description": "Disordered", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MobiDB-lite", "source": "SAM"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "CATALYTIC ACTIVITY": 3, + "SUBCELLULAR LOCATION": 1, + "SIMILARITY": 1, + }, + "countByFeatureType": {"Domain": 1, "Region": 1}, + "uniParcId": "UPI000BBDE7D1", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 5, 14), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2014, 5, 14), + "entryVersion": 37, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 4.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "NCVP5", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + } + }, + { + "fullName": { + "value": "NS28", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + } + }, + ], + "recommendedName": { + "shortNames": [ + { + "value": "NSP4", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + } + ], + "fullName": { + "value": "Non-structural glycoprotein 4", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + }, + }, + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHL44730.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_RNA"}, + ], + "id": "KJ187597", + }, + { + "database": "GO", + "properties": [ + {"key": "GoTerm", "value": "C:extracellular region"}, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-SubCell"}, + ], + "id": "GO:0005576", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "NSP4 localizes also in vesicular structures which contain autophagosomal markers and associate with viroplasms in virus-infected cells. Additionally, a soluble form of glycosylated NSP4 is secreted despite retention of its transmembrane domain", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + } + ] + }, + "subcellularLocations": [ + { + "topology": { + "id": "SL-9907", + "value": "Single-pass type III membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + }, + "location": { + "id": "SL-0423", + "value": "Host rough endoplasmic reticulum membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + }, + }, + { + "topology": { + "id": "SL-9907", + "value": "Single-pass type III membrane protein", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + }, + "location": { + "id": "SL-0428", + "value": "Host membrane, host caveola", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + }, + }, + { + "location": { + "id": "SL-0243", + "value": "Secreted", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + } + }, + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Plays an essential role in the virus replication cycle by acting as a viroporin. Creates a pore in the host endoplasmic reticulum and as a consequence releases Ca(2+) in the cytoplasm of infected cell. In turn, high levels of cytoplasmic calcium trigger membrane trafficking and transport of viral ER-associated proteins to viroplasms, sites of viral genome replication and immature particle assembly", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Biological process", + "name": "Activation of host autophagy by virus", + "id": "KW-1072", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023050", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Calcium", + "id": "KW-0106", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022837", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Coiled coil", + "id": "KW-0175", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Coils", "source": "SAM"}], + }, + { + "category": "Molecular function", + "name": "Enterotoxin", + "id": "KW-0260", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022861", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023180", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host endoplasmic reticulum", + "id": "KW-1038", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023184", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host membrane", + "id": "KW-1043", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022870", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Host-virus interaction", + "id": "KW-0945", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022581", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Molecular function", + "name": "Ion channel", + "id": "KW-0407", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023303", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Ion transport", + "id": "KW-0406", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023065", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023136", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022723", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Secreted", + "id": "KW-0964", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022525", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Signal-anchor", + "id": "KW-0735", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022968", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Molecular function", + "name": "Toxin", + "id": "KW-0800", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022656", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022692", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022989", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Transport", + "id": "KW-0813", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022448", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Molecular function", + "name": "Viral ion channel", + "id": "KW-1182", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023039", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Virulence", + "id": "KW-0843", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023026", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}, + ], + }, + ], + "sequence": { + "value": "MEKLTDLNYTLSVITLMNDTLHTIMEDPGMAYFPYIASVLTVLFTLHKASIPTMKIALRTSRCSYKVIKYCIVSIFNTLLKLAGYKEQITTKDEIEKQMDRVVKEMRRQLEMIDKLTTREIEQVELLKRIHDMLVIKPVDKVDMSQEFNQIQFKTLTEWVEGENPYEPKEVTASL", + "length": 175, + "molWeight": 20457, + "crc64": "C336CE8F0EE54E0C", + "md5": "5FEED6986EFE388FAC3C63734DF453FD", + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44730.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8Q983_9REOV", + "primaryAccession": "W8Q983", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHL44730.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [{"database": "PubMed", "id": "24818762"}], + "authors": ["Jeong S.", "Than V.T.", "Lim I.", "Kim W."], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44730.1", "source": "EMBL"}], + } + ], + "features": [ + { + "ligand": {"name": "Ca(2+)", "id": "ChEBI:CHEBI:29108"}, + "featureCrossReferences": [{"database": "ChEBI", "id": "CHEBI:29108"}], + "location": { + "start": {"value": 120, "modifier": "EXACT"}, + "end": {"value": 120, "modifier": "EXACT"}, + }, + "type": "Binding site", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}], + }, + { + "featureId": "", + "location": { + "start": {"value": 8, "modifier": "EXACT"}, + "end": {"value": 8, "modifier": "EXACT"}, + }, + "type": "Glycosylation", + "description": "N-linked (GlcNAc...) asparagine; by host", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04091", "source": "HAMAP-Rule"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "DOMAIN": 1, + "PTM": 1, + "SIMILARITY": 1, + }, + "countByFeatureType": { + "Topological domain": 2, + "Transmembrane": 1, + "Coiled coil": 1, + "Binding site": 2, + "Glycosylation": 2, + }, + "uniParcId": "UPI00042F2EA0", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 5, 14), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2014, 5, 14), + "entryVersion": 31, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "recommendedName": { + "fullName": { + "value": "Intermediate capsid protein VP6", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"} + ], + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHL44737.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_RNA"}, + ], + "id": "KJ187603", + }, + { + "database": "GO", + "properties": [ + {"key": "GoTerm", "value": "C:T=13 icosahedral viral capsid"}, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-ARBA"}, + ], + "id": "GO:0039621", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Component of the intermediate capsid. Also found in spherical cytoplasmic structures, called virus factories, that appear early after infection and are the site of viral replication and packaging", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"} + ], + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"} + ], + } + } + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Intermediate capsid protein that self assembles to form an icosahedral capsid with a T=13 symmetry, which consists of 230 trimers of VP6, with channels at each of its five-fold vertices. This capsid constitutes the middle concentric layer of the viral mature particle. The innermost VP2 capsid and the intermediate VP6 capsid remain intact following cell entry to protect the dsRNA from degradation and to prevent unfavorable antiviral responses in the host cell during all the replication cycle of the virus. Nascent transcripts are transcribed within the structural confines of this double-layered particle (DLP) and are extruded through the channels at the five-fold axes. VP6 is required for the transcription activity of the DLP", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00057631", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Ligand", + "name": "Calcium", + "id": "KW-0106", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022837", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Capsid protein", + "id": "KW-0167", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022561", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Intermediate capsid protein", + "id": "KW-1154", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022493", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022723", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "PTM", + "name": "Ubl conjugation", + "id": "KW-0832", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022843", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022844", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Zinc", + "id": "KW-0862", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022833", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}, + ], + }, + ], + "sequence": { + "value": "MDVLYSLSKTLKDARDKIVEGTLYSNVSDLIQQFNQMIITMNGNEFQTGGIGNLPVRNWNFDFGLLGTTLLNLDANYVETARNTIDYFVDFVDNVCMDEMVRESQRNGIAPQSDSLRKLSGIKFKRINFDNSSEYIENWNLQNRRQRTGFTFHKPNIFPYSASFTLNRSQPAHDNLMGTMWLNAGSEIQVAGFDYSCAINAPANIQQFEHIVQLRRVLTTATITLLPDAERFSFPRVINSADGATTWYFNPVILRPNNVEVEFLLNGQIINTYQARFGTIIARNFDTIRLSFQLMRPPNMTPAVAALFPNAQPFEHHATVGLTLRIESAVCESVLADASETMLANVTSVRQEYAIPVGPVFPPGMNWTDLITNYSPSREDNLQRVFTVASIRSMLIK", + "length": 397, + "molWeight": 44897, + "crc64": "1AC9A23DA62FBEC0", + "md5": "74A8486A6D109C92A854B58EFBEB03A4", + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44737.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8R3F3_9REOV", + "primaryAccession": "W8R3F3", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHL44737.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [{"database": "PubMed", "id": "24818762"}], + "authors": ["Jeong S.", "Than V.T.", "Lim I.", "Kim W."], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44737.1", "source": "EMBL"}], + } + ], + "features": [ + { + "ligand": { + "note": "ligand shared between all trimeric partners", + "name": "Zn(2+)", + "id": "ChEBI:CHEBI:29105", + }, + "featureCrossReferences": [{"database": "ChEBI", "id": "CHEBI:29105"}], + "location": { + "start": {"value": 153, "modifier": "EXACT"}, + "end": {"value": 153, "modifier": "EXACT"}, + }, + "type": "Binding site", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}], + }, + { + "location": { + "start": {"value": 62, "modifier": "EXACT"}, + "end": {"value": 73, "modifier": "EXACT"}, + }, + "type": "Region", + "description": "Interaction with the inner capsid protein VP2", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04129", "source": "HAMAP-Rule"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "MISCELLANEOUS": 1, + "SIMILARITY": 1, + }, + "countByFeatureType": {"Region": 1, "Binding site": 3}, + "uniParcId": "UPI00042F181F", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 5, 14), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2014, 5, 14), + "entryVersion": 50, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "recommendedName": { + "fullName": { + "value": "Outer capsid glycoprotein VP7", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "RU363022", "source": "RuleBase"}, + ], + } + } + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHL44738.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_RNA"}, + ], + "id": "KJ187604", + }, + { + "database": "GO", + "properties": [ + {"key": "GoTerm", "value": "C:host cell endoplasmic reticulum lumen"}, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-SubCell"}, + ], + "id": "GO:0044166", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "The outer layer contains 780 copies of VP7, grouped as 260 trimers. Immature double-layered particles assembled in the cytoplasm bud across the membrane of the endoplasmic reticulum, acquiring during this process a transient lipid membrane that is modified with the ER resident viral glycoproteins NSP4 and VP7; these enveloped particles also contain VP4. As the particles move towards the interior of the ER cisternae, the transient lipid membrane and the non-structural protein NSP4 are lost, while the virus surface proteins VP4 and VP7 rearrange to form the outermost virus protein layer, yielding mature infectious triple-layered particles", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"} + ], + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0389", + "value": "Host endoplasmic reticulum lumen", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "RU363022", "source": "RuleBase"}, + ], + } + }, + { + "location": { + "id": "SL-0274", + "value": "Virion", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"} + ], + } + }, + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Calcium-binding protein that interacts with rotavirus cell receptors once the initial attachment by VP4 has been achieved. Rotavirus attachment and entry into the host cell probably involves multiple sequential contacts between the outer capsid proteins VP4 and VP7, and the cell receptors. Following entry into the host cell, low intracellular or intravesicular Ca(2+) concentration probably causes the calcium-stabilized VP7 trimers to dissociate from the virion. This step is probably necessary for the membrane-disrupting entry step and the release of VP4, which is locked onto the virion by VP7", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00059356", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + } + ], + }, + ], + "keywords": [ + { + "category": "Ligand", + "name": "Calcium", + "id": "KW-0106", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022837", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Capsid protein", + "id": "KW-0167", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022561", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "PTM", + "name": "Disulfide bond", + "id": "KW-1015", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023157", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023180", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host endoplasmic reticulum", + "id": "KW-1038", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023184", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Biological process", + "name": "Host-virus interaction", + "id": "KW-0945", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022581", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022723", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Outer capsid protein", + "id": "KW-1152", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022770", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Signal", + "id": "KW-0732", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022729", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04131", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "T=13 icosahedral capsid protein", + "id": "KW-1146", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022708", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "category": "Cellular component", + "name": "Virion", + "id": "KW-0946", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022844", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04130", "source": "HAMAP-Rule"}, + ], + }, + ], + "sequence": { + "value": "MYGIEYTTVLTFLISVILLNYALKSLTRIMDFIIYRFLLIVVVLSPFLNAQNYGINLPITGSIDTPYTNSTQEEMFLTSTLCLYYPTEAATEINDNSWKDTLSQLFLTKGWPTGSVYFKDYADIASFSIDPQLYCDYNLVLMKYDAALQLDMSELADLILNEWLCNPMDITLYYYQQTDEANKWISMGSSCTIKVCPLNVQTLGIGCLTTDTSTFEEIATAEKLVITDVVDGVNHKLNVTTNTCTIRNCKKLGPRENVAVIQVGGSNVLDITADPTTAPQTERMMRVNWKKWWQVFYTIVDYVNQIVQAMSKRSRSLNSAAFYYRV", + "length": 326, + "molWeight": 37073, + "crc64": "BE3B85417F66327D", + "md5": "D5F596F60A5D582F6C2D226E70983AB9", + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44738.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8QKB9_9REOV", + "primaryAccession": "W8QKB9", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHL44738.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [{"database": "PubMed", "id": "24818762"}], + "authors": ["Jeong S.", "Than V.T.", "Lim I.", "Kim W."], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44738.1", "source": "EMBL"}], + } + ], + "features": [ + { + "ligand": {"name": "Ca(2+)", "id": "ChEBI:CHEBI:29108", "label": "1"}, + "featureCrossReferences": [{"database": "ChEBI", "id": "CHEBI:29108"}], + "location": { + "start": {"value": 95, "modifier": "EXACT"}, + "end": {"value": 95, "modifier": "EXACT"}, + }, + "type": "Binding site", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04131", "source": "HAMAP-Rule"}], + }, + { + "featureId": "PRO_5023428870", + "location": { + "start": {"value": 51, "modifier": "EXACT"}, + "end": {"value": 326, "modifier": "EXACT"}, + }, + "type": "Chain", + "description": "Outer capsid glycoprotein VP7", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04131", "source": "HAMAP-Rule"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 2, + "SUBUNIT": 2, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "MISCELLANEOUS": 2, + "SIMILARITY": 1, + "CAUTION": 1, + }, + "countByFeatureType": { + "Chain": 1, + "Transmembrane": 2, + "Region": 2, + "Binding site": 9, + "Disulfide bond": 4, + }, + "uniParcId": "UPI00042E6A3F", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2014, 5, 14), + "lastAnnotationUpdateDate": dt.date(2026, 6, 10), + "lastSequenceUpdateDate": dt.date(2014, 5, 14), + "entryVersion": 33, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 3.0, + "proteinDescription": { + "alternativeNames": [ + { + "fullName": { + "value": "NS26", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"} + ], + } + } + ], + "recommendedName": { + "shortNames": [ + { + "value": "NSP5", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR004006", "source": "PIRNR"}, + ], + } + ], + "fullName": { + "value": "Non-structural protein 5", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR004006", "source": "PIRNR"}, + ], + }, + }, + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "AHL44731.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "Genomic_RNA"}, + ], + "id": "KJ187598", + }, + { + "database": "GO", + "properties": [ + {"key": "GoTerm", "value": "C:host cell cytoplasm"}, + {"key": "GoEvidenceType", "value": "IEA:UniProtKB-SubCell"}, + ], + "id": "GO:0030430", + }, + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "note": { + "texts": [ + { + "value": "Found in spherical cytoplasmic structures, called virus factories, that appear early after infection and are the site of viral replication and packaging", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR004006", "source": "PIRNR"}, + ], + } + ] + }, + "subcellularLocations": [ + { + "location": { + "id": "SL-0381", + "value": "Host cytoplasm", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + {"evidenceCode": "ECO:0000256", "id": "PIRNR004006", "source": "PIRNR"}, + ], + } + } + ], + }, + { + "commentType": "COFACTOR", + "cofactors": [ + { + "cofactorCrossReference": {"database": "ChEBI", "id": "CHEBI:18420"}, + "name": "Mg(2+)", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"} + ], + } + ], + }, + { + "commentType": "FUNCTION", + "texts": [ + { + "value": "Plays an essential role in the viral genome replication. Participates, together with NSP2, in the formation of viral factories (viroplasms) which are large inclusions in the host cytoplasm where replication intermediates are assembled and viral RNA replication takes place. Orchestrates the recruitment of viroplasmic proteins such as capsid proteins to these factories. Participates in the selective exclusion of host proteins from stress granules (SG) and P bodies (PB). Participates also in the sequestration of these remodeled organelles in viral factories", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023180", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Cellular component", + "name": "Host cytoplasm", + "id": "KW-1035", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023200", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Magnesium", + "id": "KW-0460", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022842", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "Ligand", + "name": "Metal-binding", + "id": "KW-0479", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}], + }, + { + "category": "Ligand", + "name": "Nucleotide-binding", + "id": "KW-0547", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022741", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + ], + }, + { + "category": "PTM", + "name": "Phosphoprotein", + "id": "KW-0597", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}], + }, + { + "category": "Molecular function", + "name": "RNA-binding", + "id": "KW-0694", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022884", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}, + ], + }, + ], + "sequence": { + "value": "MSLSIDVTSLPSISSSIYKNESSSTTSTLSGKSIGRSEQYISPDAEAFGKYMLSKSPEDIGPSDSASNDPLTSFSIRSNAVKTNADASVSMDSSTQSRPSSNVGCDQVDFSFNKGIKMNTNLDSSISIATFSKKEKSKNDHKSRKHYPKVEAESDSDDYVLDDSDSDDGKCKNCKYKRKYFALRVRMKQVAMQLIEDL", + "length": 198, + "molWeight": 21726, + "crc64": "27C0104400E9CE41", + "md5": "F11A74C9F145ED70EB6CD8BCAE6D3989", + }, + "organism": { + "taxonId": 10941, + "scientificName": "Human rotavirus A", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44731.1", "source": "EMBL"}], + "lineage": [ + "Viruses", + "Riboviria", + "Orthornavirae", + "Duplornaviricota", + "Resentoviricetes", + "Reovirales", + "Sedoreoviridae", + "Rotavirus", + "Rotavirus alphagastroenteritidis", + "Rotavirus A", + ], + }, + "proteinExistence": "3: Inferred from homology", + "uniProtkbId": "W8Q8A1_9REOV", + "primaryAccession": "W8Q8A1", + "references": [ + { + "referenceComments": [ + { + "type": "STRAIN", + "value": "CAU12-2-51", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "AHL44731.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + "citation": { + "firstPage": "E97127", + "lastPage": "E97127", + "volume": "9", + "citationCrossReferences": [{"database": "PubMed", "id": "24818762"}], + "authors": ["Jeong S.", "Than V.T.", "Lim I.", "Kim W."], + "publicationDate": "2014", + "title": "Whole-Genome Analysis of a Rare Human Korean G3P Rotavirus Strain Suggests a Complex Evolutionary Origin Potentially Involving Reassortment Events between Feline and Bovine Rotaviruses.", + "id": "24818762", + "journal": "PLoS ONE", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "AHL44731.1", "source": "EMBL"}], + } + ], + "features": [ + { + "ligand": {"name": "Mg(2+)", "id": "ChEBI:CHEBI:18420"}, + "featureCrossReferences": [{"database": "ChEBI", "id": "CHEBI:18420"}], + "location": { + "start": {"value": 92, "modifier": "EXACT"}, + "end": {"value": 92, "modifier": "EXACT"}, + }, + "type": "Binding site", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MF_04092", "source": "HAMAP-Rule"}], + }, + { + "location": { + "start": {"value": 16, "modifier": "EXACT"}, + "end": {"value": 37, "modifier": "EXACT"}, + }, + "type": "Region", + "description": "Disordered", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "MobiDB-lite", "source": "SAM"}], + }, + ], + "extraAttributes": { + "countByCommentType": { + "FUNCTION": 1, + "COFACTOR": 1, + "SUBUNIT": 1, + "SUBCELLULAR LOCATION": 1, + "PTM": 2, + "SIMILARITY": 1, + }, + "countByFeatureType": { + "Region": 4, + "Compositional bias": 2, + "Binding site": 1, + "Modified residue": 5, + }, + "uniParcId": "UPI00042E2A3D", + }, + }, + { + "entryAudit": { + "sequenceVersion": 1, + "firstPublicDate": dt.date(2018, 10, 10), + "lastAnnotationUpdateDate": dt.date(2026, 9, 2), + "lastSequenceUpdateDate": dt.date(2018, 10, 10), + "entryVersion": 27, + }, + "entryType": "UniProtKB unreviewed (TrEMBL)", + "annotationScore": 2.0, + "proteinDescription": { + "submissionNames": [ + { + "fullName": { + "value": "Signaling lymphocyte activation molecule", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "BAN42597.1", "source": "EMBL"} + ], + } + } + ] + }, + "uniProtKBCrossReferences": [ + { + "database": "EMBL", + "properties": [ + {"key": "ProteinId", "value": "BAN42597.1"}, + {"key": "Status", "value": "-"}, + {"key": "MoleculeType", "value": "mRNA"}, + ], + "id": "AB771742", + }, + { + "database": "RefSeq", + "properties": [{"key": "NucleotideSequenceId", "value": "NM_001278826.1"}], + "id": "NP_001265755.1", + }, + ], + "genes": [ + { + "geneName": { + "value": "SLAM", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "BAN42597.1", "source": "EMBL"}], + } + } + ], + "comments": [ + { + "commentType": "SUBCELLULAR LOCATION", + "subcellularLocations": [ + { + "location": { + "id": "SL-0162", + "value": "Membrane", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00004370", "source": "ARBA"} + ], + } + } + ], + }, + { + "commentType": "CAUTION", + "texts": [ + { + "value": "Lacks conserved residue(s) required for the propagation of feature annotation", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PRU00114", "source": "PROSITE-ProRule"} + ], + } + ], + }, + ], + "keywords": [ + { + "category": "PTM", + "name": "Glycoprotein", + "id": "KW-0325", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "ARBA00023180", "source": "ARBA"}], + }, + { + "category": "Domain", + "name": "Immunoglobulin domain", + "id": "KW-0393", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "PRU00114", "source": "PROSITE-ProRule"} + ], + }, + { + "category": "Cellular component", + "name": "Membrane", + "id": "KW-0472", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00023136", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}, + ], + }, + { + "category": "Domain", + "name": "Signal", + "id": "KW-0732", + "evidences": [ + {"evidenceCode": "ECO:0000256", "id": "ARBA00022729", "source": "ARBA"}, + {"evidenceCode": "ECO:0000256", "id": "SignalP", "source": "SAM"}, + ], + }, + { + "category": "Domain", + "name": "Transmembrane", + "id": "KW-0812", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + { + "category": "Domain", + "name": "Transmembrane helix", + "id": "KW-1133", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "Phobius", "source": "SAM"}], + }, + ], + "sequence": { + "value": "MDLKGLLSLNFLLFLSLAFELSCGTGESLMTCQEVTEQLGRNLLLPLASEGISKSMNKSIHILVTKSESPQKNVKTKIVSLSLPERASPRFLENGYKFHPENLKLEILQSKKENEGWYFTTLEENFSVRHFCLQLKLFEQVSTPEIKVLNWTQENGNCSLTLACEVEKGDHVAYSWREETGADPPISANSSHLHLSLGPQHVHNVYVCTVSNPISNHSQTFTPGSVCMPDPPELRPWGLYVGLSSGAVIGVILTLKVAVLLLRRRGKANHYQPTTEAKSLTIYAQVQKSGSIQKKPDAPSAQDPCTTIYVAATEPVPEPVQEPKSITVYASVTLPES", + "length": 337, + "molWeight": 37201, + "crc64": "F24FDE8F6BDA985B", + "md5": "246583275688D762B2F35467EAB3FE5C", + }, + "organism": { + "taxonId": 9685, + "commonName": "Cat", + "synonyms": ["Felis silvestris catus"], + "scientificName": "Felis catus", + "evidences": [{"evidenceCode": "ECO:0000313", "id": "BAN42597.1", "source": "EMBL"}], + "lineage": [ + "Eukaryota", + "Metazoa", + "Chordata", + "Craniata", + "Vertebrata", + "Euteleostomi", + "Mammalia", + "Eutheria", + "Laurasiatheria", + "Carnivora", + "Feliformia", + "Felidae", + "Felinae", + "Felis", + ], + }, + "proteinExistence": "2: Evidence at transcript level", + "uniProtkbId": "W6JD38_FELCA", + "primaryAccession": "W6JD38", + "references": [ + { + "referenceComments": [ + { + "type": "TISSUE", + "value": "Thymus", + "evidences": [ + {"evidenceCode": "ECO:0000313", "id": "BAN42597.1", "source": "EMBL"} + ], + } + ], + "referenceNumber": 1, + "referencePositions": ["NUCLEOTIDE SEQUENCE"], + "citation": { + "firstPage": "1085", + "lastPage": "1089", + "volume": "75", + "citationCrossReferences": [ + {"database": "PubMed", "id": "23535266"}, + {"database": "DOI", "id": "10.1292/jvms.13-0003"}, + ], + "authors": [ + "Hara Y.", + "Suzuki J.", + "Noguchi K.", + "Terada Y.", + "Shimoda H.", + "Mizuno T.", + "Maeda K.", + ], + "publicationDate": "2013", + "title": "Function of feline signaling lymphocyte activation molecule as a receptor of canine distemper virus.", + "id": "23535266", + "journal": "J. Vet. Med. Sci.", + "citationType": "journal article", + }, + "evidences": [{"evidenceCode": "ECO:0000313", "id": "BAN42597.1", "source": "EMBL"}], + } + ], + "features": [ + { + "featureId": "PRO_5016357255", + "location": { + "start": {"value": 19, "modifier": "EXACT"}, + "end": {"value": 337, "modifier": "EXACT"}, + }, + "type": "Chain", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "SignalP", "source": "SAM"}], + }, + { + "location": { + "start": {"value": 1, "modifier": "EXACT"}, + "end": {"value": 18, "modifier": "EXACT"}, + }, + "type": "Signal", + "description": "", + "evidences": [{"evidenceCode": "ECO:0000256", "id": "SignalP", "source": "SAM"}], + }, + ], + "extraAttributes": { + "countByCommentType": {"SUBCELLULAR LOCATION": 1, "CAUTION": 1}, + "countByFeatureType": {"Signal": 1, "Chain": 1, "Transmembrane": 1, "Domain": 1}, + "uniParcId": "UPI000298A0A1", + }, + }, + ] + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/uniprot_search/minimal.json b/tests/data/jsonschema/examples/uniprot_search/minimal.json new file mode 100644 index 00000000..bf5c180e --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/minimal.json @@ -0,0 +1,10 @@ +[ + { + "results": [ + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "X5IFX5" + } + ] + } +] diff --git a/tests/data/jsonschema/examples/uniprot_search/minimal_expected.json b/tests/data/jsonschema/examples/uniprot_search/minimal_expected.json new file mode 100644 index 00000000..bf5c180e --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/minimal_expected.json @@ -0,0 +1,10 @@ +[ + { + "results": [ + { + "entryType": "UniProtKB unreviewed (TrEMBL)", + "primaryAccession": "X5IFX5" + } + ] + } +] diff --git a/tests/data/jsonschema/examples/uniprot_search/minimal_expected_table.py b/tests/data/jsonschema/examples/uniprot_search/minimal_expected_table.py new file mode 100644 index 00000000..694d37f7 --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/minimal_expected_table.py @@ -0,0 +1,13 @@ +"""Expected PyArrow table for minimal real-world example uniprot_search.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + {"results": [{"entryType": "UniProtKB unreviewed (TrEMBL)", "primaryAccession": "X5IFX5"}]}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/examples/uniprot_search/schema.json b/tests/data/jsonschema/examples/uniprot_search/schema.json new file mode 100644 index 00000000..e26972c6 --- /dev/null +++ b/tests/data/jsonschema/examples/uniprot_search/schema.json @@ -0,0 +1,2384 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jsonschema:searchresult.json", + "title": "SearchResult", + "$ref": "#/$defs/SearchResult", + "$defs": { + "SearchResult": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "$ref": "#/$defs/UniProtKBEntry" + } + }, + "facets": { + "type": "array", + "items": { + "$ref": "#/$defs/Facet" + } + }, + "matchedFields": { + "type": "array", + "items": { + "$ref": "#/$defs/TermInfo" + } + }, + "suggestions": { + "type": "array", + "items": { + "$ref": "#/$defs/Suggestion" + } + }, + "warnings": { + "type": "array", + "items": { + "$ref": "#/$defs/ProblemPair" + } + } + } + }, + "UniProtKBEntry": { + "type": "object", + "properties": { + "entryAudit": { + "$ref": "#/$defs/EntryAudit" + }, + "organismHosts": { + "type": "array", + "items": { + "$ref": "#/$defs/OrganismHost" + } + }, + "entryType": { + "type": "string", + "enum": [ + "UniProtKB reviewed (Swiss-Prot)", + "UniProtKB unreviewed (TrEMBL)", + "Inactive", + "AA", + "UNKNOWN" + ] + }, + "annotationScore": { + "type": "number", + "format": "double" + }, + "secondaryAccessions": { + "type": "array", + "items": { + "type": "string" + } + }, + "proteinDescription": { + "$ref": "#/$defs/ProteinDescription" + }, + "uniProtKBCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/UniProtKBCrossReference" + } + }, + "genes": { + "type": "array", + "items": { + "$ref": "#/$defs/Gene" + } + }, + "comments": { + "type": "array", + "items": { + "$ref": "#/$defs/Comment" + } + }, + "geneLocations": { + "type": "array", + "items": { + "$ref": "#/$defs/GeneLocation" + } + }, + "keywords": { + "type": "array", + "items": { + "$ref": "#/$defs/Keyword" + } + }, + "sequence": { + "$ref": "#/$defs/Sequence" + }, + "organism": { + "$ref": "#/$defs/Organism" + }, + "lineages": { + "type": "array", + "items": { + "$ref": "#/$defs/TaxonomyLineage" + } + }, + "proteinExistence": { + "type": "string", + "enum": [ + "1: Evidence at protein level", + "2: Evidence at transcript level", + "3: Inferred from homology", + "4: Predicted", + "5: Uncertain", + "UNKNOWN" + ] + }, + "uniProtkbId": { + "type": "string" + }, + "primaryAccession": { + "type": "string" + }, + "inactiveReason": { + "$ref": "#/$defs/EntryInactiveReason" + }, + "references": { + "type": "array", + "items": { + "$ref": "#/$defs/UniProtKBReference" + } + }, + "internalSection": { + "$ref": "#/$defs/InternalSection" + }, + "fragment": { + "type": "boolean" + }, + "features": { + "type": "array", + "items": { + "$ref": "#/$defs/UniProtKBFeature" + } + }, + "active": { + "type": "boolean" + }, + "extraAttributes": { + "type": "object", + "properties": { + "countByCommentType": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "int32" + } + }, + "countByFeatureType": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "int32" + } + }, + "uniParcId": { + "type": "string" + } + } + } + } + }, + "EntryAudit": { + "type": "object", + "properties": { + "sequenceVersion": { + "type": "integer", + "format": "int32" + }, + "firstPublicDate": { + "type": "string", + "format": "date" + }, + "lastAnnotationUpdateDate": { + "type": "string", + "format": "date" + }, + "lastSequenceUpdateDate": { + "type": "string", + "format": "date" + }, + "entryVersion": { + "type": "integer", + "format": "int32" + } + } + }, + "OrganismHost": { + "type": "object", + "properties": { + "taxonId": { + "type": "integer", + "format": "int64" + }, + "commonName": { + "type": "string" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + } + }, + "scientificName": { + "type": "string" + } + } + }, + "ProteinDescription": { + "type": "object", + "properties": { + "alternativeNames": { + "type": "array", + "items": { + "$ref": "#/$defs/ProteinName" + } + }, + "recommendedName": { + "$ref": "#/$defs/ProteinName" + }, + "submissionNames": { + "type": "array", + "items": { + "$ref": "#/$defs/ProteinSubName" + } + }, + "includes": { + "type": "array", + "items": { + "$ref": "#/$defs/ProteinSection" + } + }, + "allergenName": { + "$ref": "#/$defs/Name" + }, + "biotechName": { + "$ref": "#/$defs/Name" + }, + "cdAntigenNames": { + "type": "array", + "items": { + "$ref": "#/$defs/Name" + } + }, + "innNames": { + "type": "array", + "items": { + "$ref": "#/$defs/Name" + } + }, + "contains": { + "type": "array", + "items": { + "$ref": "#/$defs/ProteinSection" + } + }, + "flag": { + "type": "string" + } + } + }, + "ProteinName": { + "type": "object", + "properties": { + "shortNames": { + "type": "array", + "items": { + "$ref": "#/$defs/Name" + } + }, + "ecNumbers": { + "type": "array", + "items": { + "$ref": "#/$defs/EC" + } + }, + "valid": { + "type": "boolean" + }, + "fullName": { + "$ref": "#/$defs/Name" + } + } + }, + "Name": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidence": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Evidence": { + "type": "object", + "properties": { + "evidenceCode": { + "type": "string" + }, + "id": { + "type": "string" + }, + "source": { + "type": "string" + } + } + }, + "EC": { + "type": "object", + "properties": { + "valid": { + "type": "boolean" + }, + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "ProteinSubName": { + "type": "object", + "properties": { + "ecNumbers": { + "type": "array", + "items": { + "$ref": "#/$defs/EC" + } + }, + "valid": { + "type": "boolean" + }, + "fullName": { + "$ref": "#/$defs/Name" + } + } + }, + "ProteinSection": { + "type": "object", + "properties": { + "alternativeNames": { + "type": "array", + "items": { + "$ref": "#/$defs/ProteinName" + } + }, + "recommendedName": { + "$ref": "#/$defs/ProteinName" + }, + "allergenName": { + "$ref": "#/$defs/Name" + }, + "biotechName": { + "$ref": "#/$defs/Name" + }, + "cdAntigenNames": { + "type": "array", + "items": { + "$ref": "#/$defs/Name" + } + }, + "innNames": { + "type": "array", + "items": { + "$ref": "#/$defs/Name" + } + } + } + }, + "UniProtKBCrossReference": { + "type": "object", + "properties": { + "isoformId": { + "type": "string" + }, + "database": { + "type": "string" + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/Property" + } + }, + "id": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Property": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "Gene": { + "type": "object", + "properties": { + "orderedLocusNames": { + "type": "array", + "items": { + "$ref": "#/$defs/OrderedLocusName" + } + }, + "geneName": { + "$ref": "#/$defs/GeneName" + }, + "orfNames": { + "type": "array", + "items": { + "$ref": "#/$defs/ORFName" + } + }, + "synonyms": { + "type": "array", + "items": { + "$ref": "#/$defs/GeneNameSynonym" + } + } + } + }, + "OrderedLocusName": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "GeneName": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "ORFName": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "GeneNameSynonym": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Comment": { + "type": "object", + "properties": { + "commentType": { + "type": "string", + "enum": [ + "FUNCTION", + "CATALYTIC ACTIVITY", + "COFACTOR", + "ACTIVITY REGULATION", + "BIOPHYSICOCHEMICAL PROPERTIES", + "PATHWAY", + "SUBUNIT", + "INTERACTION", + "SUBCELLULAR LOCATION", + "ALTERNATIVE PRODUCTS", + "TISSUE SPECIFICITY", + "DEVELOPMENTAL STAGE", + "INDUCTION", + "DOMAIN", + "PTM", + "RNA EDITING", + "MASS SPECTROMETRY", + "POLYMORPHISM", + "DISEASE", + "DISRUPTION PHENOTYPE", + "ALLERGEN", + "TOXIC DOSE", + "BIOTECHNOLOGY", + "PHARMACEUTICAL", + "MISCELLANEOUS", + "SIMILARITY", + "CAUTION", + "SEQUENCE CAUTION", + "WEB RESOURCE", + "UNKOWN" + ] + } + }, + "anyOf": [ + { + "$ref": "#/$defs/AlternativeProductsComment" + }, + { + "$ref": "#/$defs/BPCPComment" + }, + { + "$ref": "#/$defs/CatalyticActivityComment" + }, + { + "$ref": "#/$defs/CofactorComment" + }, + { + "$ref": "#/$defs/DiseaseComment" + }, + { + "$ref": "#/$defs/FreeTextComment" + }, + { + "$ref": "#/$defs/InteractionComment" + }, + { + "$ref": "#/$defs/MassSpectrometryComment" + }, + { + "$ref": "#/$defs/RnaEditingComment" + }, + { + "$ref": "#/$defs/SequenceCautionComment" + }, + { + "$ref": "#/$defs/SubcellularLocationComment" + }, + { + "$ref": "#/$defs/WebResourceComment" + } + ] + }, + "AlternativeProductsComment": { + "type": "object", + "properties": { + "isoforms": { + "type": "array", + "items": { + "$ref": "#/$defs/APIsoform" + } + }, + "events": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "Alternative promoter usage", + "Alternative splicing", + "Alternative initiation", + "Ribosomal frameshifting" + ] + } + }, + "note": { + "$ref": "#/$defs/Note" + } + } + }, + "APIsoform": { + "type": "object", + "properties": { + "isoformIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "sequenceIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "isoformSequenceStatus": { + "type": "string", + "enum": [ + "Displayed", + "External", + "Not described", + "Described" + ] + }, + "note": { + "$ref": "#/$defs/Note" + }, + "synonyms": { + "type": "array", + "items": { + "$ref": "#/$defs/IsoformName" + } + }, + "name": { + "$ref": "#/$defs/IsoformName" + } + } + }, + "Note": { + "type": "object", + "properties": { + "valid": { + "type": "boolean" + }, + "texts": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidencedValue" + } + } + } + }, + "EvidencedValue": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "IsoformName": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "BPCPComment": { + "type": "object", + "properties": { + "kineticParameters": { + "$ref": "#/$defs/KineticParameters" + }, + "absorption": { + "$ref": "#/$defs/Absorption" + }, + "phDependence": { + "$ref": "#/$defs/PhDependence" + }, + "redoxPotential": { + "$ref": "#/$defs/RedoxPotential" + }, + "temperatureDependence": { + "$ref": "#/$defs/TemperatureDependence" + }, + "molecule": { + "type": "string" + } + } + }, + "KineticParameters": { + "type": "object", + "properties": { + "maximumVelocities": { + "type": "array", + "items": { + "$ref": "#/$defs/MaximumVelocity" + } + }, + "michaelisConstants": { + "type": "array", + "items": { + "$ref": "#/$defs/MichaelisConstant" + } + }, + "note": { + "$ref": "#/$defs/Note" + } + } + }, + "MaximumVelocity": { + "type": "object", + "properties": { + "unit": { + "type": "string" + }, + "enzyme": { + "type": "string" + }, + "velocity": { + "type": "number", + "format": "double" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "MichaelisConstant": { + "type": "object", + "properties": { + "constant": { + "type": "number", + "format": "double" + }, + "unit": { + "type": "string", + "enum": [ + "M", + "mM", + "uM", + "nM", + "mg/mL", + "mg/ml" + ] + }, + "substrate": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Absorption": { + "type": "object", + "properties": { + "approximate": { + "type": "boolean" + }, + "note": { + "$ref": "#/$defs/Note" + }, + "max": { + "type": "integer", + "format": "int32" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "PhDependence": { + "type": "object", + "properties": { + "texts": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidencedValue" + } + } + } + }, + "RedoxPotential": { + "type": "object", + "properties": { + "texts": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidencedValue" + } + } + } + }, + "TemperatureDependence": { + "type": "object", + "properties": { + "texts": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidencedValue" + } + } + } + }, + "CatalyticActivityComment": { + "type": "object", + "properties": { + "physiologicalReactions": { + "type": "array", + "items": { + "$ref": "#/$defs/PhysiologicalReaction" + } + }, + "reaction": { + "$ref": "#/$defs/Reaction" + }, + "molecule": { + "type": "string" + } + } + }, + "PhysiologicalReaction": { + "type": "object", + "properties": { + "reactionCrossReference": { + "$ref": "#/$defs/CrossReferenceReactionDatabase" + }, + "directionType": { + "type": "string", + "enum": [ + "left-to-right", + "right-to-left" + ] + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "CrossReferenceReactionDatabase": { + "type": "object", + "properties": { + "database": { + "type": "string", + "enum": [ + "ChEBI", + "Rhea" + ] + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/Property" + } + }, + "id": { + "type": "string" + } + } + }, + "Reaction": { + "type": "object", + "properties": { + "reactionCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceReactionDatabase" + } + }, + "ecNumber": { + "type": "string" + }, + "name": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "CofactorComment": { + "type": "object", + "properties": { + "cofactors": { + "type": "array", + "items": { + "$ref": "#/$defs/Cofactor" + } + }, + "note": { + "$ref": "#/$defs/Note" + }, + "valid": { + "type": "boolean" + }, + "molecule": { + "type": "string" + } + } + }, + "Cofactor": { + "type": "object", + "properties": { + "cofactorCrossReference": { + "$ref": "#/$defs/CrossReferenceCofactorDatabase" + }, + "name": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "CrossReferenceCofactorDatabase": { + "type": "object", + "properties": { + "database": { + "type": "string", + "enum": [ + "ChEBI", + "" + ] + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/Property" + } + }, + "id": { + "type": "string" + } + } + }, + "DiseaseComment": { + "type": "object", + "properties": { + "disease": { + "$ref": "#/$defs/Disease" + }, + "note": { + "$ref": "#/$defs/Note" + }, + "molecule": { + "type": "string" + } + } + }, + "Disease": { + "type": "object", + "properties": { + "diseaseAccession": { + "type": "string" + }, + "diseaseCrossReference": { + "$ref": "#/$defs/CrossReferenceDiseaseDatabase" + }, + "diseaseId": { + "type": "string" + }, + "acronym": { + "type": "string" + }, + "description": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "CrossReferenceDiseaseDatabase": { + "type": "object", + "properties": { + "database": { + "type": "string", + "enum": [ + "MIM", + "" + ] + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/Property" + } + }, + "id": { + "type": "string" + } + } + }, + "FreeTextComment": { + "type": "object", + "properties": { + "texts": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidencedValue" + } + }, + "molecule": { + "type": "string" + } + } + }, + "InteractionComment": { + "type": "object", + "properties": { + "interactions": { + "type": "array", + "items": { + "$ref": "#/$defs/Interaction" + } + } + } + }, + "Interaction": { + "type": "object", + "properties": { + "numberOfExperiments": { + "type": "integer", + "format": "int32" + }, + "interactantOne": { + "$ref": "#/$defs/Interactant" + }, + "organismDiffer": { + "type": "boolean" + }, + "interactantTwo": { + "$ref": "#/$defs/Interactant" + } + } + }, + "Interactant": { + "type": "object", + "properties": { + "geneName": { + "type": "string" + }, + "chainId": { + "type": "string" + }, + "intActId": { + "type": "string" + }, + "uniProtKBAccession": { + "type": "string" + } + } + }, + "MassSpectrometryComment": { + "type": "object", + "properties": { + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + }, + "molWeight": { + "type": "number", + "format": "float" + }, + "molWeightError": { + "type": "number", + "format": "float" + }, + "note": { + "type": "string" + }, + "method": { + "type": "string", + "enum": [ + "Electrospray", + "FAB", + "LSI", + "MALDI", + "Plasma desorption", + "SELDI", + "API", + "Unknown" + ] + }, + "molecule": { + "type": "string" + } + } + }, + "RnaEditingComment": { + "type": "object", + "properties": { + "positions": { + "type": "array", + "items": { + "$ref": "#/$defs/RnaEdPosition" + } + }, + "locationType": { + "type": "string", + "enum": [ + "Not_applicable", + "Undetermined", + "Unknown", + "Known" + ] + }, + "note": { + "$ref": "#/$defs/Note" + }, + "molecule": { + "type": "string" + } + } + }, + "RnaEdPosition": { + "type": "object", + "properties": { + "position": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "SequenceCautionComment": { + "type": "object", + "properties": { + "sequenceCautionType": { + "type": "string", + "enum": [ + "Frameshift", + "Erroneous initiation", + "Erroneous termination", + "Erroneous gene model prediction", + "Erroneous translation", + "Miscellaneous discrepancy", + "unknown" + ] + }, + "note": { + "type": "string" + }, + "sequence": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + }, + "molecule": { + "type": "string" + } + } + }, + "SubcellularLocationComment": { + "type": "object", + "properties": { + "subcellularLocations": { + "type": "array", + "items": { + "$ref": "#/$defs/SubcellularLocation" + } + }, + "note": { + "$ref": "#/$defs/Note" + }, + "molecule": { + "type": "string" + } + } + }, + "SubcellularLocation": { + "type": "object", + "properties": { + "topology": { + "$ref": "#/$defs/SubcellularLocationValue" + }, + "orientation": { + "$ref": "#/$defs/SubcellularLocationValue" + }, + "location": { + "$ref": "#/$defs/SubcellularLocationValue" + } + } + }, + "SubcellularLocationValue": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "WebResourceComment": { + "type": "object", + "properties": { + "resourceUrl": { + "type": "string" + }, + "ftp": { + "type": "boolean" + }, + "note": { + "type": "string" + }, + "resourceName": { + "type": "string" + }, + "molecule": { + "type": "string" + } + } + }, + "GeneLocation": { + "type": "object", + "properties": { + "geneEncodingType": { + "type": "string", + "enum": [ + "unknown", + "Hydrogenosome", + "Mitochondrion", + "Nucleomorph", + "Plasmid", + "Plastid", + "Apicoplast", + "Chloroplast", + "Cyanelle", + "Non-photosynthetic plastid", + "Organellar chromatophore" + ] + }, + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Keyword": { + "type": "object", + "properties": { + "category": { + "type": "string", + "enum": [ + "Biological process", + "Cellular component", + "Coding sequence diversity", + "Developmental stage", + "Disease", + "Domain", + "Ligand", + "Molecular function", + "PTM", + "Technical term", + "Unknown" + ] + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Sequence": { + "type": "object", + "properties": { + "type": { + "type": "string", + "xml": { + "attribute": true + } + }, + "ref": { + "type": "string", + "xml": { + "attribute": true + } + }, + "value": { + "type": "string" + }, + "length": { + "type": "integer", + "format": "int32" + }, + "molWeight": { + "type": "integer", + "format": "int32" + }, + "crc64": { + "type": "string" + }, + "md5": { + "type": "string" + } + } + }, + "Organism": { + "type": "object", + "properties": { + "lineages": { + "type": "array", + "items": { + "type": "string" + } + }, + "taxonId": { + "type": "integer", + "format": "int64" + }, + "commonName": { + "type": "string" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + } + }, + "scientificName": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + }, + "lineage": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "TaxonomyLineage": { + "type": "object", + "properties": { + "rank": { + "type": "string", + "enum": [ + "strain", + "serotype", + "series", + "morph", + "biotype", + "clade", + "genotype", + "isolate", + "forma specialis", + "pathogroup", + "serogroup", + "section", + "subsection", + "forma", + "varietas", + "subspecies", + "species", + "species subgroup", + "species group", + "subgenus", + "genus", + "subtribe", + "tribe", + "subfamily", + "family", + "superfamily", + "parvorder", + "infraorder", + "suborder", + "order", + "superorder", + "subcohort", + "cohort", + "infraclass", + "subclass", + "class", + "superclass", + "subphylum", + "phylum", + "superphylum", + "subkingdom", + "kingdom", + "superkingdom", + "realm", + "domain", + "subvariety", + "acellular root", + "cellular root", + "no rank" + ] + }, + "taxonId": { + "type": "integer", + "format": "int64" + }, + "hidden": { + "type": "boolean" + }, + "commonName": { + "type": "string" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + } + }, + "scientificName": { + "type": "string" + } + } + }, + "EntryInactiveReason": { + "type": "object", + "properties": { + "deletedReason": { + "type": "string", + "enum": [ + "", + "Undefined", + "Deleted from sequence source (EMBL)", + "Deleted from sequence source (TAIR)", + "Deleted from sequence source (SGD)", + "Deleted from sequence source (ENSEMBL)", + "Deleted from sequence source (PDB)", + "Deleted from sequence source (RefSeq)", + "Deleted from Swiss-Prot", + "Redundant sequence", + "Not part of a reference proteome", + "Excluded proteome", + "Over-represented sequence", + "Not part of a reference proteome" + ] + }, + "inactiveReasonType": { + "type": "string", + "enum": [ + "DELETED", + "MERGED", + "DEMERGED" + ] + }, + "mergeDemergeTos": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "UniProtKBReference": { + "type": "object", + "properties": { + "referenceComments": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferenceComment" + } + }, + "referenceNumber": { + "type": "integer", + "format": "int32" + }, + "referencePositions": { + "type": "array", + "items": { + "type": "string" + } + }, + "citation": { + "$ref": "#/$defs/Citation" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "ReferenceComment": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "STRAIN", + "PLASMID", + "TRANSPOSON", + "TISSUE" + ] + }, + "value": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "Citation": { + "type": "object", + "properties": { + "citationType": { + "type": "string", + "enum": [ + "journal article", + "book", + "online journal article", + "patent", + "submission", + "thesis", + "UniProt indexed literatures", + "unpublished observations" + ] + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/Book" + }, + { + "$ref": "#/$defs/ElectronicArticle" + }, + { + "$ref": "#/$defs/JournalArticle" + }, + { + "$ref": "#/$defs/Literature" + }, + { + "$ref": "#/$defs/Patent" + }, + { + "$ref": "#/$defs/Submission" + }, + { + "$ref": "#/$defs/Thesis" + } + ] + }, + "CrossReferenceCitationDatabase": { + "type": "object", + "properties": { + "database": { + "type": "string", + "enum": [ + "PubMed", + "DOI", + "AGRICOLA" + ] + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/Property" + } + }, + "id": { + "type": "string" + } + } + }, + "Book": { + "type": "object", + "properties": { + "firstPage": { + "type": "string" + }, + "lastPage": { + "type": "string" + }, + "volume": { + "type": "string" + }, + "publisher": { + "type": "string" + }, + "bookName": { + "type": "string" + }, + "editors": { + "type": "array", + "items": { + "type": "string" + } + }, + "address": { + "type": "string" + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ElectronicArticle": { + "type": "object", + "properties": { + "journal": { + "type": "string" + }, + "locator": { + "type": "string" + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "JournalArticle": { + "type": "object", + "properties": { + "journal": { + "type": "string" + }, + "firstPage": { + "type": "string" + }, + "lastPage": { + "type": "string" + }, + "volume": { + "type": "string" + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Literature": { + "type": "object", + "properties": { + "literatureAbstract": { + "type": "string" + }, + "completeAuthorList": { + "type": "boolean" + }, + "pubmedId": { + "type": "integer", + "format": "int64" + }, + "doiId": { + "type": "string" + }, + "journal": { + "type": "string" + }, + "firstPage": { + "type": "string" + }, + "lastPage": { + "type": "string" + }, + "volume": { + "type": "string" + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + } + } + }, + "Patent": { + "type": "object", + "properties": { + "patentNumber": { + "type": "string" + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Submission": { + "type": "object", + "properties": { + "submissionDatabase": { + "type": "string", + "enum": [ + "PDB data bank", + "PIR data bank", + "Swiss-Prot", + "UniProtKB", + "EMBL/GenBank/DDBJ databases" + ] + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Thesis": { + "type": "object", + "properties": { + "institute": { + "type": "string" + }, + "address": { + "type": "string" + }, + "citationCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceCitationDatabase" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "publicationDate": { + "type": "string" + }, + "authoringGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "title": { + "type": "string" + }, + "id": { + "type": "string" + }, + "authoringGroup": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "InternalSection": { + "type": "object", + "properties": { + "internalLines": { + "type": "array", + "items": { + "$ref": "#/$defs/InternalLine" + } + }, + "sourceLines": { + "type": "array", + "items": { + "$ref": "#/$defs/SourceLine" + } + }, + "evidenceLines": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidenceLine" + } + } + } + }, + "InternalLine": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "CL", + "CP", + "CX", + "DR", + "DG", + "GO", + "EV", + "HA", + "HR", + "HW", + "HU", + "HP", + "ID", + "IS", + "NI", + "PM", + "SO", + "YY", + "ZA", + "ZB", + "ZC", + "ZR", + "ZZ", + "ET", + "PE", + "RU", + "TX", + "PROSITE", + "UP", + "ZD" + ] + }, + "value": { + "type": "string" + } + } + }, + "SourceLine": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + } + }, + "EvidenceLine": { + "type": "object", + "properties": { + "evidence": { + "type": "string" + }, + "createDate": { + "type": "string", + "format": "date" + }, + "curator": { + "type": "string" + } + } + }, + "UniProtKBFeature": { + "type": "object", + "properties": { + "alternativeSequence": { + "$ref": "#/$defs/AlternativeSequence" + }, + "featureId": { + "type": "string" + }, + "ligand": { + "$ref": "#/$defs/Ligand" + }, + "ligandPart": { + "$ref": "#/$defs/LigandPart" + }, + "featureCrossReferences": { + "type": "array", + "items": { + "$ref": "#/$defs/CrossReferenceUniprotKBFeatureDatabase" + } + }, + "location": { + "$ref": "#/$defs/FeatureLocation" + }, + "type": { + "type": "string", + "enum": [ + "Initiator methionine", + "Signal", + "Propeptide", + "Transit peptide", + "Chain", + "Peptide", + "Topological domain", + "Transmembrane", + "Intramembrane", + "Domain", + "Repeat", + "Calcium binding", + "Zinc finger", + "DNA binding", + "Nucleotide binding", + "Region", + "Coiled coil", + "Motif", + "Compositional bias", + "Active site", + "Metal binding", + "Binding site", + "Site", + "Non-standard residue", + "Modified residue", + "Lipidation", + "Glycosylation", + "Disulfide bond", + "Cross-link", + "Alternative sequence", + "Natural variant", + "Mutagenesis", + "Sequence uncertainty", + "Sequence conflict", + "Non-adjacent residues", + "Non-terminal residue", + "Helix", + "Turn", + "Beta strand" + ] + }, + "description": { + "type": "string" + }, + "evidences": { + "type": "array", + "items": { + "$ref": "#/$defs/Evidence" + } + } + } + }, + "AlternativeSequence": { + "type": "object", + "properties": { + "originalSequence": { + "type": "string" + }, + "alternativeSequences": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Ligand": { + "type": "object", + "properties": { + "note": { + "type": "string" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "LigandPart": { + "type": "object", + "properties": { + "note": { + "type": "string" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "CrossReferenceUniprotKBFeatureDatabase": { + "type": "object", + "properties": { + "database": { + "type": "string", + "enum": [ + "dbSNP", + "ChEBI" + ] + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/Property" + } + }, + "id": { + "type": "string" + } + } + }, + "FeatureLocation": { + "type": "object", + "properties": { + "start": { + "$ref": "#/$defs/Position" + }, + "end": { + "$ref": "#/$defs/Position" + }, + "sequence": { + "type": "string" + } + } + }, + "Position": { + "type": "object", + "properties": { + "value": { + "type": "integer", + "format": "int32" + }, + "modifier": { + "type": "string", + "enum": [ + "EXACT", + "OUTSIDE", + "UNKNOWN", + "UNSURE" + ] + } + } + }, + "Facet": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "name": { + "type": "string" + }, + "allowMultipleSelection": { + "type": "boolean" + }, + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/FacetItem" + } + } + } + }, + "FacetItem": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "value": { + "type": "string" + }, + "count": { + "type": "integer", + "format": "int64" + } + } + }, + "TermInfo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "hits": { + "type": "integer", + "format": "int64" + } + } + }, + "Suggestion": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "hits": { + "type": "integer", + "format": "int64" + } + } + }, + "ProblemPair": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } +} diff --git a/tests/data/jsonschema/rules.yaml b/tests/data/jsonschema/rules.yaml new file mode 100644 index 00000000..6fb1720f --- /dev/null +++ b/tests/data/jsonschema/rules.yaml @@ -0,0 +1,175 @@ +# Rule catalogue for the JSON Schema -> PyArrow converter +# (cdm_data_loaders.converters.jsonschema_to_pyarrow: generate_pyarrow_schema / coerce_record_for_pyarrow). +# +# This file and the per-case `case.yaml` manifests are the single source of truth for +# docs/jsonschema_to_pyarrow_specification.md, generated by scripts/gen_converter_spec.py. +# Every rule must be exercised by at least one case (enforced by test_jsonschema_to_pyarrow_spec.py). +title: JSON Schema (Draft 2020-12) to PyArrow Specification +intro: | + This document specifies how a **JSON Schema Draft 2020-12** document determines: + + 1. The **PyArrow `pa.Schema`** produced by `generate_pyarrow_schema`. + 2. The **Python row dictionaries** produced by `coerce_record_for_pyarrow` from schema-valid JSON records, + ready for `pa.Table.from_pylist(rows, schema=pa_schema)` and Parquet serialization. + 3. The **errors** raised for constructs that cannot be represented as a static columnar schema. + +sections: + - id: resolution + title: Top-Level Resolution and Unwrapping + rules: + - id: R-DIALECT + statement: | + The root must declare `"$schema": "https://json-schema.org/draft/2020-12/schema"` or the KBase + XSV metaschema `"https://json-schema.kbase.us/draft/2020-12-xsv/schema"`; anything else raises **E1**. + - id: R-ROOT-REF + statement: | + A root consisting of `"$ref": "#/$defs/"` (without its own `properties`) is replaced by the + referenced definition before classification. That definition counts as one expansion for recursion. + - id: R-UNWRAP + statement: | + The resolved root must be a fixed-property object producing at least one field. Its fields become + the top-level columns. A scalar, array or pure-map root raises **E2**; an object with no fields raises + **E3**. + + - id: naming + title: Naming and Field Ordering + rules: + - id: R-NAME-PROPERTY + statement: | + Property keys are used verbatim as field names. + - id: R-ORDER + statement: | + Struct fields are emitted in this order: properties from each `allOf` subschema in list order, then + direct `properties` in declaration order, then properties contributed by `anyOf`, `oneOf`, + `dependentSchemas`, `then` and `else`. A property seen more than once keeps the position of its + first occurrence. + - id: R-MERGE-DUPLICATE + statement: | + When a property is declared more than once (across `allOf`, `anyOf`/`oneOf`, `dependentSchemas` or + `then`/`else`), identical declarations collapse to one; differing declarations are combined as an + `anyOf` of the declarations and resolved with the union rules ([R-UNION-SCALAR](#r-union-scalar), + [R-UNION-OBJECT](#r-union-object)). The first declared `description` is kept. + + - id: shape + title: Construct Classification (Shape) + rules: + - id: R-SHAPE-OBJECT + statement: | + An object with `properties` (or object combinators) maps to `pa.struct([...])`. Boolean + `additionalProperties` (`true`, `false` or omitted) on such an object is ignored. + - id: R-SHAPE-ARRAY + statement: | + `type: "array"` with `items: ` maps to `pa.list_(pa.field("item", ))`; nested + arrays nest lists. + - id: R-SHAPE-TUPLE + statement: | + A closed tuple (`prefixItems: [S_0, ..., S_k-1]` with `items` `false` or omitted) maps to + `pa.struct` with fields `item_0` ... `item_k-1`. Any other `items` value raises **E5**. + - id: R-SHAPE-MAP + statement: | + An object with schema-valued `additionalProperties` and/or `patternProperties` and no fixed + properties maps to `pa.map_(pa.string(), pa.field("value", ))`. Several value schemas are + combined with the union rules. Mixing fixed properties with a map schema raises **E6**. + - id: R-SHAPE-REF + statement: | + `"$ref": "#/$defs/"` is replaced by the definition; sibling keywords on the referencing node + are overlaid on the definition. Only local `#/$defs/` pointers are supported (**E9**). + - id: R-RECURSION + statement: | + Each `$defs` target may be expanded at most `max_recursion_depth` times (default `2`) along a path. + A deeper reference becomes `pa.string()` (or `pa.list_(pa.string())` inside an array), carries no + `jsonschema_type`, and its runtime value is serialized with `json.dumps`. Applies to self-recursion + and mutual recursion. + + - id: nullability + title: Field Nullability + rules: + - id: R-NULL-REQUIRED + statement: | + A property field is `nullable=False` only when it is in the enclosing object's effective `required` + set and its schema does not allow `null`. The effective `required` set is the union of `required` + across the object and its `allOf` subschemas, plus the intersection of `required` across + `anyOf`/`oneOf` object branches. + - id: R-NULL-ALLOWS-NULL + statement: | + A schema allows `null` when `"null"` is in `type`, `null` is in `enum` or is the `const`, or any + `anyOf`/`oneOf`/`allOf` branch (or `$ref` target) allows `null`. Such fields are `nullable=True` even + when required. + - id: R-NULL-CONDITIONAL + statement: | + Properties introduced only by `dependentSchemas`, `then` or `else` are `nullable=True`; + `dependentRequired` does not affect nullability. + - id: R-NULL-DEFAULT + statement: | + `default` is an annotation only: it neither changes nullability nor fills in missing values. + - id: R-NULL-ITEMS + statement: | + A list `item` field and a map `value` field are `nullable=False` unless the item/value schema allows + `null`. A tuple field `item_i` is `nullable=False` only when `minItems >= i + 1` and `prefixItems[i]` + does not allow `null`. + + - id: scalars + title: Scalar Types and Unions + generated: scalar_table + rules: + - id: R-SCALAR-TYPE + statement: | + Scalar `type`, `format` and `contentEncoding` map to Arrow types per the table below. Numeric + validation keywords (`minimum`, `maximum`, `exclusiveMinimum`, `exclusiveMaximum`, `multipleOf`) do + not change the type. Only `format: "date"` becomes a temporal type; other formats stay `pa.string()`. + - id: R-SCALAR-ENUM-CONST + statement: | + With an explicit `type`, `enum`/`const` use that type. Without one, each non-null literal's type is + inferred (`bool` -> boolean, `int` -> integer, `float` -> number, `str` -> string) and combined with + the union rules. + - id: R-UNION-SCALAR + statement: | + For `type: [...]` and scalar-only `anyOf`/`oneOf`, `null` members are dropped (making the field + nullable). If the rest share one Arrow type it is used; if they are only `integer` and `number` the + type is `pa.float64()`; otherwise `pa.string()`. `jsonschema_type` joins the distinct member labels + with `" | "` in declaration order. `object` or `array` in a `type` list raises **E8**. + - id: R-UNION-OBJECT + statement: | + `anyOf`/`oneOf` whose non-null branches are all objects merge their properties into one struct. + Branches of other structural shapes (array, map) raise **E8**. + - id: R-UNION-SCALAR-OBJECT + statement: | + `anyOf`/`oneOf` mixing object and scalar branches yields a struct of the merged object properties, + all `nullable=True`, followed by a synthesized `_value` field holding the unified scalar type. A + declared `_value` property raises **E12**. + + - id: metadata + title: Field Metadata + rules: + - id: R-META-JSONSCHEMA-TYPE + statement: | + Every scalar field (including scalar list `item`, map `value` and tuple `item_i` fields) carries + `metadata["jsonschema_type"]`. Struct, list, map and recursion-truncated fields do not. + - id: R-META-COMMENT + statement: | + A non-empty `description` is stored in `metadata["comment"]`. If the node has none, it is inherited + from its `$ref` target, or from the only non-null `$ref` branch of an `anyOf`/`oneOf`. + + - id: coercion + title: Record Coercion + rules: + - id: R-COERCE-SCALAR + statement: | + Strings for `pa.date32()` are parsed with `datetime.date.fromisoformat`; `pa.binary()` strings are + decoded as Base16 (`bytes.fromhex`) when `jsonschema_type` mentions `base16`, otherwise Base64. Ints + become floats for float fields, and integral floats become ints for integer fields. + - id: R-COERCE-STRING-FALLBACK + statement: | + Non-string values written to a `pa.string()` field are stringified: booleans, objects and arrays with + `json.dumps` (`true`/`false`), numbers with `str()`. This covers heterogeneous scalar unions and + recursion-truncated subtrees. + - id: R-COERCE-TUPLE + statement: | + A JSON array for a closed tuple becomes `{"item_0": v_0, ...}`; positions that are missing or `null` + are omitted. + - id: R-COERCE-VALUE + statement: | + A scalar written to a scalar + object union struct becomes `{"_value": }`. + - id: R-COERCE-SPARSE + statement: | + Only keys declared in the schema are kept; absent keys stay absent (they become null in the table). diff --git a/tests/data/jsonschema/spec/01_scalars_and_formats/case.yaml b/tests/data/jsonschema/spec/01_scalars_and_formats/case.yaml new file mode 100644 index 00000000..54560fd8 --- /dev/null +++ b/tests/data/jsonschema/spec/01_scalars_and_formats/case.yaml @@ -0,0 +1,11 @@ +title: Scalars, Formats, Binary Encodings, Enums and Scalar Unions +schema: schema.json +rules: [R-DIALECT, R-UNWRAP, R-NAME-PROPERTY, R-SCALAR-TYPE, R-SCALAR-ENUM-CONST, R-UNION-SCALAR, R-META-JSONSCHEMA-TYPE, R-META-COMMENT, R-NULL-REQUIRED, R-COERCE-SCALAR, R-COERCE-STRING-FALLBACK, R-COERCE-SPARSE] +notes: | + - `untyped_mixed_enum` (`enum: [1, "two", true]`) falls back to `pa.string()`; the value `1` is coerced to `"1"`. + - `numeric_union` (`["integer", "number"]`) promotes to `pa.float64()`; `scalar_union` (`["integer", "string"]`) falls back to `pa.string()`. + - `date_or_year_union` mixes `string (date)` (`pa.date32()`) and `string`, so it falls back to `pa.string()`. + - The second record omits every optional key; they stay absent in the coerced row. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/01_scalars_and_formats/data.json b/tests/data/jsonschema/spec/01_scalars_and_formats/data.json new file mode 100644 index 00000000..1cc8c1f3 --- /dev/null +++ b/tests/data/jsonschema/spec/01_scalars_and_formats/data.json @@ -0,0 +1,56 @@ +[ + { + "id": "rec1", + "status": "active", + "version": 2, + "flag": true, + "bounded_int": -42, + "big_int": 999999999999, + "float_val": 2.718281828, + "decimal_multiple": 3.14, + "created_date": "2025-06-15", + "updated_at": "2025-06-15T12:30:00Z", + "event_time": "12:30:00Z", + "elapsed": "P3DT4H", + "record_uuid": "550e8400-e29b-41d4-a716-446655440000", + "homepage": "https://example.org/rec1", + "const_tag": "v1", + "untyped_int_enum": 20, + "untyped_mixed_enum": 1, + "numeric_union": 42, + "scalar_union": 100, + "date_or_year_union": "2025-06-15", + "raw_b64": "SGVsbG8=", + "raw_hex": "48656c6c6f" + }, + { + "id": "rec2", + "status": "archived", + "version": 1, + "flag": false + }, + { + "id": "rec3", + "status": "pending", + "version": 32767, + "flag": true, + "bounded_int": 999, + "big_int": -999999999999, + "float_val": -0.5, + "decimal_multiple": 42.0, + "created_date": "1999-12-31", + "updated_at": "2025-06-15T12:30:00.123+02:00", + "event_time": "23:59:59+02:00", + "elapsed": "PT15M", + "record_uuid": "123e4567-e89b-12d3-a456-426614174000", + "homepage": "https://example.org/rec3", + "const_tag": "v1", + "untyped_int_enum": 30, + "untyped_mixed_enum": true, + "numeric_union": 3.5, + "scalar_union": "hundred", + "date_or_year_union": "1999", + "raw_b64": "V29ybGQ=", + "raw_hex": "576f726c64" + } +] diff --git a/tests/data/jsonschema/spec/01_scalars_and_formats/expected.json b/tests/data/jsonschema/spec/01_scalars_and_formats/expected.json new file mode 100644 index 00000000..6ccc637c --- /dev/null +++ b/tests/data/jsonschema/spec/01_scalars_and_formats/expected.json @@ -0,0 +1,56 @@ +[ + { + "id": "rec1", + "status": "active", + "version": 2, + "flag": true, + "bounded_int": -42, + "big_int": 999999999999, + "float_val": 2.718281828, + "decimal_multiple": 3.14, + "created_date": "2025-06-15", + "updated_at": "2025-06-15T12:30:00Z", + "event_time": "12:30:00Z", + "elapsed": "P3DT4H", + "record_uuid": "550e8400-e29b-41d4-a716-446655440000", + "homepage": "https://example.org/rec1", + "const_tag": "v1", + "untyped_int_enum": 20, + "untyped_mixed_enum": "1", + "numeric_union": 42.0, + "scalar_union": "100", + "date_or_year_union": "2025-06-15", + "raw_b64": "Hello", + "raw_hex": "Hello" + }, + { + "id": "rec2", + "status": "archived", + "version": 1, + "flag": false + }, + { + "id": "rec3", + "status": "pending", + "version": 32767, + "flag": true, + "bounded_int": 999, + "big_int": -999999999999, + "float_val": -0.5, + "decimal_multiple": 42.0, + "created_date": "1999-12-31", + "updated_at": "2025-06-15T12:30:00.123+02:00", + "event_time": "23:59:59+02:00", + "elapsed": "PT15M", + "record_uuid": "123e4567-e89b-12d3-a456-426614174000", + "homepage": "https://example.org/rec3", + "const_tag": "v1", + "untyped_int_enum": 30, + "untyped_mixed_enum": "true", + "numeric_union": 3.5, + "scalar_union": "hundred", + "date_or_year_union": "1999", + "raw_b64": "World", + "raw_hex": "World" + } +] diff --git a/tests/data/jsonschema/spec/01_scalars_and_formats/expected_schema.py b/tests/data/jsonschema/spec/01_scalars_and_formats/expected_schema.py new file mode 100644 index 00000000..564a4fb8 --- /dev/null +++ b/tests/data/jsonschema/spec/01_scalars_and_formats/expected_schema.py @@ -0,0 +1,57 @@ +"""Expected PyArrow schema for 01_scalars_and_formats.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "id", + pa.string(), + nullable=False, + metadata={"comment": "Unique record identifier.", "jsonschema_type": "string"}, + ), + pa.field( + "status", + pa.string(), + nullable=False, + metadata={"comment": "Record lifecycle status.", "jsonschema_type": "string"}, + ), + pa.field("version", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field( + "flag", + pa.bool_(), + nullable=False, + metadata={"comment": "Primary boolean flag.", "jsonschema_type": "boolean"}, + ), + pa.field("bounded_int", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("big_int", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("float_val", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field("decimal_multiple", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field("created_date", pa.date32(), nullable=True, metadata={"jsonschema_type": "string (date)"}), + pa.field("updated_at", pa.string(), nullable=True, metadata={"jsonschema_type": "string (date-time)"}), + pa.field("event_time", pa.string(), nullable=True, metadata={"jsonschema_type": "string (time)"}), + pa.field("elapsed", pa.string(), nullable=True, metadata={"jsonschema_type": "string (duration)"}), + pa.field("record_uuid", pa.string(), nullable=True, metadata={"jsonschema_type": "string (uuid)"}), + pa.field("homepage", pa.string(), nullable=True, metadata={"jsonschema_type": "string (uri)"}), + pa.field("const_tag", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("untyped_int_enum", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field( + "untyped_mixed_enum", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "integer | string | boolean"}, + ), + pa.field("numeric_union", pa.float64(), nullable=True, metadata={"jsonschema_type": "integer | number"}), + pa.field("scalar_union", pa.string(), nullable=True, metadata={"jsonschema_type": "integer | string"}), + pa.field( + "date_or_year_union", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string (date) | string"}, + ), + pa.field("raw_b64", pa.binary(), nullable=True, metadata={"jsonschema_type": "string (base64)"}), + pa.field("raw_hex", pa.binary(), nullable=True, metadata={"jsonschema_type": "string (base16)"}), + ] +) diff --git a/tests/data/jsonschema/spec/01_scalars_and_formats/expected_table.py b/tests/data/jsonschema/spec/01_scalars_and_formats/expected_table.py new file mode 100644 index 00000000..682e027c --- /dev/null +++ b/tests/data/jsonschema/spec/01_scalars_and_formats/expected_table.py @@ -0,0 +1,63 @@ +"""Expected PyArrow table for 01_scalars_and_formats.""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "rec1", + "status": "active", + "version": 2, + "flag": True, + "bounded_int": -42, + "big_int": 999999999999, + "float_val": 2.718281828, + "decimal_multiple": 3.14, + "created_date": dt.date(2025, 6, 15), + "updated_at": "2025-06-15T12:30:00Z", + "event_time": "12:30:00Z", + "elapsed": "P3DT4H", + "record_uuid": "550e8400-e29b-41d4-a716-446655440000", + "homepage": "https://example.org/rec1", + "const_tag": "v1", + "untyped_int_enum": 20, + "untyped_mixed_enum": "1", + "numeric_union": 42.0, + "scalar_union": "100", + "date_or_year_union": "2025-06-15", + "raw_b64": b"Hello", + "raw_hex": b"Hello", + }, + {"id": "rec2", "status": "archived", "version": 1, "flag": False}, + { + "id": "rec3", + "status": "pending", + "version": 32767, + "flag": True, + "bounded_int": 999, + "big_int": -999999999999, + "float_val": -0.5, + "decimal_multiple": 42.0, + "created_date": dt.date(1999, 12, 31), + "updated_at": "2025-06-15T12:30:00.123+02:00", + "event_time": "23:59:59+02:00", + "elapsed": "PT15M", + "record_uuid": "123e4567-e89b-12d3-a456-426614174000", + "homepage": "https://example.org/rec3", + "const_tag": "v1", + "untyped_int_enum": 30, + "untyped_mixed_enum": "true", + "numeric_union": 3.5, + "scalar_union": "hundred", + "date_or_year_union": "1999", + "raw_b64": b"World", + "raw_hex": b"World", + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/01_scalars_and_formats/schema.json b/tests/data/jsonschema/spec/01_scalars_and_formats/schema.json new file mode 100644 index 00000000..b8c01a21 --- /dev/null +++ b/tests/data/jsonschema/spec/01_scalars_and_formats/schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/01_scalars_and_formats.schema.json", + "title": "ScalarsAndFormatsRecord", + "type": "object", + "required": [ + "id", + "status", + "version", + "flag" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique record identifier." + }, + "status": { + "type": "string", + "enum": [ + "active", + "archived", + "pending" + ], + "description": "Record lifecycle status." + }, + "version": { + "type": "integer", + "minimum": 1, + "maximum": 32767 + }, + "flag": { + "type": "boolean", + "description": "Primary boolean flag." + }, + "bounded_int": { + "type": "integer", + "exclusiveMinimum": -1000, + "exclusiveMaximum": 1000 + }, + "big_int": { + "type": "integer" + }, + "float_val": { + "type": "number" + }, + "decimal_multiple": { + "type": "number", + "multipleOf": 0.01 + }, + "created_date": { + "type": "string", + "format": "date" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "event_time": { + "type": "string", + "format": "time" + }, + "elapsed": { + "type": "string", + "format": "duration" + }, + "record_uuid": { + "type": "string", + "format": "uuid" + }, + "homepage": { + "type": "string", + "format": "uri" + }, + "const_tag": { + "const": "v1" + }, + "untyped_int_enum": { + "enum": [ + 10, + 20, + 30 + ] + }, + "untyped_mixed_enum": { + "enum": [ + 1, + "two", + true + ] + }, + "numeric_union": { + "type": [ + "integer", + "number" + ] + }, + "scalar_union": { + "type": [ + "integer", + "string" + ] + }, + "date_or_year_union": { + "anyOf": [ + { + "type": "string", + "format": "date" + }, + { + "type": "string", + "pattern": "^[0-9]{4}$" + } + ] + }, + "raw_b64": { + "type": "string", + "contentEncoding": "base64" + }, + "raw_hex": { + "type": "string", + "contentEncoding": "base16" + } + } +} + diff --git a/tests/data/jsonschema/spec/02_nullability_and_required/case.yaml b/tests/data/jsonschema/spec/02_nullability_and_required/case.yaml new file mode 100644 index 00000000..5b571940 --- /dev/null +++ b/tests/data/jsonschema/spec/02_nullability_and_required/case.yaml @@ -0,0 +1,10 @@ +title: Nullability, Required Fields and Array Item Nullability +schema: schema.json +rules: [R-NULL-REQUIRED, R-NULL-ALLOWS-NULL, R-NULL-DEFAULT, R-NULL-ITEMS, R-SCALAR-ENUM-CONST] +notes: | + - `required_nullable_str`, `required_anyof_nullable` and `required_enum_nullable` are required but allow `null`, so they are nullable. + - `optional_with_default` has a `default` and stays nullable. + - `strict_int_list` items are non-nullable; `nullable_int_list` items (`["integer", "null"]`) are nullable. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/02_nullability_and_required/data.json b/tests/data/jsonschema/spec/02_nullability_and_required/data.json new file mode 100644 index 00000000..9175c6ea --- /dev/null +++ b/tests/data/jsonschema/spec/02_nullability_and_required/data.json @@ -0,0 +1,39 @@ +[ + { + "required_str": "alpha", + "required_int": 10, + "required_nullable_str": "present", + "required_anyof_nullable": 99, + "required_enum_nullable": "high", + "optional_str": "opt_value", + "optional_with_default": "explicit", + "optional_nullable_number": 1.25, + "strict_int_list": [ + 1, + 2, + 3 + ], + "nullable_int_list": [ + 10, + null, + 30 + ], + "optional_str_list": [ + "a", + "b" + ] + }, + { + "required_str": "beta", + "required_int": 0, + "required_nullable_str": null, + "required_anyof_nullable": null, + "required_enum_nullable": null, + "optional_nullable_number": null, + "strict_int_list": [], + "nullable_int_list": [ + null + ] + } +] + diff --git a/tests/data/jsonschema/spec/02_nullability_and_required/expected.json b/tests/data/jsonschema/spec/02_nullability_and_required/expected.json new file mode 100644 index 00000000..9175c6ea --- /dev/null +++ b/tests/data/jsonschema/spec/02_nullability_and_required/expected.json @@ -0,0 +1,39 @@ +[ + { + "required_str": "alpha", + "required_int": 10, + "required_nullable_str": "present", + "required_anyof_nullable": 99, + "required_enum_nullable": "high", + "optional_str": "opt_value", + "optional_with_default": "explicit", + "optional_nullable_number": 1.25, + "strict_int_list": [ + 1, + 2, + 3 + ], + "nullable_int_list": [ + 10, + null, + 30 + ], + "optional_str_list": [ + "a", + "b" + ] + }, + { + "required_str": "beta", + "required_int": 0, + "required_nullable_str": null, + "required_anyof_nullable": null, + "required_enum_nullable": null, + "optional_nullable_number": null, + "strict_int_list": [], + "nullable_int_list": [ + null + ] + } +] + diff --git a/tests/data/jsonschema/spec/02_nullability_and_required/expected_schema.py b/tests/data/jsonschema/spec/02_nullability_and_required/expected_schema.py new file mode 100644 index 00000000..4df9ca2c --- /dev/null +++ b/tests/data/jsonschema/spec/02_nullability_and_required/expected_schema.py @@ -0,0 +1,49 @@ +"""Expected PyArrow schema for 02_nullability_and_required.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "required_str", + pa.string(), + nullable=False, + metadata={"comment": "Strictly required non-null string.", "jsonschema_type": "string"}, + ), + pa.field("required_int", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field( + "required_nullable_str", + pa.string(), + nullable=True, + metadata={"comment": "Required key whose value may be null.", "jsonschema_type": "string"}, + ), + pa.field("required_anyof_nullable", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field("required_enum_nullable", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("optional_str", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("optional_with_default", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("optional_nullable_number", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field( + "strict_int_list", + pa.list_( + pa.field("item", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + ), + nullable=False, + ), + pa.field( + "nullable_int_list", + pa.list_( + pa.field("item", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + ), + nullable=False, + ), + pa.field( + "optional_str_list", + pa.list_( + pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/02_nullability_and_required/expected_table.py b/tests/data/jsonschema/spec/02_nullability_and_required/expected_table.py new file mode 100644 index 00000000..4f70171b --- /dev/null +++ b/tests/data/jsonschema/spec/02_nullability_and_required/expected_table.py @@ -0,0 +1,35 @@ +"""Expected PyArrow table for 02_nullability_and_required.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "required_str": "alpha", + "required_int": 10, + "required_nullable_str": "present", + "required_anyof_nullable": 99, + "required_enum_nullable": "high", + "optional_str": "opt_value", + "optional_with_default": "explicit", + "optional_nullable_number": 1.25, + "strict_int_list": [1, 2, 3], + "nullable_int_list": [10, None, 30], + "optional_str_list": ["a", "b"], + }, + { + "required_str": "beta", + "required_int": 0, + "required_nullable_str": None, + "required_anyof_nullable": None, + "required_enum_nullable": None, + "optional_nullable_number": None, + "strict_int_list": [], + "nullable_int_list": [None], + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/02_nullability_and_required/schema.json b/tests/data/jsonschema/spec/02_nullability_and_required/schema.json new file mode 100644 index 00000000..be9407b9 --- /dev/null +++ b/tests/data/jsonschema/spec/02_nullability_and_required/schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/02_nullability_and_required.schema.json", + "title": "NullabilityAndRequiredRecord", + "type": "object", + "required": [ + "required_str", + "required_int", + "required_nullable_str", + "required_anyof_nullable", + "required_enum_nullable", + "strict_int_list", + "nullable_int_list" + ], + "properties": { + "required_str": { + "type": "string", + "description": "Strictly required non-null string." + }, + "required_int": { + "type": "integer" + }, + "required_nullable_str": { + "type": [ + "string", + "null" + ], + "description": "Required key whose value may be null." + }, + "required_anyof_nullable": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "required_enum_nullable": { + "enum": [ + "low", + "medium", + "high", + null + ] + }, + "optional_str": { + "type": "string" + }, + "optional_with_default": { + "type": "string", + "default": "fallback" + }, + "optional_nullable_number": { + "type": [ + "number", + "null" + ] + }, + "strict_int_list": { + "type": "array", + "items": { + "type": "integer" + } + }, + "nullable_int_list": { + "type": "array", + "items": { + "type": [ + "integer", + "null" + ] + } + }, + "optional_str_list": { + "type": "array", + "items": { + "type": "string" + } + } + } +} + diff --git a/tests/data/jsonschema/spec/03_arrays_and_tuples/case.yaml b/tests/data/jsonschema/spec/03_arrays_and_tuples/case.yaml new file mode 100644 index 00000000..7ad14553 --- /dev/null +++ b/tests/data/jsonschema/spec/03_arrays_and_tuples/case.yaml @@ -0,0 +1,9 @@ +title: Homogeneous Arrays, 2D Arrays and Closed Tuples +schema: schema.json +rules: [R-SHAPE-ARRAY, R-SHAPE-TUPLE, R-NULL-ITEMS, R-COERCE-TUPLE, R-META-COMMENT] +notes: | + `coord_3d` has `minItems: 2`, so `item_0` and `item_1` are non-nullable and `item_2` (`["number", "null"]`) is + nullable. `[10.5, 20.25, 30.0]` becomes `{"item_0": 10.5, "item_1": 20.25, "item_2": 30.0}`. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/03_arrays_and_tuples/data.json b/tests/data/jsonschema/spec/03_arrays_and_tuples/data.json new file mode 100644 index 00000000..cf590e71 --- /dev/null +++ b/tests/data/jsonschema/spec/03_arrays_and_tuples/data.json @@ -0,0 +1,54 @@ +[ + { + "id": "arr1", + "tags": [ + "genomics", + "assembly" + ], + "measurements": [ + { + "label": "gc_content", + "value": 0.512, + "unit": "ratio" + }, + { + "label": "n50", + "value": 4500000.0 + } + ], + "matrix_2d": [ + [ + 1.0, + 2.0 + ], + [ + 3.5, + 4.5 + ] + ], + "coord_3d": [ + 10.5, + 20.25, + 30.0 + ], + "pair_tuple_list": [ + [ + "chr1", + 248956422 + ], + [ + "chr2", + 242193529 + ] + ] + }, + { + "id": "arr2", + "tags": [], + "coord_3d": [ + -1.0, + 2.0 + ] + } +] + diff --git a/tests/data/jsonschema/spec/03_arrays_and_tuples/expected.json b/tests/data/jsonschema/spec/03_arrays_and_tuples/expected.json new file mode 100644 index 00000000..252e51f5 --- /dev/null +++ b/tests/data/jsonschema/spec/03_arrays_and_tuples/expected.json @@ -0,0 +1,54 @@ +[ + { + "id": "arr1", + "tags": [ + "genomics", + "assembly" + ], + "measurements": [ + { + "label": "gc_content", + "value": 0.512, + "unit": "ratio" + }, + { + "label": "n50", + "value": 4500000.0 + } + ], + "matrix_2d": [ + [ + 1.0, + 2.0 + ], + [ + 3.5, + 4.5 + ] + ], + "coord_3d": { + "item_0": 10.5, + "item_1": 20.25, + "item_2": 30.0 + }, + "pair_tuple_list": [ + { + "item_0": "chr1", + "item_1": 248956422 + }, + { + "item_0": "chr2", + "item_1": 242193529 + } + ] + }, + { + "id": "arr2", + "tags": [], + "coord_3d": { + "item_0": -1.0, + "item_1": 2.0 + } + } +] + diff --git a/tests/data/jsonschema/spec/03_arrays_and_tuples/expected_schema.py b/tests/data/jsonschema/spec/03_arrays_and_tuples/expected_schema.py new file mode 100644 index 00000000..a00a2f1d --- /dev/null +++ b/tests/data/jsonschema/spec/03_arrays_and_tuples/expected_schema.py @@ -0,0 +1,93 @@ +"""Expected PyArrow schema for 03_arrays_and_tuples.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "tags", + pa.list_( + pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=False, + metadata={"comment": "Ordered list of string tags."}, + ), + pa.field( + "measurements", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("label", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("value", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + pa.field("unit", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + pa.field( + "matrix_2d", + pa.list_( + pa.field( + "item", + pa.list_( + pa.field("item", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + ), + nullable=False, + ), + ), + nullable=True, + metadata={"comment": "Two-dimensional numeric matrix."}, + ), + pa.field( + "coord_3d", + pa.struct( + [ + pa.field( + "item_0", + pa.float64(), + nullable=False, + metadata={"comment": "X coordinate.", "jsonschema_type": "number"}, + ), + pa.field( + "item_1", + pa.float64(), + nullable=False, + metadata={"comment": "Y coordinate.", "jsonschema_type": "number"}, + ), + pa.field( + "item_2", + pa.float64(), + nullable=True, + metadata={"comment": "Optional Z coordinate.", "jsonschema_type": "number"}, + ), + ] + ), + nullable=False, + metadata={"comment": "Closed 3D coordinate tuple (x, y, optional z)."}, + ), + pa.field( + "pair_tuple_list", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field("item_0", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("item_1", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/03_arrays_and_tuples/expected_table.py b/tests/data/jsonschema/spec/03_arrays_and_tuples/expected_table.py new file mode 100644 index 00000000..700f3502 --- /dev/null +++ b/tests/data/jsonschema/spec/03_arrays_and_tuples/expected_table.py @@ -0,0 +1,24 @@ +"""Expected PyArrow table for 03_arrays_and_tuples.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "arr1", + "tags": ["genomics", "assembly"], + "measurements": [ + {"label": "gc_content", "value": 0.512, "unit": "ratio"}, + {"label": "n50", "value": 4500000.0}, + ], + "matrix_2d": [[1.0, 2.0], [3.5, 4.5]], + "coord_3d": {"item_0": 10.5, "item_1": 20.25, "item_2": 30.0}, + "pair_tuple_list": [{"item_0": "chr1", "item_1": 248956422}, {"item_0": "chr2", "item_1": 242193529}], + }, + {"id": "arr2", "tags": [], "coord_3d": {"item_0": -1.0, "item_1": 2.0}}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/03_arrays_and_tuples/schema.json b/tests/data/jsonschema/spec/03_arrays_and_tuples/schema.json new file mode 100644 index 00000000..0cb90eb5 --- /dev/null +++ b/tests/data/jsonschema/spec/03_arrays_and_tuples/schema.json @@ -0,0 +1,94 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/03_arrays_and_tuples.schema.json", + "title": "ArraysAndTuplesRecord", + "type": "object", + "required": [ + "id", + "tags", + "coord_3d" + ], + "properties": { + "id": { + "type": "string" + }, + "tags": { + "type": "array", + "description": "Ordered list of string tags.", + "items": { + "type": "string" + } + }, + "measurements": { + "type": "array", + "items": { + "type": "object", + "required": [ + "label", + "value" + ], + "properties": { + "label": { + "type": "string" + }, + "value": { + "type": "number" + }, + "unit": { + "type": "string" + } + } + } + }, + "matrix_2d": { + "type": "array", + "description": "Two-dimensional numeric matrix.", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "coord_3d": { + "type": "array", + "description": "Closed 3D coordinate tuple (x, y, optional z).", + "minItems": 2, + "prefixItems": [ + { + "type": "number", + "description": "X coordinate." + }, + { + "type": "number", + "description": "Y coordinate." + }, + { + "type": [ + "number", + "null" + ], + "description": "Optional Z coordinate." + } + ], + "items": false + }, + "pair_tuple_list": { + "type": "array", + "items": { + "type": "array", + "minItems": 2, + "prefixItems": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "items": false + } + } + } +} + diff --git a/tests/data/jsonschema/spec/04_objects_and_maps/case.yaml b/tests/data/jsonschema/spec/04_objects_and_maps/case.yaml new file mode 100644 index 00000000..a019b682 --- /dev/null +++ b/tests/data/jsonschema/spec/04_objects_and_maps/case.yaml @@ -0,0 +1,10 @@ +title: Nested Fixed Objects and Dynamic Maps +schema: schema.json +rules: [R-SHAPE-OBJECT, R-SHAPE-MAP, R-NULL-ITEMS] +notes: | + `closed_meta` (`additionalProperties: false`) and `open_meta` (`additionalProperties: true`) are both structs. + `string_to_int_map`, `nullable_score_map` and `struct_map` use `additionalProperties`; `pattern_metric_map` + uses `patternProperties`. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/04_objects_and_maps/data.json b/tests/data/jsonschema/spec/04_objects_and_maps/data.json new file mode 100644 index 00000000..2dca1635 --- /dev/null +++ b/tests/data/jsonschema/spec/04_objects_and_maps/data.json @@ -0,0 +1,40 @@ +[ + { + "id": "obj1", + "closed_meta": { + "owner": "alice", + "priority": 1 + }, + "open_meta": { + "source": "ncbi" + }, + "string_to_int_map": { + "reads": 1500, + "contigs": 12 + }, + "nullable_score_map": { + "completeness": 99.4, + "contamination": null + }, + "pattern_metric_map": { + "metric_1": 0.95, + "metric_2": 0.88 + }, + "struct_map": { + "kmer_31": { + "count": 42000, + "note": "canonical" + }, + "kmer_21": { + "count": 38000 + } + } + }, + { + "id": "obj2", + "closed_meta": { + "owner": "bob" + } + } +] + diff --git a/tests/data/jsonschema/spec/04_objects_and_maps/expected.json b/tests/data/jsonschema/spec/04_objects_and_maps/expected.json new file mode 100644 index 00000000..2dca1635 --- /dev/null +++ b/tests/data/jsonschema/spec/04_objects_and_maps/expected.json @@ -0,0 +1,40 @@ +[ + { + "id": "obj1", + "closed_meta": { + "owner": "alice", + "priority": 1 + }, + "open_meta": { + "source": "ncbi" + }, + "string_to_int_map": { + "reads": 1500, + "contigs": 12 + }, + "nullable_score_map": { + "completeness": 99.4, + "contamination": null + }, + "pattern_metric_map": { + "metric_1": 0.95, + "metric_2": 0.88 + }, + "struct_map": { + "kmer_31": { + "count": 42000, + "note": "canonical" + }, + "kmer_21": { + "count": 38000 + } + } + }, + { + "id": "obj2", + "closed_meta": { + "owner": "bob" + } + } +] + diff --git a/tests/data/jsonschema/spec/04_objects_and_maps/expected_schema.py b/tests/data/jsonschema/spec/04_objects_and_maps/expected_schema.py new file mode 100644 index 00000000..a9c5ad63 --- /dev/null +++ b/tests/data/jsonschema/spec/04_objects_and_maps/expected_schema.py @@ -0,0 +1,74 @@ +"""Expected PyArrow schema for 04_objects_and_maps.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "closed_meta", + pa.struct( + [ + pa.field("owner", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("priority", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + ] + ), + nullable=False, + metadata={"comment": "Strict object with additionalProperties set to false."}, + ), + pa.field( + "open_meta", + pa.struct( + [ + pa.field("source", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + metadata={"comment": "Fixed-property object with boolean additionalProperties true."}, + ), + pa.field( + "string_to_int_map", + pa.map_( + pa.string(), + pa.field("value", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + ), + nullable=True, + metadata={"comment": "Dynamic map from string keys to non-null integer counts."}, + ), + pa.field( + "nullable_score_map", + pa.map_( + pa.string(), + pa.field("value", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + ), + nullable=True, + ), + pa.field( + "pattern_metric_map", + pa.map_( + pa.string(), + pa.field("value", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + ), + nullable=True, + ), + pa.field( + "struct_map", + pa.map_( + pa.string(), + pa.field( + "value", + pa.struct( + [ + pa.field("count", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("note", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/04_objects_and_maps/expected_table.py b/tests/data/jsonschema/spec/04_objects_and_maps/expected_table.py new file mode 100644 index 00000000..dbffa7f6 --- /dev/null +++ b/tests/data/jsonschema/spec/04_objects_and_maps/expected_table.py @@ -0,0 +1,22 @@ +"""Expected PyArrow table for 04_objects_and_maps.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "obj1", + "closed_meta": {"owner": "alice", "priority": 1}, + "open_meta": {"source": "ncbi"}, + "string_to_int_map": {"reads": 1500, "contigs": 12}, + "nullable_score_map": {"completeness": 99.4, "contamination": None}, + "pattern_metric_map": {"metric_1": 0.95, "metric_2": 0.88}, + "struct_map": {"kmer_31": {"count": 42000, "note": "canonical"}, "kmer_21": {"count": 38000}}, + }, + {"id": "obj2", "closed_meta": {"owner": "bob"}}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/04_objects_and_maps/schema.json b/tests/data/jsonschema/spec/04_objects_and_maps/schema.json new file mode 100644 index 00000000..93170412 --- /dev/null +++ b/tests/data/jsonschema/spec/04_objects_and_maps/schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/04_objects_and_maps.schema.json", + "title": "ObjectsAndMapsRecord", + "type": "object", + "required": [ + "id", + "closed_meta" + ], + "properties": { + "id": { + "type": "string" + }, + "closed_meta": { + "type": "object", + "description": "Strict object with additionalProperties set to false.", + "required": [ + "owner" + ], + "additionalProperties": false, + "properties": { + "owner": { + "type": "string" + }, + "priority": { + "type": "integer" + } + } + }, + "open_meta": { + "type": "object", + "description": "Fixed-property object with boolean additionalProperties true.", + "additionalProperties": true, + "properties": { + "source": { + "type": "string" + } + } + }, + "string_to_int_map": { + "type": "object", + "description": "Dynamic map from string keys to non-null integer counts.", + "additionalProperties": { + "type": "integer" + } + }, + "nullable_score_map": { + "type": "object", + "additionalProperties": { + "type": [ + "number", + "null" + ] + } + }, + "pattern_metric_map": { + "type": "object", + "patternProperties": { + "^metric_[0-9]+$": { + "type": "number" + } + }, + "additionalProperties": false + }, + "struct_map": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "count" + ], + "properties": { + "count": { + "type": "integer" + }, + "note": { + "type": "string" + } + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/05_defs_and_refs/case.yaml b/tests/data/jsonschema/spec/05_defs_and_refs/case.yaml new file mode 100644 index 00000000..29015ecf --- /dev/null +++ b/tests/data/jsonschema/spec/05_defs_and_refs/case.yaml @@ -0,0 +1,9 @@ +title: Shared Definitions, `$ref` Reuse and Root `$ref` +schema: schema.json +rules: [R-ROOT-REF, R-SHAPE-REF, R-META-COMMENT] +notes: | + The root is `"$ref": "#/$defs/Document"`. `lead_author` declares its own `description`, overriding the + `Author` definition's; `reviewer` and `co_authors` items inherit it. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/05_defs_and_refs/data.json b/tests/data/jsonschema/spec/05_defs_and_refs/data.json new file mode 100644 index 00000000..dc9d3cff --- /dev/null +++ b/tests/data/jsonschema/spec/05_defs_and_refs/data.json @@ -0,0 +1,26 @@ +[ + { + "doc_id": "doc1", + "published_on": "2025-07-01", + "lead_author": { + "name": "Ada Lovelace", + "orcid": "0000-0001-2345-6789" + }, + "reviewer": { + "name": "Grace Hopper" + }, + "co_authors": [ + { + "name": "Alan Turing", + "orcid": "0000-0002-3456-7890" + } + ] + }, + { + "doc_id": "doc2", + "lead_author": { + "name": "Barbara Liskov" + } + } +] + diff --git a/tests/data/jsonschema/spec/05_defs_and_refs/expected.json b/tests/data/jsonschema/spec/05_defs_and_refs/expected.json new file mode 100644 index 00000000..dc9d3cff --- /dev/null +++ b/tests/data/jsonschema/spec/05_defs_and_refs/expected.json @@ -0,0 +1,26 @@ +[ + { + "doc_id": "doc1", + "published_on": "2025-07-01", + "lead_author": { + "name": "Ada Lovelace", + "orcid": "0000-0001-2345-6789" + }, + "reviewer": { + "name": "Grace Hopper" + }, + "co_authors": [ + { + "name": "Alan Turing", + "orcid": "0000-0002-3456-7890" + } + ] + }, + { + "doc_id": "doc2", + "lead_author": { + "name": "Barbara Liskov" + } + } +] + diff --git a/tests/data/jsonschema/spec/05_defs_and_refs/expected_schema.py b/tests/data/jsonschema/spec/05_defs_and_refs/expected_schema.py new file mode 100644 index 00000000..ad359337 --- /dev/null +++ b/tests/data/jsonschema/spec/05_defs_and_refs/expected_schema.py @@ -0,0 +1,48 @@ +"""Expected PyArrow schema for 05_defs_and_refs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +_AUTHOR_STRUCT = pa.struct( + [ + pa.field("name", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("orcid", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] +) + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("doc_id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "published_on", + pa.date32(), + nullable=True, + metadata={"comment": "ISO-8601 calendar date.", "jsonschema_type": "string (date)"}, + ), + pa.field( + "lead_author", + _AUTHOR_STRUCT, + nullable=False, + metadata={"comment": "Primary corresponding author."}, + ), + pa.field( + "reviewer", + _AUTHOR_STRUCT, + nullable=True, + metadata={"comment": "Author contact record."}, + ), + pa.field( + "co_authors", + pa.list_( + pa.field( + "item", + _AUTHOR_STRUCT, + nullable=False, + metadata={"comment": "Author contact record."}, + ), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/05_defs_and_refs/expected_table.py b/tests/data/jsonschema/spec/05_defs_and_refs/expected_table.py new file mode 100644 index 00000000..eec7aee0 --- /dev/null +++ b/tests/data/jsonschema/spec/05_defs_and_refs/expected_table.py @@ -0,0 +1,22 @@ +"""Expected PyArrow table for 05_defs_and_refs.""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "doc_id": "doc1", + "published_on": dt.date(2025, 7, 1), + "lead_author": {"name": "Ada Lovelace", "orcid": "0000-0001-2345-6789"}, + "reviewer": {"name": "Grace Hopper"}, + "co_authors": [{"name": "Alan Turing", "orcid": "0000-0002-3456-7890"}], + }, + {"doc_id": "doc2", "lead_author": {"name": "Barbara Liskov"}}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/05_defs_and_refs/schema.json b/tests/data/jsonschema/spec/05_defs_and_refs/schema.json new file mode 100644 index 00000000..3d735f2c --- /dev/null +++ b/tests/data/jsonschema/spec/05_defs_and_refs/schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/05_defs_and_refs.schema.json", + "title": "DefsAndRefsRoot", + "$ref": "#/$defs/Document", + "$defs": { + "IsoDate": { + "type": "string", + "format": "date", + "description": "ISO-8601 calendar date." + }, + "Author": { + "type": "object", + "description": "Author contact record.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "orcid": { + "type": "string" + } + } + }, + "Document": { + "type": "object", + "required": [ + "doc_id", + "lead_author" + ], + "properties": { + "doc_id": { + "type": "string" + }, + "published_on": { + "$ref": "#/$defs/IsoDate" + }, + "lead_author": { + "$ref": "#/$defs/Author", + "description": "Primary corresponding author." + }, + "reviewer": { + "$ref": "#/$defs/Author" + }, + "co_authors": { + "type": "array", + "items": { + "$ref": "#/$defs/Author" + } + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/06_recursion_depth/case.yaml b/tests/data/jsonschema/spec/06_recursion_depth/case.yaml new file mode 100644 index 00000000..1e5e0a26 --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/case.yaml @@ -0,0 +1,16 @@ +title: Structural Recursion and `max_recursion_depth` +schema: schema.json +rules: [R-RECURSION, R-COERCE-STRING-FALLBACK] +notes: | + `TreeNode` is self-recursive (`branch: array of TreeNode`); `Section` and `Subsection` are mutually recursive. + At depth 2 the third `TreeNode` level becomes JSON strings; at depth 1 the second level already does. +runs: + - id: depth-2 + params: {max_recursion_depth: 2} + data: data.json + - id: depth-1 + params: {max_recursion_depth: 1} + data: data.json + expected_json: expected_depth_1.json + schema_attr: EXPECTED_SCHEMA_DEPTH_1 + table_attr: EXPECTED_TABLE_DEPTH_1 diff --git a/tests/data/jsonschema/spec/06_recursion_depth/data.json b/tests/data/jsonschema/spec/06_recursion_depth/data.json new file mode 100644 index 00000000..4a99d072 --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/data.json @@ -0,0 +1,48 @@ +[ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + { + "label": "L2", + "branch": [ + { + "label": "L3", + "weight": 30, + "branch": [ + { + "label": "L4" + } + ] + }, + { + "label": "L3_sibling" + } + ] + } + ] + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + { + "section_title": "Sec 1.1", + "subsection": { + "subsection_title": "Sub 1.1.a", + "nested_sections": [ + { + "section_title": "Subsec 1.1.1" + } + ] + } + } + ] + } + } + } +] + diff --git a/tests/data/jsonschema/spec/06_recursion_depth/expected.json b/tests/data/jsonschema/spec/06_recursion_depth/expected.json new file mode 100644 index 00000000..bb237289 --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/expected.json @@ -0,0 +1,36 @@ +[ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + { + "label": "L2", + "branch": [ + "{\"label\": \"L3\", \"weight\": 30, \"branch\": [{\"label\": \"L4\"}]}", + "{\"label\": \"L3_sibling\"}" + ] + } + ] + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + { + "section_title": "Sec 1.1", + "subsection": { + "subsection_title": "Sub 1.1.a", + "nested_sections": [ + "{\"section_title\": \"Subsec 1.1.1\"}" + ] + } + } + ] + } + } + } +] + diff --git a/tests/data/jsonschema/spec/06_recursion_depth/expected_depth_1.json b/tests/data/jsonschema/spec/06_recursion_depth/expected_depth_1.json new file mode 100644 index 00000000..f4c9c702 --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/expected_depth_1.json @@ -0,0 +1,22 @@ +[ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + "{\"label\": \"L2\", \"branch\": [{\"label\": \"L3\", \"weight\": 30, \"branch\": [{\"label\": \"L4\"}]}, {\"label\": \"L3_sibling\"}]}" + ] + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + "{\"section_title\": \"Sec 1.1\", \"subsection\": {\"subsection_title\": \"Sub 1.1.a\", \"nested_sections\": [{\"section_title\": \"Subsec 1.1.1\"}]}}" + ] + } + } + } +] + diff --git a/tests/data/jsonschema/spec/06_recursion_depth/expected_schema.py b/tests/data/jsonschema/spec/06_recursion_depth/expected_schema.py new file mode 100644 index 00000000..5e673f20 --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/expected_schema.py @@ -0,0 +1,171 @@ +"""Expected PyArrow schema for 06_recursion_depth.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("label", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("weight", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field( + "branch", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "label", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "weight", + pa.int64(), + nullable=True, + metadata={"jsonschema_type": "integer"}, + ), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "section", + pa.struct( + [ + pa.field("section_title", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "subsection", + pa.struct( + [ + pa.field( + "subsection_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "nested_sections", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "section_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "subsection", + pa.struct( + [ + pa.field( + "subsection_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "nested_sections", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) + +EXPECTED_SCHEMA_DEPTH_1 = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "tree", + pa.struct( + [ + pa.field("label", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("weight", pa.int64(), nullable=True, metadata={"jsonschema_type": "integer"}), + pa.field( + "branch", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field( + "section", + pa.struct( + [ + pa.field("section_title", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "subsection", + pa.struct( + [ + pa.field( + "subsection_title", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "nested_sections", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/06_recursion_depth/expected_table.py b/tests/data/jsonschema/spec/06_recursion_depth/expected_table.py new file mode 100644 index 00000000..5b8834f5 --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/expected_table.py @@ -0,0 +1,68 @@ +"""Expected PyArrow table for 06_recursion_depth.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA, EXPECTED_SCHEMA_DEPTH_1 + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + { + "label": "L2", + "branch": [ + '{"label": "L3", "weight": 30, "branch": [{"label": "L4"}]}', + '{"label": "L3_sibling"}', + ], + } + ], + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + { + "section_title": "Sec 1.1", + "subsection": { + "subsection_title": "Sub 1.1.a", + "nested_sections": ['{"section_title": "Subsec 1.1.1"}'], + }, + } + ], + }, + }, + }, + ], + schema=EXPECTED_SCHEMA, +) + +EXPECTED_TABLE_DEPTH_1 = pa.Table.from_pylist( + [ + { + "id": "r1", + "tree": { + "label": "L1", + "weight": 10, + "branch": [ + '{"label": "L2", "branch": [{"label": "L3", "weight": 30, "branch": [{"label": "L4"}]}, {"label": "L3_sibling"}]}' + ], + }, + "section": { + "section_title": "Ch 1", + "subsection": { + "subsection_title": "Sub 1.a", + "nested_sections": [ + '{"section_title": "Sec 1.1", "subsection": {"subsection_title": "Sub 1.1.a", "nested_sections": [{"section_title": "Subsec 1.1.1"}]}}' + ], + }, + }, + }, + ], + schema=EXPECTED_SCHEMA_DEPTH_1, +) diff --git a/tests/data/jsonschema/spec/06_recursion_depth/schema.json b/tests/data/jsonschema/spec/06_recursion_depth/schema.json new file mode 100644 index 00000000..8e8d40af --- /dev/null +++ b/tests/data/jsonschema/spec/06_recursion_depth/schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/06_recursion_depth.schema.json", + "title": "RecursionDepthRecord", + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "tree": { + "$ref": "#/$defs/TreeNode" + }, + "section": { + "$ref": "#/$defs/Section" + } + }, + "$defs": { + "TreeNode": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "weight": { + "type": "integer" + }, + "branch": { + "type": "array", + "items": { + "$ref": "#/$defs/TreeNode" + } + } + } + }, + "Section": { + "type": "object", + "required": [ + "section_title" + ], + "properties": { + "section_title": { + "type": "string" + }, + "subsection": { + "$ref": "#/$defs/Subsection" + } + } + }, + "Subsection": { + "type": "object", + "required": [ + "subsection_title" + ], + "properties": { + "subsection_title": { + "type": "string" + }, + "nested_sections": { + "type": "array", + "items": { + "$ref": "#/$defs/Section" + } + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/case.yaml b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/case.yaml new file mode 100644 index 00000000..41c7484c --- /dev/null +++ b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/case.yaml @@ -0,0 +1,11 @@ +title: Combinators — `allOf`, `anyOf` and `oneOf` +schema: schema.json +rules: [R-ORDER, R-MERGE-DUPLICATE, R-UNION-OBJECT, R-UNION-SCALAR, R-UNION-SCALAR-OBJECT, R-NULL-REQUIRED, R-COERCE-VALUE] +notes: | + - `allOf: [BaseEntity]` puts `entity_id` and `created_by` before the direct properties. + - `location` merges two `oneOf` object branches. `storage_kind` is required in both, so it is non-nullable; it is + declared with different `const` values, which merge to `string`. + - `note` is `string | object`; the plain string `"plain text note"` becomes `{"_value": "plain text note"}`. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/data.json b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/data.json new file mode 100644 index 00000000..20b97616 --- /dev/null +++ b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/data.json @@ -0,0 +1,36 @@ +[ + { + "entity_id": "ent1", + "created_by": "pipeline", + "sample_name": "E. coli K-12", + "location": { + "storage_kind": "s3", + "bucket": "cdm-lake", + "key": "genomes/ecoli.fna.gz" + }, + "pagination": { + "start_page": "101", + "end_page": "115" + }, + "identifier_union": 12345, + "note": { + "text": "Structured note", + "author": "curator" + } + }, + { + "entity_id": "ent2", + "sample_name": "B. subtilis", + "location": { + "storage_kind": "http", + "url": "https://example.org/bsub.fna.gz", + "checksum": "sha256:abc123" + }, + "pagination": { + "medline_pgn": "e10042" + }, + "identifier_union": "GCF_000009045.1", + "note": "Plain text note" + } +] + diff --git a/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected.json b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected.json new file mode 100644 index 00000000..48402582 --- /dev/null +++ b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected.json @@ -0,0 +1,38 @@ +[ + { + "entity_id": "ent1", + "created_by": "pipeline", + "sample_name": "E. coli K-12", + "location": { + "storage_kind": "s3", + "bucket": "cdm-lake", + "key": "genomes/ecoli.fna.gz" + }, + "pagination": { + "start_page": "101", + "end_page": "115" + }, + "identifier_union": "12345", + "note": { + "text": "Structured note", + "author": "curator" + } + }, + { + "entity_id": "ent2", + "sample_name": "B. subtilis", + "location": { + "storage_kind": "http", + "url": "https://example.org/bsub.fna.gz", + "checksum": "sha256:abc123" + }, + "pagination": { + "medline_pgn": "e10042" + }, + "identifier_union": "GCF_000009045.1", + "note": { + "_value": "Plain text note" + } + } +] + diff --git a/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_schema.py b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_schema.py new file mode 100644 index 00000000..19dbbf88 --- /dev/null +++ b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_schema.py @@ -0,0 +1,55 @@ +"""Expected PyArrow schema for 07_combinators_allof_anyof_oneof.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field( + "entity_id", + pa.string(), + nullable=False, + metadata={"comment": "Inherited base entity identifier.", "jsonschema_type": "string"}, + ), + pa.field("created_by", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("sample_name", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "location", + pa.struct( + [ + pa.field("storage_kind", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("bucket", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("key", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("url", pa.string(), nullable=True, metadata={"jsonschema_type": "string (uri)"}), + pa.field("checksum", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + metadata={"comment": "Storage location choice across S3 or HTTP."}, + ), + pa.field( + "pagination", + pa.struct( + [ + pa.field("start_page", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("end_page", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("medline_pgn", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + ), + pa.field("identifier_union", pa.string(), nullable=True, metadata={"jsonschema_type": "integer | string"}), + pa.field( + "note", + pa.struct( + [ + pa.field("text", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("author", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("_value", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_table.py b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_table.py new file mode 100644 index 00000000..93439ba5 --- /dev/null +++ b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/expected_table.py @@ -0,0 +1,29 @@ +"""Expected PyArrow table for 07_combinators_allof_anyof_oneof.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "entity_id": "ent1", + "created_by": "pipeline", + "sample_name": "E. coli K-12", + "location": {"storage_kind": "s3", "bucket": "cdm-lake", "key": "genomes/ecoli.fna.gz"}, + "pagination": {"start_page": "101", "end_page": "115"}, + "identifier_union": "12345", + "note": {"text": "Structured note", "author": "curator"}, + }, + { + "entity_id": "ent2", + "sample_name": "B. subtilis", + "location": {"storage_kind": "http", "url": "https://example.org/bsub.fna.gz", "checksum": "sha256:abc123"}, + "pagination": {"medline_pgn": "e10042"}, + "identifier_union": "GCF_000009045.1", + "note": {"_value": "Plain text note"}, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/schema.json b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/schema.json new file mode 100644 index 00000000..a913a8f2 --- /dev/null +++ b/tests/data/jsonschema/spec/07_combinators_allof_anyof_oneof/schema.json @@ -0,0 +1,143 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/07_combinators_allof_anyof_oneof.schema.json", + "title": "CombinatorsRecord", + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/BaseEntity" + } + ], + "required": [ + "sample_name", + "location" + ], + "properties": { + "sample_name": { + "type": "string" + }, + "location": { + "description": "Storage location choice across S3 or HTTP.", + "oneOf": [ + { + "type": "object", + "required": [ + "storage_kind", + "bucket", + "key" + ], + "properties": { + "storage_kind": { + "type": "string", + "const": "s3" + }, + "bucket": { + "type": "string" + }, + "key": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "storage_kind", + "url" + ], + "properties": { + "storage_kind": { + "type": "string", + "const": "http" + }, + "url": { + "type": "string", + "format": "uri" + }, + "checksum": { + "type": "string" + } + } + } + ] + }, + "pagination": { + "anyOf": [ + { + "type": "object", + "required": [ + "start_page" + ], + "properties": { + "start_page": { + "type": "string" + }, + "end_page": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "medline_pgn" + ], + "properties": { + "medline_pgn": { + "type": "string" + } + } + } + ] + }, + "identifier_union": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [ + "text" + ], + "properties": { + "text": { + "type": "string" + }, + "author": { + "type": "string" + } + } + } + ] + } + }, + "$defs": { + "BaseEntity": { + "type": "object", + "required": [ + "entity_id" + ], + "properties": { + "entity_id": { + "type": "string", + "description": "Inherited base entity identifier." + }, + "created_by": { + "type": "string" + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/case.yaml b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/case.yaml new file mode 100644 index 00000000..3b20f392 --- /dev/null +++ b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/case.yaml @@ -0,0 +1,9 @@ +title: Conditional (`if`/`then`/`else`) and Dependent Schemas +schema: schema.json +rules: [R-NULL-CONDITIONAL, R-ORDER] +notes: | + `strain_name` (from `then`), `biome` (from `else`) and `platform` (from `dependentSchemas`) are nullable. + `depth_unit` appears only in `dependentRequired` and stays nullable. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/data.json b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/data.json new file mode 100644 index 00000000..911c132a --- /dev/null +++ b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/data.json @@ -0,0 +1,16 @@ +[ + { + "id": "cond1", + "specimen_type": "isolate", + "sequencing_depth": 120.5, + "depth_unit": "x", + "platform": "Illumina NovaSeq", + "strain_name": "K-12 MG1655" + }, + { + "id": "cond2", + "specimen_type": "metagenome", + "biome": "marine pelagic" + } +] + diff --git a/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected.json b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected.json new file mode 100644 index 00000000..911c132a --- /dev/null +++ b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected.json @@ -0,0 +1,16 @@ +[ + { + "id": "cond1", + "specimen_type": "isolate", + "sequencing_depth": 120.5, + "depth_unit": "x", + "platform": "Illumina NovaSeq", + "strain_name": "K-12 MG1655" + }, + { + "id": "cond2", + "specimen_type": "metagenome", + "biome": "marine pelagic" + } +] + diff --git a/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_schema.py b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_schema.py new file mode 100644 index 00000000..337f75a2 --- /dev/null +++ b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_schema.py @@ -0,0 +1,32 @@ +"""Expected PyArrow schema for 08_conditional_and_dependent_schemas.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("specimen_type", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("sequencing_depth", pa.float64(), nullable=True, metadata={"jsonschema_type": "number"}), + pa.field("depth_unit", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "platform", + pa.string(), + nullable=True, + metadata={"comment": "Sequencing platform when depth is reported.", "jsonschema_type": "string"}, + ), + pa.field( + "strain_name", + pa.string(), + nullable=True, + metadata={"comment": "Cultured isolate strain identifier.", "jsonschema_type": "string"}, + ), + pa.field( + "biome", + pa.string(), + nullable=True, + metadata={"comment": "Environmental biome for metagenome samples.", "jsonschema_type": "string"}, + ), + ] +) diff --git a/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_table.py b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_table.py new file mode 100644 index 00000000..438a1779 --- /dev/null +++ b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/expected_table.py @@ -0,0 +1,21 @@ +"""Expected PyArrow table for 08_conditional_and_dependent_schemas.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "cond1", + "specimen_type": "isolate", + "sequencing_depth": 120.5, + "depth_unit": "x", + "platform": "Illumina NovaSeq", + "strain_name": "K-12 MG1655", + }, + {"id": "cond2", "specimen_type": "metagenome", "biome": "marine pelagic"}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/schema.json b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/schema.json new file mode 100644 index 00000000..765b6b5c --- /dev/null +++ b/tests/data/jsonschema/spec/08_conditional_and_dependent_schemas/schema.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/08_conditional_and_dependent_schemas.schema.json", + "title": "ConditionalAndDependentRecord", + "type": "object", + "required": [ + "id", + "specimen_type" + ], + "properties": { + "id": { + "type": "string" + }, + "specimen_type": { + "type": "string", + "enum": [ + "isolate", + "metagenome" + ] + }, + "sequencing_depth": { + "type": "number" + }, + "depth_unit": { + "type": "string" + } + }, + "dependentRequired": { + "sequencing_depth": [ + "depth_unit" + ] + }, + "dependentSchemas": { + "sequencing_depth": { + "properties": { + "platform": { + "type": "string", + "description": "Sequencing platform when depth is reported." + } + } + } + }, + "if": { + "properties": { + "specimen_type": { + "const": "isolate" + } + } + }, + "then": { + "required": [ + "strain_name" + ], + "properties": { + "strain_name": { + "type": "string", + "description": "Cultured isolate strain identifier." + } + } + }, + "else": { + "required": [ + "biome" + ], + "properties": { + "biome": { + "type": "string", + "description": "Environmental biome for metagenome samples." + } + } + } +} + diff --git a/tests/data/jsonschema/spec/09_context_dependent_shared_defs/case.yaml b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/case.yaml new file mode 100644 index 00000000..ab84eece --- /dev/null +++ b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/case.yaml @@ -0,0 +1,10 @@ +title: Context-Dependent Shared Definitions +schema: schema.json +rules: [R-SHAPE-REF, R-NULL-REQUIRED, R-NULL-ALLOWS-NULL, R-NULL-ITEMS, R-META-COMMENT] +notes: | + One `#/$defs/Tag` used as a required property (`primary_tag`), an optional property (`secondary_tag`), a + required-but-nullable `anyOf` property (`fallback_tag`), an array item (`tag_list`) and a map value + (`tag_lookup`). +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/09_context_dependent_shared_defs/data.json b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/data.json new file mode 100644 index 00000000..9d30e4b2 --- /dev/null +++ b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/data.json @@ -0,0 +1,41 @@ +[ + { + "id": "ctx1", + "primary_tag": { + "key": "domain", + "value": "Bacteria" + }, + "secondary_tag": { + "key": "phylum", + "value": "Proteobacteria" + }, + "fallback_tag": { + "key": "status", + "value": "curated" + }, + "tag_list": [ + { + "key": "gram", + "value": "negative" + }, + { + "key": "motile" + } + ], + "tag_lookup": { + "ncbi": { + "key": "taxid", + "value": "562" + } + } + }, + { + "id": "ctx2", + "primary_tag": { + "key": "domain", + "value": "Archaea" + }, + "fallback_tag": null + } +] + diff --git a/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected.json b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected.json new file mode 100644 index 00000000..9d30e4b2 --- /dev/null +++ b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected.json @@ -0,0 +1,41 @@ +[ + { + "id": "ctx1", + "primary_tag": { + "key": "domain", + "value": "Bacteria" + }, + "secondary_tag": { + "key": "phylum", + "value": "Proteobacteria" + }, + "fallback_tag": { + "key": "status", + "value": "curated" + }, + "tag_list": [ + { + "key": "gram", + "value": "negative" + }, + { + "key": "motile" + } + ], + "tag_lookup": { + "ncbi": { + "key": "taxid", + "value": "562" + } + } + }, + { + "id": "ctx2", + "primary_tag": { + "key": "domain", + "value": "Archaea" + }, + "fallback_tag": null + } +] + diff --git a/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_schema.py b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_schema.py new file mode 100644 index 00000000..0b591dd3 --- /dev/null +++ b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_schema.py @@ -0,0 +1,61 @@ +"""Expected PyArrow schema for 09_context_dependent_shared_defs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +_TAG_STRUCT = pa.struct( + [ + pa.field("key", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("value", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] +) + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field( + "primary_tag", + _TAG_STRUCT, + nullable=False, + metadata={"comment": "Reusable key-value tag structure."}, + ), + pa.field( + "secondary_tag", + _TAG_STRUCT, + nullable=True, + metadata={"comment": "Reusable key-value tag structure."}, + ), + pa.field( + "fallback_tag", + _TAG_STRUCT, + nullable=True, + metadata={"comment": "Reusable key-value tag structure."}, + ), + pa.field( + "tag_list", + pa.list_( + pa.field( + "item", + _TAG_STRUCT, + nullable=False, + metadata={"comment": "Reusable key-value tag structure."}, + ), + ), + nullable=True, + ), + pa.field( + "tag_lookup", + pa.map_( + pa.string(), + pa.field( + "value", + _TAG_STRUCT, + nullable=False, + metadata={"comment": "Reusable key-value tag structure."}, + ), + ), + nullable=True, + ), + ] +) diff --git a/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_table.py b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_table.py new file mode 100644 index 00000000..4ace8401 --- /dev/null +++ b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/expected_table.py @@ -0,0 +1,21 @@ +"""Expected PyArrow table for 09_context_dependent_shared_defs.""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "id": "ctx1", + "primary_tag": {"key": "domain", "value": "Bacteria"}, + "secondary_tag": {"key": "phylum", "value": "Proteobacteria"}, + "fallback_tag": {"key": "status", "value": "curated"}, + "tag_list": [{"key": "gram", "value": "negative"}, {"key": "motile"}], + "tag_lookup": {"ncbi": {"key": "taxid", "value": "562"}}, + }, + {"id": "ctx2", "primary_tag": {"key": "domain", "value": "Archaea"}, "fallback_tag": None}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/09_context_dependent_shared_defs/schema.json b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/schema.json new file mode 100644 index 00000000..bdef7892 --- /dev/null +++ b/tests/data/jsonschema/spec/09_context_dependent_shared_defs/schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/09_context_dependent_shared_defs.schema.json", + "title": "ContextDependentSharedDefsRecord", + "type": "object", + "required": [ + "id", + "primary_tag", + "fallback_tag" + ], + "properties": { + "id": { + "type": "string" + }, + "primary_tag": { + "$ref": "#/$defs/Tag" + }, + "secondary_tag": { + "$ref": "#/$defs/Tag" + }, + "fallback_tag": { + "anyOf": [ + { + "$ref": "#/$defs/Tag" + }, + { + "type": "null" + } + ] + }, + "tag_list": { + "type": "array", + "items": { + "$ref": "#/$defs/Tag" + } + }, + "tag_lookup": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/Tag" + } + } + }, + "$defs": { + "Tag": { + "type": "object", + "description": "Reusable key-value tag structure.", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/10_combinations/case.yaml b/tests/data/jsonschema/spec/10_combinations/case.yaml new file mode 100644 index 00000000..af343f0a --- /dev/null +++ b/tests/data/jsonschema/spec/10_combinations/case.yaml @@ -0,0 +1,10 @@ +title: Combinations +schema: schema.json +rules: [R-ORDER, R-SCALAR-TYPE, R-UNION-SCALAR, R-UNION-OBJECT, R-SHAPE-TUPLE, R-SHAPE-MAP, R-RECURSION, R-NULL-CONDITIONAL, R-SCALAR-ENUM-CONST, R-COERCE-SCALAR] +notes: | + Stress test combining `allOf` inheritance, `const`, date / date-time / Base64 scalars, a nullable + `["number", "string", "null"]` union, a closed tuple, a string map, a `oneOf` object choice, a recursive + `TaxNode` and a `dependentSchemas` property. +runs: + - id: default + data: data.json diff --git a/tests/data/jsonschema/spec/10_combinations/data.json b/tests/data/jsonschema/spec/10_combinations/data.json new file mode 100644 index 00000000..70f6ae33 --- /dev/null +++ b/tests/data/jsonschema/spec/10_combinations/data.json @@ -0,0 +1,65 @@ +[ + { + "schema_version": 1, + "record_id": "comb1", + "active": true, + "count": 42, + "ratio": 0.875, + "release_date": "2025-08-15", + "timestamp_str": "2025-08-15T09:00:00Z", + "payload_b64": "Q0RN", + "payload_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "score_or_label": 98.5, + "range_tuple": [ + 100, + 500, + "+" + ], + "tags": [ + "complete", + "reference" + ], + "annotations": { + "bioproject": "PRJNA225", + "biosample": "SAMN02604091" + }, + "source_choice": { + "kind": "genbank", + "accession": "GCF_000005845.2" + }, + "lineage_tree": { + "taxon_name": "Bacteria", + "rank": "domain", + "children": [ + { + "taxon_name": "Proteobacteria", + "rank": "phylum", + "children": [ + { + "taxon_name": "Gammaproteobacteria", + "rank": "class" + } + ] + } + ] + } + }, + { + "schema_version": 1, + "record_id": "comb2", + "active": false, + "count": 0, + "ratio": 0.0, + "release_date": "2025-01-01", + "score_or_label": null, + "range_tuple": [ + 1, + 10 + ], + "source_choice": { + "kind": "publication", + "doi": "10.1038/s41586-025-00001-x" + } + } +] + diff --git a/tests/data/jsonschema/spec/10_combinations/expected.json b/tests/data/jsonschema/spec/10_combinations/expected.json new file mode 100644 index 00000000..7317e240 --- /dev/null +++ b/tests/data/jsonschema/spec/10_combinations/expected.json @@ -0,0 +1,62 @@ +[ + { + "schema_version": 1, + "record_id": "comb1", + "active": true, + "count": 42, + "ratio": 0.875, + "release_date": "2025-08-15", + "timestamp_str": "2025-08-15T09:00:00Z", + "payload_b64": "CDM", + "payload_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "score_or_label": "98.5", + "range_tuple": { + "item_0": 100, + "item_1": 500, + "item_2": "+" + }, + "tags": [ + "complete", + "reference" + ], + "annotations": { + "bioproject": "PRJNA225", + "biosample": "SAMN02604091" + }, + "source_choice": { + "kind": "genbank", + "accession": "GCF_000005845.2" + }, + "lineage_tree": { + "taxon_name": "Bacteria", + "rank": "domain", + "children": [ + { + "taxon_name": "Proteobacteria", + "rank": "phylum", + "children": [ + "{\"taxon_name\": \"Gammaproteobacteria\", \"rank\": \"class\"}" + ] + } + ] + } + }, + { + "schema_version": 1, + "record_id": "comb2", + "active": false, + "count": 0, + "ratio": 0.0, + "release_date": "2025-01-01", + "score_or_label": null, + "range_tuple": { + "item_0": 1, + "item_1": 10 + }, + "source_choice": { + "kind": "publication", + "doi": "10.1038/s41586-025-00001-x" + } + } +] + diff --git a/tests/data/jsonschema/spec/10_combinations/expected_schema.py b/tests/data/jsonschema/spec/10_combinations/expected_schema.py new file mode 100644 index 00000000..aa0be3eb --- /dev/null +++ b/tests/data/jsonschema/spec/10_combinations/expected_schema.py @@ -0,0 +1,105 @@ +"""Expected PyArrow schema for 10_combinations.""" + +# ruff: noqa: INP001 + +import pyarrow as pa + +EXPECTED_SCHEMA = pa.schema( + [ + pa.field("schema_version", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field( + "record_id", + pa.string(), + nullable=False, + metadata={"comment": "Primary record identifier.", "jsonschema_type": "string"}, + ), + pa.field("active", pa.bool_(), nullable=False, metadata={"jsonschema_type": "boolean"}), + pa.field("count", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("ratio", pa.float64(), nullable=False, metadata={"jsonschema_type": "number"}), + pa.field("release_date", pa.date32(), nullable=False, metadata={"jsonschema_type": "string (date)"}), + pa.field("timestamp_str", pa.string(), nullable=True, metadata={"jsonschema_type": "string (date-time)"}), + pa.field("payload_b64", pa.binary(), nullable=True, metadata={"jsonschema_type": "string (base64)"}), + pa.field("score_or_label", pa.string(), nullable=True, metadata={"jsonschema_type": "number | string"}), + pa.field( + "range_tuple", + pa.struct( + [ + pa.field("item_0", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("item_1", pa.int64(), nullable=False, metadata={"jsonschema_type": "integer"}), + pa.field("item_2", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=False, + ), + pa.field( + "tags", + pa.list_( + pa.field("item", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=True, + ), + pa.field( + "annotations", + pa.map_( + pa.string(), + pa.field("value", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + ), + nullable=True, + ), + pa.field( + "source_choice", + pa.struct( + [ + pa.field("kind", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("accession", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field("doi", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] + ), + nullable=True, + ), + pa.field( + "lineage_tree", + pa.struct( + [ + pa.field("taxon_name", pa.string(), nullable=False, metadata={"jsonschema_type": "string"}), + pa.field("rank", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + pa.field( + "children", + pa.list_( + pa.field( + "item", + pa.struct( + [ + pa.field( + "taxon_name", + pa.string(), + nullable=False, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "rank", + pa.string(), + nullable=True, + metadata={"jsonschema_type": "string"}, + ), + pa.field( + "children", + pa.list_( + pa.field("item", pa.string(), nullable=False), + ), + nullable=True, + ), + ] + ), + nullable=False, + ), + ), + nullable=True, + ), + ] + ), + nullable=True, + ), + pa.field("payload_sha256", pa.string(), nullable=True, metadata={"jsonschema_type": "string"}), + ] +) diff --git a/tests/data/jsonschema/spec/10_combinations/expected_table.py b/tests/data/jsonschema/spec/10_combinations/expected_table.py new file mode 100644 index 00000000..89ca125c --- /dev/null +++ b/tests/data/jsonschema/spec/10_combinations/expected_table.py @@ -0,0 +1,52 @@ +"""Expected PyArrow table for 10_combinations.""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "schema_version": 1, + "record_id": "comb1", + "active": True, + "count": 42, + "ratio": 0.875, + "release_date": dt.date(2025, 8, 15), + "timestamp_str": "2025-08-15T09:00:00Z", + "payload_b64": b"CDM", + "payload_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "score_or_label": "98.5", + "range_tuple": {"item_0": 100, "item_1": 500, "item_2": "+"}, + "tags": ["complete", "reference"], + "annotations": {"bioproject": "PRJNA225", "biosample": "SAMN02604091"}, + "source_choice": {"kind": "genbank", "accession": "GCF_000005845.2"}, + "lineage_tree": { + "taxon_name": "Bacteria", + "rank": "domain", + "children": [ + { + "taxon_name": "Proteobacteria", + "rank": "phylum", + "children": ['{"taxon_name": "Gammaproteobacteria", "rank": "class"}'], + } + ], + }, + }, + { + "schema_version": 1, + "record_id": "comb2", + "active": False, + "count": 0, + "ratio": 0.0, + "release_date": dt.date(2025, 1, 1), + "score_or_label": None, + "range_tuple": {"item_0": 1, "item_1": 10}, + "source_choice": {"kind": "publication", "doi": "10.1038/s41586-025-00001-x"}, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/jsonschema/spec/10_combinations/schema.json b/tests/data/jsonschema/spec/10_combinations/schema.json new file mode 100644 index 00000000..f28dd4e4 --- /dev/null +++ b/tests/data/jsonschema/spec/10_combinations/schema.json @@ -0,0 +1,165 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.org/spec/10_combinations.schema.json", + "title": "ComprehensiveCombinationsRecord", + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/AuditHeader" + } + ], + "required": [ + "record_id", + "active", + "count", + "ratio", + "release_date", + "range_tuple" + ], + "properties": { + "record_id": { + "type": "string", + "description": "Primary record identifier." + }, + "active": { + "type": "boolean" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "ratio": { + "type": "number", + "multipleOf": 0.001 + }, + "release_date": { + "type": "string", + "format": "date" + }, + "timestamp_str": { + "type": "string", + "format": "date-time" + }, + "payload_b64": { + "type": "string", + "contentEncoding": "base64" + }, + "score_or_label": { + "type": [ + "number", + "string", + "null" + ] + }, + "range_tuple": { + "type": "array", + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "items": false + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "source_choice": { + "oneOf": [ + { + "type": "object", + "required": [ + "kind", + "accession" + ], + "properties": { + "kind": { + "type": "string", + "const": "genbank" + }, + "accession": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "doi" + ], + "properties": { + "kind": { + "type": "string", + "const": "publication" + }, + "doi": { + "type": "string" + } + } + } + ] + }, + "lineage_tree": { + "$ref": "#/$defs/TaxNode" + } + }, + "dependentSchemas": { + "payload_b64": { + "properties": { + "payload_sha256": { + "type": "string" + } + } + } + }, + "$defs": { + "AuditHeader": { + "type": "object", + "required": [ + "schema_version" + ], + "properties": { + "schema_version": { + "const": 1 + } + } + }, + "TaxNode": { + "type": "object", + "required": [ + "taxon_name" + ], + "properties": { + "taxon_name": { + "type": "string" + }, + "rank": { + "type": "string" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/TaxNode" + } + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/array_top_level.json b/tests/data/jsonschema/spec/errors/array_top_level.json new file mode 100644 index 00000000..5fc847f9 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/array_top_level.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "integer" + } +} + diff --git a/tests/data/jsonschema/spec/errors/boolean_property_schema.json b/tests/data/jsonschema/spec/errors/boolean_property_schema.json new file mode 100644 index 00000000..81e6c089 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/boolean_property_schema.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "flag": true + } +} diff --git a/tests/data/jsonschema/spec/errors/const_null_field.json b/tests/data/jsonschema/spec/errors/const_null_field.json new file mode 100644 index 00000000..fb661515 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/const_null_field.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "nothing": { + "const": null + } + } +} diff --git a/tests/data/jsonschema/spec/errors/dynamic_ref.json b/tests/data/jsonschema/spec/errors/dynamic_ref.json new file mode 100644 index 00000000..f5c89a82 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/dynamic_ref.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "node": { + "$dynamicRef": "#node" + } + } +} diff --git a/tests/data/jsonschema/spec/errors/empty_object_nested.json b/tests/data/jsonschema/spec/errors/empty_object_nested.json new file mode 100644 index 00000000..87830963 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/empty_object_nested.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "empty_child": { + "type": "object", + "properties": {} + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/empty_object_top_level.json b/tests/data/jsonschema/spec/errors/empty_object_top_level.json new file mode 100644 index 00000000..5a0e11e1 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/empty_object_top_level.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {} +} + diff --git a/tests/data/jsonschema/spec/errors/enum_null_field.json b/tests/data/jsonschema/spec/errors/enum_null_field.json new file mode 100644 index 00000000..fc7ad914 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/enum_null_field.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "nothing": { + "enum": [ + null + ] + } + } +} diff --git a/tests/data/jsonschema/spec/errors/incompatible_branch_shapes.json b/tests/data/jsonschema/spec/errors/incompatible_branch_shapes.json new file mode 100644 index 00000000..8cfe1331 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/incompatible_branch_shapes.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "payload": { + "oneOf": [ + { + "type": "object", + "properties": { + "a": { + "type": "string" + } + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/manifest.yaml b/tests/data/jsonschema/spec/errors/manifest.yaml new file mode 100644 index 00000000..09115d2d --- /dev/null +++ b/tests/data/jsonschema/spec/errors/manifest.yaml @@ -0,0 +1,89 @@ +# Error specification fixtures for the JSON Schema -> PyArrow converter. +# `code` must match the `code` of a JSONSchemaToPyArrowError subclass in +# cdm_data_loaders.converters.jsonschema_to_pyarrow. `file` is relative to this directory. +# `match` is a regex applied to the exception message. +errors: + - file: missing_schema_dialect.json + code: E1 + match: \$schema|2020-12|dialect + description: Declares the Draft-07 dialect URI. + - file: missing_schema_keyword.json + code: E1 + match: \$schema|2020-12|dialect + description: No `$schema` keyword. + - file: scalar_top_level.json + code: E2 + match: object|top-level|root + description: 'Root is `{"type": "string"}`.' + - file: array_top_level.json + code: E2 + match: object|top-level|root + description: 'Root is `{"type": "array", "items": {"type": "integer"}}`.' + - file: map_top_level.json + code: E2 + match: map|properties|top-level|root + description: 'Root is a pure map (`additionalProperties: {"type": "string"}`).' + - file: empty_object_top_level.json + code: E3 + match: empty|properties + description: 'Root object has `properties: {}`.' + - file: empty_object_nested.json + code: E3 + match: empty|properties + description: 'Nested property has `properties: {}`.' + - file: unconstrained_property.json + code: E4 + match: unconstrained|type|boolean + description: 'Property schema is `{}`.' + - file: boolean_property_schema.json + code: E4 + match: unconstrained|type|boolean + description: 'Property schema is `true`.' + - file: open_tuple_prefix_items.json + code: E5 + match: prefixItems|open tuple|items + description: '`prefixItems` with `items: {"type": "boolean"}`.' + - file: mixed_properties_and_map.json + code: E6 + match: additionalProperties|patternProperties|mixed + description: '`properties` together with `additionalProperties: {"type": "integer"}`.' + - file: negation_not_keyword.json + code: E7 + match: not + description: 'Property uses `"not": {"type": "string"}`.' + - file: incompatible_branch_shapes.json + code: E8 + match: incompatible|branch|oneOf|anyOf + description: '`oneOf` with an object branch and an array branch.' + - file: unresolvable_ref.json + code: E9 + match: \$ref|DoesNotExist|unresolvable + description: '`$ref` to the missing `#/$defs/DoesNotExist`.' + - file: dynamic_ref.json + code: E9 + match: \$dynamicRef|dynamic|ref + description: 'Property uses `$dynamicRef`.' + - file: recursive_date_type.json + code: E10 + match: recursive + description: 'Recursive `DatedNode` has a `format: "date"` property.' + - file: recursive_binary_type.json + code: E10 + match: recursive + description: 'Recursive `BlobNode` has a `contentEncoding: "base64"` property.' + - file: null_only_field.json + code: E11 + match: "null" + description: 'Property is `{"type": "null"}`.' + - file: const_null_field.json + code: E11 + match: "null" + description: 'Property is `{"const": null}`.' + - file: enum_null_field.json + code: E11 + match: "null" + description: 'Property is `{"enum": [null]}`.' + - file: value_field_collision.json + code: E12 + match: _value|collision|scalar|object + description: Object branch of a string + object `anyOf` declares `_value`. diff --git a/tests/data/jsonschema/spec/errors/map_top_level.json b/tests/data/jsonschema/spec/errors/map_top_level.json new file mode 100644 index 00000000..fcf8952a --- /dev/null +++ b/tests/data/jsonschema/spec/errors/map_top_level.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": { + "type": "string" + } +} + diff --git a/tests/data/jsonschema/spec/errors/missing_schema_dialect.json b/tests/data/jsonschema/spec/errors/missing_schema_dialect.json new file mode 100644 index 00000000..e57f34f9 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/missing_schema_dialect.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/missing_schema_keyword.json b/tests/data/jsonschema/spec/errors/missing_schema_keyword.json new file mode 100644 index 00000000..6b84d794 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/missing_schema_keyword.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + } + } +} diff --git a/tests/data/jsonschema/spec/errors/mixed_properties_and_map.json b/tests/data/jsonschema/spec/errors/mixed_properties_and_map.json new file mode 100644 index 00000000..2f4bdea7 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/mixed_properties_and_map.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "hybrid": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "additionalProperties": { + "type": "integer" + } + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/negation_not_keyword.json b/tests/data/jsonschema/spec/errors/negation_not_keyword.json new file mode 100644 index 00000000..cfe001c0 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/negation_not_keyword.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "non_string": { + "not": { + "type": "string" + } + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/null_only_field.json b/tests/data/jsonschema/spec/errors/null_only_field.json new file mode 100644 index 00000000..d93e76d0 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/null_only_field.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "always_null": { + "type": "null" + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/open_tuple_prefix_items.json b/tests/data/jsonschema/spec/errors/open_tuple_prefix_items.json new file mode 100644 index 00000000..08a63073 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/open_tuple_prefix_items.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "open_tuple": { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "items": { + "type": "boolean" + } + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/recursive_binary_type.json b/tests/data/jsonschema/spec/errors/recursive_binary_type.json new file mode 100644 index 00000000..ede3618d --- /dev/null +++ b/tests/data/jsonschema/spec/errors/recursive_binary_type.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "node": { + "$ref": "#/$defs/BlobNode" + } + }, + "$defs": { + "BlobNode": { + "type": "object", + "properties": { + "payload": { + "type": "string", + "contentEncoding": "base64" + }, + "child": { + "$ref": "#/$defs/BlobNode" + } + } + } + } +} diff --git a/tests/data/jsonschema/spec/errors/recursive_date_type.json b/tests/data/jsonschema/spec/errors/recursive_date_type.json new file mode 100644 index 00000000..306b2417 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/recursive_date_type.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "node": { + "$ref": "#/$defs/DatedNode" + } + }, + "$defs": { + "DatedNode": { + "type": "object", + "properties": { + "created_on": { + "type": "string", + "format": "date" + }, + "child": { + "$ref": "#/$defs/DatedNode" + } + } + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/scalar_top_level.json b/tests/data/jsonschema/spec/errors/scalar_top_level.json new file mode 100644 index 00000000..e4cd28b5 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/scalar_top_level.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} + diff --git a/tests/data/jsonschema/spec/errors/unconstrained_property.json b/tests/data/jsonschema/spec/errors/unconstrained_property.json new file mode 100644 index 00000000..bd92b9f4 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/unconstrained_property.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "anything": {} + } +} + diff --git a/tests/data/jsonschema/spec/errors/unresolvable_ref.json b/tests/data/jsonschema/spec/errors/unresolvable_ref.json new file mode 100644 index 00000000..68ac7250 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/unresolvable_ref.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "missing": { + "$ref": "#/$defs/DoesNotExist" + } + } +} + diff --git a/tests/data/jsonschema/spec/errors/value_field_collision.json b/tests/data/jsonschema/spec/errors/value_field_collision.json new file mode 100644 index 00000000..71f26bf9 --- /dev/null +++ b/tests/data/jsonschema/spec/errors/value_field_collision.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "_value": { + "type": "string" + }, + "author": { + "type": "string" + } + } + } + ] + } + } +} diff --git a/tests/data/xsd/examples/pubmed_article/case.yaml b/tests/data/xsd/examples/pubmed_article/case.yaml new file mode 100644 index 00000000..713e3e14 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/case.yaml @@ -0,0 +1,18 @@ +title: PubMed (`PubmedArticle`) +xml_tag: PubmedArticle +schema: ../../../pubmed/pubmed.xsd +rules: [R-SHAPE-MIXED, R-RECURSION, R-SHAPE-EMPTY-MARKER, R-NULL-CHOICE] +notes: | + DTD-derived schema with mixed-content titles and abstracts, the `(StartPage, EndPage?, MedlinePgn?) | MedlinePgn` + pagination choice and element-`ref` self-recursion. +runs: + - id: minimal + data: minimal.xml + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.xml + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/xsd/examples/pubmed_article/maximal_expected_table.py b/tests/data/xsd/examples/pubmed_article/maximal_expected_table.py new file mode 100644 index 00000000..ebf01bf1 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/maximal_expected_table.py @@ -0,0 +1,583 @@ +"""Expected PyArrow table for pubmed_article maximal.xml (xml_tag='PubmedArticle').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "MedlineCitation": { + "_Owner": "NLM", + "_Status": "MEDLINE", + "_VersionID": "1", + "_VersionDate": "2024-05-29", + "_IndexingMethod": "Automated", + "PMID": {"_Version": "1", "#text": "38123456"}, + "DateCompleted": {"Year": "2024", "Month": "05", "Day": "29"}, + "DateRevised": {"Year": "2025", "Month": "01", "Day": "16"}, + "Article": { + "_PubModel": "Print-Electronic", + "Journal": { + "ISSN": {"_IssnType": "Electronic", "#text": "1476-4687"}, + "JournalIssue": { + "_CitedMedium": "Internet", + "Volume": "629", + "Issue": "8012", + "PubDate": {"Year": "2023", "Month": "09", "Day": "15"}, + }, + "Title": "Nature", + "ISOAbbreviation": "Nature", + }, + "ArticleTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Structural basis of TP53 tetramerization and Ca2+ signaling", + }, + "Pagination": {"StartPage": "410", "EndPage": "418", "MedlinePgn": "410-418"}, + "ELocationID": [ + {"_EIdType": "doi", "_ValidYN": "Y", "#text": "10.1038/s41586-024-07123-x"}, + {"_EIdType": "pii", "_ValidYN": "N", "#text": "S0028-0836(24)00123-4"}, + ], + "Abstract": { + "AbstractText": [ + { + "_Label": "BACKGROUND", + "_NlmCategory": "BACKGROUND", + "#text": "The tumor suppressor p53 regulates transcription in response to <5 Gy stress and H2O2.", + }, + { + "_Label": "METHODS", + "_NlmCategory": "METHODS", + "#text": "We undertook 20 semi-structured interviews with participants aged >18 years studying Ca2+ signaling & DNA repair and underlined text.", + }, + {"_Label": "UNLABELLED", "_NlmCategory": "UNASSIGNED"}, + ], + "CopyrightInformation": "Copyright (c) 2024 Springer Nature Limited.", + }, + "AuthorList": { + "_CompleteYN": "Y", + "_Type": "authors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "LastName": "Morris", + "ForeName": "Charlotte", + "Initials": "C", + "Suffix": "Jr", + "Identifier": [ + {"_Source": "ORCID", "#text": "0000-0002-9047-0481"}, + {"_Source": "ISNI", "#text": "0000-0001-6450-5815"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom.", + "Identifier": [ + {"_Source": "VIAF", "#text": "0000-0003-2804-1181"}, + {"_Source": "GND", "#text": "0000-0002-0939-9842"}, + ], + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom.", + "Identifier": [ + {"_Source": "RINGGOLD", "#text": "0000-0001-5784-7403"}, + {"_Source": "GRID", "#text": "0000-0002-1825-0097"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "International Cancer Genome Consortium", + }, + "Identifier": [ + {"_Source": "ROR", "#text": "https://ror.org/027m9bs27"}, + {"_Source": "ORCID (8)", "#text": "https://ror.org/01ej9dk98"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom.", + "Identifier": [ + {"_Source": "ISNI (9)", "#text": "https://ror.org/00za53h95"}, + {"_Source": "VIAF (10)", "#text": "https://ror.org/03vek6s52"}, + ], + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA.", + "Identifier": [ + {"_Source": "GND (11)", "#text": "0000-0002-9047-0481 (11)"}, + {"_Source": "RINGGOLD (12)", "#text": "0000-0001-6450-5815 (12)"}, + ], + }, + ], + }, + ], + }, + "Language": ["eng", "fre"], + "DataBankList": { + "_CompleteYN": "Y", + "DataBank": [ + {"DataBankName": "PDB", "AccessionNumberList": {"AccessionNumber": ["1TUP", "2OCJ"]}}, + { + "DataBankName": "GenBank", + "AccessionNumberList": {"AccessionNumber": ["NM_000546", "AF307851"]}, + }, + ], + }, + "GrantList": { + "_CompleteYN": "Y", + "Grant": [ + { + "GrantID": "R01-CA043460", + "Acronym": "CA", + "Agency": "NCI NIH HHS", + "Country": "United States", + }, + { + "GrantID": "MR/T021975/1", + "Acronym": "MRC", + "Agency": "Medical Research Council", + "Country": "United Kingdom", + }, + ], + }, + "PublicationTypeList": { + "PublicationType": [ + {"_UI": "D016428", "#text": "Journal Article"}, + {"_UI": "D013485", "#text": "Research Support, Non-U.S. Gov't"}, + ] + }, + "VernacularTitle": "Bases structurales de la tetramerisation de TP53", + "ArticleDate": [ + {"_DateType": "Electronic", "Year": "2022", "Month": "11", "Day": "04"}, + {"_DateType": "Electronic", "Year": "2021", "Month": "03", "Day": "21"}, + ], + }, + "MedlineJournalInfo": { + "Country": "Belgium", + "MedlineTA": "Nature", + "NlmUniqueID": "0410462", + "ISSNLinking": "0028-0836", + }, + "ChemicalList": { + "Chemical": [ + { + "RegistryNumber": "EC 2.7.11.1", + "NameOfSubstance": {"_UI": "Sample_UI_1", "#text": "Tumor Suppressor Protein p53"}, + }, + { + "RegistryNumber": "EC 6.3.2.19", + "NameOfSubstance": {"_UI": "Sample_UI_2", "#text": "Proto-Oncogene Proteins c-mdm2"}, + }, + ] + }, + "SupplMeshList": { + "SupplMeshName": [ + {"_Type": "Disease", "_UI": "C536599", "#text": "Li-Fraumeni Syndrome"}, + {"_Type": "Protocol", "_UI": "C045812", "#text": "Chromatin Immunoprecipitation"}, + ] + }, + "CitationSubset": ["IM", "AIM"], + "CommentsCorrectionsList": { + "CommentsCorrections": [ + { + "_RefType": "Cites", + "RefSource": "Nature. 2000 Nov 16;408(6810):307-310.", + "PMID": {"_Version": "2", "#text": "11099028"}, + "Note": "Foundational review of the p53 signaling network.", + }, + { + "_RefType": "CommentIn", + "RefSource": "Science. 2024 Jun 7;384(6700):1020-1021.", + "PMID": {"_Version": "3", "#text": "38843210"}, + "Note": "Perspective highlighting TP53 tetramerization dynamics.", + }, + ] + }, + "GeneSymbolList": {"GeneSymbol": ["TP53", "MDM2"]}, + "MeshHeadingList": { + "MeshHeading": [ + { + "DescriptorName": { + "_MajorTopicYN": "Y", + "_AutoHM": "Y", + "_Type": "Geographic", + "_UI": "D016159", + "#text": "Tumor Suppressor Protein p53", + }, + "QualifierName": [ + {"_AutoHM": "Y", "_MajorTopicYN": "N", "_UI": "Q000235", "#text": "genetics"}, + {"_AutoHM": "Y", "_MajorTopicYN": "Y", "_UI": "Q000378", "#text": "metabolism"}, + ], + }, + { + "DescriptorName": { + "_MajorTopicYN": "N", + "_AutoHM": "Y", + "_Type": "Geographic", + "_UI": "D004260", + "#text": "DNA Repair", + }, + "QualifierName": [ + {"_AutoHM": "Y", "_MajorTopicYN": "N", "_UI": "Q000502", "#text": "physiology"}, + {"_AutoHM": "Y", "_MajorTopicYN": "Y", "_UI": "Q000187", "#text": "drug effects"}, + ], + }, + ] + }, + "NumberOfReferences": "42", + "PersonalNameSubjectList": { + "PersonalNameSubject": [ + {"LastName": "Kontopantelis", "ForeName": "Evangelos", "Initials": "E", "Suffix": "III"}, + {"LastName": "Robinson", "ForeName": "Louise", "Initials": "L", "Suffix": "Sr"}, + ] + }, + "OtherID": [ + {"_Source": "NASA", "#text": "NTRS-20240001234"}, + {"_Source": "PIP", "#text": "PIP-987654"}, + ], + "OtherAbstract": [ + { + "_Type": "Publisher", + "_Language": "eng", + "AbstractText": [ + { + "_Label": "RESULTS", + "_NlmCategory": "RESULTS", + "#text": "Tetramerization affinity increased in vitro by 3.5-fold (p < 0.001) upon phosphorylation.", + }, + { + "_Label": "CONCLUSIONS", + "_NlmCategory": "CONCLUSIONS", + "#text": "Clarifying TP53 & MDM2 kinetics improves targeted cancer therapy.", + }, + ], + "CopyrightInformation": "Copyright (c) 2024 The Authors. Published under CC-BY 4.0.", + }, + { + "_Type": "AAMC", + "_Language": "fre", + "AbstractText": [ + { + "_Label": "OBJECTIVE", + "_NlmCategory": "OBJECTIVE", + "#text": "French summary: Le facteur de transcription p53 maintient la stabilite du genome.", + }, + { + "_Label": "BACKGROUND (6)", + "_NlmCategory": "UNASSIGNED", + "#text": "Plain-language summary: How cells repair damaged DNA when exposed to &gt;2 Gy radiation.", + }, + ], + "CopyrightInformation": "Copyright (c) 2024 National Library of Medicine.", + }, + ], + "KeywordList": [ + { + "_Owner": "NOTNLM", + "Keyword": [ + {"_MajorTopicYN": "Y", "#text": "p53 signaling"}, + {"_MajorTopicYN": "N", "#text": "tetramerization"}, + ], + }, + { + "_Owner": "NLM", + "Keyword": [ + {"_MajorTopicYN": "Y", "#text": "DNA damage response"}, + {"_MajorTopicYN": "N", "#text": "apoptosis"}, + ], + }, + ], + "CoiStatement": "The authors declare no competing financial or non-financial interests.", + "SpaceFlightMission": ["ISS Expedition 68", "Artemis I BioSentinel"], + "InvestigatorList": [ + { + "_ID": "INV_1", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Ashcroft", + "ForeName": "Darren", + "Initials": "D", + "Suffix": "II", + "Identifier": [ + {"_Source": "GRID (13)", "#text": "0000-0003-2804-1181 (13)"}, + {"_Source": "ROR (14)", "#text": "0000-0002-0939-9842 (14)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom.", + "Identifier": [ + {"_Source": "ORCID (15)", "#text": "0000-0001-5784-7403 (15)"}, + {"_Source": "ISNI (16)", "#text": "0000-0002-1825-0097 (16)"}, + ], + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA.", + "Identifier": [ + {"_Source": "VIAF (17)", "#text": "https://ror.org/027m9bs27 (17)"}, + {"_Source": "GND (18)", "#text": "https://ror.org/01ej9dk98 (18)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "LastName": "Blakeman", + "ForeName": "Thomas", + "Initials": "T", + "Suffix": "IV", + "Identifier": [ + {"_Source": "RINGGOLD (19)", "#text": "https://ror.org/00za53h95 (19)"}, + {"_Source": "GRID (20)", "#text": "https://ror.org/03vek6s52 (20)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA.", + "Identifier": [ + {"_Source": "ROR (21)", "#text": "0000-0002-9047-0481 (21)"}, + {"_Source": "ORCID (22)", "#text": "0000-0001-6450-5815 (22)"}, + ], + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA.", + "Identifier": [ + {"_Source": "ISNI (23)", "#text": "0000-0003-2804-1181 (23)"}, + {"_Source": "VIAF (24)", "#text": "0000-0002-0939-9842 (24)"}, + ], + }, + ], + }, + ], + }, + { + "_ID": "INV_2", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Tucker", + "ForeName": "Joseph", + "Initials": "J", + "Suffix": "PhD", + "Identifier": [ + {"_Source": "GND (25)", "#text": "0000-0001-5784-7403 (25)"}, + {"_Source": "RINGGOLD (26)", "#text": "0000-0002-1825-0097 (26)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom. (9)", + "Identifier": [ + {"_Source": "GRID (27)", "#text": "https://ror.org/027m9bs27 (27)"}, + {"_Source": "ROR (28)", "#text": "https://ror.org/01ej9dk98 (28)"}, + ], + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (10)", + "Identifier": [ + {"_Source": "ORCID (29)", "#text": "https://ror.org/00za53h95 (29)"}, + {"_Source": "ISNI (30)", "#text": "https://ror.org/03vek6s52 (30)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "LastName": "Mullens", + "ForeName": "Amy", + "Initials": "A", + "Suffix": "MD", + "Identifier": [ + {"_Source": "VIAF (31)", "#text": "0000-0002-9047-0481 (31)"}, + {"_Source": "GND (32)", "#text": "0000-0001-6450-5815 (32)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (11)", + "Identifier": [ + {"_Source": "RINGGOLD (33)", "#text": "0000-0003-2804-1181 (33)"}, + {"_Source": "GRID (34)", "#text": "0000-0002-0939-9842 (34)"}, + ], + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (12)", + "Identifier": [ + {"_Source": "ROR (35)", "#text": "0000-0001-5784-7403 (35)"}, + {"_Source": "ORCID (36)", "#text": "0000-0002-1825-0097 (36)"}, + ], + }, + ], + }, + ], + }, + ], + "GeneralNote": [ + {"_Owner": "NLM", "#text": "Indexed by automated curation."}, + {"_Owner": "NASA", "#text": "Supported by NASA Space Biology grant 80NSSC22K0123."}, + ], + }, + "PubmedData": { + "History": { + "PubMedPubDate": [ + { + "_PubStatus": "received", + "Year": "2020", + "Month": "07", + "Day": "12", + "Hour": "14", + "Minute": "30", + "Second": "00", + }, + { + "_PubStatus": "revised", + "Year": "2019", + "Month": "02", + "Day": "08", + "Hour": "09", + "Minute": "15", + "Second": "25", + }, + ] + }, + "PublicationStatus": "ppublish", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pubmed", "#text": "38123456"}, + {"_IdType": "doi", "#text": "10.1038/s41586-024-07123-x"}, + ] + }, + "ObjectList": { + "Object": [ + { + "_Type": "SupplementaryMaterial", + "Param": [ + {"_Name": "id", "#text": "supp_table_1.pdf"}, + { + "_Name": "caption", + "#text": "Crystallographic data collection and refinement statistics", + }, + ], + }, + { + "_Type": "Dataset", + "Param": [ + {"_Name": "accession", "#text": "GSE245678"}, + {"_Name": "repository", "#text": "NCBI Gene Expression Omnibus"}, + ], + }, + ] + }, + "ReferenceList": [ + { + "Title": "Primary Literature", + "Reference": [ + { + "Citation": "Vogelstein B, Lane D, Levine AJ. Surfing the p53 network. Nature. 2000;408(6810):307-310.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmc", "#text": "PMC10987654"}, + {"_IdType": "pii", "#text": "S0028-0836(24)00123-4"}, + ] + }, + }, + { + "Citation": "Cho Y, Gorina S, Jeffrey PD, Pavletich NP. Crystal structure of a p53 tumor suppressor-DNA complex. Science. 1994;265(5170):346-355.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "mid", "#text": "11099028"}, + {"_IdType": "pmcid", "#text": "10.1038/35042675"}, + ] + }, + }, + ], + "ReferenceList": [ + { + "Title": "Structural Biology References", + "Reference": [ + { + "Citation": "Baker SJ, Fearon ER, Nigro JM, et al. Chromosome 17 deletions and p53 gene mutations in colorectal carcinomas. Science. 1989;244(4901):217-221.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmcpid", "#text": "7585406"}, + {"_IdType": "pmpid", "#text": "10.1126/science.7878469"}, + ] + }, + }, + { + "Citation": "Momand J, Zambetti GP, Olson DC, George D, Levine AJ. The mdm-2 oncogene product forms a complex with the p53 protein. Cell. 1992;69(7):1237-1245.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "sici", "#text": "2694717"}, + {"_IdType": "medline", "#text": "10.1016/0092-8674(89)90984-7"}, + ] + }, + }, + ], + "ReferenceList": [ + '{"Title": "Clinical &amp; Genetic Studies", "Reference": [{"Citation": "Kussie PH, Gorina S, Marechal V, et al. Structure of the MDM2 oncoprotein bound to the p53 tumor suppressor transactivation domain. Science. 1996;274(5289):948-953.", "ArticleIdList": {"ArticleId": [{"_IdType": "pmcbook", "#text": "8183552"}, {"_IdType": "bookaccession", "#text": "10.1126/science.8183552"}]}}, {"Citation": "Haupt Y, Maya R, Kazaz A, Oren M. Mdm2 promotes the rapid degradation of p53. Nature. 1997;387(6630):296-299.", "ArticleIdList": {"ArticleId": [{"_IdType": "pubmed", "#text": "9367981"}, {"_IdType": "doi", "#text": "10.1038/387296a0"}]}}]}' + ], + } + ], + }, + { + "Title": "Historical Background", + "Reference": [ + { + "Citation": "Cohen GN, Barbe V, Flament D, et al. An integrated analysis of the genome of Pyrococcus abyssi. Mol Microbiol. 2003;47(6):1495-1512.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmc", "#text": "10430560"}, + {"_IdType": "pii", "#text": "10.1093/genetics/152.4.1299"}, + ] + }, + }, + { + "Citation": "Erkel C, Kube M, Reinhardt R, Liesack W. Genome of rice cluster I archaea. Science. 2006;313(5785):370-372.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "mid", "#text": "12622808"}, + {"_IdType": "pmcid", "#text": "10.1046/j.1365-2958.2003.03381.x"}, + ] + }, + }, + ], + "ReferenceList": [ + { + "Title": "Methodological References", + "Reference": [ + { + "Citation": "Anderson IJ, Sieprawska-Lupa M, Lapidus A, et al. Complete genome sequence of Methanoculleus marisnigri JR1. Stand Genomic Sci. 2009;1(2):189-196.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmcpid", "#text": "16857943"}, + {"_IdType": "pmpid", "#text": "10.1126/science.1127062"}, + ] + }, + }, + { + "Citation": "Gao J, Wang J. Re-annotation of two hyperthermophilic archaea Pyrococcus abyssi & P. furiosus. Curr Microbiol. 2012;64(2):118-129.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "sici", "#text": "21304656"}, + {"_IdType": "medline", "#text": "10.4056/sigs.32535"}, + ] + }, + }, + ], + "ReferenceList": [ + '{"Title": "Supplementary Citations", "Reference": [{"Citation": "Morris C, Robinson L, Ashcroft D, et al. Primary care experiences of people with dementia. Br J Gen Pract. 2025;75(751):e102-e110.", "ArticleIdList": {"ArticleId": [{"_IdType": "pmcbook", "#text": "22057919"}, {"_IdType": "bookaccession", "#text": "10.1007/s00284-011-0035-x"}]}}, {"Citation": "Tucker J, Mullens A, Morroni C, Ong J. The HIV response at an inflection point: resilience &amp; equity. Sex Health. 2026;23(1):SH25236.", "ArticleIdList": {"ArticleId": [{"_IdType": "pubmed", "#text": "31234567"}, {"_IdType": "doi", "#text": "10.1016/j.cell.2019.05.012"}]}}]}' + ], + } + ], + }, + ], + }, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/pubmed_article/minimal_expected_table.py b/tests/data/xsd/examples/pubmed_article/minimal_expected_table.py new file mode 100644 index 00000000..5503e12f --- /dev/null +++ b/tests/data/xsd/examples/pubmed_article/minimal_expected_table.py @@ -0,0 +1,31 @@ +"""Expected PyArrow table for pubmed_article minimal.xml (xml_tag='PubmedArticle').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "MedlineCitation": { + "_Status": "MEDLINE", + "PMID": {"_Version": "1", "#text": "38123456"}, + "Article": { + "_PubModel": "Print-Electronic", + "Journal": { + "JournalIssue": {"_CitedMedium": "Internet", "PubDate": {"MedlineDate": "Sample_MedlineDate_1"}} + }, + "ArticleTitle": { + "#text": "Structural basis of TP53 tetramerization and Ca2+ signaling" + }, + "ELocationID": [{"_EIdType": "doi", "#text": "10.1038/s41586-024-07123-x"}], + "Language": ["eng"], + "PublicationTypeList": {"PublicationType": [{"_UI": "D016428", "#text": "Journal Article"}]}, + }, + "MedlineJournalInfo": {"MedlineTA": "Nature"}, + } + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/pubmed_book_article/case.yaml b/tests/data/xsd/examples/pubmed_book_article/case.yaml new file mode 100644 index 00000000..f34f81f4 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/case.yaml @@ -0,0 +1,18 @@ +title: PubMed (`PubmedBookArticle`) +xml_tag: PubmedBookArticle +schema: ../../../pubmed/pubmed.xsd +rules: [R-SHAPE-MIXED, R-RECURSION, R-SHAPE-EMPTY-MARKER] +notes: | + Book article variant of the PubMed schema, including recursive `Section` elements and the empty + `BookInProcess` marker. +runs: + - id: minimal + data: minimal.xml + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.xml + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/xsd/examples/pubmed_book_article/maximal_expected_table.py b/tests/data/xsd/examples/pubmed_book_article/maximal_expected_table.py new file mode 100644 index 00000000..f5ac6804 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/maximal_expected_table.py @@ -0,0 +1,705 @@ +"""Expected PyArrow table for pubmed_book_article maximal.xml (xml_tag='PubmedBookArticle').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "BookDocument": { + "PMID": {"_Version": "1", "#text": "38123456"}, + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pubmed", "#text": "38123456"}, + {"_IdType": "doi", "#text": "10.1038/s41586-024-07123-x"}, + ] + }, + "Book": { + "Publisher": { + "PublisherName": "Cold Spring Harbor Laboratory Press", + "PublisherLocation": "Cold Spring Harbor (NY)", + }, + "BookTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Molecular Biology of the Cell, 7th Edition", + }, + "PubDate": {"Year": "2024", "Month": "05", "Day": "29"}, + "BeginningDate": {"Year": "2025", "Month": "01", "Day": "16"}, + "EndingDate": {"Year": "2023", "Month": "09", "Day": "15"}, + "AuthorList": [ + { + "_CompleteYN": "Y", + "_Type": "authors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "LastName": "Morris", + "ForeName": "Charlotte", + "Initials": "C", + "Suffix": "Jr", + "Identifier": [ + {"_Source": "ORCID", "#text": "0000-0002-9047-0481"}, + {"_Source": "ISNI", "#text": "0000-0001-6450-5815"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom.", + "Identifier": [ + {"_Source": "VIAF", "#text": "0000-0003-2804-1181"}, + {"_Source": "GND", "#text": "0000-0002-0939-9842"}, + ], + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom.", + "Identifier": [ + {"_Source": "RINGGOLD", "#text": "0000-0001-5784-7403"}, + {"_Source": "GRID", "#text": "0000-0002-1825-0097"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "International Cancer Genome Consortium", + }, + "Identifier": [ + {"_Source": "ROR", "#text": "https://ror.org/027m9bs27"}, + {"_Source": "ORCID (8)", "#text": "https://ror.org/01ej9dk98"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom.", + "Identifier": [ + {"_Source": "ISNI (9)", "#text": "https://ror.org/00za53h95"}, + {"_Source": "VIAF (10)", "#text": "https://ror.org/03vek6s52"}, + ], + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA.", + "Identifier": [ + {"_Source": "GND (11)", "#text": "0000-0002-9047-0481 (11)"}, + {"_Source": "RINGGOLD (12)", "#text": "0000-0001-6450-5815 (12)"}, + ], + }, + ], + }, + ], + }, + { + "_CompleteYN": "N", + "_Type": "editors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "UK Primary Care Dementia Research Group", + }, + "Identifier": [ + {"_Source": "GRID (13)", "#text": "0000-0003-2804-1181 (13)"}, + {"_Source": "ROR (14)", "#text": "0000-0002-0939-9842 (14)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom.", + "Identifier": [ + {"_Source": "ORCID (15)", "#text": "0000-0001-5784-7403 (15)"}, + {"_Source": "ISNI (16)", "#text": "0000-0002-1825-0097 (16)"}, + ], + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA.", + "Identifier": [ + {"_Source": "VIAF (17)", "#text": "https://ror.org/027m9bs27 (17)"}, + {"_Source": "GND (18)", "#text": "https://ror.org/01ej9dk98 (18)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "LastName": "Kontopantelis", + "ForeName": "Evangelos", + "Initials": "E", + "Suffix": "III", + "Identifier": [ + {"_Source": "RINGGOLD (19)", "#text": "https://ror.org/00za53h95 (19)"}, + {"_Source": "GRID (20)", "#text": "https://ror.org/03vek6s52 (20)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA.", + "Identifier": [ + {"_Source": "ROR (21)", "#text": "0000-0002-9047-0481 (21)"}, + {"_Source": "ORCID (22)", "#text": "0000-0001-6450-5815 (22)"}, + ], + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA.", + "Identifier": [ + {"_Source": "ISNI (23)", "#text": "0000-0003-2804-1181 (23)"}, + {"_Source": "VIAF (24)", "#text": "0000-0002-0939-9842 (24)"}, + ], + }, + ], + }, + ], + }, + ], + "InvestigatorList": { + "_ID": "INV_1", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Robinson", + "ForeName": "Louise", + "Initials": "L", + "Suffix": "Sr", + "Identifier": [ + {"_Source": "GND (25)", "#text": "0000-0001-5784-7403 (25)"}, + {"_Source": "RINGGOLD (26)", "#text": "0000-0002-1825-0097 (26)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom. (9)", + "Identifier": [ + {"_Source": "GRID (27)", "#text": "https://ror.org/027m9bs27 (27)"}, + {"_Source": "ROR (28)", "#text": "https://ror.org/01ej9dk98 (28)"}, + ], + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (10)", + "Identifier": [ + {"_Source": "ORCID (29)", "#text": "https://ror.org/00za53h95 (29)"}, + {"_Source": "ISNI (30)", "#text": "https://ror.org/03vek6s52 (30)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "LastName": "Ashcroft", + "ForeName": "Darren", + "Initials": "D", + "Suffix": "II", + "Identifier": [ + {"_Source": "VIAF (31)", "#text": "0000-0002-9047-0481 (31)"}, + {"_Source": "GND (32)", "#text": "0000-0001-6450-5815 (32)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (11)", + "Identifier": [ + {"_Source": "RINGGOLD (33)", "#text": "0000-0003-2804-1181 (33)"}, + {"_Source": "GRID (34)", "#text": "0000-0002-0939-9842 (34)"}, + ], + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (12)", + "Identifier": [ + {"_Source": "ROR (35)", "#text": "0000-0001-5784-7403 (35)"}, + {"_Source": "ORCID (36)", "#text": "0000-0002-1825-0097 (36)"}, + ], + }, + ], + }, + ], + }, + "Volume": "629", + "VolumeTitle": "Part I: Introduction to the Cell", + "Edition": "7th", + "CollectionTitle": { + "_book": "nbk21054", + "_part": "seriesA", + "_sec": "vol1", + "#text": "NCBI Bookshelf Classics", + }, + "Isbn": ["978-0-8153-4432-2", "978-0-8153-4464-3"], + "ELocationID": [ + {"_EIdType": "doi", "_ValidYN": "Y", "#text": "10.1038/s41586-024-07123-x"}, + {"_EIdType": "pii", "_ValidYN": "N", "#text": "S0028-0836(24)00123-4"}, + ], + "Medium": "Internet", + "ReportNumber": "NBK21054", + }, + "LocationLabel": [{"_Type": "chapter", "#text": "Chapter 1"}, {"_Type": "section", "#text": "Part I"}], + "ArticleTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Structural basis of TP53 tetramerization and Ca2+ signaling", + }, + "VernacularTitle": "Bases structurales de la tetramerisation de TP53", + "Pagination": {"StartPage": "410", "EndPage": "418", "MedlinePgn": "410-418"}, + "Language": ["eng", "fre"], + "AuthorList": [ + { + "_CompleteYN": "Y", + "_Type": "authors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "LastName": "Blakeman", + "ForeName": "Thomas", + "Initials": "T", + "Suffix": "IV", + "Identifier": [ + {"_Source": "ISNI (37)", "#text": "https://ror.org/027m9bs27 (37)"}, + {"_Source": "VIAF (38)", "#text": "https://ror.org/01ej9dk98 (38)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. (13)", + "Identifier": [ + {"_Source": "GND (39)", "#text": "https://ror.org/00za53h95 (39)"}, + {"_Source": "RINGGOLD (40)", "#text": "https://ror.org/03vek6s52 (40)"}, + ], + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. (14)", + "Identifier": [ + {"_Source": "GRID (41)", "#text": "0000-0002-9047-0481 (41)"}, + {"_Source": "ROR (42)", "#text": "0000-0001-6450-5815 (42)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "Asia-Pacific HIV Prevention Network", + }, + "Identifier": [ + {"_Source": "ORCID (43)", "#text": "0000-0003-2804-1181 (43)"}, + {"_Source": "ISNI (44)", "#text": "0000-0002-0939-9842 (44)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. (15)", + "Identifier": [ + {"_Source": "VIAF (45)", "#text": "0000-0001-5784-7403 (45)"}, + {"_Source": "GND (46)", "#text": "0000-0002-1825-0097 (46)"}, + ], + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA. (16)", + "Identifier": [ + {"_Source": "RINGGOLD (47)", "#text": "https://ror.org/027m9bs27 (47)"}, + {"_Source": "GRID (48)", "#text": "https://ror.org/01ej9dk98 (48)"}, + ], + }, + ], + }, + ], + }, + { + "_CompleteYN": "N", + "_Type": "editors", + "Author": [ + { + "_ValidYN": "Y", + "_EqualContrib": "Y", + "CollectiveName": { + "_Investigators": "INV_1", + "#text": "European TP53 Structural Biology Consortium", + }, + "Identifier": [ + {"_Source": "ROR (49)", "#text": "https://ror.org/00za53h95 (49)"}, + {"_Source": "ORCID (50)", "#text": "https://ror.org/03vek6s52 (50)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "The University of Manchester, Division of Population Health, Manchester, United Kingdom. (17)", + "Identifier": [ + {"_Source": "ISNI (51)", "#text": "0000-0002-9047-0481 (51)"}, + {"_Source": "VIAF (52)", "#text": "0000-0001-6450-5815 (52)"}, + ], + }, + { + "Affiliation": "National Institute for Health and Care Research School for Primary Care Research, Manchester, United Kingdom. (18)", + "Identifier": [ + {"_Source": "GND (53)", "#text": "0000-0003-2804-1181 (53)"}, + {"_Source": "RINGGOLD (54)", "#text": "0000-0002-0939-9842 (54)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "_EqualContrib": "N", + "LastName": "Tucker", + "ForeName": "Joseph", + "Initials": "J", + "Suffix": "PhD", + "Identifier": [ + {"_Source": "GRID (55)", "#text": "0000-0001-5784-7403 (55)"}, + {"_Source": "ROR (56)", "#text": "0000-0002-1825-0097 (56)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Newcastle University Faculty of Medical Sciences, Population Health Sciences Institute, Newcastle upon Tyne, United Kingdom. (19)", + "Identifier": [ + {"_Source": "ORCID (57)", "#text": "https://ror.org/027m9bs27 (57)"}, + {"_Source": "ISNI (58)", "#text": "https://ror.org/01ej9dk98 (58)"}, + ], + }, + { + "Affiliation": "Sidney Kimmel Comprehensive Cancer Center, Johns Hopkins University, Baltimore, MD, USA. (20)", + "Identifier": [ + {"_Source": "VIAF (59)", "#text": "https://ror.org/00za53h95 (59)"}, + {"_Source": "GND (60)", "#text": "https://ror.org/03vek6s52 (60)"}, + ], + }, + ], + }, + ], + }, + ], + "InvestigatorList": { + "_ID": "INV_2", + "Investigator": [ + { + "_ValidYN": "Y", + "LastName": "Mullens", + "ForeName": "Amy", + "Initials": "A", + "Suffix": "MD", + "Identifier": [ + {"_Source": "RINGGOLD (61)", "#text": "0000-0002-9047-0481 (61)"}, + {"_Source": "GRID (62)", "#text": "0000-0001-6450-5815 (62)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "MRC Laboratory of Molecular Biology, Francis Crick Avenue, Cambridge, United Kingdom. (21)", + "Identifier": [ + {"_Source": "ROR (63)", "#text": "0000-0003-2804-1181 (63)"}, + {"_Source": "ORCID (64)", "#text": "0000-0002-0939-9842 (64)"}, + ], + }, + { + "Affiliation": "Cold Spring Harbor Laboratory, One Bungtown Road, Cold Spring Harbor, NY, USA. (22)", + "Identifier": [ + {"_Source": "ISNI (65)", "#text": "0000-0001-5784-7403 (65)"}, + {"_Source": "VIAF (66)", "#text": "0000-0002-1825-0097 (66)"}, + ], + }, + ], + }, + { + "_ValidYN": "N", + "LastName": "Morroni", + "ForeName": "Chelsea", + "Initials": "CM", + "Suffix": "FRS", + "Identifier": [ + {"_Source": "GND (67)", "#text": "https://ror.org/027m9bs27 (67)"}, + {"_Source": "RINGGOLD (68)", "#text": "https://ror.org/01ej9dk98 (68)"}, + ], + "AffiliationInfo": [ + { + "Affiliation": "Institute for Advanced Study, Einstein Drive, Princeton, NJ, USA. (23)", + "Identifier": [ + {"_Source": "GRID (69)", "#text": "https://ror.org/00za53h95 (69)"}, + {"_Source": "ROR (70)", "#text": "https://ror.org/03vek6s52 (70)"}, + ], + }, + { + "Affiliation": "Department of Molecular Biology, Princeton University, Princeton, NJ, USA. (24)", + "Identifier": [ + {"_Source": "ORCID (71)", "#text": "0000-0002-9047-0481 (71)"}, + {"_Source": "ISNI (72)", "#text": "0000-0001-6450-5815 (72)"}, + ], + }, + ], + }, + ], + }, + "PublicationType": [ + {"_UI": "D016428", "#text": "Journal Article"}, + {"_UI": "D013485", "#text": "Research Support, Non-U.S. Gov't"}, + ], + "Abstract": { + "AbstractText": [ + { + "_Label": "BACKGROUND", + "_NlmCategory": "BACKGROUND", + "#text": "The tumor suppressor p53 regulates transcription in response to <5 Gy stress and H2O2.", + }, + { + "_Label": "METHODS", + "_NlmCategory": "METHODS", + "#text": "We undertook 20 semi-structured interviews with participants aged >18 years studying Ca2+ signaling & DNA repair and underlined text.", + }, + {"_Label": "UNLABELLED", "_NlmCategory": "UNASSIGNED"}, + ], + "CopyrightInformation": "Copyright (c) 2024 Springer Nature Limited.", + }, + "Sections": { + "Section": [ + { + "LocationLabel": {"_Type": "part", "#text": "Chapter 1"}, + "SectionTitle": { + "_book": "nbk21054", + "_part": "part1", + "_sec": "sec1", + "#text": "Chapter 1: Cells and Genomes & Evolution", + }, + "Section": [ + { + "LocationLabel": {"_Type": "appendix", "#text": "Section 1.1"}, + "SectionTitle": { + "_book": "nbk21055", + "_part": "part1", + "_sec": "sec1.1", + "#text": "Section 1.1: The Universal Features of Cells on Earth", + }, + "Section": [ + '{"LocationLabel": {"_Type": "table", "#text": "Subsection 1.1.1"}, "SectionTitle": {"_book": "nbk21056", "_part": "part1", "_sec": "sec1.1.1", "#text": "Subsection 1.1.1: All Cells Store Their Hereditary Information in DNA"}}' + ], + } + ], + }, + { + "LocationLabel": {"_Type": "figure", "#text": "Chapter 2"}, + "SectionTitle": { + "_book": "nbk21057", + "_part": "part2", + "_sec": "sec2", + "#text": "Chapter 2: Cell Chemistry and Bioenergetics", + }, + "Section": [ + { + "LocationLabel": {"_Type": "box", "#text": "Section 2.1"}, + "SectionTitle": { + "_book": "nbk21058", + "_part": "part2", + "_sec": "sec2.1", + "#text": "Section 2.1: The Chemical Components of a Cell & CO2 Fixation", + }, + "Section": [ + '{"LocationLabel": {"_Type": "chapter", "#text": "Subsection 2.1.1"}, "SectionTitle": {"_book": "nbk21059", "_part": "part2", "_sec": "sec2.1.1", "#text": "Subsection 2.1.1: Chemical Bonds and Ca2+ Ions"}}' + ], + } + ], + }, + ] + }, + "KeywordList": [ + { + "_Owner": "NOTNLM", + "Keyword": [ + {"_MajorTopicYN": "Y", "#text": "p53 signaling"}, + {"_MajorTopicYN": "N", "#text": "tetramerization"}, + ], + }, + { + "_Owner": "NLM", + "Keyword": [ + {"_MajorTopicYN": "Y", "#text": "DNA damage response"}, + {"_MajorTopicYN": "N", "#text": "apoptosis"}, + ], + }, + ], + "ContributionDate": {"Year": "2022", "Month": "11", "Day": "04"}, + "DateRevised": {"Year": "2021", "Month": "03", "Day": "21"}, + "GrantList": { + "_CompleteYN": "Y", + "Grant": [ + { + "GrantID": "R01-CA043460", + "Acronym": "CA", + "Agency": "NCI NIH HHS", + "Country": "United States", + }, + { + "GrantID": "MR/T021975/1", + "Acronym": "MRC", + "Agency": "Medical Research Council", + "Country": "United Kingdom", + }, + ], + }, + "ItemList": [ + {"_ListType": "list", "Item": ["G1/S DNA damage checkpoint", "G2/M mitotic spindle checkpoint"]}, + { + "_ListType": "ordered", + "Item": [ + "Ataxia-telangiectasia mutated (ATM) kinase activation", + "CHK2-mediated phosphorylation of Ser-20", + ], + }, + ], + "ReferenceList": [ + { + "Title": "Nature", + "Reference": [ + { + "Citation": "Vogelstein B, Lane D, Levine AJ. Surfing the p53 network. Nature. 2000;408(6810):307-310.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmc", "#text": "PMC10987654"}, + {"_IdType": "pii", "#text": "S0028-0836(24)00123-4"}, + ] + }, + }, + { + "Citation": "Cho Y, Gorina S, Jeffrey PD, Pavletich NP. Crystal structure of a p53 tumor suppressor-DNA complex. Science. 1994;265(5170):346-355.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "mid", "#text": "11099028"}, + {"_IdType": "pmcid", "#text": "10.1038/35042675"}, + ] + }, + }, + ], + "ReferenceList": [ + { + "Title": "Primary Literature", + "Reference": [ + { + "Citation": "Baker SJ, Fearon ER, Nigro JM, et al. Chromosome 17 deletions and p53 gene mutations in colorectal carcinomas. Science. 1989;244(4901):217-221.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmcpid", "#text": "7585406"}, + {"_IdType": "pmpid", "#text": "10.1126/science.7878469"}, + ] + }, + }, + { + "Citation": "Momand J, Zambetti GP, Olson DC, George D, Levine AJ. The mdm-2 oncogene product forms a complex with the p53 protein. Cell. 1992;69(7):1237-1245.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "sici", "#text": "2694717"}, + {"_IdType": "medline", "#text": "10.1016/0092-8674(89)90984-7"}, + ] + }, + }, + ], + "ReferenceList": [ + '{"Title": "Structural Biology References", "Reference": [{"Citation": "Kussie PH, Gorina S, Marechal V, et al. Structure of the MDM2 oncoprotein bound to the p53 tumor suppressor transactivation domain. Science. 1996;274(5289):948-953.", "ArticleIdList": {"ArticleId": [{"_IdType": "pmcbook", "#text": "8183552"}, {"_IdType": "bookaccession", "#text": "10.1126/science.8183552"}]}}, {"Citation": "Haupt Y, Maya R, Kazaz A, Oren M. Mdm2 promotes the rapid degradation of p53. Nature. 1997;387(6630):296-299.", "ArticleIdList": {"ArticleId": [{"_IdType": "pubmed", "#text": "9367981"}, {"_IdType": "doi", "#text": "10.1038/387296a0"}]}}]}' + ], + } + ], + }, + { + "Title": "Clinical &amp; Genetic Studies", + "Reference": [ + { + "Citation": "Cohen GN, Barbe V, Flament D, et al. An integrated analysis of the genome of Pyrococcus abyssi. Mol Microbiol. 2003;47(6):1495-1512.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmc", "#text": "10430560"}, + {"_IdType": "pii", "#text": "10.1093/genetics/152.4.1299"}, + ] + }, + }, + { + "Citation": "Erkel C, Kube M, Reinhardt R, Liesack W. Genome of rice cluster I archaea. Science. 2006;313(5785):370-372.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "mid", "#text": "12622808"}, + {"_IdType": "pmcid", "#text": "10.1046/j.1365-2958.2003.03381.x"}, + ] + }, + }, + ], + "ReferenceList": [ + { + "Title": "Historical Background", + "Reference": [ + { + "Citation": "Anderson IJ, Sieprawska-Lupa M, Lapidus A, et al. Complete genome sequence of Methanoculleus marisnigri JR1. Stand Genomic Sci. 2009;1(2):189-196.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmcpid", "#text": "16857943"}, + {"_IdType": "pmpid", "#text": "10.1126/science.1127062"}, + ] + }, + }, + { + "Citation": "Gao J, Wang J. Re-annotation of two hyperthermophilic archaea Pyrococcus abyssi & P. furiosus. Curr Microbiol. 2012;64(2):118-129.", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "sici", "#text": "21304656"}, + {"_IdType": "medline", "#text": "10.4056/sigs.32535"}, + ] + }, + }, + ], + "ReferenceList": [ + '{"Title": "Methodological References", "Reference": [{"Citation": "Morris C, Robinson L, Ashcroft D, et al. Primary care experiences of people with dementia. Br J Gen Pract. 2025;75(751):e102-e110.", "ArticleIdList": {"ArticleId": [{"_IdType": "pmcbook", "#text": "22057919"}, {"_IdType": "bookaccession", "#text": "10.1007/s00284-011-0035-x"}]}}, {"Citation": "Tucker J, Mullens A, Morroni C, Ong J. The HIV response at an inflection point: resilience &amp; equity. Sex Health. 2026;23(1):SH25236.", "ArticleIdList": {"ArticleId": [{"_IdType": "pubmed", "#text": "31234567"}, {"_IdType": "doi", "#text": "10.1016/j.cell.2019.05.012"}]}}]}' + ], + } + ], + }, + ], + }, + "PubmedBookData": { + "History": { + "PubMedPubDate": [ + { + "_PubStatus": "received", + "Year": "2020", + "Month": "07", + "Day": "12", + "Hour": "14", + "Minute": "30", + "Second": "00", + }, + { + "_PubStatus": "revised", + "Year": "2019", + "Month": "02", + "Day": "08", + "Hour": "09", + "Minute": "15", + "Second": "25", + }, + ] + }, + "PublicationStatus": "ppublish", + "ArticleIdList": { + "ArticleId": [ + {"_IdType": "pmc", "#text": "38123456 (27)"}, + {"_IdType": "pii", "#text": "10.1038/s41586-024-07123-x (28)"}, + ] + }, + "ObjectList": { + "Object": [ + { + "_Type": "SupplementaryMaterial", + "Param": [ + {"_Name": "id", "#text": "supp_table_1.pdf"}, + { + "_Name": "caption", + "#text": "Crystallographic data collection and refinement statistics", + }, + ], + }, + { + "_Type": "Dataset", + "Param": [ + {"_Name": "accession", "#text": "GSE245678"}, + {"_Name": "repository", "#text": "NCBI Gene Expression Omnibus"}, + ], + }, + ] + }, + }, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/pubmed_book_article/minimal_expected_table.py b/tests/data/xsd/examples/pubmed_book_article/minimal_expected_table.py new file mode 100644 index 00000000..80e1efa7 --- /dev/null +++ b/tests/data/xsd/examples/pubmed_book_article/minimal_expected_table.py @@ -0,0 +1,23 @@ +"""Expected PyArrow table for pubmed_book_article minimal.xml (xml_tag='PubmedBookArticle').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "BookDocument": { + "PMID": {"_Version": "1", "#text": "38123456"}, + "ArticleIdList": {"ArticleId": [{"#text": "38123456"}]}, + "Book": { + "Publisher": {"PublisherName": "Cold Spring Harbor Laboratory Press"}, + "BookTitle": {"#text": "Molecular Biology of the Cell, 7th Edition"}, + "PubDate": {"MedlineDate": "Sample_MedlineDate_1"}, + }, + } + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/uniprot/case.yaml b/tests/data/xsd/examples/uniprot/case.yaml new file mode 100644 index 00000000..aafa40d0 --- /dev/null +++ b/tests/data/xsd/examples/uniprot/case.yaml @@ -0,0 +1,18 @@ +title: UniProtKB +xml_tag: entry +schema: ../../../uniprot/uniprot_kb/uniprot.xsd +rules: [R-SHAPE-XSLIST, R-SCALAR-UNION, R-NIL, R-NULL-NILLABLE-PARENT, R-META-COMMENT] +notes: | + Exercises `intListType` (`xs:list`), the `date | gYearMonth | gYear` union and nillable `comment` elements + with attributes. +runs: + - id: minimal + data: minimal.xml + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.xml + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/xsd/examples/uniprot/maximal_expected_table.py b/tests/data/xsd/examples/uniprot/maximal_expected_table.py new file mode 100644 index 00000000..413b0b6d --- /dev/null +++ b/tests/data/xsd/examples/uniprot/maximal_expected_table.py @@ -0,0 +1,1269 @@ +"""Expected PyArrow table for uniprot maximal.xml (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_dataset": "Swiss-Prot", + "_created": dt.date(1986, 8, 13), + "_modified": dt.date(2024, 5, 29), + "_version": 285, + "accession": ["P04637", "Q9V2L2"], + "name": ["P53_HUMAN", "1A1D_PYRAB"], + "protein": { + "recommendedName": { + "fullName": {"_evidence": [1, 2], "#text": "Cellular tumor antigen p53"}, + "shortName": [{"_evidence": [1, 3], "#text": "p53"}, {"_evidence": [2, 3], "#text": "TP53-alpha"}], + "ecNumber": [ + {"_evidence": [1, 4], "#text": "2.7.11.1"}, + {"_evidence": [2, 4], "#text": "3.5.99.7"}, + ], + }, + "alternativeName": [ + { + "fullName": {"_evidence": [3, 4], "#text": "Antigen NY-CO-13"}, + "shortName": [ + {"_evidence": [1, 2, 3], "#text": "NY-CO-13"}, + {"_evidence": [1, 2, 4], "#text": "Pp53"}, + ], + "ecNumber": [ + {"_evidence": [2, 3, 4], "#text": "3.1.3.16"}, + {"_evidence": [1, 5], "#text": "3.1.3.48"}, + ], + }, + { + "fullName": {"_evidence": [2, 5], "#text": "Phosphoprotein p53"}, + "shortName": [ + {"_evidence": [3, 5], "#text": "TRP53"}, + {"_evidence": [4, 6, 9], "#text": "p53-iso1"}, + ], + "ecNumber": [ + {"_evidence": [5, 7, 1], "#text": "2.7.10.2"}, + {"_evidence": [6, 8, 2], "#text": "2.7.12.1"}, + ], + }, + ], + "submittedName": [ + { + "fullName": {"_evidence": [7, 9, 3], "#text": "Cellular phosphoprotein p53"}, + "ecNumber": [ + {"_evidence": [8, 1, 4], "#text": "2.7.11.22"}, + {"_evidence": [9, 2, 5], "#text": "2.7.11.24"}, + ], + }, + { + "fullName": {"_evidence": [1, 3, 6], "#text": "Mutant-type p53 antigen"}, + "ecNumber": [ + {"_evidence": [2, 4, 7], "#text": "3.2.2.6"}, + {"_evidence": [3, 5, 8], "#text": "3.2.2.20"}, + ], + }, + ], + "allergenName": {"_evidence": [4, 6, 9], "#text": "Hom s p53"}, + "biotechName": {"_evidence": [5, 7, 1], "#text": "Recombinant human p53 (rHu-p53)"}, + "cdAntigenName": [ + {"_evidence": [6, 8, 2], "#text": "CD-p53a"}, + {"_evidence": [7, 9, 3], "#text": "CD-p53b"}, + ], + "innName": [ + {"_evidence": [8, 1, 4], "#text": "p53-alfa"}, + {"_evidence": [9, 2, 5], "#text": "p53-beta"}, + ], + "domain": [ + { + "recommendedName": { + "fullName": {"_evidence": [1, 3, 6], "#text": "DNA-binding core domain"}, + "shortName": [ + {"_evidence": [2, 4, 7], "#text": "p53-DBD"}, + {"_evidence": [3, 5, 8], "#text": "DBD-core"}, + ], + "ecNumber": [ + {"_evidence": [4, 6, 9], "#text": "3.2.2.-"}, + {"_evidence": [5, 7, 1], "#text": "3.1.3.16"}, + ], + }, + "alternativeName": [ + { + "fullName": {"_evidence": [6, 8, 2], "#text": "Tumor suppressor p53"}, + "shortName": [ + {"_evidence": [7, 9, 3], "#text": "CD-A1"}, + {"_evidence": [8, 1, 4], "#text": "CD-A2"}, + ], + "ecNumber": [ + {"_evidence": [9, 2, 5], "#text": "3.4.19.12"}, + {"_evidence": [1, 3, 6], "#text": "3.4.22.68"}, + ], + }, + { + "fullName": {"_evidence": [2, 4, 7], "#text": "Transformation-related protein 53"}, + "shortName": [ + {"_evidence": [3, 5, 8], "#text": "CD-B1"}, + {"_evidence": [4, 6, 9], "#text": "CD-B2"}, + ], + "ecNumber": [ + {"_evidence": [5, 7, 1], "#text": "2.4.2.30"}, + {"_evidence": [6, 8, 2], "#text": "2.1.1.43"}, + ], + }, + ], + "submittedName": [ + { + "fullName": {"_evidence": [7, 9, 3], "#text": "Zinc-coordinated DNA binding region"}, + "ecNumber": [ + {"_evidence": [8, 1, 4], "#text": "3.5.1.98"}, + {"_evidence": [9, 2, 5], "#text": "3.5.1.104"}, + ], + }, + { + "fullName": {"_evidence": [1, 3, 6], "#text": "Conserved box II-V domain"}, + "ecNumber": [ + {"_evidence": [2, 4, 7], "#text": "6.3.2.25"}, + {"_evidence": [3, 5, 8], "#text": "6.3.2.27"}, + ], + }, + ], + "allergenName": {"_evidence": [4, 6, 9], "#text": "Hom s p53-DBD"}, + "biotechName": {"_evidence": [5, 7, 1], "#text": "Engineered p53 core domain"}, + "cdAntigenName": [ + {"_evidence": [6, 8, 2], "#text": "CD-DBD1"}, + {"_evidence": [7, 9, 3], "#text": "CD-DBD2"}, + ], + "innName": [ + {"_evidence": [8, 1, 4], "#text": "p53-gamma"}, + {"_evidence": [9, 2, 5], "#text": "p53-delta"}, + ], + }, + { + "recommendedName": { + "fullName": {"_evidence": [1, 3, 6], "#text": "Tetramerization domain"}, + "shortName": [ + {"_evidence": [2, 4, 7], "#text": "p53-TD"}, + {"_evidence": [3, 5, 8], "#text": "TD-domain"}, + ], + "ecNumber": [ + {"_evidence": [4, 6, 9], "#text": "2.3.1.48"}, + {"_evidence": [5, 7, 1], "#text": "3.4.21.-"}, + ], + }, + "alternativeName": [ + { + "fullName": {"_evidence": [6, 8, 2], "#text": "Core domain isoform A"}, + "shortName": [ + {"_evidence": [7, 9, 3], "#text": "OR-1a"}, + {"_evidence": [8, 1, 4], "#text": "OR-1b"}, + ], + "ecNumber": [ + {"_evidence": [9, 2, 5], "#text": "1.1.1.1"}, + {"_evidence": [1, 3, 6], "#text": "1.2.1.12"}, + ], + }, + { + "fullName": {"_evidence": [2, 4, 7], "#text": "Core domain isoform B"}, + "shortName": [ + {"_evidence": [3, 5, 8], "#text": "OR-2a"}, + {"_evidence": [4, 6, 9], "#text": "OR-2b"}, + ], + "ecNumber": [ + {"_evidence": [5, 7, 1], "#text": "3.1.3.16 (11)"}, + {"_evidence": [6, 8, 2], "#text": "3.1.3.48 (12)"}, + ], + }, + ], + "submittedName": [ + { + "fullName": {"_evidence": [7, 9, 3], "#text": "C-terminal tetramerization helix"}, + "ecNumber": [ + {"_evidence": [8, 1, 4], "#text": "2.7.1.37"}, + {"_evidence": [9, 2, 5], "#text": "2.7.4.3"}, + ], + }, + { + "fullName": {"_evidence": [1, 3, 6], "#text": "Nuclear export signal region"}, + "ecNumber": [ + {"_evidence": [2, 4, 7], "#text": "2.7.11.22 (11)"}, + {"_evidence": [3, 5, 8], "#text": "2.7.11.24 (12)"}, + ], + }, + ], + "allergenName": {"_evidence": [4, 6, 9], "#text": "Hom s p53-TD"}, + "biotechName": {"_evidence": [5, 7, 1], "#text": "Stabilized p53 tetramer domain"}, + "cdAntigenName": [ + {"_evidence": [6, 8, 2], "#text": "CD-TD1"}, + {"_evidence": [7, 9, 3], "#text": "CD-TD2"}, + ], + "innName": [ + {"_evidence": [8, 1, 4], "#text": "p53-epsilon"}, + {"_evidence": [9, 2, 5], "#text": "p53-zeta"}, + ], + }, + ], + "component": [ + { + "recommendedName": { + "fullName": {"_evidence": [1, 3, 6], "#text": "Cleaved p53 N-terminal peptide"}, + "shortName": [ + {"_evidence": [2, 4, 7], "#text": "p53-N"}, + {"_evidence": [3, 5, 8], "#text": "N-pep"}, + ], + "ecNumber": [ + {"_evidence": [4, 6, 9], "#text": "2.7.7.7"}, + {"_evidence": [5, 7, 1], "#text": "3.6.4.12"}, + ], + }, + "alternativeName": [ + { + "fullName": {"_evidence": [6, 8, 2], "#text": "Oligomerization region 1"}, + "shortName": [ + {"_evidence": [7, 9, 3], "#text": "TA-1a"}, + {"_evidence": [8, 1, 4], "#text": "TA-1b"}, + ], + "ecNumber": [ + {"_evidence": [9, 2, 5], "#text": "2.7.10.2 (13)"}, + {"_evidence": [1, 3, 6], "#text": "2.7.12.1 (14)"}, + ], + }, + { + "fullName": {"_evidence": [2, 4, 7], "#text": "Oligomerization region 2"}, + "shortName": [ + {"_evidence": [3, 5, 8], "#text": "TA-2a"}, + {"_evidence": [4, 6, 9], "#text": "TA-2b"}, + ], + "ecNumber": [ + {"_evidence": [5, 7, 1], "#text": "3.4.19.12 (15)"}, + {"_evidence": [6, 8, 2], "#text": "3.4.22.68 (16)"}, + ], + }, + ], + "submittedName": [ + { + "fullName": {"_evidence": [7, 9, 3], "#text": "N-terminal TAD1 peptide"}, + "ecNumber": [ + {"_evidence": [8, 1, 4], "#text": "3.2.2.6 (13)"}, + {"_evidence": [9, 2, 5], "#text": "3.2.2.20 (14)"}, + ], + }, + { + "fullName": {"_evidence": [1, 3, 6], "#text": "N-terminal TAD2 peptide"}, + "ecNumber": [ + {"_evidence": [2, 4, 7], "#text": "3.5.1.98 (15)"}, + {"_evidence": [3, 5, 8], "#text": "3.5.1.104 (16)"}, + ], + }, + ], + "allergenName": {"_evidence": [4, 6, 9], "#text": "Hom s p53-N"}, + "biotechName": {"_evidence": [5, 7, 1], "#text": "Synthetic TAD peptide"}, + "cdAntigenName": [ + {"_evidence": [6, 8, 2], "#text": "CD-TAD1"}, + {"_evidence": [7, 9, 3], "#text": "CD-TAD2"}, + ], + "innName": [ + {"_evidence": [8, 1, 4], "#text": "p53-eta"}, + {"_evidence": [9, 2, 5], "#text": "p53-theta"}, + ], + }, + { + "recommendedName": { + "fullName": {"_evidence": [1, 3, 6], "#text": "Cleaved p53 C-terminal regulatory chain"}, + "shortName": [ + {"_evidence": [2, 4, 7], "#text": "p53-C"}, + {"_evidence": [3, 5, 8], "#text": "C-chain"}, + ], + "ecNumber": [ + {"_evidence": [4, 6, 9], "#text": "1.14.11.-"}, + {"_evidence": [5, 7, 1], "#text": "6.3.2.19"}, + ], + }, + "alternativeName": [ + { + "fullName": {"_evidence": [6, 8, 2], "#text": "Transactivation subunit alpha"}, + "shortName": [ + {"_evidence": [7, 9, 3], "#text": "CT-1a"}, + {"_evidence": [8, 1, 4], "#text": "CT-1b"}, + ], + "ecNumber": [ + {"_evidence": [9, 2, 5], "#text": "2.4.2.30 (17)"}, + {"_evidence": [1, 3, 6], "#text": "2.1.1.43 (18)"}, + ], + }, + { + "fullName": {"_evidence": [2, 4, 7], "#text": "Transactivation subunit beta"}, + "shortName": [ + {"_evidence": [3, 5, 8], "#text": "CT-2a"}, + {"_evidence": [4, 6, 9], "#text": "CT-2b"}, + ], + "ecNumber": [ + {"_evidence": [5, 7, 1], "#text": "1.1.1.1 (19)"}, + {"_evidence": [6, 8, 2], "#text": "1.2.1.12 (20)"}, + ], + }, + ], + "submittedName": [ + { + "fullName": {"_evidence": [7, 9, 3], "#text": "Basic regulatory region"}, + "ecNumber": [ + {"_evidence": [8, 1, 4], "#text": "6.3.2.25 (17)"}, + {"_evidence": [9, 2, 5], "#text": "6.3.2.27 (18)"}, + ], + }, + { + "fullName": {"_evidence": [1, 3, 6], "#text": "Apoptosis cofactor peptide"}, + "ecNumber": [ + {"_evidence": [2, 4, 7], "#text": "2.7.1.37 (19)"}, + {"_evidence": [3, 5, 8], "#text": "2.7.4.3 (20)"}, + ], + }, + ], + "allergenName": {"_evidence": [4, 6, 9], "#text": "Hom s p53-C"}, + "biotechName": {"_evidence": [5, 7, 1], "#text": "Fusion C-terminal peptide"}, + "cdAntigenName": [ + {"_evidence": [6, 8, 2], "#text": "CD-REG1"}, + {"_evidence": [7, 9, 3], "#text": "CD-REG2"}, + ], + "innName": [ + {"_evidence": [8, 1, 4], "#text": "p53-iota"}, + {"_evidence": [9, 2, 5], "#text": "p53-kappa"}, + ], + }, + ], + }, + "gene": [ + { + "name": [ + {"_evidence": [1, 3, 6], "_type": "primary", "#text": "TP53"}, + {"_evidence": [2, 4, 7], "_type": "synonym", "#text": "P53"}, + ] + }, + { + "name": [ + {"_evidence": [3, 5, 8], "_type": "ordered locus", "#text": "PYRAB00630"}, + {"_evidence": [4, 6, 9], "_type": "ORF", "#text": "PAB2303"}, + ] + }, + ], + "organism": { + "_evidence": [1], + "name": [{"_type": "synonym", "#text": "Homo sapiens"}, {"_type": "synonym", "#text": "Human"}], + "dbReference": [ + { + "_type": "UniProtKB ID", + "_id": "P53_HUMAN", + "_evidence": [5, 7, 1], + "molecule": {"_id": "P04637-1", "#text": "Isoform 1"}, + "property": [ + {"_type": "member count", "_value": "3"}, + {"_type": "common taxon", "_value": "Homo sapiens"}, + ], + }, + { + "_type": "UniParc ID", + "_id": "UPI000002ED67", + "_evidence": [6, 8, 2], + "molecule": {"_id": "P04637-2", "#text": "Isoform 2"}, + "property": [ + {"_type": "common taxon ID", "_value": "9606"}, + {"_type": "GO Molecular Function", "_value": "GO:0003677"}, + ], + }, + ], + "lineage": {"taxon": ["Eukaryota", "Metazoa"]}, + }, + "organismHost": [ + { + "_evidence": [1, 2], + "name": [ + {"_type": "synonym", "#text": "Pyrococcus abyssi (strain GE5 / Orsay)"}, + {"_type": "synonym", "#text": "Pyrococcus abyssi"}, + ], + "dbReference": [ + { + "_type": "RefSeq", + "_id": "NP_000537.3", + "_evidence": [7, 9, 3], + "molecule": {"_id": "P04637-3", "#text": "Isoform 3"}, + "property": [ + {"_type": "UniProtKB accession", "_value": "P04637"}, + {"_type": "UniParc ID", "_value": "UPI000002ED67"}, + ], + }, + { + "_type": "NCBI Taxonomy", + "_id": "9606", + "_evidence": [8, 1, 4], + "molecule": {"_id": "Q9V2L2-1", "#text": "Chain A"}, + "property": [ + {"_type": "UniRef100 ID", "_value": "UniRef100_P04637"}, + {"_type": "UniRef90 ID", "_value": "UniRef90_P04637"}, + ], + }, + ], + "lineage": {"taxon": ["Archaea", "Thermococci"]}, + }, + { + "_evidence": [2, 3], + "name": [ + { + "_type": "synonym", + "#text": "Methanocella arvoryzae (strain DSM 22066 / NBRC 105507 / MRE50)", + }, + {"_type": "synonym", "#text": "Rice cluster I archaeon"}, + ], + "dbReference": [ + { + "_type": "Proteomes", + "_id": "UP000005640", + "_evidence": [9, 2, 5], + "molecule": {"_id": "Q0W5C8-1", "#text": "Chain B"}, + "property": [ + {"_type": "protein name", "_value": "Cellular tumor antigen p53"}, + {"_type": "source organism", "_value": "Homo sapiens (Human)"}, + ], + }, + { + "_type": "EMBL", + "_id": "X02469", + "_evidence": [1, 3, 6], + "molecule": {"_id": "A3CTY6-1", "#text": "Monomer"}, + "property": [ + {"_type": "NCBI taxonomy", "_value": "9606"}, + {"_type": "length", "_value": "393"}, + ], + }, + ], + "lineage": {"taxon": ["Methanomicrobia", "Methanocellales"]}, + }, + ], + "geneLocation": [ + { + "_type": "mitochondrion", + "_evidence": [2, 4, 7], + "name": [ + {"_status": "known", "#text": "17p13.1"}, + {"_status": "known", "#text": "chr17:7668402-7687550"}, + ], + }, + { + "_type": "plasmid", + "_evidence": [3, 5, 8], + "name": [{"_status": "known", "#text": "pBR322"}, {"_status": "known", "#text": "pMRE50"}], + }, + ], + "reference": [ + { + "_evidence": [4, 6, 9], + "_key": "1", + "citation": { + "_type": "journal article", + "_date": "2003-09-15", + "_name": "Mol. Microbiol.", + "_volume": "47", + "_first": "1495", + "_last": "1512", + "_publisher": "Wiley-Blackwell", + "_city": "Oxford", + "_db": "EMBL/GenBank/DDBJ", + "_number": "US20230123456", + "_institute": "MRC Laboratory of Molecular Biology", + "_country": "United Kingdom", + "title": "An integrated analysis of the genome of the hyperthermophilic archaeon Pyrococcus abyssi.", + "editorList": { + "consortium": [{"_name": "The UniProt Consortium"}], + "person": [{"_name": "Cohen G.N."}], + }, + "authorList": { + "consortium": [{"_name": "International Cancer Genome Consortium"}], + "person": [{"_name": "Forterre P."}], + }, + "locator": "10.1046/j.1365-2958.2003.03381.x", + "dbReference": [ + { + "_type": "GO", + "_id": "GO:0003677", + "_evidence": [5, 7, 1], + "molecule": {"_id": "P04637-1 (7)", "#text": "Homotetramer"}, + "property": [ + {"_type": "isSeed", "_value": "true"}, + {"_type": "protein sequence ID", "_value": "CAB48986.1"}, + ], + }, + { + "_type": "Pfam", + "_id": "PF00870", + "_evidence": [6, 8, 2], + "molecule": {"_id": "P04637-2 (8)", "#text": "Core domain"}, + "property": [ + {"_type": "molecule type", "_value": "Genomic_DNA"}, + {"_type": "entry name", "_value": "ACCD_DCysDesulf"}, + ], + }, + ], + }, + "scope": [ + "NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]", + "X-RAY CRYSTALLOGRAPHY (2.2 ANGSTROMS) OF 102-292", + ], + "source": { + "strain": [{"_evidence": [7, 9, 3], "#text": "GE5 / Orsay"}], + "plasmid": [{"_evidence": [8, 1, 4], "#text": "pAB1"}], + "transposon": [{"_evidence": [9, 2, 5], "#text": "Tn10"}], + "tissue": [{"_evidence": [1, 3, 6], "#text": "Liver"}], + }, + }, + { + "_evidence": [2, 4, 7], + "_key": "2", + "citation": { + "_type": "book", + "_date": "1999", + "_name": "Curr. Microbiol.", + "_volume": "64", + "_first": "118", + "_last": "129", + "_publisher": "Springer", + "_city": "Heidelberg", + "_db": "UniProtKB", + "_number": "EP3987654A1", + "_institute": "Max Planck Institute for Biochemistry", + "_country": "Germany", + "title": "Genome of rice cluster I archaea -- the key methane producers in the rice rhizosphere.", + "editorList": { + "consortium": [{"_name": "Archaea Genome Sequencing Project"}], + "person": [{"_name": "Erkel C."}], + }, + "authorList": { + "consortium": [{"_name": "The UniProt Consortium (4)"}], + "person": [{"_name": "Liesack W."}], + }, + "locator": "10.1126/science.1127062", + "dbReference": [ + { + "_type": "InterPro", + "_id": "IPR011615", + "_evidence": [3, 5, 8], + "molecule": {"_id": "P04637-3 (9)", "#text": "Precursor"}, + "property": [ + {"_type": "match status", "_value": "1"}, + {"_type": "nucleotide sequence ID", "_value": "NC_000868.1"}, + ], + }, + { + "_type": "PDB", + "_id": "1TUP", + "_evidence": [4, 6, 9], + "molecule": {"_id": "Q9V2L2-1 (10)", "#text": "Mature peptide"}, + "property": [ + {"_type": "taxonomic scope", "_value": "Archaea"}, + {"_type": "component", "_value": "Chromosome"}, + ], + }, + ], + }, + "scope": ["GENOME REANNOTATION", "FUNCTION AND CATALYTIC ACTIVITY"], + "source": { + "strain": [{"_evidence": [5, 7, 1], "#text": "DSM 22066 / NBRC 105507 / MRE50"}], + "plasmid": [{"_evidence": [6, 8, 2], "#text": "pMRE50"}], + "transposon": [{"_evidence": [7, 9, 3], "#text": "Tn5"}], + "tissue": [{"_evidence": [8, 1, 4], "#text": "Colon epithelium"}], + }, + }, + { + "_evidence": [1, 5, 9], + "_key": "3", + "citation": {"_type": "submission", "_date": "2007-10", "_db": "EMBL/GenBank/DDBJ databases"}, + "scope": ["NUCLEOTIDE SEQUENCE [GENOMIC DNA]"], + }, + ], + "comment": [ + { + "_type": "biophysicochemical properties", + "_locationType": "topology", + "_name": "UV-Vis absorption profile", + "_mass": 43653.5, + "_error": "Frameshift at position 112", + "_method": "Electrospray ionization", + "_evidence": [9, 2, 5], + "molecule": {"_id": "Q0W5C8-1 (11)", "#text": "Isoform 1 (11)"}, + "absorption": { + "max": {"_evidence": [1, 3, 6], "#text": "280"}, + "text": [ + {"_evidence": [2, 4, 7], "#text": "Absorbs UV light with a primary peak at 280 nm."}, + { + "_evidence": [3, 5, 8], + "#text": "Secondary shoulder observed at 292 nm in native buffer.", + }, + ], + }, + "kinetics": { + "KM": [ + {"_evidence": [4, 6, 9], "#text": "12.5 uM for ATP"}, + {"_evidence": [5, 7, 1], "#text": "48.0 uM for 1-aminocyclopropane-1-carboxylate"}, + ], + "Vmax": [ + {"_evidence": [6, 8, 2], "#text": "450 nmol/min/mg enzyme"}, + {"_evidence": [7, 9, 3], "#text": "1280 nmol/min/mg in presence of PLP cofactor"}, + ], + "text": [ + {"_evidence": [8, 1, 4], "#text": "Follows classical Michaelis-Menten kinetics at pH 7.4."}, + {"_evidence": [9, 2, 5], "#text": "Shows positive cooperativity upon tetramerization."}, + ], + }, + "phDependence": { + "text": [ + {"_evidence": [1, 3, 6], "#text": "Optimum pH is 7.4-7.8 in phosphate buffer."}, + {"_evidence": [2, 4, 7], "#text": "Rapidly loses DNA-binding activity below pH 6.0."}, + ] + }, + "redoxPotential": { + "text": [ + {"_evidence": [3, 5, 8], "#text": "Midpoint potential E0 is -240 mV at pH 7.0."}, + { + "_evidence": [4, 6, 9], + "#text": "Oxidation of Cys-277 abolishes sequence-specific binding.", + }, + ] + }, + "temperatureDependence": { + "text": [ + {"_evidence": [5, 7, 1], "#text": "Optimum temperature is 37 degrees Celsius."}, + { + "_evidence": [6, 8, 2], + "#text": "Unfolds irreversibly above 44 degrees Celsius (Tm = 42.5 C).", + }, + ] + }, + "location": [ + { + "_sequence": "VSP_006535", + "begin": {"_position": 1, "_status": "certain", "_evidence": [7, 9, 3]}, + "end": {"_position": 42, "_status": "certain", "_evidence": [8, 1, 4]}, + }, + { + "_sequence": "VSP_006536", + "position": {"_position": 15, "_status": "certain", "_evidence": [9, 2, 5]}, + }, + ], + "text": [ + { + "_evidence": [1, 3, 6], + "#text": "Biophysical measurements performed on purified recombinant core domain.", + }, + {"_evidence": [2, 4, 7], "#text": "Exhibits high-affinity sequence-specific DNA binding."}, + ], + }, + { + "_type": "catalytic activity", + "_locationType": "transmembrane", + "_name": "ACC deaminase reaction", + "_mass": 35755.25, + "_error": "Erroneous initiation at residue 1", + "_method": "MALDI-TOF", + "_evidence": [3, 5, 8], + "molecule": {"_id": "A3CTY6-1 (12)", "#text": "Isoform 2 (12)"}, + "reaction": { + "_evidence": [4, 6, 9], + "text": "1-aminocyclopropane-1-carboxylate + H2O = 2-oxobutanoate + NH4(+)", + "dbReference": [ + { + "_type": "PubMed", + "_id": "12622808", + "_evidence": [5, 7, 1], + "molecule": {"_id": "P04637-1 (13)", "#text": "Isoform 3 (13)"}, + "property": [ + {"_type": "term", "_value": "F:DNA-binding transcription factor activity"}, + {"_type": "evidence", "_value": "ECO:0007669"}, + ], + }, + { + "_type": "DOI", + "_id": "10.1046/j.1365-2958.2003.03381.x", + "_evidence": [6, 8, 2], + "molecule": {"_id": "P04637-2 (14)", "#text": "Chain A (14)"}, + "property": [ + {"_type": "project", "_value": "UniProtKB-KW"}, + {"_type": "chains", "_value": "A/B=102-292"}, + ], + }, + ], + }, + "physiologicalReaction": [ + { + "_direction": "left-to-right", + "_evidence": [7, 9, 3], + "dbReference": { + "_type": "Rhea", + "_id": "RHEA:16933", + "_evidence": [8, 1, 4], + "molecule": {"_id": "P04637-3 (15)", "#text": "Chain B (15)"}, + "property": [ + {"_type": "resolution", "_value": "2.20 A"}, + {"_type": "method", "_value": "X-ray"}, + ], + }, + }, + { + "_direction": "right-to-left", + "_evidence": [9, 2, 5], + "dbReference": { + "_type": "ChEBI", + "_id": "CHEBI:15377", + "_evidence": [1, 3, 6], + "molecule": {"_id": "Q9V2L2-1 (16)", "#text": "Monomer (16)"}, + "property": [ + {"_type": "member count (27)", "_value": "Q9V2L2"}, + {"_type": "common taxon (28)", "_value": "UPI0000061B2A"}, + ], + }, + }, + ], + "location": [ + { + "_sequence": "VSP_006537", + "position": {"_position": 20, "_status": "uncertain", "_evidence": [2, 4, 7]}, + }, + { + "_sequence": "VSP_006538", + "begin": {"_position": 43, "_status": "uncertain", "_evidence": [3, 5, 8]}, + "end": {"_position": 63, "_status": "uncertain", "_evidence": [4, 6, 9]}, + }, + ], + "text": [ + {"_evidence": [5, 7, 1], "#text": "Catalyzes the ring-opening deamination of ACC."}, + {"_evidence": [6, 8, 2], "#text": "Activity is strictly dependent on divalent cations."}, + ], + }, + { + "_type": "cofactor", + "_locationType": "intramembrane", + "_name": "Zinc-binding site", + "_mass": 21879.75, + "_error": "Stop codon readthrough at residue 341", + "_method": "Analytical ultracentrifugation", + "_evidence": [7, 9, 3], + "molecule": {"_id": "Q0W5C8-1 (17)", "#text": "Homotetramer (17)"}, + "cofactor": [ + { + "_evidence": [8, 1, 4], + "name": "pyridoxal 5'-phosphate", + "dbReference": { + "_type": "EC", + "_id": "3.5.99.7", + "_evidence": [9, 2, 5], + "molecule": {"_id": "A3CTY6-1 (18)", "#text": "Core domain (18)"}, + "property": [ + {"_type": "common taxon ID (29)", "_value": "UniRef100_Q9V2L2"}, + { + "_type": "GO Molecular Function (30)", + "_value": "Putative 1-aminocyclopropane-1-carboxylate deaminase", + }, + ], + }, + }, + { + "_evidence": [1, 3, 6], + "name": "Zn(2+)", + "dbReference": { + "_type": "IntAct", + "_id": "EBI-366083", + "_evidence": [2, 4, 7], + "molecule": {"_id": "P04637-1 (19)", "#text": "Precursor (19)"}, + "property": [ + { + "_type": "UniProtKB accession (31)", + "_value": "Pyrococcus abyssi (strain GE5 / Orsay)", + }, + {"_type": "UniParc ID (32)", "_value": "272844"}, + ], + }, + }, + ], + "location": [ + { + "_sequence": "VSP_006539", + "begin": {"_position": 65, "_status": "less than", "_evidence": [3, 5, 8]}, + "end": {"_position": 92, "_status": "less than", "_evidence": [4, 6, 9]}, + }, + { + "_sequence": "VSP_006540", + "position": {"_position": 37, "_status": "less than", "_evidence": [5, 7, 1]}, + }, + ], + "text": [ + {"_evidence": [6, 8, 2], "#text": "Zinc coordination stabilizes loop-sheet-helix motif."}, + {"_evidence": [7, 9, 3], "#text": "Depletion of zinc induces molten-globule conformation."}, + ], + }, + { + "_type": "subcellular location", + "_locationType": "topological domain", + "_name": "Nuclear-cytoplasmic shuttling", + "_mass": 42110.125, + "_error": "Intron retention in exon 4", + "_method": "Size-exclusion chromatography", + "_evidence": [8, 1, 4], + "molecule": {"_id": "P04637-2 (20)", "#text": "Mature peptide (20)"}, + "subcellularLocation": [ + { + "location": [ + {"_evidence": [9, 2, 5], "#text": "Nucleus"}, + {"_evidence": [1, 3, 6], "#text": "Nucleolus"}, + ], + "topology": [ + {"_evidence": [2, 4, 7], "#text": "Peripheral membrane protein"}, + {"_evidence": [3, 5, 8], "#text": "Single-pass membrane protein"}, + ], + "orientation": [ + {"_evidence": [4, 6, 9], "#text": "Cytoplasmic side"}, + {"_evidence": [5, 7, 1], "#text": "Extracellular side"}, + ], + }, + { + "location": [ + {"_evidence": [6, 8, 2], "#text": "Cytoplasm"}, + {"_evidence": [7, 9, 3], "#text": "Mitochondrion"}, + ], + "topology": [ + {"_evidence": [8, 1, 4], "#text": "Multi-pass membrane protein"}, + {"_evidence": [9, 2, 5], "#text": "Lipid-anchor"}, + ], + "orientation": [ + {"_evidence": [1, 3, 6], "#text": "Matrix side"}, + {"_evidence": [2, 4, 7], "#text": "Lumenal side"}, + ], + }, + ], + "location": [ + { + "_sequence": "VSP_006541", + "position": {"_position": 54, "_status": "greater than", "_evidence": [3, 5, 8]}, + }, + { + "_sequence": "VSP_006542", + "begin": {"_position": 102, "_status": "greater than", "_evidence": [4, 6, 9]}, + "end": {"_position": 292, "_status": "greater than", "_evidence": [5, 7, 1]}, + }, + ], + "text": [ + { + "_evidence": [6, 8, 2], + "#text": "Shuttles between nucleus and cytoplasm in a CRM1-dependent manner.", + }, + {"_evidence": [7, 9, 3], "#text": "Accumulates in the nucleus following ionizing radiation."}, + ], + }, + { + "_type": "sequence caution", + "_locationType": "chain", + "_name": "Frameshift discrepancy", + "_mass": 39850.0, + "_error": "Sequencing artifact at position 215", + "_method": "X-ray crystallography", + "_evidence": [8, 1, 4], + "molecule": {"_id": "P04637-3 (21)", "#text": "Isoform 1 (21)"}, + "conflict": { + "_type": "frameshift", + "_ref": "1", + "sequence": {"_resource": "EMBL-CDS", "_id": "CAB48986", "_version": 1}, + }, + "location": [ + { + "_sequence": "VSP_006543", + "begin": {"_position": 113, "_status": "unknown", "_evidence": [9, 2, 5]}, + "end": {"_position": 185, "_status": "unknown", "_evidence": [1, 3, 6]}, + }, + { + "_sequence": "VSP_006544", + "position": {"_position": 72, "_status": "unknown", "_evidence": [2, 4, 7]}, + }, + ], + "text": [ + {"_evidence": [3, 5, 8], "#text": "Sequence differs from genomic reference at codon 112."}, + {"_evidence": [4, 6, 9], "#text": "Verified by mass spectrometry of tryptic peptides."}, + ], + }, + { + "_type": "online information", + "_locationType": "domain", + "_name": "UniProtKB entry page", + "_mass": 41200.5, + "_error": "Mismatch at codon 72", + "_method": "Cryo-electron microscopy", + "_evidence": [5, 7, 1], + "molecule": {"_id": "Q9V2L2-1 (22)", "#text": "Isoform 2 (22)"}, + "link": [ + {"_uri": "https://www.uniprot.org/uniprotkb/P04637"}, + {"_uri": "https://www.ebi.ac.uk/pdbe/entry/pdb/1tup"}, + ], + "location": [ + { + "_sequence": "VSP_006535 (11)", + "position": {"_position": 175, "_status": "certain", "_evidence": [6, 8, 2]}, + }, + { + "_sequence": "VSP_006536 (12)", + "begin": {"_position": 195, "_status": "certain", "_evidence": [7, 9, 3]}, + "end": {"_position": 240, "_status": "certain", "_evidence": [8, 1, 4]}, + }, + ], + "text": [ + {"_evidence": [9, 2, 5], "#text": "Full structural and clinical annotations available online."}, + {"_evidence": [1, 3, 6], "#text": "Cross-referenced with PDBe macromolecular database."}, + ], + }, + { + "_type": "alternative products", + "_locationType": "region", + "_name": "TP53 splice variants", + "_mass": 38915.25, + "_error": "Alternative exon boundary", + "_method": "NMR spectroscopy", + "_evidence": [2, 4, 7], + "molecule": {"_id": "Q0W5C8-1 (23)", "#text": "Isoform 3 (23)"}, + "event": [{"_type": "alternative splicing"}, {"_type": "alternative initiation"}], + "isoform": [ + { + "id": ["P04637-1", "P04637-alpha"], + "name": [ + {"_evidence": [3, 5, 8], "#text": "Alpha"}, + {"_evidence": [4, 6, 9], "#text": "Canonical"}, + ], + "sequence": {"_type": "displayed", "_ref": "VSP_006535"}, + "text": [ + {"_evidence": [5, 7, 1], "#text": "Chosen as the canonical sequence."}, + {"_evidence": [6, 8, 2], "#text": "Expressed ubiquitously in normal tissues."}, + ], + }, + { + "id": ["P04637-2", "P04637-beta"], + "name": [ + {"_evidence": [7, 9, 3], "#text": "Beta"}, + {"_evidence": [8, 1, 4], "#text": "Alternative"}, + ], + "sequence": {"_type": "external", "_ref": "VSP_006536"}, + "text": [ + {"_evidence": [9, 2, 5], "#text": "Produced by alternative splicing at intron 9."}, + {"_evidence": [1, 3, 6], "#text": "Lacks the C-terminal oligomerization domain."}, + ], + }, + ], + "location": [ + { + "_sequence": "VSP_006537 (13)", + "begin": {"_position": 248, "_status": "uncertain", "_evidence": [2, 4, 7]}, + "end": {"_position": 270, "_status": "uncertain", "_evidence": [3, 5, 8]}, + }, + { + "_sequence": "VSP_006538 (14)", + "position": {"_position": 245, "_status": "uncertain", "_evidence": [4, 6, 9]}, + }, + ], + "text": [ + {"_evidence": [5, 7, 1], "#text": "Contains 12 characterized splice isoforms."}, + {"_evidence": [6, 8, 2], "#text": "Alternative promoter usage produces delta133 isoforms."}, + ], + }, + { + "_type": "interaction", + "_locationType": "motif", + "_name": "p53-MDM2 complex", + "_mass": 44102.75, + "_error": "Premature termination at 305", + "_method": "Surface plasmon resonance", + "_evidence": [7, 9, 3], + "molecule": {"_id": "A3CTY6-1 (24)", "#text": "Chain A (24)"}, + "interactant": [ + { + "_intactId": "EBI-366083", + "id": "P04637", + "label": "TP53", + "dbReference": { + "_type": "MIM", + "_id": "151623", + "_evidence": [8, 1, 4], + "molecule": {"_id": "P04637-1 (25)", "#text": "Chain B (25)"}, + "property": [ + {"_type": "UniRef100 ID (33)", "_value": "330"}, + {"_type": "UniRef90 ID (34)", "_value": "false"}, + ], + }, + }, + { + "_intactId": "EBI-389668", + "id": "Q00987", + "label": "MDM2", + "dbReference": { + "_type": "Reactome", + "_id": "R-HSA-3700989", + "_evidence": [9, 2, 5], + "molecule": {"_id": "P04637-2 (26)", "#text": "Monomer (26)"}, + "property": [ + {"_type": "protein name (35)", "_value": "CAJ36415.1"}, + {"_type": "source organism (36)", "_value": "mRNA"}, + ], + }, + }, + ], + "organismsDiffer": False, + "experiments": 14, + "location": [ + { + "_sequence": "VSP_006539 (15)", + "position": {"_position": 248, "_status": "less than", "_evidence": [1, 3, 6]}, + }, + { + "_sequence": "VSP_006540 (16)", + "begin": {"_position": 273, "_status": "less than", "_evidence": [2, 4, 7]}, + "end": {"_position": 299, "_status": "less than", "_evidence": [3, 5, 8]}, + }, + ], + "text": [ + { + "_evidence": [4, 6, 9], + "#text": "Forms a tight complex with E3 ubiquitin-protein ligase MDM2.", + }, + {"_evidence": [5, 7, 1], "#text": "Phosphorylation at Ser-15 disrupts MDM2 binding."}, + ], + }, + { + "_type": "disease", + "_locationType": "site", + "_name": "Li-Fraumeni predisposition", + "_mass": 43590.875, + "_error": "Missense substitution R175H", + "_method": "Isothermal titration calorimetry", + "_evidence": [6, 8, 2], + "molecule": {"_id": "P04637-3 (27)", "#text": "Homotetramer (27)"}, + "disease": { + "_id": "DI-00682", + "name": "Li-Fraumeni syndrome", + "acronym": "LFS1", + "description": "An autosomal dominant familial cancer syndrome caused by germline TP53 mutations.", + "dbReference": { + "_type": "AlphaFoldDB", + "_id": "AF-P04637-F1", + "_evidence": [7, 9, 3], + "molecule": {"_id": "Q9V2L2-1 (28)", "#text": "Core domain (28)"}, + "property": [ + {"_type": "NCBI taxonomy (37)", "_value": "Pur_DNA_glyco"}, + {"_type": "length (38)", "_value": "2"}, + ], + }, + }, + "location": [ + { + "_sequence": "VSP_006541 (17)", + "begin": {"_position": 301, "_status": "greater than", "_evidence": [8, 1, 4]}, + "end": {"_position": 322, "_status": "greater than", "_evidence": [9, 2, 5]}, + }, + { + "_sequence": "VSP_006542 (18)", + "position": {"_position": 249, "_status": "greater than", "_evidence": [1, 3, 6]}, + }, + ], + "text": [ + { + "_evidence": [2, 4, 7], + "#text": "Mutations in the DNA-binding domain account for 80% of LFS cases.", + }, + { + "_evidence": [3, 5, 8], + "#text": "Associated with early-onset sarcomas and breast carcinomas.", + }, + ], + }, + {"_type": "miscellaneous"}, + ], + "dbReference": [ + { + "_type": "SMART", + "_id": "SM00325", + "_evidence": [4, 6, 9], + "molecule": {"_id": "Q0W5C8-1 (29)", "#text": "Precursor (29)"}, + "property": [ + {"_type": "isSeed (39)", "_value": "NC_009464.1"}, + {"_type": "protein sequence ID (40)", "_value": "Eukaryota"}, + ], + }, + { + "_type": "SUPFAM", + "_id": "SSF49417", + "_evidence": [5, 7, 1], + "molecule": {"_id": "A3CTY6-1 (30)", "#text": "Mature peptide (30)"}, + "property": [ + {"_type": "molecule type (41)", "_value": "Mitochondrion"}, + {"_type": "entry name (42)", "_value": "P:regulation of cell cycle"}, + ], + }, + ], + "proteinExistence": {"_type": "evidence at protein level"}, + "keyword": [ + {"_evidence": [6, 8, 2], "_id": "KW-0002", "#text": "3D-structure"}, + {"_evidence": [7, 9, 3], "_id": "KW-0227", "#text": "DNA damage"}, + ], + "feature": [ + { + "_type": "domain", + "_id": "PRO_0000080560", + "_description": "p53 DNA-binding core domain", + "_evidence": [8, 1, 4], + "_ref": "1", + "original": "R", + "variation": ["H", "Q"], + "location": { + "_sequence": "VSP_006543 (19)", + "begin": {"_position": 325, "_status": "unknown", "_evidence": [9, 2, 5]}, + "end": {"_position": 355, "_status": "unknown", "_evidence": [1, 3, 6]}, + }, + "ligand": { + "name": "Mg(2+)", + "dbReference": { + "_type": "PANTHER", + "_id": "PTHR11447", + "_evidence": [2, 4, 7], + "molecule": {"_id": "P04637-1 (31)", "#text": "Isoform 1 (31)"}, + "property": [ + {"_type": "match status (43)", "_value": "ECO:0000314"}, + {"_type": "nucleotide sequence ID (44)", "_value": "GO_Central"}, + ], + }, + "label": "ZN-1", + "note": "Coordinated by Cys-176, His-179, Cys-238, and Cys-242.", + }, + "ligandPart": { + "name": "catalytic zinc site", + "dbReference": { + "_type": "Gene3D", + "_id": "2.60.40.720", + "_evidence": [3, 5, 8], + "molecule": {"_id": "P04637-2 (32)", "#text": "Isoform 2 (32)"}, + "property": [ + {"_type": "taxonomic scope (45)", "_value": "3 (45)"}, + {"_type": "component (46)", "_value": "Homo sapiens (46)"}, + ], + }, + "label": "ZN-coord", + "note": "Essential for structural integrity of L2/L3 loops.", + }, + }, + { + "_type": "mutagenesis site", + "_id": "PRO_0000184521", + "_description": "Loss of sequence-specific DNA binding", + "_evidence": [4, 6, 9], + "_ref": "2", + "original": "S", + "variation": ["A", "E"], + "location": { + "_sequence": "VSP_006544 (20)", + "position": {"_position": 273, "_status": "unknown", "_evidence": [5, 7, 1]}, + }, + "ligand": { + "name": "S-adenosyl-L-methionine", + "dbReference": { + "_type": "CDD", + "_id": "cd08367", + "_evidence": [6, 8, 2], + "molecule": {"_id": "P04637-3 (33)", "#text": "Isoform 3 (33)"}, + "property": [ + {"_type": "term (47)", "_value": "9606 (47)"}, + {"_type": "evidence (48)", "_value": "GO:0003677 (48)"}, + ], + }, + "label": "PLP-54", + "note": "Covalently linked via Schiff base to Lys-54.", + }, + "ligandPart": { + "name": "phosphate binding pocket", + "dbReference": { + "_type": "HAMAP-Rule", + "_id": "MF_00527", + "_evidence": [7, 9, 3], + "molecule": {"_id": "Q9V2L2-1 (34)", "#text": "Chain A (34)"}, + "property": [ + {"_type": "project (49)", "_value": "P04637 (49)"}, + {"_type": "chains (50)", "_value": "UPI000002ED67 (50)"}, + ], + }, + "label": "PLP-OPO3", + "note": "Stabilized by hydrogen bonding with Gly-188 and Thr-190.", + }, + }, + ], + "evidence": [ + { + "_type": "ECO:0000269", + "_key": 1, + "source": { + "_ref": 1, + "dbReference": { + "_type": "PIR", + "_id": "A25224", + "_evidence": [8, 1, 4], + "molecule": {"_id": "Q0W5C8-1 (35)", "#text": "Chain B (35)"}, + "property": [ + {"_type": "resolution (51)", "_value": "UniRef100_P04637 (51)"}, + {"_type": "method (52)", "_value": "UniRef90_P04637 (52)"}, + ], + }, + }, + "importedFrom": { + "dbReference": { + "_type": "KEGG", + "_id": "hsa:7157", + "_evidence": [9, 2, 5], + "molecule": {"_id": "A3CTY6-1 (36)", "#text": "Monomer (36)"}, + "property": [ + {"_type": "member count (53)", "_value": "Cellular tumor antigen p53 (53)"}, + {"_type": "common taxon (54)", "_value": "Homo sapiens (Human) (54)"}, + ], + } + }, + }, + { + "_type": "ECO:0000305", + "_key": 2, + "source": { + "_ref": 2, + "dbReference": { + "_type": "STRING", + "_id": "9606.ENSP00000269305", + "_evidence": [1, 3, 6], + "molecule": {"_id": "P04637-1 (37)", "#text": "Homotetramer (37)"}, + "property": [ + {"_type": "common taxon ID (55)", "_value": "9606 (55)"}, + {"_type": "GO Molecular Function (56)", "_value": "393 (56)"}, + ], + }, + }, + "importedFrom": { + "dbReference": { + "_type": "OrthoDB", + "_id": "119234at2759", + "_evidence": [2, 4, 7], + "molecule": {"_id": "P04637-2 (38)", "#text": "Core domain (38)"}, + "property": [ + {"_type": "UniProtKB accession (57)", "_value": "true (57)"}, + {"_type": "UniParc ID (58)", "_value": "CAB48986.1 (58)"}, + ], + } + }, + }, + ], + "sequence": { + "_length": 393, + "_mass": 43653, + "_checksum": "AD5C149FD8106131", + "_modified": dt.date(2009, 11, 24), + "_version": 2, + "_precursor": True, + "_fragment": "single", + "#text": "MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP", + }, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/uniprot/minimal_expected_table.py b/tests/data/xsd/examples/uniprot/minimal_expected_table.py new file mode 100644 index 00000000..7d14c246 --- /dev/null +++ b/tests/data/xsd/examples/uniprot/minimal_expected_table.py @@ -0,0 +1,43 @@ +"""Expected PyArrow table for uniprot minimal.xml (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_dataset": "Swiss-Prot", + "_created": dt.date(1986, 8, 13), + "_modified": dt.date(2024, 5, 29), + "_version": 285, + "accession": ["P04637"], + "name": ["P53_HUMAN"], + "protein": {"recommendedName": {"fullName": {"#text": "Cellular tumor antigen p53"}}}, + "organism": { + "name": [{"_type": "synonym", "#text": "Homo sapiens"}], + "dbReference": [{"_type": "UniProtKB ID", "_id": "P53_HUMAN"}], + }, + "reference": [ + { + "_key": "1", + "citation": {"_type": "journal article"}, + "scope": ["NUCLEOTIDE SEQUENCE [LARGE SCALE GENOMIC DNA]"], + } + ], + "proteinExistence": {"_type": "evidence at protein level"}, + "sequence": { + "_length": 393, + "_mass": 43653, + "_checksum": "AD5C149FD8106131", + "_modified": dt.date(2009, 11, 24), + "_version": 1, + "#text": "MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP", + }, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/uniref/case.yaml b/tests/data/xsd/examples/uniref/case.yaml new file mode 100644 index 00000000..4fe91bda --- /dev/null +++ b/tests/data/xsd/examples/uniref/case.yaml @@ -0,0 +1,18 @@ +title: UniRef +xml_tag: entry +schema: ../../../uniprot/uniref/uniref.xsd +rules: [R-RESOLVE-DEPTH, R-META-COMMENT] +notes: | + Four root elements (`UniRef100`, `UniRef90`, `UniRef50`, `UniRef`) declare `entry` with the shared + `entryType`, so resolution succeeds. +runs: + - id: minimal + data: minimal.xml + expected_json: minimal_expected.json + expected_table: minimal_expected_table.py + expected_count: 1 + - id: maximal + data: maximal.xml + expected_json: maximal_expected.json + expected_table: maximal_expected_table.py + expected_count: 1 diff --git a/tests/data/xsd/examples/uniref/maximal_expected_table.py b/tests/data/xsd/examples/uniref/maximal_expected_table.py new file mode 100644 index 00000000..7e047438 --- /dev/null +++ b/tests/data/xsd/examples/uniref/maximal_expected_table.py @@ -0,0 +1,67 @@ +"""Expected PyArrow table for uniref maximal.xml (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_id": "UniRef100_P04637", + "_updated": dt.date(2025, 2, 5), + "name": "Cluster: Glycogen synthase", + "property": [{"_type": "member count", "_value": "3"}, {"_type": "common taxon", "_value": "Homo sapiens"}], + "representativeMember": { + "dbReference": { + "_type": "UniProtKB ID", + "_id": "P53_HUMAN", + "property": [ + {"_type": "common taxon ID", "_value": "9606"}, + {"_type": "GO Molecular Function", "_value": "GO:0003677"}, + ], + }, + "sequence": { + "_length": 393, + "_checksum": "AD5C149FD8106131", + "#text": "MHPKVDALLSRFPRITLIPWETPIQYLPRISRELGVDVYVKRDDLTGLGIGGNKIRKLEF", + }, + }, + "member": [ + { + "dbReference": { + "_type": "UniParc ID", + "_id": "UPI000002ED67", + "property": [ + {"_type": "UniProtKB accession", "_value": "P04637"}, + {"_type": "UniParc ID", "_value": "UPI000002ED67"}, + ], + }, + "sequence": { + "_length": 330, + "_checksum": "CC8699923C0B11CD", + "#text": "MPVLPRDFYDRPTLEVARDLLGKTLVRQLPAGRVALRIVETEAYIGENDKACHASKGMTA", + }, + }, + { + "dbReference": { + "_type": "RefSeq", + "_id": "NP_000537.3", + "property": [ + {"_type": "UniRef100 ID", "_value": "UniRef100_P04637"}, + {"_type": "UniRef90 ID", "_value": "UniRef90_P04637"}, + ], + }, + "sequence": { + "_length": 200, + "_checksum": "8DA31EA44EEB8636", + "#text": "MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGP", + }, + }, + ], + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/examples/uniref/minimal_expected_table.py b/tests/data/xsd/examples/uniref/minimal_expected_table.py new file mode 100644 index 00000000..e586ec4e --- /dev/null +++ b/tests/data/xsd/examples/uniref/minimal_expected_table.py @@ -0,0 +1,20 @@ +"""Expected PyArrow table for uniref minimal.xml (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_id": "UniRef100_P04637", + "_updated": dt.date(2025, 2, 5), + "name": "Cluster: Glycogen synthase", + "representativeMember": {"dbReference": {"_type": "UniProtKB ID", "_id": "P53_HUMAN"}}, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/rules.yaml b/tests/data/xsd/rules.yaml new file mode 100644 index 00000000..df83e283 --- /dev/null +++ b/tests/data/xsd/rules.yaml @@ -0,0 +1,169 @@ +# Rule catalogue for the XSD -> PyArrow converter (cdm_data_loaders.readers.xsd). +# +# This file and the per-case `case.yaml` manifests are the single source of truth for +# docs/xsd_to_pyarrow_specification.md, which is generated by scripts/gen_converter_spec.py. +# Every rule must be exercised by at least one case (enforced by tests/cdm_data_loaders/readers/test_xsd_spec.py). +title: XSD to PyArrow Specification +intro: | + This document specifies how an XML Schema (`.xsd`) and a target `xml_tag` determine: + + 1. The **PyArrow `pa.Schema`** of the table yielded by the pipeline (`generate_pyarrow_schema`). + 2. The **Python / JSON row dictionaries** produced by `convert_xml_element` before + `pa.Table.from_pylist(rows, schema=pa_schema)`. + 3. The **errors** raised when an XSD or `xml_tag` uses unsupported or ambiguous constructs. + +sections: + - id: resolution + title: Top-Level `xml_tag` Resolution and Unwrapping + rules: + - id: R-RESOLVE-DEPTH + statement: | + `xml_tag` is searched for among global elements (depth 0), then breadth-first through the content + models of their types (depth 1, 2, ...). Matches at the shallowest depth are deduplicated by type + identity; one type resolves, zero matches raise **E1**, and more than one distinct type raises **E2**. + - id: R-RESOLVE-CLARK + statement: | + `xml_tag` may be a local name (`entry`) or Clark notation (`{http://example.org/ns}entry`); both + resolve to the same declaration. + - id: R-UNWRAP + statement: | + Each `` element becomes one row. Its type must map to a `pa.struct` (see **E3**), and the + struct's fields become the top-level columns of the `pa.Schema`. + + - id: naming + title: Naming and Field Ordering + rules: + - id: R-NAME-ATTR + statement: | + Attributes become fields prefixed with `_` (`id` -> `_id`). Namespace declarations (`xmlns`, + `xmlns:*`) and `xsi:*` attributes are stripped. + - id: R-NAME-TEXT + statement: | + When a type has attributes and text (`simpleContent` with attributes, or `mixed="true"` with + attributes), the text is stored in a field named `#text`. + - id: R-ORDER + statement: | + Struct fields are ordered: declared attributes in declaration order (base type before extension), + then `#text`, then child elements in content-model traversal order. A child name appearing in + several branches or positions is emitted once, at its first appearance. + + - id: shape + title: Element Classification (Shape) + intro: | + | XSD element type | Single (`overall_max_occurs == 1`) | Repeated (`> 1` or `unbounded`) | + | :--- | :--- | :--- | + | atomic `simpleType` | scalar | `pa.list_(scalar)` | + | `xs:list` `simpleType` | `pa.list_(item)` | `pa.list_(pa.list_(item))` | + | `simpleContent`, no attributes | as underlying `simpleType` | as underlying `simpleType`, listed | + | `simpleContent` with attributes | `pa.struct([_attr..., #text])` | `pa.list_(pa.struct(...))` | + | `mixed="true"`, no attributes | `pa.string()` (raw inner XML) | `pa.list_(pa.string())` | + | `mixed="true"` with attributes | `pa.struct([_attr..., #text: string])` | `pa.list_(pa.struct(...))` | + | attribute-only / element-only `complexType` | `pa.struct([...])` | `pa.list_(pa.struct(...))` | + | empty `complexType` (no attrs, no children, not mixed) | `pa.string()` | `pa.list_(pa.string())` | + rules: + - id: R-SHAPE-CARDINALITY + statement: | + A child's `overall_max_occurs` is computed across the whole content model: element `maxOccurs` + multiplied through enclosing compositors, summed over `xs:sequence` / `xs:all` positions and + maximised over `xs:choice` branches. A value greater than 1 (including a child of a repeated + compositor that itself has `maxOccurs="1"`) makes the field a `pa.list_`. `maxOccurs="0"` children + are omitted. + - id: R-SHAPE-GROUPS + statement: | + `xs:group` and `xs:attributeGroup` references are expanded in place; `xs:sequence`, `xs:choice` + and `xs:all` contribute their elements to the enclosing struct. + - id: R-SHAPE-EXTENSION + statement: | + `complexContent` / `xs:extension` merges base-type attributes and children with the extension's, + base first. + - id: R-SHAPE-XSLIST + statement: | + `xs:list` simple types, and the built-in list types `xs:NMTOKENS`, `xs:IDREFS`, `xs:ENTITIES`, map to + `pa.list_(item)` and are coerced by splitting on whitespace and coercing each token. + - id: R-SHAPE-SIMPLECONTENT + statement: | + `simpleContent` without attributes maps exactly like its base `simpleType`. With attributes it is a + struct of the attribute fields followed by `#text`. + - id: R-SHAPE-MIXED + statement: | + `mixed="true"` content is captured as the raw inner XML string; declared child elements are not + emitted as fields. Inline tags (``, ``, ...) are kept, text entities stay escaped (`&`, + `<`, `>`), and namespace declarations are dropped from inline tags. + - id: R-SHAPE-ATTR-ONLY + statement: | + A `complexType` with attributes but no content maps to a struct of its attribute fields. + - id: R-SHAPE-EMPTY-MARKER + statement: | + A `complexType` with no attributes, no children and no mixed content is an empty marker: it maps + to `pa.string()`, with value `""` when present and `None` when absent. + - id: R-SHAPE-CONTEXT-PATH + statement: | + Types are resolved per ancestor path from `xml_tag`, not per `(parent, child)` name pair, so the + same child name under differently-typed parents gets each parent's exact type. + - id: R-RECURSION + statement: | + Each complex type may be expanded at most `max_recursion_depth` times (default `2`) along a path. + A deeper occurrence becomes `pa.string()` (or `pa.list_(pa.string())` when repeated) and its + subtree is serialized with `json.dumps`. Applies to type self-recursion, element-`ref` + self-recursion and mutual recursion. Truncated `pa.string()` fields carry no `xsd_type`. + + - id: nullability + title: Field Nullability and Empty Tag Coercion + rules: + - id: R-NULL-ATTR + statement: | + Attribute fields are `nullable=False` when `use="required"` and the enclosing element is not + `nillable="true"`; otherwise `nullable=True`. + - id: R-NULL-CHILD + statement: | + A child field is `nullable=False` only when its overall `minOccurs` (multiplied through all + enclosing compositors) is at least 1, it is not `nillable="true"`, and its parent element is not + `nillable="true"`. Otherwise it is `nullable=True`. + - id: R-NULL-CHOICE + statement: | + Elements that appear in only some branches of an `xs:choice` have overall `minOccurs == 0` and are + therefore `nullable=True`. + - id: R-NULL-NILLABLE-PARENT + statement: | + All attribute and child fields inside a `nillable="true"` complex element are `nullable=True`, since + `` produces `{"_type": "x"}` with no children. + - id: R-NULL-TEXT + statement: | + `#text` fields are always `nullable=True` (`` has no text), and list `item` fields + are always `nullable=True`. + - id: R-NIL + statement: | + `xsi:nil="true"` yields `None` for an element with no attributes, and a struct holding only its + attributes otherwise. + - id: R-EMPTY-COERCE + statement: | + A present empty element (``, no `xsi:nil`) that maps to a non-nullable `pa.string()` field is + coerced to `""`, distinguishing it from an absent optional element (`None`). Empty markers are + always `""` when present. + + - id: scalars + title: Scalar Type Mapping and Field Metadata + generated: scalar_table + rules: + - id: R-SCALAR-BUILTIN + statement: | + Built-in XSD types map to Arrow types per the table below. Unlisted built-ins fall back to + `pa.string()`. `xs:boolean` accepts `true`/`1` and `false`/`0`; `xs:date` is coerced with + `datetime.date.fromisoformat`; `xs:hexBinary` with `bytes.fromhex`; `xs:base64Binary` with + `base64.b64decode`. + - id: R-SCALAR-RESTRICTION + statement: | + `xs:restriction` types (including enumerations) inherit the mapping of their built-in base type. + - id: R-SCALAR-UNION + statement: | + An `xs:union` uses the shared Arrow type when every member maps to the same type; otherwise it + falls back to `pa.string()`. Its `xsd_type` joins the member names with `" | "`. + - id: R-META-XSD-TYPE + statement: | + Every leaf scalar field (including `#text` and scalar list `item` fields) carries + `metadata["xsd_type"]` with the built-in `xs:` name. Struct, outer list, empty-marker and + recursion-truncated fields do not. + - id: R-META-COMMENT + statement: | + `` on an element or attribute (or, failing that, on its `ref` + target or type) is whitespace-normalized and stored in `metadata["comment"]`. diff --git a/tests/data/xsd/spec/01_scalars_and_simple_types/case.yaml b/tests/data/xsd/spec/01_scalars_and_simple_types/case.yaml new file mode 100644 index 00000000..dca99927 --- /dev/null +++ b/tests/data/xsd/spec/01_scalars_and_simple_types/case.yaml @@ -0,0 +1,11 @@ +title: Scalar Types, Restrictions, Lists and Unions +xml_tag: record +schema: schema.xsd +rules: [R-UNWRAP, R-NAME-ATTR, R-ORDER, R-SHAPE-XSLIST, R-SCALAR-BUILTIN, R-SCALAR-RESTRICTION, R-SCALAR-UNION, R-META-XSD-TYPE, R-META-COMMENT, R-NULL-ATTR] +notes: | + - `code` is `xs:int | xs:short`: members map to different Arrow types (`int32` vs `int16`), so it falls back to `pa.string()`. + - `published` is `xs:date | xs:gYearMonth | xs:gYear` and also resolves to `pa.string()`. + - `scores` and `_evidence` use the `xs:list` type `intListType` and become `pa.list_(pa.int32())`. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/01_scalars_and_simple_types/expected_table.py b/tests/data/xsd/spec/01_scalars_and_simple_types/expected_table.py new file mode 100644 index 00000000..fd77dc00 --- /dev/null +++ b/tests/data/xsd/spec/01_scalars_and_simple_types/expected_table.py @@ -0,0 +1,67 @@ +"""Expected PyArrow table for 01_scalars_and_simple_types (xml_tag='record').""" + +# ruff: noqa: INP001 + +import datetime as dt + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_id": "rec1", + "_status": "active", + "_evidence": [1, 4, 9], + "_version": 2, + "flag": True, + "smallInt": -5, + "medInt": 1024, + "stdInt": 100000, + "bigInt": 999999999999, + "uByte": 255, + "uShort": 65535, + "uInt": 4000000000, + "uLong": 18446744073709551615, + "f32": 1.5, + "f64": 2.718281828, + "dec": 3.14, + "created": dt.date(2025, 6, 15), + "updatedAt": "2025-06-15T12:30:00Z", + "published": "2025-06", + "homogeneousUnion": 42000000000, + "scores": [10, 20, 30], + "tokens": ["alpha", "beta", "gamma"], + "rawHex": b"Hello", + "rawB64": b"Hello", + }, + {"_id": "rec2", "_status": "archived", "_version": 1, "flag": False}, + { + "_id": "rec3", + "_status": "active", + "_evidence": [10, 20], + "_version": -42, + "flag": True, + "smallInt": -128, + "medInt": -32768, + "stdInt": -2147483648, + "bigInt": -999999999999, + "uByte": 0, + "uShort": 0, + "uInt": 0, + "uLong": 0, + "f32": -0.5, + "f64": 0.000125, + "dec": 42.0, + "created": dt.date(2025, 6, 15), + "updatedAt": "2025-06-15T12:30:00.123+02:00", + "published": "1999", + "homogeneousUnion": -42, + "scores": [100, 200], + "tokens": ["one", "two"], + "rawHex": b"Hello", + "rawB64": b"Hello", + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/02_namespaces_and_nillable/case.yaml b/tests/data/xsd/spec/02_namespaces_and_nillable/case.yaml new file mode 100644 index 00000000..07c8dd34 --- /dev/null +++ b/tests/data/xsd/spec/02_namespaces_and_nillable/case.yaml @@ -0,0 +1,16 @@ +title: Namespaces and `nillable="true"` +xml_tag: entry +schema: schema.xsd +rules: [R-RESOLVE-CLARK, R-NAME-ATTR, R-NAME-TEXT, R-SHAPE-SIMPLECONTENT, R-NIL, R-NULL-NILLABLE-PARENT, R-NULL-TEXT] +notes: | + Covers `targetNamespace` with `elementFormDefault="qualified"` and the three `xsi:nil="true"` cases: + + 1. Nillable element with no attributes (``) -> `None`. + 2. Nillable complex element with attributes (`<comment type="function" xsi:nil="true"/>`) -> `{"_type": "function"}`. + 3. Nillable `simpleContent` element with attributes (`<label lang="en" xsi:nil="true"/>`) -> `{"_lang": "en"}`. +runs: + - id: default + data: data.xml + - id: clark-notation + xml_tag: "{http://example.org/ns}entry" + data: data.xml diff --git a/tests/data/xsd/spec/02_namespaces_and_nillable/expected_table.py b/tests/data/xsd/spec/02_namespaces_and_nillable/expected_table.py new file mode 100644 index 00000000..11b5f886 --- /dev/null +++ b/tests/data/xsd/spec/02_namespaces_and_nillable/expected_table.py @@ -0,0 +1,18 @@ +"""Expected PyArrow table for 02_namespaces_and_nillable (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_id": "e1", + "label": {"_lang": "en"}, + "comment": [{"_type": "function"}, {"_type": "subunit", "text": "Homodimer"}], + }, + {"_id": "e2", "title": "Normal Title"}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/03_mixed_content_raw_text/case.yaml b/tests/data/xsd/spec/03_mixed_content_raw_text/case.yaml new file mode 100644 index 00000000..b30c2bd5 --- /dev/null +++ b/tests/data/xsd/spec/03_mixed_content_raw_text/case.yaml @@ -0,0 +1,11 @@ +title: Mixed Content Kept as Raw Text +xml_tag: article +schema: schema.xsd +rules: [R-SHAPE-MIXED, R-NAME-TEXT, R-SHAPE-GROUPS, R-EMPTY-COERCE, R-NULL-TEXT] +notes: | + - `title` is `mixed="true"` without attributes -> `pa.string()`; the empty `<title/>` coerces to `""`. + - `abstractText` is `mixed="true"` with a `Label` attribute -> `pa.struct([_Label, #text])`. + - `&` in text remains escaped so literal `<`/`>` are distinguishable from inline tags. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/03_mixed_content_raw_text/expected_table.py b/tests/data/xsd/spec/03_mixed_content_raw_text/expected_table.py new file mode 100644 index 00000000..5a5ec9f3 --- /dev/null +++ b/tests/data/xsd/spec/03_mixed_content_raw_text/expected_table.py @@ -0,0 +1,22 @@ +"""Expected PyArrow table for 03_mixed_content_raw_text (xml_tag='article').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "title": "Ca<sup>2+</sup> signaling in <i>E. <b>coli</b></i> cells", + "abstractText": [ + {"_Label": "BACKGROUND", "#text": "Studied <b>bold</b> & <i>italic</i> text where x < y."}, + {"#text": "Unlabelled paragraph with <sub>2</sub>O and empty <sup/> tag."}, + {"_Label": "EMPTY"}, + None, + ], + }, + {"title": ""}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/04_recursion_depth/case.yaml b/tests/data/xsd/spec/04_recursion_depth/case.yaml new file mode 100644 index 00000000..2416ed7e --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/case.yaml @@ -0,0 +1,17 @@ +title: Structural Recursion with Configurable `max_recursion_depth` +xml_tag: record +schema: schema.xsd +rules: [R-RECURSION, R-SHAPE-CARDINALITY] +notes: | + With `max_recursion_depth=2`, `tree` (depth 1) and its `branch` children (depth 2) are structs; grandchild + `branch` elements are serialized to JSON strings. With `max_recursion_depth=1`, `branch` is already truncated. +runs: + - id: depth-2 + params: {max_recursion_depth: 2} + data: data.xml + - id: depth-1 + params: {max_recursion_depth: 1} + data: data.xml + expected_json: expected_depth_1.json + schema_attr: EXPECTED_SCHEMA_DEPTH_1 + table_attr: EXPECTED_TABLE_DEPTH_1 diff --git a/tests/data/xsd/spec/04_recursion_depth/expected_table.py b/tests/data/xsd/spec/04_recursion_depth/expected_table.py new file mode 100644 index 00000000..80eba2cf --- /dev/null +++ b/tests/data/xsd/spec/04_recursion_depth/expected_table.py @@ -0,0 +1,53 @@ +"""Expected PyArrow table for 04_recursion_depth (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA, EXPECTED_SCHEMA_DEPTH_1 + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_id": "r1", + "tree": { + "_label": "L1", + "_weight": 10, + "branch": [ + { + "_label": "L2", + "branch": [ + '{"_label": "L3", "_weight": 30, "branch": [{"_label": "L4"}]}', + '{"_label": "L3_sibling"}', + ], + } + ], + }, + "Section": { + "SectionTitle": "Ch 1", + "Section": [{"SectionTitle": "Sec 1.1", "Section": ['{"SectionTitle": "Subsec 1.1.1"}']}], + }, + }, + ], + schema=EXPECTED_SCHEMA, +) + + +EXPECTED_TABLE_DEPTH_1 = pa.Table.from_pylist( + [ + { + "_id": "r1", + "tree": { + "_label": "L1", + "_weight": 10, + "branch": [ + '{"_label": "L2", "branch": [{"_label": "L3", "_weight": 30, "branch": [{"_label": "L4"}]}, {"_label": "L3_sibling"}]}' + ], + }, + "Section": { + "SectionTitle": "Ch 1", + "Section": ['{"SectionTitle": "Sec 1.1", "Section": [{"SectionTitle": "Subsec 1.1.1"}]}'], + }, + }, + ], + schema=EXPECTED_SCHEMA_DEPTH_1, +) diff --git a/tests/data/xsd/spec/05_groups_and_compositors/case.yaml b/tests/data/xsd/spec/05_groups_and_compositors/case.yaml new file mode 100644 index 00000000..29e640d9 --- /dev/null +++ b/tests/data/xsd/spec/05_groups_and_compositors/case.yaml @@ -0,0 +1,12 @@ +title: Model Groups, Attribute Groups, `xs:all` and Duplicate Element Names +xml_tag: item +schema: schema.xsd +rules: [R-SHAPE-GROUPS, R-SHAPE-CARDINALITY, R-ORDER, R-NULL-CHOICE, R-NULL-CHILD] +notes: | + - `pagination` is `(StartPage, EndPage?, MedlinePgn?) | MedlinePgn`: `MedlinePgn` appears in both choice + branches but has `overall_max_occurs == 1`, so it is a single field emitted once. + - `pairs` repeats `<xs:group ref="kvGroup" maxOccurs="unbounded"/>`, so `k` and `v` are lists. + - `meta` uses `<xs:all>`; `auditAttrs` is an `xs:attributeGroup`. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/05_groups_and_compositors/expected_table.py b/tests/data/xsd/spec/05_groups_and_compositors/expected_table.py new file mode 100644 index 00000000..8c41b6cd --- /dev/null +++ b/tests/data/xsd/spec/05_groups_and_compositors/expected_table.py @@ -0,0 +1,21 @@ +"""Expected PyArrow table for 05_groups_and_compositors (xml_tag='item').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "_createdBy": "alice", + "_version": 1, + "pagination": {"StartPage": "10", "EndPage": "20", "MedlinePgn": "10-20"}, + "interaction": {"interactant": ["P12345", "Q67890"]}, + "meta": {"author": "Bob", "year": 2025}, + "pairs": {"k": ["a", "b"], "v": [1, 2]}, + }, + {"pagination": {"MedlinePgn": "e12345"}}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/06_context_dependent_paths/case.yaml b/tests/data/xsd/spec/06_context_dependent_paths/case.yaml new file mode 100644 index 00000000..708264c7 --- /dev/null +++ b/tests/data/xsd/spec/06_context_dependent_paths/case.yaml @@ -0,0 +1,10 @@ +title: Context-Dependent Same `(parent, child)` Names with Different Types +xml_tag: root +schema: schema.xsd +rules: [R-SHAPE-CONTEXT-PATH, R-SHAPE-SIMPLECONTENT, R-NULL-CHILD] +notes: | + `left/node/leaf` is a single `xs:string`; `right/node/leaf` is an unbounded `simpleContent` type with a `lang` + attribute. The ancestor paths differ, so each `leaf` gets its own type. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/06_context_dependent_paths/expected_table.py b/tests/data/xsd/spec/06_context_dependent_paths/expected_table.py new file mode 100644 index 00000000..bce89b1d --- /dev/null +++ b/tests/data/xsd/spec/06_context_dependent_paths/expected_table.py @@ -0,0 +1,16 @@ +"""Expected PyArrow table for 06_context_dependent_paths (xml_tag='root').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + { + "left": {"node": {"leaf": "l1"}}, + "right": {"node": {"leaf": [{"#text": "r1"}, {"_lang": "en", "#text": "r2"}]}}, + }, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/07_multi_root_shared_type/case.yaml b/tests/data/xsd/spec/07_multi_root_shared_type/case.yaml new file mode 100644 index 00000000..73247285 --- /dev/null +++ b/tests/data/xsd/spec/07_multi_root_shared_type/case.yaml @@ -0,0 +1,10 @@ +title: Multiple Root Elements Sharing the Same Child `xml_tag` +xml_tag: entry +schema: schema.xsd +rules: [R-RESOLVE-DEPTH] +notes: | + Several root elements each declare a child `entry` of the same named type (the `uniref.xsd` pattern), so + resolving `xml_tag="entry"` is unambiguous. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/07_multi_root_shared_type/expected_table.py b/tests/data/xsd/spec/07_multi_root_shared_type/expected_table.py new file mode 100644 index 00000000..c675ea4f --- /dev/null +++ b/tests/data/xsd/spec/07_multi_root_shared_type/expected_table.py @@ -0,0 +1,14 @@ +"""Expected PyArrow table for 07_multi_root_shared_type (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + {"_id": "C1", "name": "First Cluster"}, + {"_id": "C2", "name": "Second Cluster"}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/08_nullability_and_empty_tags/case.yaml b/tests/data/xsd/spec/08_nullability_and_empty_tags/case.yaml new file mode 100644 index 00000000..b7355c1e --- /dev/null +++ b/tests/data/xsd/spec/08_nullability_and_empty_tags/case.yaml @@ -0,0 +1,12 @@ +title: Field Nullability and Empty String Tag Coercion +xml_tag: entry +schema: schema.xsd +rules: [R-NULL-ATTR, R-NULL-CHILD, R-NULL-CHOICE, R-NULL-NILLABLE-PARENT, R-EMPTY-COERCE, R-NIL] +notes: | + - `_id` (`use="required"`) is non-nullable; `_source` is nullable. + - `summary` is required and non-nillable, so the present `<summary/>` coerces to `""`. + - `geneId` / `proteinId` sit in an `xs:choice` and are nullable. + - `comment` is `nillable="true"`, so its required child `text` is nullable. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_table.py b/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_table.py new file mode 100644 index 00000000..b9b94625 --- /dev/null +++ b/tests/data/xsd/spec/08_nullability_and_empty_tags/expected_table.py @@ -0,0 +1,13 @@ +"""Expected PyArrow table for 08_nullability_and_empty_tags (xml_tag='entry').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + {"_id": "e1", "name": "BRCA1", "summary": "", "geneId": 672, "comment": {"_type": "function"}}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/09_empty_marker_elements/case.yaml b/tests/data/xsd/spec/09_empty_marker_elements/case.yaml new file mode 100644 index 00000000..b62227ea --- /dev/null +++ b/tests/data/xsd/spec/09_empty_marker_elements/case.yaml @@ -0,0 +1,10 @@ +title: Purely Empty `complexType` Marker Elements +xml_tag: book +schema: schema.xsd +rules: [R-SHAPE-EMPTY-MARKER, R-EMPTY-COERCE, R-SHAPE-CARDINALITY] +notes: | + DTD-derived markers such as PubMed's `<BookInProcess/>` map to `pa.string()` (`""` when present) and repeated + markers to `pa.list_(pa.string())`. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/09_empty_marker_elements/expected_table.py b/tests/data/xsd/spec/09_empty_marker_elements/expected_table.py new file mode 100644 index 00000000..9fe073c8 --- /dev/null +++ b/tests/data/xsd/spec/09_empty_marker_elements/expected_table.py @@ -0,0 +1,14 @@ +"""Expected PyArrow table for 09_empty_marker_elements (xml_tag='book').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + {"title": "In-Process Book", "BookInProcess": "", "flag": ["", ""]}, + {"title": "Completed Book"}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/10_combinations/case.yaml b/tests/data/xsd/spec/10_combinations/case.yaml new file mode 100644 index 00000000..201750d0 --- /dev/null +++ b/tests/data/xsd/spec/10_combinations/case.yaml @@ -0,0 +1,12 @@ +title: Combinations +xml_tag: record +schema: schema.xsd +rules: [R-SHAPE-CARDINALITY, R-SHAPE-EXTENSION, R-SHAPE-ATTR-ONLY, R-SHAPE-SIMPLECONTENT, R-SHAPE-MIXED, R-SHAPE-EMPTY-MARKER, R-SHAPE-CONTEXT-PATH, R-RECURSION, R-ORDER, R-NAME-TEXT] +notes: | + Stress test combining the constructs above in one type: bounded and unbounded repeats, `maxOccurs="0"` + children, children of repeated `xs:choice` / `xs:sequence` compositors, `simpleContent` with string and + decimal bases, attribute-only types, mixed types with and without attributes, element `ref`, a + self-recursive type and `complexContent` extension. +runs: + - id: default + data: data.xml diff --git a/tests/data/xsd/spec/10_combinations/expected_table.py b/tests/data/xsd/spec/10_combinations/expected_table.py new file mode 100644 index 00000000..a5958959 --- /dev/null +++ b/tests/data/xsd/spec/10_combinations/expected_table.py @@ -0,0 +1,101 @@ +"""Expected PyArrow table for 10_combinations (xml_tag='record').""" + +# ruff: noqa: INP001 + +import pyarrow as pa +from expected_schema import EXPECTED_SCHEMA + +EXPECTED_TABLE = pa.Table.from_pylist( + [ + {"_id": "r01"}, + {"_id": "r02", "plain": "a", "plainList": ["x"], "bounded": ["only"]}, + {"_id": "r03", "plainList": ["x", None, None, "y"], "bounded": ["b1", "b2"]}, + {"_id": "r04", "plain": "a < b & c", "plainList": ["padded", "Tom & Jerry"]}, + { + "_id": "r05", + "label": {"_lang": "en", "#text": "Hello"}, + "labelList": [ + {"_lang": "en", "_source": "curated", "#text": "one"}, + {"#text": "two"}, + {"_lang": "fr"}, + None, + ], + }, + {"_id": "r06", "label": {"#text": "Hello"}, "labelList": [{"#text": "solo"}]}, + {"_id": "r07", "label": {"_lang": "de"}, "textNoAttr": "t", "measure": {"_unit": "kg", "#text": 3.5}}, + {"_id": "r08", "measure": {"#text": 7.0}}, + {"_id": "r09", "flag": {"_on": True}, "flagList": [{"_on": False}, None], "marker": ["", ""]}, + {"_id": "r10", "flagList": [{"_on": True}], "marker": [""]}, + { + "_id": "r11", + "group": {"name": "g1", "alias": ["a1", "a2"]}, + "entity": [ + { + "_type": "person", + "_name": "attr-name", + "name": "child-name", + "prop": [{"_key": "k1", "_value": "v1"}, {"_key": "k2"}], + }, + {"_type": "org"}, + ], + }, + {"_id": "r12", "group": {"name": "g2"}, "box": {"item": ["1"]}, "entity": [{"prop": [{"_key": "only"}]}]}, + { + "_id": "r13", + "para": [ + {"#text": "plain text"}, + {"#text": "Hello\n <b>world</b> again"}, + {"#text": "<b>x</b>"}, + None, + {"#text": "A\n <b>1</b>B\n <i>2</i>C\n <b>3</b>"}, + {"_style": "em", "#text": "Intro\n <b>bold</b>"}, + ], + "note": {"_kind": "aside", "#text": "side text"}, + "remark": "just text", + }, + {"_id": "r14", "para": [{"#text": "<i>only italic</i>"}, {"_style": "x"}], "note": {"#text": "bare note"}}, + { + "_id": "r15", + "note": {"_kind": "empty"}, + "optA": ["a1", "a2"], + "optB": [{"_lang": "en", "#text": "b1"}, {"#text": "b2"}], + }, + {"_id": "r16", "optA": ["solo"], "pairs": {"k": ["a"], "v": ["1"]}}, + {"_id": "r17", "optB": [None], "pairs": {"k": ["a", "b"], "v": ["1", "2"]}}, + { + "_id": "r18", + "book": { + "title": "Book title", + "chapter": [ + {"title": [{"#text": "Ch1"}]}, + {"title": [{"_lang": "en", "#text": "Ch2"}, {"#text": "Ch2 alt"}]}, + ], + }, + }, + {"_id": "r19", "book": {"title": "Lonely"}}, + { + "_id": "r20", + "left": {"node": {"leaf": "l1"}}, + "right": {"node": {"leaf": [{"#text": "r1"}, {"_lang": "en", "#text": "r2"}]}}, + }, + {"_id": "r21", "left": {"node": {"leaf": ""}}, "right": {"node": {"leaf": [{"#text": "r"}]}}}, + { + "_id": "r22", + "tree": { + "_label": "root", + "branch": [ + {"_label": "a", "branch": ['{"_label": "a1"}', '{"_label": "a2", "branch": [{"_label": "a2x"}]}']}, + None, + ], + }, + }, + {"_id": "r23", "tree": {"_label": "t", "branch": [{"branch": ['{"_label": "deep"}']}]}}, + { + "_id": "r24", + "shared": ["s1"], + "derived": {"_baseAttr": "b", "_extraAttr": "e", "baseField": "base", "extraField": ["x1"]}, + }, + {"_id": "r25", "shared": ["s1", "s2"], "derived": {"extraField": ["x1", "x2"]}}, + ], + schema=EXPECTED_SCHEMA, +) diff --git a/tests/data/xsd/spec/errors/manifest.yaml b/tests/data/xsd/spec/errors/manifest.yaml new file mode 100644 index 00000000..9b2a6505 --- /dev/null +++ b/tests/data/xsd/spec/errors/manifest.yaml @@ -0,0 +1,84 @@ +# Error specification fixtures for the XSD -> PyArrow converter. +# `code` must match the `code` of an XsdToPyArrowError subclass in cdm_data_loaders.readers.xsd. +# `file` is relative to this directory. `match` is a regex applied to the exception message. +errors: + - file: ../../empty_schema.xsd + xml_tag: root + code: E1 + match: not found + description: Schema declares no elements. + - file: ../../../uniprot/uniref/uniref.xsd + xml_tag: nonExistentElement + code: E1 + match: not found + description: "`xml_tag` is not declared in an otherwise valid schema." + - file: ambiguous_xml_tag.xsd + xml_tag: entry + code: E2 + match: Ambiguous + description: Root `A` declares `<entry type="Type1"/>` and root `B` declares `<entry type="Type2"/>`. + - file: scalar_top_level.xsd + xml_tag: leaf + code: E3 + match: complex type + description: "`xml_tag` has a simple type (`xs:string`)." + - file: empty_marker_top_level.xsd + xml_tag: marker + code: E3 + match: complex type + description: "`xml_tag` has an empty `complexType` with no attributes or children." + - file: ../../no_children.xsd + xml_tag: leaf + code: E3 + match: complex type + description: "`xml_tag` is a scalar root element." + - file: wildcard_any_element.xsd + xml_tag: entry + code: E4 + match: xs:any + description: "`<xs:sequence><xs:any processContents=\"lax\"/></xs:sequence>`" + - file: wildcard_any_attribute.xsd + xml_tag: entry + code: E5 + match: xs:anyAttribute + description: "`<xs:anyAttribute processContents=\"skip\"/>`" + - file: ../../cdata_variety.xsd + xml_tag: root + code: E5 + match: xs:anyAttribute + description: Legacy fixture containing `xs:anyAttribute`. + - file: ../../combinations.xsd + xml_tag: record + code: E5 + match: xs:anyAttribute + description: Legacy fixture containing `xs:anyAttribute`. + - file: abstract_element.xsd + xml_tag: entry + code: E6 + match: abstract + description: Child element declared with `abstract="true"`. + - file: abstract_type.xsd + xml_tag: entry + code: E6 + match: abstract + description: Child element typed by a `complexType` with `abstract="true"`. + - file: substitution_group.xsd + xml_tag: entry + code: E7 + match: substitutionGroup + description: '`<xs:element name="sub" substitutionGroup="head" type="OtherType"/>`' + - file: attr_child_name_collision.xsd + xml_tag: entry + code: E8 + match: collision + description: Attribute `id` (field `_id`) and child element `_id` in the same type. + - file: nested_date_under_recursion.xsd + xml_tag: entry + code: E9 + match: recursive type + description: Recursive `NodeType` has an `xs:date` child element. + - file: nested_binary_under_recursion.xsd + xml_tag: entry + code: E9 + match: recursive type + description: Recursive `NodeType` has an `xs:hexBinary` attribute. diff --git a/tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd b/tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd new file mode 100644 index 00000000..07b42cd9 --- /dev/null +++ b/tests/data/xsd/spec/errors/nested_binary_under_recursion.xsd @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="NodeType"> + <xs:sequence> + <xs:element name="node" type="NodeType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + <xs:attribute name="checksum" type="xs:hexBinary"/> + </xs:complexType> + <xs:element name="root"> + <xs:complexType> + <xs:sequence> + <xs:element name="entry" type="NodeType" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/data/xsd/spec/errors/nested_date_under_recursion.xsd b/tests/data/xsd/spec/errors/nested_date_under_recursion.xsd new file mode 100644 index 00000000..f92032b6 --- /dev/null +++ b/tests/data/xsd/spec/errors/nested_date_under_recursion.xsd @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:complexType name="NodeType"> + <xs:sequence> + <xs:element name="created" type="xs:date" minOccurs="0"/> + <xs:element name="node" type="NodeType" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + <xs:element name="root"> + <xs:complexType> + <xs:sequence> + <xs:element name="entry" type="NodeType" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> diff --git a/tests/pyarrow_spec_helpers.py b/tests/pyarrow_spec_helpers.py new file mode 100644 index 00000000..adee8d09 --- /dev/null +++ b/tests/pyarrow_spec_helpers.py @@ -0,0 +1,132 @@ +"""Shared helpers for XSD-to-PyArrow and JSON Schema-to-PyArrow specification tests.""" + +import datetime as dt +import importlib.util +import io +import json +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq + + +def list_field( + name: str, + item_type: pa.DataType, + *, + nullable: bool = True, + item_nullable: bool = True, +) -> pa.Field: + """Build a PyArrow list field with configurable outer and item nullability.""" + return pa.field(name, pa.list_(pa.field("item", item_type, nullable=item_nullable)), nullable=nullable) + + +def load_module(module_file: Path) -> ModuleType: + """Load and execute a Python module from a file path.""" + spec = importlib.util.spec_from_file_location(module_file.stem, module_file) + if spec is None or spec.loader is None: + msg = f"Could not load module from {module_file}" + raise RuntimeError(msg) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_fixture_attr[T](module_file: Path, attr_name: str, expected_type: type[T]) -> T: + """Load a typed constant attribute from a fixture Python module.""" + module = load_module(module_file) + value = getattr(module, attr_name) + if not isinstance(value, expected_type): + msg = f"Expected {attr_name} in {module_file} to be a {expected_type.__name__}, got {type(value)}" + raise TypeError(msg) + return value + + +def load_expected_schema(schema_file: Path, attr_name: str = "EXPECTED_SCHEMA") -> pa.Schema: + """Load a pa.Schema constant from an expected_schema.py fixture file.""" + return load_fixture_attr(schema_file, attr_name, pa.Schema) + + +def load_expected_table(table_file: Path, attr_name: str = "EXPECTED_TABLE") -> pa.Table: + """Load a pa.Table constant from an expected_table.py fixture file.""" + previous_schema_mod = sys.modules.get("expected_schema") + sys.modules["expected_schema"] = load_module(table_file.parent / "expected_schema.py") + try: + return load_fixture_attr(table_file, attr_name, pa.Table) + finally: + if previous_schema_mod is None: + sys.modules.pop("expected_schema", None) + else: + sys.modules["expected_schema"] = previous_schema_mod + + +def json_default(obj: object) -> str: + """Serialize Python date and bytes values from Arrow tables for comparison with expected JSON.""" + if isinstance(obj, dt.date): + return obj.isoformat() + if isinstance(obj, bytes): + return obj.decode("utf-8") + msg = f"Object of type {type(obj)} is not JSON serializable" + raise TypeError(msg) + + +def assert_expected_table_valid( + records: list[Any], + expected_json_path: Path, + expected_table_path: Path, + *, + expected_count: int | None = None, + schema_attr: str = "EXPECTED_SCHEMA", + table_attr: str = "EXPECTED_TABLE", +) -> None: + """Verify input record count, expected JSON, and expected PyArrow table against expected_schema.py.""" + expected = json.loads(expected_json_path.read_text(encoding="utf-8")) + assert len(records) == len(expected) + if expected_count is not None: + assert len(expected) == expected_count + + expected_pa_schema = load_expected_schema(expected_table_path.parent / "expected_schema.py", schema_attr) + expected_table = load_expected_table(expected_table_path, table_attr) + expected_table.validate(full=True) + assert expected_table.schema.equals(expected_pa_schema, check_metadata=True) + assert expected_table.num_rows == len(expected) + + +def assert_schema_matches_expected( + pa_schema: pa.Schema, + expected_schema_path: Path, + schema_attr: str = "EXPECTED_SCHEMA", +) -> None: + """Verify a generated PyArrow schema equals the expected schema in expected_schema.py.""" + expected_schema = load_expected_schema(expected_schema_path, schema_attr) + assert pa_schema.equals(expected_schema, check_metadata=True) + + +def assert_rows_and_table_match_expected( + rows: list[dict[str, Any]], + pa_schema: pa.Schema, + expected_json_path: Path, + expected_table_path: Path, + *, + table_attr: str = "EXPECTED_TABLE", +) -> None: + """Verify converted rows match expected JSON, equal expected PyArrow Table, and round-trip via Parquet.""" + expected = json.loads(expected_json_path.read_text(encoding="utf-8")) + serialized_rows = json.loads(json.dumps(rows, default=json_default)) + assert serialized_rows == expected + + expected_table = load_expected_table(expected_table_path, table_attr) + table = pa.Table.from_pylist(rows, schema=pa_schema) + table.validate(full=True) + assert table.equals(expected_table, check_metadata=True) + + buf = io.BytesIO() + pq.write_table(table, buf) + buf.seek(0) + roundtrip_table = pq.read_table(buf) + assert roundtrip_table.num_rows == expected_table.num_rows + assert roundtrip_table.to_pylist() == expected_table.to_pylist() + diff --git a/uv.lock b/uv.lock index f244ce77..46dd9c77 100644 --- a/uv.lock +++ b/uv.lock @@ -505,7 +505,6 @@ dependencies = [ dev = [ { name = "linkml" }, { name = "moto", extra = ["s3"] }, - { name = "platformdirs" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -515,6 +514,9 @@ dev = [ { name = "ruff" }, { name = "types-boto3", extra = ["s3"] }, ] +lakehouse = [ + { name = "cdm-task-service-client" }, +] models = [ { name = "genson" }, { name = "json2python-models" }, @@ -561,7 +563,6 @@ requires-dist = [ dev = [ { name = "linkml", specifier = ">=1.9.5" }, { name = "moto", extras = ["s3"], specifier = ">=5.2.1" }, - { name = "platformdirs", specifier = ">=4.11.15" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, { name = "pytest-cov", specifier = ">=7.1.0" }, @@ -571,7 +572,7 @@ dev = [ { name = "ruff", specifier = ">=0.15.14" }, { name = "types-boto3", extras = ["s3"], specifier = ">=1.43.62" }, ] -lakehouse = [] +lakehouse = [{ name = "cdm-task-service-client", git = "https://github.com/kbase/cdm-task-service-client.git" }] models = [ { name = "genson", specifier = ">=1.3.0" }, { name = "json2python-models", specifier = ">=0.3.1" }, @@ -582,6 +583,14 @@ mutants = [ ] xml = [{ name = "xsdata", extras = ["cli", "lxml"], specifier = ">=26.2" }] +[[package]] +name = "cdm-task-service-client" +version = "0.2.1" +source = { git = "https://github.com/kbase/cdm-task-service-client.git#4b5da002b4c6484fe55a9ddc0056427353c352a3" } +dependencies = [ + { name = "requests" }, +] + [[package]] name = "certifi" version = "2026.10.19" From cd7419bf9b2b6346793b4471e948283c546a9d48 Mon Sep 17 00:00:00 2001 From: ialarmedalien <ialarmedalien@gmail.com> Date: Tue, 27 Oct 2026 11:53:11 -0700 Subject: [PATCH 2/2] Fixing formatting; removing unnecessary logging --- src/cdm_data_loaders/pipelines/core.py | 4 ---- tests/pyarrow_spec_helpers.py | 1 - 2 files changed, 5 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/core.py b/src/cdm_data_loaders/pipelines/core.py index a1aa1a68..5b1ef88d 100644 --- a/src/cdm_data_loaders/pipelines/core.py +++ b/src/cdm_data_loaders/pipelines/core.py @@ -290,8 +290,6 @@ def run_pipeline( pipeline = dlt.pipeline(**pipeline_kwargs) - logger.info("Prepping pipeline...") - try: pipeline.destination = dlt.destination(settings.use_destination, **(destination_kwargs or {})) except Exception as e: @@ -307,9 +305,7 @@ def run_pipeline( load_info: LoadInfo | None = None err_msg: str | None = None try: - logger.info("Running pipeline!") load_info = pipeline.run(resource, **(pipeline_run_kwargs or {})) - logger.info("Pipeline complete!") except PipelineStepFailed as e: # errors raise PipelineStepFailed err_msg = f"Pipeline failed: {e!s}" diff --git a/tests/pyarrow_spec_helpers.py b/tests/pyarrow_spec_helpers.py index adee8d09..3aa3cc98 100644 --- a/tests/pyarrow_spec_helpers.py +++ b/tests/pyarrow_spec_helpers.py @@ -129,4 +129,3 @@ def assert_rows_and_table_match_expected( roundtrip_table = pq.read_table(buf) assert roundtrip_table.num_rows == expected_table.num_rows assert roundtrip_table.to_pylist() == expected_table.to_pylist() -