[ialarmedalien]

• reinstate uniprot/uniref endpoints
• add in schemas for uniprot/uniref: XSD and pyarrow
• clean up duplication in tests

[codecov]

❌ 2 Tests Failed:

Tests completed	Failed	Passed	Skipped
5472	2	5470	220

View the top 2 failed test(s) by shortest run time

tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py::test_jsonlines_pass_no_data_lost_across_worker_configs[jsonlines_extract-jsonl-extract_3-normalize_2-load_2]

Stack Traces | 5.54s run time

run_reference_pipeline = <function run_reference_pipeline.<locals>._factory at 0x7f4a9753dd00>
sorted_reference_entities = ['{"accession": "GCA_000003115.1", "assembly_info": {"assembly_level": "Chromosome", "assembly_name": "catChrV17e", "a...ype 5", "tax_id": 2025339}, "paired_accession": "GCF_002271065.1", "source_database": "SOURCE_DATABASE_GENBANK"}', ...]
extract_workers = 3, normalize_workers = 2, load_workers = 2
loader_file_format = <LoaderFileFormatEnum.JSONL: 'jsonl'>

    @pytest.mark.parametrize(
        ("extract_workers", "normalize_workers", "load_workers"),
        WORKER_CONFIGS,
        ids=[f"extract_{extract}-normalize_{normalize}-load_{load}" for extract, normalize, load in WORKER_CONFIGS],
    )
    @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str)
    def test_jsonlines_pass_no_data_lost_across_worker_configs(
        *,
        run_reference_pipeline: Callable[..., Any],
        sorted_reference_entities: list[str],
        extract_workers: int,
        normalize_workers: int,
        load_workers: int,
        loader_file_format: str,
    ) -> None:
        """No data is lost regardless of how the job is split across extract/normalize/load workers."""
        load_info_and_dir = {}
        with dlt.config.values(
            {
                "extract.workers": extract_workers,
                "normalize.workers": normalize_workers,
                "load.workers": load_workers,
            }
        ):
>           load_info_and_dir["chunk_4"] = run_and_read(
                run_reference_pipeline, "chunk_4", loader_file_format=loader_file_format
            )

.../jsonlines/integration/test_jsonlines_reference_tests.py:219: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

run_pipeline = <function run_reference_pipeline.<locals>._factory at 0x7f4a9753dd00>
chunk_dir = 'chunk_4'
overrides = {'loader_file_format': <LoaderFileFormatEnum.JSONL: 'jsonl'>}

    def run_and_read(
        run_pipeline: Any,
        chunk_dir: str,
        **overrides: Any,
    ) -> tuple[LoadInfo, Path]:
        """Run the pipeline on a chunk dir and return the run result with its tables."""
        (load_info, output_dir) = run_pipeline(chunk_dir, **overrides)
>       assert load_info is not None
               ^^^^^^^^^^^^^^^^^^^^^
E       AssertionError

.../cdm_data_loaders/pipelines/helpers.py:269: AssertionError

tests/cdm_data_loaders/pipelines/xmltodict/integration/test_xmltodict_reference_tests.py::test_xml_pass_no_data_lost_across_worker_configs[jsonl-extract_3-normalize_2-load_2]

Stack Traces | 21s run time

run_xmltodict_pipeline_fxt = <function run_xmltodict_pipeline_fxt.<locals>._factory at 0x7f497001fd80>
sorted_reference_xml_entries = ['{"_id": "UniRef50_A0A091DBV8", "_updated": "2025-10-08", "member": [{"dbReference": {"_id": "A0ABM1K697_GEKJA", "_ty...Seed", "_value": "true"}]}, "sequence": {"#text": "SNIP", "_checksum": "4C9209D51C7EBAED", "_length": "34580"}}}', ...]
extract_workers = 3, normalize_workers = 2, load_workers = 2
loader_file_format = <LoaderFileFormatEnum.JSONL: 'jsonl'>

    @pytest.mark.parametrize(
        ("extract_workers", "normalize_workers", "load_workers"),
        WORKER_CONFIGS,
        ids=[f"extract_{e}-normalize_{n}-load_{l}" for e, n, l in WORKER_CONFIGS],  # noqa: E741
    )
    @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str)
    def test_xml_pass_no_data_lost_across_worker_configs(
        *,
        run_xmltodict_pipeline_fxt: Callable[..., Any],
        sorted_reference_xml_entries: list[str],
        extract_workers: int,
        normalize_workers: int,
        load_workers: int,
        loader_file_format: str,
    ) -> None:
        """No data is lost regardless of how the job is split across extract/normalize/load workers."""
        load_info_and_dir = {}
        for worker_key in ("baseline", "current"):
            worker_config = (
                {}
                if worker_key == "baseline"
                else {
                    "extract.workers": extract_workers,
                    "normalize.workers": normalize_workers,
                    "load.workers": load_workers,
                }
            )
            with dlt.config.values(worker_config):
>               load_info_and_dir[worker_key] = run_and_read(
                    run_xmltodict_pipeline_fxt, "chunk_5_el", loader_file_format=loader_file_format
                )

.../xmltodict/integration/test_xmltodict_reference_tests.py:169: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

run_pipeline = <function run_xmltodict_pipeline_fxt.<locals>._factory at 0x7f497001fd80>
chunk_dir = 'chunk_5_el'
overrides = {'loader_file_format': <LoaderFileFormatEnum.JSONL: 'jsonl'>}

    def run_and_read(
        run_pipeline: Any,
        chunk_dir: str,
        **overrides: Any,
    ) -> tuple[LoadInfo, Path]:
        """Run the pipeline on a chunk dir and return the run result with its tables."""
        (load_info, output_dir) = run_pipeline(chunk_dir, **overrides)
>       assert load_info is not None
               ^^^^^^^^^^^^^^^^^^^^^
E       AssertionError

.../cdm_data_loaders/pipelines/helpers.py:269: AssertionError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}