[ialarmedalien]

• Update JSONSchema to pyarrow converter to act similarly to the XSD to pyarrow converter.
• Update test cases and examples
• add in autogeneration of specs document from examples/tests

[codecov]

❌ 1 Tests Failed:

Tests completed	Failed	Passed	Skipped
5622	1	5621	220

View the top 2 failed test(s) by shortest run time

tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py::test_jsonlines_pass_no_data_lost_across_worker_configs[jsonlines_extract-parquet-extract_5-normalize_3-load_4]

Stack Traces | 4.43s run time

run_reference_pipeline = <function run_reference_pipeline.<locals>._factory at 0x7f14a57634c0>
sorted_reference_entities = ['{"accession": "GCA_000003115.1", "assembly_info": {"assembly_level": "Chromosome", "assembly_name": "catChrV17e", "a...ype 5", "tax_id": 2025339}, "paired_accession": "GCF_002271065.1", "source_database": "SOURCE_DATABASE_GENBANK"}', ...]
extract_workers = 5, normalize_workers = 3, load_workers = 4
loader_file_format = <LoaderFileFormatEnum.PARQUET: 'parquet'>

    @pytest.mark.parametrize(
        ("extract_workers", "normalize_workers", "load_workers"),
        WORKER_CONFIGS,
        ids=[f"extract_{extract}-normalize_{normalize}-load_{load}" for extract, normalize, load in WORKER_CONFIGS],
    )
    @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str)
    def test_jsonlines_pass_no_data_lost_across_worker_configs(
        *,
        run_reference_pipeline: Callable[..., Any],
        sorted_reference_entities: list[str],
        extract_workers: int,
        normalize_workers: int,
        load_workers: int,
        loader_file_format: str,
    ) -> None:
        """No data is lost regardless of how the job is split across extract/normalize/load workers."""
        load_info_and_dir = {}
        with dlt.config.values(
            {
                "extract.workers": extract_workers,
                "normalize.workers": normalize_workers,
                "load.workers": load_workers,
            }
        ):
>           load_info_and_dir["chunk_4"] = run_and_read(
                run_reference_pipeline, "chunk_4", loader_file_format=loader_file_format
            )

.../jsonlines/integration/test_jsonlines_reference_tests.py:219: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

run_pipeline = <function run_reference_pipeline.<locals>._factory at 0x7f14a57634c0>
chunk_dir = 'chunk_4'
overrides = {'loader_file_format': <LoaderFileFormatEnum.PARQUET: 'parquet'>}

    def run_and_read(
        run_pipeline: Any,
        chunk_dir: str,
        **overrides: Any,
    ) -> tuple[LoadInfo, Path]:
        """Run the pipeline on a chunk dir and return the run result with its tables."""
        (load_info, output_dir) = run_pipeline(chunk_dir, **overrides)
>       assert load_info is not None
               ^^^^^^^^^^^^^^^^^^^^^
E       AssertionError

.../cdm_data_loaders/pipelines/helpers.py:211: AssertionError

tests/cdm_data_loaders/pipelines/jsonlines/integration/test_jsonlines_reference_tests.py::test_jsonlines_pass_no_data_lost_across_worker_configs[jsonlines_jsonschema-parquet-extract_5-normalize_3-load_4]

Stack Traces | 9.24s run time

run_reference_pipeline = <function run_reference_pipeline.<locals>._factory at 0x7f7dc888cf40>
sorted_reference_entities = ['{"accession": "GCA_000003115.1", "assembly_info": {"assembly_level": "Chromosome", "assembly_name": "catChrV17e", "a...ype 5", "tax_id": 2025339}, "paired_accession": "GCF_002271065.1", "source_database": "SOURCE_DATABASE_GENBANK"}', ...]
extract_workers = 5, normalize_workers = 3, load_workers = 4
loader_file_format = <LoaderFileFormatEnum.PARQUET: 'parquet'>

    @pytest.mark.parametrize(
        ("extract_workers", "normalize_workers", "load_workers"),
        WORKER_CONFIGS,
        ids=[f"extract_{extract}-normalize_{normalize}-load_{load}" for extract, normalize, load in WORKER_CONFIGS],
    )
    @pytest.mark.parametrize("loader_file_format", LOADER_FILE_FORMATS, ids=str)
    def test_jsonlines_pass_no_data_lost_across_worker_configs(
        *,
        run_reference_pipeline: Callable[..., Any],
        sorted_reference_entities: list[str],
        extract_workers: int,
        normalize_workers: int,
        load_workers: int,
        loader_file_format: str,
    ) -> None:
        """No data is lost regardless of how the job is split across extract/normalize/load workers."""
        load_info_and_dir = {}
        with dlt.config.values(
            {
                "extract.workers": extract_workers,
                "normalize.workers": normalize_workers,
                "load.workers": load_workers,
            }
        ):
>           load_info_and_dir["chunk_4"] = run_and_read(
                run_reference_pipeline, "chunk_4", loader_file_format=loader_file_format
            )

.../jsonlines/integration/test_jsonlines_reference_tests.py:219: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

run_pipeline = <function run_reference_pipeline.<locals>._factory at 0x7f7dc888cf40>
chunk_dir = 'chunk_4'
overrides = {'loader_file_format': <LoaderFileFormatEnum.PARQUET: 'parquet'>}

    def run_and_read(
        run_pipeline: Any,
        chunk_dir: str,
        **overrides: Any,
    ) -> tuple[LoadInfo, Path]:
        """Run the pipeline on a chunk dir and return the run result with its tables."""
        (load_info, output_dir) = run_pipeline(chunk_dir, **overrides)
>       assert load_info is not None
               ^^^^^^^^^^^^^^^^^^^^^
E       AssertionError

.../cdm_data_loaders/pipelines/helpers.py:211: AssertionError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}