diff --git a/cdisc_rules_engine/constants/domains.py b/cdisc_rules_engine/constants/domains.py index 166b3598f..9e74f4cb9 100644 --- a/cdisc_rules_engine/constants/domains.py +++ b/cdisc_rules_engine/constants/domains.py @@ -1,4 +1 @@ SUPPLEMENTARY_DOMAINS = ("SUPP", "SQ") -AP_DOMAIN: str = "AP" -APFA_DOMAIN: str = "APFA" -APRELSUB_DOMAIN: str = "APRELSUB" diff --git a/cdisc_rules_engine/constants/rule_constants.py b/cdisc_rules_engine/constants/rule_constants.py index a0b35dac1..6d18dae11 100644 --- a/cdisc_rules_engine/constants/rule_constants.py +++ b/cdisc_rules_engine/constants/rule_constants.py @@ -1 +1,2 @@ ALL_KEYWORD: str = "ALL" +SPLIT_DATASETS_KEYWORD = "SPLIT DATASETS" diff --git a/cdisc_rules_engine/models/sdtm_dataset_metadata.py b/cdisc_rules_engine/models/sdtm_dataset_metadata.py index b8eec552e..36a7ca800 100644 --- a/cdisc_rules_engine/models/sdtm_dataset_metadata.py +++ b/cdisc_rules_engine/models/sdtm_dataset_metadata.py @@ -12,26 +12,28 @@ class SDTMDatasetMetadata(DatasetMetadata): """ Examples - | name | unsplit_name | is_supp | domain | wildcard_replacement | rdomain | is_ap | ap_suffix | domain_is_custom | related_domain | related_domain_is_custom | - | -------- | ------------ | ------- | ------ | -------------------- | ------- | ----- | --------- | ---------------- | -------------- | ------------------------ | - | QS | QS | False | QS | QS | None | False | | False | | | - | QSX | QS | False | QS | QS | None | False | | False | | | - | QSXX | QS | False | QS | QS | None | False | | False | | | - | SUPPQS | SUPPQS | True | None | | QS | False | | False | QS | | - | SUPPQSX | SUPPQS | True | None | | QS | False | | False | QS | | - | SUPPQSXX | SUPPQS | True | None | | QS | False | | False | QS | | - | APQS | APQS | False | APQS | QS | None | True | QS | False | QS | | - | APQSX | APQS | False | APQS | QS | None | True | QS | False | QS | | - | APQSXX | APQS | False | APQS | QS | None | True | QS | False | QS | | - | SQAPQS | SQAPQS | True | None | | APQS | True | | False | QS | | - | SQAPQSX | SQAPQS | True | None | | APQS | True | | False | QS | | - | SQAPQSXX | SQAPQS | True | None | | APQS | True | | False | | | - | RELREC | RELREC | False | None | | None | False | | False | | | - | XX | XX | False | XX | XX | None | False | | True | | | - | SUPPXX | SUPPXX | True | None | | XX | False | | False | XX | True | - | APXX | APXX | False | APXX | XX | None | True | XX | False | XX | True | - | SQAPXX | SQAPXX | True | None | | APXX | True | | False | XX | True | - | FA | FA | False | FA | FA | None | False | | False | | | +| name | unsplit_name | is_supp | is_split | domain | wildcard_replacement | rdomain | is_ap | ap_suffix | domain_is_custom | related_domain | related_domain_is_custom | +| --------- | ------------ | ------- | -------- | ------ | --------------------- | ------- | ----- | --------- | ---------------- | --------------- | ------------------------- | +| QS | QS | False | False | QS | QS | None | False | | False | | | +| QSX | QS | False | True | QS | QS | None | False | | False | | | +| QSXX | QS | False | True | QS | QS | None | False | | False | | | +| SUPPQS | SUPPQS | True | False | | | QS | False | | False | QS | | +| SUPPQSX | SUPPQS | True | True | | | QS | False | | False | QS | | +| SUPPQSXX | SUPPQS | True | True | | | QS | False | | False | QS | | +| APQS | APQS | False | False | QS | QS | None | True | QS | False | QS | | +| APQSX | APQS | False | True | QS | QS | None | True | QS | False | QS | | +| APQSXX | APQS | False | True | QS | QS | None | True | QS | False | QS | | +| SQAPQS | SQAPQS | True | False | | | APQS | False | | False | QS | | +| SQAPQSX | SQAPQS | True | True | | | APQS | False | | False | QS | | +| SQAPQSXX | SQAPQS | True | True | | | APQS | False | | False | | | +| RELREC | RELREC | False | False | | | None | False | | False | | | +| XX | XX | False | False | XX | XX | None | False | | True | | | +| SUPPXX | SUPPXX | True | False | | | XX | False | | False | XX | True | +| APXX | APXX | False | False | XX | XX | None | True | XX | False | XX | True | +| SQAPXX | SQAPXX | True | False | | | APXX | False | | False | XX | True | +| FA | FA | False | False | FA | FA | None | False | | False | | | +| APRELSUB | APRELSUB | False | False | | RELSUB | None | True | RELSUB | False | | | +| APRELSPEC | APRELSPEC | False | False | | RELSPEC | None | True | RELSPEC | False | | | """ # noqa: E501 W291 @property @@ -72,24 +74,20 @@ def is_ap(self) -> bool: """ Returns true if APID variable exists in first_record for non-supp datasets, or if the dataset name starts with AP for non-supp datasets, - or if rdomain starts with AP for supp datasets. """ if self.is_supp: - return ( - isinstance(self.rdomain, str) - and len(self.rdomain) == 4 - and self.rdomain.startswith("AP") - ) + return False first_record = self.first_record or {} if "APID" in first_record: return True - return self.name.startswith("AP") @property def ap_suffix(self) -> str: """ Returns the 2-character suffix (characters 3-4) from AP domains. + Falls back to the dataset name when there's no DOMAIN variable + (e.g. APRELSUB, APRELSPEC) Returns empty string if not an AP domain or for supp datasets. """ if not self.is_ap: @@ -98,4 +96,10 @@ def ap_suffix(self) -> str: return "" if isinstance(self.domain, str) and len(self.domain) >= 4: return self.domain[2::] + if ( + isinstance(self.name, str) + and len(self.name) >= 4 + and self.name.upper().startswith("AP") + ): + return self.name[2::] return "" diff --git a/cdisc_rules_engine/operations/domain_is_custom.py b/cdisc_rules_engine/operations/domain_is_custom.py index 641b3476d..7c34d887c 100644 --- a/cdisc_rules_engine/operations/domain_is_custom.py +++ b/cdisc_rules_engine/operations/domain_is_custom.py @@ -1,5 +1,6 @@ from cdisc_rules_engine.operations.base_operation import BaseOperation from cdisc_rules_engine.utilities.sdtm_utilities import is_custom_domain +from cdisc_rules_engine.utilities.utils import get_dataset_standard_check_name class DomainIsCustom(BaseOperation): @@ -9,4 +10,5 @@ def _execute_operation(self): given domain is in standard domains. If no -> the domain is custom. """ - return is_custom_domain(self.library_metadata, self.params.domain) + check_name = get_dataset_standard_check_name(self.params.dataframe_metadata) + return is_custom_domain(self.library_metadata, check_name) diff --git a/cdisc_rules_engine/services/data_services/base_data_service.py b/cdisc_rules_engine/services/data_services/base_data_service.py index d1552d1aa..6f8a2c8de 100644 --- a/cdisc_rules_engine/services/data_services/base_data_service.py +++ b/cdisc_rules_engine/services/data_services/base_data_service.py @@ -31,7 +31,6 @@ from cdisc_rules_engine.services.data_readers import DataReaderFactory from cdisc_rules_engine.utilities.utils import ( get_dataset_cache_key_from_path, - search_in_list, replace_nan_values_in_df, ) from cdisc_rules_engine.utilities.sdtm_utilities import ( @@ -242,47 +241,46 @@ def handle_custom_domains( dataset: DatasetInterface, dataset_metadata: SDTMDatasetMetadata, ): - if self._contains_topic_variable(dataset, dataset_metadata.domain, "TERM"): + class_name = self._detect_class_from_topic_variable( + dataset, dataset_metadata.domain + ) + if class_name: + return class_name + if dataset_metadata.is_ap: + return self._get_associated_persons_inherit_class(dataset, dataset_metadata) + return None + + def _detect_class_from_topic_variable( + self, dataset: DatasetInterface, domain_key: str + ) -> Optional[str]: + if self._contains_topic_variable(dataset, domain_key, "TERM"): return EVENTS - if self._contains_topic_variable(dataset, dataset_metadata.domain, "TRT"): + if self._contains_topic_variable(dataset, domain_key, "TRT"): return INTERVENTIONS - if self._contains_topic_variable(dataset, dataset_metadata.domain, "QNAM"): + if self._contains_topic_variable(dataset, domain_key, "QNAM"): return RELATIONSHIP - if self._contains_topic_variable(dataset, dataset_metadata.domain, "TESTCD"): - if self._contains_topic_variable(dataset, dataset_metadata.domain, "OBJ"): + if self._contains_topic_variable(dataset, domain_key, "TESTCD"): + if self._contains_topic_variable(dataset, domain_key, "OBJ"): return FINDINGS_ABOUT return FINDINGS - if dataset_metadata.is_ap: - return self._get_associated_persons_inherit_class(dataset_metadata) return None def _get_associated_persons_inherit_class( self, + dataset: DatasetInterface, dataset_metadata: SDTMDatasetMetadata, - ): - """ - Check with inherit class AP-- belongs to. - """ + ) -> Optional[str]: ap_suffix = dataset_metadata.ap_suffix - if not ap_suffix: - return None - datasets = self.get_datasets() - if len(datasets) > 1: - new_dataset_metadata: SDTMDatasetMetadata = search_in_list( - datasets, lambda item: item.domain == ap_suffix - ) - if new_dataset_metadata: - if new_dataset_metadata.is_ap: - raise ValueError("Nested Associated Persons domain reference") - new_dataset = self.get_dataset(dataset_name=new_dataset_metadata.name) - else: - raise ValueError("Filename for domain doesn't exist") - return self.get_dataset_class( - new_dataset, - new_dataset_metadata, + if self.library_metadata: + class_data, _ = get_class_and_dataset_metadata( + self.library_metadata, + ap_suffix, ) - else: - return None + name = (class_data or {}).get("name") + if name: + return convert_library_class_name_to_ct_class(name) + + return self._detect_class_from_topic_variable(dataset, ap_suffix) def _contains_topic_variable( self, diff --git a/cdisc_rules_engine/utilities/rule_processor.py b/cdisc_rules_engine/utilities/rule_processor.py index dce7ec69c..3c97bdb59 100644 --- a/cdisc_rules_engine/utilities/rule_processor.py +++ b/cdisc_rules_engine/utilities/rule_processor.py @@ -18,11 +18,6 @@ FINDINGS_ABOUT, FINDINGS, ) -from cdisc_rules_engine.constants.domains import ( - AP_DOMAIN, - APFA_DOMAIN, - SUPPLEMENTARY_DOMAINS, -) from cdisc_rules_engine.constants.rule_constants import ALL_KEYWORD from cdisc_rules_engine.constants.use_cases import USE_CASE_DOMAINS from cdisc_rules_engine.interfaces import ConditionInterface @@ -60,32 +55,36 @@ def __init__( self.cache = cache self.library_metadata = library_metadata + DATASET_PROPERTY_KEYWORDS: dict = { + "SUPP--": lambda md: bool(md.is_supp), + "SQ--": lambda md: bool(md.is_supp), + "AP--": lambda md: bool(md.is_ap), + "SPLIT DATASETS": lambda md: bool(md.is_split), + "AP SPLIT DATASETS": lambda md: ( + bool(md.is_ap) and bool(md.is_split) and not bool(md.is_supp) + ), + } + + @classmethod + def _matched_property_keyword( + cls, dataset_metadata: SDTMDatasetMetadata, domains_to_check: List[str] + ) -> bool: + for keyword, predicate in cls.DATASET_PROPERTY_KEYWORDS.items(): + if keyword in domains_to_check and predicate(dataset_metadata): + return True + return False + @classmethod def rule_applies_to_domain( cls, dataset_metadata: SDTMDatasetMetadata, rule: dict ) -> bool: - """ - Check that rule is applicable to dataset domain - """ domains = rule.get("domains") or {} - include_split_datasets: bool = domains.get("include_split_datasets") - included_domains = domains.get("Include", []) excluded_domains = domains.get("Exclude", []) - is_included = cls._is_domain_name_included( - dataset_metadata, included_domains, include_split_datasets - ) + is_included = cls._is_domain_name_included(dataset_metadata, included_domains) is_excluded = cls._is_domain_name_excluded(dataset_metadata, excluded_domains) - # additional check for split domains based on the flag - is_excluded, is_included = cls._handle_split_domains( - dataset_metadata.is_split, - include_split_datasets, - is_excluded, - is_included, - ) - return is_included and not is_excluded @classmethod @@ -93,22 +92,8 @@ def _is_domain_name_included( cls, dataset_metadata: SDTMDatasetMetadata, included_domains: List[str], - include_split_datasets: bool, ) -> bool: - """ - If included domains aren't specified - and include_split_datasets is True, - and it is not a split dataset - -> domain is not included - If included domains are specified, - and the domain is not in the list of included domains, - and domain doesn't match with AP / APFA / APRELSUB / SUPP / SQ naming pattern - -> domain is not included. - In other cases domain is included - """ if not included_domains: - if include_split_datasets is True and not dataset_metadata.is_split: - return False return True if ( @@ -117,7 +102,7 @@ def _is_domain_name_included( or ALL_KEYWORD in included_domains ): return True - if cls._domain_matched_ap_or_supp(dataset_metadata, included_domains): + if cls._matched_property_keyword(dataset_metadata, included_domains): return True return False @@ -128,7 +113,6 @@ def _is_domain_name_excluded( """ If excluded domains are specified, and the domain is in the list of excluded domains, - or domain name match with AP / APFA / APRELSUB / SUPP / SQ naming pattern domain is excluded. In other cases domain is not excluded. @@ -143,49 +127,7 @@ def _is_domain_name_excluded( or ALL_KEYWORD in excluded_domains ): return True - if cls._domain_matched_ap_or_supp(dataset_metadata, excluded_domains): - return True - return False - - @classmethod - def _handle_split_domains( - cls, - is_split_domain: bool, - include_split_datasets: bool, - is_excluded: bool, - is_included: bool, - ) -> Tuple[bool, bool]: - """ - HANDLING SPLIT DOMAINS - - If include_split_datasets is True - - add split domains to the list of included domains. - If no included domains specified, only validate split domains - - If include_split_datasets is False - Exclude split domains - If include_split_datasets is None - Do nothing - """ - if include_split_datasets is True and is_split_domain and not is_excluded: - is_included = True - if include_split_datasets is False and is_split_domain: - is_excluded = True - return is_excluded, is_included - - @classmethod - def _domain_matched_ap_or_supp( - cls, dataset_metadata: SDTMDatasetMetadata, domains_to_check: List[str] - ) -> bool: - """ - Check that domain name match with only - AP / APFA / APRELSUB / SUPP / SQ naming pattern - """ - domains_to_check = set(domains_to_check) - supp_domains = {f"{domain}--" for domain in SUPPLEMENTARY_DOMAINS} - ap_domains = {f"{AP_DOMAIN}--", f"{APFA_DOMAIN}--"} - - if dataset_metadata.is_supp and (domains_to_check & supp_domains): - return True - if dataset_metadata.is_ap and (domains_to_check & ap_domains): + if cls._matched_property_keyword(dataset_metadata, excluded_domains): return True return False @@ -214,7 +156,7 @@ def rule_applies_to_data_structure( def rule_applies_to_class( self, rule, - dataset_metadata: SDTMDatasetMetadata, + dataset_metadata: "SDTMDatasetMetadata", ): """ If included classes are specified and the class diff --git a/cdisc_rules_engine/utilities/utils.py b/cdisc_rules_engine/utilities/utils.py index 40fe038d7..3749096cb 100644 --- a/cdisc_rules_engine/utilities/utils.py +++ b/cdisc_rules_engine/utilities/utils.py @@ -19,6 +19,7 @@ from cdisc_rules_engine.models.base_validation_entity import BaseValidationEntity from cdisc_rules_engine.check_operators.helpers import is_valid_date from cdisc_rules_engine.constants.adam_products import ADAM_PRODUCTS +from cdisc_rules_engine.models.sdtm_dataset_metadata import SDTMDatasetMetadata def convert_dataclass_to_superclass[T](instance: object, superclass: type[T]) -> T: @@ -414,3 +415,15 @@ def custom_str_conversion(x): elif isinstance(x, float): return f"{x:.0f}" if x.is_integer() else str(x).strip() return x + + +def get_dataset_standard_check_name( + dataset_metadata: Optional[SDTMDatasetMetadata], +) -> Optional[str]: + if dataset_metadata is None: + return None + if dataset_metadata.is_supp: + return "SUPPQUAL" + if dataset_metadata.is_ap: + return dataset_metadata.ap_suffix or None + return dataset_metadata.domain or dataset_metadata.name diff --git a/resources/schema/rule-merged/CORE-base.json b/resources/schema/rule-merged/CORE-base.json index c7ce4f5e8..6bf8ede12 100644 --- a/resources/schema/rule-merged/CORE-base.json +++ b/resources/schema/rule-merged/CORE-base.json @@ -74,13 +74,17 @@ "ALL", "AP--", "APRELSUB", + "APRELSPEC", "POOLDEF", "RELREC", "RELREF", "RELSPEC", "RELSUB", + "SQ--", "SUPP--", - "TUMOR" + "TUMOR", + "SPLIT DATASETS", + "AP SPLIT DATASETS" ], "type": "string" }, diff --git a/resources/schema/rule/CORE-base.json b/resources/schema/rule/CORE-base.json index 16dc33f8d..8b1b647f3 100644 --- a/resources/schema/rule/CORE-base.json +++ b/resources/schema/rule/CORE-base.json @@ -74,13 +74,17 @@ "ALL", "AP--", "APRELSUB", + "APRELSPEC", "POOLDEF", "RELREC", "RELREF", "RELSPEC", "RELSUB", + "SQ--", "SUPP--", - "TUMOR" + "TUMOR", + "SPLIT DATASETS", + "AP SPLIT DATASETS" ], "type": "string" }, diff --git a/tests/unit/test_operations/test_domain_is_custom.py b/tests/unit/test_operations/test_domain_is_custom.py index d1caf924c..111a70219 100644 --- a/tests/unit/test_operations/test_domain_is_custom.py +++ b/tests/unit/test_operations/test_domain_is_custom.py @@ -5,6 +5,7 @@ from cdisc_rules_engine.models.library_metadata_container import ( LibraryMetadataContainer, ) +from cdisc_rules_engine.models.sdtm_dataset_metadata import SDTMDatasetMetadata import pytest from cdisc_rules_engine.models.operation_params import OperationParams from cdisc_rules_engine.operations.domain_is_custom import DomainIsCustom @@ -13,89 +14,81 @@ @pytest.mark.parametrize( - "dataframe, domain, standard, standard_version, expected", + "dataframe, domain, dataframe_metadata, standard, standard_version, expected", [ ( PandasDataset.from_dict( - { - "STUDYID": [ - "TEST_STUDY", - "TEST_STUDY", - "TEST_STUDY", - ], - "AETERM": [ - "test", - "test", - "test", - ], - } + {"STUDYID": ["TEST_STUDY"] * 3, "AETERM": ["test"] * 3} ), "AE", + SDTMDatasetMetadata(name="AE", first_record={"DOMAIN": "AE"}), "sdtmig", "3-4", False, ), ( DaskDataset.from_dict( - { - "STUDYID": [ - "TEST_STUDY", - "TEST_STUDY", - "TEST_STUDY", - ], - "AETERM": [ - "test", - "test", - "test", - ], - } + {"STUDYID": ["TEST_STUDY"] * 3, "AETERM": ["test"] * 3} ), "AE", + SDTMDatasetMetadata(name="AE", first_record={"DOMAIN": "AE"}), "sdtmig", "3-4", False, ), ( PandasDataset.from_dict( - { - "STUDYID": [ - "TEST_STUDY", - "TEST_STUDY", - "TEST_STUDY", - ], - "BCTERM": [ - "test", - "test", - "test", - ], - } + {"STUDYID": ["TEST_STUDY"] * 3, "BCTERM": ["test"] * 3} ), "BC", + SDTMDatasetMetadata(name="BC", first_record={"DOMAIN": "BC"}), "sdtmig", "3-4", True, ), + ( + PandasDataset.from_dict( + {"STUDYID": ["TEST_STUDY"] * 3, "APID": ["AP001"] * 3} + ), + "APMH", + SDTMDatasetMetadata( + name="APMH", first_record={"DOMAIN": "APMH", "APID": "AP001"} + ), + "sdtmig-ap", + "1-0", + False, + ), + ( + PandasDataset.from_dict( + {"STUDYID": ["TEST_STUDY"] * 3, "RDOMAIN": ["AE"] * 3} + ), + "SUPPAE", + SDTMDatasetMetadata(name="SUPPAE", first_record={"RDOMAIN": "AE"}), + "sdtmig", + "3-4", + False, + ), ], ) def test_domain_is_custom( operation_params: OperationParams, dataframe: DatasetInterface, domain: str, + dataframe_metadata: SDTMDatasetMetadata, standard: str, standard_version: str, expected: bool, ): standard_metadata = { - "dataset_names": {"AE"}, + "dataset_names": {"AE", "MH", "SUPPQUAL"}, } operation_params.dataframe = dataframe operation_params.domain = domain + operation_params.dataframe_metadata = dataframe_metadata operation_params.standard = standard operation_params.standard_version = standard_version - # save model metadata to cache cache = InMemoryCacheService.get_instance() library_metadata = LibraryMetadataContainer(standard_metadata=standard_metadata) - # execute operation data_service = LocalDataService.get_instance( cache_service=cache, config=ConfigService() ) diff --git a/tests/unit/test_operations/test_extract_metadata.py b/tests/unit/test_operations/test_extract_metadata.py index 4bc5afe7c..e9bfefbdc 100644 --- a/tests/unit/test_operations/test_extract_metadata.py +++ b/tests/unit/test_operations/test_extract_metadata.py @@ -66,22 +66,20 @@ def _create_mock_service(dataset_name, first_record=None): @pytest.mark.parametrize( "dataset_name, first_record, expected_suffix", [ - ("APFA", None, ""), - ("APXX", None, ""), - ("APLB", None, ""), - ("", {"DOMAIN": "APFA", "APID": "AP001"}, "FA"), - ("AE", {"DOMAIN": "APFA", "APID": "AP001"}, "FA"), - ("AP", {"DOMAIN": "APFA", "APID": "AP001"}, "FA"), - ("APF", {"DOMAIN": "APFA", "APID": "AP001"}, "FA"), - ("AE", None, ""), - ("LB", None, ""), - ("AP", None, ""), - ("APF", None, ""), ("AE", {"DOMAIN": "AE"}, ""), - ("AE", {"DOMAIN": "LB"}, ""), - ("AE", {"DOMAIN": "AP"}, ""), + ("LB", {"DOMAIN": "LB"}, ""), + ("AE1", {"DOMAIN": "AE"}, ""), + ("APFA", {"DOMAIN": "APFA", "APID": "AP001"}, "FA"), + ("APLB", {"DOMAIN": "APLB", "APID": "AP001"}, "LB"), + ("APFA1", {"DOMAIN": "APFA", "APID": "AP001"}, "FA"), + ("APXX", None, "XX"), + ("APFA", {"APID": "AP001"}, "FA"), ("AE", {"DOMAIN": ""}, ""), ("AE", {"DOMAIN": None}, ""), + ("APRELSUB", {"APID": "AP001"}, "RELSUB"), + ("APRELSPEC", {"APID": "AP001"}, "RELSPEC"), + ("APRELSUB", None, "RELSUB"), + ("APRELSPEC", None, "RELSPEC"), ], ) @pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) @@ -111,9 +109,12 @@ def test_extract_metadata_domain_suffix( ) result = operation.execute() assert operation_params.operation_id in result - assert all( - item == expected_suffix for item in result[operation_params.operation_id] - ) + actual_values = result[operation_params.operation_id] + for i, item in enumerate(actual_values): + assert item == expected_suffix, ( + f"row {i}: dataset_name={dataset_name!r} first_record={first_record!r} " + f"expected={expected_suffix!r} got={item!r}" + ) @pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) @@ -135,22 +136,3 @@ def test_extract_metadata_domain_suffix_uses_domain( result = operation.execute() assert operation_params.operation_id in result assert all(item == "XX" for item in result[operation_params.operation_id]) - - -@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) -def test_extract_metadata_domain_suffix_empty_metadata( - operation_params: OperationParams, dataset_type -): - mock_data_service = _create_mock_service("APFA", None) - operation_params.dataframe = dataset_type.from_dict( - {"STUDYID": ["TEST_STUDY"], "DOMAIN": ["APFA"]} - ) - operation_params.dataframe_metadata = SDTMDatasetMetadata(name="APFA") - operation_params.target = "ap_suffix" - cache = InMemoryCacheService.get_instance() - operation = ExtractMetadata( - operation_params, operation_params.dataframe, cache, mock_data_service - ) - result = operation.execute() - assert operation_params.operation_id in result - assert all(item == "" for item in result[operation_params.operation_id]) diff --git a/tests/unit/test_services/test_data_service/test_data_service.py b/tests/unit/test_services/test_data_service/test_data_service.py index b87677991..9091e965f 100644 --- a/tests/unit/test_services/test_data_service/test_data_service.py +++ b/tests/unit/test_services/test_data_service/test_data_service.py @@ -452,3 +452,111 @@ def func(self, **kwargs): keys = [call[0][0] for call in cache_service.get.call_args_list] assert keys[0] != keys[1] + + +def test_get_associated_persons_inherit_class_fallback_to_library(): + ap_metadata = SDTMDatasetMetadata( + name="APMH", + first_record={"DOMAIN": "APMH", "APID": "AP001"}, + filename="apmh.xpt", + ) + ap_dataset = PandasDataset.from_dict({"DOMAIN": ["APMH"], "APID": ["test"]}) + library_metadata: LibraryMetadataContainer = get_library_metadata_from_cache( + Validation_args( + f"{os.path.dirname(__file__)}/../../../../resources/cache", + 10, + [], + "", + "", + "sdtmig", + "3-4", + None, + None, + "", + "", + "", + False, + None, + None, + "", + "", + None, + "", + None, + False, + None, + None, + None, + None, + None, + None, + ) + ) + data_service = LocalDataService( + MagicMock(), + MagicMock(), + MagicMock(), + standard="sdtmig", + standard_version="3-4", + library_metadata=library_metadata, + ) + data_service.get_datasets = lambda: [ap_metadata] + + result = data_service._get_associated_persons_inherit_class(ap_dataset, ap_metadata) + + assert result == EVENTS + + +def test_get_associated_persons_inherit_class_custom_base_domain_resolves_via_topic_variable(): + dataset_metadata = SDTMDatasetMetadata( + name="APZZ", first_record={"DOMAIN": "APZZ", "APID": "AP001"} + ) + ap_dataset = PandasDataset.from_dict( + {"DOMAIN": ["APZZ"], "APID": ["test"], "ZZTERM": ["test"]} + ) + library_metadata: LibraryMetadataContainer = get_library_metadata_from_cache( + Validation_args( + f"{os.path.dirname(__file__)}/../../../../resources/cache", + 10, + [], + "", + "", + "sdtmig", + "3-4", + None, + None, + "", + "", + "", + False, + None, + None, + "", + "", + None, + "", + None, + False, + None, + None, + None, + None, + None, + None, + ) + ) + data_service = LocalDataService( + MagicMock(), + MagicMock(), + MagicMock(), + standard="sdtmig", + standard_version="3-4", + library_metadata=library_metadata, + ) + data_service.get_datasets = lambda: [dataset_metadata] + + result = data_service._get_associated_persons_inherit_class( + ap_dataset, dataset_metadata + ) + + assert result == EVENTS diff --git a/tests/unit/test_utilities/test_rule_processor.py b/tests/unit/test_utilities/test_rule_processor.py index 273e8606c..7509e7aa6 100644 --- a/tests/unit/test_utilities/test_rule_processor.py +++ b/tests/unit/test_utilities/test_rule_processor.py @@ -28,10 +28,10 @@ "name, rule_metadata, outcome", [ ("SQAE", {"domains": {"Exclude": ["SUPP--"]}}, False), - ("SQAE", {"domains": {"Exclude": ["SUPP--", "SQ--"]}}, False), + ("SQAE", {"domains": {"Exclude": ["SQ--"]}}, False), ("SQAE", {"domains": {"Include": ["SQ--"]}}, True), - ("SQAE", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), ("SQAE", {"domains": {"Include": ["SUPP--"]}}, True), + ("SQAE", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), ("SQAE", {"domains": {"Include": [ALL_KEYWORD]}}, True), ("AE", {"domains": {"Include": ["AE"]}}, True), ("AE", {"domains": {"Include": [ALL_KEYWORD]}}, True), @@ -39,8 +39,10 @@ ("AE", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), ("AE", {"domains": {"Include": ["TV"]}}, False), ("SUPPAE", {"domains": {"Exclude": ["SUPP--"]}}, False), - ("SUPPAE", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), + ("SUPPAE", {"domains": {"Exclude": ["SQ--"]}}, False), ("SUPPAE", {"domains": {"Include": ["SUPP--"]}}, True), + ("SUPPAE", {"domains": {"Include": ["SQ--"]}}, True), + ("SUPPAE", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), ("SUPPAE", {"domains": {"Include": [ALL_KEYWORD]}}, True), ("APTE", {"domains": {"Exclude": ["AP--"]}}, False), ("APTE", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), @@ -50,9 +52,9 @@ ("APRELSUB", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), ("APRELSUB", {"domains": {"Include": ["APRELSUB"]}}, True), ("APRELSUB", {"domains": {"Include": [ALL_KEYWORD]}}, True), - ("APFASU", {"domains": {"Exclude": ["APFA--"]}}, False), + ("APFASU", {"domains": {"Exclude": ["AP--"]}}, False), ("APFASU", {"domains": {"Exclude": [ALL_KEYWORD]}}, False), - ("APFASU", {"domains": {"Include": ["APFA--"]}}, True), + ("APFASU", {"domains": {"Include": ["AP--"]}}, True), ("APFASU", {"domains": {"Include": [ALL_KEYWORD]}}, True), ], ) @@ -73,133 +75,84 @@ def test_rule_applies_to_domain(mock_data_service, name, rule_metadata, outcome) "rule_domains, expected_results", [ ( - { - "Include": [ALL_KEYWORD], - "include_split_datasets": True, # Includes all - }, - [True, True, True, True, True, True], + {"Include": ["AP SPLIT DATASETS"]}, + [True, True, False, False], ), ( - {}, # Domains are included by default + {"Include": ["SPLIT DATASETS"]}, + [True, True, True, False], + ), + ( + {"Exclude": ["AP SPLIT DATASETS"]}, + [False, False, True, True], + ), + ( + {"Include": ["SPLIT DATASETS"], "Exclude": ["AP SPLIT DATASETS"]}, + [False, False, True, False], + ), + ], +) +def test_rule_applies_to_domain_ap_split_datasets( + mock_data_service, rule_domains: dict, expected_results: List[bool] +): + rule = {"domains": rule_domains} + datasets: List[dict] = [ + {"name": "APFA1", "domain": "APFA"}, # AP + split + {"name": "APFA2", "domain": "APFA"}, # AP + split + {"name": "QS1", "domain": "QS"}, # split, but not AP + {"name": "AE", "domain": "AE"}, # neither split nor AP + ] + processor = RuleProcessor(mock_data_service, InMemoryCacheService()) + results = [ + processor.rule_applies_to_domain( + SDTMDatasetMetadata( + name=dataset["name"], first_record={"DOMAIN": dataset["domain"]} + ), + rule, + ) + for dataset in datasets + ] + assert results == expected_results + + +@pytest.mark.parametrize( + "rule_domains, expected_results", + [ + ( + {"Include": [ALL_KEYWORD]}, [True, True, True, True, True, True], ), ( - { - "include_split_datasets": True, # Only include split datasets - }, - [ - False, - False, - True, - True, - True, - True, - ], + {}, # Domains are included by default + [True, True, True, True, True, True], ), ( - { - "Include": ["AE"], - "include_split_datasets": True, - }, - [ - True, - False, - True, - True, - True, - True, - ], + {"Include": ["SPLIT DATASETS"]}, # Only split datasets + [False, False, True, True, True, True], ), ( - # Only run on split datasets except SUPP - { - "Exclude": ["SUPP--"], - "include_split_datasets": True, - }, - [ - False, - False, - True, - True, - False, - False, - ], + {"Include": ["AE", "SPLIT DATASETS"]}, + [True, False, True, True, True, True], ), ( - { - "Include": [ - "EC", - ], - "Exclude": ["SUPP--"], - "include_split_datasets": True, - }, - [ - False, - True, - True, - True, - False, - False, - ], + {"Include": ["SPLIT DATASETS"], "Exclude": ["SUPP--"]}, + [False, False, True, True, False, False], ), ( - { - "Include": [ - "EC", - "QS", - ], - "include_split_datasets": False, - }, - [ - False, - True, - False, - False, - False, - False, - ], + {"Include": ["EC", "SPLIT DATASETS"], "Exclude": ["SUPP--"]}, + [False, True, True, True, False, False], ), ( - { - "Include": [ - "EC", - "QS", - ], - "include_split_datasets": True, - }, - [ - False, - True, - True, - True, - True, - True, - ], + {"Include": ["EC", "QS"]}, + [False, True, True, True, False, False], ), ( {"Include": ["QS"]}, - [ - False, - False, - True, - True, - False, - False, - ], + [False, False, True, True, False, False], ), ( - { - "Exclude": ["QS", "SUPPQS"], - "include_split_datasets": True, - }, - [ - False, - False, - False, - False, - False, - False, - ], + {"Exclude": ["SPLIT DATASETS"]}, + [True, True, False, False, False, False], ), ( {"Include": ["EC"]}, diff --git a/tests/unit/test_utilities/test_sdtm_utils.py b/tests/unit/test_utilities/test_sdtm_utils.py index aee50aa8b..2e2c30f0b 100644 --- a/tests/unit/test_utilities/test_sdtm_utils.py +++ b/tests/unit/test_utilities/test_sdtm_utils.py @@ -281,9 +281,9 @@ def test_is_supp_dataset(mock_dataset, expected): ({"first_record": None}, False), ({"first_record": {}}, False), ({}, False), - ({"name": "SQAPQS", "first_record": {"RDOMAIN": "APQS"}}, True), - ({"name": "SQAPQSX", "first_record": {"RDOMAIN": "APQS"}}, True), - ({"name": "SQAPQSXX", "first_record": {"RDOMAIN": "APQS"}}, True), + ({"name": "SQAPQS", "first_record": {"RDOMAIN": "APQS"}}, False), + ({"name": "SQAPQSX", "first_record": {"RDOMAIN": "APQS"}}, False), + ({"name": "SQAPQSXX", "first_record": {"RDOMAIN": "APQS"}}, False), ({"name": "SUPPQS", "first_record": {"RDOMAIN": "QS"}}, False), ({"name": "SQAPQS", "first_record": {"RDOMAIN": "AP"}}, False), ({"name": "SQAPQS", "first_record": {"RDOMAIN": "APF"}}, False),