core.py

#!/usr/bin/env python3
"""
CLI entrypoint for the CDISC Rules Engine.
"""

import asyncio
import codecs
import json
import logging
import os
import pickle
import sys
import tempfile
from datetime import datetime
from multiprocessing import freeze_support
from pathlib import Path

import click
from dotenv import load_dotenv

from cdisc_rules_engine.config import config
from cdisc_rules_engine.enums.dataformat_types import DataFormatTypes
from cdisc_rules_engine.enums.default_file_paths import DefaultFilePaths
from cdisc_rules_engine.enums.progress_parameter_options import ProgressParameterOptions
from cdisc_rules_engine.enums.report_types import ReportTypes
from cdisc_rules_engine.enums.standard_types import StandardTypes
from cdisc_rules_engine.exceptions.custom_exceptions import InvalidCSVFile
from cdisc_rules_engine.models.external_dictionaries_container import (
    DictionaryTypes,
    ExternalDictionariesContainer,
)
from cdisc_rules_engine.models.validation_args import Validation_args
from cdisc_rules_engine.services.cache.cache_populator_service import CachePopulator
from cdisc_rules_engine.services.cache.cache_service_factory import CacheServiceFactory
from cdisc_rules_engine.services.cdisc_library_service import CDISCLibraryService
from cdisc_rules_engine.utilities.utils import (
    generate_report_filename,
    get_rules_cache_key,
    validate_dataset_files_exist,
)
from cdisc_rules_engine.constants import VALIDATION_FORMATS_MESSAGE, DEFAULT_ENCODING
from scripts.list_dataset_metadata_handler import list_dataset_metadata_handler
from scripts.run_validation import run_validation
from version import __version__

DEFAULT_CACHE_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), DefaultFilePaths.CACHE.value
)


def validate_encoding(_ctx, _param, value):
    if value is None:
        return DEFAULT_ENCODING
    try:
        codecs.lookup(value)
        return value
    except LookupError:
        raise click.BadParameter(
            f"Invalid encoding '{value}'. Please provide a valid encoding name "
            f"(e.g., utf-8, utf-16, utf-32, cp1252, latin-1)."
        )


def valid_data_file(data_path: list) -> tuple[list, set]:
    allowed_formats = [
        DataFormatTypes.XPT.value,
        DataFormatTypes.JSON.value,
        DataFormatTypes.NDJSON.value,
        DataFormatTypes.XLSX.value,
        DataFormatTypes.CSV.value,
    ]
    found_formats = set()
    file_list = []
    ignored_files = []

    for file in data_path:
        file_extension = os.path.splitext(file)[1][1:].upper()
        if file_extension in allowed_formats:
            found_formats.add(file_extension)
            file_list.append(file)
        elif file_extension:
            ignored_files.append(os.path.basename(file))

    if ignored_files:
        logger = logging.getLogger("validator")
        logger.info(
            f"Ignoring {len(ignored_files)} file(s) with unsupported formats: {', '.join(ignored_files[:5])}"
            + ("..." if len(ignored_files) > 5 else "")
        )

    if DataFormatTypes.XLSX.value in found_formats:
        if len(found_formats) > 1:
            return [], found_formats
        elif len(file_list) > 1:
            return [], found_formats
        else:
            return file_list, found_formats
    if len(found_formats) >= 1:
        return file_list, found_formats
    else:
        return [], set()


@click.group()
def cli():
    pass


def _validate_csv_data_paths(
    dataset_paths: list[str], encoding: str = DEFAULT_ENCODING
) -> list[str]:
    """
    Filters dataset paths based on _datasets.csv content.

    Raises InvalidCSVFile error if there are no proper _datasets.csv files in provided path.

    Keeps only datasets listed in _datasets.csv (Filename column).
    Always excludes _datasets.csv and _variables.csv from result.
    """
    import pandas as pd

    paths = [Path(p) for p in dataset_paths]

    datasets_path = list({p for p in paths if p.name.lower() == "_datasets.csv"})
    if len(datasets_path) > 1:
        raise InvalidCSVFile(
            "There is more than one _datasets.csv file in provided path."
        )
    elif len(datasets_path) == 0:
        raise InvalidCSVFile("There is no _datasets.csv file in provided path.")
    else:
        datasets_path = datasets_path[0]

    dataset_files = [
        p for p in paths if p.name.lower() not in ("_datasets.csv", "_variables.csv")
    ]

    datasets_df = pd.read_csv(datasets_path, encoding=encoding)

    if "Filename" not in datasets_df.columns or "Label" not in datasets_df.columns:
        raise InvalidCSVFile(
            "Metadata files is malformed. One of [Filename, Label] columns is missing."
        )

    allowed_datasets = {
        Path(str(name)).stem.lower() for name in datasets_df["Filename"].dropna()
    }

    filtered = {
        str(p)
        for p in dataset_files
        if p.suffix.lower() == ".csv" and p.stem.lower() in allowed_datasets
    }
    return list(filtered)


def _has_multiple_usdm_json_files(
    standard: str, found_formats: set, dataset_paths: list
) -> bool:
    """Returns True if USDM standard is used with more than one JSON file found."""
    is_usdm_standard = bool(standard) and standard.lower() == "usdm"
    return (
        is_usdm_standard
        and DataFormatTypes.JSON.value in found_formats
        and len(dataset_paths) > 1
    )


def _check_mixed_xlsx_formats(arg_name: str, found_formats: set) -> str | None:
    """Returns the error message if XLSX is mixed with other formats, else None."""
    if DataFormatTypes.XLSX.value in found_formats and len(found_formats) > 1:
        return (
            f"Argument {arg_name} contains XLSX files mixed with other formats ({', '.join(found_formats)}).\n"
            f"Excel format (XLSX) validation only supports single files.\n"
            f"Please provide either a single XLSX file or use other supported formats: "
            f"{VALIDATION_FORMATS_MESSAGE}"
        )
    return None


def _gather_csv_metadata_files(dataset_path: tuple[str]) -> list[str]:
    """Gathers provided dataset paths plus any sibling datasets.csv/variables.csv files."""
    all_files_in_dp = []
    for dp in dataset_path:
        all_files_in_dp.append(dp)
        dp_path = Path(dp)
        all_files_in_dp.extend(
            [
                str(p)
                for p in dp_path.parent.glob("*")
                if p.is_file() and p.name in {"_datasets.csv", "_variables.csv"}
            ]
        )
    return all_files_in_dp


def _directory_empty_message(data: str, found_formats: set) -> str:
    """Returns the appropriate error message when no valid dataset paths remain."""
    if DataFormatTypes.XLSX.value in found_formats and len(found_formats) == 1:
        return (
            f"Multiple XLSX files found in directory: {data}\n"
            f"Excel format (XLSX) validation only supports single files.\n"
            f"Please provide either a single XLSX file or use other supported formats: "
            f"{VALIDATION_FORMATS_MESSAGE}"
        )
    return (
        f"No valid dataset files found in directory: {data}\n"
        f"Supported formats: {VALIDATION_FORMATS_MESSAGE}\n"
        f"Please ensure your directory contains files in one of these formats."
    )


def _dataset_path_empty_message(found_formats: set, filetype: str) -> str:
    """Returns the appropriate error message when no valid dataset paths remain."""
    if DataFormatTypes.XLSX.value in found_formats and len(found_formats) == 1:
        return (
            f"Multiple XLSX files provided.\n"
            f"Excel format (XLSX) validation only supports single files.\n"
            f"Please provide either a single XLSX file or use other supported formats: "
            f"{VALIDATION_FORMATS_MESSAGE}"
        )
    if filetype:
        return (
            f"Provided dataset path does not match the specified file type.\n"
            f"Specified format: {filetype}\n"
            f"Please ensure the file extension matches the selected format."
        )
    return (
        f"No valid dataset files provided.\n"
        f"Supported formats: {VALIDATION_FORMATS_MESSAGE}\n"
        f"Please ensure your files are in one of these formats."
    )


def _validate_data_directory(
    data: str, logger, filetype: str = None, standard: str = None
) -> tuple[list, set]:
    """Validate data directory and return dataset paths and found formats."""
    # Added filetype argument to filter files by extension if provided
    if filetype:
        pattern = f"*.{filetype}"
        dataset_paths, found_formats = valid_data_file(
            [str(p) for p in Path(data).rglob(pattern) if p.is_file()]
        )
    else:
        dataset_paths, found_formats = valid_data_file(
            [str(p) for p in Path(data).rglob("*") if p.is_file()]
        )

    if _has_multiple_usdm_json_files(standard, found_formats, dataset_paths):
        logger.error(
            f"Multiple JSON files found in directory: {data}\n"
            f"USDM validation only supports a single JSON file."
        )
        return [], set()

    xlsx_mixed_message = _check_mixed_xlsx_formats("--data", found_formats)
    if xlsx_mixed_message:
        logger.error(xlsx_mixed_message)
        return [], set()

    if DataFormatTypes.CSV.value in found_formats:
        try:
            dataset_paths = _validate_csv_data_paths(dataset_paths)
        except InvalidCSVFile as e:
            logger.error(e)
            return [], set()

    if not dataset_paths:
        logger.error(_directory_empty_message(data, found_formats))
        return [], set()

    return dataset_paths, found_formats


def _validate_dataset_paths(
    dataset_path: tuple[str], logger, filetype: str, standard
) -> tuple[list, set]:
    """Validate dataset paths and return dataset paths and found formats."""
    if filetype:
        pattern = f"*.{filetype}"
        dataset_paths, found_formats = valid_data_file(
            [
                str(p)
                for p in dataset_path
                if Path(p).match(pattern)
                if Path(p).is_file()
            ]
        )
    else:
        dataset_paths, found_formats = valid_data_file([dp for dp in dataset_path])

    if _has_multiple_usdm_json_files(standard, found_formats, dataset_paths):
        logger.error(
            "Multiple JSON files provided for --dataset-path.\n"
            "USDM validation only supports a single JSON file."
        )
        return [], set()

    xlsx_mixed_message = _check_mixed_xlsx_formats("--dataset-path", found_formats)
    if xlsx_mixed_message:
        logger.error(xlsx_mixed_message)
        return [], set()

    if DataFormatTypes.CSV.value in found_formats:
        try:
            dataset_paths = _validate_csv_data_paths(
                _gather_csv_metadata_files(dataset_path)
            )
        except InvalidCSVFile as e:
            logger.error(e)
            return [], set()

    if not dataset_paths:
        logger.error(_dataset_path_empty_message(found_formats, filetype))
        return [], set()

    return dataset_paths, found_formats


def _validate_no_arguments(logger) -> None:
    """Validate that at least one dataset argument is provided."""
    logger.error("You must pass one of the following arguments: --dataset-path, --data")


def load_custom_dotenv(_ctx, _param, value):
    if not value:
        return value
    if os.path.exists(value):
        load_dotenv(value, override=False)
    return value


def load_custom_dotenv_from_data_options(_ctx, _param, value):
    if not value:
        return value
    if isinstance(value, str):
        env_path = os.path.join(value, ".env")
    elif isinstance(value, tuple):
        env_path = next((os.path.join(Path(v).parent, ".env") for v in value), None)
    else:
        return value
    if os.path.exists(env_path):
        load_dotenv(env_path, override=False)
    return value


@click.command()
@click.option(
    "-ca",
    "--cache",
    default=DEFAULT_CACHE_PATH,
    help="Relative path to cache files containing pre loaded metadata and rules",
)
@click.option(
    "-ps",
    "--pool-size",
    default=10,
    type=int,
    help="Number of parallel processes for validation",
)
@click.option(
    "-dep",
    "--dotenv-path",
    required=False,
    help="Path to .env file containing custom dotenv values",
    callback=load_custom_dotenv,
)
@click.option(
    "-d",
    "--data",
    required=False,
    help=f"Path to directory containing data files ({VALIDATION_FORMATS_MESSAGE}). "
    "Should be provided once. If provided more than once, only the last value will be recorded",
    callback=load_custom_dotenv_from_data_options,
)
@click.option(
    "-ft",
    "--filetype",
    default=None,
    required=False,
    help="File extension to use for input files in the data directory (e.g., 'json', 'xpt', 'xlsx', 'ndjson')",
)
@click.option(
    "-dp",
    "--dataset-path",
    required=False,
    multiple=True,
    help=f"Absolute path to dataset file ({VALIDATION_FORMATS_MESSAGE})",
    callback=load_custom_dotenv_from_data_options,
)
@click.option(
    "-l",
    "--log-level",
    default="disabled",
    type=click.Choice(["info", "debug", "error", "critical", "disabled", "warn"]),
    help="Sets log level for engine logs, logs are disabled by default",
)
@click.option(
    "-rt",
    "--report-template",
    help="File path of report template to use for excel output",
)
@click.option(
    "-s",
    "--standard",
    required=True,
    help="CDISC standard to validate against",
    envvar="PRODUCT",
)
@click.option(
    "-v",
    "--version",
    required=True,
    help="Standard version to validate against",
    envvar="VERSION",
)
@click.option(
    "-ss",
    "--substandard",
    default=None,
    type=click.Choice(["sdtm", "send", "adam", "cdash"], case_sensitive=False),
    help="CDISC Substandard to validate against. Any of SDTM, SEND, ADaM, CDASH",
    envvar="SUBSTANDARD",
)
@click.option(
    "-uc",
    "--use-case",
    required=False,
    default=None,
    type=click.Choice(["INDH", "PROD", "NONCLIN", "ANALYSIS"], case_sensitive=True),
    help=(
        "Specifies the CDISC TIG use case for all custom domains in a validation run"
        "Any of INDH, PROD, NONCLIN, or ANALYSIS."
    ),
    envvar="USE_CASE",
)
@click.option(
    "-ct",
    "--controlled-terminology-package",
    multiple=True,
    help=(
        "Controlled terminology package to validate against, "
        "can provide more than one"
    ),
    envvar="CT",
)
@click.option(
    "-o",
    "--output",
    default=generate_report_filename(datetime.now().isoformat()),
    help="Report output file destination",
)
@click.option(
    "-of",
    "--output-format",
    multiple=True,
    default=[ReportTypes.XLSX.value],
    type=click.Choice(list(ReportTypes.values()), case_sensitive=False),
    help="Output file format",
)
@click.option(
    "-rr",
    "--raw-report",
    default=False,
    show_default=True,
    is_flag=True,
    help=(
        "Report in a raw format as it is generated by the engine. "
        "This flag must be used only with --output-format JSON."
    ),
)
@click.option("--whodrug", help="Path to directory with WHODrug dictionary files")
@click.option("--meddra", help="Path to directory with MedDRA dictionary files")
@click.option("--loinc", help="Path to directory with LOINC dictionary files")
@click.option("--medrt", help="Path to directory with MEDRT dictionary files")
@click.option("--unii", help="Path to directory with UNII dictionary files")
@click.option("--snomed-version", help="Version of snomed to use.")
@click.option(
    "--snomed-url",
    help="The Base URL of snomed to use. Defaults to snowstorm test instance",
    default="https://snowstorm.snomedtools.org/snowstorm/snomed-ct/",
)
@click.option("--snomed-edition", help="Edition of snomed to use.")
@click.option(
    "--rules",
    "-r",
    multiple=True,
    help="Specify rule core ID ex. CORE-000001. Can be specified multiple times",
)
@click.option(
    "--exclude-rules",
    "-er",
    multiple=True,
    help="Specify rule core ID to exclude, ex. CORE-000001. Can be specified multiple times",
)
@click.option(
    "--local-rules",
    "-lr",
    required=False,
    type=click.Path(exists=True, readable=True, resolve_path=True),
    help="Path to directory containing local rules.",
    multiple=True,
)
@click.option(
    "--custom-standard",
    "-cs",
    required=False,
    is_flag=True,
    default=False,
    help=("Flag to run a validation using a custom standard from the cache"),
)
@click.option(
    "-p",
    "--progress",
    default=ProgressParameterOptions.BAR.value,
    type=click.Choice(list(ProgressParameterOptions.values())),
    help=(
        "Defines how to display the validation progress. "
        'By default a progress bar like "[████████████████████████████--------]   78%"'
        "is printed."
    ),
)
@click.option(
    "-dxp",
    "--define-xml-path",
    required=False,
    help="Path to Define-XML",
    envvar="DEFINE_XML",
)
@click.option(
    "-vx",
    "--validate-xml",
    default="y",
    help="Enable XML validation (default 'y' to enable, otherwise disable)",
)
@click.option(
    "-jcf",
    "--jsonata-custom-functions",
    default=[],
    multiple=True,
    required=False,
    type=(
        str,
        click.Path(exists=True, file_okay=False, readable=True, resolve_path=True),
    ),
    help="Variable Name and Path to directory containing a set of custom JSONata functions.",
)
@click.option(
    "-mr",
    "--max-report-rows",
    type=int,
    default=None,
    required=False,
    help="Maximum number of rows per report sheet.",
)
@click.option(
    "-me",
    "--max-errors-per-rule",
    type=(int, bool),
    default=(0, False),
    required=False,
    help=(
        "Maximum number of errors per rule. "
        "Usage: -me <limit> <per_dataset_flag>. "
        "Example: -me 100 true. "
        "If per_dataset_flag is false (default), applies cumulative limit across datasets. "
        "If true, limits reported issues per dataset per rule."
    ),
)
@click.option(
    "-e",
    "--encoding",
    default=DEFAULT_ENCODING,
    required=False,
    callback=validate_encoding,
    help=(
        f"File encoding for reading datasets. "
        f"Defaults to {DEFAULT_ENCODING}. "
        f"Supported encodings: utf-8, utf-16, utf-32, cp1252, latin-1, etc."
    ),
)
@click.option(
    "-vcp",
    "--variables-csv-path",
    required=False,
    help="Path to _variables.csv",
)
@click.option(
    "-dcp",
    "--datasets-csv-path",
    required=False,
    help="Path to _datasets.csv",
)
@click.pass_context
def validate(  # noqa
    ctx,
    cache: str,
    pool_size: int,
    dotenv_path: str,
    data: str,
    filetype: str,
    dataset_path: tuple[str],
    log_level: str,
    report_template: str,
    standard: str,
    version: str,
    substandard: str,
    use_case: str,
    controlled_terminology_package: tuple[str],
    output: str,
    output_format: tuple[str],
    raw_report: bool,
    whodrug: str,
    meddra: str,
    loinc: str,
    medrt: str,
    unii: str,
    snomed_version: str,
    snomed_edition: str,
    snomed_url: str,
    rules: tuple[str],
    exclude_rules: tuple[str],
    local_rules: str,
    custom_standard: bool,
    progress: str,
    define_xml_path: str,
    validate_xml: str,
    jsonata_custom_functions: tuple[()] | tuple[tuple[str, str], ...],
    max_report_rows: int,
    max_errors_per_rule: tuple[int, bool],
    encoding: str,
    variables_csv_path: str,
    datasets_csv_path: str,
):
    """
    Validate data using CDISC Rules Engine

    Example:

    python core.py -s SDTM -v 3.4 -d /path/to/datasets
    """

    # Validate conditional options
    logger = logging.getLogger("validator")
    load_dotenv(dotenv_path)
    validate_dataset_files_exist(dataset_path, logger, ctx)

    if define_xml_path and dotenv_path and os.getenv("DEFINE_XML"):
        dxp = Path(define_xml_path)
        if not dxp.is_absolute() and dxp.parent == Path("."):
            resolved = Path(dotenv_path).parent / define_xml_path
            if resolved.is_file():
                define_xml_path = str(resolved)

    if not custom_standard:
        standard = standard.lower()
        supported_standards = StandardTypes.values()
        if standard not in supported_standards:
            supported_list = ", ".join(sorted(supported_standards))
            logger.error(
                f"Standard '{standard}' is not a supported standard. "
                f"Supported standards: {supported_list}. "
                f"Use --custom-standard flag for custom standards."
            )
            ctx.exit(2)

    if raw_report:
        if not (len(output_format) == 1 and output_format[0] == ReportTypes.JSON.value):
            logger.error(
                "Flag --raw-report can be used only when --output-format is JSON"
            )
            ctx.exit(2)

    if exclude_rules and rules:
        logger.error("Cannot use both --rules and --exclude-rules flags together.")
        ctx.exit(2)

    cache_path: str = os.path.join(os.path.dirname(__file__), cache)

    if standard == "tig":
        if not substandard:
            logger.error("Standard 'tig' requires --substandard to be specified.")
            ctx.exit(2)
    # Construct ExternalDictionariesContainer:
    external_dictionaries = ExternalDictionariesContainer(
        {
            DictionaryTypes.UNII.value: unii,
            DictionaryTypes.MEDRT.value: medrt,
            DictionaryTypes.MEDDRA.value: meddra,
            DictionaryTypes.WHODRUG.value: whodrug,
            DictionaryTypes.LOINC.value: loinc,
            DictionaryTypes.SNOMED.value: {
                "edition": snomed_edition,
                "version": snomed_version,
                "base_url": snomed_url,
            },
        }
    )
    # Validate dataset arguments
    if data:
        if dataset_path:
            logger.error(
                "Argument --dataset-path cannot be used together with argument --data"
            )
            ctx.exit(2)
        dataset_paths, found_formats = _validate_data_directory(
            data, logger, filetype, standard
        )
        if not dataset_paths:
            ctx.exit(2)
    elif dataset_path:
        dataset_paths, found_formats = _validate_dataset_paths(
            dataset_path, logger, filetype, standard
        )
        if not dataset_paths:
            ctx.exit(2)
    else:
        _validate_no_arguments(logger)
        ctx.exit(2)
    validate_xml_bool = True if validate_xml.lower() in ("y", "yes") else False
    run_validation(
        Validation_args(
            cache_path,
            pool_size,
            dataset_paths,
            log_level,
            report_template,
            standard,
            version,
            substandard,
            use_case,
            set(controlled_terminology_package),
            output,
            set(output_format),
            raw_report,
            external_dictionaries,
            rules,
            exclude_rules,
            local_rules,
            custom_standard,
            progress,
            define_xml_path,
            validate_xml_bool,
            jsonata_custom_functions,
            max_report_rows,
            max_errors_per_rule,
            encoding,
            variables_csv_path,
            datasets_csv_path,
        )
    )


@click.command()
@click.option(
    "-c",
    "--cache-path",
    default=DEFAULT_CACHE_PATH,
    help="Relative path to cache files containing pre loaded metadata and rules",
)
@click.option(
    "--apikey",
    envvar="CDISC_LIBRARY_API_KEY",
    help=(
        "CDISC Library api key. "
        "Can be provided in the environment "
        "variable CDISC_LIBRARY_API_KEY"
    ),
    required=False,
    default=None,
)
@click.option(
    "-crd",
    "--custom-rules-directory",
    help=(
        "Relative path to directory containing local rules in yaml or JSON formats"
        "to be added to the cache. "
    ),
)
@click.option(
    "-cr",
    "--custom-rule",
    multiple=True,
    help=(
        "Relative path to rule file in yaml or JSON formats"
        "to be added to the cache. "
    ),
)
@click.option(
    "-rcr",
    "--remove-custom-rules",
    help=(
        "Remove rules from the cache. Can be a single rule ID, a comma-separated list of IDs, "
        "or 'ALL' to remove all custom rules."
    ),
)
@click.option(
    "-ucr",
    "--update-custom-rule",
    help=(
        "Relative path to rule file in yaml or JSON formats"
        "Rule will be updated in cache with this file. "
    ),
)
@click.option(
    "-cs",
    "--custom-standard",
    help=(
        "Relative path to JSON file containing custom standard details."
        "Will update the standard if it already exists."
    ),
)
@click.option(
    "-cse",
    "--custom-standard-encoding",
    help="Encoding for custom standard details. ",
)
@click.option(
    "-rcs",
    "--remove-custom-standard",
    help="Removes a custom standard and version from the cache. ",
    multiple=True,
)
@click.pass_context
def update_cache(
    ctx: click.Context,
    cache_path: str,
    apikey: str,
    custom_rules_directory: str,
    custom_rule: str,
    remove_custom_rules: str,
    update_custom_rule: str,
    custom_standard: str,
    custom_standard_encoding: str,
    remove_custom_standard: str,
):
    cache = CacheServiceFactory(config).get_cache_service()
    # CDISC library service should log failed requests instead of failing the
    # cache update process
    library_service = CDISCLibraryService(apikey, cache, raise_on_error=False)
    update_rules_only = not apikey
    cache_populator = CachePopulator(
        cache,
        library_service,
        custom_rules_directory,
        custom_rule,
        remove_custom_rules,
        update_custom_rule,
        custom_standard,
        custom_standard_encoding,
        remove_custom_standard,
        cache_path,
        rules_only=update_rules_only,
    )

    if update_rules_only:
        logger = logging.getLogger("validator")
        logger.warning("API key was not provided. Only CORE rules will be updated.")

    if custom_rule or custom_rules_directory:
        cache_populator.add_custom_rules()
    elif remove_custom_rules:
        cache_populator.remove_custom_rules_from_cache()
    elif update_custom_rule:
        cache_populator.update_custom_rule_in_cache()
    elif custom_standard:
        cache_populator.add_custom_standard_to_cache()
    elif remove_custom_standard:
        cache_populator.remove_custom_standards_from_cache()
    else:
        asyncio.run(cache_populator.update_cache())

    print("Cache update complete")


@click.command()
@click.option(
    "-c",
    "--cache-path",
    default=DEFAULT_CACHE_PATH,
    help="Relative path to cache files containing pre loaded metadata and rules",
)
@click.option(
    "-s", "--standard", required=False, help="CDISC standard to get rules for"
)
@click.option(
    "-v", "--version", required=False, help="Standard version to get rules for"
)
@click.option(
    "-ss",
    "--substandard",
    required=False,
    default=None,
    help="CDISC substandard to get rules for. Any of SDTM, SEND, ADaM, CDASH",
)
@click.option(
    "-cr",
    "--custom-rules",
    is_flag=True,
    default=False,
    required=False,
    help="Flag to list custom rules in the cache",
)
@click.option(
    "-r",
    "--rule-id",
    required=False,
    help="Rule ID to get rule for.",
    multiple=True,
)
@click.pass_context
def list_rules(
    ctx: click.Context,
    cache_path: str,
    standard: str,
    version: str,
    substandard: str,
    custom_rules: bool,
    rule_id: str,
):
    # Load all rules
    if custom_rules:
        rules_file = DefaultFilePaths.CUSTOM_RULES_CACHE_FILE.value
        dict_file = DefaultFilePaths.CUSTOM_RULES_DICTIONARY.value
    else:
        rules_file = DefaultFilePaths.RULES_CACHE_FILE.value
        dict_file = DefaultFilePaths.RULES_DICTIONARY.value
    with open(os.path.join(cache_path, rules_file), "rb") as f:
        rules_data = pickle.load(f)
    with open(os.path.join(cache_path, dict_file), "rb") as f:
        rules_dict = pickle.load(f)
    rules = []
    if rule_id:
        for id in rule_id:
            if id in rules_data:
                rules.append(rules_data[id])
    elif standard and version:
        key_prefix = get_rules_cache_key(
            standard, version.replace(".", "-"), substandard
        )
        if key_prefix in rules_dict:
            rule_ids = rules_dict[key_prefix]
            for rid in rule_ids:
                if rid in rules_data:
                    rules.append(rules_data[rid])
    else:
        # Print all rules
        rules = list(rules_data.values())
    print(json.dumps(rules, indent=4))


@click.command()
@click.option(
    "-c",
    "--cache-path",
    default=DEFAULT_CACHE_PATH,
    help="Relative path to cache files containing pre loaded metadata and rules",
)
@click.option(
    "-o",
    "--custom",
    is_flag=True,
    default=False,
    help="Flag to list all custom standards and versions in the cache",
)
@click.pass_context
def list_rule_sets(ctx: click.Context, cache_path: str, custom: bool):
    """Lists all standards and versions for which rules are available."""
    if custom:
        rules_file = DefaultFilePaths.CUSTOM_RULES_DICTIONARY.value
    else:
        rules_file = DefaultFilePaths.RULES_DICTIONARY.value
    with open(os.path.join(cache_path, rules_file), "rb") as f:
        rules_data = pickle.load(f)
    rule_sets = {}
    for key in rules_data.keys():
        if "/" in key:
            parts = key.split("/")
            standard = parts[0]
            version = parts[1]
            substandard = parts[2] if len(parts) > 2 else None
            if standard not in rule_sets:
                rule_sets[standard] = set()
            rule_sets[standard].add((version, substandard))
    for standard in sorted(rule_sets.keys()):
        versions = sorted(rule_sets[standard], key=lambda x: (x[0], x[1] or ""))
        for version, substandard in versions:
            if substandard:
                print(f"{standard}, {version}, {substandard}")
            else:
                print(f"{standard}, {version}")


@click.command()
@click.option(
    "-dp",
    "--dataset-path",
    required=True,
    multiple=True,
)
@click.pass_context
def list_dataset_metadata(ctx: click.Context, dataset_path: tuple[str]):
    """
    Command that lists metadata of given datasets.

    Input:
        core.py list-ds-metadata -dp=path_1 -dp=path_2 -dp=path_3 ...
    Output:
        [
           {
              "domain":"AE",
              "filename":"ae.xpt",
              "full_path":"/Users/Aleksei_Furmenkov/PycharmProjects/cdisc-rules-engine/resources/data/ae.xpt",
              "file_size":"38000",
              "label":"Adverse Events",
              "modification_date":"2020-08-21T09:14:26"
           },
           {
              "domain":"EX",
              "filename":"ex.xpt",
              "full_path":"/Users/Aleksei_Furmenkov/PycharmProjects/cdisc-rules-engine/resources/data/ex.xpt",
              "file_size":"78050",
              "label":"Exposure",
              "modification_date":"2021-09-17T09:23:22"
           },
           ...
        ]
    """
    print(json.dumps(list_dataset_metadata_handler(dataset_path), indent=4))


@click.command()
def version():
    print(__version__)


@click.command()
@click.option(
    "-c",
    "--cache-path",
    default=DEFAULT_CACHE_PATH,
    help="Relative path to cache files containing pre loaded metadata and rules",
)
@click.option(
    "-s",
    "--subsets",
    help="CT package subset type. Ex: sdtmct. Multiple values allowed",
    required=False,
    multiple=True,
)
def list_ct(cache_path: str, subsets: set[str]):
    """
    Command to list the ct packages available in the cache.
    """
    if subsets:
        subsets = set([subset.lower() for subset in subsets])

    for file in os.listdir(cache_path):
        file_prefix = file[0 : file.find("ct-") + 2]
        if file_prefix.endswith("ct") and (not subsets or file_prefix in subsets):
            print(os.path.splitext(file)[0])


@click.command()
@click.argument("filetype", type=click.Choice(["json", "xpt"], case_sensitive=False))
def test_validate(filetype):
    """
    Verify CORE with a test validation.
    Requires FILETYPE argument: 'json' or 'xpt'.
    Report file is confirmed and automatically cleaned up. For actual validation, use 'validate' command.
    """
    try:
        base_path = os.path.join("resources", "datasets")
        if filetype.lower() == "json":
            test_file = os.path.join(base_path, "TS.json")
            output_name = "json_validation_output"
        else:
            test_file = os.path.join(base_path, "ae.xpt")
            output_name = "xpt_validation_output"
        if not os.path.exists(test_file):
            raise FileNotFoundError(f"Test dataset not found: {test_file}")
        cache_path = DEFAULT_CACHE_PATH
        pool_size = 10
        log_level = "disabled"
        standard = "sdtmig"
        version = "3.4"
        output_format = {ReportTypes.XLSX.value}
        external_dictionaries = ExternalDictionariesContainer({})
        progress = ProgressParameterOptions.BAR.value
        max_report_errors = (0, False)

        with tempfile.TemporaryDirectory() as temp_dir:
            output = os.path.join(temp_dir, output_name)
            run_validation(
                Validation_args(
                    cache_path,
                    pool_size,
                    [test_file],
                    log_level,
                    None,
                    standard,
                    version,
                    None,
                    None,
                    set(),
                    output,
                    output_format,
                    False,
                    external_dictionaries,
                    [],
                    [],
                    None,
                    False,
                    progress,
                    None,
                    False,
                    (),
                    None,
                    max_report_errors,
                    None,
                    None,
                    None,
                )
            )
            print(f"{filetype.upper()} validation completed successfully!")
        sys.exit(0)
    except Exception as e:
        import traceback

        print(f"{filetype.upper()} validation test failed: {str(e)}")
        print(traceback.format_exc())
        sys.exit(1)


cli.add_command(test_validate)
cli.add_command(validate)
cli.add_command(update_cache)
cli.add_command(list_rules)
cli.add_command(list_rule_sets)
cli.add_command(list_dataset_metadata)
cli.add_command(version)
cli.add_command(list_ct)

if __name__ == "__main__":
    freeze_support()
    cli()