Release 0.12.0

bedboss/bbuploader/__init__.py

@@ -1,13 +1 @@
 """Package-level data"""
-
-import coloredlogs
-import logmuse
-
-from bedboss.bbuploader.constants import PKG_NAME
-
-_LOGGER = logmuse.init_logger(PKG_NAME)
-coloredlogs.install(
-    logger=_LOGGER,
-    datefmt="%H:%M:%S",
-    fmt="[%(levelname)s] [%(asctime)s] [BBUPLOADER] %(message)s",
-)

bedboss/bbuploader/main.py

@@ -753,7 +753,6 @@ def _upload_gse(
                 bedbase_config=bedbase_config,
                 narrowpeak=required_metadata.narrowpeak,
                 other_metadata=required_metadata.pep.model_dump(),
-                upload_pephub=True,
                 upload_s3=True,
                 upload_qdrant=True,
                 force_overwrite=overwrite,
@@ -831,7 +830,6 @@ def _upload_gse(
             name=gse,
             description=project.description,
             heavy=False,  # TODO: set to False because can't handle bedset > 10 files
-            upload_pephub=True,
             upload_s3=True,
             no_fail=True,
             force_overwrite=overwrite_bedset,

bedboss/bedboss.py

@@ -79,7 +79,6 @@ def run_all(
     update: bool = False,
     upload_qdrant: bool = False,
     upload_s3: bool = False,
-    upload_pephub: bool = False,
     lite: bool = False,
     # Universes
     universe: bool = False,
@@ -90,7 +89,7 @@ def run_all(
     reference_genome_validator: ReferenceValidator = None,
 ) -> str:
     """
-    Run bedboss: bedmaker -> bedqc -> bedclassifier -> bedstat -> upload to s3, qdrant, pephub, and bedbase.
+    Run bedboss: bedmaker -> bedqc -> bedclassifier -> bedstat -> upload to s3, qdrant, and bedbase.
 
     Args:
         input_file: Input file path.
@@ -113,7 +112,6 @@ def run_all(
         update: Whether to update the record in the database. If True, overwrites 'force_overwrite'. Default: False.
         upload_qdrant: Whether to upload to qdrant. Default: False.
         upload_s3: Whether to upload to s3.
-        upload_pephub: Whether to push bedfiles and metadata to pephub. Default: False.
         lite: Whether to run lite version of the pipeline. Default: False.
         universe: Whether to add the sample as the universe. Default: False.
         universe_method: Method used to create the universe.
@@ -280,7 +278,6 @@ def run_all(
             ref_validation=ref_valid_stats,
             license_id=license_id,
             upload_qdrant=upload_qdrant and not lite,
-            upload_pephub=upload_pephub,
             upload_s3=upload_s3,
             local_path=outfolder,
             overwrite=True,
@@ -298,7 +295,6 @@ def run_all(
             ref_validation=ref_valid_stats,
             license_id=license_id,
             upload_qdrant=upload_qdrant and not lite,
-            upload_pephub=upload_pephub,
             upload_s3=upload_s3,
             local_path=outfolder,
             overwrite=force_overwrite,
@@ -337,7 +333,6 @@ def insert_pep(
     force_overwrite: bool = False,
     update: bool = False,
     upload_s3: bool = False,
-    upload_pephub: bool = False,
     upload_qdrant: bool = False,
     no_fail: bool = False,
     standardize_pep: bool = False,
@@ -366,7 +361,6 @@ def insert_pep(
         force_overwrite: Whether to overwrite the existing record.
         update: Whether to update the record in the database. Overwrites force_overwrite. Default: False.
         upload_s3: Whether to upload to s3.
-        upload_pephub: Whether to push bedfiles and metadata to pephub. Default: False.
         upload_qdrant: Whether to execute qdrant indexing.
         no_fail: Whether to raise an error if bedset was not added to the database.
         standardize_pep: Whether to standardize the pep file before processing by using bedms. Default: False.
@@ -458,7 +452,6 @@ def insert_pep(
                 update=update,
                 upload_qdrant=upload_qdrant,
                 upload_s3=upload_s3,
-                upload_pephub=upload_pephub,
                 universe=pep_sample.get("universe"),
                 universe_method=pep_sample.get("universe_method"),
                 universe_bedset=pep_sample.get("universe_bedset"),
@@ -486,7 +479,6 @@ def insert_pep(
             output_folder=output_folder,
             description=pep.description,
             heavy=bedset_heavy,
-            upload_pephub=upload_pephub,
             upload_s3=upload_s3,
             no_fail=no_fail,
             force_overwrite=force_overwrite,
@@ -608,7 +600,6 @@ def reprocess_all(
                 update=True,
                 upload_qdrant=True,
                 upload_s3=True,
-                upload_pephub=True,
                 lite=False,
                 universe=False,
                 universe_method=None,
@@ -714,7 +705,6 @@ def reprocess_one(
         update=True,
         upload_qdrant=True,
         upload_s3=True,
-        upload_pephub=True,
         lite=False,
         universe=False,
         universe_method=None,
@@ -764,7 +754,6 @@ def reprocess_bedset(
         output_folder=output_folder,
         description=bedset_annot.description,
         heavy=heavy,
-        upload_pephub=False,
         upload_s3=heavy,
         no_fail=no_fail,
         force_overwrite=True,

bedboss/bedboss_hpc.py

@@ -90,7 +90,6 @@ class BoolFlagSpec(BaseModel):
     "update": BoolFlagSpec(on="--update"),
     "upload_qdrant": BoolFlagSpec(on="--upload-qdrant", off="--no-upload-qdrant"),
     "upload_s3": BoolFlagSpec(on="--upload-s3", off="--no-upload-s3"),
-    "upload_pephub": BoolFlagSpec(on="--upload-pephub", off="--no-upload-pephub"),
     "no_fail": BoolFlagSpec(on="--no-fail"),
     "standardize_pep": BoolFlagSpec(on="--standardize-pep"),
     "lite": BoolFlagSpec(on="--lite"),
@@ -160,7 +159,6 @@ class RunPepArgs(BaseModel):
     update: bool = False
     upload_qdrant: bool = True
     upload_s3: bool = True
-    upload_pephub: bool = True
     no_fail: bool = False
     license_id: Optional[str] = None
     standardize_pep: bool = False

bedboss/bedbuncher/bedbuncher.py

@@ -99,7 +99,6 @@ def run_bedbuncher(
     description: str = None,
     annotation: dict = None,
     heavy: bool = False,
-    upload_pephub: bool = False,
     upload_s3: bool = False,
     no_fail: bool = False,
     force_overwrite: bool = False,
@@ -118,7 +117,6 @@ def run_bedbuncher(
         annotation: Bedset annotation (author, source, summary, etc.).
         heavy: Whether to use heavy processing (add all columns to the database).
             If False, R-script won't be executed, only basic statistics will be calculated.
-        upload_pephub: Whether to create a view in pephub.
         upload_s3: Whether to upload files to s3.
         no_fail: Whether to raise an error if bedset was not added to the database.
         force_overwrite: Whether to overwrite the record in the database.
@@ -158,7 +156,6 @@ def run_bedbuncher(
         bedid_list=bed_set,
         statistics=True,
         description=description,
-        upload_pephub=upload_pephub,
         upload_s3=upload_s3,
         plots=plots.model_dump(exclude_none=True, exclude_unset=True) if plots else {},
         local_path=output_folder,
@@ -175,7 +172,6 @@ def run_bedbuncher_form_pep(
     output_folder: str,
     bedset_name: str = None,
     heavy: bool = False,
-    upload_pephub: bool = False,
     upload_s3: bool = False,
     no_fail: bool = False,
     force_overwrite: bool = False,
@@ -190,7 +186,6 @@ def run_bedbuncher_form_pep(
         bedset_name: Name of the bedset.
         heavy: Whether to use heavy processing (add all columns to the database).
             If False, R-script won't be executed, only basic statistics will be calculated.
-        upload_pephub: Whether to create a view in pephub.
         upload_s3: Whether to upload files to s3.
         no_fail: Whether to raise an error if bedset was not added to the database.
         force_overwrite: Whether to overwrite the record in the database.
@@ -223,7 +218,6 @@ def run_bedbuncher_form_pep(
         name=bedset_name or pep_of_bed.name,
         description=pep_of_bed.description,
         heavy=heavy,
-        upload_pephub=upload_pephub,
         upload_s3=upload_s3,
         no_fail=no_fail,
         force_overwrite=force_overwrite,
@@ -244,13 +238,11 @@ def run_bedbuncher_form_pep(
 #         description="This is a description",
 #         upload_s3=True,
 #         no_fail=True,
-#         upload_pephub=True,
 #         heavy=True,
 #     )
 
 # run_bedbuncher_form_pep(
 #     "/home/bnt4me/virginia/repos/bbuploader/config_db_local.yaml",
 #     bedset_name="pephub_test",
 #     bedset_pep="khoroshevskyi/bedbunch:default",
-#     upload_pephub=True,
 # )

bedboss/bedclassifier/bedclassifier.py

@@ -108,7 +108,7 @@ def _read_bed_file(filepath: str, skiprows: int = 0) -> pd.DataFrame | None:
         else:
             raise BedTypeException(reason="Input is not a string or dataframe.")
 
-    df = df.dropna(axis=1)
+    df = df.dropna(axis=1, how="all")
     df.columns = range(len(df.columns))
     num_cols = len(df.columns)
     compliant_columns = 0

bedboss/cli.py

@@ -1,13 +1,16 @@
 import os
 from importlib.metadata import version as _pkg_version
 
+import logmuse
 import typer
 
 __version__ = _pkg_version("bedboss")
 from pephubclient.helpers import MessageHandler as printm
 
 from bedboss.bbuploader.cli import app_bbuploader
 from bedboss.qdrant_index.qdrant_cli import qdrant_app
+from bedboss.scripts.analysis_files import files_app
+from bedboss.scripts.snapshot import snapshot_app
 
 # commented and made new const here, because it speeds up help function,
 # from bbconf.const import DEFAULT_LICENSE
@@ -100,7 +103,6 @@ def run_all(
     ),
     upload_qdrant: bool = typer.Option(False, help="Upload to Qdrant"),
     upload_s3: bool = typer.Option(False, help="Upload to S3"),
-    upload_pephub: bool = typer.Option(False, help="Upload to PEPHub"),
     # Universes
     universe: bool = typer.Option(False, help="Create a universe"),
     universe_method: str = typer.Option(
@@ -144,7 +146,6 @@ def run_all(
         update=update,
         upload_qdrant=upload_qdrant,
         upload_s3=upload_s3,
-        upload_pephub=upload_pephub,
         universe=universe,
         universe_method=universe_method,
         universe_bedset=universe_bedset,
@@ -181,7 +182,6 @@ def run_pep(
     ),
     upload_qdrant: bool = typer.Option(True, help="Upload to Qdrant"),
     upload_s3: bool = typer.Option(True, help="Upload to S3"),
-    upload_pephub: bool = typer.Option(True, help="Upload to PEPHub"),
     no_fail: bool = typer.Option(False, help="Do not fail on error"),
     license_id: str = typer.Option(DEFAULT_LICENSE, help="License ID"),
     standardize_pep: bool = typer.Option(False, help="Standardize the PEP using bedMS"),
@@ -222,7 +222,6 @@ def run_pep(
         update=update,
         license_id=license_id,
         upload_s3=upload_s3,
-        upload_pephub=upload_pephub,
         upload_qdrant=upload_qdrant,
         no_fail=no_fail,
         standardize_pep=standardize_pep,
@@ -267,7 +266,6 @@ def run_pep_hpc(
     update: bool = typer.Option(False, help="Update existing records"),
     upload_qdrant: bool = typer.Option(True, help="Upload to Qdrant"),
     upload_s3: bool = typer.Option(True, help="Upload to S3"),
-    upload_pephub: bool = typer.Option(True, help="Upload to PEPHub"),
     no_fail: bool = typer.Option(False, help="Do not fail on error"),
     license_id: str = typer.Option(DEFAULT_LICENSE, help="License ID"),
     standardize_pep: bool = typer.Option(False, help="Standardize the PEP using bedMS"),
@@ -307,7 +305,6 @@ def run_pep_hpc(
         update=update,
         upload_qdrant=upload_qdrant,
         upload_s3=upload_s3,
-        upload_pephub=upload_pephub,
         no_fail=no_fail,
         license_id=license_id,
         standardize_pep=standardize_pep,
@@ -421,7 +418,7 @@ def reprocess_bedset(
     )
 
 
-@app.command(help=f"Create a bed files form a [{', '.join(options_list)}] file")
+@app.command(help=f"Create a bed files from a [{', '.join(options_list)}] file")
 def make_bed(
     input_file: str = typer.Option(
         ...,
@@ -584,7 +581,6 @@ def make_bedset(
         False, help="Force overwrite the output files"
     ),
     upload_s3: bool = typer.Option(False, help="Upload to S3"),
-    upload_pephub: bool = typer.Option(False, help="Upload to PEPHub"),
     no_fail: bool = typer.Option(False, help="Do not fail on error"),
 ):
     from bedboss.bedbuncher.bedbuncher import run_bedbuncher_form_pep
@@ -595,7 +591,6 @@ def make_bedset(
         output_folder=outfolder,
         bedset_name=bedset_name,
         heavy=heavy,
-        upload_pephub=upload_pephub,
         upload_s3=upload_s3,
         no_fail=no_fail,
         force_overwrite=force_overwrite,
@@ -900,9 +895,25 @@ def common(
     version: bool = typer.Option(
         None, "--version", "-v", callback=version_callback, help="App version"
     ),
+    verbosity: int = typer.Option(
+        None, "--verbosity", help="Set logging level: 1 (CRITICAL) to 5 (DEBUG)"
+    ),
+    logdev: bool = typer.Option(False, "--logdev", help="Use developer logging format"),
+    silent: bool = typer.Option(False, "--silent", help="Silence logging"),
 ):
-    pass
+    # This callback runs before any command, so it is where logging gets
+    # configured. Don't move this into `__init__.py`.
+    #
+    # Configure the root logger so that bedboss and its dependencies (pipestat,
+    # geniml, bbconf, pephubclient, refgenconf, bbuploader) all log through one
+    # handler. Each line is tagged with its logger name, so there is no need to
+    # attach a per-package handler to identify the source.
+    logmuse.init_logger(
+        "", make_root=True, verbosity=verbosity, devmode=logdev, silent=silent
+    )
 
 
 app.add_typer(app_bbuploader, name="geo")
 app.add_typer(qdrant_app, name="qdrant")
+app.add_typer(snapshot_app, name="snapshot")
+app.add_typer(files_app, name="files")

bedboss/refgenome_validator/main.py

@@ -368,9 +368,7 @@ def calculate_rating(self, compat_stats: CompatibilityStats) -> RatingModel:
             if sequence_fit < 0.90:
                 points_rating += 1
             if sequence_fit < 0.60:
-                points_rating += 1
-            if sequence_fit < 0.60:
-                points_rating += 1
+                points_rating += 2
 
         else:
             # if no chrom names were found during assessment