Store balsamic

cg/apps/hk.py

@@ -1,22 +1,26 @@
-# -*- coding: utf-8 -*-
+""" Module to decouple cg code from Housekeeper code """
 import datetime as dt
 import logging
 import os
 from pathlib import Path
 
-from housekeeper.exc import VersionIncludedError
 from housekeeper.include import include_version, checksum as hk_checksum
 from housekeeper.store import Store, models
 
-log = logging.getLogger(__name__)
+LOG = logging.getLogger(__name__)
 
 
-class HousekeeperAPI(Store):
+class HousekeeperAPI:
+    """ API to decouple cg code from Housekeeper """
 
     def __init__(self, config):
-        super(HousekeeperAPI, self).__init__(config['housekeeper']['database'],
-                                             config['housekeeper']['root'])
-        self.root_dir = config['housekeeper']['root']
+        self.store = Store(
+            config["housekeeper"]["database"], config["housekeeper"]["root"]
+        )
+
+    def add_bundle(self, bundle_data):
+        """ Wrap method in Housekeeper Store """
+        return self.store.add_bundle(bundle_data)
 
     def include(self, version_obj: models.Version):
         """Call the include version function to import related assets."""
@@ -30,26 +34,28 @@ def include_file(self, file_obj: models.File, version_obj: models.Version):
         # generate root directory
         version_root_dir = global_root_dir / version_obj.relative_root_dir
         version_root_dir.mkdir(parents=True, exist_ok=True)
-        log.info(f"created new bundle version dir: {version_root_dir}")
+        LOG.info("created new bundle version dir: %s", version_root_dir)
 
         if file_obj.to_archive:
             # calculate sha1 checksum if file is to be archived
             file_obj.checksum = HousekeeperAPI.checksum(file_obj.path)
         # hardlink file to the internal structure
         new_path = version_root_dir / Path(file_obj.path).name
         os.link(file_obj.path, new_path)
-        log.info(f"linked file: {file_obj.path} -> {new_path}")
-        file_obj.path = str(new_path).replace(f"{global_root_dir}/", '', 1)
+        LOG.info("linked file: %s -> %s", file_obj.path, new_path)
+        file_obj.path = str(new_path).replace(f"{global_root_dir}/", "", 1)
 
     def last_version(self, bundle: str) -> models.Version:
-        return (self.Version.query
-                            .join(models.Version.bundle)
-                            .filter(models.Bundle.name == bundle)
-                            .order_by(models.Version.created_at.desc())
-                            .first())
+        return (
+            self.store.Version.query.join(models.Version.bundle)
+            .filter(models.Bundle.name == bundle)
+            .order_by(models.Version.created_at.desc())
+            .first()
+        )
 
     def get_root_dir(self):
-        return self.root_dir
+        """Returns the root dir of Housekeeper"""
+        return self.store.root_dir
 
     def get_files(self, bundle: str, tags: list, version: int = None):
         """Fetch all the files in housekeeper, optionally filtered by bundle and/or tags and/or
@@ -58,17 +64,17 @@ def get_files(self, bundle: str, tags: list, version: int = None):
         Returns:
             iterable(hk.Models.File)
         """
-        return self.files(bundle=bundle, tags=tags, version=version)
+        return self.store.files(bundle=bundle, tags=tags, version=version)
 
     def add_file(self, file, version_obj: models.Version, tag_name, to_archive=False):
         """Add a file to housekeeper."""
-        new_file = self.new_file(
+        new_file = self.store.new_file(
             path=str(Path(file).absolute()),
             to_archive=to_archive,
-            tags=[self.tag(tag_name)]
+            tags=[self.store.tag(tag_name)],
         )
         new_file.version = version_obj
-        self.add_commit(new_file)
+        self.store.add_commit(new_file)
         return new_file
 
     @staticmethod

cg/cli/workflow/balsamic/store.py

@@ -1,14 +1,16 @@
 """Click commands to store balsamic analyses"""
-import datetime as dt
-import logging
-from pathlib import Path
 
+import logging
 import click
 
 from cg.apps import hk, tb
-from cg.exc import AnalysisNotFinishedError, AnalysisDuplicationError
+from cg.meta.store.balsamic import _gather_files_and_bundle_in_housekeeper
+
 from cg.store import Store
 
+from cg.exc import AnalysisNotFinishedError, AnalysisDuplicationError
+from housekeeper.exc import VersionIncludedError
+
 LOG = logging.getLogger(__name__)
 
 
@@ -22,117 +24,37 @@ def store(context):
 
 
 @store.command()
-@click.argument("config-stream", type=click.File("r"), required=False)
+@click.argument("case_id")
+@click.argument("config-stream", type=click.File("r"))
 @click.pass_context
-def analysis(context, config_stream):
+def analysis(context, case_id, config_stream):
     """Store a finished analysis in Housekeeper."""
+
     status = context.obj["db"]
-    tb_api = context.obj["tb_api"]
-    hk_api = context.obj["hk_api"]
+    case_obj = status.family(case_id)
 
-    if not config_stream:
-        LOG.error("provide a config, suggestions:")
-        for analysis_obj in tb_api.analyses(
-            status="completed", deleted=False
-        )[:25]:
-            click.echo(analysis_obj.config_path)
+    if not case_obj:
+        click.echo(click.style(f"Case {case_id} not found", fg="red"))
         context.abort()
 
-    new_analysis = _gather_files_and_bundle_in_housekeeper(
-        config_stream, context, hk_api, status, tb_api
-    )
-
-    status.add_commit(new_analysis)
-    click.echo(click.style("included files in Housekeeper", fg="green"))
-
-
-def _gather_files_and_bundle_in_housekeeper(
-    config_stream, context, hk_api, status, tb_api
-):
-    """Function to gather files and bundle in housekeeper"""
+    hk_api = context.obj["hk_api"]
 
     try:
-        bundle_data = tb_api.add_analysis(config_stream)
+        new_analysis = _gather_files_and_bundle_in_housekeeper(
+            config_stream, hk_api, status
+        )
     except AnalysisNotFinishedError as error:
         click.echo(click.style(error.message, fg="red"))
         context.abort()
-
-    try:
-        results = hk_api.add_bundle(bundle_data)
-        if results is None:
-            print(click.style("analysis version already added", fg="yellow"))
-            context.abort()
-        bundle_obj, version_obj = results
     except FileNotFoundError as error:
         click.echo(click.style(f"missing file: {error.args[0]}", fg="red"))
         context.abort()
-
-    family_obj = _add_new_analysis_to_the_status_api(bundle_obj, status)
-    _reset_action_from_running_on_family(family_obj)
-    new_analysis = _add_new_complete_analysis_record(
-        bundle_data, family_obj, status, version_obj
-    )
-    version_date = version_obj.created_at.date()
-    click.echo(f"new bundle added: {bundle_obj.name}, version {version_date}")
-    _include_files_in_housekeeper(bundle_obj, context, hk_api, version_obj)
-
-    return new_analysis
-
-
-def _include_files_in_housekeeper(bundle_obj, context, hk_api, version_obj):
-    """Function to include files in housekeeper"""
-    try:
-        hk_api.include(version_obj)
-    except hk.VersionIncludedError as error:
+    except AnalysisDuplicationError as error:
+        print(click.style("analysis version already added", fg="yellow"))
+        context.abort()
+    except VersionIncludedError as error:
         click.echo(click.style(error.message, fg="red"))
         context.abort()
-    hk_api.add_commit(bundle_obj, version_obj)
-
-
-def _add_new_complete_analysis_record(bundle_data, family_obj, status, version_obj):
-    """Function to create and return a new analysis database record"""
-    pipeline = family_obj.links[0].sample.data_analysis
-    pipeline = pipeline if pipeline else "mip"  # TODO remove this default from here
-
-    if status.analysis(family=family_obj, started_at=version_obj.created_at):
-        raise AnalysisDuplicationError(
-            f"Analysis object already exists for {family_obj.internal_id}{version_obj.created_at}"
-        )
-
-    new_analysis = status.add_analysis(
-        pipeline=pipeline,
-        version=bundle_data["pipeline_version"],
-        started_at=version_obj.created_at,
-        completed_at=dt.datetime.now(),
-        primary=(len(family_obj.analyses) == 0),
-    )
-    new_analysis.family = family_obj
-    return new_analysis
-
 
-def _reset_action_from_running_on_family(family_obj):
-    family_obj.action = None
-
-
-def _add_new_analysis_to_the_status_api(bundle_obj, status):
-    family_obj = status.family(bundle_obj.name)
-    return family_obj
-
-
-@store.command()
-@click.pass_context
-def completed(context):
-    """Store all completed analyses."""
-    hk_api = context.obj["hk_api"]
-    for analysis_obj in context.obj["tb_api"].analyses(
-        status="completed", deleted=False
-    ):
-        existing_record = hk_api.version(analysis_obj.family, analysis_obj.started_at)
-        if existing_record:
-            LOG.debug(
-                "analysis stored: %s - %s", analysis_obj.family, analysis_obj.started_at
-            )
-            continue
-        click.echo(click.style(f"storing family: {analysis_obj.family}", fg="blue"))
-        with Path(analysis_obj.config_path).open() as config_stream:
-            context.invoke(analysis, config_stream=config_stream)
+    status.add_commit(new_analysis)
+    click.echo(click.style("included files in Housekeeper", fg="green"))

cg/meta/store/balsamic.py

@@ -0,0 +1,102 @@
+"""Builds balsamic bundle for linking in Housekeeper"""
+
+
+import datetime as dt
+import logging
+import ruamel.yaml
+from cg.exc import AnalysisDuplicationError
+
+LOG = logging.getLogger(__name__)
+
+
+def _gather_files_and_bundle_in_housekeeper(config_stream, hk_api, status):
+    """Function to gather files and bundle in housekeeper"""
+
+    bundle_data = _add_analysis(config_stream)
+
+    results = hk_api.add_bundle(bundle_data)
+    if not results:
+        raise AnalysisDuplicationError("analysis version already added")
+    bundle_obj, version_obj = results
+
+    case_obj = _get_case(bundle_obj, status)
+    _reset_analysis_action(case_obj)
+    new_analysis = _create_analysis(bundle_data, case_obj, status, version_obj)
+    version_date = version_obj.created_at.date()
+    LOG.info(f"new bundle added: {bundle_obj.name}, version {version_date}")
+    _include_files_in_housekeeper(bundle_obj, hk_api, version_obj)
+    return new_analysis
+
+
+def _include_files_in_housekeeper(bundle_obj, hk_api, version_obj):
+    """Function to include files in housekeeper"""
+    hk_api.include(version_obj)
+    hk_api.store.add_commit(bundle_obj, version_obj)
+
+
+def _create_analysis(bundle_data, case_obj, status, version_obj):
+    """Function to create and return a new analysis database record"""
+    pipeline = case_obj.links[0].sample.data_analysis
+    pipeline = pipeline if pipeline else "balsamic"
+
+    if status.analysis(family=case_obj, started_at=version_obj.created_at):
+        raise AnalysisDuplicationError(
+            f"Analysis object already exists for {case_obj.internal_id}{version_obj.created_at}"
+        )
+
+    new_analysis = status.add_analysis(
+        pipeline=pipeline,
+        version=bundle_data["pipeline_version"],
+        started_at=version_obj.created_at,
+        completed_at=dt.datetime.now(),
+        primary=(len(case_obj.analyses) == 0),
+    )
+    new_analysis.family = case_obj
+    return new_analysis
+
+
+def _reset_analysis_action(case_obj):
+    case_obj.action = None
+
+
+def _get_case(bundle_obj, status):
+    case_obj = status.family(bundle_obj.name)
+    return case_obj
+
+
+def _parse_meta_data(config_raw):
+    return config_raw
+
+
+def _add_analysis(config_stream):
+    """Gather information from balsamic analysis to store."""
+    meta_raw = ruamel.yaml.safe_load(config_stream)
+    new_bundle = _build_bundle(meta_raw)
+    return new_bundle
+
+
+def _build_bundle(meta_data: dict) -> dict:
+    """Create a new bundle."""
+    data = {
+        "name": "not-implemented",
+        "created": "not-implemented",
+        "pipeline_version": "not-implemented",
+        "files": _get_files(meta_data),
+    }
+    return data
+
+
+def _get_files(meta_data: dict) -> list:
+    """Get all the files from the balsamic files."""
+
+    data = []
+
+    for tag in meta_data["files"]:
+
+        paths = meta_data["files"][tag]
+
+        for path in paths:
+
+            data.append({"path": path, "tags": [tag], "archive": False})
+
+    return data