Merge pull request #19 from arteria-project/devel

Merge devel branch for new release

bcl2fastq/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.0"
+__version__ = "1.1.1"

bcl2fastq/app.py

@@ -15,7 +15,8 @@ def routes(**kwargs):
         url(r"/api/1.0/versions", VersionsHandler, name="versions", kwargs=kwargs),
         url(r"/api/1.0/start/([\w_-]+)", StartHandler, name="start", kwargs=kwargs),
         url(r"/api/1.0/status/(\d*)", StatusHandler, name="status", kwargs=kwargs),
-        url(r"/api/1.0/stop/([\d|all]*)", StopHandler, name="stop", kwargs=kwargs)
+        url(r"/api/1.0/stop/([\d|all]*)", StopHandler, name="stop", kwargs=kwargs),
+        url(r"/api/1.0/logs/([\w_-]+)", Bcl2FastqLogHandler, name="logs", kwargs=kwargs)
     ]
 
 def start():

bcl2fastq/handlers/bcl2fastq_handlers.py

@@ -6,9 +6,12 @@
 from bcl2fastq.lib.jobrunner import LocalQAdapter
 from bcl2fastq.lib.bcl2fastq_utils import BCL2FastqRunnerFactory, Bcl2FastqConfig
 from bcl2fastq import __version__ as version
+from bcl2fastq.lib.bcl2fastq_logs import Bcl2FastqLogFileProvider
+from arteria.exceptions import ArteriaUsageException
 from arteria.web.state import State
 from arteria.web.handlers import BaseRestHandler
 
+
 log = logging.getLogger(__name__)
 
 class Bcl2FastqServiceMixin:
@@ -59,6 +62,7 @@ def initialize(self, config):
         to subclasses.
         """
         self.config = config
+        self.bcl2fastq_log_file_provider = Bcl2FastqLogFileProvider(self.config)
 
 
 class VersionsHandler(BaseBcl2FastqHandler):
@@ -109,7 +113,7 @@ def create_config_from_request(self, runfolder, request_body):
         runfolder_input = "{0}/{1}".format(runfolder_base_path, runfolder)
 
         if not os.path.isdir(runfolder_input):
-            raise RuntimeError("No such file: {0}".format(runfolder_input))
+            raise ArteriaUsageException("No such file: {0}".format(runfolder_input))
 
         if "bcl2fastq_version" in request_data:
             bcl2fastq_version = request_data["bcl2fastq_version"]
@@ -172,8 +176,7 @@ def post(self, runfolder):
             cmd = job_runner.construct_command()
             job_runner.symlink_output_to_unaligned()
 
-            log_base_path = self.config["bcl2fastq_logs_path"]
-            log_file = "{0}/{1}.log".format(log_base_path, runfolder)
+            log_file = self.bcl2fastq_log_file_provider.log_file_path(runfolder)
 
             job_id = self.runner_service().start(
                 cmd,
@@ -202,11 +205,12 @@ def post(self, runfolder):
 
             self.set_status(202, reason="started processing")
             self.write_json(response_data)
-        except RuntimeError as e:
+        except ArteriaUsageException as e:
             log.warning("Failed starting {0}. Message: {1}".format(runfolder, e.message))
             self.send_error(status_code=500, reason=e.message)
 
 
+
 class StatusHandler(BaseBcl2FastqHandler, Bcl2FastqServiceMixin):
     """
     Get the status of one or all jobs.
@@ -252,10 +256,28 @@ def post(self, job_id):
                 self.runner_service().stop(job_id)
                 self.set_status(200)
             else:
-                ValueError("Unknown job to stop")
-        except ValueError as e:
+                ArteriaUsageException("Unknown job to stop")
+        except ArteriaUsageException as e:
             log.warning("Failed stopping job: {}. Message: ".format(job_id, e.message))
             self.send_error(500, reason=e.message)
 
 
+class Bcl2FastqLogHandler(BaseBcl2FastqHandler):
+    """
+    Gets the content of the log for a particular runfolder
+    """
 
+    def get(self, runfolder):
+        """
+        Get the content of the log for a particular runfolder
+        :param runfolder:
+        :return:
+        """
+        try:
+            log_content = self.bcl2fastq_log_file_provider.get_log_for_runfolder(runfolder)
+            response_data = {"runfolder": runfolder, "log": log_content}
+            self.set_status(200)
+            self.write_json(response_data)
+        except IOError as e:
+            log.warning("Problem with accessing {}, message: {}".format(runfolder, e.message))
+            self.send_error(500, reason=e.message)

bcl2fastq/lib/bcl2fastq_logs.py

@@ -0,0 +1,18 @@
+
+
+class Bcl2FastqLogFileProvider:
+
+    def __init__(self, config):
+        self.config = config
+
+    def log_file_path(self, runfolder):
+        log_base_path = self.config["bcl2fastq_logs_path"]
+        log_file = "{0}/{1}.log".format(log_base_path, runfolder)
+        return log_file
+
+    def get_log_for_runfolder(self, runfolder):
+        log_path = self.log_file_path(runfolder)
+        with open(log_path) as f:
+            file_content = f.read()
+        return file_content
+

bcl2fastq/lib/bcl2fastq_utils.py

@@ -7,6 +7,8 @@
 
 from illuminate.metadata import InteropMetadata
 
+
+from arteria.exceptions import ArteriaUsageException
 from bcl2fastq.lib.illumina import Samplesheet
 
 log = logging.getLogger(__name__)
@@ -169,7 +171,8 @@ def build_index_string(length_tuple):
             difference = length_of_index_read - length_of_index_in_samplesheet
 
             if not difference >= 0:
-                raise ValueError("Sample sheet indicates that index is longer than what was read by the sequencer!")
+                raise ArteriaUsageException("Sample sheet indicates that index is "
+                                       "longer than what was read by the sequencer!")
 
             if length_of_index_in_samplesheet == 0:
                 # If there is no index in the samplesheet, ignore it in the base-mask
@@ -427,8 +430,9 @@ def construct_command(self):
                     is_single_read_run)
             base_masks_as_set = set(lanes_and_base_mask.values())
 
-            assert len(base_masks_as_set) is 1, "For bcl2fastq 1.8.4 there is no support for " \
-                                                "mixing different bases masks for different lanes"
+            if len(base_masks_as_set) is 1:
+                raise ArteriaUsageException("For bcl2fastq 1.8.4 there is no support for "
+                                       "mixing different bases masks for different lanes")
 
             # Here we are forced to use the same bases mask was always used for all lanes.
             commandline_collection.append("--use_bases_mask " + lanes_and_base_mask.values()[0])

requirements/prod

@@ -1,7 +1,7 @@
 jsonpickle==0.9.2
 tornado==4.2.1
 git+https://github.com/johandahlberg/localq.git@with_shell_true # Get from pip in future - localq
-git+https://github.com/arteria-project/arteria-core.git@v1.0.0#egg=arteria-core
+git+https://github.com/arteria-project/arteria-core.git@v1.0.1#egg=arteria-core
 illuminate==0.6.2
 pandas==0.14.1
 

tests/test_bcl2fastq_handlers.py

@@ -7,6 +7,7 @@
 
 from bcl2fastq.handlers.bcl2fastq_handlers import *
 from bcl2fastq.lib.bcl2fastq_utils import BCL2Fastq2xRunner, BCL2FastqRunner
+from bcl2fastq.lib.bcl2fastq_logs import Bcl2FastqLogFileProvider
 from bcl2fastq.app import routes
 from tornado.web import Application
 from test_utils import FakeRunner
@@ -129,3 +130,13 @@ def test_stop_handler(self):
     def test_exception_stop_handler(self):
         response = self.fetch(self.API_BASE + "/stop/lll", method="POST", body = "")
         self.assertEqual(response.code, 500)
+
+    def test_get_logs(self):
+        with mock.patch.object(Bcl2FastqLogFileProvider, "get_log_for_runfolder", return_value="This is a string"):
+            response = self.fetch(self.API_BASE + "/logs/coolest_runfolder", method="GET")
+            self.assertEqual(response.code, 200)
+            self.assertEqual(json.loads(response.body)["log"], "This is a string")
+
+    def test_get_logs_trying_to_reach_other_files(self):
+        response = self.fetch(self.API_BASE + "/logs/../../../etc/shadow", method="GET")
+        self.assertEqual(response.code, 404)

tests/test_bcl2fastq_logs.py

@@ -0,0 +1,35 @@
+
+import unittest
+from mock import MagicMock, patch, mock_open
+
+from bcl2fastq.lib.bcl2fastq_logs import Bcl2FastqLogFileProvider
+
+class TestBcl2FastqLogFileProvider(unittest.TestCase):
+
+    fake_file_content = """
+    This is a nice multi-line
+    string for
+    you
+    my friend
+    """
+    fake_path = "/fake/path"
+    mock_config = MagicMock()
+    mock_config.__getitem__.return_value = fake_path
+    runfolder = "160218_ST-E00215_0070_BHKGLFCCXX"
+
+    log_filer_provider = Bcl2FastqLogFileProvider(mock_config)
+
+    def test_log_file_path(self):
+        log_file = self.log_filer_provider.log_file_path(self.runfolder)
+        self.assertEqual(log_file, "{}/{}.log".format(self.fake_path, self.runfolder))
+
+    def test_get_log_for_runfolder(self):
+        with patch("__builtin__.open", mock_open(read_data=self.fake_file_content), create=True):
+            file_content = self.log_filer_provider.get_log_for_runfolder(self.runfolder)
+            self.assertEqual(file_content, self.fake_file_content)
+
+    def test_get_log_for_runfolder_does_not_exist(self):
+        with self.assertRaises(IOError):
+            self.log_filer_provider.get_log_for_runfolder(self.runfolder)
+
+

tests/tests_bcl2fastq_utils.py

@@ -93,7 +93,7 @@ def test_get_bases_mask_per_lane_from_samplesheet_invalid_length_combo(self):
         mock_read_index_lengths = {2: 4, 3: 4}
         samplesheet = Samplesheet(TestBcl2FastqConfig.samplesheet_file)
 
-        with self.assertRaises(ValueError):
+        with self.assertRaises(ArteriaUsageException):
             Bcl2FastqConfig. \
                 get_bases_mask_per_lane_from_samplesheet(samplesheet, mock_read_index_lengths, False)