Karma BrowserStack reporting enhancements

README.md

@@ -55,16 +55,17 @@ module.exports = function(config) {
 
 ### Global options
 
-- `username` your BS username, you can also use `BROWSERSTACK_USERNAME` env variable.
-- `accessKey` your BS access key, you can also use `BROWSERSTACK_ACCESS_KEY` env variable.
+- `username` your BS username, you can also use `BROWSER_STACK_USERNAME` env variable.
+- `accessKey` your BS access key, you can also use `BROWSER_STACK_ACCESS_KEY` env variable.
 - `startTunnel` do you wanna establish the BrowserStack tunnel ? (defaults to `true`)
-- `tunnelIdentifier` in case you want to start the BrowserStack tunnel outside `karma` by setting `startTunnel` to `false`, set the identifier passed to the `-localIdentifier` option here (optional)
+- `tunnelIdentifier`/`localIdentifier` in case you want to start the BrowserStack tunnel outside `karma` by setting `startTunnel` to `false`, set the identifier passed to the `-localIdentifier` option here (optional)
 - `retryLimit` how many times do you want to retry to capture the browser ? (defaults to `3`)
 - `captureTimeout` the browser capture timeout (defaults to `120`)
 - `timeout` the BS worker timeout (defaults to `300`
 - `build` the BS worker build name (optional)
 - `name` the BS worker name (optional)
 - `project` the BS worker project name (optional)
+- `binaryBasePath` the BS binary base bath, you can also use `BROWSER_STACK_BINARY_BASE_PATH` env variable. This will override the default and set the base path to the BS local binary (optional)
 - `proxyHost` the host of your proxy for communicating with BrowserStack REST API and BrowserStackLocal (optional)
 - `proxyPort` the port of your proxy (optional)
 - `proxyUser` the username used for authentication with your proxy (optional)
@@ -85,7 +86,7 @@ module.exports = function(config) {
 - `name` the BS worker name (optional, defaults to global)
 - `project` the BS worker project name (optional, defaults to global)
 
-> **Note:** you can also pass through any additional options supported by browserstack. (EG. `url`, `resolution`, etc.)  
+> **Note:** you can also pass through any additional options supported by browserstack. (EG. `timezone`, `resolution`, etc.)  
 See https://www.browserstack.com/automate/capabilities for a full list of supported options.
 
 ### BrowserStack reporter

browserstack-reporter.js

@@ -1,53 +1,207 @@
 var Browserstack = require('browserstack')
+var common = require('./common')
 
-var BrowserStackReporter = function (logger, /* BrowserStack:sessionMapping */ sessionMapping) {
-  var log = logger.create('reporter.browserlabs')
+var BrowserStackReporter = function (baseReporterDecorator, logger, /* BrowserStack:sessionMapping */ sessionMapping, config) {
+  try {
+    var log = logger.create('reporter.browserstack')
+    var suiteResults = []
+    var buildId
+    var allMessages = []
+    baseReporterDecorator(this)
+    var callWhenFinished = function () {}
 
-  var pendingUpdates = 0
-  var callWhenFinished = function () {}
-
-  var exitIfAllFinished = function () {
-    if (pendingUpdates === 0) {
+    var exitIfAllFinished = function () {
       callWhenFinished()
     }
-  }
 
-  // We're only interested in the final results per browser
-  this.onBrowserComplete = function (browser) {
-    var result = browser.lastResult
+    this.onRunStart = function (browsers) {
+      // implement this function if you want to perform custom actions post the overall test batch run has started
+
+    }
+
+    this.onBrowserStart = function (browser) {
+      // implement this function if you want to perform custom actions post browser start
+      log.debug('Browser: ' + browser.name + ' has been started')
+    }
+
+    this.onBrowserError = function (browser, error) {
+      var browserId = browser.launchId || browser.id
+      if (browserId in sessionMapping) {
+        var existingObj = suiteResults.find(o => o.browserId === browserId)
+        if (typeof existingObj === 'undefined') {
+          suiteResults.push({
+            browserId: browserId,
+            records: [{
+              apiStatus: 'Error',
+              suite: 'N/A [Karma <-> Browser error]',
+              log: error
+            }]
+          })
+        } else {
+          existingObj.records.push({
+            apiStatus: 'Error',
+            suite: 'N/A [Karma <-> Browser error]',
+            log: error
+          })
+        }
+      }
+    }
 
-    if (result.disconnected) {
-      log.error('✖ Test Disconnected')
+    this.specSuccess = function (browser, result) {
+      try {
+        var browserId = browser.launchId || browser.id
+        if (browserId in sessionMapping) {
+          var existingObj = suiteResults.find(o => o.browserId === browserId)
+          if (typeof existingObj === 'undefined') {
+            suiteResults.push({
+              browserId: browserId,
+              records: [{
+                apiStatus: 'Passed',
+                suite: result.suite,
+                specs: [result.description]
+              }]
+            })
+          } else {
+            existingObj['records'][0]['specs'].push(result.description)
+          }
+        }
+      } catch (e) {
+        log.debug('Browserstack reporter module specSuccess function encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
+      }
     }
 
-    if (result.error) {
-      log.error('✖ Test Errored')
+    this.specFailure = this.specSkipped = function (browser, result) {
+      try {
+        var browserId = browser.launchId || browser.id
+        log.debug('Spec Failure:' + JSON.stringify(result))
+        if (browserId in sessionMapping) {
+          var existingObj = suiteResults.find(o => o.browserId === browserId)
+          if (typeof existingObj === 'undefined') {
+            suiteResults.push({
+              browserId: browserId,
+              records: [{
+                apiStatus: 'Failed',
+                suite: result.suite,
+                log: result.log
+              }]
+            })
+          } else {
+            existingObj.records.push({
+              apiStatus: 'Failed',
+              suite: result.suite,
+              log: result.log
+            })
+          }
+        }
+      } catch (e) {
+        log.debug('Browserstack reporter module specFailure function encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
+      }
     }
 
-    var browserId = browser.launchId || browser.id
-    if (browserId in sessionMapping) {
-      pendingUpdates++
-      var browserstackClient = Browserstack.createAutomateClient(sessionMapping.credentials)
-      var apiStatus = !(result.failed || result.error || result.disconnected) ? 'completed' : 'error'
+    this.onSpecComplete = function (browser, result) {
+      if (result.skipped) {
+        this.specSkipped(browser, result)
+      } else if (result.success) {
+        this.specSuccess(browser, result)
+      } else {
+        this.specFailure(browser, result)
+      }
+      log.debug('Suite: ' + result.suite + ', Spec completed: ' + result.description)
+    }
+
+    this.onRunComplete = function (browsersCollection, results) {
+      // implement this function if you want to perform custom actions post test run completion
+      try {
+        log.debug('Test run completed\n' + JSON.stringify(results))
+        var browserId = browsersCollection.browsers[0].id
 
-      browserstackClient.updateSession(sessionMapping[browserId], {
-        status: apiStatus
-      }, function (error) {
-        if (error) {
-          log.error('✖ Could not update BrowserStack status')
-          log.debug(error)
+        if (typeof buildId === 'undefined') {
+          var browserstackClient = Browserstack.createAutomateClient(sessionMapping.credentials)
+          common.getSessionDetails(log, browserstackClient, sessionMapping, browserId, config, function (sessionObj) {
+            var browserUrl = sessionObj['automation_session']['browser_url']
+            if (typeof browserUrl !== 'undefined') {
+              buildId = browserUrl.split('/').slice(-3)[0]
+              var originalBuildName = config.browserStack.build
+              if (typeof buildId !== 'undefined' || buildId !== undefined) {
+                common.updateBuildName(log, buildId, originalBuildName, results, config)
+              }
+            }
+          })
         }
-        pendingUpdates--
-        exitIfAllFinished()
+        allMessages = []
+      } catch (e) {
+        log.debug('onRunComplete: Browserstack reporter module encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
+      }
+    }
+
+    var findSpecObjectRecords = function (browserId, specStatusSearchCondition) {
+      var suiteObj = suiteResults.find(o => o.browserId === browserId)
+      var specRecord
+      if (typeof suiteObj !== 'undefined') {
+        if (specStatusSearchCondition !== '') {
+          specRecord = suiteObj.records.find(o => o.apiStatus === specStatusSearchCondition)
+        }
+      }
+      return [suiteObj, specRecord]
+    }
+
+    var removeFromSuiteResults = function (objectListToRemove) {
+      suiteResults = suiteResults.filter(function (item) {
+        return item !== objectListToRemove
       })
     }
-  }
 
-  // Wait until all updates have been pushed to Browserstack
-  this.onExit = function (done) {
-    callWhenFinished = done
-    exitIfAllFinished()
+    this.adapters = [function (msg) {
+      allMessages.push(msg)
+    }]
+
+    this.onBrowserComplete = function (browser) {
+      try {
+        var browserId = browser.launchId || browser.id
+        var failedObjRecord = findSpecObjectRecords(browserId, 'Failed')
+        var errorObjRecord = findSpecObjectRecords(browserId, 'Error')
+        var browserstackClient = Browserstack.createAutomateClient(sessionMapping.credentials)
+        if (browserId in sessionMapping) {
+          if (typeof failedObjRecord[1] !== 'undefined') {
+            var reason = 'Suite: ' + failedObjRecord[1]['suite'] + ', Error log: ' + failedObjRecord[1]['log']
+            common.updateStatusSession(log, browserstackClient, sessionMapping, browserId, failedObjRecord[1].apiStatus, reason, failedObjRecord[1]['suite'])
+            removeFromSuiteResults(failedObjRecord[0])
+          } else if (typeof errorObjRecord[1] === 'undefined') {
+            var passedObjRecord = findSpecObjectRecords(browserId, 'Passed')
+            if (typeof passedObjRecord[1] !== 'undefined') {
+              reason = 'Suite: ' + passedObjRecord[1]['suite'] + ', Specs run: ' + JSON.stringify(passedObjRecord[1]['specs'])
+              common.updateStatusSession(log, browserstackClient, sessionMapping, browserId, passedObjRecord[1].apiStatus, reason, passedObjRecord[1]['suite'])
+              removeFromSuiteResults(passedObjRecord[0])
+              allMessages = []
+            }
+          }
+        }
+        var result = browser.lastResult
+        log.debug('Browser Last Result: ' + JSON.stringify(result, null, 2))
+        if (result.disconnected) {
+          log.error('✖ Test Disconnected')
+        } else if (result.error) {
+          log.error('✖ Test Errored')
+        }
+        errorObjRecord = findSpecObjectRecords(browserId, 'Error')
+        if (typeof errorObjRecord[1] !== 'undefined') {
+          reason = 'Suite: ' + errorObjRecord[1]['suite'] + ', Error log: ' + errorObjRecord[1]['log']
+          common.updateStatusSession(log, browserstackClient, sessionMapping, browserId, errorObjRecord[1].apiStatus, reason, errorObjRecord[1]['suite'])
+          removeFromSuiteResults(errorObjRecord[0])
+        }
+      } catch (e) {
+        log.debug('Browserstack reporter module encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
+      }
+    }
+
+    // Wait until all updates have been pushed to Browserstack
+    this.onExit = function (done) {
+      callWhenFinished = done
+      exitIfAllFinished()
+    }
+  } catch (e) {
+    log.debug('Browserstack reporter full module encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
   }
 }
-
+// PUBLISH DI MODULE
 module.exports = BrowserStackReporter

common.js

@@ -0,0 +1,93 @@
+var RestClient = require('node-rest-client').Client
+
+module.exports = {
+  updateStatusSession: function (log, browserstackClient, sessionMapping, browserId, status, reason, sessionName) {
+    browserstackClient.updateSession(sessionMapping[browserId], {
+      status: status,
+      reason: reason,
+      name: sessionName.length !== 0 ? sessionName.toString() : 'Karma Test Suite'
+    }, function (error) {
+      if (error) {
+        log.error('✖ Could not update BrowserStack status' + error)
+      }
+    })
+  },
+  getSessionDetails: function (log, browserstackClient, sessionMapping, browserId, config, callback) {
+    try {
+      var apiClientEndpoint = config.browserStack.apiClientEndpoint
+      var creds = sessionMapping.credentials.username + ':' + sessionMapping.credentials.password
+
+      let buff = Buffer.from(creds)
+      let base64data = buff.toString('base64')
+      var encodedCreds = base64data
+      var client = new RestClient()
+      var args = {
+        data: {},
+        headers: {
+          'Authorization': 'Basic ' + encodedCreds
+        }
+      }
+      client.get(apiClientEndpoint + '/automate/sessions/' + sessionMapping[browserId] + '.json', args, function (data, response) {
+        callback(data)
+      })
+    } catch (e) {
+      log.debug('Browserstack reporter module encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
+    }
+  },
+  updateBuildName: function (log, buildId, originalBuildName, results, config) {
+    try {
+      log.debug('Attempting build name update')
+      if (typeof buildId === 'undefined') {
+        log.debug('BrowserStack build id is not defined. Not attempting update for build id!')
+        return
+      }
+
+      var apiClientEndpoint = config.browserStack.apiClientEndpoint
+      var client = new RestClient()
+      var newBuildName
+      var t = new Date().toISOString()
+
+      if (originalBuildName === null || typeof originalBuildName === 'undefined') {
+        originalBuildName = 'Karma build'
+      }
+
+      if (typeof results !== 'undefined') {
+        if (results.exitCode === 1) {
+          if (results.disconnected === true) {
+            newBuildName = originalBuildName + ' completed at ' + t + ' with 1 or more session disconnects: [Specs passed: ' + results.success + ' | Specs failed: ' + results.failed + ']'
+          } else if (results.error === true) {
+            newBuildName = originalBuildName + ' completed at ' + t + ' with 1 or more session errors: [Specs passed: ' + results.success + ' | Specs failed: ' + results.failed + ']'
+          } else if (results.failed !== 0) {
+            newBuildName = originalBuildName + ' completed at ' + t + ' with spec failures: [Specs passed: ' + results.success + ' | Specs failed: ' + results.failed + ']'
+          }
+        } else if (results.failed === 0) {
+          newBuildName = originalBuildName + ' completed successfully at ' + t + ': [Specs passed: ' + results.success + ']'
+        }
+      } else {
+        newBuildName = originalBuildName + ' started at ' + t + ' | In Progress | '
+      }
+
+      var creds = config.browserStack.username + ':' + config.browserStack.accessKey
+
+      let buff = Buffer.from(creds)
+      let base64data = buff.toString('base64')
+      var encodedCreds = base64data
+      var args = {
+        data: {
+          name: newBuildName
+        },
+        headers: {
+          'Authorization': 'Basic ' + encodedCreds,
+          'Content-Type': 'application/json'
+        }
+      }
+      var completeEndpoint = apiClientEndpoint + '/automate/builds/' + buildId + '.json'
+      log.debug('Complete endpoint for build name update: ' + completeEndpoint)
+      client.put(completeEndpoint, args, function (data, response) {
+      })
+      log.debug('Exiting updateBuildName function')
+    } catch (e) {
+      log.debug('Browserstack reporter module update build name function encountered issues.\nError message: ' + e.message + '\nStacktrace: ' + e.stack)
+    }
+  }
+}

gruntfile.js

@@ -41,7 +41,7 @@ module.exports = function (grunt) {
     },
     eslint: {
       target: [
-        'index.js',
+        '*.js',
         'worker.js',
         'worker-manager.js',
         'gruntfile.js'