Allow non-optional requests split into multiple separate requests to … (

#364)

* Allow non-optional requests split into multiple separate requests to be processed as a group rather than immediately failing as soon as one part of the group fails

* remove old code

* linefeed cleanup

* clarify id recommendation

lib/BaseDownloadController.js

@@ -44,6 +44,18 @@ module.exports = class BaseDownloadController extends CompositeController {
     let lastModifiedArray = objectPath.get(this.data, ['object', 'status', 'last-modified'], []);
     let newLastModifiedArray = [];
 
+    /*
+    A request may have been split into mulitple requests (e.g. a request that returns multiple files split into separate requests per file).
+    If this is done and the original request is non-optional, all the requests in the group should be processed as a group rather than aborting on the first failure.
+    These two should behave the same:
+    - A single original request for a single file containing multiple resources
+    - A single original request for multiple files each containing a single resource
+    To get the same behavior, track failures and messages for the group and only fail once the entire has been attempted.
+    The group is defined by having a `splitRequestId` attribute added to each new request added by splitting the original.  Using the hash of the original request is recommended.
+    */
+    let mandatoryResourceFailure = 0;
+    let mandatoryResourceFailureMsgs = [];
+
     for (var i = 0; i < requests.length; i++) {
       let request = requests[i];
       let requestHash = hash(request);
@@ -132,10 +144,25 @@ module.exports = class BaseDownloadController extends CompositeController {
           this.log.warn(msg);
           this.updateRazeeLogs('warn', { controller: 'BaseDownload', warn: `Error applying file to kubernetes, see logs for details. StatusCode: ${e.statusCode}`, url: url });
         } else {
-          return Promise.reject(msg);
+          ++mandatoryResourceFailure;
+          mandatoryResourceFailureMsgs.push(msg);
         }
       }
 
+      // Get the next request to check whether it's time to process mandatory request failures.
+      const nextRequest = (i+1 == requests.length) ? null : requests[i+1];
+      // If this is the end of an original request or a split original request...
+      if( !request.splitRequestId || !nextRequest || !nextRequest.splitRequestId || nextRequest.splitRequestId != request.splitRequestId ) {
+        // If mandatory request failures occurred...
+        if( mandatoryResourceFailure > 0 ) {
+          // Non-optional requests failed.  Fail.
+          this.log.error(`${mandatoryResourceFailure} mandatory resource(s) failed to process: ${mandatoryResourceFailureMsgs.join(', ')}`);
+          return Promise.reject(`${mandatoryResourceFailure} errors occurred: ${mandatoryResourceFailureMsgs.join(', ')}`);
+        }
+        // Reset the mandatory request failure trackers.
+        mandatoryResourceFailure = 0;
+        mandatoryResourceFailureMsgs = [];
+      }
     }
     this.log.info(`requests processed: ${requests.length}`);