Spike 5000 queries round deux

.env.example

@@ -19,6 +19,10 @@ SOLID_CABLE_POLLING=0.1.seconds
 # /case page.
 FORCE_SSL=false
 
+# This makes Quepid show detailed error messages in the UI instead of a generic 500 page,
+# useful while testing a deployment in Production.
+QUEPID_CONSIDER_ALL_REQUESTS_LOCAL=false
+
 DB_HOST=mysql
 DB_USERNAME=root
 DB_PASSWORD=password

Gemfile

@@ -93,3 +93,5 @@ group :test do
   gem 'capybara'
   gem 'selenium-webdriver'
 end
+
+gem 'mini_racer', '~> 0.16.0'

Gemfile.lock

@@ -253,6 +253,8 @@ GEM
       childprocess (~> 5.0)
     letter_opener (1.10.0)
       launchy (>= 2.2, < 4)
+    libv8-node (18.19.0.0-x86_64-darwin)
+    libv8-node (18.19.0.0-x86_64-linux)
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
@@ -271,6 +273,8 @@ GEM
     memory_profiler (1.1.0)
     mini_histogram (0.3.1)
     mini_mime (1.1.5)
+    mini_racer (0.16.0)
+      libv8-node (~> 18.19.0.0)
     minitest (5.25.4)
     minitest-reporters (1.7.1)
       ansi
@@ -578,6 +582,7 @@ DEPENDENCIES
   listen (~> 3.3)
   local_time
   memory_profiler
+  mini_racer (~> 0.16.0)
   minitest-reporters (>= 0.5.0)
   mission_control-jobs (~> 0.5.0)
   mocha (~> 2.7)

Procfile

@@ -1,2 +1,3 @@
 web:    bundle exec puma -C config/puma.rb
-worker: bundle exec rake solid_queue:start
+# worker: bundle exec bin/jobs
+worker: bundle exec rake solid_queue:start  # per https://github.com/rails/solid_queue/issues/405

README.md

@@ -272,9 +272,9 @@ bin/docker r bundle exec derailed bundle:mem
 
 ### Debugging JS
 
-While running the application, you can debug the javascript using your favorite tool, the way you've always done it.
+While running the application, you can debug the JavaScript using your favorite tool, the way you've always done it.
 
-The javascript files will be concatenated into one file, using the rails asset pipeline.
+The JavaScript files will be concatenated into one file, using the rails asset pipeline.
 
 You can turn that off by toggling the following flag in `config/environments/development.rb`:
 

app.json

@@ -83,13 +83,14 @@
     "worker": {
       "quantity": 1,
       "size": "standard-1x",
-      "command": "bundle exec rake solid_queue:start"
+      "command": "bundle exec bin/jobs"
     },
     "web": {
       "quantity": 1,
       "size": "standard-1x"
     }
   },
   "scripts": {
+    "postdeploy": "bundle exec rake db:migrate"
   }
 }

app/assets/javascripts/components/case_listing/case_listing.html

@@ -11,6 +11,8 @@
           ng-click="ctrl.goToCase()"
         >
           {{ ctrl.thisCase.caseName }}
+
+          <i class="bi bi-repeat" ng-show="ctrl.thisCase.nightly"></i>
         </span>
         <br/>
         <span class="item-actions">

app/assets/javascripts/components/export_case/_modal.html

@@ -29,7 +29,7 @@ <h3 class="modal-title">Export Case: <span class="modal-case">{{ ctrl.theCase.ca
       Detailed export is only supported from the individual Case view.
     </p>
     <span class="help-block">
-      CSV file with <code>Team Name,Case Name,Case ID,Query Text,Doc ID,Title,Rating,Field1,...,FieldN</code> where <code>Field1,...,FieldN</code> are specified under <strong>Settings</strong> in the <strong>Displayed Fields</strong> field.
+      CSV file with <code>Team Name,Case Name,Case ID,Query Text,Doc ID,Position,Title,Rating,Field1,...,FieldN</code> where <code>Field1,...,FieldN</code> are specified under <strong>Settings</strong> in the <strong>Displayed Fields</strong> field.
     </span>
   </div>
 

app/assets/javascripts/controllers/case.js

@@ -14,6 +14,11 @@ angular.module('QuepidApp')
       $scope.caseModel.reorderEnabled = false;
       $scope.scores  = [];
       $scope.theCase = caseSvc.getSelectedCase();
+
+      $scope.updateNightly = function () {
+        caseSvc.updateNightly($scope.theCase);
+      };
+
       $scope.caseName = {
         name: null,
         startRename: false,

app/assets/javascripts/factories/ScorerFactory.js

@@ -7,6 +7,9 @@
       ScorerFactory
     ]);
 
+  // This file contains JavaScript logic that supports running the Scorers on the client side.  
+  // Many of the methods are duplicated in lib/scorer_logic.js to support running Scorers on the server side,
+  // so be aware if you make any changes.
   function ScorerFactory($q, $timeout) {
     var Scorer = function(data) {
       var self = this;
@@ -63,10 +66,12 @@
 
 
       // public functions
+      // NOT in scorer_logic.js
       function getColors() {
         return scaleToColors(self.scale);
       }
 
+      // NOT in scorer_logic.js
       function scaleToArray(string) {
         return string.replace(/^\s+|\s+$/g,'')
           .split(/\s*,\s*/)
@@ -75,6 +80,7 @@
           });
       }
 
+      // NOT in scorer_logic.js
       function scaleToColors (scale) {
         var colorMap = {};
 
@@ -110,6 +116,7 @@
         return colorMap;
       }
 
+      // NOT in scorer_logic.js
       function scaleToScaleWithLabels(scale, scaleWithLabels) {
         if ( angular.isUndefined(scaleWithLabels) || scaleWithLabels === null ) {
           scaleWithLabels = {};
@@ -128,6 +135,7 @@
         return scaleWithLabels;
       }
 
+      // NOT in scorer_logic.js
       function setDisplayName(name, communal) {
         if ( communal === true ) {
           return name + ' (Communal)';
@@ -136,13 +144,15 @@
         }
       }
 
+      // NOT in scorer_logic.js
       function showScaleLabel(value) {
         return  self.showScaleLabels === true &&
                 self.scaleWithLabels !== null &&
                 angular.isDefined(self.scaleWithLabels) &&
                 angular.isDefined(self.scaleWithLabels[value]);
       }
 
+      // NOT in scorer_logic.js
       function teamNames() {
         var teams = [];
         angular.forEach(self.teams, function(team) {
@@ -152,6 +162,7 @@
         return self.teamName || teams.join(', ');
       }
 
+      // NOT in scorer_logic.js
       function baseAvg(docs, count) {
         var sum = 0.0;
         var docsRated = 0;
@@ -177,11 +188,13 @@
         }
       }
 
+      // NOT in scorer_logic.js
       function baseAvgRounded(docs, count) {
         var avg = self.baseAvg(docs, count);
         return Math.floor(avg);
       }
 
+      // NOT in scorer_logic.js
       function avg100(docs, count) {
         var max         = self.scale[self.scale.length -1];
         var multiplier  = 100 / max;
@@ -194,6 +207,7 @@
         }
       }
 
+      // NOT in scorer_logic.js
       function editDistance(str1, str2) {
 
         var makeZeroArr = function(len) {
@@ -240,6 +254,7 @@
         return bestDocsRatings;
       }
 
+      // NOT in scorer_logic.js
       function distanceFromBest(docs, bestDocs, count) {
         if ( angular.isUndefined(count) ) {
           count = DEFAULT_NUM_DOCS;
@@ -301,6 +316,7 @@
       // We could not get this to work/test, and spent too much time on it.
       // Leaving it here until we do figure out
       // -YC
+      // NOT in scorer_logic.js
       function checkCodeExecutionTime() {
         return $q(function(resolve, reject) {
           var myWorker = new Worker('scripts/scorerEvalTest.js');
@@ -320,6 +336,7 @@
         });
       }
 
+      // NOT in scorer_logic.js
       function checkCode() {
         var deferred    = $q.defer();
         var loopPromise = hasLoop();
@@ -336,6 +353,7 @@
       }
 
       // mode may no longer be used..  maybe it was for unit test style scorers?
+      // NOT in scorer_logic.js
       function runCode(query, total, docs, bestDocs, mode, options) {
         var scale     = self.scale;
         var max       = scale[scale.length-1];
@@ -423,12 +441,12 @@
           return baseAvg(docs, count);
         };
 
-        // Used in the original v1 scorer, which were replaced by the
-        // classic search relevance metrics @P etc.
+        // NOT in scorer_logic.js
         var avgRating100 = function(count) {
           return avg100(docs, count);
         };
 
+        // NOT in scorer_logic.js
         var editDistanceFromBest = function(count) {
           return distanceFromBest(docs, bestDocs, count);
         };
@@ -459,6 +477,8 @@
           }
         };
 
+        // may not be called
+        // NOT in scorer_logic.js
         var refreshRatedDocs = function(k) {
           return query.refreshRatedDocs(k);
         };
@@ -468,6 +488,8 @@
         // param that is passed, and calls the callback function on
         // each doc.
         // Even those that are not in the top 10 current.
+        // may not be used?
+        // NOT in scorer_logic.js
         //
         // @param score,  Int
         // @param f,      Callback
@@ -503,16 +525,19 @@
           }
         };
 
+        // NOT in scorer_logic.js
         var recordDepthOfRanking = function (k){
           query.depthOfRating = k;
           self.depthOfRating = k;
         };
 
+        // NOT in scorer_logic.js
         /*jshint unused:false */
         function pass() {
           scorerDeferred.resolve(100);
         }
 
+        // NOT in scorer_logic.js
         function fail() {
           scorerDeferred.reject(0);
         }
@@ -521,12 +546,14 @@
           scorerDeferred.resolve(score);
         }
 
+        // NOT in scorer_logic.js
         function assert(cond) {
           if (!cond) {
             fail();
           }
         }
 
+        // NOT in scorer_logic.js
         function assertOrScore(cond, score) {
           if (!cond) {
             setScore(score);

app/assets/javascripts/factories/snapshotFactory.js

@@ -6,12 +6,13 @@
   angular.module('QuepidApp')
     .factory('SnapshotFactory', [
       '$log',
+      '$filter',
       'docCacheSvc',
       'normalDocsSvc',
       SnapshotFactory
   ]);
 
-  function SnapshotFactory($log, docCacheSvc, normalDocsSvc) {
+  function SnapshotFactory($log, $filter, docCacheSvc, normalDocsSvc) {
     var Snapshot = function(params) {
       var self  = this;
 
@@ -24,8 +25,7 @@
 
       self.allDocIds        = allDocIds;
       self.getSearchResults = getSearchResults;
-      self.timestamp        = timestamp;
-
+
       self.docIdsPerQuery   = {};
 
       // Map from snake_case to camelCase.
@@ -45,7 +45,7 @@
       });
 
       function snapshotName () {
-        return 'Snapshot: ' + params.name;
+        return '(' + $filter('date')(params.time, 'shortDate') + ') ' + params.name;
       }
 
       function allDocIds () {
@@ -97,24 +97,6 @@
 
         return searchResults;
       }
-
-      function timestamp () {
-        var date    = new Date(self.time * 1000);
-
-        var hour    = date.getHours();
-        var minutes = date.getMinutes();
-        var year    = date.getFullYear();
-        var day     = date.getDate();
-
-        var months  = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'];
-        var month   = months[date.getMonth()];
-
-        if (minutes < 10) {
-          minutes = '0' + ('' + minutes);
-        }
-
-        return day + '-' + month + '-' + year + ' ' + hour + ':' + minutes;
-      }
     };
 
     // Return factory object

app/assets/javascripts/services/caseCSVSvc.js

@@ -79,6 +79,7 @@
             'Case ID',
             'Query Text',
             'Doc ID',
+            'Doc Position',
             'Title',
             'Rating',
           ];
@@ -353,7 +354,7 @@
               csvContent += dataString + EOL;
             }
             else {
-              angular.forEach(docs, function (doc) {
+              angular.forEach(docs, function (doc, index) {
                 var dataString;
                 var infoArray = [];
 
@@ -362,6 +363,7 @@
                 infoArray.push(stringifyField(aCase.lastScore.case_id));
                 infoArray.push(stringifyField(query.queryText));
                 infoArray.push(stringifyField(doc.id));
+                infoArray.push(stringifyField(index+1));
                 infoArray.push(stringifyField(doc.title));
                 infoArray.push(stringifyField(doc.getRating()));