Use websockets to display autotest run errors. (#6878)

Also improve display of autotest run status in general by
differentiating between AutotestRunJob and AutotestResultsJob
completing. Also add websocket support for autotest results to
submission table.

Changelog.md

@@ -6,6 +6,7 @@
 - Enable plotly rendering for jupyter notebooks (#6871)
 - Bug Fix: Prevent assigning inactive graders to a group (#6801)
 - Added new API routes for the `create` and `destroy` actions of SectionsController (#6873)
+- Display error message in real-time when running automated tests, rather than waiting for page refresh (#6878)
 
 ## [v2.4.1]
 - Internal changes only

app/assets/javascripts/Components/submission_table.jsx

@@ -12,7 +12,7 @@ import {
 import CollectSubmissionsModal from "./Modals/collect_submissions_modal";
 import ReleaseUrlsModal from "./Modals/release_urls_modal";
 import consumer from "/app/javascript/channels/consumer";
-import {generateFlashMessageContentsUsingStatus, renderFlashMessages} from "../flash";
+import {renderFlashMessages} from "../flash";
 
 class RawSubmissionTable extends React.Component {
   constructor() {
@@ -31,7 +31,7 @@ class RawSubmissionTable extends React.Component {
 
   componentDidMount() {
     this.fetchData();
-    this.createCollectSubmissionsChannelSubscription();
+    this.createChannelSubscriptions();
   }
 
   fetchData = () => {
@@ -364,25 +364,47 @@ class RawSubmissionTable extends React.Component {
     }, after_function);
   };
 
-  createCollectSubmissionsChannelSubscription = () => {
+  createChannelSubscriptions = () => {
+    // Subscribe to submission collection job status
     consumer.subscriptions.create(
       {channel: "CollectSubmissionsChannel", course_id: this.props.course_id},
       {
-        connected: () => {
-          // Called when the subscription is ready for use on the server
-        },
-
-        disconnected: () => {
-          // Called when the subscription has been terminated by the server
+        connected: () => {},
+        disconnected: () => {},
+        received: data => {
+          // Called when there's incoming data on the websocket for this channel
+          if (data["status"] != null) {
+            let message_data = generateMessage(data);
+            renderFlashMessages(message_data);
+          }
+          if (data["update_table"] != null) {
+            this.fetchData();
+          }
         },
+      }
+    );
 
+    // Subscribe to autotest runs job status
+    consumer.subscriptions.create(
+      {
+        channel: "TestRunsChannel",
+        course_id: this.props.course_id,
+        assignment_id: this.props.assignment_id,
+      },
+      {
+        connected: () => {},
+        disconnected: () => {},
         received: data => {
           // Called when there's incoming data on the websocket for this channel
           if (data["status"] != null) {
-            let message_data = generateFlashMessageContentsUsingStatus(data);
+            let message_data = generateMessage(data);
             renderFlashMessages(message_data);
           }
-          if (data["update_table"] != null) {
+          if (
+            data["update_table"] !== undefined &&
+            data["assignment_ids"] !== undefined &&
+            data["assignment_ids"].includes(this.props.assignment_id)
+          ) {
             this.fetchData();
           }
         },
@@ -599,3 +621,45 @@ class SubmissionsActionBox extends React.Component {
 export function makeSubmissionTable(elem, props) {
   return render(<SubmissionTable {...props} />, elem);
 }
+
+function generateMessage(status_data) {
+  let message_data = {};
+  switch (status_data["status"]) {
+    case "failed":
+      if (!status_data["exception"] || !status_data["exception"]["message"]) {
+        message_data["error"] = I18n.t("job.status.failed.no_message");
+      } else {
+        message_data["error"] = I18n.t("job.status.failed.message", {
+          error: status_data["exception"]["message"],
+        });
+      }
+      break;
+    case "completed":
+      if (status_data["job_class"] === "AutotestRunJob") {
+        message_data["success"] = I18n.t("automated_tests.autotest_run_job.status.completed");
+      } else if (status_data["job_class"] === "AutotestResultsJob") {
+        message_data["success"] = I18n.t("automated_tests.autotest_results_job.status.completed");
+      } else {
+        message_data["success"] = I18n.t("job.status.completed");
+      }
+      break;
+    case "queued":
+      message_data["notice"] = I18n.t("job.status.queued");
+      break;
+    default:
+      if (status_data["job_class"] === "SubmissionsJob") {
+        let progress = status_data["progress"];
+        let total = status_data["total"];
+        message_data["notice"] = I18n.t("submissions.collect.status.in_progress", {
+          progress,
+          total,
+        });
+      } else {
+        message_data["notice"] = I18n.t("job.status.in_progress", {progress, total});
+      }
+  }
+  if (status_data["warning_message"]) {
+    message_data["warning"] = status_data["warning_message"];
+  }
+  return message_data;
+}

app/assets/javascripts/Components/test_run_table.jsx

@@ -5,6 +5,7 @@ import {getType} from "mime/lite";
 import {dateSort, selectFilter} from "./Helpers/table_helpers";
 import {FileViewer} from "./Result/file_viewer";
 import consumer from "../../../javascript/channels/consumer";
+import {renderFlashMessages} from "../flash";
 
 export class TestRunTable extends React.Component {
   constructor(props) {
@@ -85,11 +86,19 @@ export class TestRunTable extends React.Component {
       },
       {
         connected: () => {},
-
         disconnected: () => {},
-
         received: data => {
-          this.fetchData();
+          // Called when there's incoming data on the websocket for this channel
+          if (data["status"] !== null) {
+            let message_data = generateMessage(data);
+            renderFlashMessages(message_data);
+          }
+          if (data["status"] === "completed") {
+            // Note: this gets called after AutotestRunJob completes (when a new
+            // TestRun is created), and after an AutotestResultsJob completed
+            // (when test results are available).
+            this.fetchData();
+          }
         },
       }
     );
@@ -393,3 +402,34 @@ class TestGroupFeedbackFileTable extends React.Component {
 export function makeTestRunTable(elem, props) {
   return render(<TestRunTable {...props} />, elem);
 }
+
+function generateMessage(status_data) {
+  let message_data = {};
+  switch (status_data["status"]) {
+    case "failed":
+      if (!status_data["exception"] || !status_data["exception"]["message"]) {
+        message_data["error"] = I18n.t("job.status.failed.no_message");
+      } else {
+        message_data["error"] = I18n.t("job.status.failed.message", {
+          error: status_data["exception"]["message"],
+        });
+      }
+      break;
+    case "completed":
+      if (status_data["job_class"] === "AutotestRunJob") {
+        message_data["success"] = I18n.t("automated_tests.autotest_run_job.status.completed");
+      } else {
+        message_data["success"] = I18n.t("automated_tests.autotest_results_job.status.completed");
+      }
+      break;
+    case "queued":
+      message_data["notice"] = I18n.t("job.status.queued");
+      break;
+    default:
+      message_data["notice"] = I18n.t("automated_tests.autotest_run_job.status.in_progress");
+  }
+  if (status_data["warning_message"]) {
+    message_data["warning"] = status_data["warning_message"];
+  }
+  return message_data;
+}

app/assets/javascripts/flash.js

@@ -5,18 +5,18 @@ const generateFlashMessageContentsUsingStatus = status_data => {
   switch (status_data["status"]) {
     case "failed":
       if (!status_data["exception"] || !status_data["exception"]["message"]) {
-        message_data["error"] = I18n.t("submissions.collect.status.failed.no_message");
+        message_data["error"] = I18n.t("job.status.failed.no_message");
       } else {
-        message_data["error"] = I18n.t("submissions.collect.status.failed.message", {
+        message_data["error"] = I18n.t("job.status.failed.message", {
           error: status_data["exception"]["message"],
         });
       }
       break;
     case "completed":
-      message_data["success"] = I18n.t("submissions.collect.status.completed");
+      message_data["success"] = I18n.t("job.status.completed");
       break;
     case "queued":
-      message_data["notice"] = I18n.t("submissions.collect.status.queued");
+      message_data["notice"] = I18n.t("job.status.queued");
       break;
     default:
       let progress = status_data["progress"];

app/controllers/automated_tests_controller.rb

@@ -55,13 +55,13 @@ def execute_test_run
                                                    context: { assignment: assignment, grouping: grouping })).value
     if allowed
       grouping.decrease_test_tokens
-      @current_job = AutotestRunJob.perform_later(request.protocol + request.host_with_port,
-                                                  current_role.id,
-                                                  assignment.id,
-                                                  [grouping.group_id],
-                                                  collected: false)
-      session[:job_id] = @current_job.job_id
-      flash_message(:success, I18n.t('automated_tests.test_run_table.tests_running'))
+      flash_message(:notice, I18n.t('automated_tests.autotest_run_job.status.in_progress'))
+      AutotestRunJob.perform_later(request.protocol + request.host_with_port,
+                                   current_role.id,
+                                   assignment.id,
+                                   [grouping.group_id],
+                                   user: current_user,
+                                   collected: false)
     end
   rescue StandardError => e
     flash_message(:error, e.message)

app/controllers/results_controller.rb

@@ -281,12 +281,12 @@ def edit
 
   def run_tests
     submission = record.submission
-    @current_job = AutotestRunJob.perform_later(request.protocol + request.host_with_port,
-                                                current_role.id,
-                                                submission.assignment.id,
-                                                [submission.grouping.group_id])
-    session[:job_id] = @current_job.job_id
-    flash_message(:success, I18n.t('automated_tests.test_run_table.tests_running'))
+    flash_message(:notice, I18n.t('automated_tests.autotest_run_job.status.in_progress'))
+    AutotestRunJob.perform_later(request.protocol + request.host_with_port,
+                                 current_role.id,
+                                 submission.assignment.id,
+                                 [submission.grouping.group_id],
+                                 user: current_user)
   end
 
   ##  Tag Methods  ##

app/controllers/submissions_controller.rb

@@ -252,12 +252,12 @@ def run_tests
     begin
       if !group_ids.empty?
         if flash_allowance(:error, allowance_to(:run_tests?, current_role, context: { assignment: assignment })).value
-          @current_job = AutotestRunJob.perform_later(request.protocol + request.host_with_port,
-                                                      current_role.id,
-                                                      assignment.id,
-                                                      group_ids)
-          session[:job_id] = @current_job.job_id
-          success = I18n.t('automated_tests.tests_running', assignment_identifier: assignment.short_identifier)
+          AutotestRunJob.perform_later(request.protocol + request.host_with_port,
+                                       current_role.id,
+                                       assignment.id,
+                                       group_ids,
+                                       user: current_user)
+          success = I18n.t('automated_tests.autotest_run_job.status.in_progress')
         end
       else
         error = I18n.t('automated_tests.need_submission')

app/helpers/automated_tests_helper.rb

@@ -222,9 +222,6 @@ def run_tests(assignment, host_with_port, group_ids, role, collected: true, batc
           autotest_test_id: test_id_hash[grouping.group.id],
           status: :in_progress
         )
-        if batch.nil?
-          TestRunsChannel.broadcast_to(role.user, body: 'sent')
-        end
       end
     end
 

app/jobs/autotest_results_job.rb

@@ -25,26 +25,46 @@ def perform(_retry: 3)
                     .pluck('assessments.id', 'test_runs.id')
                     .group_by(&:first)
                     .transform_values { |v| v.map(&:second) }
+
+    # Map users to the ids of assignments with updated results from their test runs
+    updated_assignment_batch_runs = Hash.new { |h, k| h[k] = [] }
+
     ids.each do |assignment_id, test_run_ids|
       assignment = Assignment.find(assignment_id)
       test_runs = TestRun.where(id: test_run_ids)
-      statuses(assignment, test_runs).each do |autotest_test_id, status|
+      updated_users = []
+      statuses(assignment, test_runs).each do |autotest_test_id, test_run_status|
         # statuses from rq: https://python-rq.org/docs/jobs/#retrieving-a-job-from-redis
-        status = 'not_found' if status.nil?
-        if %(started queued deferred).include? status
+        test_run_status = 'not_found' if test_run_status.nil?
+        if %(started queued deferred).include? test_run_status
           outstanding_results = true
         else
           test_run = test_runs.find_by(autotest_test_id: autotest_test_id)
-          if %(finished failed).include? status
+          if %(finished failed).include? test_run_status
             results(test_run.grouping.assignment, test_run) unless test_run.nil?
           else
-            test_run&.failure(status)
+            test_run&.failure(test_run_status)
           end
-          if !test_run.nil? && test_run.test_batch_id.nil?
-            TestRunsChannel.broadcast_to(test_run.role.user, body: 'sent')
+          unless test_run.nil?
+            if test_run.test_batch_id.nil?
+              TestRunsChannel.broadcast_to(test_run.role.user, { status: 'completed', job_class: 'AutotestResultsJob' })
+            else
+              updated_users << test_run.role.user
+            end
           end
         end
       end
+      updated_users.each do |user|
+        updated_assignment_batch_runs[user] << assignment_id
+      end
+    end
+
+    updated_assignment_batch_runs.each do |user, assignment_ids|
+      TestRunsChannel.broadcast_to(user,
+                                   { status: 'completed',
+                                     job_class: 'AutotestResultsJob',
+                                     assignment_ids: assignment_ids,
+                                     update_table: true })
     end
     outstanding_results
   ensure

app/jobs/autotest_run_job.rb

@@ -4,7 +4,7 @@ def self.show_status(_status)
     I18n.t('poll_job.autotest_run_job_enqueuing')
   end
 
-  def perform(host_with_port, role_id, assignment_id, group_ids, collected: true)
+  def perform(host_with_port, role_id, assignment_id, group_ids, user: nil, collected: true)
     # create and enqueue test runs
     role = Role.find(role_id)
     test_batch = group_ids.size > 1 ? TestBatch.create(course: role.course) : nil # create 1 batch object if needed
@@ -14,5 +14,14 @@ def perform(host_with_port, role_id, assignment_id, group_ids, collected: true)
       run_tests(assignment, host_with_port, group_id_slice, role, collected: collected, batch: test_batch)
     end
     AutotestResultsJob.perform_later
+    unless user.nil?
+      TestRunsChannel.broadcast_to(user, { status: 'completed', job_class: 'AutotestRunJob' })
+    end
+  rescue StandardError => e
+    status.catch_exception(e)
+    unless user.nil?
+      TestRunsChannel.broadcast_to(user, { **status.to_h, job_class: 'AutotestRunJob' })
+    end
+    raise e
   end
 end

config/locales/defaults/job/en.yml

@@ -0,0 +1,10 @@
+---
+en:
+  job:
+    status:
+      completed: This job is now complete.
+      failed:
+        message: 'This job failed due to the following error: {{error}}'
+        no_message: This job failed due to an unexpected error.
+      in_progress: This job is in progress.
+      queued: This job is waiting to be run.