Use websockets to display autotest run errors.

Also improve display of autotest run status in general by
differentiating between AutotestRunJob and AutotestResultsJob
completing. Also add websocket support for autotest results to
submission table.

Changelog.md

@@ -3,6 +3,7 @@
 ## [unreleased]
 - Allow inactive groups in the graders table to be toggled for display (#6778)
 - Enable plotly rendering for jupyter notebooks (#6871)
+- Display error message in real-time when running automated tests, rather than waiting for page refresh (#6878)
 
 ## [v2.4.1]
 - Internal changes only

app/assets/javascripts/Components/submission_table.jsx

@@ -12,7 +12,7 @@ import {
 import CollectSubmissionsModal from "./Modals/collect_submissions_modal";
 import ReleaseUrlsModal from "./Modals/release_urls_modal";
 import consumer from "/app/javascript/channels/consumer";
-import {generateFlashMessageContentsUsingStatus, renderFlashMessages} from "../flash";
+import {renderFlashMessages} from "../flash";
 
 class RawSubmissionTable extends React.Component {
   constructor() {
@@ -31,7 +31,7 @@ class RawSubmissionTable extends React.Component {
 
   componentDidMount() {
     this.fetchData();
-    this.createCollectSubmissionsChannelSubscription();
+    this.createChannelSubscriptions();
   }
 
   fetchData = () => {
@@ -364,25 +364,47 @@ class RawSubmissionTable extends React.Component {
     }, after_function);
   };
 
-  createCollectSubmissionsChannelSubscription = () => {
+  createChannelSubscriptions = () => {
+    // Subscribe to submission collection job status
     consumer.subscriptions.create(
       {channel: "CollectSubmissionsChannel", course_id: this.props.course_id},
       {
-        connected: () => {
-          // Called when the subscription is ready for use on the server
-        },
-
-        disconnected: () => {
-          // Called when the subscription has been terminated by the server
+        connected: () => {},
+        disconnected: () => {},
+        received: data => {
+          // Called when there's incoming data on the websocket for this channel
+          if (data["status"] != null) {
+            let message_data = generateMessage(data);
+            renderFlashMessages(message_data);
+          }
+          if (data["update_table"] != null) {
+            this.fetchData();
+          }
         },
+      }
+    );
 
+    // Subscribe to autotest runs job status
+    consumer.subscriptions.create(
+      {
+        channel: "TestRunsChannel",
+        course_id: this.props.course_id,
+        assignment_id: this.props.assignment_id,
+      },
+      {
+        connected: () => {},
+        disconnected: () => {},
         received: data => {
           // Called when there's incoming data on the websocket for this channel
           if (data["status"] != null) {
-            let message_data = generateFlashMessageContentsUsingStatus(data);
+            let message_data = generateMessage(data);
             renderFlashMessages(message_data);
           }
-          if (data["update_table"] != null) {
+          if (
+            data["update_table"] !== undefined &&
+            data["assignment_ids"] !== undefined &&
+            data["assignment_ids"].includes(this.props.assignment_id)
+          ) {
             this.fetchData();
           }
         },
@@ -599,3 +621,45 @@ class SubmissionsActionBox extends React.Component {
 export function makeSubmissionTable(elem, props) {
   return render(<SubmissionTable {...props} />, elem);
 }
+
+function generateMessage(status_data) {
+  let message_data = {};
+  switch (status_data["status"]) {
+    case "failed":
+      if (!status_data["exception"] || !status_data["exception"]["message"]) {
+        message_data["error"] = I18n.t("job.status.failed.no_message");
+      } else {
+        message_data["error"] = I18n.t("job.status.failed.message", {
+          error: status_data["exception"]["message"],
+        });
+      }
+      break;
+    case "completed":
+      if (status_data["job_class"] === "AutotestRunJob") {
+        message_data["success"] = I18n.t("automated_tests.autotest_run_job.status.completed");
+      } else if (status_data["job_class"] === "AutotestResultsJob") {
+        message_data["success"] = I18n.t("automated_tests.autotest_results_job.status.completed");
+      } else {
+        message_data["success"] = I18n.t("job.status.completed");
+      }
+      break;
+    case "queued":
+      message_data["notice"] = I18n.t("job.status.queued");
+      break;
+    default:
+      if (status_data["job_class"] === "SubmissionsJob") {
+        let progress = status_data["progress"];
+        let total = status_data["total"];
+        message_data["notice"] = I18n.t("submissions.collect.status.in_progress", {
+          progress,
+          total,
+        });
+      } else {
+        message_data["notice"] = I18n.t("job.status.in_progress", {progress, total});
+      }
+  }
+  if (status_data["warning_message"]) {
+    message_data["warning"] = status_data["warning_message"];
+  }
+  return message_data;
+}

app/assets/javascripts/Components/test_run_table.jsx

@@ -5,6 +5,7 @@ import {getType} from "mime/lite";
 import {dateSort, selectFilter} from "./Helpers/table_helpers";
 import {FileViewer} from "./Result/file_viewer";
 import consumer from "../../../javascript/channels/consumer";
+import {renderFlashMessages} from "../flash";
 
 export class TestRunTable extends React.Component {
   constructor(props) {
@@ -85,11 +86,19 @@ export class TestRunTable extends React.Component {
       },
       {
         connected: () => {},
-
         disconnected: () => {},
-
         received: data => {
-          this.fetchData();
+          // Called when there's incoming data on the websocket for this channel
+          if (data["status"] !== null) {
+            let message_data = generateMessage(data);
+            renderFlashMessages(message_data);
+          }
+          if (data["status"] === "completed") {
+            // Note: this gets called after AutotestRunJob completes (when a new
+            // TestRun is created), and after an AutotestResultsJob completed
+            // (when test results are available).
+            this.fetchData();
+          }
         },
       }
     );
@@ -393,3 +402,34 @@ class TestGroupFeedbackFileTable extends React.Component {
 export function makeTestRunTable(elem, props) {
   return render(<TestRunTable {...props} />, elem);
 }
+
+function generateMessage(status_data) {
+  let message_data = {};
+  switch (status_data["status"]) {
+    case "failed":
+      if (!status_data["exception"] || !status_data["exception"]["message"]) {
+        message_data["error"] = I18n.t("job.status.failed.no_message");
+      } else {
+        message_data["error"] = I18n.t("job.status.failed.message", {
+          error: status_data["exception"]["message"],
+        });
+      }
+      break;
+    case "completed":
+      if (status_data["job_class"] === "AutotestRunJob") {
+        message_data["success"] = I18n.t("automated_tests.autotest_run_job.status.completed");
+      } else {
+        message_data["success"] = I18n.t("automated_tests.autotest_results_job.status.completed");
+      }
+      break;
+    case "queued":
+      message_data["notice"] = I18n.t("job.status.queued");
+      break;
+    default:
+      message_data["notice"] = I18n.t("automated_tests.autotest_run_job.status.in_progress");
+  }
+  if (status_data["warning_message"]) {
+    message_data["warning"] = status_data["warning_message"];
+  }
+  return message_data;
+}

app/assets/javascripts/flash.js

@@ -5,18 +5,18 @@ const generateFlashMessageContentsUsingStatus = status_data => {
   switch (status_data["status"]) {
     case "failed":
       if (!status_data["exception"] || !status_data["exception"]["message"]) {
-        message_data["error"] = I18n.t("submissions.collect.status.failed.no_message");
+        message_data["error"] = I18n.t("job.status.failed.no_message");
       } else {
-        message_data["error"] = I18n.t("submissions.collect.status.failed.message", {
+        message_data["error"] = I18n.t("job.status.failed.message", {
           error: status_data["exception"]["message"],
         });
       }
       break;
     case "completed":
-      message_data["success"] = I18n.t("submissions.collect.status.completed");
+      message_data["success"] = I18n.t("job.status.completed");
       break;
     case "queued":
-      message_data["notice"] = I18n.t("submissions.collect.status.queued");
+      message_data["notice"] = I18n.t("job.status.queued");
       break;
     default:
       let progress = status_data["progress"];

app/controllers/automated_tests_controller.rb

@@ -55,13 +55,13 @@ def execute_test_run
                                                    context: { assignment: assignment, grouping: grouping })).value
     if allowed
       grouping.decrease_test_tokens
-      @current_job = AutotestRunJob.perform_later(request.protocol + request.host_with_port,
-                                                  current_role.id,
-                                                  assignment.id,
-                                                  [grouping.group_id],
-                                                  collected: false)
-      session[:job_id] = @current_job.job_id
-      flash_message(:success, I18n.t('automated_tests.test_run_table.tests_running'))
+      flash_message(:notice, I18n.t('automated_tests.autotest_run_job.status.in_progress'))
+      AutotestRunJob.perform_later(request.protocol + request.host_with_port,
+                                   current_role.id,
+                                   assignment.id,
+                                   [grouping.group_id],
+                                   user: current_user,
+                                   collected: false)
     end
   rescue StandardError => e
     flash_message(:error, e.message)

app/controllers/results_controller.rb

@@ -281,12 +281,12 @@ def edit
 
   def run_tests
     submission = record.submission
-    @current_job = AutotestRunJob.perform_later(request.protocol + request.host_with_port,
-                                                current_role.id,
-                                                submission.assignment.id,
-                                                [submission.grouping.group_id])
-    session[:job_id] = @current_job.job_id
-    flash_message(:success, I18n.t('automated_tests.test_run_table.tests_running'))
+    flash_message(:notice, I18n.t('automated_tests.autotest_run_job.status.in_progress'))
+    AutotestRunJob.perform_later(request.protocol + request.host_with_port,
+                                 current_role.id,
+                                 submission.assignment.id,
+                                 [submission.grouping.group_id],
+                                 user: current_user)
   end
 
   ##  Tag Methods  ##

app/controllers/submissions_controller.rb

@@ -252,12 +252,12 @@ def run_tests
     begin
       if !group_ids.empty?
         if flash_allowance(:error, allowance_to(:run_tests?, current_role, context: { assignment: assignment })).value
-          @current_job = AutotestRunJob.perform_later(request.protocol + request.host_with_port,
-                                                      current_role.id,
-                                                      assignment.id,
-                                                      group_ids)
-          session[:job_id] = @current_job.job_id
-          success = I18n.t('automated_tests.tests_running', assignment_identifier: assignment.short_identifier)
+          AutotestRunJob.perform_later(request.protocol + request.host_with_port,
+                                       current_role.id,
+                                       assignment.id,
+                                       group_ids,
+                                       user: current_user)
+          success = I18n.t('automated_tests.autotest_run_job.status.in_progress')
         end
       else
         error = I18n.t('automated_tests.need_submission')

app/helpers/automated_tests_helper.rb

@@ -222,9 +222,6 @@ def run_tests(assignment, host_with_port, group_ids, role, collected: true, batc
           autotest_test_id: test_id_hash[grouping.group.id],
           status: :in_progress
         )
-        if batch.nil?
-          TestRunsChannel.broadcast_to(role.user, body: 'sent')
-        end
       end
     end
 

app/jobs/autotest_results_job.rb

@@ -25,26 +25,46 @@ def perform(_retry: 3)
                     .pluck('assessments.id', 'test_runs.id')
                     .group_by(&:first)
                     .transform_values { |v| v.map(&:second) }
+
+    # Map users to the ids of assignments with updated results from their test runs
+    updated_assignment_batch_runs = Hash.new { |h, k| h[k] = [] }
+
     ids.each do |assignment_id, test_run_ids|
       assignment = Assignment.find(assignment_id)
       test_runs = TestRun.where(id: test_run_ids)
-      statuses(assignment, test_runs).each do |autotest_test_id, status|
+      updated_users = []
+      statuses(assignment, test_runs).each do |autotest_test_id, test_run_status|
         # statuses from rq: https://python-rq.org/docs/jobs/#retrieving-a-job-from-redis
-        status = 'not_found' if status.nil?
-        if %(started queued deferred).include? status
+        test_run_status = 'not_found' if test_run_status.nil?
+        if %(started queued deferred).include? test_run_status
           outstanding_results = true
         else
           test_run = test_runs.find_by(autotest_test_id: autotest_test_id)
-          if %(finished failed).include? status
+          if %(finished failed).include? test_run_status
             results(test_run.grouping.assignment, test_run) unless test_run.nil?
           else
-            test_run&.failure(status)
+            test_run&.failure(test_run_status)
           end
-          if !test_run.nil? && test_run.test_batch_id.nil?
-            TestRunsChannel.broadcast_to(test_run.role.user, body: 'sent')
+          unless test_run.nil?
+            if test_run.test_batch_id.nil?
+              TestRunsChannel.broadcast_to(test_run.role.user, { status: 'completed', job_class: 'AutotestResultsJob' })
+            else
+              updated_users << test_run.role.user
+            end
           end
         end
       end
+      updated_users.each do |user|
+        updated_assignment_batch_runs[user] << assignment_id
+      end
+    end
+
+    updated_assignment_batch_runs.each do |user, assignment_ids|
+      TestRunsChannel.broadcast_to(user,
+                                   { status: 'completed',
+                                     job_class: 'AutotestResultsJob',
+                                     assignment_ids: assignment_ids,
+                                     update_table: true })
     end
     outstanding_results
   ensure

app/jobs/autotest_run_job.rb

@@ -4,7 +4,7 @@ def self.show_status(_status)
     I18n.t('poll_job.autotest_run_job_enqueuing')
   end
 
-  def perform(host_with_port, role_id, assignment_id, group_ids, collected: true)
+  def perform(host_with_port, role_id, assignment_id, group_ids, user: nil, collected: true)
     # create and enqueue test runs
     role = Role.find(role_id)
     test_batch = group_ids.size > 1 ? TestBatch.create(course: role.course) : nil # create 1 batch object if needed
@@ -14,5 +14,14 @@ def perform(host_with_port, role_id, assignment_id, group_ids, collected: true)
       run_tests(assignment, host_with_port, group_id_slice, role, collected: collected, batch: test_batch)
     end
     AutotestResultsJob.perform_later
+    unless user.nil?
+      TestRunsChannel.broadcast_to(user, { status: 'completed', job_class: 'AutotestRunJob' })
+    end
+  rescue StandardError => e
+    status.catch_exception(e)
+    unless user.nil?
+      TestRunsChannel.broadcast_to(user, { **status.to_h, job_class: 'AutotestRunJob' })
+    end
+    raise e
   end
 end

config/locales/defaults/job/en.yml

@@ -0,0 +1,10 @@
+---
+en:
+  job:
+    status:
+      completed: This job is now complete.
+      failed:
+        message: 'This job failed due to the following error: {{error}}'
+        no_message: This job failed due to an unexpected error.
+      in_progress: This job is in progress.
+      queued: This job is waiting to be run.