Fixed: Queued jobs are not considered in deferring logic

.vscode/settings.json

@@ -40,9 +40,4 @@
     "cSpell.words": [
         "crowdsourcing"
     ],
-    "isort.args":["--profile", "black"],
-    "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter",
-        "editor.formatOnSave": true
-    }
 }

cvat/apps/engine/views.py

@@ -2972,9 +2972,14 @@ def _export_annotations(
         rq_request = rq_job.meta.get('request', None)
         request_time = rq_request.get('timestamp', None) if rq_request else None
         if request_time is None or request_time < last_instance_update_time:
-            # in case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER
-            # we have to enqueue dependent jobs after canceling one
-            rq_job.cancel(enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER)
+            if rq_job.worker_name:
+                send_stop_job_command(rq_job.connection, rq_job.id)
+            else:
+                # in case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER
+                # we have to enqueue dependent jobs after canceling one
+                rq_job.cancel(
+                    enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER
+                )
             rq_job.delete()
         else:
             if rq_job.is_finished:

cvat/apps/events/handlers.py

@@ -521,6 +521,11 @@ def handle_dataset_import(
     handle_dataset_io(instance, "import", format_name=format_name, cloud_storage=cloud_storage)
 
 def handle_rq_exception(rq_job, exc_type, exc_value, tb):
+    if isinstance(exc_type, SystemExit):
+        # rq process was killed intentionally by SIGTERM
+        # we do not need to log it
+        return False
+
     oid = rq_job.meta.get("org_id", None)
     oslug = rq_job.meta.get("org_slug", None)
     pid = rq_job.meta.get("project_id", None)

cvat/rqworker.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 import os
-
+import signal
 from rq import Worker
 
 import cvat.utils.remote_debugger as debug
@@ -42,12 +42,18 @@ def execute_job(self, *args, **kwargs):
         # errors during debugging
         # https://stackoverflow.com/questions/8242837/django-multiprocessing-and-database-connections/10684672#10684672
         from django import db
-        db.connections.close_all()
 
+        db.connections.close_all()
         return self.perform_job(*args, **kwargs)
 
+    def kill_horse(self, sig: signal.Signals = signal.SIGTERM):
+        # Send SIGTERM instead of default SIGKILL in debug mode as SIGKILL can't be handled
+        # to prevent killing rq worker process as rq code handles SIGTERM properly
+        super().kill_horse(sig)
+
 
 if debug.is_debugging_enabled():
+
     class RemoteDebugWorker(SimpleWorker):
         """
         Support for VS code debugger
@@ -68,6 +74,7 @@ def execute_job(self, *args, **kwargs):
 
 if os.environ.get("COVERAGE_PROCESS_START"):
     import coverage
+
     default_exit = os._exit
 
     def coverage_exit(*args, **kwargs):