Human in the loop

compute_worker/compute_worker.py

@@ -314,6 +314,9 @@ def run_wrapper(run_args):
         run.prepare()
         run.start()
         if run.is_scoring:
+            if run.human_in_the_loop:
+                run.wait_for_human_validation()
+                run._update_status(SubmissionStatus.FINISHED)
             run.push_scores()
         run.push_output()
     except DockerImagePullException as e:
@@ -470,6 +473,7 @@ def __init__(self, run_args):
         self.prediction_result = run_args["prediction_result"]
         self.scoring_result = run_args.get("scoring_result")
         self.execution_time_limit = run_args["execution_time_limit"]
+        self.human_in_the_loop = run_args.get("human_in_the_loop", False)
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -1445,11 +1449,55 @@ def start(self):
                 )
                 # Raise so upstream marks failed immediately
                 raise SubmissionException("Child task failed or non-zero return code")
-            self._update_status(SubmissionStatus.FINISHED)
+
+            if not self.human_in_the_loop:
+                self._update_status(SubmissionStatus.FINISHED)
 
         else:
             self._update_status(SubmissionStatus.SCORING)
 
+    def wait_for_human_validation(self):
+        container_output_dir = self.output_dir
+        host_output_dir = self._get_host_path(self.output_dir)
+
+        scores_path = os.path.join(host_output_dir, "scores.json")
+        if not os.path.exists(os.path.join(container_output_dir, "scores.json")):
+            scores_path = os.path.join(host_output_dir, "scores.txt")
+
+        approved_container = os.path.join(container_output_dir, "hitl_approved")
+        rejected_container = os.path.join(container_output_dir, "hitl_rejected")
+        approved_host = os.path.join(host_output_dir, "hitl_approved")
+        rejected_host = os.path.join(host_output_dir, "hitl_rejected")
+
+        logger.info("=" * 60)
+        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
+        logger.info("Inspect the scores file:")
+        logger.info(f"  cat {scores_path}")
+        logger.info(f"To approve : touch {approved_host}")
+        logger.info(f"To reject  : touch {rejected_host}")
+        logger.info("=" * 60)
+
+        poll_interval = 3
+        max_wait = 60 * 60 * 24
+
+        elapsed = 0
+        while elapsed < max_wait:
+            if os.path.exists(approved_container):
+                logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
+                return
+            if os.path.exists(rejected_container):
+                raise SubmissionException(
+                    f"HITL: scores rejected by the compute node operator "
+                    f"(submission {self.submission_id})"
+                )
+            time.sleep(poll_interval)
+            elapsed += poll_interval
+
+        raise SubmissionException(
+            f"HITL: 24h timeout reached without validation "
+            f"(submission {self.submission_id})"
+        )
+
     def push_scores(self):
         """This is only ran at the end of the scoring step"""
         # POST to some endpoint:

src/apps/api/serializers/competitions.py

@@ -272,7 +272,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def validate_phases(self, phases):
@@ -417,7 +418,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def get_leaderboards(self, instance):

src/apps/competitions/models.py

@@ -89,6 +89,7 @@ class Competition(models.Model):
 
     # If true, forum is enabled (default=True)
     forum_enabled = models.BooleanField(default=True)
+    enable_human_in_the_loop = models.BooleanField(default=False)
 
     def __str__(self):
         return f"competition-{self.title}-{self.pk}-{self.competition_type}"

src/apps/competitions/tasks.py

@@ -59,6 +59,7 @@
     "contact_email",
     "fact_sheet",
     "forum_enabled",
+    "enable_human_in_the_loop"
 ]
 
 TASK_FIELDS = [
@@ -124,6 +125,11 @@
 
 
 def _send_to_compute_worker(submission, is_scoring):
+    hitl_active = (
+        submission.phase.competition.enable_human_in_the_loop
+        and submission.queue is not None
+    )
+
     run_args = {
         "user_pk": submission.owner.pk,
         "submissions_api_url": settings.SUBMISSIONS_API_URL,
@@ -134,6 +140,7 @@ def _send_to_compute_worker(submission, is_scoring):
         ),
         "id": submission.pk,
         "is_scoring": is_scoring,
+        "human_in_the_loop": hitl_active,
     }
 
     if (
@@ -209,9 +216,12 @@ def _send_to_compute_worker(submission, is_scoring):
     logger.info(run_args)
 
     # Pad timelimit so worker has time to cleanup
-    time_padding = 60 * 20  # 20 minutes
+    time_padding = 60 * 20
     time_limit = submission.phase.execution_time_limit + time_padding
 
+    if is_scoring and hitl_active:
+        time_limit = 60 * 60 * 25
+
     if (
         submission.phase.competition.queue
     ):  # if the competition is running on a custom queue, not the default queue
@@ -321,6 +331,23 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False):
     )
     submission = qs.get(pk=submission_pk)
 
+    # ── HUMAN IN THE LOOP SECURE (private CW only)
+    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+        if submission.queue is None:
+            logger.error(
+                f"HITL: submission {submission_pk} rejected — "
+                f"Human in the Loop is enabled but no private queue is configured "
+                f"(competition or participant group)."
+            )
+            submission.status = Submission.FAILED
+            submission.status_details = (
+                "This competition requires Human in the Loop validation, but no "
+                "private compute queue is configured. Contact the organizer."
+            )
+            submission.save(update_fields=["status", "status_details"])
+            return
+    # ── HITL SECURE (private CW only)
+
     if submission.is_specific_task_re_run:
         # Should only be one task for a specified task submission
         tasks = Task.objects.filter(pk__in=task_pks)

src/static/riot/competitions/detail/submission_manager.tag

@@ -107,7 +107,8 @@
                 <td class="right aligned collapsing">
                     { submission.status }
                     <sup data-tooltip="{submission.status_details}">
-                        <i if="{submission.status === 'Failed'}" class="failed question circle icon"></i>
+                        <i if="{submission.status === 'Failed' && !is_hitl_failure(submission)}" class="failed question circle icon"></i>
+                        <i if="{submission.status === 'Failed' && is_hitl_failure(submission)}" class="orange lock icon"></i>
                     </sup>
                     <sup data-tooltip="An organizer will run your submission soon">
                         <i if="{submission.status === 'Submitting' && !submission.auto_run}"
@@ -311,7 +312,10 @@
             self.update()
         }
 
-
+        self.is_hitl_failure = function (submission) {
+            return !!(submission.status_details && submission.status_details.indexOf('Human in the Loop') !== -1)
+        }
+
         self.update_submissions = function (filters) {
             self.loading = true
             self.update()

src/static/riot/competitions/editor/_competition_details.tag

@@ -211,6 +211,27 @@
             </sup>
         </div>
 
+        <!--  Human in the Loop  -->
+        <div class="field">
+            <label>Human in the Loop</label>
+            <div class="ui checkbox">
+                <label>Enable Human in the Loop validation</label>
+                <input type="checkbox" ref="enable_human_in_the_loop" onchange="{form_updated}">
+            </div>
+            <sup>
+                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform. Only active on private queues."
+                        data-inverted=""
+                        data-position="bottom center">
+                    <i class="help icon circle"></i>
+                </span>
+            </sup>
+            <div ref="hitl_warning" class="ui yellow message" style="display:none; margin-top: 0.5em;">
+                <i class="exclamation triangle icon"></i>
+                Human in the Loop is only active on <strong>private queues</strong>.
+                It will have no effect until a private queue is selected for this competition or its participant groups.
+            </div>
+        </div>
+
         <!--  Public submissions  -->
         <div class="field">
             <label>Public Submissions</label>
@@ -314,6 +335,12 @@
             self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked
             self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked
             self.data["forum_enabled"] = self.refs.forum_enabled.checked
+            self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked
+            const hitlChecked = self.refs.enable_human_in_the_loop.checked
+            const hasPrivateQueue = !!(self.data["queue"] && self.data["queue"] !== "")
+            if (self.refs.hitl_warning) {
+                self.refs.hitl_warning.style.display = (hitlChecked && !hasPrivateQueue) ? 'block' : 'none'
+            }
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()
@@ -452,6 +479,7 @@
             self.refs.auto_run_submissions.checked = competition.auto_run_submissions
             self.refs.can_participants_make_submissions_public.checked = competition.can_participants_make_submissions_public
             self.refs.forum_enabled.checked = competition.forum_enabled
+            self.refs.enable_human_in_the_loop.checked = competition.enable_human_in_the_loop
             self.refs.make_programs_available.checked = competition.make_programs_available
             self.refs.make_input_data_available.checked = competition.make_input_data_available
             $(self.refs.docker_image).val(competition.docker_image)