From 0ccc352113c711a83bdded0c9a51473f71531ff6 Mon Sep 17 00:00:00 2001 From: Idir Chikhoune Date: Wed, 24 Jun 2026 12:51:01 +0200 Subject: [PATCH 1/3] HITL enabled and working, needs polish and test --- compute_worker/compute_worker.py | 50 ++++++++++++++++++- src/apps/api/serializers/competitions.py | 6 ++- src/apps/competitions/models.py | 1 + src/apps/competitions/tasks.py | 5 ++ .../editor/_competition_details.tag | 18 +++++++ 5 files changed, 77 insertions(+), 3 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 243dcd16d..75a3540c3 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -314,6 +314,9 @@ def run_wrapper(run_args): run.prepare() run.start() if run.is_scoring: + if run.human_in_the_loop: + run.wait_for_human_validation() + run._update_status(SubmissionStatus.FINISHED) run.push_scores() run.push_output() except DockerImagePullException as e: @@ -470,6 +473,7 @@ def __init__(self, run_args): self.prediction_result = run_args["prediction_result"] self.scoring_result = run_args.get("scoring_result") self.execution_time_limit = run_args["execution_time_limit"] + self.human_in_the_loop = run_args.get("human_in_the_loop", False) # stdout and stderr self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = ( self._get_stdout_stderr_file_names(run_args) @@ -1445,11 +1449,55 @@ def start(self): ) # Raise so upstream marks failed immediately raise SubmissionException("Child task failed or non-zero return code") - self._update_status(SubmissionStatus.FINISHED) + + if not self.human_in_the_loop: + self._update_status(SubmissionStatus.FINISHED) else: self._update_status(SubmissionStatus.SCORING) + def wait_for_human_validation(self): + container_output_dir = self.output_dir + host_output_dir = self._get_host_path(self.output_dir) + + scores_path = os.path.join(host_output_dir, "scores.json") + if not os.path.exists(os.path.join(container_output_dir, "scores.json")): + scores_path = os.path.join(host_output_dir, "scores.txt") + + approved_container = os.path.join(container_output_dir, "hitl_approved") + rejected_container = os.path.join(container_output_dir, "hitl_rejected") + approved_host = os.path.join(host_output_dir, "hitl_approved") + rejected_host = os.path.join(host_output_dir, "hitl_rejected") + + logger.info("=" * 60) + logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}") + logger.info("Inspect the scores file:") + logger.info(f" cat {scores_path}") + logger.info(f"To approve : touch {approved_host}") + logger.info(f"To reject : touch {rejected_host}") + logger.info("=" * 60) + + poll_interval = 3 + max_wait = 60 * 60 * 24 + + elapsed = 0 + while elapsed < max_wait: + if os.path.exists(approved_container): # ← poll le chemin conteneur + logger.info(f"HITL: submission {self.submission_id} approved, sending scores.") + return + if os.path.exists(rejected_container): # ← poll le chemin conteneur + raise SubmissionException( + f"HITL: scores rejected by the compute node operator " + f"(submission {self.submission_id})" + ) + time.sleep(poll_interval) + elapsed += poll_interval + + raise SubmissionException( + f"HITL: 24h timeout reached without validation " + f"(submission {self.submission_id})" + ) + def push_scores(self): """This is only ran at the end of the scoring step""" # POST to some endpoint: diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index a0fb14fda..3a566446e 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -272,7 +272,8 @@ class Meta: 'contact_email', 'report', 'whitelist_emails', - 'forum_enabled' + 'forum_enabled', + 'enable_human_in_the_loop' ) def validate_phases(self, phases): @@ -417,7 +418,8 @@ class Meta: 'contact_email', 'report', 'whitelist_emails', - 'forum_enabled' + 'forum_enabled', + 'enable_human_in_the_loop' ) def get_leaderboards(self, instance): diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index f919ae432..164e66536 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -89,6 +89,7 @@ class Competition(models.Model): # If true, forum is enabled (default=True) forum_enabled = models.BooleanField(default=True) + enable_human_in_the_loop = models.BooleanField(default=False) def __str__(self): return f"competition-{self.title}-{self.pk}-{self.competition_type}" diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 71acb85c1..2e052e9ca 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -59,6 +59,7 @@ "contact_email", "fact_sheet", "forum_enabled", + "enable_human_in_the_loop" ] TASK_FIELDS = [ @@ -134,6 +135,7 @@ def _send_to_compute_worker(submission, is_scoring): ), "id": submission.pk, "is_scoring": is_scoring, + "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop, } if ( @@ -212,6 +214,9 @@ def _send_to_compute_worker(submission, is_scoring): time_padding = 60 * 20 # 20 minutes time_limit = submission.phase.execution_time_limit + time_padding + if is_scoring and submission.phase.competition.enable_human_in_the_loop: + time_limit = 60 * 60 * 25 + if ( submission.phase.competition.queue ): # if the competition is running on a custom queue, not the default queue diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag index 199edd404..da135fef0 100644 --- a/src/static/riot/competitions/editor/_competition_details.tag +++ b/src/static/riot/competitions/editor/_competition_details.tag @@ -211,6 +211,22 @@ + +
+ +
+ + +
+ + + + + +
+
@@ -314,6 +330,7 @@ self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked self.data["forum_enabled"] = self.refs.forum_enabled.checked + self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked self.data["make_programs_available"] = self.refs.make_programs_available.checked self.data["make_input_data_available"] = self.refs.make_input_data_available.checked self.data["docker_image"] = $(self.refs.docker_image).val() @@ -452,6 +469,7 @@ self.refs.auto_run_submissions.checked = competition.auto_run_submissions self.refs.can_participants_make_submissions_public.checked = competition.can_participants_make_submissions_public self.refs.forum_enabled.checked = competition.forum_enabled + self.refs.enable_human_in_the_loop.checked = competition.enable_human_in_the_loop self.refs.make_programs_available.checked = competition.make_programs_available self.refs.make_input_data_available.checked = competition.make_input_data_available $(self.refs.docker_image).val(competition.docker_image) From 8b79a61c204451538550029ed3a390aaed277ebb Mon Sep 17 00:00:00 2001 From: Idir Chikhoune Date: Mon, 29 Jun 2026 15:23:32 +0200 Subject: [PATCH 2/3] disable HITL for public CWs --- compute_worker/compute_worker.py | 4 ++-- src/apps/competitions/tasks.py | 11 ++++++++--- .../competitions/editor/_competition_details.tag | 12 +++++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 75a3540c3..1d1a7ea11 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -1482,10 +1482,10 @@ def wait_for_human_validation(self): elapsed = 0 while elapsed < max_wait: - if os.path.exists(approved_container): # ← poll le chemin conteneur + if os.path.exists(approved_container): logger.info(f"HITL: submission {self.submission_id} approved, sending scores.") return - if os.path.exists(rejected_container): # ← poll le chemin conteneur + if os.path.exists(rejected_container): raise SubmissionException( f"HITL: scores rejected by the compute node operator " f"(submission {self.submission_id})" diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 2e052e9ca..813075ebe 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -125,6 +125,11 @@ def _send_to_compute_worker(submission, is_scoring): + hitl_active = ( + submission.phase.competition.enable_human_in_the_loop + and submission.queue is not None + ) + run_args = { "user_pk": submission.owner.pk, "submissions_api_url": settings.SUBMISSIONS_API_URL, @@ -135,7 +140,7 @@ def _send_to_compute_worker(submission, is_scoring): ), "id": submission.pk, "is_scoring": is_scoring, - "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop, + "human_in_the_loop": hitl_active, } if ( @@ -211,10 +216,10 @@ def _send_to_compute_worker(submission, is_scoring): logger.info(run_args) # Pad timelimit so worker has time to cleanup - time_padding = 60 * 20 # 20 minutes + time_padding = 60 * 20 time_limit = submission.phase.execution_time_limit + time_padding - if is_scoring and submission.phase.competition.enable_human_in_the_loop: + if is_scoring and hitl_active: time_limit = 60 * 60 * 25 if ( diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag index da135fef0..25afa4099 100644 --- a/src/static/riot/competitions/editor/_competition_details.tag +++ b/src/static/riot/competitions/editor/_competition_details.tag @@ -219,12 +219,17 @@
- + @@ -331,6 +336,11 @@ self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked self.data["forum_enabled"] = self.refs.forum_enabled.checked self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked + const hitlChecked = self.refs.enable_human_in_the_loop.checked + const hasPrivateQueue = !!(self.data["queue"] && self.data["queue"] !== "") + if (self.refs.hitl_warning) { + self.refs.hitl_warning.style.display = (hitlChecked && !hasPrivateQueue) ? 'block' : 'none' + } self.data["make_programs_available"] = self.refs.make_programs_available.checked self.data["make_input_data_available"] = self.refs.make_input_data_available.checked self.data["docker_image"] = $(self.refs.docker_image).val() From a0fe2a7753b6ed7f5fd6a04a4e5ebb2d519901bf Mon Sep 17 00:00:00 2001 From: Idir Chikhoune Date: Tue, 30 Jun 2026 10:56:34 +0200 Subject: [PATCH 3/3] HITL secute on public CW + UI improvement --- src/apps/competitions/tasks.py | 17 +++++++++++++++++ .../competitions/detail/submission_manager.tag | 8 ++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 813075ebe..e2788deed 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -331,6 +331,23 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False): ) submission = qs.get(pk=submission_pk) + # ── HUMAN IN THE LOOP SECURE (private CW only) + if is_scoring and submission.phase.competition.enable_human_in_the_loop: + if submission.queue is None: + logger.error( + f"HITL: submission {submission_pk} rejected — " + f"Human in the Loop is enabled but no private queue is configured " + f"(competition or participant group)." + ) + submission.status = Submission.FAILED + submission.status_details = ( + "This competition requires Human in the Loop validation, but no " + "private compute queue is configured. Contact the organizer." + ) + submission.save(update_fields=["status", "status_details"]) + return + # ── HITL SECURE (private CW only) + if submission.is_specific_task_re_run: # Should only be one task for a specified task submission tasks = Task.objects.filter(pk__in=task_pks) diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 3f41a69eb..37530345c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -107,7 +107,8 @@ { submission.status } - + +