feat: Add a script to monitor the judges

Author: MatthieuNICOLAS

scripts/README.md

@@ -0,0 +1,31 @@
+# Scripts
+
+## monitor_judges
+
+Despite our efforts to stop and restart properly the judges after each execution request, in some cases the judges stay alive but as zombies.
+
+In this weird state, the judge behaves accordingly to one of the following:
+  - The judge seems to be idle, but is still connected to the message queue. Thus it still receives execution requests but does not handle them. Then all execution requests send to the judge result in timeouts.
+  - The judge seems to be still running the student's code despite the scheduled interruption (10s of execution). Thus it is still consuming ressources uselessly.
+
+This goal of this script is to monitor started containers and to restart containers identified as zombies.
+A judge is identified as a zombie if:
+  - It is alive for longer than 30s.
+  - It is not waiting for an execution request (thus it is processing one).
+  - It is still processing the same execution request 10s later (the PID of the java command is still the same).
+
+### Usage
+
+This script is compatible with Python >= 2.7
+
+First, you may need to install the dependency of this script:
+
+```
+pip install python-daemon
+```
+
+Then run the script with:
+
+```
+python monitor_judges.py
+```

scripts/monitor_judges.py

@@ -0,0 +1,71 @@
+import daemon
+import subprocess
+from datetime import date, datetime, time, timedelta
+from threading import Timer
+
+LOG_WAITING = "[INFO] Waiting for request...\n"
+MARKED_JUDGES = dict()
+
+def mark_judge(judge_name, pid):
+    print "Marking judge: {}".format(judge_name)
+    MARKED_JUDGES[judge_name] = pid
+
+def unmark_judge(judge_name):
+    print "Unmarking judge: {}".format(judge_name)
+    del MARKED_JUDGES[judge_name]
+
+def is_waiting(judge_name):
+    last_log = subprocess.check_output(["docker", "logs", "--tail=1", judge_name])
+    return LOG_WAITING == last_log
+
+def restart_judge(judge_name):
+    print "Restarting judge: {}".format(judge_name)
+    subprocess.check_output(["docker", "restart", judge_name])
+    unmark_judge(judge_name)
+
+def retrieve_start_time(judge_name):
+    started_at = subprocess.check_output(["docker", "inspect", "--format", "{{.State.StartedAt}}", judge_name])
+    started_at = started_at.split(".")[0]
+
+    started_date = started_at.split("T")[0].split("-")
+    started_time = started_at.split("T")[1].split(":")
+
+    year = int(started_date[0])
+    month = int(started_date[1])
+    day = int(started_date[2])
+
+    started_date = date(year, month, day)
+
+    hour = int(started_time[0]) + 2 # To use the set timezone as now()
+    minute = int(started_time[1])
+    second = int(started_time[2])
+
+    started_time = time(hour, minute, second)
+
+    return datetime.combine(started_date, started_time)
+
+def monitor_judge(judge_name):
+    started_at = retrieve_start_time(judge_name)
+    now = datetime.now()
+    if (now - started_at) > timedelta(seconds=30):
+        if not is_waiting(judge_name):
+            pid = int(subprocess.check_output(["docker top {} | tail -n 1".format(judge_name)], shell=True).split()[1])
+            if judge_name in MARKED_JUDGES and pid == MARKED_JUDGES[judge_name]:
+                restart_judge(judge_name)
+            else:
+                mark_judge(judge_name, pid)
+    else:
+        if judge_name in MARKED_JUDGES:
+            unmark_judge(judge_name)
+
+def main():
+    print "Marked judges: {}".format(MARKED_JUDGES)
+    nb_judges = int(subprocess.check_output(["docker ps | grep judge | wc -l"], shell=True))
+    for i in range(1, nb_judges + 1):
+        judge_name = "judge_judge_{}".format(i)
+        monitor_judge(judge_name)
+    thread = Timer(10.0, main)
+    thread.start()
+
+with daemon.DaemonContext():
+    main()