Fix running with time_limits and memory_limits present in config (#68)

* Fix time and memory limits

* Add package for testing

* Add tests

* Update package to work on macOS

* Fix tests for macOS

* Fix again

* Possible fix

* Add comments to `print_view`

* Add function for stringifying keys

* Bump version for release

Author: MasloMaslane

src/sinol_make/__init__.py

@@ -6,7 +6,7 @@
 
 from sinol_make import util
 
-__version__ = "1.4.1"
+__version__ = "1.4.2"
 
 def configure_parsers():
     parser = argparse.ArgumentParser(

src/sinol_make/commands/run/__init__.py

@@ -55,7 +55,7 @@ def update_group_status(group_status, new_status):
 
 
 def print_view(term_width, term_height, program_groups_scores, all_results, print_data: PrintData, names, executions,
-               groups, scores, tests, possible_score, time_limit, memory_limit, cpus, hide_memory):
+               groups, scores, tests, possible_score, cpus, hide_memory, config):
     width = term_width - 13  # First column has 6 characters, the " | " separator has 3 characters and 4 for margin
     programs_in_row = width // 13  # Each program has 10 characters and the " | " separator has 3 characters
 
@@ -64,9 +64,18 @@ def print_view(term_width, term_height, program_groups_scores, all_results, prin
     sys.stdout = output
 
     program_scores = collections.defaultdict(int)
-    program_times = collections.defaultdict(lambda: -1)
-    program_memory = collections.defaultdict(lambda: -1)
-    time_remaining = (len(executions) - print_data.i - 1) * 2 * time_limit / cpus / 1000.0
+    # program_times and program_memory are dictionaries of tuples (max, limit),
+    # where max is the maximum time/memory used by a program and
+    # limit is the time/memory limit of the test that caused the maximum
+    # time/memory usage.
+    program_times = collections.defaultdict(lambda: (-1, 0))
+    program_memory = collections.defaultdict(lambda: (-1, 0))
+
+    time_sum = 0
+    for test in tests:
+        time_sum += package_util.get_time_limit(test, config)
+
+    time_remaining = (len(executions) - print_data.i - 1) * 2 * time_sum / cpus / 1000.0
     title = 'Done %4d/%4d. Time remaining (in the worst case): %5d seconds.' \
             % (print_data.i + 1, len(executions), time_remaining)
     title = title.center(term_width)
@@ -104,15 +113,17 @@ def print_table_end():
                     min_points = min(min_points, results[test].Points)
                     status = results[test].Status
                     if getattr(results[test], "Time") is not None:
-                        program_times[program] = max(
-                            program_times[program], results[test].Time)
+                        if program_times[program][0] < results[test].Time:
+                            program_times[program] = (results[test].Time, package_util.get_time_limit(test, config))
                     elif status == "TL":
-                        program_times[program] = 2 * time_limit
+                        program_times[program] = (2 * package_util.get_time_limit(test, config),
+                                                  package_util.get_time_limit(test, config))
                     if getattr(results[test], "Memory") is not None:
-                        program_memory[program] = max(
-                            program_memory[program], results[test].Memory)
+                        if program_memory[program][0] < results[test].Memory:
+                            program_memory[program] = (results[test].Memory, package_util.get_memory_limit(test, config))
                     elif status == "ML":
-                        program_memory[program] = 2 * memory_limit
+                        program_memory[program] = (2 * package_util.get_memory_limit(test, config),
+                                                   package_util.get_memory_limit(test, config))
                     if status == "  ":
                         group_status = "  "
                         min_points = 0
@@ -138,15 +149,15 @@ def print_table_end():
         print(margin + "  time", end=" | ")
         for program in program_group:
             program_time = program_times[program]
-            print(util.bold(("%20s" % color_time(program_time, time_limit))
-                            if program_time < 2 * time_limit and program_time >= 0
+            print(util.bold(("%20s" % color_time(program_time[0], program_time[1]))
+                            if program_time[0] < 2 * program_time[1] and program_time[0] >= 0
                             else "   " + 7 * '-'), end=" | ")
         print()
         print(margin + "memory", end=" | ")
         for program in program_group:
             program_mem = program_memory[program]
-            print(util.bold(("%20s" % color_memory(program_mem, memory_limit))
-                            if program_mem < 2 * memory_limit and program_mem >= 0
+            print(util.bold(("%20s" % color_memory(program_mem[0], program_mem[1]))
+                            if program_mem[0] < 2 * program_mem[1] and program_mem[0] >= 0
                             else "   " + 7 * '-'), end=" | ")
         print()
         print(8*" ", end=" | ")
@@ -177,13 +188,15 @@ def print_group_seperator():
                 if status == "  ": print(10*' ', end=" | ")
                 else:
                     print("%3s" % colorize_status(status),
-                         ("%17s" % color_time(result.Time, time_limit)) if getattr(result, "Time") is not None else 7*" ", end=" | ")
+                         ("%17s" % color_time(result.Time, package_util.get_time_limit(test, config)))
+                         if getattr(result, "Time") is not None else 7*" ", end=" | ")
             print()
             if not hide_memory:
                 print(8*" ", end=" | ")
                 for program in program_group:
                     result = all_results[program][package_util.get_group(test)][test]
-                    print(("%20s" % color_memory(result.Memory, memory_limit)) if getattr(result, "Memory") is not None else 10*" ", end=" | ")
+                    print(("%20s" % color_memory(result.Memory, package_util.get_memory_limit(test, config)))
+                          if getattr(result, "Memory") is not None else 10*" ", end=" | ")
                 print()
 
         print_table_end()
@@ -580,7 +593,8 @@ def run_solutions(self, compiled_commands, names, solutions):
         for (name, executable, result) in compiled_commands:
             if result:
                 for test in self.tests:
-                    executions.append((name, executable, test, self.time_limit, self.memory_limit, self.timetool_path))
+                    executions.append((name, executable, test, package_util.get_time_limit(test, self.config),
+                                       package_util.get_memory_limit(test, self.config), self.timetool_path))
                     all_results[name][self.get_group(test)][test] = ExecutionResult("  ")
                 os.makedirs(os.path.join(self.EXECUTIONS_DIR, name), exist_ok=True)
             else:
@@ -599,7 +613,7 @@ def run_solutions(self, compiled_commands, names, solutions):
             thr = threading.Thread(target=printer.printer_thread,
                                    args=(run_event, print_view, program_groups_scores, all_results, print_data, names,
                                          executions, self.groups, self.scores, self.tests, self.possible_score,
-                                         self.time_limit, self.memory_limit, self.cpus, self.args.hide_memory))
+                                         self.cpus, self.args.hide_memory, self.config))
             thr.start()
 
         pool = mp.Pool(self.cpus)
@@ -620,7 +634,7 @@ def run_solutions(self, compiled_commands, names, solutions):
 
         print("\n".join(print_view(terminal_width, terminal_height, program_groups_scores, all_results, print_data,
                                    names, executions, self.groups, self.scores, self.tests, self.possible_score,
-                                   self.time_limit, self.memory_limit, self.cpus, self.args.hide_memory)[0]))
+                                   self.cpus, self.args.hide_memory, self.config)[0]))
 
         if keyboard_interrupt:
             util.exit_with_error("Stopped due to keyboard interrupt.")

src/sinol_make/helpers/package_util.py

@@ -1,6 +1,8 @@
 import os
 from typing import List, Union
 
+from sinol_make import util
+
 
 def get_task_id() -> str:
     return os.path.split(os.getcwd())[-1]
@@ -55,3 +57,35 @@ def get_executable_path(solution: str) -> str:
     Returns path to compiled executable for given solution.
     """
     return os.path.join(os.getcwd(), 'cache', 'executables', get_executable(solution))
+
+
+def get_time_limit(test_path, config):
+    """
+    Returns time limit for given test.
+    """
+    str_config = util.stringify_keys(config)
+    test_id = extract_test_id(test_path)
+    test_group = str(get_group(test_path))
+
+    if "time_limits" in str_config:
+        if test_id in str_config["time_limits"]:
+            return str_config["time_limits"][test_id]
+        elif test_group in str_config["time_limits"]:
+            return str_config["time_limits"][test_group]
+    return str_config["time_limit"]
+
+
+def get_memory_limit(test_path, config):
+    """
+    Returns memory limit for given test.
+    """
+    str_config = util.stringify_keys(config)
+    test_id = extract_test_id(test_path)
+    test_group = str(get_group(test_path))
+
+    if "memory_limits" in str_config:
+        if test_id in str_config["memory_limits"]:
+            return str_config["memory_limits"][test_id]
+        elif test_group in str_config["memory_limits"]:
+            return str_config["memory_limits"][test_group]
+    return str_config["memory_limit"]

src/sinol_make/util.py

@@ -304,6 +304,18 @@ def fix_file_endings(file):
         f.write(content.replace(b"\r\n", b"\n"))
 
 
+def stringify_keys(d):
+    """
+    Function to stringify all keys in a dict.
+    """
+    if isinstance(d, dict):
+        return {str(k): stringify_keys(v) for k, v in d.items()}
+    elif isinstance(d, list):
+        return [stringify_keys(x) for x in d]
+    else:
+        return d
+
+
 def color_red(text): return "\033[91m{}\033[00m".format(text)
 def color_green(text): return "\033[92m{}\033[00m".format(text)
 def color_yellow(text): return "\033[93m{}\033[00m".format(text)

tests/commands/run/test_integration.py

@@ -4,7 +4,8 @@
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path(), get_library_package_path()], indirect=True)
+                                            get_checker_package_path(), get_library_package_path(),
+                                            get_limits_package_path()], indirect=True)
 def test_simple(create_package, time_tool):
     """
     Test a simple run.
@@ -20,7 +21,8 @@ def test_simple(create_package, time_tool):
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path(), get_library_package_path()], indirect=True)
+                                            get_checker_package_path(), get_library_package_path(),
+                                            get_limits_package_path()], indirect=True)
 def test_no_expected_scores(capsys, create_package, time_tool):
     """
     Test with no sinol_expected_scores in config.yml.
@@ -53,7 +55,8 @@ def test_no_expected_scores(capsys, create_package, time_tool):
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path(), get_library_package_path()], indirect=True)
+                                            get_checker_package_path(), get_library_package_path(),
+                                            get_limits_package_path()], indirect=True)
 def test_apply_suggestions(create_package, time_tool):
     """
     Test with no sinol_expected_scores in config.yml.
@@ -225,3 +228,30 @@ def test_missing_output_files(capsys, create_package):
 
     out = capsys.readouterr().out
     assert f'Missing output files for tests: {out1}, {out2}' in out
+
+
+@pytest.mark.parametrize("create_package", [get_limits_package_path()], indirect=True)
+def test_no_limits_in_config(capsys, create_package, time_tool):
+    """
+    Test with missing `time_limits` and `memory_limits` keys in config.yml.
+    """
+    package_path = create_package
+    command = get_command()
+    create_ins_outs(package_path)
+
+    config_path = os.path.join(package_path, "config.yml")
+    with open(config_path, "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
+    del config["time_limits"]
+    del config["memory_limits"]
+    with open(config_path, "w") as config_file:
+        config_file.write(yaml.dump(config))
+
+    parser = configure_parsers()
+    args = parser.parse_args(["run", "--time-tool", time_tool])
+    command = Command()
+    with pytest.raises(SystemExit):
+        command.run(args)
+
+    out = capsys.readouterr().out
+    assert "Use flag --apply-suggestions to apply suggestions." in out

tests/helpers/test_package_util.py

@@ -31,3 +31,39 @@ def test_extract_file_name():
 
 def test_get_executable():
     assert package_util.get_executable("abc.cpp") == "abc.e"
+
+
+def test_get_time_limit():
+    config = {
+        "time_limit": 1000,
+        "time_limits": {
+            "2": 2000,
+            "2a": 3000,
+            "3ocen": 5000
+        }
+    }
+
+    assert package_util.get_time_limit("in/abc1a.in", config) == 1000
+    assert package_util.get_time_limit("in/abc2a.in", config) == 3000
+    assert package_util.get_time_limit("in/abc2b.in", config) == 2000
+    assert package_util.get_time_limit("in/abc3a.in", config) == 1000
+    assert package_util.get_time_limit("in/abc3ocen.in", config) == 5000
+
+
+def test_get_memory_limit():
+    config = {
+        "memory_limit": 256,
+        "memory_limits": {
+            "2": 512,
+            "2c": 1024,
+            "3ocen": 2048,
+            "3": 128
+        }
+    }
+
+    assert package_util.get_memory_limit("in/abc1a.in", config) == 256
+    assert package_util.get_memory_limit("in/abc2a.in", config) == 512
+    assert package_util.get_memory_limit("in/abc2b.in", config) == 512
+    assert package_util.get_memory_limit("in/abc2c.in", config) == 1024
+    assert package_util.get_memory_limit("in/abc3a.in", config) == 128
+    assert package_util.get_memory_limit("in/abc3ocen.in", config) == 2048

tests/packages/lim/config.yml

@@ -0,0 +1,32 @@
+title: Package with `time_limits` and `memory_limits` set
+
+memory_limit: 30000
+memory_limits:
+  1a: 15000
+  2: 60000
+
+time_limit: 1000
+time_limits:
+  1a: 500
+  2: 2000
+
+scores:
+  1: 50
+  2: 50
+
+sinol_expected_scores:
+  lim.cpp:
+    expected: {1: OK, 2: OK}
+    points: 100
+  lim1.cpp:
+    expected: {1: TL, 2: OK}
+    points: 50
+  lim2.cpp:
+    expected: {1: TL, 2: TL}
+    points: 0
+  lim3.cpp:
+    expected: {1: ML, 2: OK}
+    points: 50
+  lim4.cpp:
+    expected: {1: ML, 2: ML}
+    points: 0

tests/packages/lim/in/.gitkeep

@@ -0,0 +1,9 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+    cout << a + b;
+}

tests/packages/lim/out/.gitkeep

@@ -0,0 +1,15 @@
+#include <bits/stdc++.h>
+#include <chrono>
+
+using namespace std;
+using namespace std::chrono_literals;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+
+    vector<int> v(5, 0);
+    this_thread::sleep_for(1.1s);
+
+    cout << a + b << endl;
+}

tests/packages/lim/prog/lim.cpp

@@ -0,0 +1,19 @@
+#include <bits/stdc++.h>
+#include <chrono>
+
+using namespace std;
+using namespace std::chrono_literals;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+
+    if (a == 2 && b == 1) {
+        this_thread::sleep_for(6s);
+    }
+    else {
+        this_thread::sleep_for(1.1s);
+    }
+
+    cout << a + b << endl;
+}

tests/packages/lim/prog/lim1.cpp

@@ -0,0 +1,16 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int a, b;
+    cin >> a >> b;
+
+    vector<long long> mem;
+    if (a == 1) {
+        for (int i = 1; i <= 1280000; i++) {
+            mem.push_back(i);
+        }
+    }
+    cout << a + b;
+}