Add support for package with checker in run (#40)

* Add support for checker in `run`

* Add simple package for checker

* Fix tests and refactor

* Add checker package to tests

* Fix tests

* Apply suggestions from code review

Co-authored-by: Tomasz Nowak <[email protected]>

* Add file diff function

* Add tests for file_diff function

* Fix indentation

* Redirect checker's stderr to /dev/null

* Assume that correct answer files are generated when using checker

* Fix tests

* Fixes errors

* Change point calculation

* Change how sinol_expected_scores looks

* Fix checker package config

* Apply suggestions from code review

Co-authored-by: Tomasz Nowak <[email protected]>

* Refactor changes list creation

* Refactor point's change print message

* Pretty error printing from checker

* Version bump

---------

Co-authored-by: Tomasz Nowak <[email protected]>

Author: MasloMaslane

src/sinol_make/__init__.py

@@ -3,7 +3,7 @@
 
 from sinol_make import util
 
-__version__ = "1.2.2"
+__version__ = "1.3.0"
 
 def configure_parsers():
     parser = argparse.ArgumentParser(

src/sinol_make/commands/run/__init__.py

@@ -8,6 +8,13 @@ class ResultChange:
     old_result: str
     result: str
 
+@dataclass
+class PointsChange:
+    solution: str
+    group: int
+    old_points: int
+    new_points: int
+
 @dataclass
 class ValidationResult:
     added_solutions: set
@@ -27,6 +34,17 @@ class ExecutionResult:
     Time: float
     # Memory in KB
     Memory: int
+    # Points for this test
+    Points: int
+    # Error message
+    Error: str
+
+    def __init__(self, status=None):
+        self.Status = status
+        self.Time = None
+        self.Memory = None
+        self.Points = 0
+        self.Error = None
 
 @dataclass
 class ExecutionData:

src/sinol_make/commands/run/structs.py

@@ -4,3 +4,10 @@ def __init__(self, message):
 
     def __str__(self):
         return self.message
+
+class CheckerOutputException(Exception):
+    def __init__(self, message):
+        self.message = message
+
+    def __str__(self):
+        return self.message

src/sinol_make/interfaces/Errors.py

@@ -1,17 +1,15 @@
-import yaml, pytest
-from ...util import *
 from ...fixtures import *
 from .util import *
 from sinol_make import configure_parsers
 
 
-@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path()], indirect=True)
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
+                                            get_checker_package_path()], indirect=True)
 def test_simple(create_package, time_tool):
     """
     Test a simple run.
     """
     package_path = create_package
-    command = get_command()
     create_ins_outs(package_path)
 
     parser = configure_parsers()
@@ -21,15 +19,15 @@ def test_simple(create_package, time_tool):
     command.run(args)
 
 
-@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path()], indirect=True)
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
+                                            get_checker_package_path()], indirect=True)
 def test_no_expected_scores(capsys, create_package, time_tool):
     """
     Test with no sinol_expected_scores in config.yml.
     Should run, but exit with exit code 1.
     Checks if a message about added solutions is printed.
     """
     package_path = create_package
-    command = get_command()
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
@@ -48,17 +46,19 @@ def test_no_expected_scores(capsys, create_package, time_tool):
 
     out = capsys.readouterr().out
     assert "Solutions were added:" in out
+    solution = glob.glob(os.path.join(package_path, "prog", "???.*"))[0]
+    assert os.path.basename(solution) in out
 
 
-@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path()], indirect=True)
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
+                                            get_checker_package_path()], indirect=True)
 def test_apply_suggestions(create_package, time_tool):
     """
     Test with no sinol_expected_scores in config.yml.
     Verifies that suggestions are applied.
     Checks if the genereated config.yml is correct.
     """
     package_path = create_package
-    command = get_command()
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
@@ -83,7 +83,6 @@ def test_incorrect_expected_scores(capsys, create_package, time_tool):
     Checks if a message about incorrect result is printed.
     """
     package_path = create_package
-    command = get_command()
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
@@ -106,42 +105,47 @@ def test_incorrect_expected_scores(capsys, create_package, time_tool):
     assert "Solution abc.cpp passed group 1 with status OK while it should pass with status WA." in out
 
 
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_checker_package_path()], indirect=True)
 def test_flag_tests(create_package, time_tool):
     """
     Test flag --tests.
     Checks if correct tests are run.
     """
     package_path = create_package
-    command = get_command()
     create_ins_outs(package_path)
 
+    test = glob.glob(os.path.join(package_path, "in", "???*.in"))[0]
     parser = configure_parsers()
-    args = parser.parse_args(["run", "--tests", "in/abc1a.in", "--time_tool", time_tool])
+    args = parser.parse_args(["run", "--tests", test, "--time_tool", time_tool])
     command = Command()
-    command.run(args)
+    try:
+        command.run(args)
+    except SystemExit:
+        pass
 
-    assert command.tests == ["in/abc1a.in"]
+    assert command.tests == [test]
 
 
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
+                                            get_checker_package_path()], indirect=True)
 def test_flag_solutions(capsys, create_package, time_tool):
     """
     Test flag --solutions.
     Checks if correct solutions are run (by checking the output).
     """
     package_path = create_package
-    command = get_command()
     create_ins_outs(package_path)
 
+    solutions = glob.glob(os.path.join(package_path, "prog", "????.*"))
     parser = configure_parsers()
-    args = parser.parse_args(["run", "--solutions", "prog/abc1.cpp", "prog/abc2.cpp", "--time_tool", time_tool])
+    args = parser.parse_args(["run", "--solutions", solutions[0], "--time_tool", time_tool])
     command = Command()
     command.run(args)
 
     out = capsys.readouterr().out
 
-    assert "abc1.cpp" in out
-    assert "abc2.cpp" in out
-    assert "abc3.cpp" not in out
+    assert os.path.basename(solutions[0]) in out
+    assert os.path.basename(solutions[1]) not in out
 
 
 @pytest.mark.parametrize("create_package", [get_weak_compilation_flags_package_path()], indirect=True)

tests/commands/run/test_integration.py

@@ -82,9 +82,15 @@ def test_run_solutions(create_package, time_tool):
     command.time_limit = command.config["time_limit"]
     command.timetool_path = util.get_oiejq_path()
 
-    print(command.compile_and_run(["abc.cpp"]))
-    assert command.compile_and_run(["abc.cpp"]) == {"abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"}}
-    assert command.compile_and_run(["abc.cpp", "abc4.cpp"]) == {
+    def flatten_results(results):
+        new_results = {}
+        for solution in results.keys():
+            new_results[solution] = dict((group, group_result["status"])
+                                         for group, group_result in results[solution].items())
+        return new_results
+
+    assert flatten_results(command.compile_and_run(["abc.cpp"])[0]) == {"abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"}}
+    assert flatten_results(command.compile_and_run(["abc.cpp", "abc4.cpp"])[0]) == {
         "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
         "abc4.cpp": {1: "OK", 2: "OK", 3: "WA", 4: "RE"}
     }
@@ -126,7 +132,7 @@ def test_validate_expected_scores_success():
     # Test with correct expected scores.
     command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
     results = {
-        "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
+        "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "OK", "points": 25}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores == results.new_expected_scores
@@ -135,7 +141,7 @@ def test_validate_expected_scores_success():
     # Test with incorrect result.
     command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
     results = {
-        "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "WA"},
+        "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "WA", "points": 0}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores != results.new_expected_scores
@@ -144,10 +150,10 @@ def test_validate_expected_scores_success():
     # Test with removed solution.
     command.args = argparse.Namespace(solutions=None, tests=None)
     results = {
-        "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK"},
-        "abc1.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "WA"},
-        "abc2.cpp": {1: "OK", 2: "WA", 3: "WA", 4: "TL"},
-        "abc3.cpp": {1: "OK", 2: "WA", 3: "WA", 4: "ML"},
+        "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "OK", "points": 25}},
+        "abc1.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}, 4: {"status": "WA", "points": 0}},
+        "abc2.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "WA", "points": 0}, 3: {"status": "WA", "points": 0}, 4: {"status": "TL", "points": 0}},
+        "abc3.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "WA", "points": 0}, 3: {"status": "WA", "points": 0}, 4: {"status": "ML", "points": 0}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores != results.new_expected_scores
@@ -157,8 +163,8 @@ def test_validate_expected_scores_success():
     command.config["scores"][5] = 0
     command.args = argparse.Namespace(solutions=["prog/abc.cpp", "prog/abc5.cpp"], tests=None)
     results = {
-        "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK", 5: "WA"},
-        "abc5.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK", 5: "WA"},
+        "abc.cpp": {1: {"status": "OK", "points": 20}, 2: {"status": "OK", "points": 20}, 3: {"status": "OK", "points": 20}, 4: {"status": "OK", "points": 20}, 5: {"status": "WA", "points": 0}},
+        "abc5.cpp": {1: {"status": "OK", "points": 20}, 2: {"status": "OK", "points": 20}, 3: {"status": "OK", "points": 20}, 4: {"status": "OK", "points": 20}, 5: {"status": "WA", "points": 0}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores != results.new_expected_scores
@@ -168,7 +174,7 @@ def test_validate_expected_scores_success():
     # Test with removed group.
     command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
     results = {
-        "abc.cpp": {1: "OK", 2: "OK", 3: "OK"},
+        "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}, 3: {"status": "OK", "points": 25}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores != results.new_expected_scores
@@ -177,7 +183,7 @@ def test_validate_expected_scores_success():
     # Test with correct expected scores and --tests flag.
     command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=["in/abc1a.in", "in/abc2a.in"])
     results = {
-        "abc.cpp": {1: "OK", 2: "OK"},
+        "abc.cpp": {1: {"status": "OK", "points": 25}, 2: {"status": "OK", "points": 25}},
     }
     results = command.validate_expected_scores(results)
     assert results.expected_scores == results.new_expected_scores
@@ -186,12 +192,12 @@ def test_validate_expected_scores_success():
 def test_validate_expected_scores_fail(capsys):
     command = get_command()
     os.chdir(get_simple_package_path())
-    command.scores = command.config["scores"]
+    command.scores = {1: 20, 2: 20, 3: 20, 4: 20}
 
     # Test with missing points for group in config.
     command.args = argparse.Namespace(solutions=["prog/abc.cpp"], tests=None)
     results = {
-        "abc.cpp": {1: "OK", 2: "OK", 3: "OK", 4: "OK", 5: "OK"},
+        "abc.cpp": {1: {"status": "OK", "points": 20}, 2: {"status": "OK", "points": 20}, 3: {"status": "OK", "points": 20}, 4: {"status": "OK", "points": 20}, 5: {"status": "OK", "points": 20}},
     }
     with pytest.raises(SystemExit) as e:
         command.validate_expected_scores(results)

tests/commands/run/test_unit.py

@@ -7,6 +7,7 @@
 from ...util import *
 from sinol_make.commands.run import Command
 from sinol_make.helpers import compiler
+from sinol_make.helpers import compile
 
 
 def get_command(path = None):
@@ -25,6 +26,7 @@ def get_command(path = None):
         java_compiler_path=compiler.get_java_compiler_path()
     )
     command.config = yaml.load(open(os.path.join(path, "config.yml"), "r"), Loader=yaml.FullLoader)
+    command.checker = None
     command.failed_compilations = []
     set_default_args(command)
     return command

tests/commands/run/util.py

@@ -0,0 +1,21 @@
+title: Package with output checker
+memory_limit: 16000
+time_limit: 1000
+scores:
+  1: 50
+  2: 50
+sinol_expected_scores:
+  chk.cpp:
+    expected: {1: OK, 2: OK}
+    points: 100
+  chk1.cpp:
+    expected: {1: WA, 2: WA}
+    points: 0
+  chk2.cpp:
+    expected:
+      1: {points: 25, status: OK}
+      2: {points: 25, status: OK}
+    points: 50
+  chk3.cpp:
+    expected: {1: OK, 2: WA}
+    points: 50

tests/packages/chk/config.yml

@@ -0,0 +1 @@
+6

tests/packages/chk/in/.gitkeep

@@ -0,0 +1 @@
+-1

tests/packages/chk/out/.gitkeep

@@ -0,0 +1 @@
+6

tests/packages/chk/out/chk1a.out

@@ -0,0 +1 @@
+5

tests/packages/chk/out/chk1b.out

@@ -0,0 +1 @@
+2

tests/packages/chk/out/chk1c.out

@@ -0,0 +1 @@
+-1

tests/packages/chk/out/chk2a.out

@@ -0,0 +1,19 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int n, s = 0;
+
+    cin >> n;
+    for (int i = 0; i < n; i++) {
+        int a;
+        cin >> a;
+        s += a;
+    }
+
+    if (s % n != 0)
+        cout << -1 << endl;
+    else
+        cout << s / 2 - 1 << endl;
+}

tests/packages/chk/out/chk2b.out

@@ -0,0 +1,16 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int n, s = 0;
+    cin >> n;
+
+    for (int i = 0; i < n; i++) {
+        int a;
+        cin >> a;
+        s += a;
+    }
+
+    cout << s + 1 << endl;
+}

tests/packages/chk/out/chk2c.out

@@ -0,0 +1,21 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int n, s = 0;
+
+    cin >> n;
+    for (int i = 0; i < n; i++) {
+        int a;
+        cin >> a;
+        s += a;
+    }
+
+    if (s % n != 0) {
+        cout << "-1";
+        return 0;
+    }
+
+    cout << s / 2 + 1;
+}