Add support for packages with library (#64)

* Add support for packages with library

* Add test package with library

* Add package with library to tests

* Change to use `with` statements everywhere

* Apply suggestions from code review

Co-authored-by: Tomasz Nowak <[email protected]>

* Use `with statements in oiejq installation`

* Fix oiejq installation

* Add comment describing oiejq installation

* Add short flag for --weak-compilation-flags

* Bump version for release

---------

Co-authored-by: Tomasz Nowak <[email protected]>

Author: MasloMaslane

src/sinol_make/__init__.py

@@ -6,7 +6,7 @@
 
 from sinol_make import util
 
-__version__ = "1.4.0"
+__version__ = "1.4.1"
 
 def configure_parsers():
     parser = argparse.ArgumentParser(

src/sinol_make/commands/run/__init__.py

@@ -315,20 +315,33 @@ def compile_solutions(self, solutions):
         os.makedirs(self.COMPILATION_DIR, exist_ok=True)
         os.makedirs(self.EXECUTABLES_DIR, exist_ok=True)
         print("Compiling %d solutions..." % len(solutions))
+        args = [(solution, True) for solution in solutions]
         with mp.Pool(self.cpus) as pool:
-            compilation_results = pool.map(self.compile, solutions)
+            compilation_results = pool.starmap(self.compile, args)
         return compilation_results
 
 
-    def compile(self, solution):
+    def compile(self, solution, use_extras = False):
         compile_log_file = os.path.join(
             self.COMPILATION_DIR, "%s.compile_log" % package_util.get_file_name(solution))
         source_file = os.path.join(os.getcwd(), "prog", self.get_solution_from_exe(solution))
         output = os.path.join(self.EXECUTABLES_DIR, package_util.get_executable(solution))
 
+        extra_compilation_args = []
+        extra_compilation_files = []
+        if use_extras:
+            if "extra_compilation_args" in self.config:
+                lang = os.path.splitext(source_file)[1][1:]
+                for file in self.config["extra_compilation_args"].get(lang, []):
+                    extra_compilation_args.append(os.path.join(os.getcwd(), "prog", file))
+
+            for file in self.config.get("extra_compilation_files", []):
+                extra_compilation_files.append(os.path.join(os.getcwd(), "prog", file))
+
         try:
-            compile.compile(source_file, output, self.compilers,
-                            open(compile_log_file, "w"), self.args.weak_compilation_flags)
+            with open(compile_log_file, "w") as compile_log:
+                compile.compile(source_file, output, self.compilers, compile_log, self.args.weak_compilation_flags,
+                                extra_compilation_args, extra_compilation_files)
             print(util.info("Compilation of file %s was successful."
                             % package_util.get_file_name(solution)))
             return True
@@ -376,10 +389,12 @@ def check_output(self, name, input_file, output_file_path, output, answer_file_p
         Returns a tuple (is correct, number of points).
         """
         if not hasattr(self, "checker") or self.checker is None:
-            correct = util.lines_diff(output, open(answer_file_path, "r").readlines())
+            with open(answer_file_path, "r") as answer_file:
+                correct = util.lines_diff(output, answer_file.readlines())
             return correct, 100 if correct else 0
         else:
-            open(output_file_path, "w").write("\n".join(output))
+            with open(output_file_path, "w") as output_file:
+                output_file.write("\n".join(output))
             return self.check_output_checker(name, input_file, output_file_path, answer_file_path)
 
 
@@ -479,7 +494,8 @@ def sigint_handler(signum, frame):
         program_exit_code = None
         if not timeout:
             output = output.decode("utf-8").splitlines()
-            lines = open(result_file_path).readlines()
+            with open(result_file_path, "r") as result_file:
+                lines = result_file.readlines()
             if len(lines) == 3:
                 """
                 If programs runs successfully, the output looks like this:
@@ -967,7 +983,8 @@ def check_are_any_tests_to_run(self):
             if len(example_tests) == len(self.tests):
                 print(util.warning('Running only on example tests.'))
 
-            self.validate_existence_of_outputs()
+            if not self.has_lib:
+                self.validate_existence_of_outputs()
         else:
             print(util.warning('There are no tests to run.'))
 
@@ -988,10 +1005,11 @@ def run(self, args):
 
         self.set_constants()
         self.args = args
-        try:
-            self.config = yaml.load(open("config.yml"), Loader=yaml.FullLoader)
-        except AttributeError:
-            self.config = yaml.load(open("config.yml"))
+        with open(os.path.join(os.getcwd(), "config.yml"), 'r') as config:
+            try:
+                self.config = yaml.load(config, Loader=yaml.FullLoader)
+            except AttributeError:
+                self.config = yaml.load(config)
 
         if not 'title' in self.config.keys():
             util.exit_with_error('Title was not defined in config.yml.')
@@ -1033,6 +1051,9 @@ def run(self, args):
         else:
             self.checker = None
 
+        lib = glob.glob(os.path.join(os.getcwd(), "prog", f'{self.ID}lib.*'))
+        self.has_lib = len(lib) != 0
+
         self.set_scores()
         self.check_are_any_tests_to_run()
 

src/sinol_make/helpers/compile.py

@@ -1,6 +1,7 @@
 from typing import Tuple
 import os
 import sys
+import shutil
 import stat
 import subprocess
 
@@ -9,33 +10,52 @@
 from sinol_make.structs.compiler_structs import Compilers
 
 
-def compile(program, output, compilers: Compilers = None, compile_log = None, weak_compilation_flags = False):
+def compile(program, output, compilers: Compilers = None, compile_log = None, weak_compilation_flags = False,
+            extra_compilation_args = None, extra_compilation_files = None):
     """
-    Compile a program
-    compilers - A Compilers object with compilers to use. If None, default compilers will be used.
+    Compile a program.
+    :param program: Path to the program to compile
+    :param output: Path to the output file
+    :param compilers: Compilers object
+    :param compile_log: File to write the compilation log to
+    :param weak_compilation_flags: If True, disable all warnings
+    :param extra_compilation_args: Extra compilation arguments
+    :param extra_compilation_files: Extra compilation files
     """
+    if extra_compilation_args is None:
+        extra_compilation_args = []
+    if extra_compilation_files is None:
+        extra_compilation_files = []
+
+    for file in extra_compilation_files:
+        shutil.copy(file, os.path.join(os.path.dirname(output), os.path.basename(file)))
+
     gcc_compilation_flags = '-Werror -Wall -Wextra -Wshadow -Wconversion -Wno-unused-result -Wfloat-equal'
     if weak_compilation_flags:
-        gcc_compilation_flags = '-w' # Disable all warnings
+        gcc_compilation_flags = '-w'  # Disable all warnings
 
     if compilers is None:
         compilers = Compilers()
 
     ext = os.path.splitext(program)[1]
     arguments = []
     if ext == '.cpp':
-        arguments = [compilers.cpp_compiler_path or compiler.get_cpp_compiler_path(), program, '-o', output] + \
+        arguments = [compilers.cpp_compiler_path or compiler.get_cpp_compiler_path(), program] + \
+                    extra_compilation_args + ['-o', output] + \
                     f'--std=c++17 -O3 -lm {gcc_compilation_flags} -fdiagnostics-color'.split(' ')
     elif ext == '.c':
-        arguments = [compilers.c_compiler_path, program, '-o', output] + \
+        arguments = [compilers.c_compiler_path or compiler.get_c_compiler_path(), program] + \
+                    extra_compilation_args + ['-o', output] + \
                     f'--std=c17 -O3 -lm {gcc_compilation_flags} -fdiagnostics-color'.split(' ')
     elif ext == '.py':
         if sys.platform == 'win32' or sys.platform == 'cygwin':
             # TODO: Make this work on Windows
             pass
         else:
-            open(output, 'w').write('#!/usr/bin/python3\n')
-            open(output, 'a').write(open(program, 'r').read())
+            with open(output, 'w') as output_file, open(program, 'r') as program_file:
+                output_file.write('#!/usr/bin/python3\n')
+                output_file.write(program_file.read())
+
             st = os.stat(output)
             os.chmod(output, st.st_mode | stat.S_IEXEC)
         arguments = [compilers.python_interpreter_path, '-m', 'py_compile', program]
@@ -76,14 +96,12 @@ def compile_file(file_path: str, name: str, compilers: Compilers, weak_compilati
 
     output = os.path.join(executable_dir, name)
     compile_log_path = os.path.join(compile_log_dir, os.path.splitext(name)[0] + '.compile_log')
-    compile_log = open(compile_log_path, 'w')
-
-    try:
-        if compile(file_path, output, compilers, compile_log, weak_compilation_flags):
-            return output, compile_log_path
-        else:
-            return None, compile_log_path
-    except CompilationError:
+    with open(compile_log_path, 'w') as compile_log:
+        try:
+            if compile(file_path, output, compilers, compile_log, weak_compilation_flags):
+                return output, compile_log_path
+        except CompilationError:
+            pass
         return None, compile_log_path
 
 
@@ -93,8 +111,7 @@ def print_compile_log(compile_log_path: str):
     :param compile_log_path: path to the compilation log
     """
 
-    compile_log = open(compile_log_path, 'r')
-    lines = compile_log.readlines()
-    compile_log.close()
-    for line in lines[:500]:
-        print(line, end='')
+    with open(compile_log_path, 'r') as compile_log:
+        lines = compile_log.readlines()
+        for line in lines[:500]:
+            print(line, end='')

src/sinol_make/helpers/parsers.py

@@ -21,5 +21,5 @@ def add_compilation_arguments(parser: argparse.ArgumentParser):
                         help='Python interpreter to use (default: python3)')
     parser.add_argument('--java-compiler-path', dest='java_compiler_path', type=str, default=compiler.get_java_compiler_path(),
                         help='Java compiler to use (default: javac)')
-    parser.add_argument('--weak-compilation-flags', dest='weak_compilation_flags', action='store_true',
+    parser.add_argument('-W', '--weak-compilation-flags', dest='weak_compilation_flags', action='store_true',
                         help='use weaker compilation flags')

src/sinol_make/util.py

@@ -1,4 +1,5 @@
 import glob, importlib, os, sys, subprocess, requests, tarfile, yaml
+import tempfile
 import importlib.resources
 import threading
 
@@ -79,19 +80,25 @@ def install_oiejq():
         raise Exception('Couldn\'t download oiejq (https://oij.edu.pl/zawodnik/srodowisko/oiejq.tar.gz couldn\'t connect)')
     if request.status_code != 200:
         raise Exception('Couldn\'t download oiejq (https://oij.edu.pl/zawodnik/srodowisko/oiejq.tar.gz returned status code: ' + str(request.status_code) + ')')
-    open('/tmp/oiejq.tar.gz', 'wb').write(request.content)
 
-    def strip(tar):
-        l = len('oiejq/')
-        for member in tar.getmembers():
-            member.name = member.name[l:]
-            yield member
-
-    tar = tarfile.open('/tmp/oiejq.tar.gz')
-    tar.extractall(path=os.path.expanduser('~/.local/bin'), members=strip(tar))
-    tar.close()
-    os.remove('/tmp/oiejq.tar.gz')
-    os.rename(os.path.expanduser('~/.local/bin/oiejq.sh'), os.path.expanduser('~/.local/bin/oiejq'))
+    # oiejq is downloaded to a temporary directory and not to the `cache` dir,
+    # as there is no guarantee that the current directory is the package directory.
+    # The `cache` dir is only used for files that are part of the package and those
+    # that the package creator might want to look into.
+    with tempfile.TemporaryDirectory() as tmpdir:
+        oiejq_path = os.path.join(tmpdir, 'oiejq.tar.gz')
+        with open(oiejq_path, 'wb') as oiejq_file:
+            oiejq_file.write(request.content)
+
+        def strip(tar):
+            l = len('oiejq/')
+            for member in tar.getmembers():
+                member.name = member.name[l:]
+                yield member
+
+        with tarfile.open(oiejq_path) as tar:
+            tar.extractall(path=os.path.expanduser('~/.local/bin'), members=strip(tar))
+        os.rename(os.path.expanduser('~/.local/bin/oiejq.sh'), os.path.expanduser('~/.local/bin/oiejq'))
 
     return check_oiejq()
 
@@ -131,7 +138,14 @@ def save_config(config):
         "time_limits",
         "override_limits",
         "scores",
-        "extra_compilation_files",
+        {
+            "key": "extra_compilation_files",
+            "default_flow_style": None
+        },
+        {
+            "key": "extra_compilation_args",
+            "default_flow_style": None
+        },
         {
             "key": "sinol_expected_scores",
             "default_flow_style": None
@@ -240,12 +254,13 @@ def lines_diff(lines1, lines2):
     return True
 
 
-def file_diff(file1, file2):
+def file_diff(file1_path, file2_path):
     """
     Function to compare two files.
     Returns True if they are the same, False otherwise.
     """
-    return lines_diff(open(file1).readlines(), open(file2).readlines())
+    with open(file1_path) as file1, open(file2_path) as file2:
+        return lines_diff(file1.readlines(), file2.readlines())
 
 
 def get_terminal_size():

tests/commands/run/test_integration.py

@@ -4,7 +4,7 @@
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path()], indirect=True)
+                                            get_checker_package_path(), get_library_package_path()], indirect=True)
 def test_simple(create_package, time_tool):
     """
     Test a simple run.
@@ -20,7 +20,7 @@ def test_simple(create_package, time_tool):
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path()], indirect=True)
+                                            get_checker_package_path(), get_library_package_path()], indirect=True)
 def test_no_expected_scores(capsys, create_package, time_tool):
     """
     Test with no sinol_expected_scores in config.yml.
@@ -31,9 +31,11 @@ def test_no_expected_scores(capsys, create_package, time_tool):
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
-    config = yaml.load(open(config_path, "r"), Loader=yaml.SafeLoader)
+    with open(config_path, "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
     del config["sinol_expected_scores"]
-    open(config_path, "w").write(yaml.dump(config))
+    with open(config_path, "w") as config_file:
+        config_file.write(yaml.dump(config))
 
     parser = configure_parsers()
     args = parser.parse_args(["run", "--time-tool", time_tool])
@@ -51,7 +53,7 @@ def test_no_expected_scores(capsys, create_package, time_tool):
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
-                                            get_checker_package_path()], indirect=True)
+                                            get_checker_package_path(), get_library_package_path()], indirect=True)
 def test_apply_suggestions(create_package, time_tool):
     """
     Test with no sinol_expected_scores in config.yml.
@@ -62,17 +64,20 @@ def test_apply_suggestions(create_package, time_tool):
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
-    config = yaml.load(open(config_path, "r"), Loader=yaml.SafeLoader)
+    with open(config_path, "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
     expected_scores = config["sinol_expected_scores"]
     del config["sinol_expected_scores"]
-    open(config_path, "w").write(yaml.dump(config))
+    with open(config_path, "w") as config_file:
+        config_file.write(yaml.dump(config))
 
     parser = configure_parsers()
     args = parser.parse_args(["run", "--apply-suggestions", "--time-tool", time_tool])
     command = Command()
     command.run(args)
 
-    config = yaml.load(open(config_path, "r"), Loader=yaml.SafeLoader)
+    with open(config_path, "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
     assert config["sinol_expected_scores"] == expected_scores
 
 
@@ -86,10 +91,12 @@ def test_incorrect_expected_scores(capsys, create_package, time_tool):
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
-    config = yaml.load(open(config_path, "r"), Loader=yaml.SafeLoader)
+    with open(config_path, "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
     config["sinol_expected_scores"]["abc.cpp"]["expected"][1] = "WA"
     config["sinol_expected_scores"]["abc.cpp"]["points"] = 75
-    open(config_path, "w").write(yaml.dump(config))
+    with open(config_path, "w") as config_file:
+        config_file.write(yaml.dump(config))
 
     parser = configure_parsers()
     args = parser.parse_args(["run", "--time-tool", time_tool])
@@ -105,7 +112,8 @@ def test_incorrect_expected_scores(capsys, create_package, time_tool):
     assert "Solution abc.cpp passed group 1 with status OK while it should pass with status WA." in out
 
 
-@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_checker_package_path()], indirect=True)
+@pytest.mark.parametrize("create_package", [get_simple_package_path(), get_checker_package_path(),
+                                            get_library_package_path()], indirect=True)
 def test_flag_tests(create_package, time_tool):
     """
     Test flag --tests.
@@ -178,9 +186,11 @@ def test_no_scores(capsys, create_package, time_tool):
     create_ins_outs(package_path)
 
     config_path = os.path.join(package_path, "config.yml")
-    config = yaml.load(open(config_path, "r"), Loader=yaml.SafeLoader)
+    with open(config_path, "r") as config_file:
+        config = yaml.load(config_file, Loader=yaml.SafeLoader)
     del config["scores"]
-    open(config_path, "w").write(yaml.dump(config))
+    with open(config_path, "w") as config_file:
+        config_file.write(yaml.dump(config))
 
     parser = configure_parsers()
     args = parser.parse_args(["run", "--time-tool", time_tool])