Implement a simple test filter

cf. #29

Author: yamt

README.md

@@ -38,6 +38,16 @@ python3 test-runner/wasi_test_runner.py
     -r adapters/wasmtime.sh # path to a runtime adapter
 ```
 
+Optionally you can specify test cases to skip with the `--filter` option.
+
+```bash
+python3 test-runner/wasi_test_runner.py                                                  \
+    -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
+       ./tests/c/testsuite/                                                              \
+    --filter examples/skip.json                                                          \
+    -r adapters/wasmtime.sh # path to a runtime adapter
+```
+
 ## Contributing
 
 All contributions are very welcome. Contributors can help with:

examples/skip.json

@@ -0,0 +1,5 @@
+{
+    "WASI C tests": {
+        "stat-dev-ino": "d_ino is known broken in this combination of engine and wasi-sdk version."
+    }
+}

test-runner/.pylintrc

@@ -3,5 +3,6 @@ disable=
     C0114, # Missing module docstring
     C0115, # Missing class docstring
     C0116, # Missing function or method docstring
+    R0903, # Too few public methods
 [FORMAT]
 max-line-length=120

test-runner/tests/test_test_suite_runner.py

@@ -62,8 +62,12 @@ def test_runner_end_to_end() -> None:
 
     reporters = [Mock(), Mock()]
 
+    filt = Mock()
+    filt.should_skip.return_value = (False, None)
+    filters = [filt]
+
     with patch("glob.glob", return_value=test_paths):
-        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters)  # type: ignore
+        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters)  # type: ignore
 
     # Assert manifest was read correctly
     assert suite.name == "test-suite"
@@ -91,9 +95,15 @@ def test_runner_end_to_end() -> None:
         for config, output in zip(expected_config, outputs):
             validator.assert_any_call(config, output)
 
+    # Assert filter calls
+    for filt in filters:
+        assert filt.should_skip.call_count == 3
+        for test_case in expected_test_cases:
+            filt.should_skip.assert_any_call(suite.name, test_case.name)
+
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
-    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [])
+    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
 
     assert suite.name == "my-path"

test-runner/wasi_test_runner/__main__.py

@@ -5,6 +5,8 @@
 
 from .runtime_adapter import RuntimeAdapter
 from .harness import run_all_tests
+from .filters import TestFilter
+from .filters import JSONTestFilter
 from .reporters import TestReporter
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
@@ -23,6 +25,14 @@ def main() -> int:
         nargs="+",
         help="Locations of suites (directories with *.wasm test files).",
     )
+    parser.add_argument(
+        "-f",
+        "--filter",
+        required=False,
+        nargs="+",
+        default=[],
+        help="Locations of test filters (JSON files).",
+    )
     parser.add_argument(
         "-r", "--runtime-adapter", required=True, help="Path to a runtime adapter."
     )
@@ -45,11 +55,16 @@ def main() -> int:
 
     validators: List[Validator] = [exit_code_validator, stdout_validator]
 
+    filters: List[TestFilter] = []
+    for filt in options.filter:
+        filters.append(JSONTestFilter(filt))
+
     return run_all_tests(
         RuntimeAdapter(options.runtime_adapter),
         options.test_suite,
         validators,
         reporters,
+        filters,
     )
 
 

test-runner/wasi_test_runner/filters.py

@@ -0,0 +1,24 @@
+from typing import Tuple, Any
+from abc import ABC
+from abc import abstractmethod
+
+import json
+
+
+class TestFilter(ABC):
+    @abstractmethod
+    def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]:
+        pass
+
+
+class JSONTestFilter(TestFilter):
+    def __init__(self, filename: str) -> None:
+        with open(filename, encoding="utf-8") as file:
+            self.filter_dict = json.load(file)
+
+    def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]:
+        test_suite_filter = self.filter_dict.get(test_suite_name)
+        if test_suite_filter is None:
+            return False, None
+        why = test_suite_filter.get(test_name)
+        return why is not None, why

test-runner/wasi_test_runner/harness.py

@@ -1,5 +1,6 @@
 from typing import List
 
+from .filters import TestFilter
 from .reporters import TestReporter
 from .test_suite_runner import run_tests_from_test_suite
 from .runtime_adapter import RuntimeAdapter
@@ -11,12 +12,13 @@ def run_all_tests(
     test_suite_paths: List[str],
     validators: List[Validator],
     reporters: List[TestReporter],
+    filters: List[TestFilter],
 ) -> int:
     ret = 0
 
     for test_suite_path in test_suite_paths:
         test_suite = run_tests_from_test_suite(
-            test_suite_path, runtime, validators, reporters
+            test_suite_path, runtime, validators, reporters, filters,
         )
         for reporter in reporters:
             reporter.report_test_suite(test_suite)

test-runner/wasi_test_runner/reporters/console.py

@@ -54,6 +54,7 @@ def finalize(self, version: RuntimeVersion) -> None:
             print(f"  Total: {suite.test_count}")
             self._print_pass(f"  Passed:  {suite.pass_count}")
             self._print_fail(f"  Failed:  {suite.fail_count}")
+            self._print_skip(f"  Skipped: {suite.skip_count}")
             print("")
 
         print(

test-runner/wasi_test_runner/test_suite_runner.py

@@ -8,6 +8,7 @@
 from datetime import datetime
 from typing import List, cast
 
+from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
     Result,
@@ -25,28 +26,52 @@ def run_tests_from_test_suite(
     runtime: RuntimeAdapter,
     validators: List[Validator],
     reporters: List[TestReporter],
+    filters: List[TestFilter],
 ) -> TestSuite:
     test_cases: List[TestCase] = []
     test_start = datetime.now()
 
     _cleanup_test_output(test_suite_path)
 
+    test_suite_name = _read_manifest(test_suite_path)
+
     for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")):
-        test_case = _execute_single_test(runtime, validators, test_path)
+        test_name = os.path.splitext(os.path.basename(test_path))[0]
+        for filt in filters:
+            # for now, just drop the skip reason string. it might be
+            # useful to make reporters report it.
+            skip, _ = filt.should_skip(test_suite_name, test_name)
+            if skip:
+                test_case = _skip_single_test(runtime, validators, test_path)
+                break
+        else:
+            test_case = _execute_single_test(runtime, validators, test_path)
         test_cases.append(test_case)
         for reporter in reporters:
             reporter.report_test(test_case)
 
     elapsed = (datetime.now() - test_start).total_seconds()
 
     return TestSuite(
-        name=_read_manifest(test_suite_path),
+        name=test_suite_name,
         time=test_start,
         duration_s=elapsed,
         test_cases=test_cases,
     )
 
 
+def _skip_single_test(
+    _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str
+) -> TestCase:
+    config = _read_test_config(test_path)
+    return TestCase(
+        name=os.path.splitext(os.path.basename(test_path))[0],
+        config=config,
+        result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
+        duration_s=0,
+    )
+
+
 def _execute_single_test(
     runtime: RuntimeAdapter, validators: List[Validator], test_path: str
 ) -> TestCase: