Refactor Result class

test-runner/wasi_test_runner/reporters/console.py

@@ -2,7 +2,7 @@
 from colorama import Fore, init
 
 from . import TestReporter
-from ..test_case import TestCase
+from ..test_case import TestCase, Skipped
 from ..test_suite import TestSuite
 from ..runtime_adapter import RuntimeVersion
 
@@ -19,18 +19,18 @@ def __init__(self, colored: bool = True) -> None:
         self._colored = colored
 
     def report_test(self, test: TestCase) -> None:
-        if test.result.failed:
+        if isinstance(test.result, Skipped):
+            self._print_skip(f"Test {test.name} skipped: {test.result.reason}")
+        elif test.result.failures:
             self._print_fail(f"Test {test.name} failed")
             for reason in test.result.failures:
                 self._print_fail(f"  [{reason.type}] {reason.message}")
             print("STDOUT:")
             print(test.result.output.stdout)
             print("STDERR:")
             print(test.result.output.stderr)
-        elif test.result.is_executed:
-            self._print_pass(f"Test {test.name} passed")
         else:
-            self._print_skip(f"Test {test.name} skipped")
+            self._print_pass(f"Test {test.name} passed")
 
     def report_test_suite(self, test_suite: TestSuite) -> None:
         self._test_suites.append(test_suite)

test-runner/wasi_test_runner/test_case.py

@@ -1,6 +1,6 @@
 import logging
 import json
-from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional
+from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional, Union
 
 
 class Output(NamedTuple):
@@ -14,14 +14,16 @@ class Failure(NamedTuple):
     message: str
 
 
-class Result(NamedTuple):
+class Executed(NamedTuple):
     output: Output
-    is_executed: bool
     failures: List[Failure]
 
-    @property
-    def failed(self) -> bool:
-        return len(self.failures) > 0
+
+class Skipped(NamedTuple):
+    reason: str
+
+
+Result = Union[Executed, Skipped]
 
 
 T = TypeVar("T", bound="Config")

test-runner/wasi_test_runner/test_suite.py

@@ -1,6 +1,6 @@
 from typing import NamedTuple, List
 from datetime import datetime
-from .test_case import TestCase
+from .test_case import TestCase, Executed, Skipped
 
 
 class TestSuite(NamedTuple):
@@ -19,7 +19,7 @@ def pass_count(self) -> int:
             [
                 1
                 for test in self.test_cases
-                if test.result.is_executed and test.result.failed is False
+                if isinstance(test.result, Executed) and not test.result.failures
             ]
         )
 
@@ -29,10 +29,10 @@ def fail_count(self) -> int:
             [
                 1
                 for test in self.test_cases
-                if test.result.is_executed and test.result.failed
+                if isinstance(test.result, Executed) and test.result.failures
             ]
         )
 
     @property
     def skip_count(self) -> int:
-        return len([1 for test in self.test_cases if not test.result.is_executed])
+        return len([1 for test in self.test_cases if isinstance(test.result, Skipped)])

test-runner/wasi_test_runner/test_suite_runner.py

@@ -11,7 +11,8 @@
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
-    Result,
+    Executed,
+    Skipped,
     Config,
     Output,
     TestCase,
@@ -38,11 +39,13 @@ def run_tests_from_test_suite(
     for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")):
         test_name = os.path.splitext(os.path.basename(test_path))[0]
         for filt in filters:
-            # for now, just drop the skip reason string. it might be
-            # useful to make reporters report it.
-            skip, _ = filt.should_skip(test_suite_name, test_name)
+            skip, reason = filt.should_skip(test_suite_name, test_name)
             if skip:
-                test_case = _skip_single_test(runtime, validators, test_path)
+                # Given the type of should_skip, "if skip" implies reason
+                # is str. But mypy doesn't seem to recognize it. Use an
+                # explicit cast for now.
+                reason = cast(str, reason)
+                test_case = _skip_single_test(runtime, validators, test_path, reason)
                 break
         else:
             test_case = _execute_single_test(runtime, validators, test_path)
@@ -61,13 +64,13 @@ def run_tests_from_test_suite(
 
 
 def _skip_single_test(
-    _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str
+    _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str, reason: str
 ) -> TestCase:
     config = _read_test_config(test_path)
     return TestCase(
         name=os.path.splitext(os.path.basename(test_path))[0],
         config=config,
-        result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
+        result=Skipped(reason),
         duration_s=0,
     )
 
@@ -88,14 +91,14 @@ def _execute_single_test(
     )
 
 
-def _validate(validators: List[Validator], config: Config, output: Output) -> Result:
+def _validate(validators: List[Validator], config: Config, output: Output) -> Executed:
     failures = [
         result
         for result in [validator(config, output) for validator in validators]
         if result is not None
     ]
 
-    return Result(failures=failures, is_executed=True, output=output)
+    return Executed(failures=failures, output=output)
 
 
 def _read_test_config(test_path: str) -> Config: