misc: Test for actual failure vs skipped

Signed-off-by: Jason Lowe-Power <[email protected]>

.github/workflows/test-materials.yml

@@ -39,4 +39,37 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Run test
-      run: python3 materials/test-material.py "${{ matrix.file }}"
+      id: run-test
+      continue-on-error: true
+      run: |
+        python3 materials/test-material.py "${{ matrix.file }}"
+        echo "${{ matrix.file }}=$?" >> results.txt
+
+    - name: Upload test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-results
+        path: results.txt
+
+  aggregate-results:
+    needs: run-tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Download test results
+      uses: actions/download-artifact@v3
+      with:
+        name: test-results
+        path: ./results
+
+    - name: Aggregate test results
+      run: |
+        failed=0
+        for test in $(cat ./results/results.txt); do
+          result=$(echo $test | cut -d'=' -f2)
+          if [ "$result" -eq 1 ]; then
+            failed=1
+            echo "Test failed for $(echo $test | cut -d'=' -f1)"
+          fi
+        done
+        exit $failed

materials/test-material.py

@@ -31,11 +31,15 @@ def main():
     with open(args.script, "r") as f:
         script = f.read()
 
+    if '"""' not in script:
+        print("Warning: No docstring found")
+        return 2
+
     docstring = script.split('"""')[1]
 
     if "$ gem5" not in docstring:
         print("Warning: No gem5 command found in docstring")
-        return 0
+        return 2
 
     gem5_command = "gem5" + docstring.split("$ gem5")[1].split("\n")[0].strip()
     expected_output = docstring.split("$ gem5")[1].split("\n")[1].strip()