From 254c5dfec25cc28777c5ed5e566927fa848aa023 Mon Sep 17 00:00:00 2001
From: Jason Lowe-Power <jason@lowepower.com>
Date: Tue, 22 Oct 2024 16:08:05 -0700
Subject: [PATCH] misc: Test for actual failure vs skipped

Signed-off-by: Jason Lowe-Power <jason@lowepower.com>
---
 .github/workflows/test-materials.yml | 35 +++++++++++++++++++++++++++-
 materials/test-material.py           |  6 ++++-
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-materials.yml b/.github/workflows/test-materials.yml
index 10d1ff6..4102bee 100644
--- a/.github/workflows/test-materials.yml
+++ b/.github/workflows/test-materials.yml
@@ -39,4 +39,37 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Run test
-      run: python3 materials/test-material.py "${{ matrix.file }}"
+      id: run-test
+      continue-on-error: true
+      run: |
+        python3 materials/test-material.py "${{ matrix.file }}"
+        echo "${{ matrix.file }}=$?" >> results.txt
+
+    - name: Upload test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-results
+        path: results.txt
+
+  aggregate-results:
+    needs: run-tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Download test results
+      uses: actions/download-artifact@v3
+      with:
+        name: test-results
+        path: ./results
+
+    - name: Aggregate test results
+      run: |
+        failed=0
+        for test in $(cat ./results/results.txt); do
+          result=$(echo $test | cut -d'=' -f2)
+          if [ "$result" -eq 1 ]; then
+            failed=1
+            echo "Test failed for $(echo $test | cut -d'=' -f1)"
+          fi
+        done
+        exit $failed
diff --git a/materials/test-material.py b/materials/test-material.py
index a5cb0dd..0ba7d72 100755
--- a/materials/test-material.py
+++ b/materials/test-material.py
@@ -31,11 +31,15 @@ def main():
     with open(args.script, "r") as f:
         script = f.read()
 
+    if '"""' not in script:
+        print("Warning: No docstring found")
+        return 2
+
     docstring = script.split('"""')[1]
 
     if "$ gem5" not in docstring:
         print("Warning: No gem5 command found in docstring")
-        return 0
+        return 2
 
     gem5_command = "gem5" + docstring.split("$ gem5")[1].split("\n")[0].strip()
     expected_output = docstring.split("$ gem5")[1].split("\n")[1].strip()