Update nist-validation test to provide more granular results

conf/waivers/30-permanent

@@ -84,4 +84,11 @@
 /hardening/.*/ospp/configure_crypto_policy
     rhel.is_centos() and rhel == 9
 
+# scapval waivers
+#
+# Caused by SCE content being built by default, enabled
+# in https://github.com/ComplianceAsCode/content/pull/12488
+/static-checks/nist-validation/ssg-rhel9-ds/SRC-118
+    rhel >= 9
+
 # vim: syntax=python

static-checks/nist-validation/main.fmf

@@ -4,14 +4,10 @@ result: custom
 environment+:
     PYTHONPATH: ../..
 duration: 15m
-require+:
-  # we use java-17 specifically here because the NIST tool needs it and does not
-  # work with any newer version
+recommend+:
   - java-17-openjdk
+  - java-21-openjdk
 adjust:
   - when: arch != x86_64
     enabled: false
     because: the test is not architecture-specific, one is enough
-  - when: distro == rhel-10
-    enabled: false
-    because: TODO - RHEL-10 doesn't have Java 17, see requires above

static-checks/nist-validation/test.py

@@ -5,6 +5,7 @@
 import zipfile
 import requests
 import subprocess
+import xml.etree.ElementTree as ET
 
 from lib import util, results
 
@@ -16,6 +17,8 @@
 zip.extractall()
 os.chmod('scapval.sh', 0o755)
 
+ns = {'nist': 'http://csrc.nist.gov/ns/decima/results/1.0'}
+
 for datastream in util.iter_datastreams():
     ds_name = datastream.stem
     report_file = f'{ds_name}.report.html'
@@ -27,12 +30,19 @@
         '-valresultfile', result_file,
         '-file', datastream,
     ]
-    proc = util.subprocess_run(cmd, stdout=subprocess.PIPE, check=True, universal_newlines=True)
-    if 'The target is valid' in proc.stdout:
-        results.report('pass', ds_name)
-    elif 'The target is invalid' in proc.stdout:
-        results.report('fail', ds_name, logs=[report_file, result_file])
-    else:
-        raise RuntimeError("SCAPval out has not been correctly parsed")
+    util.subprocess_run(cmd, stdout=subprocess.DEVNULL, check=True)
+    tree = ET.parse(result_file)
+    root = tree.getroot()
+    for elem in root.findall('./nist:results/nist:base-requirement', ns):
+        name = f'{ds_name}/{elem.attrib["id"]}'
+        status = elem.find('./nist:status', ns).text
+        if status in ['NOT_TESTED', 'NOT_APPLICABLE']:
+            continue
+        elif status in ['PASS', 'WARNING', 'INFORMATIONAL']:
+            results.report('pass', name)
+        elif status == 'FAIL':
+            results.report('fail', name, logs=[report_file, result_file])
+        else:
+            results.report('error', name, logs=[report_file, result_file])
 
 results.report_and_exit()