Skip to content

Commit 3010c7e

Browse files
committed
Add CMS automated tests with OpenSSL 3.6
1 parent 747e9dc commit 3010c7e

File tree

6 files changed

+231
-6
lines changed

6 files changed

+231
-6
lines changed

.github/workflows/artifact_validation.yaml

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,27 @@ jobs:
4040
with:
4141
name: Compatibility_openssl_csv
4242
path: ./output/
43+
openssl36_validation:
44+
runs-on: ubuntu-latest
45+
container: ubuntu:24.04
46+
steps:
47+
- name: Checkout code
48+
uses: actions/checkout@v4
49+
- name: Install OpenSSL 3.6
50+
run: |
51+
apt update
52+
apt install -y zip gnupg
53+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6ED0E7B82643E131
54+
echo "deb http://deb.debian.org/debian experimental main" > /etc/apt/sources.list
55+
apt update
56+
apt install -y openssl/experimental
57+
- name: Test CMS artifacts with OpenSSL 3.6
58+
run: ./src/test_cms_v3_openssl.sh
59+
- name: Save artifacts
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: Compatibility_openssl36_csv
63+
path: ./output/
4364
ssai_validation:
4465
runs-on: ubuntu-latest
4566
container: jethrolow/quantcrypt_validator
@@ -100,7 +121,7 @@ jobs:
100121
build_results_html:
101122
runs-on: ubuntu-latest
102123
container: ubuntu:latest
103-
needs: [bc_validation, openssl_validation, ssai_validation, composite_ref_impl_validation]
124+
needs: [bc_validation, openssl_validation, openssl36_validation, ssai_validation, composite_ref_impl_validation]
104125
steps:
105126
- name: Checkout code
106127
uses: actions/checkout@v4
@@ -124,6 +145,11 @@ jobs:
124145
with:
125146
name: Compatibility_openssl_csv
126147
path: output/
148+
- name: Get OpenSSL 3.6 results from previous job
149+
uses: actions/download-artifact@v4
150+
with:
151+
name: Compatibility_openssl36_csv
152+
path: output/
127153
- name: Get SSAI results from previous job
128154
uses: actions/download-artifact@v4
129155
with:
@@ -138,8 +164,10 @@ jobs:
138164
run: ./src/rebuild_results_certs_r5.sh
139165
- name: Build compat matrix (cms_v3)
140166
run: ./src/rebuild_results_cms_v3.sh
141-
- name: Copy output files (automated r5/v1)
142-
run: cp ./docs/pqc_hackathon_results_certs_r5_automated_tests.html ./output/certs/bc_certs.log ./output/certs/openssl_certs.log ./output/certs/ssai_certs.log ./output/certs/composite-sigs-ref-impl.log ./output/certs/composite-kem-ref-impl.log ./docs/gh-pages
167+
- name: Copy output files (automated certs_r5)
168+
run: cp ./docs/pqc_hackathon_results_certs_r5_automated_tests.html ./output/certs/bc_certs.log ./output/certs/openssl_certs.log ./output/certs/ssai_certs.log ./output/certs/composite-sigs-ref-impl.log ./output/certs/composite-kem-ref-impl.log ./output/cms/openssl_cms.log ./docs/gh-pages
169+
- name: Copy output files (automated cms_v3)
170+
run: cp ./docs/pqc_hackathon_results_cms_v3_automated_tests.html ./output/cms/openssl_cms.log ./docs/gh-pages
143171
- name: Copy output files (manual r5/v1)
144172
run: cp ./docs/pqc_hackathon_results_certs_r5.html ./docs/pqc_hackathon_results_cms_v1.html ./docs/gh-pages
145173
- name: Copy output files (manual r5/v3)

docs/gh-pages/index.html

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
<body>
77
<p>Test matrix generated with the automated test suite:</p>
88
<p><a href="pqc_hackathon_results_certs_r5_automated_tests.html">pqc_hackathon_results_certs_r5_automated_tests.html</a></p>
9+
<p><a href="pqc_hackathon_results_cms_v3_automated_tests.html">pqc_hackathon_results_cms_v3_automated_tests.html</a></p>
910
<br>
1011
<p>Test matrix generated from manually-uploaded test results:</p>
1112
<p><a href="pqc_hackathon_results_certs_r5.html">pqc_hackathon_results_certs_r5.html</a></p>
@@ -17,6 +18,7 @@
1718
<p><a href="composite-sigs-ref-impl.log">composite-sigs-ref-impl.log</a></p>
1819
<p><a href="composite-kem-ref-impl.log">composite-kem-ref-impl.log</a></p>
1920
<p><a href="openssl_certs.log">openssl_certs.log</a></p>
21+
<p><a href="openssl_cms.log">openssl_cms.log</a></p>
2022
<p><a href="ssai_certs.log">ssai_certs.log</a></p>
2123
</body>
2224
</html>

docs/pqc_hackathon_results_cms_v3.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<h1 id="ietf-pqc-hackathon-cms-interoperability-results">IETF PQC
22
Hackathon CMS Interoperability Results</h1>
33
<style> table { border-collapse: collapse; width:auto !important; } th, td { border: solid black 1px; padding: 0 1ex; } col { width: auto !important; } </style>
4-
<p>Generated: 2025-11-02 14:52 UTC</p>
4+
<p>Generated: 2025-11-02 18:35 UTC</p>
55
<h1 id="algorithms-submitted">Algorithms Submitted</h1>
66
<p>✅ = passing all verifiers<br>◒ = passing some verifiers<br>⚪︎ = not
77
passing any verifiers<br>Columns represent producers who submitted

docs/pqc_hackathon_results_cms_v3.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ IETF PQC Hackathon CMS Interoperability Results
55

66
<style> table { border-collapse: collapse; width:auto !important; } th, td { border: solid black 1px; padding: 0 1ex; } col { width: auto !important; } </style>
77

8-
Generated: 2025-11-02 14:52 UTC
8+
Generated: 2025-11-02 18:35 UTC
99

1010

1111
# Algorithms Submitted

src/rebuild_results_cms_v3.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,10 @@ python3 src/pqc_report_writer_common.py docs/oid_mapping.md pqc_hackathon_result
1414

1515
# convert to html
1616
pandoc -f markdown pqc_hackathon_results_cms_v3.md > pqc_hackathon_results_cms_v3.html
17-
mv pqc_hackathon_results_cms_v3.md pqc_hackathon_results_cms_v3.html docs
17+
mv pqc_hackathon_results_cms_v3.md pqc_hackathon_results_cms_v3.html docs
18+
19+
if [ -d ./output/cms ]; then
20+
python3 src/pqc_report_writer_common.py docs/oid_mapping.md pqc_hackathon_results_cms_v3_automated_tests.md "Certificate Automated Verification" $(find ./output/cms -name "*.csv")
21+
pandoc -f markdown pqc_hackathon_results_cms_v3_automated_tests.md > pqc_hackathon_results_cms_v3_automated_tests.html
22+
mv pqc_hackathon_results_cms_v3_automated_tests.md pqc_hackathon_results_cms_v3_automated_tests.html docs
23+
fi

src/test_cms_v3_openssl.sh

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
echo "Running OpenSSL 3.6 verifier." | tee $logfile
6+
7+
certsdir="artifacts_cms_v3"
8+
cmszip="artifacts_cms_v3.zip"
9+
inputdir="./providers"
10+
outputdir="./output/cms"
11+
logfile=$outputdir/openssl_cms.log
12+
13+
# Start the results CSV file
14+
mkdir -p $outputdir
15+
printf "Build time: %s\n\n" "$(date)" | tee -a $logfile
16+
17+
alreadyTestedSigOIDs=";"
18+
alreadyTestedKEMOIDs=";"
19+
20+
check_signed_attrs() {
21+
signed_attrs=$1
22+
ta_file=$2
23+
24+
openssl cms -verify -no_check_time -in $signed_attrs -inform DER -CAfile $ta_file
25+
if [ $? -ne 0 ]; then
26+
echo "ERROR: verifying signed attributes" | tee -a $logfile
27+
return 1
28+
fi
29+
30+
return 0
31+
}
32+
33+
# Requires an input: the TA file to test
34+
test_signed_attrs () {
35+
signed_attrs_file=$1
36+
resultsfile=$2
37+
38+
output=""
39+
signed_attrs_filename=$(basename $signed_attrs_file)
40+
41+
# strip off the friendly name
42+
signed_attrs_fileBasename=$(echo $signed_attrs_filename | egrep -o '[^-]*_signed_attrs.der$')
43+
44+
# strip off the file suffix to get the OID name
45+
if [ $(expr match "$signed_attrs_fileBasename" ".*_signed_attrs\.der$") != 0 ]; then
46+
oid=${signed_attrs_fileBasename%_signed_attrs.der}
47+
else # It's some other filename
48+
printf "\nERROR: file name is not in the expected format: %s\n" $signed_attrs_file | tee -a $logfile
49+
return
50+
fi
51+
52+
# some artifacts submit multiple copies of the same cert as .pem, .der, etc. Just skip the second one
53+
if [ $(expr match "$alreadyTestedSigOIDs" ".*\;$oid\;.*") != 0 ]; then
54+
printf "\nWarning: %s has been submitted multiple times by this provider. Skipping\n" $oid | tee -a $logfile
55+
return
56+
fi
57+
58+
openssl list --signature-algorithms --kem-algorithms | grep " $oid,"
59+
if [ $? != 0 ]; then
60+
printf "\nSkipping %s, unsupported\n" $signed_attrs_filename | tee -a $logfile
61+
return
62+
fi
63+
64+
alreadyTestedSigOIDs=${alreadyTestedSigOIDs}$oid";"
65+
66+
printf "\nTesting signed attributes %s\n" $signed_attrs_file | tee -a $logfile
67+
output+=$(check_signed_attrs $signed_attrs_file $(dirname $signed_attrs_file)/ta.der)
68+
status=$?
69+
if [ $status != 0 ]; then
70+
echo "Signed Attributes Result: FAIL" | tee -a $logfile
71+
echo "${oid},N" >> $resultsfile
72+
else
73+
echo "Signed Attributes Result: SUCCESS" | tee -a $logfile
74+
echo "${oid},Y" >> $resultsfile
75+
fi
76+
77+
# log it to file and to stdout
78+
echo "$output" | tee -a $logfile
79+
}
80+
81+
check_kemri() {
82+
eefile=$1
83+
oid=$2
84+
85+
priv_key=$( find $(dirname $eefile) -name "*${oid}*_priv.der" -print -quit )
86+
kemri_base=${eefile%_ee.der}
87+
for kemri_file in ${kemri_base}_kemri*.der; do
88+
if [[ "$kemri_file" == *"kmac"* ]]; then
89+
printf "\nSkipping KMAC %s\n" $kemri_file | tee -a $logfile
90+
continue
91+
fi
92+
OUTFILE=$(mktemp)
93+
rm -f $OUTFILE
94+
95+
printf "\nTesting KEMRI %s\n" $kemri_file | tee -a $logfile
96+
97+
openssl cms -decrypt -inform DER -in ${kemri_file} -recip ${eefile} \
98+
-inkey ${priv_key} -keyform DER -out $OUTFILE
99+
if [ $? -ne 0 ]; then
100+
echo "ERROR: Decrypting KEMRI" | tee -a $logfile
101+
return 1
102+
fi
103+
rm -f $OUTFILE
104+
done
105+
106+
return 0
107+
}
108+
109+
# Requires an input: the TA file to test
110+
test_kemri() {
111+
eefile=$1
112+
resultsfile=$2
113+
114+
output=""
115+
eefileBasename=$(basename $eefile)
116+
117+
# strip off the friendly name
118+
eefileBasename=$(echo $eefileBasename | egrep -o '[^-]*_ee.der$')
119+
120+
# strip off the file suffix to get the OID name
121+
if [ $(expr match "$eefileBasename" ".*_ee\.der$") != 0 ]; then
122+
oid=${eefileBasename%_ee.der}
123+
else # It's some other filename
124+
printf "\nERROR: file name is not in the expected format: %s\n" $eefile | tee -a $logfile
125+
return
126+
fi
127+
128+
# some artifacts submit multiple copies of the same cert as .pem, .der, etc. Just skip the second one
129+
if [ $(expr match "$alreadyTestedKEMOIDs" ".*\;$oid\;.*") != 0 ]; then
130+
printf "\nWarning: %s has been submitted multiple times by this provider. Skipping\n" $oid | tee -a $logfile
131+
return
132+
fi
133+
134+
openssl list --signature-algorithms --kem-algorithms | grep " $oid,"
135+
if [ $? != 0 ]; then
136+
printf "\nSkipping %s, unsupported\n" $eefileBasename | tee -a $logfile
137+
return
138+
fi
139+
140+
alreadyTestedKEMOIDs=${alreadyTestedKEMOIDs}$oid";"
141+
142+
printf "\nTesting EE %s\n" $eefile | tee -a $logfile
143+
output+=$(check_kemri $eefile $oid)
144+
status=$?
145+
if [ $status != 0 ]; then
146+
echo "KemRecipientInfo Result: FAIL" | tee -a $logfile
147+
echo "${oid},N" >> $resultsfile
148+
else
149+
echo "KemRecipientInfo Result: SUCCESS" | tee -a $logfile
150+
echo "${oid},Y" >> $resultsfile
151+
fi
152+
153+
# log it to file and to stdout
154+
echo "$output" | tee -a $logfile
155+
}
156+
157+
# First, recurse into any provider dir
158+
for providerdir in $(ls -d $inputdir/*/); do
159+
provider=$(basename $providerdir)
160+
161+
printf "\n**** BEGIN provider %s **** \n" $provider | tee -a $logfile
162+
163+
# process certs
164+
zip=${providerdir}$cmszip
165+
if ! [ -f $zip ]; then
166+
continue
167+
fi
168+
unzipdir=${providerdir}$certsdir
169+
printf "\nUnzipping %s to %s\n" $zip $unzipdir | tee -a $logfile
170+
unzip -o $zip -d $unzipdir
171+
172+
resultsfile=${outputdir}/${provider}_openssl.csv
173+
echo "key_algorithm_oid,type,test_result" > $resultsfile # CSV header row
174+
175+
alreadyTestedSigOIDs=";" # for a guard to skip testing the same cert multiple times
176+
alreadyTestedKEMOIDs=";" # for a guard to skip testing the same cert multiple times
177+
# test each signed attributes fileartifact
178+
for signed_attrs_file in $(find $unzipdir \( -iname "*_signed_attrs.der" \)); do
179+
test_signed_attrs "$signed_attrs_file" "$resultsfile"
180+
done
181+
# test each KEMRI artifact
182+
for eefile in $(find $unzipdir \( -iname "*_ee.der" \)); do
183+
test_kemri "$eefile" "$resultsfile"
184+
done
185+
186+
printf "\n**** DONE provider %s **** \n" $provider | tee -a $logfile
187+
done
188+
189+
printf "\n**** DONE **** \n" | tee -a $logfile

0 commit comments

Comments
 (0)