Skip to content

Commit 38698da

Browse files
authored
Add CMS automated tests with OpenSSL 3.6 (#247)
Add CMS automated tests with OpenSSL 3.6
1 parent 747e9dc commit 38698da

File tree

7 files changed

+249
-17
lines changed

7 files changed

+249
-17
lines changed

.github/workflows/artifact_validation.yaml

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,27 @@ jobs:
4040
with:
4141
name: Compatibility_openssl_csv
4242
path: ./output/
43+
openssl36_validation:
44+
runs-on: ubuntu-latest
45+
container: ubuntu:24.04
46+
steps:
47+
- name: Checkout code
48+
uses: actions/checkout@v4
49+
- name: Install OpenSSL 3.6
50+
run: |
51+
apt update
52+
apt install -y zip gnupg
53+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6ED0E7B82643E131
54+
echo "deb http://deb.debian.org/debian experimental main" > /etc/apt/sources.list
55+
apt update
56+
apt install -y openssl/experimental
57+
- name: Test CMS artifacts with OpenSSL 3.6
58+
run: ./src/test_cms_v3_openssl.sh
59+
- name: Save artifacts
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: Compatibility_openssl36_csv
63+
path: ./output/
4364
ssai_validation:
4465
runs-on: ubuntu-latest
4566
container: jethrolow/quantcrypt_validator
@@ -100,7 +121,7 @@ jobs:
100121
build_results_html:
101122
runs-on: ubuntu-latest
102123
container: ubuntu:latest
103-
needs: [bc_validation, openssl_validation, ssai_validation, composite_ref_impl_validation]
124+
needs: [bc_validation, openssl_validation, openssl36_validation, ssai_validation, composite_ref_impl_validation]
104125
steps:
105126
- name: Checkout code
106127
uses: actions/checkout@v4
@@ -124,6 +145,11 @@ jobs:
124145
with:
125146
name: Compatibility_openssl_csv
126147
path: output/
148+
- name: Get OpenSSL 3.6 results from previous job
149+
uses: actions/download-artifact@v4
150+
with:
151+
name: Compatibility_openssl36_csv
152+
path: output/
127153
- name: Get SSAI results from previous job
128154
uses: actions/download-artifact@v4
129155
with:
@@ -138,8 +164,10 @@ jobs:
138164
run: ./src/rebuild_results_certs_r5.sh
139165
- name: Build compat matrix (cms_v3)
140166
run: ./src/rebuild_results_cms_v3.sh
141-
- name: Copy output files (automated r5/v1)
142-
run: cp ./docs/pqc_hackathon_results_certs_r5_automated_tests.html ./output/certs/bc_certs.log ./output/certs/openssl_certs.log ./output/certs/ssai_certs.log ./output/certs/composite-sigs-ref-impl.log ./output/certs/composite-kem-ref-impl.log ./docs/gh-pages
167+
- name: Copy output files (automated certs_r5)
168+
run: cp ./docs/pqc_hackathon_results_certs_r5_automated_tests.html ./output/certs/bc_certs.log ./output/certs/openssl_certs.log ./output/certs/ssai_certs.log ./output/certs/composite-sigs-ref-impl.log ./output/certs/composite-kem-ref-impl.log ./output/cms/openssl_cms.log ./docs/gh-pages
169+
- name: Copy output files (automated cms_v3)
170+
run: cp ./docs/pqc_hackathon_results_cms_v3_automated_tests.html ./output/cms/openssl_cms.log ./docs/gh-pages
143171
- name: Copy output files (manual r5/v1)
144172
run: cp ./docs/pqc_hackathon_results_certs_r5.html ./docs/pqc_hackathon_results_cms_v1.html ./docs/gh-pages
145173
- name: Copy output files (manual r5/v3)

docs/gh-pages/index.html

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
<body>
77
<p>Test matrix generated with the automated test suite:</p>
88
<p><a href="pqc_hackathon_results_certs_r5_automated_tests.html">pqc_hackathon_results_certs_r5_automated_tests.html</a></p>
9+
<p><a href="pqc_hackathon_results_cms_v3_automated_tests.html">pqc_hackathon_results_cms_v3_automated_tests.html</a></p>
910
<br>
1011
<p>Test matrix generated from manually-uploaded test results:</p>
1112
<p><a href="pqc_hackathon_results_certs_r5.html">pqc_hackathon_results_certs_r5.html</a></p>
@@ -17,6 +18,7 @@
1718
<p><a href="composite-sigs-ref-impl.log">composite-sigs-ref-impl.log</a></p>
1819
<p><a href="composite-kem-ref-impl.log">composite-kem-ref-impl.log</a></p>
1920
<p><a href="openssl_certs.log">openssl_certs.log</a></p>
21+
<p><a href="openssl_cms.log">openssl_cms.log</a></p>
2022
<p><a href="ssai_certs.log">ssai_certs.log</a></p>
2123
</body>
2224
</html>

docs/pqc_hackathon_results_cms_v3.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<h1 id="ietf-pqc-hackathon-cms-interoperability-results">IETF PQC
22
Hackathon CMS Interoperability Results</h1>
33
<style> table { border-collapse: collapse; width:auto !important; } th, td { border: solid black 1px; padding: 0 1ex; } col { width: auto !important; } </style>
4-
<p>Generated: 2025-11-02 14:52 UTC</p>
4+
<p>Generated: 2025-11-02 20:07 UTC</p>
55
<h1 id="algorithms-submitted">Algorithms Submitted</h1>
66
<p>✅ = passing all verifiers<br>◒ = passing some verifiers<br>⚪︎ = not
77
passing any verifiers<br>Columns represent producers who submitted

docs/pqc_hackathon_results_cms_v3.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ IETF PQC Hackathon CMS Interoperability Results
55

66
<style> table { border-collapse: collapse; width:auto !important; } th, td { border: solid black 1px; padding: 0 1ex; } col { width: auto !important; } </style>
77

8-
Generated: 2025-11-02 14:52 UTC
8+
Generated: 2025-11-02 20:07 UTC
99

1010

1111
# Algorithms Submitted

readme.md

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -154,13 +154,13 @@ Within `providers/<provider_name>/[implementation_name/]`
154154
- artifacts_cms_v3.zip
155155
- `artifacts_cms_v3/` subfolder which will contain the artifacts
156156
- `artifacts_cms_v3/expected_plaintext.txt` # The message which was encrypted and can be compared against the decrypted artifacts.
157-
- `artifacts_cms_v3/ukm.txt` # The User Keying Material (UKM) included in some of the enveloped messages.
158157
- `artifacts_cms_v3/ta.der` # ML-DSA-44 trust anchor used to sign the end-entity certificates.
159158
- `artifacts_cms_v3/<friendly>-<oid>_ee.der` # The KEM certificate that the message is enveloped to.
160-
- `artifacts_cms_v3/<friendly>-<oid>_both_priv.der` # The private KEM key to decrypt the enveloped messages.
159+
- `artifacts_cms_v3/<friendly>-<oid>_priv.der` # The private KEM key to decrypt the enveloped messages.
161160
- `artifacts_cms_v3/<friendly>-<oid>_kemri_ukm.der` # An Enveloped artifact using KEMRI’s UKM field and one of the MTI KDFs for the KEM algorithm.
162161
- `artifacts_cms_v3/<friendly>-<oid>_kemri_auth.der` # An AuthEnveloped artifact using KEMRI without UKM and one of the MTI KDFs for the KEM algorithm.
163162
- `artifacts_cms_v3/<friendly>-<oid>_kemri_<kdf>.der` # Enveloped artifacts using KEMRI without UKM, and the specified KDF. Implementations must provide artifacts for each of the MTI KDFs for the OID, and may provide artifacts for other KDFs.
163+
- `artifacts_cms_v3/<friendly>-<oid>_kemri_auth_<kdf>.der` # An AuthEnveloped artifact using KEMRI without UKM and the specified KDF.
164164
- `artifacts_cms_v3/<friendly>-<oid>_signed_attrs.der` # Signed artifact, with attached content and signed attributes.
165165

166166
#### Friendly
@@ -169,7 +169,7 @@ Per https://github.com/IETF-Hackathon/pqc-certificates/issues/96 we would like a
169169

170170
#### Trust Anchor
171171

172-
A trust anchor isn't necessary to verify the KEMRecipientInfo artifacts, but some implementations may find it useful. We're using dilithium2 at the moment since some might not have implemented ML-DSA.ipd.
172+
A trust anchor isn't necessary to verify the KEMRecipientInfo artifacts, but some implementations may find it useful. We're using ML-DSA-44.
173173

174174
#### DER vs PEM
175175

@@ -188,14 +188,21 @@ Each RFC will specify mandatory KDFs, and probably allow for others as well. You
188188
| I-D/RFC | Algorithm | MTI KDF | `<kdf> string` |
189189
| - | - | - | - |
190190
| rfc5990bis | RSA-KEM | KDF3 w/ SHA-256 | id-kdf-kdf3 |
191-
| cms-kyber | ML-KEM-512 | KMAC128-KDF\* | id-kmac128\* |
192-
| cms-kyber | ML-KEM-768 | KMAC256-KDF\* | id-kmac256\* |
193-
| cms-kyber | ML-KEM-1024 | KMAC256-KDF\* | id-kmac256\* |
194-
| - | kyber512 | KMAC256-KDF\* | id-kmac128\* |
195-
| - | kyber768 | KMAC256-KDF\* | id-kmac256\* |
196-
| - | kyber1024 | KMAC256-KDF\* | id-kmac256\* |
197-
198-
\* The MTI artifacts were updated to KMAC-based KDFs in draft-ietf-lamps-cms-kyber-03.
191+
| cms-kyber | ML-KEM-512 | HKDF with SHA256 | id-alg-hkdf-with-sha256 |
192+
| cms-kyber | ML-KEM-768 | HKDF with SHA256 | id-alg-hkdf-with-sha256 |
193+
| cms-kyber | ML-KEM-1024 | HKDF with SHA256 | id-alg-hkdf-with-sha256 |
194+
| TBD | id-MLKEM768-RSA2048-SHA3-256 | KMAC256-KDF | id-kmac256 |
195+
| TBD | id-MLKEM768-RSA3072-SHA3-256 | KMAC256-KDF | id-kmac256 |
196+
| TBD | id-MLKEM768-RSA4096-SHA3-256 | KMAC256-KDF | id-kmac256 |
197+
| TBD | id-MLKEM768-X25519-SHA3-256 | KMAC256-KDF | id-kmac256 |
198+
| TBD | id-MLKEM768-ECDH-P256-SHA3-256 | KMAC256-KDF | id-kmac256 |
199+
| TBD | id-MLKEM768-ECDH-P384-SHA3-256 | KMAC256-KDF | id-kmac256 |
200+
| TBD | id-MLKEM768-ECDH-brainpoolP256r1-SHA3-256 | KMAC256-KDF | id-kmac256 |
201+
| TBD | id-MLKEM1024-RSA3072-SHA3-256 | KMAC256-KDF | id-kmac256 |
202+
| TBD | id-MLKEM1024-ECDH-P384-SHA3-256 | KMAC256-KDF | id-kmac256 |
203+
| TBD | id-MLKEM1024-ECDH-brainpoolP384r1-SHA3-256 | KMAC256-KDF | id-kmac256 |
204+
| TBD | id-MLKEM1024-X448-SHA3-256 | KMAC256-KDF | id-kmac256 |
205+
| TBD | id-MLKEM1024-ECDH-P521-SHA3-256 | KMAC256-KDF | id-kmac256 |
199206

200207
### CMP -- artifacts_cmp.zip
201208

src/rebuild_results_cms_v3.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,10 @@ python3 src/pqc_report_writer_common.py docs/oid_mapping.md pqc_hackathon_result
1414

1515
# convert to html
1616
pandoc -f markdown pqc_hackathon_results_cms_v3.md > pqc_hackathon_results_cms_v3.html
17-
mv pqc_hackathon_results_cms_v3.md pqc_hackathon_results_cms_v3.html docs
17+
mv pqc_hackathon_results_cms_v3.md pqc_hackathon_results_cms_v3.html docs
18+
19+
if [ -d ./output/cms ]; then
20+
python3 src/pqc_report_writer_common.py docs/oid_mapping.md pqc_hackathon_results_cms_v3_automated_tests.md "Certificate Automated Verification" $(find ./output/cms -name "*.csv")
21+
pandoc -f markdown pqc_hackathon_results_cms_v3_automated_tests.md > pqc_hackathon_results_cms_v3_automated_tests.html
22+
mv pqc_hackathon_results_cms_v3_automated_tests.md pqc_hackathon_results_cms_v3_automated_tests.html docs
23+
fi

src/test_cms_v3_openssl.sh

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
echo "Running OpenSSL 3.6 verifier." | tee $logfile
6+
7+
certsdir="artifacts_cms_v3"
8+
cmszip="artifacts_cms_v3.zip"
9+
inputdir="./providers"
10+
outputdir="./output/cms"
11+
logfile=$outputdir/openssl_cms.log
12+
13+
# Start the results CSV file
14+
mkdir -p $outputdir
15+
printf "Build time: %s\n\n" "$(date)" | tee -a $logfile
16+
17+
alreadyTestedSigOIDs=";"
18+
alreadyTestedKEMOIDs=";"
19+
20+
check_signed_attrs() {
21+
signed_attrs=$1
22+
ta_file=$2
23+
24+
openssl cms -verify -no_check_time -in $signed_attrs -inform DER -CAfile $ta_file
25+
if [ $? -ne 0 ]; then
26+
echo "ERROR: verifying signed attributes" | tee -a $logfile
27+
return 1
28+
fi
29+
30+
return 0
31+
}
32+
33+
# Requires an input: the TA file to test
34+
test_signed_attrs () {
35+
signed_attrs_file=$1
36+
resultsfile=$2
37+
38+
output=""
39+
signed_attrs_filename=$(basename $signed_attrs_file)
40+
41+
# strip off the friendly name
42+
signed_attrs_fileBasename=$(echo $signed_attrs_filename | egrep -o '[^-]*_signed_attrs.der$')
43+
44+
# strip off the file suffix to get the OID name
45+
if [ $(expr match "$signed_attrs_fileBasename" ".*_signed_attrs\.der$") != 0 ]; then
46+
oid=${signed_attrs_fileBasename%_signed_attrs.der}
47+
else # It's some other filename
48+
printf "\nERROR: file name is not in the expected format: %s\n" $signed_attrs_file | tee -a $logfile
49+
return
50+
fi
51+
52+
# some artifacts submit multiple copies of the same cert as .pem, .der, etc. Just skip the second one
53+
if [ $(expr match "$alreadyTestedSigOIDs" ".*\;$oid\;.*") != 0 ]; then
54+
printf "\nWarning: %s has been submitted multiple times by this provider. Skipping\n" $oid | tee -a $logfile
55+
return
56+
fi
57+
58+
openssl list --signature-algorithms --kem-algorithms | grep " $oid,"
59+
if [ $? != 0 ]; then
60+
printf "\nSkipping %s, unsupported\n" $signed_attrs_filename | tee -a $logfile
61+
return
62+
fi
63+
64+
alreadyTestedSigOIDs=${alreadyTestedSigOIDs}$oid";"
65+
66+
printf "\nTesting signed attributes %s\n" $signed_attrs_file | tee -a $logfile
67+
output+=$(check_signed_attrs $signed_attrs_file $(dirname $signed_attrs_file)/ta.der)
68+
status=$?
69+
if [ $status != 0 ]; then
70+
echo "Signed Attributes Result: FAIL" | tee -a $logfile
71+
echo "${oid},N" >> $resultsfile
72+
else
73+
echo "Signed Attributes Result: SUCCESS" | tee -a $logfile
74+
echo "${oid},Y" >> $resultsfile
75+
fi
76+
77+
# log it to file and to stdout
78+
echo "$output" | tee -a $logfile
79+
}
80+
81+
check_kemri() {
82+
eefile=$1
83+
oid=$2
84+
85+
priv_key=$( find $(dirname $eefile) -name "*${oid}*_priv.der" -print -quit )
86+
kemri_base=${eefile%_ee.der}
87+
for kemri_file in ${kemri_base}_kemri*.der; do
88+
if [[ "$kemri_file" == *"kmac"* ]]; then
89+
printf "\nSkipping KMAC %s\n" $kemri_file | tee -a $logfile
90+
continue
91+
fi
92+
OUTFILE=$(mktemp)
93+
rm -f $OUTFILE
94+
95+
printf "\nTesting KEMRI %s\n" $kemri_file | tee -a $logfile
96+
97+
openssl cms -decrypt -inform DER -in ${kemri_file} -recip ${eefile} \
98+
-inkey ${priv_key} -keyform DER -out $OUTFILE
99+
if [ $? -ne 0 ]; then
100+
echo "ERROR: Decrypting KEMRI" | tee -a $logfile
101+
return 1
102+
fi
103+
rm -f $OUTFILE
104+
done
105+
106+
return 0
107+
}
108+
109+
# Requires an input: the TA file to test
110+
test_kemri() {
111+
eefile=$1
112+
resultsfile=$2
113+
114+
output=""
115+
eefileBasename=$(basename $eefile)
116+
117+
# strip off the friendly name
118+
eefileBasename=$(echo $eefileBasename | egrep -o '[^-]*_ee.der$')
119+
120+
# strip off the file suffix to get the OID name
121+
if [ $(expr match "$eefileBasename" ".*_ee\.der$") != 0 ]; then
122+
oid=${eefileBasename%_ee.der}
123+
else # It's some other filename
124+
printf "\nERROR: file name is not in the expected format: %s\n" $eefile | tee -a $logfile
125+
return
126+
fi
127+
128+
# some artifacts submit multiple copies of the same cert as .pem, .der, etc. Just skip the second one
129+
if [ $(expr match "$alreadyTestedKEMOIDs" ".*\;$oid\;.*") != 0 ]; then
130+
printf "\nWarning: %s has been submitted multiple times by this provider. Skipping\n" $oid | tee -a $logfile
131+
return
132+
fi
133+
134+
openssl list --signature-algorithms --kem-algorithms | grep " $oid,"
135+
if [ $? != 0 ]; then
136+
printf "\nSkipping %s, unsupported\n" $eefileBasename | tee -a $logfile
137+
return
138+
fi
139+
140+
alreadyTestedKEMOIDs=${alreadyTestedKEMOIDs}$oid";"
141+
142+
printf "\nTesting EE %s\n" $eefile | tee -a $logfile
143+
output+=$(check_kemri $eefile $oid)
144+
status=$?
145+
if [ $status != 0 ]; then
146+
echo "KemRecipientInfo Result: FAIL" | tee -a $logfile
147+
echo "${oid},N" >> $resultsfile
148+
else
149+
echo "KemRecipientInfo Result: SUCCESS" | tee -a $logfile
150+
echo "${oid},Y" >> $resultsfile
151+
fi
152+
153+
# log it to file and to stdout
154+
echo "$output" | tee -a $logfile
155+
}
156+
157+
# First, recurse into any provider dir
158+
for providerdir in $(ls -d $inputdir/*/); do
159+
provider=$(basename $providerdir)
160+
161+
printf "\n**** BEGIN provider %s **** \n" $provider | tee -a $logfile
162+
163+
# process certs
164+
zip=${providerdir}$cmszip
165+
if ! [ -f $zip ]; then
166+
continue
167+
fi
168+
unzipdir=${providerdir}$certsdir
169+
printf "\nUnzipping %s to %s\n" $zip $unzipdir | tee -a $logfile
170+
unzip -o $zip -d $unzipdir
171+
172+
resultsfile=${outputdir}/${provider}_openssl.csv
173+
echo "key_algorithm_oid,test_result" > $resultsfile # CSV header row
174+
175+
alreadyTestedSigOIDs=";" # for a guard to skip testing the same cert multiple times
176+
alreadyTestedKEMOIDs=";" # for a guard to skip testing the same cert multiple times
177+
# test each signed attributes fileartifact
178+
for signed_attrs_file in $(find $unzipdir \( -iname "*_signed_attrs.der" \)); do
179+
test_signed_attrs "$signed_attrs_file" "$resultsfile"
180+
done
181+
# test each KEMRI artifact
182+
for eefile in $(find $unzipdir \( -iname "*_ee.der" \)); do
183+
test_kemri "$eefile" "$resultsfile"
184+
done
185+
186+
printf "\n**** DONE provider %s **** \n" $provider | tee -a $logfile
187+
done
188+
189+
printf "\n**** DONE **** \n" | tee -a $logfile

0 commit comments

Comments
 (0)