xmlsec1-1.3.5.legacy.patch

--- xmlsec1-1.3.7.orig/src/openssl/signatures.c	2025-02-11 16:33:03
+++ xmlsec1-1.3.7/src/openssl/signatures.c	2025-02-12 15:32:02
@@ -35,6 +35,7 @@
 #ifdef XMLSEC_OPENSSL_API_300
 #include <openssl/core_names.h>
 #endif /* XMLSEC_OPENSSL_API_300 */
+#include <openssl/x509.h>
 
 #include "../cast_helpers.h"
 #include "openssl_compat.h"
@@ -921,15 +922,22 @@
                 "ret=%d", ret);
             goto error;
         }
+        ret = EVP_PKEY_CTX_set_signature_md(pKeyCtx, ctx->digest);
     } else {
-        ret = EVP_PKEY_verify_init(pKeyCtx);
+        if((ctx->mode == xmlSecOpenSSLEvpSignatureMode_RsaPadding) && (ctx->rsaPadding == RSA_PKCS1_PADDING)) {
+            ret = EVP_PKEY_verify_recover_init(pKeyCtx);
+        } else {
+            ret = EVP_PKEY_verify_init(pKeyCtx);
+        }
         if(ret <= 0) {
             xmlSecOpenSSLError2("EVP_PKEY_verify_init", xmlSecTransformGetName(transform),
                 "ret=%d", ret);
             goto error;
         }
+        if((ctx->mode != xmlSecOpenSSLEvpSignatureMode_RsaPadding) || (ctx->rsaPadding != RSA_PKCS1_PADDING)) {
+            ret = EVP_PKEY_CTX_set_signature_md(pKeyCtx, ctx->digest);
+        }
     }
-    ret = EVP_PKEY_CTX_set_signature_md(pKeyCtx, ctx->digest);
     if(ret <= 0) {
         xmlSecOpenSSLError2("EVP_PKEY_CTX_set_signature_md", xmlSecTransformGetName(transform),
             "ret=%d", ret);
@@ -983,6 +991,9 @@
     xmlSecByte dgst[EVP_MAX_MD_SIZE];
     unsigned int dgstSize = sizeof(dgst);
     EVP_PKEY_CTX *pKeyCtx = NULL;
+    unsigned char * recvData = NULL;
+    size_t recvDataLen = 0;
+    const unsigned char * recvDataPtr;
 #if !defined(XMLSEC_NO_DSA) || !defined(XMLSEC_NO_EC)
     unsigned char * fixedData = NULL;
     int fixedDataLen = 0;
@@ -990,6 +1001,9 @@
     unsigned int dataLen;
     int ret;
     int res = -1;
+    X509_SIG * sig = NULL;
+    const X509_ALGOR *algor = NULL;
+    const ASN1_OCTET_STRING *value = NULL;
 
     xmlSecAssert2(xmlSecOpenSSLEvpSignatureCheckId(transform), -1);
     xmlSecAssert2(transform->operation == xmlSecTransformOperationVerify, -1);
@@ -1021,8 +1035,51 @@
 
     switch(ctx->mode) {
     case xmlSecOpenSSLEvpSignatureMode_RsaPadding:
+        XMLSEC_SAFE_CAST_SIZE_TO_UINT(dataSize, dataLen, goto done, xmlSecTransformGetName(transform));
+        if(ctx->rsaPadding != RSA_PKCS1_PADDING) {
+            ret = EVP_PKEY_verify(pKeyCtx, (xmlSecByte*)data, dataLen, dgst, dgstSize);
+            break;
+        }
+
+        ret = EVP_PKEY_verify_recover(pKeyCtx, NULL, &recvDataLen, data, dataLen);
+        if((ret <= 0) || (recvDataLen <= 0)) {
+            xmlSecOpenSSLError("EVP_PKEY_verify_recover", xmlSecTransformGetName(transform));
+            goto done;
+        }
+
+        recvData = OPENSSL_malloc(recvDataLen);
+        ret = EVP_PKEY_verify_recover(pKeyCtx, recvData, &recvDataLen, data, dataLen);
+        if((ret <= 0) || (recvDataLen <= 0)) {
+            xmlSecOpenSSLError("EVP_PKEY_verify_recover", xmlSecTransformGetName(transform));
+            goto done;
+        }
+
+        recvDataPtr = recvData;
+        sig = d2i_X509_SIG(NULL, &recvDataPtr, (long)recvDataLen);
+        if(!sig) {
+            xmlSecOpenSSLError("d2i_X509_SIG", xmlSecTransformGetName(transform));
+            goto done;
+        }
+
+        X509_SIG_get0(sig, &algor, &value);
+        if((algor->parameter != NULL) && (ASN1_TYPE_get(algor->parameter) != V_ASN1_NULL)) {
+            xmlSecInternalError("Signature algorithm parameter type is not ASN1 NULL", xmlSecTransformGetName(transform));
+            goto done;
+        }
+
+        if(EVP_MD_nid(ctx->digest) != OBJ_obj2nid(algor->algorithm)) {
+            xmlSecInternalError("Signature digest method does not match excpected digest method", xmlSecTransformGetName(transform));
+            goto done;
+        }
+
+        if(((const unsigned int)value->length == dgstSize) && (memcmp(value->data, dgst, dgstSize) == 0)) {
+            ret = 1;
+        } else {
+            ret = 0;
+        }
+        break;
+
     case xmlSecOpenSSLEvpSignatureMode_Gost:
-        /* simple RSA or GOST padding */
         XMLSEC_SAFE_CAST_SIZE_TO_UINT(dataSize, dataLen, goto done, xmlSecTransformGetName(transform));
         ret = EVP_PKEY_verify(pKeyCtx, (xmlSecByte*)data, dataLen, dgst, dgstSize);
         break;
@@ -1077,11 +1134,17 @@
     res = 0;
 
 done:
+    if(sig != NULL) {
+        X509_SIG_free(sig);
+    }
 #if !defined(XMLSEC_NO_DSA) || !defined(XMLSEC_NO_EC)
     if(fixedData != NULL) {
         OPENSSL_free(fixedData);
     }
 #endif
+    if(recvData != NULL) {
+        OPENSSL_free(recvData);
+    }
     if(pKeyCtx != NULL) {
         EVP_PKEY_CTX_free(pKeyCtx);
     }