From 07c7332b4a01a1ad53aa8ab9b3bc412d8ce8aa01 Mon Sep 17 00:00:00 2001
From: qmuntal <qmuntaldiaz@microsoft.com>
Date: Tue, 9 Apr 2024 16:00:26 +0200
Subject: [PATCH] set implicit rejection of invalid RSA PKCS#1 v1.5 padding

---
 evp.go      | 11 +++++++++++
 rsa_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 shims.h     |  3 ++-
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/evp.go b/evp.go
index a9237a6a..4cecc1a6 100644
--- a/evp.go
+++ b/evp.go
@@ -249,6 +249,17 @@ func setupEVP(withKey withKeyFunc, padding C.int,
 		}
 
 	case C.GO_RSA_PKCS1_PADDING:
+		if vMajor >= 3 {
+			// OpenSSL 3.2 changed the EVP_PKEY_decrypt behavior to not return an error
+			// when the padding is invalid. Instead, it returns a random value.
+			// See https://github.com/openssl/openssl/pull/13817.
+			// This is a security improvement, but it breaks compatibility with [rsa.DecryptPKCS1v15],
+			// which is documented to return an error when the padding is invalid.
+			// To maintain compatibility, we need to enable implicit rejection of invalid padding.
+			// Unconditionally enable implicit rejection of invalid padding, even in OpenSSL 3.0 and 3.1,
+			// as some distributions have backported this change. Ignore the error, is is an optional feature.
+			_ = C.go_openssl_EVP_PKEY_CTX_ctrl(ctx, C.GO_EVP_PKEY_RSA, -1, C.GO_EVP_PKEY_CTRL_RSA_IMPLICIT_REJECTION, 0, nil)
+		}
 		if ch != 0 {
 			// We support unhashed messages.
 			md := cryptoHashToMD(ch)
diff --git a/rsa_test.go b/rsa_test.go
index 1d2c2cab..328bdcca 100644
--- a/rsa_test.go
+++ b/rsa_test.go
@@ -4,8 +4,12 @@ import (
 	"bytes"
 	"crypto"
 	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
 	"math/big"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/golang-fips/openssl/v2"
@@ -264,3 +268,47 @@ func BenchmarkGenerateKeyRSA(b *testing.B) {
 		}
 	}
 }
+
+func TestOverlongMessagePKCS1v15(t *testing.T) {
+	priv := parseKey(rsaPrivateKey)
+	ciphertext := decodeBase64("fjOVdirUzFoLlukv80dBllMLjXythIf22feqPrNo0YoIjzyzyoMFiLjAc/Y4krkeZ11XFThIrEvw\nkRiZcCq5ng==")
+	_, err := openssl.DecryptRSAPKCS1(priv, ciphertext)
+	if err == nil {
+		t.Error("RSA decrypted a message that was too long.")
+	}
+}
+
+var rsaPrivateKey = testingKey(`-----BEGIN RSA TESTING KEY-----
+MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0
+fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu
+/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu
+RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/
+EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A
+IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS
+tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V
+-----END RSA TESTING KEY-----`)
+
+func decodeBase64(in string) []byte {
+	out := make([]byte, base64.StdEncoding.DecodedLen(len(in)))
+	n, err := base64.StdEncoding.Decode(out, []byte(in))
+	if err != nil {
+		return nil
+	}
+	return out[0:n]
+}
+
+func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") }
+
+func parseKey(s string) *openssl.PrivateKeyRSA {
+	p, _ := pem.Decode([]byte(s))
+	var err error
+	k, err := x509.ParsePKCS1PrivateKey(p.Bytes)
+	if err != nil {
+		panic(err)
+	}
+	key, err := openssl.NewPrivateKeyRSA(bbig.Enc(k.N), bbig.Enc(big.NewInt(int64(k.E))), bbig.Enc(k.D), bbig.Enc(k.Primes[0]), bbig.Enc(k.Primes[1]), nil, nil, nil)
+	if err != nil {
+		panic(err)
+	}
+	return key
+}
diff --git a/shims.h b/shims.h
index 99656f0c..bf8839a2 100644
--- a/shims.h
+++ b/shims.h
@@ -76,7 +76,8 @@ enum {
     GO_EVP_PKEY_CTRL_RSA_KEYGEN_BITS = 0x1003,
     GO_EVP_PKEY_CTRL_RSA_MGF1_MD = 0x1005,
     GO_EVP_PKEY_CTRL_RSA_OAEP_MD = 0x1009,
-    GO_EVP_PKEY_CTRL_RSA_OAEP_LABEL = 0x100A
+    GO_EVP_PKEY_CTRL_RSA_OAEP_LABEL = 0x100A,
+    GO_EVP_PKEY_CTRL_RSA_IMPLICIT_REJECTION = 0x100E
 };
 
 typedef void* GO_OPENSSL_INIT_SETTINGS_PTR;