Add support for RSA-PSS Keys

This is not as straightforward as it looked like initially and consists of
several changes that surface on different occasions:

 * The private RSA keys can have CKA_ALLOWED_MECHANISMS which can be used to
   determine if a key is generic key (can be used for any operation) or RSA-PSS
   key, so that it could be used only for any or specific RSA-PSS operations.

 * These keys get different identifier on OpenSSL level. They also get different
   OIDs on the ASN.1 in various places of the X.509 certificates:

   * The signatures in X.509 has a OID + parameters describing the hashes, mgf
     and salt length used. This is mandatory.

   * The public key encoding can contain the RSA-PSS restrictions. These are not
     mandatory so it allows us to indicate the key is restricted to PSS
     operations without the need to stick to specific combination of parameters.

 * When we force all operation in pkcs11 provider, the rsapss table was missing
   the match() callback, making key comparison fail when using RSA-PSS keys.

Given that the certtool we use for signing certificates during setup can not
distinguish RSA-PSS restricted keys and therefore generates unrestricted
certificates, we need to generate the certificate later using openssl, making
the tests a bit more ugly.

Signed-off-by: Jakub Jelen <[email protected]>

.reuse/dep5

@@ -30,6 +30,7 @@ Files: .github/*
        tests/*.pem
        tests/cert.json.in
        tests/cert.json.rsa.in
+       tests/cert.json.rsapss.in
        tests/cert.json.ecdsa.in
        tests/cert.json.ed25519.in
        tests/cert.json.ed448.in

src/encoder.c

@@ -251,12 +251,212 @@ static int p11prov_rsa_pubkey_to_der(P11PROV_OBJ *key, unsigned char **der,
     return RET_OSSL_OK;
 }
 
+/* allocate and set algorithm ID from given ptype */
+/* taken from OpenSSL */
+static X509_ALGOR *ossl_X509_ALGOR_from_nid(int nid, int ptype, void *pval)
+{
+    ASN1_OBJECT *algo = OBJ_nid2obj(nid);
+    X509_ALGOR *alg = NULL;
+
+    if (algo == NULL) {
+        return NULL;
+    }
+    if ((alg = X509_ALGOR_new()) == NULL) {
+        goto err;
+    }
+    if (X509_ALGOR_set0(alg, algo, ptype, pval)) {
+        return alg;
+    }
+    alg->algorithm = NULL; /* precaution to prevent double free */
+
+err:
+    X509_ALGOR_free(alg);
+    /* ASN1_OBJECT_free(algo) is not needed due to OBJ_nid2obj() */
+    return NULL;
+}
+
+/* allocate and set algorithm ID from EVP_MD, default SHA1 */
+/* taken from OpenSSL */
+static int ossl_x509_algor_new_from_md(X509_ALGOR **palg, const EVP_MD *md)
+{
+    X509_ALGOR *alg;
+
+    /* Default is SHA1 so no need to create it - still success */
+    if (md == NULL || EVP_MD_is_a(md, "SHA1")) {
+        return 1;
+    }
+    if ((alg = X509_ALGOR_new()) == NULL) {
+        return 0;
+    }
+    X509_ALGOR_set_md(alg, md);
+    *palg = alg;
+    return 1;
+}
+
+/* taken from OpenSSL */
+static int ossl_x509_algor_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
+{
+    X509_ALGOR *algtmp = NULL;
+    ASN1_STRING *stmp = NULL;
+
+    *palg = NULL;
+    if (mgf1md == NULL || EVP_MD_is_a(mgf1md, "SHA1")) {
+        return 1;
+    }
+    /* need to embed algorithm ID inside another */
+    if (!ossl_x509_algor_new_from_md(&algtmp, mgf1md)) {
+        goto err;
+    }
+    if (ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp) == NULL) {
+         goto err;
+    }
+    *palg = ossl_X509_ALGOR_from_nid(NID_mgf1, V_ASN1_SEQUENCE, stmp);
+    if (*palg == NULL) {
+        goto err;
+    }
+    stmp = NULL;
+err:
+    ASN1_STRING_free(stmp);
+    X509_ALGOR_free(algtmp);
+    return *palg != NULL;
+}
+
+RSA_PSS_PARAMS *p11prov_encode_rsa_pss_params(const EVP_MD *md,
+                                              const EVP_MD *mgf1_md,
+                                              int saltlen)
+{
+    RSA_PSS_PARAMS *pss = NULL;
+
+    pss = RSA_PSS_PARAMS_new();
+    if (pss == NULL) {
+        return NULL;
+    }
+
+    pss->saltLength = ASN1_INTEGER_new();
+    if (pss->saltLength == NULL) {
+        goto err;
+    }
+    if (!ASN1_INTEGER_set(pss->saltLength, saltlen)) {
+        goto err;
+    }
+
+    if (!ossl_x509_algor_new_from_md(&pss->hashAlgorithm, md)) {
+        goto err;
+    }
+
+    if (!ossl_x509_algor_md_to_mgf1(&pss->maskGenAlgorithm, mgf1_md)) {
+        goto err;
+    }
+
+    if (!ossl_x509_algor_new_from_md(&pss->maskHash, mgf1_md)) {
+        goto err;
+    }
+    /* the pss->trailerField has a default value 1 so it is optional and we do
+     * not need to include it in the generated ASN1 for public key restrictions
+     */
+    return pss;
+
+err:
+    RSA_PSS_PARAMS_free(pss);
+    return NULL;
+}
+
+/* RSA-PSS mechanism map, sorted by priority */
+struct rsa_pss_map {
+    CK_MECHANISM_TYPE pkcs11_mechanism;
+    const EVP_MD *(*hash_md)(void);
+} rsa_pss_map[P11PROV_N_RSAPSS_MECHS] = {
+    { CKM_RSA_PKCS_PSS, EVP_sha512 },
+    { CKM_SHA512_RSA_PKCS_PSS, EVP_sha512 },
+    { CKM_SHA384_RSA_PKCS_PSS, EVP_sha384 },
+    { CKM_SHA256_RSA_PKCS_PSS, EVP_sha256 },
+    { CKM_SHA224_RSA_PKCS_PSS, EVP_sha224 },
+    /* Technically OpenSSL will not encode the SHA3 these days so if we will get
+     * these on PKCS#11 level, we will just fall back to non-PSS Key encoding:
+     * https://tools.ietf.org/html/rfc8017#appendix-A.2.1
+     */
+    { CKM_SHA3_512_RSA_PKCS_PSS, EVP_sha3_512 },
+    { CKM_SHA3_384_RSA_PKCS_PSS, EVP_sha3_384 },
+    { CKM_SHA3_256_RSA_PKCS_PSS, EVP_sha3_256 },
+    { CKM_SHA3_224_RSA_PKCS_PSS, EVP_sha3_224 },
+    { CKM_SHA1_RSA_PKCS_PSS, EVP_sha1 },
+};
+
+/* The PKCS#11 defines CKA_ALLOWED_MECHANISMS listing all allowed mechanisms on
+ * given key.
+ * OTOH X509 define parameters that either allow the key operation with any
+ * parameters or only with one particular parameter combination
+ * (including hash, mgf, mgf1 hash, salt length and trailer field).
+ *
+ * In case we have only one hash algorithm, such as CKM_SHA256_RSA_PKCS_PSS, we
+ * can derive hash and default parameters.
+ *
+ * If there are more or there is a generic CKM_RSA_PKCS_PSS or all the PSS
+ * mechanisms, we need to generate unrestricted RSA-PSS mechanism
+ * (V_ASN1_UNDEF). Note, that this is possible to do only on the key itself and
+ * not on the signature!
+ */
+static ASN1_STRING *p11prov_encode_rsa_pss(P11PROV_OBJ *obj)
+{
+    CK_ATTRIBUTE *am = p11prov_obj_get_attr(obj, CKA_ALLOWED_MECHANISMS);
+    CK_MECHANISM_TYPE *allowed;
+    int am_nmechs, i, saltlen, nfound, first = -1;
+    RSA_PSS_PARAMS *pss = NULL;
+    const EVP_MD *md;
+    ASN1_STRING *pstr = NULL;
+
+    if (am == NULL || am->ulValueLen == 0) {
+        /* no limitations or no support for allowed mechs. Should not be
+         * reached.
+         * TODO we can try also certificate restrictions */
+        return NULL;
+    }
+    allowed = (CK_MECHANISM_TYPE *)am->pValue;
+    am_nmechs = am->ulValueLen / sizeof(CK_MECHANISM_TYPE);
+    for (i = 0; i < P11PROV_N_RSAPSS_MECHS; i++) {
+        bool found = false;
+        for (int j = 0; j < am_nmechs; j++) {
+            if (rsa_pss_map[i].pkcs11_mechanism == allowed[j]) {
+                found = true;
+                break;
+            }
+        }
+        if (found) {
+            nfound++;
+            if (first == -1) {
+                first = i;
+            }
+        }
+    }
+    if (i == P11PROV_N_RSAPSS_MECHS) {
+        /* no RSA-PSS mechanism -- should not be reached */
+        return NULL;
+    }
+
+    if (nfound > 1 || rsa_pss_map[first].pkcs11_mechanism == CKM_RSA_PKCS_PSS) {
+        /* no restrictions -- we can not better express limitation on multiple
+         * mechanisms or this generic PSS mechanism */
+        return NULL;
+    }
+
+    md = rsa_pss_map[i].hash_md();
+    saltlen = EVP_MD_get_size(md);
+    pss = p11prov_encode_rsa_pss_params(md, md, saltlen);
+    if (ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &pstr) == NULL) {
+        RSA_PSS_PARAMS_free(pss);
+        return NULL;
+    }
+    RSA_PSS_PARAMS_free(pss);
+    return pstr;
+}
+
 static X509_PUBKEY *p11prov_rsa_pubkey_to_x509(P11PROV_OBJ *key)
 {
     X509_PUBKEY *pubkey;
     unsigned char *der = NULL;
     int derlen = 0;
-    int ret;
+    int ret, nid, ptype;
+    ASN1_STRING *pval = NULL;
 
     ret = p11prov_rsa_pubkey_to_der(key, &der, &derlen);
     if (ret != RET_OSSL_OK) {
@@ -269,8 +469,23 @@ static X509_PUBKEY *p11prov_rsa_pubkey_to_x509(P11PROV_OBJ *key)
         return NULL;
     }
 
-    ret = X509_PUBKEY_set0_param(pubkey, OBJ_nid2obj(NID_rsaEncryption),
-                                 V_ASN1_NULL, NULL, der, derlen);
+    if (p11prov_obj_is_rsa_pss(key)) {
+        nid = NID_rsassaPss;
+        pval = p11prov_encode_rsa_pss(key);
+        if (pval != NULL) {
+            /* This is RSA-PSS key with restrictions */
+            ptype = V_ASN1_SEQUENCE;
+        } else {
+            /* This is RSA-PSS key without additional restrictions */
+            ptype = V_ASN1_UNDEF;
+        }
+    } else {
+        /* this is generic RSA key without restrictions */
+        nid = NID_rsaEncryption;
+        ptype = V_ASN1_NULL;
+    }
+    ret = X509_PUBKEY_set0_param(pubkey, OBJ_nid2obj(nid), ptype, pval, der,
+                                 derlen);
     if (ret != RET_OSSL_OK) {
         OPENSSL_free(der);
         X509_PUBKEY_free(pubkey);

src/encoder.h

@@ -42,4 +42,7 @@ extern const OSSL_DISPATCH
     p11prov_ec_edwards_encoder_priv_key_info_pem_functions[];
 extern const OSSL_DISPATCH p11prov_ec_edwards_encoder_text_functions[];
 
+RSA_PSS_PARAMS *p11prov_encode_rsa_pss_params(const EVP_MD *md,
+                                              const EVP_MD *mgf1_md,
+                                              int saltlen);
 #endif /* _ENCODER_H */

src/keymgmt.c

@@ -131,6 +131,14 @@ struct key_generator {
     void *cb_arg;
 };
 
+const CK_MECHANISM_TYPE p11prov_rsapss_mechs[P11PROV_N_RSAPSS_MECHS] = {
+    CKM_SHA1_RSA_PKCS_PSS,     CKM_SHA224_RSA_PKCS_PSS,
+    CKM_SHA256_RSA_PKCS_PSS,   CKM_SHA384_RSA_PKCS_PSS,
+    CKM_SHA512_RSA_PKCS_PSS,   CKM_SHA3_224_RSA_PKCS_PSS,
+    CKM_SHA3_256_RSA_PKCS_PSS, CKM_SHA3_384_RSA_PKCS_PSS,
+    CKM_SHA3_512_RSA_PKCS_PSS, CKM_RSA_PKCS_PSS
+};
+
 static int p11prov_common_gen_set_params(void *genctx,
                                          const OSSL_PARAM params[])
 {
@@ -784,7 +792,7 @@ static int p11prov_rsa_export(void *keydata, int selection,
     P11PROV_CTX *ctx = p11prov_obj_get_prov_ctx(key);
     CK_OBJECT_CLASS class = p11prov_obj_get_class(key);
 
-    P11PROV_debug("rsa export %p", keydata);
+    P11PROV_debug("rsa export %p, selection= %d", keydata, selection);
 
     if (key == NULL) {
         return RET_OSSL_ERR;
@@ -998,21 +1006,14 @@ DISPATCH_KEYMGMT_FN(rsa, gen_settable_params);
 
 static CK_RV set_default_rsapss_mechanisms(struct key_generator *ctx)
 {
-    CK_MECHANISM_TYPE rsapss_mechs[] = {
-        CKM_SHA1_RSA_PKCS_PSS,     CKM_SHA224_RSA_PKCS_PSS,
-        CKM_SHA256_RSA_PKCS_PSS,   CKM_SHA384_RSA_PKCS_PSS,
-        CKM_SHA512_RSA_PKCS_PSS,   CKM_SHA3_224_RSA_PKCS_PSS,
-        CKM_SHA3_256_RSA_PKCS_PSS, CKM_SHA3_384_RSA_PKCS_PSS,
-        CKM_SHA3_512_RSA_PKCS_PSS, CKM_RSA_PKCS_PSS
-    };
-
-    ctx->data.rsa.allowed_types = OPENSSL_malloc(sizeof(rsapss_mechs));
+    size_t mechs_size = sizeof(CK_MECHANISM_TYPE) * P11PROV_N_RSAPSS_MECHS;
+    ctx->data.rsa.allowed_types = OPENSSL_malloc(mechs_size);
     if (ctx->data.rsa.allowed_types == NULL) {
         P11PROV_raise(ctx->provctx, CKR_HOST_MEMORY, "Allocating data");
         return CKR_HOST_MEMORY;
     }
-    memcpy(ctx->data.rsa.allowed_types, rsapss_mechs, sizeof(rsapss_mechs));
-    ctx->data.rsa.allowed_types_size = sizeof(rsapss_mechs);
+    memcpy(ctx->data.rsa.allowed_types, p11prov_rsapss_mechs, mechs_size);
+    ctx->data.rsa.allowed_types_size = mechs_size;
 
     return CKR_OK;
 }
@@ -1194,6 +1195,7 @@ const OSSL_DISPATCH p11prov_rsapss_keymgmt_functions[] = {
     DISPATCH_KEYMGMT_ELEM(rsa, LOAD, load),
     DISPATCH_KEYMGMT_ELEM(rsa, FREE, free),
     DISPATCH_KEYMGMT_ELEM(rsa, HAS, has),
+    DISPATCH_KEYMGMT_ELEM(rsa, MATCH, match),
     DISPATCH_KEYMGMT_ELEM(rsa, IMPORT, import),
     DISPATCH_KEYMGMT_ELEM(rsa, IMPORT_TYPES, import_types),
     DISPATCH_KEYMGMT_ELEM(rsa, EXPORT, export),

src/keymgmt.h

@@ -5,6 +5,10 @@
 #define _KEYMGMT_H
 
 /* keymgmt */
+
+#define P11PROV_N_RSAPSS_MECHS 10
+extern const CK_MECHANISM_TYPE p11prov_rsapss_mechs[P11PROV_N_RSAPSS_MECHS];
+
 #define DISPATCH_KEYMGMT_FN(type, name) \
     DECL_DISPATCH_FUNC(keymgmt, p11prov_##type, name)
 #define DISPATCH_KEYMGMT_ELEM(type, NAME, name) \