RSA: Add helper functions to reduce duplication

Setting up operation pairs largely set the same arguments in both ops, so introduce helper functions to reduce duplication and avoid them getting out of sync. Signed-off-by: Simo Sorce <[email protected]>
latchset · Dec 11, 2024 · cae63cc · cae63cc
1 parent f9e4082
commit cae63cc
Showing 1 changed file with 52 additions and 91 deletions.
diff --git a/src/ossl/rsa.rs b/src/ossl/rsa.rs
@@ -228,6 +228,17 @@ impl RsaPKCSOperation {
         }
     }
 
+    fn get_key_size(key: &Object, info: &CK_MECHANISM_INFO) -> Result<usize> {
+        let modulus = key.get_attr_as_bytes(CKA_MODULUS)?;
+        let modulus_bits: CK_ULONG = modulus.len() as CK_ULONG * 8;
+        if modulus_bits < info.ulMinKeySize
+            || (info.ulMaxKeySize != 0 && modulus_bits > info.ulMaxKeySize)
+        {
+            return Err(CKR_KEY_SIZE_RANGE)?;
+        }
+        Ok(modulus.len())
+    }
+
     fn max_message_len(
         modulus: usize,
         mech: CK_MECHANISM_TYPE,
@@ -244,30 +255,23 @@ impl RsaPKCSOperation {
         }
     }
 
-    pub fn encrypt_new(
+    fn encdec_new(
         mech: &CK_MECHANISM,
-        key: &Object,
-        info: &CK_MECHANISM_INFO,
+        pubkey: Option<EvpPkey>,
+        privkey: Option<EvpPkey>,
+        keysize: usize,
     ) -> Result<RsaPKCSOperation> {
-        let modulus = key.get_attr_as_bytes(CKA_MODULUS)?;
-        let modulus_bits: CK_ULONG = modulus.len() as CK_ULONG * 8;
-        if modulus_bits < info.ulMinKeySize
-            || (info.ulMaxKeySize != 0 && modulus_bits > info.ulMaxKeySize)
-        {
-            return Err(CKR_KEY_SIZE_RANGE)?;
-        }
         let oaep_params = parse_oaep_params(mech)?;
-        let pubkey = EvpPkey::pubkey_from_object(key)?;
         Ok(RsaPKCSOperation {
             mech: mech.mechanism,
             max_input: Self::max_message_len(
-                modulus.len(),
+                keysize,
                 mech.mechanism,
                 oaep_params.hash,
             )?,
-            output_len: modulus.len(),
-            public_key: Some(pubkey),
-            private_key: None,
+            output_len: keysize,
+            public_key: pubkey,
+            private_key: privkey,
             finalized: false,
             in_use: false,
             sigctx: None,
@@ -278,68 +282,45 @@ impl RsaPKCSOperation {
         })
     }
 
-    pub fn decrypt_new(
+    pub fn encrypt_new(
         mech: &CK_MECHANISM,
         key: &Object,
         info: &CK_MECHANISM_INFO,
     ) -> Result<RsaPKCSOperation> {
-        let modulus = key.get_attr_as_bytes(CKA_MODULUS)?;
-        let modulus_bits: CK_ULONG = modulus.len() as CK_ULONG * 8;
-        if modulus_bits < info.ulMinKeySize
-            || (info.ulMaxKeySize != 0 && modulus_bits > info.ulMaxKeySize)
-        {
-            return Err(CKR_KEY_SIZE_RANGE)?;
-        }
-        let oaep_params = parse_oaep_params(mech)?;
+        let keysize = Self::get_key_size(key, info)?;
         let pubkey = EvpPkey::pubkey_from_object(key)?;
-        let privkey = EvpPkey::privkey_from_object(key)?;
-        Ok(RsaPKCSOperation {
-            mech: mech.mechanism,
-            max_input: modulus.len(),
-            output_len: Self::max_message_len(
-                modulus.len(),
-                mech.mechanism,
-                oaep_params.hash,
-            )?,
-            public_key: Some(pubkey),
-            private_key: Some(privkey),
-            finalized: false,
-            in_use: false,
-            sigctx: None,
-            pss: no_pss_params(),
-            oaep: oaep_params,
-            #[cfg(feature = "fips")]
-            fips_approved: None,
-        })
+        Self::encdec_new(mech, Some(pubkey), None, keysize)
     }
 
-    pub fn sign_new(
+    pub fn decrypt_new(
         mech: &CK_MECHANISM,
         key: &Object,
         info: &CK_MECHANISM_INFO,
     ) -> Result<RsaPKCSOperation> {
-        let modulus = key.get_attr_as_bytes(CKA_MODULUS)?;
-        let modulus_bits: CK_ULONG = modulus.len() as CK_ULONG * 8;
-        if modulus_bits < info.ulMinKeySize
-            || (info.ulMaxKeySize != 0 && modulus_bits > info.ulMaxKeySize)
-        {
-            return Err(CKR_KEY_SIZE_RANGE)?;
-        }
-
-        let pss_params = parse_pss_params(mech)?;
+        let keysize = Self::get_key_size(key, info)?;
         let pubkey = EvpPkey::pubkey_from_object(key)?;
         let privkey = EvpPkey::privkey_from_object(key)?;
+        Self::encdec_new(mech, Some(pubkey), Some(privkey), keysize)
+    }
+
+    fn sigver_new(
+        mech: &CK_MECHANISM,
+        pubkey: Option<EvpPkey>,
+        privkey: Option<EvpPkey>,
+        keysize: usize,
+    ) -> Result<RsaPKCSOperation> {
+        let pss_params = parse_pss_params(mech)?;
         Ok(RsaPKCSOperation {
             mech: mech.mechanism,
             max_input: match mech.mechanism {
-                CKM_RSA_X_509 => modulus.len(),
-                CKM_RSA_PKCS => modulus.len() - 11,
+                CKM_RSA_X_509 => keysize,
+                CKM_RSA_PKCS => keysize - 11,
                 CKM_RSA_PKCS_PSS => Self::hash_len(pss_params.hash)?,
                 _ => 0,
             },
-            output_len: modulus.len(),
-            public_key: Some(pubkey),
-            private_key: Some(privkey),
+            output_len: keysize,
+            public_key: pubkey,
+            private_key: privkey,
             finalized: false,
             in_use: false,
             sigctx: match mech.mechanism {
@@ -356,45 +337,25 @@ impl RsaPKCSOperation {
         })
     }
 
-    pub fn verify_new(
+    pub fn sign_new(
         mech: &CK_MECHANISM,
         key: &Object,
         info: &CK_MECHANISM_INFO,
     ) -> Result<RsaPKCSOperation> {
-        let modulus = key.get_attr_as_bytes(CKA_MODULUS)?;
-        let modulus_bits: CK_ULONG = modulus.len() as CK_ULONG * 8;
-        if modulus_bits < info.ulMinKeySize
-            || (info.ulMaxKeySize != 0 && modulus_bits > info.ulMaxKeySize)
-        {
-            return Err(CKR_KEY_SIZE_RANGE)?;
-        }
+        let keysize = Self::get_key_size(key, info)?;
+        let pubkey = EvpPkey::pubkey_from_object(key)?;
+        let privkey = EvpPkey::privkey_from_object(key)?;
+        Self::sigver_new(mech, Some(pubkey), Some(privkey), keysize)
+    }
 
-        let pss_params = parse_pss_params(mech)?;
+    pub fn verify_new(
+        mech: &CK_MECHANISM,
+        key: &Object,
+        info: &CK_MECHANISM_INFO,
+    ) -> Result<RsaPKCSOperation> {
+        let keysize = Self::get_key_size(key, info)?;
         let pubkey = EvpPkey::pubkey_from_object(key)?;
-        Ok(RsaPKCSOperation {
-            mech: mech.mechanism,
-            max_input: match mech.mechanism {
-                CKM_RSA_X_509 => modulus.len(),
-                CKM_RSA_PKCS => modulus.len() - 11,
-                _ => 0,
-            },
-            output_len: modulus.len(),
-            public_key: Some(pubkey),
-            private_key: None,
-            finalized: false,
-            in_use: false,
-            sigctx: match mech.mechanism {
-                CKM_RSA_X_509 | CKM_RSA_PKCS => None,
-                #[cfg(feature = "fips")]
-                _ => Some(ProviderSignatureCtx::new(name_as_char(RSA_NAME))?),
-                #[cfg(not(feature = "fips"))]
-                _ => Some(EvpMdCtx::new()?),
-            },
-            pss: pss_params,
-            oaep: no_oaep_params(),
-            #[cfg(feature = "fips")]
-            fips_approved: None,
-        })
+        Self::sigver_new(mech, Some(pubkey), None, keysize)
     }
 
     pub fn generate_keypair(