From 9768fe8adddadd00734a5518be5261b736b30b74 Mon Sep 17 00:00:00 2001
From: John Nunley <jnunley@marvell.com>
Date: Fri, 20 Dec 2024 06:47:45 +0000
Subject: [PATCH] feat: Expose operations for RSA flags

This commit adds the functions needed to manipulated the flags on the
RSA object. These flags are not used in normal SSL as far as I know, but
are used by custom providers/engines to change some functionality. The
functions I've added are as follows:

- RSA_test_flags
- RSA_set_flags
- RSA_clear_flags

Since these operations are not available on OpenSSL 1.0.2 or earlier,
I've also added shims that allow one to directly manipulate the "flags"
variable on these older versions.

This patch is made on behalf of Marvell Technology Inc.

Signed-off-by: John Nunley <jnunley@marvell.com>
---
 openssl-sys/src/handwritten/rsa.rs |  7 +++++
 openssl/src/rsa.rs                 | 43 +++++++++++++++++++++++++++++-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/openssl-sys/src/handwritten/rsa.rs b/openssl-sys/src/handwritten/rsa.rs
index d05edfc301..266488f122 100644
--- a/openssl-sys/src/handwritten/rsa.rs
+++ b/openssl-sys/src/handwritten/rsa.rs
@@ -45,6 +45,13 @@ extern "C" {
         iqmp: *mut *const BIGNUM,
     );
 
+    #[cfg(any(ossl110, libressl273))]
+    pub fn RSA_test_flags(r: *const RSA, flags: c_int) -> c_int;
+    #[cfg(any(ossl110, libressl273))]
+    pub fn RSA_set_flags(r: *mut RSA, flags: c_int);
+    #[cfg(any(ossl110, libressl273))]
+    pub fn RSA_clear_flags(r: *mut RSA, flags: c_int);
+
     #[cfg(not(ossl110))]
     pub fn RSA_generate_key(
         modsz: c_int,
diff --git a/openssl/src/rsa.rs b/openssl/src/rsa.rs
index 2e6614aed3..7c637973d4 100644
--- a/openssl/src/rsa.rs
+++ b/openssl/src/rsa.rs
@@ -91,6 +91,24 @@ impl<T> ToOwned for RsaRef<T> {
     }
 }
 
+impl<T> Rsa<T> {
+    /// Sets the RSA flags on the object.
+    #[corresponds(RSA_set_flags)]
+    pub fn set_flags(&mut self, flags: i32) {
+        unsafe {
+            RSA_set_flags(self.as_ptr(), flags);
+        }
+    }
+
+    /// Clears the RSA flags on the object.
+    #[corresponds(RSA_set_flags)]
+    pub fn clear_flags(&mut self, flags: i32) {
+        unsafe {
+            RSA_clear_flags(self.as_ptr(), flags);
+        }
+    }
+}
+
 impl<T> RsaRef<T>
 where
     T: HasPrivate,
@@ -366,6 +384,14 @@ where
             BigNumRef::from_const_ptr(e)
         }
     }
+
+    /// Tells if the provided set of RSA flags are set.
+    ///
+    /// This function returns the union of all flags that were set on the RSA object.
+    #[corresponds(RSA_test_flags)]
+    pub fn test_flags(&self, flags: i32) -> i32 {
+        unsafe { RSA_test_flags(self.as_ptr(), flags) }
+    }
 }
 
 impl Rsa<Public> {
@@ -588,7 +614,7 @@ cfg_if! {
     if #[cfg(any(ossl110, libressl273, boringssl))] {
         use ffi::{
             RSA_get0_key, RSA_get0_factors, RSA_get0_crt_params, RSA_set0_key, RSA_set0_factors,
-            RSA_set0_crt_params,
+            RSA_set0_crt_params, RSA_test_flags, RSA_set_flags, RSA_clear_flags
         };
     } else {
         #[allow(bad_style)]
@@ -677,6 +703,21 @@ cfg_if! {
             (*r).iqmp = iqmp;
             1
         }
+
+        #[allow(bad_style)]
+        unsafe fn RSA_test_flags(r: *const ffi::RSA, flags: c_int) -> c_int {
+            (*r).flags & flags
+        }
+
+        #[allow(bad_style)]
+        unsafe fn RSA_set_flags(r: *mut ffi::RSA, flags: c_int) {
+            (*r).flags |= flags;
+        }
+
+        #[allow(bad_style)]
+        unsafe fn RSA_clear_flags(r: *mut ffi::RSA, flags: c_int) {
+            (*r).flags &= !flags;
+        }
     }
 }