[rsa] Add PSS signing

- Add RsaSignatureScheme trait, and an RsaPss implementation
- Add RsaSignature, parametric on bit size (RsaKeyBits), signature
  scheme (RsaSignatureScheme), and hash (Hasher)
- Modify existing tests to exercise signing and verification
- Add tests specific to signing and verification

Closes #4

Change-Id: I15db6ba0ddab0f49037ef72e9214ac0026fc42e7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfc4dc0..7aa65bb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,10 +12,17 @@
 
 ## [Unreleased]
 
+### Added
+- Added `public::rsa` module which supports RSA-PSS signing.
+
 ### Changed
 - In the `public` module, functions to parse and marshal DER-encoded
   public/private keys have been moved from bare functions to methods on the
   `DerPublicKey` and `DerPrivateKey` traits.
 - In the `public::ec` module, functions to parse and marshal DER-encoded
   public/private keys as the `EcPubKeyAnyCurve` and `EcPrivKeyAnyCurve` types
-  have been moved from bare functions to methods on those types.
\ No newline at end of file
+  have been moved from bare functions to methods on those types.
+- The `public::Signature::verify` method has been renamed to `is_valid` to make
+  the meaning of its return value more self-evident.
+- The `public::ec` module added experimental support for ECDSA-SHA512 under the
+  `experimental-sha512-ec` feature.
diff --git a/boringssl/bindgen.sh b/boringssl/bindgen.sh
index a8262b8..23fb135 100755
--- a/boringssl/bindgen.sh
+++ b/boringssl/bindgen.sh
@@ -98,11 +98,9 @@
 RSA_marshal_private_key|\
 RSA_new|\
 RSA_parse_private_key|\
-RSA_sign|\
 RSA_sign_pss_mgf1|\
 RSA_size|\
 RSA_up_ref|\
-RSA_verify|\
 RSA_verify_pss_mgf1|\
 SHA1_Init|\
 SHA1_Update|\
diff --git a/boringssl/boringssl.rs b/boringssl/boringssl.rs
index ddf4fc8..fa23ea6 100644
--- a/boringssl/boringssl.rs
+++ b/boringssl/boringssl.rs
@@ -1014,17 +1014,6 @@
     ) -> ::std::os::raw::c_int;
 }
 extern "C" {
-    #[link_name = "__RUST_MUNDANE_0_2_2_RSA_sign"]
-    pub fn RSA_sign(
-        hash_nid: ::std::os::raw::c_int,
-        in_: *const u8,
-        in_len: ::std::os::raw::c_uint,
-        out: *mut u8,
-        out_len: *mut ::std::os::raw::c_uint,
-        rsa: *mut RSA,
-    ) -> ::std::os::raw::c_int;
-}
-extern "C" {
     #[link_name = "__RUST_MUNDANE_0_2_2_RSA_sign_pss_mgf1"]
     pub fn RSA_sign_pss_mgf1(
         rsa: *mut RSA,
@@ -1039,17 +1028,6 @@
     ) -> ::std::os::raw::c_int;
 }
 extern "C" {
-    #[link_name = "__RUST_MUNDANE_0_2_2_RSA_verify"]
-    pub fn RSA_verify(
-        hash_nid: ::std::os::raw::c_int,
-        msg: *const u8,
-        msg_len: usize,
-        sig: *const u8,
-        sig_len: usize,
-        rsa: *mut RSA,
-    ) -> ::std::os::raw::c_int;
-}
-extern "C" {
     #[link_name = "__RUST_MUNDANE_0_2_2_RSA_verify_pss_mgf1"]
     pub fn RSA_verify_pss_mgf1(
         rsa: *mut RSA,
diff --git a/build.rs b/build.rs
index 1af1bd1..dfc95b4 100644
--- a/build.rs
+++ b/build.rs
@@ -109,11 +109,7 @@
 
     build(
         &abs_build_dir_2,
-        &[
-            &abs_boringssl_src,
-            &cmake_version_flag,
-            "-DBORINGSSL_PREFIX_SYMBOLS=../symbols.txt",
-        ],
+        &[&abs_boringssl_src, &cmake_version_flag, "-DBORINGSSL_PREFIX_SYMBOLS=../symbols.txt"],
     );
 
     // NOTE(joshlf): We symlink rather than renaming so that the BoringSSL build
@@ -233,11 +229,7 @@
 // `have` checks whether `name` is installed by running it with the provided
 // `args`. It must exist successfully.
 fn have(name: &str, args: &[&str]) -> bool {
-    Command::new(name)
-        .args(args)
-        .output()
-        .map(|output| output.status.success())
-        .unwrap_or(false)
+    Command::new(name).args(args).output().map(|output| output.status.success()).unwrap_or(false)
 }
 
 enum BuildSystem {
@@ -248,9 +240,7 @@
 // Checks which build tool was used for the previous build.
 fn built_with(abs_dir: &str) -> Option<BuildSystem> {
     let is_file = |file| {
-        fs::metadata(format!("{}/{}", abs_dir, file))
-            .map(|meta| meta.is_file())
-            .unwrap_or(false)
+        fs::metadata(format!("{}/{}", abs_dir, file)).map(|meta| meta.is_file()).unwrap_or(false)
     };
     if is_file("build.ninja") {
         Some(BuildSystem::Ninja)
diff --git a/src/boringssl/mod.rs b/src/boringssl/mod.rs
index ce5af6a..70b6d12 100644
--- a/src/boringssl/mod.rs
+++ b/src/boringssl/mod.rs
@@ -110,7 +110,7 @@
     EVP_PKEY_assign_RSA, EVP_PKEY_get1_EC_KEY, EVP_PKEY_get1_RSA, EVP_marshal_public_key,
     EVP_parse_public_key, HMAC_CTX_init, HMAC_Final, HMAC_Init_ex, HMAC_Update, HMAC_size,
     RAND_bytes, RSA_bits, RSA_generate_key_ex, RSA_marshal_private_key, RSA_parse_private_key,
-    RSA_size, SHA384_Init,
+    RSA_sign_pss_mgf1, RSA_size, RSA_verify_pss_mgf1, SHA384_Init,
 };
 
 impl CStackWrapper<BIGNUM> {
@@ -612,6 +612,68 @@
     }
 }
 
+/// The `rsa_sign_pss_mgf1` function.
+#[must_use]
+pub fn rsa_sign_pss_mgf1(
+    key: &CHeapWrapper<RSA>,
+    sig: &mut [u8],
+    digest: &[u8],
+    md: &CRef<'static, EVP_MD>,
+    mgf1_md: Option<&CRef<'static, EVP_MD>>,
+    salt_len: c_int,
+) -> Result<usize, BoringError> {
+    unsafe {
+        let mut sig_len: usize = 0;
+        RSA_sign_pss_mgf1(
+            // RSA_sign_pss_mgf1 does not mutate its argument but, for
+            // backwards-compatibility reasons, continues to take a normal
+            // (non-const) pointer.
+            key.as_const() as *mut _,
+            &mut sig_len,
+            sig.as_mut_ptr(),
+            sig.len(),
+            digest.as_ptr(),
+            digest.len(),
+            md.as_const(),
+            mgf1_md.map(CRef::as_const).unwrap_or(ptr::null()),
+            salt_len,
+        )?;
+
+        // RSA_sign_pss_mgf1 guarantees that it only needs RSA_size bytes for
+        // the signature.
+        let rsa_size = key.rsa_size().unwrap_abort();
+        assert_abort!(sig_len <= rsa_size.get());
+        Ok(sig_len)
+    }
+}
+
+/// The `RSA_verify_pss_mgf1` function.
+#[must_use]
+pub fn rsa_verify_pss_mgf1(
+    key: &CHeapWrapper<RSA>,
+    digest: &[u8],
+    md: &CRef<'static, EVP_MD>,
+    mgf1_md: Option<&CRef<'static, EVP_MD>>,
+    salt_len: c_int,
+    sig: &[u8],
+) -> bool {
+    unsafe {
+        RSA_verify_pss_mgf1(
+            // RSA_verify_pss_mgf1 does not mutate its argument but, for
+            // backwards-compatibility reasons, continues to take a normal
+            // (non-const) pointer.
+            key.as_const() as *mut _,
+            digest.as_ptr(),
+            digest.len(),
+            md.as_const(),
+            mgf1_md.map(CRef::as_const).unwrap_or(ptr::null()),
+            salt_len,
+            sig.as_ptr(),
+            sig.len(),
+        )
+    }
+}
+
 /// Implements `CStackWrapper` for a hash context type.
 ///
 /// The caller provides doc comments, a public method name, and a private
diff --git a/src/boringssl/raw.rs b/src/boringssl/raw.rs
index 8f94ee7..eb0ac02 100644
--- a/src/boringssl/raw.rs
+++ b/src/boringssl/raw.rs
@@ -394,11 +394,50 @@
 
 #[allow(non_snake_case)]
 #[must_use]
+pub unsafe fn RSA_sign_pss_mgf1(
+    rsa: *mut RSA,
+    out_len: *mut usize,
+    out: *mut u8,
+    max_out: usize,
+    in_: *const u8,
+    in_len: usize,
+    md: *const EVP_MD,
+    mgf1_md: *const EVP_MD,
+    salt_len: c_int,
+) -> Result<(), BoringError> {
+    one_or_err(
+        "RSA_sign_pss_mgf1",
+        ffi::RSA_sign_pss_mgf1(rsa, out_len, out, max_out, in_, in_len, md, mgf1_md, salt_len),
+    )
+}
+
+#[allow(non_snake_case)]
+#[must_use]
 pub unsafe fn RSA_size(key: *const RSA) -> Result<NonZeroUsize, BoringError> {
     NonZeroUsize::new(ffi::RSA_size(key).try_into().unwrap_abort())
         .ok_or_else(|| BoringError::consume_stack("RSA_size"))
 }
 
+#[allow(non_snake_case)]
+#[must_use]
+pub unsafe fn RSA_verify_pss_mgf1(
+    rsa: *mut RSA,
+    msg: *const u8,
+    msg_len: usize,
+    md: *const EVP_MD,
+    mgf1_md: *const EVP_MD,
+    salt_len: c_int,
+    sig: *const u8,
+    sig_len: usize,
+) -> bool {
+    match ffi::RSA_verify_pss_mgf1(rsa, msg, msg_len, md, mgf1_md, salt_len, sig, sig_len) {
+        0 => false,
+        1 => true,
+        // RSA_verify_pss_mgf1 promises to only return 0 or 1
+        _ => unreachable_abort!(),
+    }
+}
+
 // sha.h
 
 #[allow(non_snake_case)]
diff --git a/src/public/rsa/mod.rs b/src/public/rsa/mod.rs
index d2f9a35..ed8b0d3 100644
--- a/src/public/rsa/mod.rs
+++ b/src/public/rsa/mod.rs
@@ -11,11 +11,13 @@
 pub use public::rsa::bits::{RsaKeyBits, B2048, B3072, B4096, B6144, B8192};
 
 use std::convert::TryInto;
-use std::fmt::{self, Debug, Formatter};
+use std::fmt::{self, Debug, Display, Formatter};
+use std::marker::PhantomData;
 
-use boringssl::{CHeapWrapper, CStackWrapper};
+use boringssl::{self, BoringError, CHeapWrapper, CStackWrapper};
+use hash::{inner::Digest, Hasher};
 use public::rsa::inner::RsaKey;
-use public::{inner::DerKey, DerPrivateKey, DerPublicKey, PrivateKey, PublicKey};
+use public::{inner::DerKey, DerPrivateKey, DerPublicKey, PrivateKey, PublicKey, Signature};
 use util::Sealed;
 use Error;
 
@@ -25,8 +27,10 @@
     use std::os::raw::c_uint;
 
     use boringssl::{self, BoringError, CHeapWrapper, CStackWrapper};
+    use hash::Hasher;
     use public::inner::BoringDerKey;
     use public::rsa::RsaKeyBits;
+    use util::Sealed;
     use Error;
 
     // A convenience wrapper around boringssl::RSA.
@@ -101,6 +105,15 @@
 
     impl<B: RsaKeyBits> RsaKeyBitsExt for B {}
 
+    pub trait RsaSignatureScheme: Sealed {
+        fn sign<B: RsaKeyBits, H: Hasher>(
+            rsa: &RsaKey<B>,
+            digest: &[u8],
+            sig: &mut [u8],
+        ) -> Result<usize, BoringError>;
+        fn verify<B: RsaKeyBits, H: Hasher>(rsa: &RsaKey<B>, digest: &[u8], sig: &[u8]) -> bool;
+    }
+
     #[cfg(test)]
     mod tests {
         use std::mem;
@@ -340,9 +353,148 @@
     }
 }
 
+/// An RSA signature scheme.
+///
+/// An `RsaSignatureScheme` defines how to compute an RSA signature. The primary
+/// detail defined by a signature scheme is how to perform padding.
+///
+/// `RsaSignatureScheme` is implemented by [`RsaPss`].
+pub trait RsaSignatureScheme:
+    Sized + Copy + Clone + Default + Display + Debug + self::inner::RsaSignatureScheme
+{
+}
+
+/// The RSA-PSS signature scheme.
+#[derive(Copy, Clone, Default, Debug, Eq, PartialEq, Hash)]
+pub struct RsaPss;
+
+impl Display for RsaPss {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+        write!(f, "RSA-PSS")
+    }
+}
+
+impl Sealed for RsaPss {}
+impl RsaSignatureScheme for RsaPss {}
+
+impl self::inner::RsaSignatureScheme for RsaPss {
+    fn sign<B: RsaKeyBits, H: Hasher>(
+        rsa: &RsaKey<B>,
+        digest: &[u8],
+        sig: &mut [u8],
+    ) -> Result<usize, BoringError> {
+        // This is not needed for memory safety, but is an important security
+        // and correctness check, as passing a too-short signature would result
+        // in the signature being truncated.
+        assert!(sig.len() >= rsa.key.rsa_size().unwrap().get());
+        // A salt_len of -1 means to use a salt of the same length as the hash
+        // output. This is a reasonable default and, for bit lengths larger than
+        // 2048, ensures that the salt will never need to be truncated.
+        boringssl::rsa_sign_pss_mgf1(&rsa.key, sig, digest, &H::evp_md(), None, -1)
+    }
+    fn verify<B: RsaKeyBits, H: Hasher>(rsa: &RsaKey<B>, digest: &[u8], sig: &[u8]) -> bool {
+        // A salt_len of -2 means to recover the salt length from the signature,
+        // and thus to tolerate any salt length.
+        boringssl::rsa_verify_pss_mgf1(&rsa.key, digest, &H::evp_md(), None, -2, sig)
+    }
+}
+
+// The maximum length of an RSA-8192 signature. Since this isn't exposed in the
+// API, we can increase later if we add support for larger bit sizes.
+const MAX_SIGNATURE_LEN: usize = 1024;
+
+/// An RSA signature.
+///
+/// `RsaSignature` is an RSA signature generated by keys of length `B`, using
+/// the signature scheme `S`, and the hash function `H`.
+pub struct RsaSignature<B: RsaKeyBits, S: RsaSignatureScheme, H: Hasher> {
+    bytes: [u8; MAX_SIGNATURE_LEN],
+    // Invariant: len is in [0; MAX_SIGNATURE_LEN). If len is 0, it indicates an
+    // invalid signature. Invalid signatures can be produced when a caller
+    // invokes from_bytes with a byte slice longer than MAX_SIGNATURE_LEN. Such
+    // signatures cannot possibly have been generated by an RSA signature for
+    // any of the key sizes or signature schemes we support, and so it could not
+    // possibly be valid. In other words, it would never be correct for
+    // rsa_verify to return true when invoked on such a signature.
+    //
+    // However, if we were to simply truncate the byte slice and store a subset
+    // of it, then we might open ourselves up to attacks in which an attacker
+    // induces a mismatch between the signature that the caller /thinks/ is
+    // being verified and the signature that is /actually/ being verified. Thus,
+    // it's important that we always reject such signatures.
+    //
+    // Finally, it's OK for us to use 0 as the sentinal value to mean "invalid
+    // signature" because RSA can never produce a 0-byte signature. Thus, we
+    // will never produce a 0-byte signature from rsa_sign, and similarly, if
+    // the caller constructs a 0-byte signature using from_bytes, it's correct
+    // for us to treat it as invalid.
+    len: usize,
+    _marker: PhantomData<(B, S, H)>,
+}
+
+impl<B: RsaKeyBits, S: RsaSignatureScheme, H: Hasher> RsaSignature<B, S, H> {
+    /// Constructs an `RsaSignature` from raw bytes.
+    #[must_use]
+    pub fn from_bytes(bytes: &[u8]) -> RsaSignature<B, S, H> {
+        if bytes.len() > MAX_SIGNATURE_LEN {
+            // see comment on the len field for why we do this
+            return Self::empty();
+        }
+        let mut ret = Self::empty();
+        (&mut ret.bytes[..bytes.len()]).copy_from_slice(bytes);
+        ret.len = bytes.len();
+        ret
+    }
+
+    // TODO(joshlf): Once we have const generics, have this return a
+    // fixed-length array.
+
+    /// Gets the raw bytes of this `RsaSignature`.
+    #[must_use]
+    pub fn bytes(&self) -> &[u8] {
+        &self.bytes[..self.len]
+    }
+
+    fn is_valid_format(&self) -> bool {
+        self.len != 0
+    }
+
+    fn empty() -> RsaSignature<B, S, H> {
+        RsaSignature { bytes: [0u8; MAX_SIGNATURE_LEN], len: 0, _marker: PhantomData }
+    }
+}
+
+impl<B: RsaKeyBits, S: RsaSignatureScheme, H: Hasher> Sealed for RsaSignature<B, S, H> {}
+impl<B: RsaKeyBits, S: RsaSignatureScheme, H: Hasher> Signature for RsaSignature<B, S, H> {
+    type PrivateKey = RsaPrivKey<B>;
+
+    fn sign(key: &RsaPrivKey<B>, message: &[u8]) -> Result<RsaSignature<B, S, H>, Error> {
+        let digest = H::hash(message);
+        let mut sig = RsaSignature::empty();
+        sig.len = S::sign::<B, H>(&key.inner, digest.as_ref(), &mut sig.bytes[..])?;
+        Ok(sig)
+    }
+
+    fn is_valid(&self, key: &RsaPubKey<B>, message: &[u8]) -> bool {
+        if !self.is_valid_format() {
+            // see comment on RsaSignature::len for why we do this
+            return false;
+        }
+        let digest = H::hash(message);
+        S::verify::<B, H>(&key.inner, digest.as_ref(), self.bytes())
+    }
+}
+
+impl<B: RsaKeyBits, S: RsaSignatureScheme, H: Hasher> Debug for RsaSignature<B, S, H> {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+        write!(f, "RsaSignature")
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use hash::Sha256;
     use util::should_fail;
 
     #[test]
@@ -372,16 +524,35 @@
             unwrap_priv_any: F,
             unwrap_pub_any: G,
         ) {
+            const MESSAGE: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7];
             let key = RsaPrivKey::<B>::generate().unwrap();
 
-            let _ = RsaPrivKey::<B>::parse_from_der(&key.marshal_to_der()).unwrap();
-            let _ =
+            let parsed_key = RsaPrivKey::<B>::parse_from_der(&key.marshal_to_der()).unwrap();
+            let parsed_key_any_bits =
                 unwrap_priv_any(RsaPrivKeyAnyBits::parse_from_der(&key.marshal_to_der()).unwrap());
             let pubkey = key.public();
-            let _ = RsaPubKey::<B>::parse_from_der(&pubkey.marshal_to_der()).unwrap();
-            let _ =
+            let parsed_pubkey = RsaPubKey::<B>::parse_from_der(&pubkey.marshal_to_der()).unwrap();
+            let parsed_pubkey_any_bits =
                 unwrap_pub_any(RsaPubKeyAnyBits::parse_from_der(&pubkey.marshal_to_der()).unwrap());
 
+            fn sign_and_verify<B: RsaKeyBits>(privkey: &RsaPrivKey<B>, pubkey: &RsaPubKey<B>) {
+                let sig = RsaSignature::<B, RsaPss, Sha256>::sign(&privkey, MESSAGE).unwrap();
+                assert!(RsaSignature::<B, RsaPss, Sha256>::from_bytes(sig.bytes())
+                    .is_valid(&pubkey, MESSAGE));
+            }
+
+            // Sign and verify with every pair of keys to make sure we parsed
+            // the same key we marshaled.
+            sign_and_verify(&key, &pubkey);
+            sign_and_verify(&key, &parsed_pubkey);
+            sign_and_verify(&key, &parsed_pubkey_any_bits);
+            sign_and_verify(&parsed_key, &pubkey);
+            sign_and_verify(&parsed_key, &parsed_pubkey);
+            sign_and_verify(&parsed_key, &parsed_pubkey_any_bits);
+            sign_and_verify(&parsed_key_any_bits, &pubkey);
+            sign_and_verify(&parsed_key_any_bits, &parsed_pubkey);
+            sign_and_verify(&parsed_key_any_bits, &parsed_pubkey_any_bits);
+
             let _ = RsaPubKey::<B>::marshal_to_der;
             let _ = RsaPubKey::<B>::parse_from_der;
         }
@@ -537,4 +708,34 @@
         test_parse_wrong_bit_size::<B8192, B4096>();
         test_parse_wrong_bit_size::<B8192, B6144>();
     }
+
+    #[test]
+    fn test_signature_smoke() {
+        fn test<B: RsaKeyBits>() {
+            use public::testutil::test_signature_smoke;
+            let key = RsaPrivKey::<B>::generate().unwrap();
+            test_signature_smoke(
+                &key,
+                RsaSignature::<_, RsaPss, Sha256>::from_bytes,
+                RsaSignature::bytes,
+            );
+        }
+
+        test::<B2048>();
+        test::<B3072>();
+        test::<B4096>();
+        test::<B6144>();
+        test::<B8192>();
+    }
+
+    #[test]
+    fn test_invalid_signature() {
+        fn test_is_invalid<S: RsaSignatureScheme>(sig: &RsaSignature<B2048, S, Sha256>) {
+            assert_eq!(sig.len, 0);
+            assert!(!sig.is_valid_format());
+            assert!(!sig.is_valid(&RsaPrivKey::<B2048>::generate().unwrap().public(), &[],));
+        }
+        test_is_invalid::<RsaPss>(&RsaSignature::from_bytes(&[0; MAX_SIGNATURE_LEN + 1]));
+        test_is_invalid::<RsaPss>(&RsaSignature::from_bytes(&[]));
+    }
 }