Adjust the SNI callback

Brings it more in line with how the raw callback is structured.

openssl-sys/src/lib.rs

@@ -1198,6 +1198,9 @@ pub const RSA_X931_PADDING: c_int = 5;
 
 pub const SHA_LBLOCK: c_int = 16;
 
+pub const TLS1_AD_UNRECOGNIZED_NAME: c_int = 112;
+pub const SSL_AD_UNRECOGNIZED_NAME: c_int = TLS1_AD_UNRECOGNIZED_NAME;
+
 pub const SSL_CTRL_SET_TMP_DH: c_int = 3;
 pub const SSL_CTRL_SET_TMP_ECDH: c_int = 4;
 pub const SSL_CTRL_EXTRA_CHAIN_CERT: c_int = 14;

openssl/src/ssl/callbacks.rs

@@ -11,7 +11,7 @@ use dh::Dh;
 #[cfg(any(all(feature = "v101", ossl101), all(feature = "v102", ossl102)))]
 use ec::EcKey;
 use pkey::Params;
-use ssl::{get_callback_idx, get_ssl_callback_idx, SniError, SslRef};
+use ssl::{get_callback_idx, get_ssl_callback_idx, SniError, SslAlert, SslRef};
 #[cfg(any(all(feature = "v102", ossl102), all(feature = "v110", ossl110)))]
 use ssl::AlpnError;
 use x509::X509StoreContextRef;
@@ -89,25 +89,20 @@ where
 
 pub extern "C" fn raw_sni<F>(ssl: *mut ffi::SSL, al: *mut c_int, _arg: *mut c_void) -> c_int
 where
-    F: Fn(&mut SslRef) -> Result<(), SniError> + 'static + Sync + Send,
+    F: Fn(&mut SslRef, &mut SslAlert) -> Result<(), SniError> + 'static + Sync + Send,
 {
     unsafe {
         let ssl_ctx = ffi::SSL_get_SSL_CTX(ssl);
         let callback = ffi::SSL_CTX_get_ex_data(ssl_ctx, get_callback_idx::<F>());
         let callback: &F = &*(callback as *mut F);
         let ssl = SslRef::from_ptr_mut(ssl);
+        let mut alert = SslAlert(*al);
 
-        match callback(ssl) {
+        let r = callback(ssl, &mut alert);
+        *al = alert.0;
+        match r {
             Ok(()) => ffi::SSL_TLSEXT_ERR_OK,
-            Err(SniError::Fatal(e)) => {
-                *al = e;
-                ffi::SSL_TLSEXT_ERR_ALERT_FATAL
-            }
-            Err(SniError::Warning(e)) => {
-                *al = e;
-                ffi::SSL_TLSEXT_ERR_ALERT_WARNING
-            }
-            Err(SniError::NoAck) => ffi::SSL_TLSEXT_ERR_NOACK,
+            Err(e) => e.0,
         }
     }
 }

openssl/src/ssl/mod.rs

@@ -428,18 +428,34 @@ fn get_new_ssl_idx<T>() -> c_int {
     }
 }
 
-// FIXME look into this
-/// An error returned from an SNI callback.
-pub enum SniError {
-    Fatal(c_int),
-    Warning(c_int),
-    NoAck,
+/// An error returned from the SNI callback.
+#[derive(Debug, Copy, Clone)]
+pub struct SniError(c_int);
+
+impl SniError {
+    /// Abort the handshake with a fatal alert.
+    pub const ALERT_FATAL: SniError = SniError(ffi::SSL_TLSEXT_ERR_ALERT_FATAL);
+
+    /// Send a warning alert to the client and continue the handshake.
+    pub const ALERT_WARNING: SniError = SniError(ffi::SSL_TLSEXT_ERR_ALERT_WARNING);
+
+    pub const NOACK: SniError = SniError(ffi::SSL_TLSEXT_ERR_NOACK);
+}
+
+/// An SSL/TLS alert.
+#[derive(Debug, Copy, Clone)]
+pub struct SslAlert(c_int);
+
+impl SslAlert {
+    /// Alert 112 - `unrecognized_name`.
+    pub const UNRECOGNIZED_NAME: SslAlert = SslAlert(ffi::SSL_AD_UNRECOGNIZED_NAME);
 }
 
 /// An error returned from an ALPN selection callback.
 ///
 /// Requires the `v102` or `v110` features and OpenSSL 1.0.2 or OpenSSL 1.1.0.
 #[cfg(any(all(feature = "v102", ossl102), all(feature = "v110", ossl110)))]
+#[derive(Debug, Copy, Clone)]
 pub struct AlpnError(c_int);
 
 #[cfg(any(all(feature = "v102", ossl102), all(feature = "v110", ossl110)))]
@@ -564,7 +580,7 @@ impl SslContextBuilder {
     /// [`SSL_CTX_set_tlsext_servername_callback`]: https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_tlsext_servername_callback.html
     pub fn set_servername_callback<F>(&mut self, callback: F)
     where
-        F: Fn(&mut SslRef) -> Result<(), SniError> + 'static + Sync + Send,
+        F: Fn(&mut SslRef, &mut SslAlert) -> Result<(), SniError> + 'static + Sync + Send,
     {
         unsafe {
             let callback = Box::new(callback);