Merge pull request #963 from alexcrichton/vendored

Support builds of OpenSSL from vendored source (take 2)

.circleci/config.yml

@@ -1,9 +1,9 @@
 restore_registry: &RESTORE_REGISTRY
   restore_cache:
-    key: registry-3
+    key: registry-4
 save_registry: &SAVE_REGISTRY
   save_cache:
-    key: registry-3-{{ .BuildNum }}
+    key: registry-4-{{ .BuildNum }}
     paths:
     - /usr/local/cargo/registry/index
 openssl_key: &OPENSSL_KEY
@@ -38,14 +38,14 @@ job: &JOB
   - run: apt-get remove -y libssl-dev
   - run: ./test/add_target.sh
   - *RESTORE_REGISTRY
-  - run: cargo generate-lockfile
+  - run: cargo generate-lockfile --verbose
   - *SAVE_REGISTRY
   - run: echo "${LIBRARY}-${VERSION}-${TARGET}" > ~/lib_key
   - *RESTORE_OPENSSL
   - run: ./test/build_openssl.sh
   - *SAVE_OPENSSL
   - *RESTORE_DEPS
-  - run: cargo run --manifest-path=systest/Cargo.toml --target $TARGET
+  - run: cargo run --manifest-path=systest/Cargo.toml --target $TARGET --features "$FEATURES"
   - run: |
       ulimit -c unlimited
       export PATH=$OPENSSL_DIR/bin:$PATH
@@ -55,6 +55,7 @@ job: &JOB
       cargo test \
         --manifest-path=openssl/Cargo.toml \
         --target $TARGET \
+        --features "$FEATURES" \
         $TEST_ARGS
   - run:
       command: |
@@ -69,9 +70,6 @@ job: &JOB
 macos_job: &MACOS_JOB
   macos:
     xcode: "9.0"
-  environment:
-    RUSTUP_HOME: /usr/local/rustup
-    CARGO_HOME: /usr/local/cargo
   steps:
   - checkout
   - run: sudo mkdir /opt
@@ -83,12 +81,18 @@ macos_job: &MACOS_JOB
   - *SAVE_REGISTRY
   - run: echo "homebrew-x86_64-apple-darwin" > ~/lib_key
   - *RESTORE_DEPS
-  - run: cargo run --manifest-path=systest/Cargo.toml
+  - run: cargo run --manifest-path=systest/Cargo.toml --features "$FEATURES"
   - run: |
       PATH=/usr/local/opt/openssl/bin:$PATH
-      cargo test --manifest-path=openssl/Cargo.toml
+      cargo test --manifest-path=openssl/Cargo.toml --features "$FEATURES"
   - *SAVE_DEPS
+macos_env: &MACOS_ENV
+  RUSTUP_HOME: /usr/local/rustup
+  CARGO_HOME: /usr/local/cargo
 
+vendored: &VENDORED
+  FEATURES: vendored
+  LIBRARY: ""
 openssl_111: &OPENSSL_111
   LIBRARY: openssl
   VERSION: 1.1.1-pre8
@@ -110,6 +114,8 @@ libressl_270: &LIBRESSL_272
 
 x86_64: &X86_64
   TARGET: x86_64-unknown-linux-gnu
+musl: &MUSL
+  TARGET: x86_64-unknown-linux-musl
 i686: &I686
   TARGET: i686-unknown-linux-gnu
 armhf: &ARMHF
@@ -127,6 +133,16 @@ base: &BASE
 
 version: 2
 jobs:
+  musl-vendored:
+    <<: *JOB
+    docker:
+    - image: rust:1.21.0
+    environment:
+      <<: [*VENDORED, *MUSL, *BASE]
+  x86_64-vendored:
+    <<: *JOB
+    environment:
+      <<: [*VENDORED, *X86_64, *BASE]
   x86_64-openssl-1.1.1:
     <<: *JOB
     environment:
@@ -143,6 +159,10 @@ jobs:
     <<: *JOB
     environment:
       <<: [*OPENSSL_101, *X86_64, *BASE]
+  i686-vendored:
+    <<: *JOB
+    environment:
+      <<: [*VENDORED, *I686, *BASE]
   i686-openssl-1.1.1:
     <<: *JOB
     environment:
@@ -155,6 +175,10 @@ jobs:
     <<: *JOB
     environment:
       <<: [*OPENSSL_102, *I686, *BASE]
+  armhf-vendored:
+    <<: *JOB
+    environment:
+      <<: [*VENDORED, *ARMHF, *BASE]
   armhf-openssl-1.1.1:
     <<: *JOB
     environment:
@@ -177,20 +201,31 @@ jobs:
       <<: [*LIBRESSL_272, *X86_64, *BASE]
   macos:
     <<: *MACOS_JOB
+    environment:
+      <<: [*MACOS_ENV]
+  macos-vendored:
+    <<: *MACOS_JOB
+    environment:
+      <<: [*VENDORED, *MACOS_ENV]
 workflows:
   version: 2
   tests:
     jobs:
+    - musl-vendored
+    - x86_64-vendored
     - x86_64-openssl-1.1.1
     - x86_64-openssl-1.1.0
     - x86_64-openssl-1.0.2
     - x86_64-openssl-1.0.1
+    - i686-vendored
     - i686-openssl-1.1.1
     - i686-openssl-1.1.0
     - i686-openssl-1.0.2
+    - armhf-vendored
     - armhf-openssl-1.1.1
     - armhf-openssl-1.1.0
     - armhf-openssl-1.0.2
     - x86_64-libressl-2.5.0
     - x86_64-libressl-2.7.2
     - macos
+    - macos-vendored

openssl-sys/Cargo.toml

@@ -11,12 +11,16 @@ categories = ["cryptography", "external-ffi-bindings"]
 links = "openssl"
 build = "build/main.rs"
 
+[features]
+vendored = ['openssl-src']
+
 [dependencies]
 libc = "0.2"
 
 [build-dependencies]
 pkg-config = "0.3.9"
 cc = "1.0"
+openssl-src = { version = "110.0.4", optional = true }
 
 [target.'cfg(target_env = "msvc")'.build-dependencies]
 vcpkg = "0.2"

openssl-sys/build/main.rs

@@ -2,6 +2,8 @@ extern crate cc;
 extern crate pkg_config;
 #[cfg(target_env = "msvc")]
 extern crate vcpkg;
+#[cfg(feature = "vendored")]
+extern crate openssl_src;
 
 use std::collections::HashSet;
 use std::env;
@@ -9,7 +11,6 @@ use std::ffi::OsString;
 use std::fs::File;
 use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
-use std::process::Command;
 
 mod cfgs;
 
@@ -52,18 +53,7 @@ fn env(name: &str) -> Option<OsString> {
 fn main() {
     let target = env::var("TARGET").unwrap();
 
-    let lib_dir = env("OPENSSL_LIB_DIR").map(PathBuf::from);
+    let (lib_dir, include_dir) = imp::get_openssl(&target);
-    let include_dir = env("OPENSSL_INCLUDE_DIR").map(PathBuf::from);
-
-    let (lib_dir, include_dir) = if lib_dir.is_none() || include_dir.is_none() {
-        let openssl_dir = env("OPENSSL_DIR").unwrap_or_else(|| find_openssl_dir(&target));
-        let openssl_dir = Path::new(&openssl_dir);
-        let lib_dir = lib_dir.unwrap_or_else(|| openssl_dir.join("lib"));
-        let include_dir = include_dir.unwrap_or_else(|| openssl_dir.join("include"));
-        (lib_dir, include_dir)
-    } else {
-        (lib_dir.unwrap(), include_dir.unwrap())
-    };
 
     if !Path::new(&lib_dir).exists() {
         panic!(
@@ -110,7 +100,42 @@ fn main() {
     }
 }
 
-fn find_openssl_dir(target: &str) -> OsString {
+#[cfg(feature = "vendored")]
+mod imp {
+	use std::path::PathBuf;
+	use openssl_src;
+
+	pub fn get_openssl(_target: &str) -> (PathBuf, PathBuf) {
+		let artifacts = openssl_src::Build::new().build();
+		(artifacts.lib_dir().to_path_buf(), artifacts.include_dir().to_path_buf())
+	}
+}
+
+#[cfg(not(feature = "vendored"))]
+mod imp {
+    use pkg_config;
+    use std::path::{Path, PathBuf};
+    use std::ffi::OsString;
+    use std::process::{self, Command};
+
+    use super::env;
+
+    pub fn get_openssl(target: &str) -> (PathBuf, PathBuf) {
+        let lib_dir = env("OPENSSL_LIB_DIR").map(PathBuf::from);
+        let include_dir = env("OPENSSL_INCLUDE_DIR").map(PathBuf::from);
+
+        if lib_dir.is_none() || include_dir.is_none() {
+            let openssl_dir = env("OPENSSL_DIR").unwrap_or_else(|| find_openssl_dir(&target));
+            let openssl_dir = Path::new(&openssl_dir);
+            let lib_dir = lib_dir.unwrap_or_else(|| openssl_dir.join("lib"));
+            let include_dir = include_dir.unwrap_or_else(|| openssl_dir.join("include"));
+            (lib_dir, include_dir)
+        } else {
+            (lib_dir.unwrap(), include_dir.unwrap())
+        }
+    }
+
+    fn find_openssl_dir(target: &str) -> OsString {
         let host = env::var("HOST").unwrap();
 
         if host == target && target.contains("apple-darwin") {
@@ -238,13 +263,13 @@ OpenSSL:
         }
 
         panic!(msg);
-}
+    }
 
-/// Attempt to find OpenSSL through pkg-config.
+    /// Attempt to find OpenSSL through pkg-config.
-///
+    ///
-/// Note that if this succeeds then the function does not return as pkg-config
+    /// Note that if this succeeds then the function does not return as pkg-config
-/// typically tells us all the information that we need.
+    /// typically tells us all the information that we need.
-fn try_pkg_config() {
+    fn try_pkg_config() {
         let target = env::var("TARGET").unwrap();
         let host = env::var("HOST").unwrap();
 
@@ -269,21 +294,23 @@ fn try_pkg_config() {
             }
         };
 
-    validate_headers(&lib.include_paths);
+        super::validate_headers(&lib.include_paths);
 
         for include in lib.include_paths.iter() {
             println!("cargo:include={}", include.display());
         }
 
-    std::process::exit(0);
+        process::exit(0);
-}
+    }
+
+    /// Attempt to find OpenSSL through vcpkg.
+    ///
+    /// Note that if this succeeds then the function does not return as vcpkg
+    /// should emit all of the cargo metadata that we need.
+    #[cfg(target_env = "msvc")]
+    fn try_vcpkg() {
+        use vcpkg;
 
-/// Attempt to find OpenSSL through vcpkg.
-///
-/// Note that if this succeeds then the function does not return as vcpkg
-/// should emit all of the cargo metadata that we need.
-#[cfg(target_env = "msvc")]
-fn try_vcpkg() {
         // vcpkg will not emit any metadata if it can not find libraries
         // appropriate for the target triple with the desired linkage.
 
@@ -313,17 +340,31 @@ fn try_vcpkg() {
         }
 
         let lib = lib.unwrap();
-    validate_headers(&lib.include_paths);
+        super::validate_headers(&lib.include_paths);
 
         println!("cargo:rustc-link-lib=user32");
         println!("cargo:rustc-link-lib=gdi32");
         println!("cargo:rustc-link-lib=crypt32");
 
-    std::process::exit(0);
+        process::exit(0);
-}
+    }
 
-#[cfg(not(target_env = "msvc"))]
+    #[cfg(not(target_env = "msvc"))]
-fn try_vcpkg() {}
+    fn try_vcpkg() {}
+
+    fn execute_command_and_get_output(cmd: &str, args: &[&str]) -> Option<String> {
+        let out = Command::new(cmd).args(args).output();
+        if let Ok(ref r1) = out {
+            if r1.status.success() {
+                let r2 = String::from_utf8(r1.stdout.clone());
+                if let Ok(r3) = r2 {
+                    return Some(r3.trim().to_string());
+                }
+            }
+        }
+        return None;
+    }
+}
 
 /// Validates the header files found in `include_dir` and then returns the
 /// version string of OpenSSL.
@@ -565,16 +606,3 @@ fn determine_mode(libdir: &Path, libs: &[&str]) -> &'static str {
     // practices with security libs", let's link dynamically.
     "dylib"
 }
-
-fn execute_command_and_get_output(cmd: &str, args: &[&str]) -> Option<String> {
-    let out = Command::new(cmd).args(args).output();
-    if let Ok(ref r1) = out {
-        if r1.status.success() {
-            let r2 = String::from_utf8(r1.stdout.clone());
-            if let Ok(r3) = r2 {
-                return Some(r3.trim().to_string());
-            }
-        }
-    }
-    return None;
-}

openssl/Cargo.toml

@@ -16,6 +16,8 @@ v102 = []
 v110 = []
 v111 = []
 
+vendored = ['openssl-sys/vendored']
+
 [dependencies]
 bitflags = "1.0"
 cfg-if = "0.1"

openssl/src/ssl/test.rs

@@ -775,6 +775,7 @@ fn refcount_ssl_context() {
 
 #[test]
 #[cfg_attr(libressl250, ignore)]
+#[cfg_attr(all(target_os = "macos", feature = "vendored"), ignore)]
 fn default_verify_paths() {
     let mut ctx = SslContext::builder(SslMethod::tls()).unwrap();
     ctx.set_default_verify_paths().unwrap();
@@ -804,6 +805,7 @@ fn add_extra_chain_cert() {
 
 #[test]
 #[cfg(any(ossl102, ossl110))]
+#[cfg_attr(all(target_os = "macos", feature = "vendored"), ignore)]
 fn verify_valid_hostname() {
     let mut ctx = SslContext::builder(SslMethod::tls()).unwrap();
     ctx.set_default_verify_paths().unwrap();
@@ -845,6 +847,7 @@ fn verify_invalid_hostname() {
 
 #[test]
 #[cfg_attr(libressl250, ignore)]
+#[cfg_attr(all(target_os = "macos", feature = "vendored"), ignore)]
 fn connector_valid_hostname() {
     let connector = SslConnector::builder(SslMethod::tls()).unwrap().build();
 
@@ -861,6 +864,7 @@ fn connector_valid_hostname() {
 }
 
 #[test]
+#[cfg_attr(all(target_os = "macos", feature = "vendored"), ignore)]
 fn connector_invalid_hostname() {
     let connector = SslConnector::builder(SslMethod::tls()).unwrap().build();
 
@@ -870,6 +874,7 @@ fn connector_invalid_hostname() {
 
 #[test]
 #[cfg_attr(libressl250, ignore)]
+#[cfg_attr(all(target_os = "macos", feature = "vendored"), ignore)]
 fn connector_invalid_no_hostname_verification() {
     let connector = SslConnector::builder(SslMethod::tls()).unwrap().build();
 
@@ -1202,6 +1207,7 @@ fn idle_session() {
 
 #[test]
 #[cfg_attr(libressl250, ignore)]
+#[cfg_attr(all(target_os = "macos", feature = "vendored"), ignore)]
 fn active_session() {
     let connector = SslConnector::builder(SslMethod::tls()).unwrap().build();
 

systest/Cargo.toml

@@ -9,3 +9,6 @@ openssl-sys = { path = "../openssl-sys" }
 
 [build-dependencies]
 ctest = "0.1"
+
+[features]
+vendored = ['openssl-sys/vendored']

test/add_target.sh

@@ -8,6 +8,9 @@ case "${TARGET}" in
 "i686-unknown-linux-gnu")
     apt-get install -y --no-install-recommends gcc-multilib
     ;;
+"x86_64-unknown-linux-musl")
+    apt-get install -y --no-install-recommends musl-tools
+    ;;
 "arm-unknown-linux-gnueabihf")
     dpkg --add-architecture armhf
     apt-get update

test/build_openssl.sh

@@ -5,8 +5,6 @@ if [ -d "${OPENSSL_DIR}" ]; then
     exit 0
 fi
 
-apt-get install -y --no-install-recommends curl
-
 case "${LIBRARY}" in
 "libressl")
     URL1="http://ftp3.usa.openbsd.org/pub/OpenBSD/LibreSSL/libressl-${VERSION}.tar.gz"
@@ -16,8 +14,14 @@ case "${LIBRARY}" in
     URL1="https://openssl.org/source/openssl-${VERSION}.tar.gz"
     URL2="http://mirrors.ibiblio.org/openssl/source/openssl-${VERSION}.tar.gz"
     ;;
+"")
+    # using the vendored builds
+    exit 0;
+    ;;
 esac
 
+apt-get install -y --no-install-recommends curl
+
 case "${TARGET}" in
 "x86_64-unknown-linux-gnu")
     OS_COMPILER=linux-x86_64