keep in line with MRI if possible

keep the default x509 certs and directories in line with MRI, only if
they do not exists fallback on cacerts from the java.home/lib/security/cacerts

fixes #49, #70 and keeps the idea of b914091

Sponsored by Lookout Inc.

Author: mkristian

src/main/java/org/jruby/ext/openssl/x509store/Lookup.java

@@ -352,8 +352,7 @@ else if ( cert instanceof CRL ) {
         }
     }
 
-    public int loadDefaultJavaCACertsFile() throws IOException, GeneralSecurityException {
-        final String certsFile = X509Utils.X509_CERT_FILE.replace('/', File.separatorChar);
+    public int loadDefaultJavaCACertsFile(String certsFile) throws IOException, GeneralSecurityException {
         final FileInputStream fin = new FileInputStream(certsFile);
         int count = 0;
         try {
@@ -517,15 +516,15 @@ public int call(final Lookup ctx, final Integer cmd, final String argp, final Nu
                         file = ctx.envEntry( getDefaultCertificateFileEnvironment() );
                     }
                     catch (RuntimeException e) { }
-
-                    if (file != null) {
+                    if (file == null) {
+                        file = X509Utils.X509_CERT_FILE.replace('/', File.separatorChar);
+                    }
+                    if (file.matches(".*\\.(crt|cer|pem)$")) {
                         ok = ctx.loadCertificateOrCRLFile(file, X509_FILETYPE_PEM) != 0 ? 1 : 0;
                     } else {
-                        ok = (ctx.loadDefaultJavaCACertsFile() != 0) ? 1: 0;
-                    }
-                    if (ok == 0) {
-                        X509Error.addError(X509_R_LOADING_DEFAULTS);
+                        ok = (ctx.loadDefaultJavaCACertsFile(file) != 0) ? 1: 0;
                     }
+                    // we ignore errors on loading default paths the same way MRI does it
                 } else {
                     if (arglInt == X509_FILETYPE_PEM) {
                         ok = (ctx.loadCertificateOrCRLFile(argp, X509_FILETYPE_PEM) != 0) ? 1 : 0;

src/main/java/org/jruby/ext/openssl/x509store/X509Utils.java

@@ -28,6 +28,7 @@
 package org.jruby.ext.openssl.x509store;
 
 
+import java.io.File;
 import java.io.IOException;
 import java.math.BigInteger;
 import java.util.Arrays;
@@ -292,13 +293,48 @@ else if ( keyUsage != null && ! keyUsage[5] ) { // KU_KEY_CERT_SIGN
     public static final String X509_PRIVATE_DIR;
 
     static {
-        OPENSSLDIR = "/usr/local/openssl"; // NOTE: blindly follow?!
+        // roughly following the ideas from https://www.happyassassin.net/2015/01/12/a-note-about-ssltls-trusted-certificate-stores-and-platforms/
+        // and falling back to trust store from java to be on the save side 
+        
         // TODO usability in limited environments should be tested/reviewed
         final String JAVA_HOME = SafePropertyAccessor.getProperty("java.home", "");
-        X509_CERT_AREA = JAVA_HOME + "/lib/security";
-        X509_CERT_DIR = X509_CERT_AREA;
-        X509_CERT_FILE = X509_CERT_DIR + "/cacerts";
-        X509_PRIVATE_DIR = "/usr/lib/ssl/private"; // NOTE: blindly follow?!
+
+        // if the default files/dirs exist we use them. with this a switch
+        // from MRI to JRuby produces the same results. otherwise we use the
+        // certs from JAVA_HOME.
+        final String MAYBE_CERT_FILE;
+        final String LINUX_CERT_AREA = "/etc/ssl";
+        final String MACOS_CERT_AREA = "/System/Library/OpenSSL";
+        final String MAYBE_PKI_CERT_FILE = "/etc/pki/tls/certs/ca-bundle.crt";
+        if (new File(LINUX_CERT_AREA).exists()) {
+            X509_CERT_AREA = LINUX_CERT_AREA;
+            X509_CERT_DIR = X509_CERT_AREA + "/certs";
+            X509_PRIVATE_DIR = X509_CERT_AREA + "/private";
+            MAYBE_CERT_FILE = X509_CERT_DIR + "/cert.pem";
+        }
+        else if (new File(MACOS_CERT_AREA).exists()) {
+            X509_CERT_AREA = MACOS_CERT_AREA;
+            X509_CERT_DIR = X509_CERT_AREA + "/certs";
+            X509_PRIVATE_DIR = X509_CERT_AREA + "/private";
+            MAYBE_CERT_FILE = X509_CERT_AREA + "/cert.pem";
+        }
+        else {
+            X509_CERT_AREA = JAVA_HOME + "/lib/security";
+            X509_CERT_DIR = X509_CERT_AREA;
+            X509_PRIVATE_DIR = X509_CERT_AREA;
+            MAYBE_CERT_FILE = MAYBE_PKI_CERT_FILE;
+        }
+        if (new File(MAYBE_PKI_CERT_FILE).exists()) {
+            X509_CERT_FILE = MAYBE_PKI_CERT_FILE;
+        }
+        else if (new File(MAYBE_CERT_FILE).exists()) {
+            X509_CERT_FILE = MAYBE_CERT_FILE;
+        }
+        else {
+            X509_CERT_FILE = JAVA_HOME + "/lib/security/cacerts";
+        }
+        // keep it with some meaninful content as it is a public constant
+        OPENSSLDIR = X509_CERT_AREA;
     }
 
     public static final String X509_CERT_DIR_EVP = "SSL_CERT_DIR";

src/test/ruby/x509/ca.crt

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICATCCAWoCCQDbLK0ArLx6CTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB
+VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMB4XDTE1MDkxMDE2NDIyNVoXDTQzMDEyNjE2NDIyNVowRTELMAkG
+A1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0
+IFdpZGdpdHMgUHR5IEx0ZDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAy3aH
+N4Pci0FwOpfwxFXKQ1IlXWrfUIuWTJs4bPek3kKmM4eJOfMLnQ2f/i0nrI1JHJBn
+GTMqiXti8lRhdy0HONVCVolc9rycdIj/batAwbCLxhg5lQMVOWzIJ3fW5R5DEcKK
+87PN8Dpf4nn9b2zyszGroExqgVFcp5q2zDzD+b0CAwEAATANBgkqhkiG9w0BAQUF
+AAOBgQCu+bibmVksOHFauNRRt2uDLzNw6rDJyL9X3UC6Wu2Dp8Sk44RxR+qNQMS/
+CqgW9P7TWZdhsRUcnVRpaRMsr8CpTzHMmKaR0APON7v1C0HaV0CBYfzuNraAnlsq
+e7KpTHJ+mwxwVvN1XDv/jGqiw83p1wA4jyxQe59Cmubv+jGksQ==
+-----END CERTIFICATE-----

src/test/ruby/x509/digicert.pem

@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIEtjCCA56gAwIBAgIQDHmpRLCMEZUgkmFf4msdgzANBgkqhkiG9w0BAQsFADBs
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
+ZSBFViBSb290IENBMB4XDTEzMTAyMjEyMDAwMFoXDTI4MTAyMjEyMDAwMFowdTEL
+MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
+LmRpZ2ljZXJ0LmNvbTE0MDIGA1UEAxMrRGlnaUNlcnQgU0hBMiBFeHRlbmRlZCBW
+YWxpZGF0aW9uIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBANdTpARR+JmmFkhLZyeqk0nQOe0MsLAAh/FnKIaFjI5j2ryxQDji0/XspQUY
+uD0+xZkXMuwYjPrxDKZkIYXLBxA0sFKIKx9om9KxjxKws9LniB8f7zh3VFNfgHk/
+LhqqqB5LKw2rt2O5Nbd9FLxZS99RStKh4gzikIKHaq7q12TWmFXo/a8aUGxUvBHy
+/Urynbt/DvTVvo4WiRJV2MBxNO723C3sxIclho3YIeSwTQyJ3DkmF93215SF2AQh
+cJ1vb/9cuhnhRctWVyh+HA1BV6q3uCe7seT6Ku8hI3UarS2bhjWMnHe1c63YlC3k
+8wyd7sFOYn4XwHGeLN7x+RAoGTMCAwEAAaOCAUkwggFFMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
+BQcDAjA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp
+Z2ljZXJ0LmNvbTBLBgNVHR8ERDBCMECgPqA8hjpodHRwOi8vY3JsNC5kaWdpY2Vy
+dC5jb20vRGlnaUNlcnRIaWdoQXNzdXJhbmNlRVZSb290Q0EuY3JsMD0GA1UdIAQ2
+MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5j
+b20vQ1BTMB0GA1UdDgQWBBQ901Cl1qCt7vNKYApl0yHU+PjWDzAfBgNVHSMEGDAW
+gBSxPsNpA/i/RwHUmCYaCALvY2QrwzANBgkqhkiG9w0BAQsFAAOCAQEAnbbQkIbh
+hgLtxaDwNBx0wY12zIYKqPBKikLWP8ipTa18CK3mtlC4ohpNiAexKSHc59rGPCHg
+4xFJcKx6HQGkyhE6V6t9VypAdP3THYUYUN9XR3WhfVUgLkc3UHKMf4Ib0mKPLQNa
+2sPIoc4sUqIAY+tzunHISScjl2SFnjgOrWNoPLpSgVh5oywM395t6zHyuqB8bPEs
+1OG9d4Q3A84ytciagRpKkk47RpqF/oOi+Z6Mo8wNXrM9zwR4jxQUezKcxwCmXMS1
+oVWNWlZopCJwqjyBcdmdqEU79OX2olHdx3ti6G8MdOu42vi/hw15UJGQmxg7kVkn
+8TUoE6smftX3eg==
+-----END CERTIFICATE-----

src/test/ruby/x509/gibberish.pem

@@ -0,0 +1 @@
+something but not any pem section

src/test/ruby/x509/server.crt

@@ -0,0 +1,47 @@
+Certificate:
+    Data:
+        Version: 1 (0x0)
+        Serial Number: 1 (0x1)
+        Signature Algorithm: md5WithRSAEncryption
+        Issuer: C=AU, ST=Some-State, O=Internet Widgits Pty Ltd
+        Validity
+            Not Before: Sep 10 16:45:04 2015 GMT
+            Not After : Sep  9 16:45:04 2016 GMT
+        Subject: C=AU, ST=Some-State, O=Internet Widgits Pty Ltd, CN=example.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+            RSA Public Key: (1024 bit)
+                Modulus (1024 bit):
+                    00:db:95:d0:12:c8:78:dc:eb:c2:6f:4a:ea:d2:cc:
+                    21:e1:e9:16:69:11:84:25:c6:d0:2a:4c:a9:50:7a:
+                    23:cb:89:ba:40:d9:14:7a:72:19:90:44:20:46:b0:
+                    5d:f2:f6:6f:53:e2:f0:6a:e5:1c:0e:08:4e:0e:89:
+                    96:af:97:6b:a3:bd:0a:bb:5f:b8:64:d2:12:ed:66:
+                    2c:64:36:90:41:78:61:95:72:84:da:50:53:f2:a4:
+                    02:a2:b2:9b:ee:7d:d4:bd:26:07:06:28:80:af:16:
+                    e1:3b:73:9a:93:dd:48:98:52:0a:cd:6a:d2:9a:95:
+                    02:a5:2c:b6:b7:eb:1f:de:5d
+                Exponent: 65537 (0x10001)
+    Signature Algorithm: md5WithRSAEncryption
+        76:a4:3c:20:c8:4d:a2:bf:ac:34:93:a4:50:58:68:f1:a6:72:
+        db:eb:27:00:f4:2d:81:97:cb:b8:7f:a6:21:45:53:dc:2d:a4:
+        3a:62:66:bb:8b:a3:30:b1:e8:2d:d4:19:0d:36:22:2e:1d:77:
+        f9:bc:b8:9a:47:7a:60:70:83:04:52:a5:e0:14:c9:2e:18:38:
+        8a:7b:10:da:0a:48:45:3f:7c:97:65:aa:ea:0c:ae:9f:77:c0:
+        9f:30:1c:e4:55:29:21:cf:b4:ba:60:72:b2:be:4e:29:9f:d3:
+        1f:c3:82:ed:04:87:b7:11:3f:4a:71:84:9d:34:b8:bf:4f:41:
+        f0:ed
+-----BEGIN CERTIFICATE-----
+MIICDzCCAXgCAQEwDQYJKoZIhvcNAQEEBQAwRTELMAkGA1UEBhMCQVUxEzARBgNV
+BAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
+ZDAeFw0xNTA5MTAxNjQ1MDRaFw0xNjA5MDkxNjQ1MDRaMFsxCzAJBgNVBAYTAkFV
+MRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRz
+IFB0eSBMdGQxFDASBgNVBAMTC2V4YW1wbGUuY29tMIGfMA0GCSqGSIb3DQEBAQUA
+A4GNADCBiQKBgQDbldASyHjc68JvSurSzCHh6RZpEYQlxtAqTKlQeiPLibpA2RR6
+chmQRCBGsF3y9m9T4vBq5RwOCE4OiZavl2ujvQq7X7hk0hLtZixkNpBBeGGVcoTa
+UFPypAKispvufdS9JgcGKICvFuE7c5qT3UiYUgrNatKalQKlLLa36x/eXQIDAQAB
+MA0GCSqGSIb3DQEBBAUAA4GBAHakPCDITaK/rDSTpFBYaPGmctvrJwD0LYGXy7h/
+piFFU9wtpDpiZruLozCx6C3UGQ02Ii4dd/m8uJpHemBwgwRSpeAUyS4YOIp7ENoK
+SEU/fJdlquoMrp93wJ8wHORVKSHPtLpgcrK+Timf0x/Dgu0Eh7cRP0pxhJ00uL9P
+QfDt
+-----END CERTIFICATE-----

src/test/ruby/x509/store

@@ -3,21 +3,66 @@
 
 class TestX509Store < TestCase
 
-  if defined? JRUBY_VERSION
-    def setup; require 'jopenssl/load' end
-  else
-    def setup; require 'openssl' end
+  def setup
+    require 'openssl'
+    @cert = OpenSSL::X509::Certificate.new(File.read(File.expand_path('../server.crt', __FILE__)))
+    @ca_cert = File.expand_path('../ca.crt', __FILE__)
+    @javastore = File.expand_path('../store', __FILE__)
+    @pem = File.expand_path('../EntrustnetSecureServerCertificationAuthority.pem', __FILE__)
+  end
+
+  def test_store_location_with_pem
+    ENV['SSL_CERT_FILE'] = nil
+    store = OpenSSL::X509::Store.new
+    store.set_default_paths
+    assert !store.verify(@cert)
+    
+    ENV['SSL_CERT_FILE'] = @ca_cert
+    store = OpenSSL::X509::Store.new
+    assert !store.verify(@cert)
+    store.set_default_paths
+    assert store.verify(@cert)
+  end
+
+  def test_store_location_with_java_truststore
+    ENV['SSL_CERT_FILE'] = @javastore
+    store = OpenSSL::X509::Store.new
+    assert !store.verify(@cert)
+    store.set_default_paths
+    assert store.verify(@cert)
+  end
+
+  def test_use_gibberish_cert_file
+    ENV['SSL_CERT_FILE'] = File.expand_path('../gibberish.pem', __FILE__)
+    store = OpenSSL::X509::Store.new
+    store.set_default_paths
+    assert !store.verify(@cert)
+  end
+
+  def test_use_default_cert_file_as_custom_file
+    ENV['SSL_CERT_FILE'] = OpenSSL::X509::DEFAULT_CERT_FILE
+    store = OpenSSL::X509::Store.new
+    store.set_default_paths
+    cert = OpenSSL::X509::Certificate.new(File.read(File.expand_path('../digicert.pem', __FILE__)))
+    assert store.verify(cert)
+  end
+
+  def test_add_file_to_store_with_custom_cert_file
+    ENV['SSL_CERT_FILE'] = @ca_cert
+    store = OpenSSL::X509::Store.new
+    store.set_default_paths
+    store.add_file @pem
+    assert store.verify( OpenSSL::X509::Certificate.new(File.read(@pem)))
   end
 
   def test_add_cert_concurrently
-    pem = File.expand_path('../EntrustnetSecureServerCertificationAuthority.pem', __FILE__)
     store = OpenSSL::X509::Store.new
     t = []
     (0..25).each do |i|
 
       t << Thread.new do
         (0..2).each do
-          store.add_file pem
+          store.add_file @pem
         end
       end
     end