Merge branch 'maint'

* maint:
  Ruby/OpenSSL 2.0.6
  test/test_engine: check if RC4 is supported
  test/test_engine: suppress stderr
  ossl.c: make legacy locking callbacks reentrant
  ossl.c: use struct CRYPTO_dynlock_value for non-dynamic locks
  ssl: prevent SSLSocket#sysread* from leaking uninitialized data
  test/test_pair: replace sleep with IO.select
  tool/ruby-openssl-docker: update
  test/test_ssl: do not run NPN tests for LibreSSL >= 2.6.1
  test/test_ssl: skip tmp_ecdh_callback test for LibreSSL >= 2.6.1
  test/test_pair: disable compression
  test/test_ssl: suppress warning in test_alpn_protocol_selection_cancel
  ruby.h: unnormalized Fixnum value
  test/test_pair: fix test_write_nonblock{,_no_exceptions}

Author: rhenium

.travis.yml

@@ -23,6 +23,7 @@ matrix:
     - env: RUBY_VERSION=ruby-2.4 OPENSSL_VERSION=openssl-1.1.0
     - env: RUBY_VERSION=ruby-2.4 OPENSSL_VERSION=libressl-2.4
     - env: RUBY_VERSION=ruby-2.4 OPENSSL_VERSION=libressl-2.5
+    - env: RUBY_VERSION=ruby-2.4 OPENSSL_VERSION=libressl-2.6
     - language: ruby
       rvm: ruby-head
       before_install:

History.md

@@ -29,6 +29,26 @@ Notable changes
   [[GitHub #143]](https://github.com/ruby/openssl/pull/143)
 
 
+Version 2.0.6
+=============
+
+Bug fixes
+---------
+
+* The session_remove_cb set to an OpenSSL::SSL::SSLContext is no longer called
+  during GC.
+* A possible deadlock in OpenSSL::SSL::SSLSocket#sysread is fixed.
+  [[GitHub #139]](https://github.com/ruby/openssl/pull/139)
+* OpenSSL::BN#hash could return an unnormalized fixnum value on Windows.
+  [[Bug #13877]](https://bugs.ruby-lang.org/issues/13877)
+* OpenSSL::SSL::SSLSocket#sysread and #sysread_nonblock set the length of the
+  destination buffer String to 0 on error.
+  [[GitHub #153]](https://github.com/ruby/openssl/pull/153)
+* Possible deadlock is fixed. This happened only when built with older versions
+  of OpenSSL (before 1.1.0) or LibreSSL.
+  [[GitHub #155]](https://github.com/ruby/openssl/pull/155)
+
+
 Version 2.0.5
 =============
 

ext/openssl/ossl.c

@@ -517,40 +517,53 @@ print_mem_leaks(VALUE self)
 /**
  * Stores locks needed for OpenSSL thread safety
  */
-static rb_nativethread_lock_t *ossl_locks;
+struct CRYPTO_dynlock_value {
+    rb_nativethread_lock_t lock;
+    rb_nativethread_id_t owner;
+    size_t count;
+};
 
 static void
-ossl_lock_unlock(int mode, rb_nativethread_lock_t *lock)
+ossl_lock_init(struct CRYPTO_dynlock_value *l)
 {
-    if (mode & CRYPTO_LOCK) {
-	rb_nativethread_lock_lock(lock);
-    } else {
-	rb_nativethread_lock_unlock(lock);
-    }
+    rb_nativethread_lock_initialize(&l->lock);
+    l->count = 0;
 }
 
 static void
-ossl_lock_callback(int mode, int type, const char *file, int line)
+ossl_lock_unlock(int mode, struct CRYPTO_dynlock_value *l)
 {
-    ossl_lock_unlock(mode, &ossl_locks[type]);
+    if (mode & CRYPTO_LOCK) {
+	/* TODO: rb_nativethread_id_t is not necessarily compared with ==. */
+	rb_nativethread_id_t tid = rb_nativethread_self();
+	if (l->count && l->owner == tid) {
+	    l->count++;
+	    return;
+	}
+	rb_nativethread_lock_lock(&l->lock);
+	l->owner = tid;
+	l->count = 1;
+    } else {
+	if (!--l->count)
+	    rb_nativethread_lock_unlock(&l->lock);
+    }
 }
 
-struct CRYPTO_dynlock_value {
-    rb_nativethread_lock_t lock;
-};
-
 static struct CRYPTO_dynlock_value *
 ossl_dyn_create_callback(const char *file, int line)
 {
-    struct CRYPTO_dynlock_value *dynlock = (struct CRYPTO_dynlock_value *)OPENSSL_malloc((int)sizeof(struct CRYPTO_dynlock_value));
-    rb_nativethread_lock_initialize(&dynlock->lock);
+    /* Do not use xmalloc() here, since it may raise NoMemoryError */
+    struct CRYPTO_dynlock_value *dynlock =
+	OPENSSL_malloc(sizeof(struct CRYPTO_dynlock_value));
+    if (dynlock)
+	ossl_lock_init(dynlock);
     return dynlock;
 }
 
 static void
 ossl_dyn_lock_callback(int mode, struct CRYPTO_dynlock_value *l, const char *file, int line)
 {
-    ossl_lock_unlock(mode, &l->lock);
+    ossl_lock_unlock(mode, l);
 }
 
 static void
@@ -566,21 +579,22 @@ static void ossl_threadid_func(CRYPTO_THREADID *id)
     CRYPTO_THREADID_set_pointer(id, (void *)rb_nativethread_self());
 }
 
+static struct CRYPTO_dynlock_value *ossl_locks;
+
+static void
+ossl_lock_callback(int mode, int type, const char *file, int line)
+{
+    ossl_lock_unlock(mode, &ossl_locks[type]);
+}
+
 static void Init_ossl_locks(void)
 {
     int i;
     int num_locks = CRYPTO_num_locks();
 
-    if ((unsigned)num_locks >= INT_MAX / (int)sizeof(VALUE)) {
-	rb_raise(rb_eRuntimeError, "CRYPTO_num_locks() is too big: %d", num_locks);
-    }
-    ossl_locks = (rb_nativethread_lock_t *) OPENSSL_malloc(num_locks * (int)sizeof(rb_nativethread_lock_t));
-    if (!ossl_locks) {
-	rb_raise(rb_eNoMemError, "CRYPTO_num_locks() is too big: %d", num_locks);
-    }
-    for (i = 0; i < num_locks; i++) {
-	rb_nativethread_lock_initialize(&ossl_locks[i]);
-    }
+    ossl_locks = ALLOC_N(struct CRYPTO_dynlock_value, num_locks);
+    for (i = 0; i < num_locks; i++)
+	ossl_lock_init(&ossl_locks[i]);
 
     CRYPTO_THREADID_set_callback(ossl_threadid_func);
     CRYPTO_set_locking_callback(ossl_lock_callback);

ext/openssl/ossl_bn.c

@@ -991,7 +991,7 @@ ossl_bn_hash(VALUE self)
 	ossl_raise(eBNError, NULL);
     }
 
-    hash = INT2FIX(rb_memhash(buf, len));
+    hash = ST2FIX(rb_memhash(buf, len));
     xfree(buf);
 
     return hash;

ext/openssl/ossl_ssl.c

@@ -1694,20 +1694,26 @@ ossl_ssl_read_internal(int argc, VALUE *argv, VALUE self, int nonblock)
     }
 
     ilen = NUM2INT(len);
-    if(NIL_P(str)) str = rb_str_new(0, ilen);
-    else{
-        StringValue(str);
-        rb_str_modify(str);
-        rb_str_resize(str, ilen);
+    if (NIL_P(str))
+	str = rb_str_new(0, ilen);
+    else {
+	StringValue(str);
+	if (RSTRING_LEN(str) >= ilen)
+	    rb_str_modify(str);
+	else
+	    rb_str_modify_expand(str, ilen - RSTRING_LEN(str));
     }
-    if(ilen == 0) return str;
+    OBJ_TAINT(str);
+    rb_str_set_len(str, 0);
+    if (ilen == 0)
+	return str;
 
     GetSSL(self, ssl);
     io = rb_attr_get(self, id_i_io);
     GetOpenFile(io, fptr);
     if (ssl_started(ssl)) {
 	for (;;){
-	    nread = SSL_read(ssl, RSTRING_PTR(str), RSTRING_LENINT(str));
+	    nread = SSL_read(ssl, RSTRING_PTR(str), ilen);
 	    switch(ssl_get_error(ssl, nread)){
 	    case SSL_ERROR_NONE:
 		goto end;
@@ -1757,8 +1763,6 @@ ossl_ssl_read_internal(int argc, VALUE *argv, VALUE self, int nonblock)
 
   end:
     rb_str_set_len(str, nread);
-    OBJ_TAINT(str);
-
     return str;
 }
 

ext/openssl/ruby_missing.h

@@ -10,9 +10,15 @@
 #if !defined(_OSSL_RUBY_MISSING_H_)
 #define _OSSL_RUBY_MISSING_H_
 
+/* Ruby 2.4 */
 #ifndef RB_INTEGER_TYPE_P
-/* for Ruby 2.3 compatibility */
-#define RB_INTEGER_TYPE_P(obj) (RB_FIXNUM_P(obj) || RB_TYPE_P(obj, T_BIGNUM))
+# define RB_INTEGER_TYPE_P(obj) (RB_FIXNUM_P(obj) || RB_TYPE_P(obj, T_BIGNUM))
+#endif
+
+/* Ruby 2.5 */
+#ifndef ST2FIX
+# define RB_ST2FIX(h) LONG2FIX((long)(h))
+# define ST2FIX(h) RB_ST2FIX(h)
 #endif
 
 #endif /* _OSSL_RUBY_MISSING_H_ */

test/test_bn.rb

@@ -270,6 +270,7 @@ def test_comparison
     assert_equal(1, @e1.cmp(-999))
     assert_equal(0, @e1.ucmp(999))
     assert_equal(0, @e1.ucmp(-999))
+    assert_instance_of(String, @e1.hash.to_s)
   end
 end
 

test/test_engine.rb

@@ -52,32 +52,28 @@ def test_openssl_engine_digest_sha1
   end
 
   def test_openssl_engine_cipher_rc4
-    with_openssl <<-'end;'
-      begin
-        engine = get_engine
-        algo = "RC4" #AES is not supported by openssl Engine (<=1.0.0e)
-        data = "a" * 1000
-        key = OpenSSL::Random.random_bytes(16)
-        # suppress message from openssl Engine's RC4 cipher [ruby-core:41026]
-        err_back = $stderr.dup
-        $stderr.reopen(IO::NULL)
-        encrypted = crypt_data(data, key, :encrypt) { engine.cipher(algo) }
-        decrypted = crypt_data(encrypted, key, :decrypt) { OpenSSL::Cipher.new(algo) }
-        assert_equal(data, decrypted)
-      ensure
-        if err_back
-          $stderr.reopen(err_back)
-          err_back.close
-        end
-      end
+    begin
+      OpenSSL::Cipher.new("rc4")
+    rescue OpenSSL::Cipher::CipherError
+      pend "RC4 is not supported"
+    end
+
+    with_openssl(<<-'end;', ignore_stderr: true)
+      engine = get_engine
+      algo = "RC4"
+      data = "a" * 1000
+      key = OpenSSL::Random.random_bytes(16)
+      encrypted = crypt_data(data, key, :encrypt) { engine.cipher(algo) }
+      decrypted = crypt_data(encrypted, key, :decrypt) { OpenSSL::Cipher.new(algo) }
+      assert_equal(data, decrypted)
     end;
   end
 
   private
 
   # this is required because OpenSSL::Engine methods change global state
-  def with_openssl(code)
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+  def with_openssl(code, **opts)
+    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;", **opts)
       require #{__FILE__.dump}
       include OpenSSL::TestEngine::Utils
       #{code}

test/test_pair.rb

@@ -24,6 +24,7 @@ def ssl_pair
       sctx.cert = @svr_cert
       sctx.key = @svr_key
       sctx.tmp_dh_callback = proc { OpenSSL::TestUtils::Fixtures.pkey_dh("dh1024") }
+      sctx.options |= OpenSSL::SSL::OP_NO_COMPRESSION
       ssls = OpenSSL::SSL::SSLServer.new(tcps, sctx)
       ns = ssls.accept
       ssls.close
@@ -217,7 +218,7 @@ def test_read_nonblock
       assert_nothing_raised("[ruby-core:20298]") { ret = s2.read_nonblock(10) }
       assert_equal("def\n", ret)
       s1.close
-      sleep 0.1
+      IO.select([s2])
       assert_raise(EOFError) { s2.read_nonblock(10) }
     }
   end
@@ -233,49 +234,71 @@ def test_read_nonblock_no_exception
       assert_nothing_raised("[ruby-core:20298]") { ret = s2.read_nonblock(10, exception: false) }
       assert_equal("def\n", ret)
       s1.close
-      sleep 0.1
+      IO.select([s2])
       assert_equal(nil, s2.read_nonblock(10, exception: false))
     }
   end
 
-  def write_nonblock(socket, meth, str)
-    ret = socket.send(meth, str)
-    ret.is_a?(Symbol) ? 0 : ret
-  end
+  def test_read_with_outbuf
+    ssl_pair { |s1, s2|
+      s1.write("abc\n")
+      buf = ""
+      ret = s2.read(2, buf)
+      assert_same ret, buf
+      assert_equal "ab", ret
+
+      buf = "garbage"
+      ret = s2.read(2, buf)
+      assert_same ret, buf
+      assert_equal "c\n", ret
 
-  def write_nonblock_no_ex(socket, str)
-    ret = socket.write_nonblock str, exception: false
-    ret.is_a?(Symbol) ? 0 : ret
+      buf = "garbage"
+      assert_equal :wait_readable, s2.read_nonblock(100, buf, exception: false)
+      assert_equal "", buf
+
+      s1.close
+      buf = "garbage"
+      assert_equal nil, s2.read(100, buf)
+      assert_equal "", buf
+    }
   end
 
   def test_write_nonblock
     ssl_pair {|s1, s2|
-      n = 0
-      begin
-        n += write_nonblock s1, :write_nonblock, "a" * 100000
-        n += write_nonblock s1, :write_nonblock, "b" * 100000
-        n += write_nonblock s1, :write_nonblock, "c" * 100000
-        n += write_nonblock s1, :write_nonblock, "d" * 100000
-        n += write_nonblock s1, :write_nonblock, "e" * 100000
-        n += write_nonblock s1, :write_nonblock, "f" * 100000
-      rescue IO::WaitWritable
+      assert_equal 3, s1.write_nonblock("foo")
+      assert_equal "foo", s2.read(3)
+
+      data = "x" * 16384
+      written = 0
+      while true
+        begin
+          written += s1.write_nonblock(data)
+        rescue IO::WaitWritable, IO::WaitReadable
+          break
+        end
       end
-      s1.close
-      assert_equal(n, s2.read.length)
+      assert written > 0
+      assert_equal written, s2.read(written).bytesize
     }
   end
 
   def test_write_nonblock_no_exceptions
     ssl_pair {|s1, s2|
-      n = 0
-      n += write_nonblock_no_ex s1, "a" * 100000
-      n += write_nonblock_no_ex s1, "b" * 100000
-      n += write_nonblock_no_ex s1, "c" * 100000
-      n += write_nonblock_no_ex s1, "d" * 100000
-      n += write_nonblock_no_ex s1, "e" * 100000
-      n += write_nonblock_no_ex s1, "f" * 100000
-      s1.close
-      assert_equal(n, s2.read.length)
+      assert_equal 3, s1.write_nonblock("foo", exception: false)
+      assert_equal "foo", s2.read(3)
+
+      data = "x" * 16384
+      written = 0
+      while true
+        case ret = s1.write_nonblock(data, exception: false)
+        when :wait_readable, :wait_writable
+          break
+        else
+          written += ret
+        end
+      end
+      assert written > 0
+      assert_equal written, s2.read(written).bytesize
     }
   end