Merge pull request #155 from rhenium/ky/fix-legacy-locking-callback-relock

ossl.c: make legacy locking callbacks reentrant

Author: rhenium

ext/openssl/ossl.c

@@ -484,40 +484,53 @@ print_mem_leaks(VALUE self)
 /**
  * Stores locks needed for OpenSSL thread safety
  */
-static rb_nativethread_lock_t *ossl_locks;
+struct CRYPTO_dynlock_value {
+    rb_nativethread_lock_t lock;
+    rb_nativethread_id_t owner;
+    size_t count;
+};
 
 static void
-ossl_lock_unlock(int mode, rb_nativethread_lock_t *lock)
+ossl_lock_init(struct CRYPTO_dynlock_value *l)
 {
-    if (mode & CRYPTO_LOCK) {
-	rb_nativethread_lock_lock(lock);
-    } else {
-	rb_nativethread_lock_unlock(lock);
-    }
+    rb_nativethread_lock_initialize(&l->lock);
+    l->count = 0;
 }
 
 static void
-ossl_lock_callback(int mode, int type, const char *file, int line)
+ossl_lock_unlock(int mode, struct CRYPTO_dynlock_value *l)
 {
-    ossl_lock_unlock(mode, &ossl_locks[type]);
+    if (mode & CRYPTO_LOCK) {
+	/* TODO: rb_nativethread_id_t is not necessarily compared with ==. */
+	rb_nativethread_id_t tid = rb_nativethread_self();
+	if (l->count && l->owner == tid) {
+	    l->count++;
+	    return;
+	}
+	rb_nativethread_lock_lock(&l->lock);
+	l->owner = tid;
+	l->count = 1;
+    } else {
+	if (!--l->count)
+	    rb_nativethread_lock_unlock(&l->lock);
+    }
 }
 
-struct CRYPTO_dynlock_value {
-    rb_nativethread_lock_t lock;
-};
-
 static struct CRYPTO_dynlock_value *
 ossl_dyn_create_callback(const char *file, int line)
 {
-    struct CRYPTO_dynlock_value *dynlock = (struct CRYPTO_dynlock_value *)OPENSSL_malloc((int)sizeof(struct CRYPTO_dynlock_value));
-    rb_nativethread_lock_initialize(&dynlock->lock);
+    /* Do not use xmalloc() here, since it may raise NoMemoryError */
+    struct CRYPTO_dynlock_value *dynlock =
+	OPENSSL_malloc(sizeof(struct CRYPTO_dynlock_value));
+    if (dynlock)
+	ossl_lock_init(dynlock);
     return dynlock;
 }
 
 static void
 ossl_dyn_lock_callback(int mode, struct CRYPTO_dynlock_value *l, const char *file, int line)
 {
-    ossl_lock_unlock(mode, &l->lock);
+    ossl_lock_unlock(mode, l);
 }
 
 static void
@@ -541,21 +554,22 @@ static unsigned long ossl_thread_id(void)
 }
 #endif
 
+static struct CRYPTO_dynlock_value *ossl_locks;
+
+static void
+ossl_lock_callback(int mode, int type, const char *file, int line)
+{
+    ossl_lock_unlock(mode, &ossl_locks[type]);
+}
+
 static void Init_ossl_locks(void)
 {
     int i;
     int num_locks = CRYPTO_num_locks();
 
-    if ((unsigned)num_locks >= INT_MAX / (int)sizeof(VALUE)) {
-	rb_raise(rb_eRuntimeError, "CRYPTO_num_locks() is too big: %d", num_locks);
-    }
-    ossl_locks = (rb_nativethread_lock_t *) OPENSSL_malloc(num_locks * (int)sizeof(rb_nativethread_lock_t));
-    if (!ossl_locks) {
-	rb_raise(rb_eNoMemError, "CRYPTO_num_locks() is too big: %d", num_locks);
-    }
-    for (i = 0; i < num_locks; i++) {
-	rb_nativethread_lock_initialize(&ossl_locks[i]);
-    }
+    ossl_locks = ALLOC_N(struct CRYPTO_dynlock_value, num_locks);
+    for (i = 0; i < num_locks; i++)
+	ossl_lock_init(&ossl_locks[i]);
 
 #ifdef HAVE_CRYPTO_THREADID_PTR
     CRYPTO_THREADID_set_callback(ossl_threadid_func);