pkey: resume key generation after interrupt

Key/parameter generation (OpenSSL::PKey::*.{new,generate}) immediately
aborts when it is done with GVL released (in other words, no block is
given) and the thread is interrupted (e.g., by a signal) during the
operation.

Have ossl_generate_cb_2() acquire GVL and call rb_thread_check_ints()
if needed to process the pending interrupt rather than abort the
operation completely by returning 0.

Reference: https://bugs.ruby-lang.org/issues/14882

Author: rhenium

ext/openssl/ossl_pkey.c

@@ -20,6 +20,21 @@ static ID id_private_q;
 /*
  * callback for generating keys
  */
+static VALUE
+call_check_ints0(VALUE arg)
+{
+    rb_thread_check_ints();
+    return Qnil;
+}
+
+static void *
+call_check_ints(void *arg)
+{
+    int state;
+    rb_protect(call_check_ints0, Qnil, &state);
+    return (void *)(VALUE)state;
+}
+
 int
 ossl_generate_cb_2(int p, int n, BN_GENCB *cb)
 {
@@ -38,19 +53,26 @@ ossl_generate_cb_2(int p, int n, BN_GENCB *cb)
 	*/
 	rb_protect(rb_yield, ary, &state);
 	if (state) {
-	    arg->stop = 1;
 	    arg->state = state;
+	    return 0;
+	}
+    }
+    if (arg->interrupted) {
+	arg->interrupted = 0;
+	state = (int)(VALUE)rb_thread_call_with_gvl(call_check_ints, NULL);
+	if (state) {
+	    arg->state = state;
+	    return 0;
 	}
     }
-    if (arg->stop) return 0;
     return 1;
 }
 
 void
 ossl_generate_cb_stop(void *ptr)
 {
     struct ossl_generate_cb_arg *arg = (struct ossl_generate_cb_arg *)ptr;
-    arg->stop = 1;
+    arg->interrupted = 1;
 }
 
 static void

ext/openssl/ossl_pkey.h

@@ -41,7 +41,7 @@ extern const rb_data_type_t ossl_evp_pkey_type;
 
 struct ossl_generate_cb_arg {
     int yield;
-    int stop;
+    int interrupted;
     int state;
 };
 int ossl_generate_cb_2(int p, int n, BN_GENCB *cb);