Merge pull request #531 from no6v/pkey-sign-option-type-check

OpenSSL::PKey::PKey#sign etc. with wrong type option causes SEGV

Author: rhenium

ext/openssl/ossl_pkey.c

@@ -200,6 +200,7 @@ static VALUE
 pkey_ctx_apply_options0(VALUE args_v)
 {
     VALUE *args = (VALUE *)args_v;
+    Check_Type(args[1], T_HASH);
 
     rb_block_call(args[1], rb_intern("each"), 0, NULL,
                   pkey_ctx_apply_options_i, args[0]);

test/openssl/test_pkey_rsa.rb

@@ -108,6 +108,11 @@ def test_sign_verify_options
                                       salt_length: 20, mgf1_hash: "SHA1")
     # Defaults to PKCS #1 v1.5 padding => verification failure
     assert_equal false, key.verify("SHA256", sig_pss, data)
+
+    # option type check
+    assert_raise_with_message(TypeError, /expected Hash/) {
+      key.sign("SHA256", data, ["x"])
+    }
   end
 
   def test_sign_verify_raw