pkcs7: fix error queue leak in OpenSSL::PKCS7#detached

Only call PKCS7_get_detached() if the PKCS7 object is a signed-data.
This is only useful for the content type, and leaves an error entry if
called on a PKCS7 object with a different content type.

Author: rhenium

ext/openssl/ossl_pkcs7.c

@@ -510,6 +510,8 @@ ossl_pkcs7_get_detached(VALUE self)
 {
     PKCS7 *p7;
     GetPKCS7(self, p7);
+    if (!PKCS7_type_is_signed(p7))
+        return Qfalse;
     return PKCS7_get_detached(p7) ? Qtrue : Qfalse;
 }
 

test/openssl/test_pkcs7.rb

@@ -265,6 +265,7 @@ def test_data
 
     p7 = OpenSSL::PKCS7.new(asn1)
     assert_equal(:data, p7.type)
+    assert_equal(false, p7.detached)
     assert_equal(false, p7.detached?)
     # Not applicable
     assert_nil(p7.certificates)