x509name: fix handling of X509_NAME_{oneline,print_ex}() return value

X509_NAME_print_ex() behaves differently depending on the passed flags.
When XN_FLAG_COMPAT is specified, it returns either 1 on success or 0
on error. Otherwise, it returns the byte size written or -1 on error.
This means 0 return is not necessarily an error.

Also, X509_NAME_oneline() return value needs to be checked as it may
fail with a NULL return.

Fixes: #200

Author: rhenium

ext/openssl/ossl_x509name.c

@@ -239,27 +239,27 @@ ossl_x509name_to_s_old(VALUE self)
 {
     X509_NAME *name;
     char *buf;
-    VALUE str;
 
     GetX509Name(self, name);
     buf = X509_NAME_oneline(name, NULL, 0);
-    str = rb_str_new2(buf);
-    OPENSSL_free(buf);
-
-    return str;
+    if (!buf)
+	ossl_raise(eX509NameError, "X509_NAME_oneline");
+    return ossl_buf2str(buf, rb_long2int(strlen(buf)));
 }
 
 static VALUE
 x509name_print(VALUE self, unsigned long iflag)
 {
     X509_NAME *name;
     BIO *out;
+    int ret;
 
     GetX509Name(self, name);
     out = BIO_new(BIO_s_mem());
     if (!out)
 	ossl_raise(eX509NameError, NULL);
-    if (!X509_NAME_print_ex(out, name, 0, iflag)) {
+    ret = X509_NAME_print_ex(out, name, 0, iflag);
+    if (ret < 0 || iflag == XN_FLAG_COMPAT && ret == 0) {
 	BIO_free(out);
 	ossl_raise(eX509NameError, "X509_NAME_print_ex");
     }

test/test_x509name.rb

@@ -322,6 +322,34 @@ def test_add_entry_street
     assert_equal("Namiki", ary[5][1])
   end
 
+  def test_to_s
+    dn = [
+      ["DC", "org"],
+      ["DC", "ruby-lang"],
+      ["CN", "フー, バー"],
+    ]
+    name = OpenSSL::X509::Name.new
+    dn.each { |x| name.add_entry(*x) }
+
+    assert_equal "/DC=org/DC=ruby-lang/" \
+      "CN=\\xE3\\x83\\x95\\xE3\\x83\\xBC, \\xE3\\x83\\x90\\xE3\\x83\\xBC",
+      name.to_s
+    # OpenSSL escapes characters with MSB by default
+    assert_equal \
+      "CN=\\E3\\83\\95\\E3\\83\\BC\\, \\E3\\83\\90\\E3\\83\\BC," \
+      "DC=ruby-lang,DC=org",
+      name.to_s(OpenSSL::X509::Name::RFC2253)
+    assert_equal "DC = org, DC = ruby-lang, " \
+      "CN = \"\\E3\\83\\95\\E3\\83\\BC, \\E3\\83\\90\\E3\\83\\BC\"",
+      name.to_s(OpenSSL::X509::Name::ONELINE)
+
+    empty = OpenSSL::X509::Name.new
+    assert_equal "", empty.to_s
+    assert_equal "", empty.to_s(OpenSSL::X509::Name::COMPAT)
+    assert_equal "", empty.to_s(OpenSSL::X509::Name::RFC2253)
+    assert_equal "", empty.to_s(OpenSSL::X509::Name::ONELINE)
+  end
+
   def test_equals2
     n1 = OpenSSL::X509::Name.parse 'CN=a'
     n2 = OpenSSL::X509::Name.parse 'CN=a'