Merge pull request #717 from rhenium/ky/cipher-update-fix-buffer-size

cipher: fix buffer overflow in Cipher#update

Author: rhenium

ext/openssl/ossl_cipher.c

@@ -387,11 +387,23 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
     if ((in_len = RSTRING_LEN(data)) == 0)
         ossl_raise(rb_eArgError, "data must not be empty");
     GetCipher(self, ctx);
-    out_len = in_len+EVP_CIPHER_CTX_block_size(ctx);
-    if (out_len <= 0) {
+
+    /*
+     * As of OpenSSL 3.2, there is no reliable way to determine the required
+     * output buffer size for arbitrary cipher modes.
+     * https://github.com/openssl/openssl/issues/22628
+     *
+     * in_len+block_size is usually sufficient, but AES key wrap with padding
+     * ciphers require in_len+15 even though they have a block size of 8 bytes.
+     *
+     * Using EVP_MAX_BLOCK_LENGTH (32) as a safe upper bound for ciphers
+     * currently implemented in OpenSSL, but this can change in the future.
+     */
+    if (in_len > LONG_MAX - EVP_MAX_BLOCK_LENGTH) {
 	ossl_raise(rb_eRangeError,
 		   "data too big to make output buffer: %ld bytes", in_len);
     }
+    out_len = in_len + EVP_MAX_BLOCK_LENGTH;
 
     if (NIL_P(str)) {
         str = rb_str_new(0, out_len);
@@ -402,7 +414,7 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
 
     if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len))
 	ossl_raise(eCipherError, NULL);
-    assert(out_len < RSTRING_LEN(str));
+    assert(out_len <= RSTRING_LEN(str));
     rb_str_set_len(str, out_len);
 
     return str;

test/openssl/test_cipher.rb

@@ -337,6 +337,22 @@ def test_aes_gcm_key_iv_order_issue
     assert_equal tag1, tag2
   end
 
+  def test_aes_keywrap_pad
+    # RFC 5649 Section 6; The second example
+    kek = ["5840df6e29b02af1ab493b705bf16ea1ae8338f4dcc176a8"].pack("H*")
+    key = ["466f7250617369"].pack("H*")
+    wrap = ["afbeb0f07dfbf5419200f2ccb50bb24f"].pack("H*")
+
+    begin
+      cipher = OpenSSL::Cipher.new("id-aes192-wrap-pad").encrypt
+    rescue OpenSSL::Cipher::CipherError, RuntimeError
+      omit "id-aes192-wrap-pad is not supported: #$!"
+    end
+    cipher.key = kek
+    ct = cipher.update(key) << cipher.final
+    assert_equal wrap, ct
+  end
+
   def test_non_aead_cipher_set_auth_data
     assert_raise(OpenSSL::Cipher::CipherError) {
       cipher = OpenSSL::Cipher.new("aes-128-cfb").encrypt