diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c index db08b9374d9..ba58594b7be 100644 --- a/ext/openssl/ossl_cipher.c +++ b/ext/openssl/ossl_cipher.c @@ -325,6 +325,9 @@ ossl_cipher_update_long(EVP_CIPHER_CTX *ctx, unsigned char *out, int limit = INT_MAX / 2 + 1; rb_len_t out_len = 0; + if (out == in && in_len > limit) + ossl_raise(eCipherError, "input too large for in-place update"); + do { int in_part_len = in_len > limit ? limit : (int)in_len; @@ -361,19 +364,19 @@ static VALUE ossl_cipher_update(int argc, VALUE *argv, VALUE self) { EVP_CIPHER_CTX *ctx; - unsigned char *in; rb_len_t in_len, out_len; VALUE data, str; rb_scan_args(argc, argv, "11", &data, &str); + GetCipher(self, ctx); if (!RTEST(rb_attr_get(self, id_key_set))) ossl_raise(eCipherError, "key not set"); StringValue(data); - in = (unsigned char *)RSTRING_PTR(data); + if (!NIL_P(str)) + StringValue(str); in_len = RSTRING_LEN(data); - GetCipher(self, ctx); /* * As of OpenSSL 3.2, there is no reliable way to determine the required @@ -396,15 +399,15 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self) if (NIL_P(str)) str = rb_str_buf_new(out_len); else { - StringValue(str); if ((rb_len_t)rb_str_capacity(str) >= out_len) rb_str_modify(str); else rb_str_modify_expand(str, out_len - RSTRING_LEN(str)); } - if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), - &out_len, in, in_len)) + if (!ossl_cipher_update_long(ctx, + (unsigned char *)RSTRING_PTR(str), &out_len, + (unsigned char *)RSTRING_PTR(data), in_len)) ossl_raise(eCipherError, "EVP_CipherUpdate"); rb_str_set_len(str, out_len); diff --git a/test/openssl/test_cipher.rb b/test/openssl/test_cipher.rb index b7926780c5b..890dc760d58 100644 --- a/test/openssl/test_cipher.rb +++ b/test/openssl/test_cipher.rb @@ -117,7 +117,7 @@ class OpenSSL::TestCipher < OpenSSL::TestCase } end - def test_ctr_if_exists + def test_aes_ctr # NIST SP 800-38A F.5.1 key = ["2b7e151628aed2a6abf7158809cf4f3c"].pack("H*") iv = ["f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"].pack("H*") @@ -125,10 +125,16 @@ class OpenSSL::TestCipher < OpenSSL::TestCase "ae2d8a571e03ac9c9eb76fac45af8e51"].pack("H*") ct = ["874d6191b620e3261bef6864990db6ce" \ "9806f66b7970fdff8617187bb9fffdff"].pack("H*") - cipher = new_encryptor("aes-128-ctr", key: key, iv: iv, padding: 0) + cipher = new_encryptor("aes-128-ctr", key: key, iv: iv) assert_equal ct, cipher.update(pt) << cipher.final - cipher = new_decryptor("aes-128-ctr", key: key, iv: iv, padding: 0) + cipher = new_decryptor("aes-128-ctr", key: key, iv: iv) assert_equal pt, cipher.update(ct) << cipher.final + + # in-place update + cipher = new_encryptor("aes-128-ctr", key: key, iv: iv) + buf = Marshal.load(Marshal.dump(pt)) + assert_equal(ct, cipher.update(buf, buf)) + assert_equal(ct, buf) end def test_update_with_buffer