MDEV-40287 AES_ENCRYPT() and KDF() return NULL with OpenSSL 4.0

include/ssl_compat.h

@@ -24,12 +24,9 @@
 #define HAVE_OPENSSL11 1
 #define SSL_LIBRARY OpenSSL_version(OPENSSL_VERSION)
 #define ERR_remove_state(X) ERR_clear_error()
-#define EVP_CIPHER_CTX_SIZE 200
 #define EVP_MD_CTX_SIZE 80
 #undef EVP_MD_CTX_init
 #define EVP_MD_CTX_init(X) do { memset((X), 0, EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0)
-#undef EVP_CIPHER_CTX_init
-#define EVP_CIPHER_CTX_init(X) do { memset((X), 0, EVP_CIPHER_CTX_SIZE); EVP_CIPHER_CTX_reset(X); } while(0)
 
 /*
   Macros below are deprecated. OpenSSL 1.1 may define them or not,
@@ -83,7 +80,6 @@
 #endif
 
 #define EVP_CIPHER_CTX_encrypting(ctx)  ((ctx)->encrypt)
-#define EVP_CIPHER_CTX_SIZE             sizeof(EVP_CIPHER_CTX)
 
 #ifndef HAVE_WOLFSSL
 #define OPENSSL_init_ssl(X,Y)           SSL_library_init()
@@ -97,13 +93,3 @@
 #ifndef TLS1_3_VERSION
 #define SSL_CTX_set_ciphersuites(X,Y) 0
 #endif
-
-#ifdef	__cplusplus
-extern "C" {
-#endif /* __cplusplus */
-
-int check_openssl_compatibility();
-
-#ifdef	__cplusplus
-}
-#endif

mysys_ssl/CMakeLists.txt

@@ -28,7 +28,6 @@ SET(MYSYS_SSL_HIDDEN_SOURCES
     my_sha384.cc
     my_sha512.cc
     my_md5.cc
-    openssl.c
    )
 
 SET(MYSYS_SSL_SOURCES

mysys_ssl/my_crypt.cc

@@ -27,36 +27,27 @@
 
 #include <my_crypt.h>
 #include <ssl_compat.h>
-#include <cstdint>
-
-#define CTX_ALIGN 16
 
 class MyCTX
 {
 public:
-  char ctx_buf[EVP_CIPHER_CTX_SIZE + CTX_ALIGN];
   EVP_CIPHER_CTX* ctx;
   MyCTX()
   {
-#if CTX_ALIGN > 0
-    uintptr_t p= ((uintptr_t)ctx_buf + (CTX_ALIGN - 1)) & ~(CTX_ALIGN - 1);
-    ctx = reinterpret_cast<EVP_CIPHER_CTX*>(p);
-#else
-    ctx = (EVP_CIPHER_CTX*)ctx_buf;
-#endif
-
-    EVP_CIPHER_CTX_init(ctx);
+    ctx= EVP_CIPHER_CTX_new();
   }
   virtual ~MyCTX()
   {
-    EVP_CIPHER_CTX_reset(ctx);
+    EVP_CIPHER_CTX_free(ctx);
     ERR_remove_state(0);
   }
 
   virtual int init(const EVP_CIPHER *cipher, int encrypt, const uchar *key,
                    uint klen, const uchar *iv, uint ivlen)
   {
     compile_time_assert(MY_AES_CTX_SIZE >= sizeof(MyCTX));
+    if (unlikely(!ctx))
+      return MY_AES_OPENSSL_ERROR;
     if (unlikely(!cipher))
       return MY_AES_BAD_KEYSIZE;
 
@@ -115,9 +106,11 @@ class MyCTX_nopad : public MyCTX
     DBUG_ASSERT(ivlen == 0 || ivlen == sizeof(oiv));
 
     int res= MyCTX::init(cipher, encrypt, key, klen, iv, ivlen);
+    if (res)
+      return res;
 
     EVP_CIPHER_CTX_set_padding(ctx, 0);
-    return res;
+    return MY_AES_OK;
   }
 
   /** Update last partial source block, stored in source_tail array. */
@@ -213,10 +206,12 @@ class MyCTX_gcm : public MyCTX
   {
     compile_time_assert(MY_AES_CTX_SIZE >= sizeof(MyCTX_gcm));
     int res= MyCTX::init(cipher, encrypt, key, klen, iv, ivlen);
+    if (res)
+      return res;
     int real_ivlen= EVP_CIPHER_CTX_iv_length(ctx);
     aad= iv + real_ivlen;
     aadlen= ivlen - real_ivlen;
-    return res;
+    return MY_AES_OK;
   }
 
   int update(const uchar *src, uint slen, uchar *dst, uint *dlen) override
@@ -298,8 +293,11 @@ int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags,
     new (ctx) MyCTX_nopad();
   else
     new (ctx) MyCTX();
-  return ((MyCTX*)ctx)->init(ciphers[mode](klen), flags & 1,
-                             key, klen, iv, ivlen);
+  int res= ((MyCTX*)ctx)->init(ciphers[mode](klen), flags & 1,
+                               key, klen, iv, ivlen);
+  if (res)
+    ((MyCTX*)ctx)->~MyCTX();
+  return res;
 }
 
 int my_aes_crypt_update(void *ctx, const uchar *src, uint slen,

sql/mysqld.cc

@@ -4089,14 +4089,6 @@ static int init_common_variables()
     exit(1);
   }
 
-#ifdef HAVE_OPENSSL
-  if (check_openssl_compatibility())
-  {
-    sql_print_error("Incompatible OpenSSL version. Cannot continue...");
-    exit(1);
-  }
-#endif
-
   if (init_thread_environment() || mysql_init_variables())
     exit(1);