ssl: fix tests using TLS 1.1 or older

Commit aa7f03e broke test_minmax_version and test_fallback_scsv
on systems using OpenSSL 1.1.1 with a system-wide configuration file
that specifies MinProtocol=TLSv1.2.

http://rubyci.s3.amazonaws.com/debian11/ruby-master/log/20250228T003003Z.fail.html.gz
http://rubyci.s3.amazonaws.com/rhel8/ruby-master/log/20250228T003003Z.fail.html.gz

These test cases were already broken before the commit, but they were
being skipped because check_supported_protocol_versions failed to detect
TLS 1.1 support. To avoid affected by the configuration file, explicitly
reset SSLContext#min_version when TLS 1.1 or older is required.

The test cases are also broken with OpenSSL 3.0 or later, but this is
not currently visible because it still fails to detect TLS 1.1 support.
This is caused by the default SSLContext#security_level value, as
OpenSSL 3.0 changed TLS 1.1 to be disabled at level 1.

Author: rhenium

test/openssl/test_ssl.rb

@@ -1247,12 +1247,16 @@ def check_supported_protocol_versions
 
     supported = []
     ctx_proc = proc { |ctx|
+      # The default security level is 1 in OpenSSL <= 3.1, 2 in OpenSSL >= 3.2
+      # In OpenSSL >= 3.0, TLS 1.1 or older is disabled at level 1
+      ctx.security_level = 0
       # Explicitly reset them to avoid influenced by OPENSSL_CONF
       ctx.min_version = ctx.max_version = nil
     }
     start_server(ctx_proc: ctx_proc, ignore_listener_error: true) do |port|
       possible_versions.each do |ver|
         ctx = OpenSSL::SSL::SSLContext.new
+        ctx.security_level = 0
         ctx.min_version = ctx.max_version = ver
         server_connect(port, ctx) { |ssl|
           ssl.puts "abc"; assert_equal "abc\n", ssl.gets
@@ -1303,11 +1307,15 @@ def test_minmax_version
 
     # Server enables a single version
     supported.each do |ver|
-      ctx_proc = proc { |ctx| ctx.min_version = ctx.max_version = ver }
+      ctx_proc = proc { |ctx|
+        ctx.security_level = 0
+        ctx.min_version = ctx.max_version = ver
+      }
       start_server(ctx_proc: ctx_proc, ignore_listener_error: true) { |port|
         supported.each do |cver|
           # Client enables a single version
           ctx1 = OpenSSL::SSL::SSLContext.new
+          ctx1.security_level = 0
           ctx1.min_version = ctx1.max_version = cver
           if ver == cver
             server_connect(port, ctx1) { |ssl|
@@ -1322,6 +1330,7 @@ def test_minmax_version
           if cver <= OpenSSL::SSL::TLS1_2_VERSION
             # Client enables a single version using #ssl_version=
             ctx2 = OpenSSL::SSL::SSLContext.new
+            ctx2.security_level = 0
             ctx2.ssl_version = vmap[cver][:method]
             if ver == cver
               server_connect(port, ctx2) { |ssl|
@@ -1336,6 +1345,7 @@ def test_minmax_version
 
         # Client enables all supported versions
         ctx3 = OpenSSL::SSL::SSLContext.new
+        ctx3.security_level = 0
         ctx3.min_version = ctx3.max_version = nil
         server_connect(port, ctx3) { |ssl|
           assert_equal vmap[ver][:name], ssl.ssl_version
@@ -1350,19 +1360,26 @@ def test_minmax_version
 
     # Server sets min_version (earliest is disabled)
     sver = supported[1]
-    ctx_proc = proc { |ctx| ctx.min_version = sver }
+    ctx_proc = proc { |ctx|
+      ctx.security_level = 0
+      ctx.min_version = sver
+    }
     start_server(ctx_proc: ctx_proc, ignore_listener_error: true) { |port|
       supported.each do |cver|
         # Client sets min_version
         ctx1 = OpenSSL::SSL::SSLContext.new
+        ctx1.security_level = 0
         ctx1.min_version = cver
+        ctx1.max_version = 0
         server_connect(port, ctx1) { |ssl|
           assert_equal vmap[supported.last][:name], ssl.ssl_version
           ssl.puts "abc"; assert_equal "abc\n", ssl.gets
         }
 
         # Client sets max_version
         ctx2 = OpenSSL::SSL::SSLContext.new
+        ctx2.security_level = 0
+        ctx2.min_version = 0
         ctx2.max_version = cver
         if cver >= sver
           server_connect(port, ctx2) { |ssl|
@@ -1377,7 +1394,11 @@ def test_minmax_version
 
     # Server sets max_version (latest is disabled)
     sver = supported[-2]
-    ctx_proc = proc { |ctx| ctx.max_version = sver }
+    ctx_proc = proc { |ctx|
+      ctx.security_level = 0
+      ctx.min_version = 0
+      ctx.max_version = sver
+    }
     start_server(ctx_proc: ctx_proc, ignore_listener_error: true) { |port|
       supported.each do |cver|
         # Client sets min_version
@@ -1394,6 +1415,8 @@ def test_minmax_version
 
         # Client sets max_version
         ctx2 = OpenSSL::SSL::SSLContext.new
+        ctx2.security_level = 0
+        ctx2.min_version = 0
         ctx2.max_version = cver
         server_connect(port, ctx2) { |ssl|
           if cver >= sver
@@ -1770,11 +1793,11 @@ def test_get_ephemeral_key
 
   def test_fallback_scsv
     supported = check_supported_protocol_versions
-    return unless supported.include?(OpenSSL::SSL::TLS1_1_VERSION) &&
-      supported.include?(OpenSSL::SSL::TLS1_2_VERSION)
+    unless supported.include?(OpenSSL::SSL::TLS1_1_VERSION)
+      omit "TLS 1.1 support is required to run this test case"
+    end
 
-    pend "Fallback SCSV is not supported" unless \
-      OpenSSL::SSL::SSLContext.method_defined?(:enable_fallback_scsv)
+    omit "Fallback SCSV is not supported" if libressl?
 
     start_server do |port|
       ctx = OpenSSL::SSL::SSLContext.new
@@ -1785,11 +1808,15 @@ def test_fallback_scsv
     end
 
     ctx_proc = proc { |ctx|
+      ctx.security_level = 0
+      ctx.min_version = 0
       ctx.max_version = OpenSSL::SSL::TLS1_1_VERSION
     }
     start_server(ctx_proc: ctx_proc) do |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.enable_fallback_scsv
+      ctx.security_level = 0
+      ctx.min_version = 0
       ctx.max_version = OpenSSL::SSL::TLS1_1_VERSION
       # Here is OK too
       # TLS1.2 not supported, fallback to TLS1.1 and signaling the fallback
@@ -1807,11 +1834,15 @@ def test_fallback_scsv
       # Otherwise, this test fails when using openssl 1.1.1 (or later) that supports TLS1.3.
       # TODO: We may need another test for TLS1.3 because it seems to have a different mechanism.
       ctx1 = OpenSSL::SSL::SSLContext.new
+      ctx1.security_level = 0
+      ctx1.min_version = 0
       ctx1.max_version = OpenSSL::SSL::TLS1_2_VERSION
       s1 = OpenSSL::SSL::SSLSocket.new(sock1, ctx1)
 
       ctx2 = OpenSSL::SSL::SSLContext.new
       ctx2.enable_fallback_scsv
+      ctx2.security_level = 0
+      ctx2.min_version = 0
       ctx2.max_version = OpenSSL::SSL::TLS1_1_VERSION
       s2 = OpenSSL::SSL::SSLSocket.new(sock2, ctx2)
       # AWS-LC has slightly different error messages in all-caps.