feat(requests-ca): respect REQUESTS_CA_BUNDLE environment variable (#755)

Co-authored-by: octavia-squidington-iii <contact@airbyte.com>
Co-authored-by: Aaron ("AJ") Steers <aj@airbyte.io>

Author: 3 people

airbyte_cdk/sources/streams/http/http_client.py

@@ -560,6 +560,15 @@ def send_request(
             data=data,
         )
 
+        env_settings = self._session.merge_environment_settings(
+            url=request.url,
+            proxies=request_kwargs.get("proxies"),
+            stream=request_kwargs.get("stream"),
+            verify=request_kwargs.get("verify"),
+            cert=request_kwargs.get("cert"),
+        )
+        request_kwargs = {**request_kwargs, **env_settings}
+
         response: requests.Response = self._send_with_retry(
             request=request,
             request_kwargs=request_kwargs,

unit_tests/sources/streams/http/test_http.py

@@ -92,7 +92,10 @@ def test_requests_native_token_authenticator():
 
 def test_request_kwargs_used(mocker, requests_mock):
     stream = StubBasicReadHttpStream()
-    request_kwargs = {"cert": None, "proxies": "google.com"}
+    request_kwargs = {
+        "cert": None,
+        "proxies": {"http": "http://example.com", "https": "http://example.com"},
+    }
     mocker.patch.object(stream, "request_kwargs", return_value=request_kwargs)
     send_mock = mocker.patch.object(
         stream._http_client._session, "send", wraps=stream._http_client._session.send
@@ -101,8 +104,16 @@ def test_request_kwargs_used(mocker, requests_mock):
 
     list(stream.read_records(sync_mode=SyncMode.full_refresh))
 
-    stream._http_client._session.send.assert_any_call(ANY, **request_kwargs)
     assert send_mock.call_count == 1
+    call_args = send_mock.call_args_list[0]
+    call_kwargs = call_args.kwargs
+
+    assert call_kwargs.get("cert") is None
+
+    proxies = call_kwargs.get("proxies")
+    assert proxies is not None
+    assert proxies["http"] == "http://example.com"
+    assert proxies["https"] == "http://example.com"
 
 
 def test_stub_basic_read_http_stream_read_records(mocker):

unit_tests/sources/streams/http/test_http_client.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 import logging
+import os
 from datetime import timedelta
 from unittest.mock import MagicMock, patch
 
@@ -746,6 +747,25 @@ def test_given_different_headers_then_response_is_not_cached(requests_mock):
     assert second_response.json()["test"] == "second response"
 
 
+@patch.dict("os.environ", {"REQUESTS_CA_BUNDLE": "/path/to/ca-bundle.crt"})
+def test_send_request_respects_environment_variables():
+    """Test that send_request respects REQUESTS_CA_BUNDLE environment variable."""
+    http_client = HttpClient(
+        name="test",
+        logger=MagicMock(),
+    )
+
+    with patch.object(http_client, "_send_with_retry") as mock_send_with_retry:
+        http_client.send_request(
+            http_method="GET", url="https://api.example.com", request_kwargs={"timeout": 10}
+        )
+
+        passed_kwargs = mock_send_with_retry.call_args[1]["request_kwargs"]
+
+        assert "verify" in passed_kwargs
+        assert passed_kwargs["verify"] == "/path/to/ca-bundle.crt"
+
+
 @pytest.mark.usefixtures("mock_sleep")
 @pytest.mark.parametrize(
     "response_code, expected_failure_type, error_message, exception_class",