Update API client parameters for legacy alignment and CDP control plane region (#215)

* Add API parameters for endpoint_region, endpoint_tls.
* Add CDP_REGION environment variable for endpoint_region.
* Add legacy parameter aliases.
* Read CDP control plane region from configuration.
* Align TLS validation with underlying URL request parameters
* Update pagination to handle multiple "next token" parameters

Signed-off-by: Webster Mudge <wmudge@cloudera.com>
Signed-off-by: rsuplina <rsuplina@cloudera.com>
Co-authored-by: rsuplina <rsuplina@cloudera.com>

Author: wmudge

plugins/doc_fragments/cdp_client.py

@@ -50,20 +50,63 @@ class ModuleDocFragment(object):
   endpoint:
     description:
       - The Cloudera on cloud API endpoint to use.
+      - Mutually exclusive with O(endpoint_region).
     type: str
-    required: True
+    required: False
     aliases:
+      - endpoint_url
       - url
+  endpoint_region:
+    description:
+      - Specify the Cloudera on cloud API endpoint region.
+      - See L(Cloudera Control Plane regions,https://docs.cloudera.com/cdp-public-cloud/cloud/cp-regions/topics/cdp-control-plane-regions.html) for more information.
+      - If not provided, the API will attempt to use the value from the environment variable E(CDP_REGION).
+      - V(default) is an alias for the V(us-west-1) region.
+      - Mutually exclusive with O(endpoint).
+    type: str
+    required: False
+    default: "us-west-1"
+    choices:
+      - default
+      - us-west-1
+      - eu-1
+      - ap-1
+    aliases:
+      - cdp_endpoint_region
+      - cdp_region
+      - region
+  endpoint_tls:
+    description:
+      - Verify the TLS certificates for the Cloudera on cloud API endpoint.
+    type: bool
+    required: False
+    default: True
+    aliases:
+      - verify_endpoint_tls
+      - verify_tls
+      - verify_api_tls
   debug:
     description:
       - If C(true), the module will capture the Cloudera on cloud HTTP log and return it in the RV(sdk_out) and RV(sdk_out_lines) fields.
     type: bool
     required: False
     default: False
+    aliases:
+      - debug_endpoints
   http_agent:
     description:
       - The HTTP user agent to use for Cloudera on cloud API requests.
     type: str
     required: False
     default: "cloudera.cloud"
+    aliases:
+      - agent_header
+  strict:
+    description:
+      - Legacy CDPy SDK error handling.
+    type: bool
+    required: False
+    default: False
+    aliases:
+      - strict_errors
 """

plugins/module_utils/cdp_client.py

@@ -44,16 +44,17 @@ class CdpCredentialError(Exception):
 def load_cdp_config(
     credentials_path: str,
     profile: str,
-) -> Tuple[str, str]:
+) -> Tuple[str, str, str]:
     """
-    Load CDP credential configuration by parsing credential file.
+    Load CDP credential configuration by parsing credential file. If the profile has a region specified, it will be
+    returned; otherwise, the default region "us-west-1" will be used.
 
     Args:
         credentials_path: Path to CDP credentials file (supports ~ expansion)
         profile: Profile name to load from the credentials file
 
     Returns:
-        Tuple of (access_key, private_key)
+        Tuple of (access_key, private_key, region)
 
     Raises:
         CdpCredentialError: If file doesn't exist, profile not found, or keys missing
@@ -85,7 +86,13 @@ def load_cdp_config(
         msg = "CDP profile '{0}' is missing 'cdp_private_key'"
         raise CdpCredentialError(msg.format(profile))
 
-    return access_key, private_key
+    # Load region
+    if config.has_option(profile, "cdp_region"):
+        region = config.get(profile, "cdp_region")
+    else:
+        region = "us-west-1"
+
+    return access_key, private_key, region
 
 
 def create_canonical_request_string(
@@ -305,7 +312,17 @@ def wrapper(self, *args, **kwargs):
                 # Get the initial response
                 response = func(self, *args, **paginated_kwargs)
 
-                if not isinstance(response, dict) or "nextPageToken" not in response:
+                if not isinstance(response, dict):
+                    return response
+
+                # Determine which pagination token is used
+                next_token_key = None
+                if "nextPageToken" in response:
+                    next_token_key = "nextPageToken"
+                elif "nextToken" in response:
+                    next_token_key = "nextToken"
+                else:
+                    # No pagination token found, return as-is
                     return response
 
                 # Collect all items from paginated responses
@@ -320,9 +337,9 @@ def wrapper(self, *args, **kwargs):
                     else:
                         all_items[key] = value
 
-                # Continue pagination while nextPageToken exists
-                while "nextPageToken" in all_items:
-                    token = all_items.pop("nextPageToken")
+                # Continue pagination while nextToken exists
+                while next_token_key in all_items:
+                    token = all_items.pop(next_token_key)
 
                     # Add pagination parameters
                     paginated_kwargs = kwargs.copy()
@@ -492,6 +509,12 @@ def _make_request(
                 self.private_key,
             )
 
+            # Populate validate_certs from endpoint_tls
+            self.module.params["validate_certs"] = self.module.params.get(
+                "endpoint_tls",
+                True,
+            )
+
             # Add query parameters to URL if provided
             if params:
                 # Handle list parameters (e.g., guid=[guid1, guid2])

plugins/module_utils/common.py

@@ -135,19 +135,46 @@ def __init__(
                     fallback=(env_fallback, ["CDP_PROFILE"]),
                     default="default",
                 ),
-                endpoint=dict(required=True, type="str", aliases=["url"]),
-                debug=dict(required=False, type="bool", default=False),
+                endpoint=dict(
+                    required=False,
+                    type="str",
+                    aliases=["endpoint_url", "url"],
+                ),
+                endpoint_region=dict(
+                    required=False,
+                    type="str",
+                    fallback=(env_fallback, ["CDP_REGION"]),
+                    # default="us-west-1", # NOTE: Handled by load_cdp_region()
+                    aliases=["cdp_endpoint_region", "cdp_region", "region"],
+                    choices=["default", "us-west-1", "eu-1", "ap-1"],
+                ),
+                endpoint_tls=dict(
+                    required=False,
+                    type="bool",
+                    default=True,
+                    aliases=["verify_endpoint_tls", "verify_tls", "verify_api_tls"],
+                ),
+                debug=dict(
+                    required=False,
+                    type="bool",
+                    default=False,
+                    aliases=["debug_endpoints"],
+                ),
                 http_agent=dict(
                     required=False,
                     type="str",
                     default="cloudera.cloud",
+                    aliases=["agent_header"],
                 ),
             ),
             required_together=required_together + [["access_key", "private_key"]],
             bypass_checks=bypass_checks,
             no_log=no_log,
             mutually_exclusive=mutually_exclusive
-            + [["access_key", "credentials_path"]],
+            + [
+                ["access_key", "credentials_path"],
+                ["endpoint", "endpoint_region"],
+            ],
             required_one_of=required_one_of,
             add_file_common_args=add_file_common_args,
             supports_check_mode=supports_check_mode,
@@ -162,10 +189,11 @@ def __init__(
         # Load CDP credentials - check if provided via parameters first
         access_key = self.get_param("access_key")
         private_key = self.get_param("private_key")
+        region = self.get_param("endpoint_region")
 
-        # If either credential is missing, load from credentials file
-        if access_key is None or private_key is None:
-            file_access_key, file_private_key = load_cdp_config(
+        # If any credential is missing, load from credentials file
+        if access_key is None or private_key is None or region is None:
+            file_access_key, file_private_key, file_region = load_cdp_config(
                 credentials_path=self.get_param("credentials_path"),
                 profile=self.get_param("profile"),
             )
@@ -174,10 +202,23 @@ def __init__(
                 access_key = file_access_key
             if private_key is None:
                 private_key = file_private_key
+            if region is None:
+                region = file_region
 
         self.access_key: str = access_key
         self.private_key: str = private_key
 
+        # Handle legacy parameter value
+        if region == "default":
+            self.endpoint_region = "us-west-1"
+        else:
+            self.endpoint_region: str = region
+
+        # NOTE: If endpoint is not provided, construct the endpoint parameter from the region
+        # NOTE: IAM endpoints for us-west-1 need to be explicitly set to iamapi.us-west-1.altus.cloudera.com
+        if self.endpoint is None:
+            self.endpoint = f"https://api.{self.endpoint_region}.cdp.cloudera.com"
+
         # Initialize mixins parameters
         for base in self.__class__.__mro__:
             if (

tests/unit/conftest.py

@@ -133,9 +133,9 @@ def mock_ansible_module(mocker):
 @pytest.fixture()
 def mock_load_cdp_config(mocker):
     """Mock the load_cdp_config function."""
-    mocker.patch(
+    return mocker.patch(
         "ansible_collections.cloudera.cloud.plugins.module_utils.common.load_cdp_config",
-        return_value=("test-access-key", "test-private-key"),
+        return_value=("test-access-key", "test-private-key", "test-region"),
     )
 
 
@@ -146,6 +146,7 @@ def unset_cdp_env_vars(monkeypatch):
     monkeypatch.delenv("CDP_PRIVATE_KEY", raising=False)
     monkeypatch.delenv("CDP_CREDENTIALS_PATH", raising=False)
     monkeypatch.delenv("CDP_PROFILE", raising=False)
+    monkeypatch.delenv("CDP_REGION", raising=False)
 
 
 @pytest.fixture()

tests/unit/plugins/module_utils/cdp_client/test_load_cdp_config.py

@@ -38,13 +38,37 @@ def test_load_cdp_config_reads_from_file_successfully(mocker):
     mock_open_object = mocker.mock_open(read_data=config_content)
     mocker.patch("builtins.open", mock_open_object)
 
-    result_access, result_private = load_cdp_config(
+    result_access, result_private, result_region = load_cdp_config(
         credentials_path="/mock/path",
         profile="default",
     )
 
     assert result_access == "file-access-key"
     assert result_private == "file-private-key"
+    assert result_region == "us-west-1"
+
+
+def test_load_cdp_config_reads_from_file_successfully_region(mocker):
+    """Test successful reading of credentials from configuration file with cdp_region."""
+
+    config_content = """[default]
+cdp_access_key_id = file-access-key
+cdp_private_key = file-private-key
+cdp_region = file-region
+"""
+
+    mocker.patch("os.path.exists", return_value=True)
+    mock_open_object = mocker.mock_open(read_data=config_content)
+    mocker.patch("builtins.open", mock_open_object)
+
+    result_access, result_private, result_region = load_cdp_config(
+        credentials_path="/mock/path",
+        profile="default",
+    )
+
+    assert result_access == "file-access-key"
+    assert result_private == "file-private-key"
+    assert result_region == "file-region"
 
 
 def test_load_cdp_config_missing_credentials_file(mocker):
@@ -124,19 +148,21 @@ def test_load_cdp_config_custom_profile(mocker):
 [production]
 cdp_access_key_id = prod-access-key
 cdp_private_key = prod-private-key
+cdp_region = prod-region
 """
 
     mocker.patch("os.path.exists", return_value=True)
     mock_open_object = mocker.mock_open(read_data=config_content)
     mocker.patch("builtins.open", mock_open_object)
 
-    result_access, result_private = load_cdp_config(
+    result_access, result_private, result_region = load_cdp_config(
         credentials_path="/mock/path",
         profile="production",
     )
 
     assert result_access == "prod-access-key"
     assert result_private == "prod-private-key"
+    assert result_region == "prod-region"
 
 
 def test_load_cdp_config_expands_user_path(mocker):
@@ -160,7 +186,7 @@ def test_load_cdp_config_expands_user_path(mocker):
     mock_open_object = mocker.mock_open(read_data=config_content)
     mocker.patch("builtins.open", mock_open_object)
 
-    result_access, result_private = load_cdp_config(
+    result_access, result_private, result_region = load_cdp_config(
         credentials_path="~/.cdp/credentials",
         profile="default",
     )
@@ -172,6 +198,7 @@ def test_load_cdp_config_expands_user_path(mocker):
     # Verify credentials were loaded
     assert result_access == "test-access-key"
     assert result_private == "test-private-key"
+    assert result_region == "us-west-1"
 
 
 def test_load_cdp_config_path_expansion_error_message(mocker):