diff --git a/plugins/doc_fragments/cdp_client.py b/plugins/doc_fragments/cdp_client.py index 65608378..a684eb2e 100644 --- a/plugins/doc_fragments/cdp_client.py +++ b/plugins/doc_fragments/cdp_client.py @@ -50,20 +50,63 @@ class ModuleDocFragment(object): endpoint: description: - The Cloudera on cloud API endpoint to use. + - Mutually exclusive with O(endpoint_region). type: str - required: True + required: False aliases: + - endpoint_url - url + endpoint_region: + description: + - Specify the Cloudera on cloud API endpoint region. + - See L(Cloudera Control Plane regions,https://docs.cloudera.com/cdp-public-cloud/cloud/cp-regions/topics/cdp-control-plane-regions.html) for more information. + - If not provided, the API will attempt to use the value from the environment variable E(CDP_REGION). + - V(default) is an alias for the V(us-west-1) region. + - Mutually exclusive with O(endpoint). + type: str + required: False + default: "us-west-1" + choices: + - default + - us-west-1 + - eu-1 + - ap-1 + aliases: + - cdp_endpoint_region + - cdp_region + - region + endpoint_tls: + description: + - Verify the TLS certificates for the Cloudera on cloud API endpoint. + type: bool + required: False + default: True + aliases: + - verify_endpoint_tls + - verify_tls + - verify_api_tls debug: description: - If C(true), the module will capture the Cloudera on cloud HTTP log and return it in the RV(sdk_out) and RV(sdk_out_lines) fields. type: bool required: False default: False + aliases: + - debug_endpoints http_agent: description: - The HTTP user agent to use for Cloudera on cloud API requests. type: str required: False default: "cloudera.cloud" + aliases: + - agent_header + strict: + description: + - Legacy CDPy SDK error handling. + type: bool + required: False + default: False + aliases: + - strict_errors """ diff --git a/plugins/module_utils/cdp_client.py b/plugins/module_utils/cdp_client.py index ed8547ab..54f138d3 100644 --- a/plugins/module_utils/cdp_client.py +++ b/plugins/module_utils/cdp_client.py @@ -44,16 +44,17 @@ class CdpCredentialError(Exception): def load_cdp_config( credentials_path: str, profile: str, -) -> Tuple[str, str]: +) -> Tuple[str, str, str]: """ - Load CDP credential configuration by parsing credential file. + Load CDP credential configuration by parsing credential file. If the profile has a region specified, it will be + returned; otherwise, the default region "us-west-1" will be used. Args: credentials_path: Path to CDP credentials file (supports ~ expansion) profile: Profile name to load from the credentials file Returns: - Tuple of (access_key, private_key) + Tuple of (access_key, private_key, region) Raises: CdpCredentialError: If file doesn't exist, profile not found, or keys missing @@ -85,7 +86,13 @@ def load_cdp_config( msg = "CDP profile '{0}' is missing 'cdp_private_key'" raise CdpCredentialError(msg.format(profile)) - return access_key, private_key + # Load region + if config.has_option(profile, "cdp_region"): + region = config.get(profile, "cdp_region") + else: + region = "us-west-1" + + return access_key, private_key, region def create_canonical_request_string( @@ -305,7 +312,17 @@ def wrapper(self, *args, **kwargs): # Get the initial response response = func(self, *args, **paginated_kwargs) - if not isinstance(response, dict) or "nextPageToken" not in response: + if not isinstance(response, dict): + return response + + # Determine which pagination token is used + next_token_key = None + if "nextPageToken" in response: + next_token_key = "nextPageToken" + elif "nextToken" in response: + next_token_key = "nextToken" + else: + # No pagination token found, return as-is return response # Collect all items from paginated responses @@ -320,9 +337,9 @@ def wrapper(self, *args, **kwargs): else: all_items[key] = value - # Continue pagination while nextPageToken exists - while "nextPageToken" in all_items: - token = all_items.pop("nextPageToken") + # Continue pagination while nextToken exists + while next_token_key in all_items: + token = all_items.pop(next_token_key) # Add pagination parameters paginated_kwargs = kwargs.copy() @@ -492,6 +509,12 @@ def _make_request( self.private_key, ) + # Populate validate_certs from endpoint_tls + self.module.params["validate_certs"] = self.module.params.get( + "endpoint_tls", + True, + ) + # Add query parameters to URL if provided if params: # Handle list parameters (e.g., guid=[guid1, guid2]) diff --git a/plugins/module_utils/common.py b/plugins/module_utils/common.py index 47a0531f..eb497c05 100644 --- a/plugins/module_utils/common.py +++ b/plugins/module_utils/common.py @@ -135,19 +135,46 @@ def __init__( fallback=(env_fallback, ["CDP_PROFILE"]), default="default", ), - endpoint=dict(required=True, type="str", aliases=["url"]), - debug=dict(required=False, type="bool", default=False), + endpoint=dict( + required=False, + type="str", + aliases=["endpoint_url", "url"], + ), + endpoint_region=dict( + required=False, + type="str", + fallback=(env_fallback, ["CDP_REGION"]), + # default="us-west-1", # NOTE: Handled by load_cdp_region() + aliases=["cdp_endpoint_region", "cdp_region", "region"], + choices=["default", "us-west-1", "eu-1", "ap-1"], + ), + endpoint_tls=dict( + required=False, + type="bool", + default=True, + aliases=["verify_endpoint_tls", "verify_tls", "verify_api_tls"], + ), + debug=dict( + required=False, + type="bool", + default=False, + aliases=["debug_endpoints"], + ), http_agent=dict( required=False, type="str", default="cloudera.cloud", + aliases=["agent_header"], ), ), required_together=required_together + [["access_key", "private_key"]], bypass_checks=bypass_checks, no_log=no_log, mutually_exclusive=mutually_exclusive - + [["access_key", "credentials_path"]], + + [ + ["access_key", "credentials_path"], + ["endpoint", "endpoint_region"], + ], required_one_of=required_one_of, add_file_common_args=add_file_common_args, supports_check_mode=supports_check_mode, @@ -162,10 +189,11 @@ def __init__( # Load CDP credentials - check if provided via parameters first access_key = self.get_param("access_key") private_key = self.get_param("private_key") + region = self.get_param("endpoint_region") - # If either credential is missing, load from credentials file - if access_key is None or private_key is None: - file_access_key, file_private_key = load_cdp_config( + # If any credential is missing, load from credentials file + if access_key is None or private_key is None or region is None: + file_access_key, file_private_key, file_region = load_cdp_config( credentials_path=self.get_param("credentials_path"), profile=self.get_param("profile"), ) @@ -174,10 +202,23 @@ def __init__( access_key = file_access_key if private_key is None: private_key = file_private_key + if region is None: + region = file_region self.access_key: str = access_key self.private_key: str = private_key + # Handle legacy parameter value + if region == "default": + self.endpoint_region = "us-west-1" + else: + self.endpoint_region: str = region + + # NOTE: If endpoint is not provided, construct the endpoint parameter from the region + # NOTE: IAM endpoints for us-west-1 need to be explicitly set to iamapi.us-west-1.altus.cloudera.com + if self.endpoint is None: + self.endpoint = f"https://api.{self.endpoint_region}.cdp.cloudera.com" + # Initialize mixins parameters for base in self.__class__.__mro__: if ( diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 7165b6d3..059f8b1f 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -133,9 +133,9 @@ def mock_ansible_module(mocker): @pytest.fixture() def mock_load_cdp_config(mocker): """Mock the load_cdp_config function.""" - mocker.patch( + return mocker.patch( "ansible_collections.cloudera.cloud.plugins.module_utils.common.load_cdp_config", - return_value=("test-access-key", "test-private-key"), + return_value=("test-access-key", "test-private-key", "test-region"), ) @@ -146,6 +146,7 @@ def unset_cdp_env_vars(monkeypatch): monkeypatch.delenv("CDP_PRIVATE_KEY", raising=False) monkeypatch.delenv("CDP_CREDENTIALS_PATH", raising=False) monkeypatch.delenv("CDP_PROFILE", raising=False) + monkeypatch.delenv("CDP_REGION", raising=False) @pytest.fixture() diff --git a/tests/unit/plugins/module_utils/cdp_client/test_load_cdp_config.py b/tests/unit/plugins/module_utils/cdp_client/test_load_cdp_config.py index 3055d066..79a677ff 100644 --- a/tests/unit/plugins/module_utils/cdp_client/test_load_cdp_config.py +++ b/tests/unit/plugins/module_utils/cdp_client/test_load_cdp_config.py @@ -38,13 +38,37 @@ def test_load_cdp_config_reads_from_file_successfully(mocker): mock_open_object = mocker.mock_open(read_data=config_content) mocker.patch("builtins.open", mock_open_object) - result_access, result_private = load_cdp_config( + result_access, result_private, result_region = load_cdp_config( credentials_path="/mock/path", profile="default", ) assert result_access == "file-access-key" assert result_private == "file-private-key" + assert result_region == "us-west-1" + + +def test_load_cdp_config_reads_from_file_successfully_region(mocker): + """Test successful reading of credentials from configuration file with cdp_region.""" + + config_content = """[default] +cdp_access_key_id = file-access-key +cdp_private_key = file-private-key +cdp_region = file-region +""" + + mocker.patch("os.path.exists", return_value=True) + mock_open_object = mocker.mock_open(read_data=config_content) + mocker.patch("builtins.open", mock_open_object) + + result_access, result_private, result_region = load_cdp_config( + credentials_path="/mock/path", + profile="default", + ) + + assert result_access == "file-access-key" + assert result_private == "file-private-key" + assert result_region == "file-region" def test_load_cdp_config_missing_credentials_file(mocker): @@ -124,19 +148,21 @@ def test_load_cdp_config_custom_profile(mocker): [production] cdp_access_key_id = prod-access-key cdp_private_key = prod-private-key +cdp_region = prod-region """ mocker.patch("os.path.exists", return_value=True) mock_open_object = mocker.mock_open(read_data=config_content) mocker.patch("builtins.open", mock_open_object) - result_access, result_private = load_cdp_config( + result_access, result_private, result_region = load_cdp_config( credentials_path="/mock/path", profile="production", ) assert result_access == "prod-access-key" assert result_private == "prod-private-key" + assert result_region == "prod-region" def test_load_cdp_config_expands_user_path(mocker): @@ -160,7 +186,7 @@ def test_load_cdp_config_expands_user_path(mocker): mock_open_object = mocker.mock_open(read_data=config_content) mocker.patch("builtins.open", mock_open_object) - result_access, result_private = load_cdp_config( + result_access, result_private, result_region = load_cdp_config( credentials_path="~/.cdp/credentials", profile="default", ) @@ -172,6 +198,7 @@ def test_load_cdp_config_expands_user_path(mocker): # Verify credentials were loaded assert result_access == "test-access-key" assert result_private == "test-private-key" + assert result_region == "us-west-1" def test_load_cdp_config_path_expansion_error_message(mocker): diff --git a/tests/unit/plugins/module_utils/cdp_client/test_service_module.py b/tests/unit/plugins/module_utils/cdp_client/test_service_module.py index c4cf7c68..c54b5452 100644 --- a/tests/unit/plugins/module_utils/cdp_client/test_service_module.py +++ b/tests/unit/plugins/module_utils/cdp_client/test_service_module.py @@ -227,6 +227,26 @@ def test_services_module_initialization_basic( ): """Test basic ServicesModule initialization.""" + module_args( + {}, + ) + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.test-region.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_endpoint_explicit( + self, + module_args, + ): + """Test ServicesModule explicit endpoint.""" + module_args( { "endpoint": "example-endpoint", @@ -243,6 +263,259 @@ def test_services_module_initialization_basic( assert module.api_client is not None assert isinstance(module.api_client, RestClient) + def test_services_module_initialization_endpoint_region_default( + self, + module_args, + ): + """Test ServicesModule default endpoint.""" + + module_args( + { + "endpoint_region": "default", + }, + ) + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.us-west-1.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_endpoint_region_us_west_1( + self, + module_args, + ): + """Test ServicesModule US-WEST-1 endpoint.""" + + module_args( + { + "endpoint_region": "us-west-1", + }, + ) + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.us-west-1.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_endpoint_region_eu_1( + self, + module_args, + ): + """Test ServicesModule EU-1 endpoint.""" + + module_args( + { + "endpoint_region": "eu-1", + }, + ) + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.eu-1.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_endpoint_region_ap_1( + self, + module_args, + ): + """Test ServicesModule AP-1 endpoint.""" + + module_args( + { + "endpoint_region": "ap-1", + }, + ) + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.ap-1.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_endpoint_region_env( + self, + module_args, + monkeypatch, + ): + """Test ServicesModule environment variable endpoint.""" + + module_args( + {}, + ) + + monkeypatch.setenv("CDP_REGION", "eu-1") + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.eu-1.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_credentials( + self, + module_args, + ): + """Test ServicesModule explicit credentials.""" + + module_args( + { + "access_key": "explicit-access-key", + "private_key": "explicit-private-key", + }, + ) + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.test-region.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "explicit-access-key" + assert module.private_key == "explicit-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_credentials_env( + self, + module_args, + monkeypatch, + ): + """Test ServicesModule environment variable credentials.""" + + module_args( + {}, + ) + + monkeypatch.setenv("CDP_ACCESS_KEY_ID", "env-access-key") + monkeypatch.setenv("CDP_PRIVATE_KEY", "env-private-key") + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + assert module.endpoint == "https://api.test-region.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "env-access-key" + assert module.private_key == "env-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_profile_env( + self, + module_args, + monkeypatch, + mock_load_cdp_config, + ): + """Test ServicesModule environment variable credentials.""" + + module_args( + {}, + ) + + monkeypatch.setenv("CDP_PROFILE", "env-profile") + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + mock_load_cdp_config.assert_called_once_with( + credentials_path="~/.cdp/credentials", + profile="env-profile", + ) + + assert module.endpoint == "https://api.test-region.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_cred_path_env( + self, + module_args, + monkeypatch, + mock_load_cdp_config, + ): + """Test ServicesModule environment variable credentials.""" + + module_args( + {}, + ) + + monkeypatch.setenv("CDP_CREDENTIALS_PATH", "env-cred-path") + + module = ConcreteServicesModule() + + # Verify default (or mock) attributes are set + mock_load_cdp_config.assert_called_once_with( + credentials_path="env-cred-path", + profile="default", + ) + + assert module.endpoint == "https://api.test-region.cdp.cloudera.com" + assert module.debug_log is False + assert module.access_key == "test-access-key" + assert module.private_key == "test-private-key" + assert module.api_client is not None + assert isinstance(module.api_client, RestClient) + + def test_services_module_initialization_invalid_endpoint_region( + self, + module_args, + ): + """Test invalid endpoint region in ServicesModule initialization.""" + + module_args( + { + "endpoint_region": "invalid-region", + }, + ) + + with pytest.raises( + AnsibleFailJson, + match="value of endpoint_region must be one of: default, us-west-1, eu-1, ap-1, got: invalid-region", + ): + ConcreteServicesModule() + + def test_services_module_initialization_invalid_endpoint_parameters( + self, + module_args, + ): + """Test invalid parameters in ServicesModule endpoint initialization.""" + + module_args( + { + "endpoint": "example-endpoint", + "endpoint_region": "example-region", + }, + ) + + with pytest.raises( + AnsibleFailJson, + match="parameters are mutually exclusive: endpoint|endpoint_region", + ): + ConcreteServicesModule() + def test_services_module_initialization_missing_private_key( self, module_args, @@ -251,9 +524,7 @@ def test_services_module_initialization_missing_private_key( module_args( { - "endpoint": "example-endpoint", "access_key": "example-access-key", - # "private_key": "test-private-key", }, ) @@ -271,8 +542,6 @@ def test_services_module_initialization_missing_access_key( module_args( { - "endpoint": "example-endpoint", - # "access_key": "example-access-key", "private_key": "test-private-key", }, ) @@ -283,7 +552,7 @@ def test_services_module_initialization_missing_access_key( ): ConcreteServicesModule() - def test_services_module_initialization_invalid_parameters( + def test_services_module_initialization_invalid_credential_parameters( self, module_args, ): @@ -291,7 +560,6 @@ def test_services_module_initialization_invalid_parameters( module_args( { - "endpoint": "example-endpoint", "access_key": "example-access-key", "credentials_path": "test-credentials-path", },