Merge branch 'release/v1.5.0'

Author: wmudge

.gitignore

@@ -1,6 +1,12 @@
 # Keep the Galaxy builds
 !cloudera-exe-*.tar.gz
 
+# Remove ansible-test output
+tests/output
+
+# Remove molecule testing deployments
+**/molecule/*/deployment
+
 ### Python template
 # Byte-compiled / optimized / DLL files
 __pycache__/

docs/configuration.yml

@@ -15,14 +15,24 @@ datahub:
   suffix:
   tags:
 de:
+  definitions:
+  suffix:
+  tags:
+  force_delete:
+  vc_suffix:
 df:
   suffix:
   min_k8s_nodes:
   max_k8s_nodes:
   public_loadbalancer:
-  ip_ranges:
+  loadbalancer_ip_ranges:
+  kube_ip_ranges:
+  cluster_subnets:
+  loadbalancer_subnets:
   teardown:
     persist:
+  force_delete:
+  terminate_deployments:
 dw:
   definitions:
   suffix:
@@ -37,6 +47,7 @@ env:
         log:
         ranger_audit_s3:
       suffix:
+      tags:
     role:
       label:
         cross_account:
@@ -51,8 +62,10 @@ env:
         log:
         ranger_audit:
       suffix:
+      tags:
     storage:
       suffix:
+      tags:
   azure:
     app:
       name:
@@ -182,12 +195,14 @@ globals:
   dynamic_inventory:
     vm:
       count:
+      os:
   gcloud_credential_file:
   infra_type:
   labels:
     admin:
     app:
     cml:
+    cde:
     credential:
     cross_account:
     data:
@@ -212,6 +227,7 @@ globals:
     table:
     user:
     vpc:
+    vpce:
   name_prefix:
   namespace_cdp:
   region:
@@ -243,6 +259,13 @@ infra:
         vpc_id:
         public_subnet_ids:
         private_subnet_ids:
+    role:
+      tags:
+    policy:
+      tags:
+    storage:
+      tags:
+    private_endpoints:
   azure:
     metagroup:
       name:
@@ -277,7 +300,6 @@ infra:
     tag_key:
     tag_value:
     vm:
-      os:
       suffix:
       type:
   gcp:
@@ -294,6 +316,9 @@ infra:
     knox:
       name:
       suffix:
+    vpce:
+      name:
+      suffix:
   storage:
     name:
     path:
@@ -361,4 +386,4 @@ data:
       suffix:
   teardown:
       delete_policies:
-      delete_roles: 
+      delete_roles: 

docs/design.md

@@ -109,12 +109,20 @@ For example, to add a pre-deploy role:
             - plat
             - run
             - animals
+            - ml
+            - dw
+            - opdb
+            - dh
       tags:
         - validate
         - infra
         - plat
         - run
         - animals
+        - ml
+        - dw
+        - opdb
+        - dh
     - name: Import the core Runlevels (and their tags)
       ansible.builtin.import_role:
         name: cloudera.exe.sequence 
@@ -158,12 +166,20 @@ For example, adding an explicit `install` tag to execute the [Installation](runl
             - plat
             - run
             - install
+            - ml
+            - dw
+            - opdb
+            - dh
       tags:
         - validate
         - infra
         - plat
         - run
         - install
+        - ml
+        - dw
+        - opdb
+        - dh
     
     - name: Validate Platform Configuration
       ansible.builtin.include_role:
@@ -175,11 +191,19 @@ For example, adding an explicit `install` tag to execute the [Installation](runl
             - plat
             - run
             - install
+            - ml
+            - dw
+            - opdb
+            - dh
       tags:
         - validate
         - plat
         - run
         - install
+        - ml
+        - dw
+        - opdb
+        - dh
 
     - name: Validate Runtime Configuration
       ansible.builtin.include_role:
@@ -190,10 +214,18 @@ For example, adding an explicit `install` tag to execute the [Installation](runl
             - validate
             - run
             - install
+            - ml
+            - dw
+            - opdb
+            - dh
       tags:
         - validate
         - run
         - install
+        - ml
+        - dw
+        - opdb
+        - dh
 
     - name: Validate Installation Configuration
       ansible.builtin.include_role:

requirements.txt

@@ -7,8 +7,9 @@ urllib3>1.24.2
 cryptography>=2.3.1
 
 # Ansible 
-jmespath    # community.general.json_query
-netaddr     # ansible.netcommon.ipaddr
+jmespath                # community.general.json_query
+netaddr                 # ansible.netcommon.ipaddr
+molecule[lint]==3.4     # Pinned due to https://github.com/ansible-community/molecule/issues/3243
 
 # CDPCLI / cdpy
 git+git://github.com/cloudera-labs/cdpy@main#egg=cdpy

roles/common/defaults/main.yml

@@ -34,6 +34,7 @@ common__public_suffix:                    "{{ globals.labels.public | default('p
 common__private_suffix:                   "{{ globals.labels.private | default('pvt') }}"
 common__security_group_knox_suffix:       "{{ globals.labels.knox | default('knox') }}"
 common__security_group_default_suffix:    "{{ globals.labels.default | default('default') }}"
+common__security_group_vpce_suffix:       "{{ globals.labels.vpce | default('vpce') }}"
 common__role_suffix:                      "{{ globals.labels.role | default('role') }}"
 common__policy_suffix:                    "{{ globals.labels.policy | default('policy') }}"
 common__storage_suffix:                   "{{ globals.labels.storage | default('storage') }}"
@@ -45,6 +46,7 @@ common__external_data_suffix:             "{{ globals.labels.external_data | def
 common__datalake_admin_suffix:            "{{ globals.labels.datalake_admin | default('dladmin') }}"
 common__ranger_audit_suffix:              "{{ globals.labels.ranger_audit | default('audit') }}"
 common__cml_suffix:                       "{{ globals.labels.cml | default('cml') }}"
+common__cde_suffix:                       "{{ globals.labels.cde | default('cde') }}"
 common__igw_suffix:                       "{{ globals.labels.internet_gateway | default('igw') }}"
 common__app_suffix:                       "{{ globals.labels.app | default('app') }}"
 common__group_suffix:                     "{{ globals.labels.group | default('group') }}"
@@ -72,9 +74,11 @@ common__vpc_public_subnets_suffix:        "{{ infra.vpc.private_subnets_suffix |
 
 common__security_group_knox_name:         "{{ infra.security_group.knox.name | default([common__namespace, common__security_group_knox_name_suffix] | join('-')) }}"
 common__security_group_default_name:      "{{ infra.security_group.default.name | default([common__namespace, common__security_group_default_name_suffix] | join('-')) }}"
+common__security_group_vpce_name:         "{{ infra.security_group.vpce.name | default([common__namespace, common__security_group_vpce_name_suffix] | join('-')) }}"
 
 common__security_group_knox_name_suffix:    "{{ infra.security_group.knox.suffix | default(common__security_group_knox_suffix) }}"
 common__security_group_default_name_suffix: "{{ infra.security_group.default.suffix | default(common__security_group_default_suffix) }}"
+common__security_group_vpce_name_suffix:    "{{ infra.security_group.vpce.suffix | default(common__security_group_vpce_suffix) }}"
 
 common__ml_path:                          "{{ infra.storage.path.ml | default('datasci') }}"
 common__de_path:                          "{{ infra.storage.path.de | default('dataeng') }}"
@@ -83,6 +87,9 @@ common__data_path:                        "{{ infra.storage.path.data | default(
 common__ranger_audit_path:                "{{ infra.storage.path.ranger_audit | default('ranger/audit') }}"
 
 # AWS Infra
+common__aws_vpc_id:                       "{{ infra.aws.vpc.existing.vpc_id | default('') }}"
+common__aws_public_subnet_ids:            "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}"
+common__aws_private_subnet_ids:           "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}"
 common__aws_region:                       "{{ infra.aws.region | default('eu-west-1') }}"
 common__aws_profile:                      "{{ infra.aws.profile | default('') }}"
 common__aws_role_suffix:                  "{{ infra.aws.role.suffix | default(common__role_suffix) }}"
@@ -141,4 +148,4 @@ common__include_datahub:                   "{{ datahub is defined | bool }}"
 common__include_opdb:                      "{{ opdb is defined | bool }}"
 
 # Teardown
-common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution
+common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution

roles/freeipa_host_group/defaults/main.yml

@@ -23,5 +23,7 @@ freeipa_host_group__env_name:  "{{ common__env_name }}"
 freeipa_host_group__infra_type:  "{{ common__infra_type }}"
 freeipa_host_group__region:    "{{ common__region }}"
 
+freeipa_host_group__gcp_project: "{{ common__gcp_project }}"
+
 # Outputs
 freeipa_host_group__host_group_name: "freeipa_server_hosts"

roles/freeipa_host_group/tasks/main.yml

@@ -46,11 +46,22 @@
 #   when: freeipa_host_group__infra_type == "azure"
 #   block:
 
-# TODO: A block per cloud provider - GCP
 # Get instance details for specific infra_type - GCP
-# - name: Gather FreeIPA instance details on GCP
-#   when: freeipa_host_group__infra_type == "gcp"
-#   block:
+- name: Gather FreeIPA instance details on GCP
+  when: freeipa_host_group__infra_type == "gcp"
+  block:
+    - name: Gather Address information used by FreeIPA GCP instance
+      google.cloud.gcp_compute_address_info:
+        region: "{{ freeipa_host_group__region }}"
+        project: "{{ freeipa_host_group__gcp_project }}"
+        # Filter on the freeipa instance name with the timestamp stripped
+        filters:
+        - "name : {{ __freeipa_server_instance_id | regex_replace('[^-]+$', '') }}*"
+      register: __gcp_freeipa_address_info
+
+    - name: Set facts for the FreeIPA server IP
+      ansible.builtin.set_fact:
+        __freeipa_server_public_ip: "{{ __gcp_freeipa_address_info.resources | map(attribute='address') }}"
 
 # Add the FreeIPA server and username to the inventory
 - name: Add FreeIPA servers to inventory

roles/info/tasks/main.yml

@@ -34,7 +34,7 @@
         - name: Set fact for the artifacts directory path
           ansible.builtin.set_fact:
             __artifacts_directory_path: "{{ __artifacts_directory_create.path }}"
-      
+
     - name: Validate artifacts directory if it does exist
       when: __artifacts_directory.stat.exists
       block:
@@ -70,6 +70,11 @@
     env: "{{ info__env_name }}"
   register: __ml_info
 
+- name: Query CDP DE Services
+  cloudera.cloud.de_info:
+    env: "{{ info__env_name }}"
+  register: __de_info
+
 - name: Query CDP Operational DBs
   cloudera.cloud.opdb_info:
     env:  "{{ info__env_name }}"
@@ -82,11 +87,12 @@
       datalake: "{{ __datalake_info.datalakes | first | default({}) }}"
       datahubs: "{{ __datahubs_info.datahubs }}"
       workspaces: "{{ __ml_info.workspaces }}"
+      services: "{{ __de_info.services }}"
       operational_dbs: "{{ __opdb_info.databases }}"
 
 - name: Save the CDP deployment details locally
   when: info__create_deployment_details
   ansible.builtin.copy:
     content: "{{ { 'deployment': deployment } | to_nice_yaml(indent=2) }}"
     dest: "{{ __artifacts_directory_path | default('.') }}/deployment_info.yml"
-  delegate_to: localhost
+  delegate_to: localhost

roles/infrastructure/defaults/main.yml

@@ -34,8 +34,8 @@ infra__public_endpoint_access:      "{{ common__public_endpoint_access }}"
 # Dynamic Inventory for Clusters
 infra__private_key_file:                 "{{ globals.ssh.private_key_file | default('') }}"
 infra__dynamic_inventory_count:          "{{ globals.dynamic_inventory.vm.count | default(0) }}"
+infra__dynamic_inventory_os:             "{{ globals.dynamic_inventory.vm.os | default('el7') }}"
 infra__dynamic_inventory_vm_suffix:      "{{ infra.dynamic_inventory.vm.suffix | default('vm') }}"
-infra__dynamic_inventory_os:             "{{ infra.dynamic_inventory.vm.os | default('centos7') }}"
 infra__dynamic_inventory_vm_type:        "{{ infra.dynamic_inventory.vm.type | default('std') }}"
 infra__dynamic_inventory_storage_type:   "{{ infra.dynamic_inventory.storage.type | default('std') }}"
 infra__dynamic_inventory_storage_size:   "{{ infra.dynamic_inventory.storage.size | default('200') }}"
@@ -89,13 +89,15 @@ infra__aws_private_subnet_ids:      "{{ infra.aws.vpc.existing.private_subnet_id
 
 infra__security_group_knox_name:    "{{ common__security_group_knox_name }}"
 infra__security_group_default_name: "{{ common__security_group_default_name }}"
+infra__security_group_vpce_name:    "{{ common__security_group_vpce_name }}"
 
 infra__ml_deploy:                   "{{ common__include_ml }}"
 infra__ml_path:                     "{{ common__ml_path }}"
 
 infra__de_deploy:                   "{{ common__include_de }}"
 infra__de_path:                     "{{ common__de_path }}"
 
+# AWS
 infra__aws_profile:                 "{{ common__aws_profile }}"
 infra__aws_vpc_az_count:            "{{ infra.aws.vpc.az_count | default(3) }}"
 infra__aws_igw_name:                "{{ infra.aws.vpc.internet_gateway.name | default([infra__namespace, infra__aws_igw_suffix] | join('-')) }}"
@@ -109,6 +111,11 @@ infra__aws_private_route_table_name: "{{ infra.aws.vpc.labels.private_route_tabl
 infra__aws_nat_gateway_name:        "{{ infra.aws.vpc.nat_gateway.name | default([infra__namespace, infra__aws_nat_gateway_suffix] | join('-')) }}"
 infra__aws_nat_gateway_suffix:      "{{ infra.aws.vpc.nat_gateway.suffix | default(common__ngw_suffix) }}"
 
+infra__aws_role_tags:               "{{ infra.aws.role.tags | default({}) }}"
+infra__aws_policy_tags:             "{{ infra.aws.policy.tags | default({}) }}"
+infra__aws_storage_tags:            "{{ infra.aws.storage.tags | default({}) }}"
+infra__aws_private_endpoints:       "{{ infra.aws.vpc.private_endpoints | default(common__tunnel) }}"
+
 # GCP
 infra__gcp_project:                 "{{ common__gcp_project }}"
 

roles/infrastructure/tasks/initialize_aws.yml

@@ -67,7 +67,6 @@
 
         - name: Set facts for existing AWS Private Subnet IDs and associate VPC ID
           ansible.builtin.set_fact:
-            infra__aws_private_subnet_ids: "{{ infra__aws_private_subnet_ids }}"
             infra__aws_subnet_ids: "{{ infra__aws_private_subnet_ids }}"
             infra__aws_vpc_id: "{{ __aws_private_subnets_info.subnets | map(attribute='vpc_id') | list | first }}"