Update to support Datahub definitions (#16)

* Add tasks for retrieving datahub definitions and filtering by datalake version. 
* Update tasks for constructing datahub definition requesting datahub deployment to take either the instance group details OR a definition
* Add task to `initialize_setup_azure.yml` to grab the subnet information needed for datahub creation. 
* Modify task in `initialize_setup_gcp.yml` to work properly. No selflink in Subnet Details.

Signed-off-by: Chris Perro <cmperro@gmail.com>

Author: cmperro

roles/runtime/tasks/initialize_base.yml

@@ -94,23 +94,34 @@
       ansible.builtin.set_fact:
         run__datahub_available_templates: "{{ __datahub_template_info.templates | selectattr('productVersion', 'search', run__cdp_datalake_version) | map(attribute='clusterTemplateName') | list }}"
 
+    - name: Retrieve available CDP Datahub definitions
+      cloudera.cloud.datahub_definition_info:
+      register: __datahub_definition_info
+
+    - name: Set fact for available CDP Datahub definitions by Datalake version
+      ansible.builtin.set_fact:
+        run__datahub_available_definitions: "{{ __datahub_definition_info.definitions | selectattr('productVersion', 'search', run__cdp_datalake_version) | map(attribute='clusterDefinitionName') | list }}"
+
     - name: Construct CDP Datahub configurations
       ansible.builtin.set_fact:
         run__datahub_configs: "{{ run__datahub_configs | default([]) | union([config]) }}"
       vars:
         include: "{{ lookup('template', __datahub_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
-        overlay: "{{ __datahub_config.instance_groups | default(include.instance_groups) | mandatory }}"
+        overlay: "{{ __datahub_config.instance_groups | default(include.instance_groups | default([]) ) }}"
         base: "{{ run__datahub_instance_group_base }}"
         config:
           name: "{{ __datahub_config.name | default([run__namespace, __datahub_config.suffix | default(include.suffix) | default(run__datahub_suffix)] | join('-')) }}"
-          template: "{{ __datahub_config.template | default(include.template) | mandatory }}"
-          instance_groups: "{{ overlay | map('cloudera.exe.combine_onto', base, recursive=True) | list }}"
+          template: "{{ __datahub_config.template | default(include.template | default(omit) ) }}"
+          definition: "{{ __datahub_config.definition | default(include.definition | default(omit) ) }}"
+          instance_groups: "{{ (overlay | length > 0) | ternary(overlay | map('cloudera.exe.combine_onto', base, recursive=True) | list, omit) }}"
           tags: "{{ __datahub_config.tags | default(run__datahub_tags) }}"
       loop: "{{ run__datahub_definitions }}"
       loop_control:
         loop_var: __datahub_config
         label: "{{ config.name }}"
-      failed_when: run__datahub_available_templates | select('search', config.template) | list | length < 1
+      failed_when:
+        - "'template' in __datahub_config or 'template' in include"
+        - run__datahub_available_templates | select('search', config.template) | list | length < 1
 
     - name: Check CDP Datahub names
       ansible.builtin.assert:

roles/runtime/tasks/initialize_setup_azure.yml

@@ -10,4 +10,9 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.
+
+- name: Set fact for Azure Subnet Names by assignment
+  when: infra__azure_subnets is defined
+  ansible.builtin.set_fact:
+    run__datahub_subnet_ids: "{{ infra__azure_subnets }}"

roles/runtime/tasks/initialize_setup_gcp.yml

@@ -46,6 +46,6 @@
     - __gcp_subnets_discovered is defined
     - __gcp_subnets_discovered.resources is defined
     - __gcp_subnets_discovered.resources | length > 0
-    - __gcp_subnet_item.selfLink in run__gcp_vpc_discovered.subnetworks
+    #- __gcp_subnet_item.selfLink in run__gcp_vpc_discovered.subnetworks
   ansible.builtin.set_fact:
-    run__datahub_subnet_ids: "{{ __gcp_subnets_discovered | json_query('resources[*].name') }}"
+    run__datahub_subnet_ids: "{{ __gcp_subnets_discovered | json_query('resources[*].name') }}"

roles/runtime/tasks/setup_base.yml

@@ -23,8 +23,9 @@
     subnet: "{{ run__datahub_subnet_ids | first }}"
     image: "{{ run__datahub_image_catalog_entry.uuid }}"
     catalog: "{{ run__datahub_image_catalog_name }}"
-    template: "{{ run__datahub_available_templates | select('ansible.builtin.search', __datahub_config_item.template) | list | first }}"
-    groups: "{{ __datahub_config_item.instance_groups }}"
+    template: "{{ ('template' in __datahub_config_item) | ternary(run__datahub_available_templates | select('ansible.builtin.search', __datahub_config_item.template | default(omit)) | list | first, omit, omit) }}"
+    definition: "{{ ('definition' in __datahub_config_item) |  ternary(run__datahub_available_definitions | select('ansible.builtin.search', __datahub_config_item.definition | default(omit)) | list | first, omit, omit) }}"
+    groups: "{{ __datahub_config_item.instance_groups | default(omit) }}"
     tags: "{{ __datahub_config_item.tags }}"
     wait: yes
   loop_control: