Merge pull request #697 from stackhpc/os-capacity-more-fixes

Various os_capacity fixes

Author: markgoddard

.yamllint

@@ -20,3 +20,4 @@ ignore: |
   .github/
   .gitlab/
   .gitlab-ci.yml
+  etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml

doc/source/configuration/monitoring.rst

@@ -140,33 +140,58 @@ enable the ceph mgr exporter.
 OpenStack Capacity
 ==================
 
-OpenStack Capacity allows you to see how much space you have avaliable
-in your cloud. StackHPC Kayobe Config includes this exporter by default
-and it's necessary that some variables are set to allow deployment.
+OpenStack Capacity allows you to see how much space you have available
+in your cloud. StackHPC Kayobe Config includes a playbook for manual
+deployment, and it's necessary that some variables are set before
+running this playbook.
 
 To successfully deploy OpenStack Capacity, you are required to specify
 the OpenStack application credentials in ``kayobe/secrets.yml`` as:
 
 .. code-block:: yaml
 
-    secrets_os_exporter_auth_url: <some_auth_url>
-    secrets_os_exporter_credential_id: <some_credential_id>
-    secrets_os_exporter_credential_secret: <some_credential_secret>
+    secrets_os_capacity_credential_id: <some_credential_id>
+    secrets_os_capacity_credential_secret: <some_credential_secret>
 
-After defining your credentials, You may deploy OpenStack Capacity
+The Keystone authentication URL and OpenStack region can be changed
+from their defaults in ``stackhpc-monitoring.yml`` should you need to
+set a different OpenStack region for your cloud. The authentication
+URL is set to use ``kolla_internal_fqdn`` by default:
+
+.. code-block:: yaml
+
+    stackhpc_os_capacity_auth_url: <some_authentication_url>
+    stackhpc_os_capacity_openstack_region_name: <some_openstack_region>
+
+Additionally, you are required to enable a conditional flag to allow
+HAProxy and Prometheus configuration to be templated during deployment.
+
+.. code-block:: yaml
+
+    stackhpc_enable_os_capacity: true
+
+If you are deploying in a cloud with internal TLS, you may be required
+to disable certificate verification for the OpenStack Capacity exporter
+if your certificate is not signed by a trusted CA.
+
+.. code-block:: yaml
+
+    stackhpc_os_capacity_openstack_verify: false
+
+After defining your credentials, you may deploy OpenStack Capacity
 using the ``ansible/deploy-os-capacity-exporter.yml`` Ansible playbook
 via Kayobe.
 
 .. code-block:: console
 
-    kayobe playbook run ansible/deploy-os-capacity-exporter.yml
+    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-os-capacity-exporter.yml
 
 It is required that you re-configure the Prometheus, Grafana and HAProxy
 services following deployment, to do this run the following Kayobe command.
 
 .. code-block:: console
 
-    kayobe overcloud service reconfigure -kt grafana,prometheus,haproxy
+    kayobe overcloud service reconfigure -kt grafana,prometheus,loadbalancer
 
 If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus
 alerts after deployment it's likely the os_exporter secrets have not been

etc/kayobe/ansible/deploy-os-capacity-exporter.yml

@@ -3,6 +3,13 @@
   gather_facts: false
 
   tasks:
+    - name: Ensure legacy os_exporter.cfg config file is deleted
+      ansible.builtin.file:
+        path: /etc/kolla/haproxy/services.d/os_exporter.cfg
+        state: absent
+      delegate_to: network
+      become: true
+
     - name: Create os-capacity directory
       ansible.builtin.file:
         path: /opt/kayobe/os-capacity/

etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2

@@ -1,10 +1,13 @@
 clouds:
   openstack:
     auth:
-      auth_url: "{{ secrets_os_exporter_auth_url }}"
-      application_credential_id: "{{ secrets_os_exporter_credential_id }}"
-      application_credential_secret: "{{ secrets_os_exporter_credential_secret }}"
-    region_name: "RegionOne"
+      auth_url: "{{ stackhpc_os_capacity_auth_url }}"
+      application_credential_id: "{{ secrets_os_capacity_credential_id }}"
+      application_credential_secret: "{{ secrets_os_capacity_credential_secret }}"
+    region_name: "{{ stackhpc_os_capacity_openstack_region_name }}"
     interface: "internal"
     identity_api_version: 3
     auth_type: "v3applicationcredential"
+{% if not stackhpc_os_capacity_openstack_verify | bool %}
+    verify: False
+{% endif %}

etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json

@@ -25,7 +25,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 2084495,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -66,7 +65,7 @@
       },
       "gridPos": {
         "h": 4,
-        "w": 2.4,
+        "w": 4.8,
         "x": 0,
         "y": 1
       },
@@ -86,7 +85,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "9.4.7",
+      "pluginVersion": "10.1.5",
       "repeat": "flavors",
       "repeatDirection": "h",
       "targets": [
@@ -96,7 +95,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "openstack_free_capacity_by_flavor_total{flavor_name=~\"$flavors\"}",
+          "expr": "round(avg_over_time(openstack_free_capacity_by_flavor_total{flavor_name=~\"$flavors\"}[30m]), 1)",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -424,6 +423,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -465,6 +465,7 @@
         "y": 17
       },
       "id": 5,
+      "interval": "10m",
       "options": {
         "legend": {
           "calcs": [
@@ -489,7 +490,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "openstack_project_usage{placement_resource=\"MEMORY_MB\"}",
+          "expr": "avg_over_time(openstack_project_usage{placement_resource=\"MEMORY_MB\"}[30m])",
           "legendFormat": "{{project_name}}",
           "range": true,
           "refId": "A"
@@ -522,6 +523,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -552,7 +554,7 @@
               }
             ]
           },
-          "unit": "decmbytes"
+          "unit": "none"
         },
         "overrides": []
       },
@@ -563,6 +565,7 @@
         "y": 17
       },
       "id": 16,
+      "interval": "10m",
       "options": {
         "legend": {
           "calcs": [
@@ -587,7 +590,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "openstack_project_usage{placement_resource=\"VCPU\"}",
+          "expr": "avg_over_time(openstack_project_usage{placement_resource=\"VCPU\"}[30m])",
           "legendFormat": "VCPU {{project_name}}",
           "range": true,
           "refId": "A"
@@ -598,7 +601,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "openstack_project_usage{placement_resource=\"PCPU\"}",
+          "expr": "avg_over_time(openstack_project_usage{placement_resource=\"PCPU\"}[30m])",
           "hide": false,
           "legendFormat": "PCPU {{project_name}}",
           "range": true,
@@ -646,6 +649,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineStyle": {
               "fill": "solid"
@@ -689,6 +693,7 @@
         "y": 26
       },
       "id": 6,
+      "interval": "10m",
       "options": {
         "legend": {
           "calcs": [
@@ -715,15 +720,15 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "openstack_free_capacity_hypervisor_by_flavor{flavor_name=~\"$flavors\"}",
+          "expr": "avg_over_time(openstack_free_capacity_hypervisor_by_flavor{flavor_name=~\"$flavors\"}[30m])",
           "format": "time_series",
           "instant": false,
           "legendFormat": "{{flavor_name}} on {{hypervisor}}",
           "range": true,
-          "refId": "Avaliable Capacity on Hypervisors"
+          "refId": "Available Capacity on Hypervisors"
         }
       ],
-      "title": "Avaliable Capacity for $flavors",
+      "title": "Available Capacity for $flavors",
       "type": "timeseries"
     },
     {
@@ -750,6 +755,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -791,6 +797,7 @@
         "y": 26
       },
       "id": 4,
+      "interval": "10m",
       "options": {
         "legend": {
           "calcs": [
@@ -814,8 +821,8 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "editorMode": "builder",
-          "expr": "openstack_hypervisor_placement_allocatable_capacity{resource=\"MEMORY_MB\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"}",
+          "editorMode": "code",
+          "expr": "avg_over_time(openstack_hypervisor_placement_allocatable_capacity{resource=\"MEMORY_MB\"}[30m]) - on(hypervisor) avg_over_time(openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"}[30m])",
           "legendFormat": "{{hypervisor}}",
           "range": true,
           "refId": "A"
@@ -885,7 +892,7 @@
     ]
   },
   "time": {
-    "from": "now-24h",
+    "from": "now-2d",
     "to": "now"
   },
   "timepicker": {},
@@ -895,4 +902,4 @@
   "version": 1,
   "weekStart": ""
 }
-{% endraw %}
+{% endraw %}