Update metric scraping for new webservers

vmcj · vmcj · commit 90c50057cbd3 · 2022-04-15T10:47:27.000+02:00
diff --git a/icpc-wf/ansible/cds.yml b/icpc-wf/ansible/cds.yml
@@ -27,3 +27,12 @@
       tags: ssh
     - role: cds
       tags: cds
+    - role: prometheus_target_web
+      tags: prometheus_target_web
+      vars:
+        MARIADB: false
+        FPM: false
+      when: GRAFANA_MONITORING
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml
@@ -39,6 +39,9 @@
       when: KEEPALIVED_PRIORITY is defined
     - role: prometheus_target_web
       tags: prometheus_target_web
+      vars:
+        MARIADB: true
+        FPM: true
       when: GRAFANA_MONITORING
     - role: prometheus_target_all
       tags: prometheus_target_all
diff --git a/icpc-wf/ansible/roles/grafana/defaults/main.yml b/icpc-wf/ansible/roles/grafana/defaults/main.yml
@@ -0,0 +1 @@
+grafana_port: 8443
diff --git a/icpc-wf/ansible/roles/grafana/handlers/main.yml b/icpc-wf/ansible/roles/grafana/handlers/main.yml
@@ -2,7 +2,25 @@
 # Define here handlers associated to this role.
 
 - name: restart nginx-exporter
-  service: name=prometheus-nginx-exporter enabled=yes state=restarted
+  service:
+    name: prometheus-nginx-exporter
+    enabled: true
+    state: restarted
 
 - name: restart php-exporter
-  service: name=php-fpm-exporter enabled=yes state=restarted
+  service:
+    name: php-fpm-exporter
+    enabled: true
+    state: restarted
+
+- name: restart nginx
+  service:
+    name: nginx
+    enabled: true
+    state: restarted
+
+- name: restart grafana
+  service:
+    name: grafana-server
+    enabled: true
+    state: restarted
diff --git a/icpc-wf/ansible/roles/grafana/tasks/main.yml b/icpc-wf/ansible/roles/grafana/tasks/main.yml
@@ -15,49 +15,58 @@
   template:
     src: prometheus.yml.j2
     dest: /etc/prometheus/prometheus.yml
-
-- name: restart prometheus
-  service: name=prometheus state=restarted
+  notify: restart prometheus
 
 ## Setup grafana
+- name: Install grafana
+  apt:
+    deb: https://dl.grafana.com/enterprise/release/grafana-enterprise_8.4.6_amd64.deb
+    state: present
+  notify: restart grafana
+
 - name: configure grafana
   synchronize:
     src: files/grafana/environment
     dest: /etc/default/grafana-server
+  notify: restart grafana
 
 - name: set up grafana datasources
   synchronize:
     src: files/grafana/datasources.yml
     dest: /etc/grafana/provisioning/datasources/default.yml
+  notify: restart grafana
 
 - name: set up grafana dashboards
   synchronize:
     src: files/grafana/dashboards.yml
     dest: /etc/grafana/provisioning/dashboards/default.yml
+  notify: restart grafana
 
 - name: copy grafana dashboards
   synchronize:
     src: dashboards/
     dest: /etc/grafana/dashboards/
-
-- name: restart grafana
-  service: name=grafana-server state=restarted
+  notify: restart grafana
 
 # Setup nginx with selfsigned certificate
 - name: copy ssl cert
   synchronize:
-    src=ssl.crt
-    dest=/etc/ssl/certs/grafana.crt
+    src: ssl.crt
+    dest: /etc/ssl/certs/grafana.crt
+  notify: restart nginx
 
 - name: copy ssl key
   synchronize:
-    src=ssl.key
-    dest=/etc/ssl/private/grafana.key
+    src: ssl.key
+    dest: /etc/ssl/private/grafana.key
+  notify: restart nginx
 
 - name: copy default nginx config
-  synchronize:
-    src=nginx.conf
-    dest=/etc/nginx/sites-enabled/grafana.conf
+  template:
+    src: nginx.conf.j2
+    dest: /etc/nginx/sites-enabled/grafana.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: restart nginx
 
-- name: restart nginx
-  service: name=nginx enabled=yes state=restarted
diff --git a/icpc-wf/ansible/roles/grafana/templates/nginx.conf.j2 b/icpc-wf/ansible/roles/grafana/templates/nginx.conf.j2
diff --git a/icpc-wf/ansible/roles/grafana/templates/prometheus.yml.j2 b/icpc-wf/ansible/roles/grafana/templates/prometheus.yml.j2
@@ -3,6 +3,9 @@ global:
   evaluation_interval: 15s # By default, scrape targets every 15 seconds.
   # scrape_timeout is set to the global default (10s).
 scrape_configs:
+  - job_name: 'grafana'
+    static_configs:
+      - targets: ['localhost:443']
   - job_name: 'prometheus'
     static_configs:
       - targets: ['localhost:9090']
@@ -29,6 +32,24 @@ scrape_configs:
       - targets:
 {% for host in groups["grafana"] %}
         - {{ hostvars[host].ansible_host }}:9100
+{% endfor %}
+  - job_name: node_mgmt
+    static_configs:
+      - targets:
+{% for host in groups["mgmt"] %}
+        - {{ hostvars[host].ansible_host }}:9100
+{% endfor %}
+  - job_name: node_scoreboard
+    static_configs:
+      - targets:
+{% for host in groups["scoreboard"] %}
+        - {{ hostvars[host].ansible_host }}:9100
+{% endfor %}
+  - job_name: node_cds
+    static_configs:
+      - targets:
+{% for host in groups["cds"] %}
+        - {{ hostvars[host].ansible_host }}:9100
 {% endfor %}
   - job_name: domjudge
     basic_auth:
@@ -42,6 +63,18 @@ scrape_configs:
       - targets:
 {% for host in groups["domserver"] %}
         - {{ hostvars[host].ansible_host }}
+{% endfor %}
+  - job_name: 'nginx_cds'
+    static_configs:
+      - targets:
+{% for host in groups["cds"] %}
+        - {{ hostvars[host].ansible_host }}:9113
+{% endfor %}
+  - job_name: 'nginx_scoreboard'
+    static_configs:
+      - targets:
+{% for host in groups["scoreboard"] %}
+        - {{ hostvars[host].ansible_host }}:9113
 {% endfor %}
   - job_name: 'nginx_domjudge'
     static_configs:
diff --git a/icpc-wf/ansible/roles/grafana/vars/.gitignore b/icpc-wf/ansible/roles/grafana/vars/.gitignore
@@ -0,0 +1 @@
+*.yml
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/files/nginx-status.conf b/icpc-wf/ansible/roles/prometheus_target_web/files/nginx-status.conf
@@ -1,6 +1,6 @@
 server {
-       listen   8080;
-       listen   [::]:8080;
+       listen   8787;
+       listen   [::]:8787;
        server_name _default_;
 
        location = /basic_status {
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/handlers/main.yml b/icpc-wf/ansible/roles/prometheus_target_web/handlers/main.yml
@@ -1,6 +1,4 @@
 ---
-# Define here handlers associated to this role.
-
 - name: restart php-exporter
   service: name=php-fpm-exporter enabled=yes state=restarted
 
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/meta/main.yml b/icpc-wf/ansible/roles/prometheus_target_web/meta/main.yml
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/tasks/main.yml b/icpc-wf/ansible/roles/prometheus_target_web/tasks/main.yml
@@ -1,49 +1,71 @@
 ---
 # These tasks configure metric collectors
 
-- name: install required packages
+- name: Every webserver runs nginx so expose metrics
   apt:
     state: present
     pkg:
       - prometheus-mysqld-exporter
-      - prometheus-nginx-exporter
+  notify: restart nginx-exporter
+
+- name: Expose MariaDB metrics
+  when: MARIADB
+  apt:
+    state: present
+    pkg:
+      - prometheus-mysqld-exporter
+  notify: restart mysqld-exporter
 
 # Gather PHP-FPM statistics
 # The exporter from this is currently not in deb sources
 # so we need to download this from GitHub see the README in files
 - name: Install PHP-fpm exporter binary
+  when: FPM
   ansible.builtin.unarchive:
-    src: php-fpm_exporter.tar.gz
+    src: https://github.com/hipages/php-fpm_exporter/releases/download/v2.0.4/php-fpm_exporter_2.0.4_linux_amd64.tar.gz
     dest: /usr/bin/
+    remote_src: true
     exclude:
       - LICENSE
       - README.md
+  notify: restart php-exporter
 
 - name: Export PHP-FPM metrics
+  when: FPM
   synchronize:
-    src=php-fpm-exporter.service
-    dest=/etc/systemd/system/php-fpm-exporter.service
+    src: php-fpm-exporter.service
+    dest: /etc/systemd/system/php-fpm-exporter.service
   notify: restart php-exporter
 
 # Gather NGINX statistics,
 # Observe that we use the observed process itself in the monitoring
 - name: Get NGINX status
   synchronize:
-    src=nginx-status.conf
-    dest=/etc/nginx/sites-enabled/nginx-status.conf
+    src: nginx-status.conf
+    dest: /etc/nginx/sites-enabled/nginx-status.conf
   notify: restart nginx
 
 - name: Prometheus nginx exporter
   lineinfile:
     dest: /etc/default/prometheus-nginx-exporter
     state: present
     regexp: '^ARGS=""'
-    line: 'ARGS="-nginx.scrape-uri=http://localhost:8080/basic_status"'
+    line: 'ARGS="-nginx.scrape-uri=http://localhost:8787/basic_status"'
   notify: restart nginx-exporter
 
+- name: Create storage dir for exporter settings
+  when: MARIADB
+  file:
+    state: directory
+    owner: prometheus
+    group: prometheus
+    mode: 0700
+    path: /var/lib/prometheus
+
 # Because the scrape happens inside the same machine we reuse the DB password
 # which is also used for the normal installation
 - name: Create MySQL authentication file
+  when: MARIADB
   template:
     src: mysqld-exporter-authentication.cnf.j2
     dest: /var/lib/prometheus/.my.cnf
diff --git a/icpc-wf/ansible/scoreboard.yml b/icpc-wf/ansible/scoreboard.yml
@@ -26,3 +26,12 @@
       tags: ssh
     - role: scoreboard
       tags: scoreboard
+    - role: prometheus_target_web
+      tags: prometheus_target_web
+      vars:
+        MARIADB: false
+        FPM: false
+      when: GRAFANA_MONITORING
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
