Use promtail to gather local logs and PUSH these to grafana

Both loki and prometheus can now be used as datasourse in grafana,
this is useful to make dashboards with metrics (prometheus) and the logs
(loki receiving logs from promtail on the client)

Author: vmcj

icpc-wf/ansible/roles/grafana/defaults/main.yml

@@ -1 +1,3 @@
 grafana_port: 8443
+LOKI: false
+

icpc-wf/ansible/roles/grafana/files/loki-local-config.yaml

@@ -0,0 +1,47 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+    - from: 2018-04-15
+      store: boltdb
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 168h
+
+storage_config:
+  boltdb:
+    directory: /data/loki/index
+
+  filesystem:
+    directory: /data/loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+

icpc-wf/ansible/roles/grafana/files/loki.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Loki service
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/loki-linux-amd64 --config.file /etc/grafana/loki/loki-local-config.yaml
+
+[Install]
+WantedBy=multi-user.target

icpc-wf/ansible/roles/grafana/handlers/main.yml

@@ -24,3 +24,10 @@
     name: grafana-server
     enabled: true
     state: restarted
+
+- name: restart loki
+  service:
+    name: loki
+    enabled: yes
+    state: restarted
+

icpc-wf/ansible/roles/grafana/tasks/main.yml

@@ -26,6 +26,50 @@
     dest: /etc/prometheus/prometheus.yml
   notify: restart prometheus
 
+# Setup loki which gathers our logs
+- name: Install loki
+  unarchive:
+    src: https://github.com/grafana/loki/releases/download/v2.5.0/loki-linux-amd64.zip
+    dest: /usr/bin/
+    remote_src: true
+    owner: domjudge
+    group: domjudge
+  when: LOKI
+
+- name: Dir for loki settings
+  file:
+    state: directory
+    path: /etc/grafana/loki/
+    owner: root
+    group: root
+    mode: 0755
+  when: LOKI
+
+- name: Set loki settings
+  copy:
+    src: loki-local-config.yaml
+    dest: /etc/grafana/loki/
+    owner: root
+    group: root
+    mode: 0644
+  when: LOKI
+  notify: restart loki
+
+- name: Setup loki systemd
+  copy:
+    src: loki.service
+    dest: /etc/systemd/system/
+    mode: 0655
+  when: LOKI
+  notify: restart loki
+
+- name: Start loki service
+  service:
+    name: loki
+    state: started
+    enabled: true
+  when: LOKI
+
 ## Setup grafana
 - name: Install grafana
   apt:

icpc-wf/ansible/roles/grafana/templates/nginx.conf.j2

@@ -9,6 +9,20 @@ server {
       proxy_pass          http://127.0.0.1:3000/;
       proxy_read_timeout  90;
     }
+
+{% if LOKI %}
+    location /loki/ {
+       proxy_set_header X-Forwarded-Proto https;
+       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+       proxy_set_header X-Forwarded-Host $host;
+       proxy_set_header X-Real-IP $remote_addr;
+       proxy_pass http://localhost:3099/;
+       proxy_buffering off;
+       proxy_redirect off;
+       proxy_set_header Host $host;
+    }
+{% endif $}
+
     access_log /var/log/nginx/prometheus.log;
     error_log /var/log/nginx/prometheus.log;
 

icpc-wf/ansible/roles/prometheus_target_all/defaults/main.yml

@@ -0,0 +1,2 @@
+PROM: true
+

icpc-wf/ansible/roles/prometheus_target_all/files/promtail.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Promtail service
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/promtail-linux-amd64 --config.file /etc/promtail/promtail-local-config.yaml
+
+[Install]
+WantedBy=multi-user.target

icpc-wf/ansible/roles/prometheus_target_all/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: restart promtail
+  service:
+    name: promtail
+    enabled: yes
+    state: restarted
+    daemon_reload: true
+

icpc-wf/ansible/roles/prometheus_target_all/tasks/main.yml

@@ -6,3 +6,50 @@
     state: present
     pkg:
       - prometheus-node-exporter
+
+# Setup promtail which sends our logs
+- name: Install promtail
+  unarchive:
+    src: https://github.com/grafana/loki/releases/download/v2.5.0/promtail-linux-amd64.zip
+    dest: /usr/bin/
+    remote_src: true
+    owner: domjudge
+    group: domjudge
+  when: PROM
+
+- name: Dir for promtail settings
+  file:
+    state: directory
+    path: /etc/promtail
+    owner: root
+    group: root
+    mode: 0755
+  when: PROM
+
+- name: Set promtail settings
+  copy:
+    src: promtail-local-config.yaml
+    dest: /etc/promtail/
+    owner: root
+    group: root
+    mode: 0644
+  when: PROM
+  notify: restart promtail
+
+- name: Setup promtail systemd
+  copy:
+    src: promtail.service
+    dest: /etc/systemd/system/
+    mode: 0655
+    owner: root
+    group: root
+  when: PROM
+  notify: restart promtail
+
+- name: Start promtail service
+  service:
+    name: promtail
+    state: started
+    enabled: true
+  when: PROM
+