Logging: Add Loki + Promtail

coreos-config/plays/services/prometheus/docker-compose.yaml

@@ -159,6 +159,27 @@ services:
       - backend
     restart: unless-stopped
 
+  loki:
+    image: grafana/loki:2.8.0
+    restart: unless-stopped
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - backend
+
+  promtail:
+    image: grafana/promtail:2.8.0
+    security_opt:
+      - label:disable
+    restart: unless-stopped
+    volumes:
+      - ./promtail.yml:/etc/promtail/docker-config.yaml:ro
+      - /var/log:/var/log:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/config.yml
+    networks:
+      - backend
+
 
 volumes:
   prom_data:

coreos-config/plays/services/prometheus/grafana-ds.yml

@@ -7,3 +7,12 @@ datasources:
   isDefault: true
   access: proxy
   editable: true
+- name: Loki
+  type: loki
+  access: proxy
+  orgId: 1
+  url: http://loki:3100
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: true

coreos-config/plays/services/prometheus/promtail.yml

@@ -0,0 +1,22 @@
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: flog_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        # filters:
+        #   - name: label
+        #     values: ["logging=promtail"]
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_jobname']
+        target_label: 'job'