Split monitoring/logging stack so clients can federate onto the server

ansible/plays/services/metric-export/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=metrics

ansible/plays/services/metric-export/docker-compose.yaml

@@ -0,0 +1,103 @@
+version: "3.4"
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z
+      - prom_data:/prometheus
+      - label_discovery:/label_discovery:ro
+    labels:
+      - "traefik.enable=false"
+    depends_on:
+      - prometheus-docker-sd
+      - cadvisor
+      - node-exporter
+    networks:
+      - default # send
+      - backend
+      - metrics
+
+  prometheus-docker-sd:
+    image: registry.tobiasmanske.de/prometheus-docker-sd:latest
+    restart: unless-stopped
+    privileged: true
+    networks:
+      - backend
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro,Z
+      - label_discovery:/prometheus-docker-sd:rw
+    logging: # this service generates a HUGE amout of logs.
+      driver: "none"
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    container_name: "{{ inventory_hostname | replace('.', '-') }}-node-exporter"
+    privileged: true
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=9100"
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /:/host:ro,rslave
+      - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro
+    command:
+      - '--path.rootfs=/host'
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points'
+      - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
+      - '--collector.systemd'
+    networks:
+      - backend
+    restart: unless-stopped
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    privileged: true
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=8080"
+    command:
+      - "-docker_only=true"
+      - "-housekeeping_interval=10s"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    networks:
+      - backend
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    security_opt:
+      - label:disable
+    restart: unless-stopped
+    volumes:
+      - ./promtail.yml:/etc/promtail/config.yml:ro
+      - /var/log:/var/log:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/config.yml
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=8080"
+    networks:
+      - default # send
+      - backend
+      - metrics
+
+volumes:
+  prom_data:
+  label_discovery:
+networks:
+  backend:
+    internal: true
+  metrics:
+    external: true

ansible/plays/services/metric-export/prometheus.yml

@@ -0,0 +1,30 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: 'service_discovery'
+  metric_relabel_configs:
+    - source_labels:
+        - "container_name"
+      target_label: "instance"
+      action: replace
+  file_sd_configs:
+    - files:
+      - /label_discovery/docker-targets.json
+remote_write:
+  - url: https://{{ common.mimir.host | mandatory }}/api/v1/push
+    headers:
+      X-Scope-OrgID: "{{ inventory_hostname | replace('.', '-') }}"
+    basic_auth:
+      username: "{{ common.mimir.username | mandatory }}"
+      password: "{{ common.mimir.password | mandatory }}"

ansible/plays/services/metric-export/promtail.yml

@@ -0,0 +1,28 @@
+positions:
+  filename: /positions.yaml
+server:
+  http_listen_port: 8080
+
+clients:
+  - url: https://{{ common.loki.host | mandatory }}/loki/api/v1/push
+    tenant_id: "{{ inventory_hostname | replace('.', '-') }}"
+    basic_auth:
+      username: "{{ common.loki.username | mandatory }}"
+      password: "{{ common.loki.password | mandatory }}"
+
+scrape_configs:
+  - job_name: flog_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        # filters:
+        #   - name: label
+        #     values: ["logging=promtail"]
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_jobname']
+        target_label: 'job'