Add Prometheus, Grafana and Targets

coreos-config/roles/compose_project/templates/prometheus/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=prometheus

coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml

@@ -0,0 +1,98 @@
+version: "3.4"
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z
+      - prom_data:/prometheus
+      - label_discovery:/label_discovery:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.prometheus.rule=Host(`prometheus.tobiasmanske.de`)"
+      - "traefik.http.routers.prometheus.entryPoints=websecure"
+      - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
+      - "traefik.http.routers.prometheus.middlewares=oauth@file"
+    depends_on:
+      - prometheus-docker-sd
+      - cadvisor
+      - node-exporter
+    networks:
+      - gateway
+      - backend
+
+  prometheus-docker-sd:
+    image: registry.tobiasmanske.de/prometheus-docker-sd:latest
+    restart: unless-stopped
+    privileged: true
+    networks:
+      - backend
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro,Z
+      - label_discovery:/prometheus-docker-sd:rw
+
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.grafana.rule=Host(`grafana.tobiasmanske.de`)"
+      - "traefik.http.routers.grafana.entryPoints=websecure"
+      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+      - "traefik.http.routers.grafana.middlewares=oauth@file"
+    networks:
+      - gateway
+      - backend
+    environment:
+      - "GF_SECURITY_ADMIN_USER={{ grafana.admin.user }}"
+      - "GF_SECURITY_ADMIN_PASSWORD={{ grafana.admin.password }}"
+    volumes:
+      - ./grafana-ds.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro,Z
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    privileged: true
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=9100"
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /:/host:ro,rslave
+    command:
+      - '--path.rootfs=/host'
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points'
+      - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
+    networks:
+      - backend
+    restart: always
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    privileged: true
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    networks:
+      - backend
+    restart: always
+
+
+volumes:
+  prom_data:
+  label_discovery:
+networks:
+  gateway:
+    external: true
+  backend:
+    internal: true

coreos-config/roles/compose_project/templates/prometheus/grafana-ds.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  url: http://prometheus:9090
+  isDefault: true
+  access: proxy
+  editable: true

coreos-config/roles/compose_project/templates/prometheus/prometheus.yml

@@ -0,0 +1,36 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: []
+    scheme: http
+    timeout: 10s
+    api_version: v1
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: 'service_discovery'
+  file_sd_configs:
+    - files:
+      - /label_discovery/docker-targets.json
+- job_name: minio-job
+  bearer_token: "{{ prometheus.scrape.s3.bearer_token }}"
+  metrics_path: /minio/v2/metrics/cluster
+  scheme: https
+  static_configs:
+  - targets: [s3.tobiasmanske.de]
+- job_name: drone-job
+  bearer_token: "{{ prometheus.scrape.drone.bearer_token }}"
+  scheme: https
+  static_configs:
+  - targets: [drone.tobiasmanske.de]