Add Prometheus, Grafana and Targets

2023-04-16 16:10:44 +02:00
parent 3468572ee5
commit f107c0c3c5
12 changed files with 547 additions and 359 deletions
--- a/coreos-config/roles/compose_project/templates/hedgedoc/docker-compose.yaml
+++ b/coreos-config/roles/compose_project/templates/hedgedoc/docker-compose.yaml
@ -56,9 +56,11 @@ services:
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.hedgedoc.rule=Host(`doc.tobiasmanske.de`)"
+      - "traefik.http.routers.hedgedoc.middlewares=deny-metrics@file"
      - "traefik.http.routers.hedgedoc.entryPoints=websecure"
      - "traefik.http.services.hedgedoc.loadbalancer.server.port=3000"
-      - "com.centurylinklabs.watchtower.scope=update"
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=3000"
    depends_on:
      database:
        condition: service_healthy
--- a/coreos-config/roles/compose_project/templates/matrix/docker-compose.yaml
+++ b/coreos-config/roles/compose_project/templates/matrix/docker-compose.yaml
@ -33,16 +33,17 @@ services:
      - gateway
      - backend
    labels:
-      # FIXME: /_synapse/admin is exposed.
      - "traefik.enable=true"
      - "traefik.http.routers.http-synapse.rule=Host(`synapse.{{ matrix.baseurl }}`)"
      - "traefik.http.routers.http-synapse.entryPoints=websecure"
      - "traefik.http.routers.http-synapse.service=matrix-synapse"
-
      - "traefik.http.routers.matrix-synapse.rule=Host(`{{ matrix.baseurl }}`) && PathPrefix(`/_{path:(matrix|synapse)}/`)"
      - "traefik.http.routers.matrix-synapse.entryPoints=websecure"
      - "traefik.http.routers.matrix-synapse.service=matrix-synapse"
      - "traefik.http.services.matrix-synapse.loadbalancer.server.port=8008"
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=9091"
+      - "prometheus-scrape.metrics_path=/_synapse/metrics"

  db:
    image: postgres:15
--- a/coreos-config/roles/compose_project/templates/matrix/synapse-config/homeserver.yaml
+++ b/coreos-config/roles/compose_project/templates/matrix/synapse-config/homeserver.yaml
@ -11,6 +11,7 @@
 # https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html
 server_name: "{{ matrix.baseurl }}"
 pid_file: /data/homeserver.pid
+enable_metrics: true
 listeners:
  - port: 8008
    tls: false
@ -19,6 +20,9 @@ listeners:
    resources:
      - names: [client, federation]
        compress: false
+  - port: 9091
+    tls: false
+    type: metrics
 database:
  name: psycopg2
  args:
--- a/coreos-config/roles/compose_project/templates/miniflux/docker-compose.yaml
+++ b/coreos-config/roles/compose_project/templates/miniflux/docker-compose.yaml
@ -21,11 +21,16 @@ services:
      - OAUTH2_OIDC_DISCOVERY_ENDPOINT={{ miniflux.oauth.discovery_endpoint }}
      - OAUTH2_PROVIDER=oidc
      - OAUTH2_REDIRECT_URL={{ miniflux.oauth.redirect_url }}
+      - METRICS_COLLECTOR=1
+      - METRICS_ALLOWED_NETWORKS=0.0.0.0/0
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.miniflux.rule=Host(`rss.tobiasmanske.de`)"
      - "traefik.http.routers.miniflux.entryPoints=websecure"
+      - "traefik.http.routers.miniflux.middlewares=deny-metrics@file"
      - "traefik.http.services.miniflux.loadbalancer.server.port=8080"
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=8080"
    networks:
      - backend
      - gateway
--- a/coreos-config/roles/compose_project/templates/prometheus/.env
+++ b/coreos-config/roles/compose_project/templates/prometheus/.env
@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=prometheus
--- a/coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml
+++ b/coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml
@ -0,0 +1,98 @@
+version: "3.4"
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z
+      - prom_data:/prometheus
+      - label_discovery:/label_discovery:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.prometheus.rule=Host(`prometheus.tobiasmanske.de`)"
+      - "traefik.http.routers.prometheus.entryPoints=websecure"
+      - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
+      - "traefik.http.routers.prometheus.middlewares=oauth@file"
+    depends_on:
+      - prometheus-docker-sd
+      - cadvisor
+      - node-exporter
+    networks:
+      - gateway
+      - backend
+
+  prometheus-docker-sd:
+    image: registry.tobiasmanske.de/prometheus-docker-sd:latest
+    restart: unless-stopped
+    privileged: true
+    networks:
+      - backend
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro,Z
+      - label_discovery:/prometheus-docker-sd:rw
+
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.grafana.rule=Host(`grafana.tobiasmanske.de`)"
+      - "traefik.http.routers.grafana.entryPoints=websecure"
+      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+      - "traefik.http.routers.grafana.middlewares=oauth@file"
+    networks:
+      - gateway
+      - backend
+    environment:
+      - "GF_SECURITY_ADMIN_USER={{ grafana.admin.user }}"
+      - "GF_SECURITY_ADMIN_PASSWORD={{ grafana.admin.password }}"
+    volumes:
+      - ./grafana-ds.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro,Z
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    privileged: true
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=9100"
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /:/host:ro,rslave
+    command:
+      - '--path.rootfs=/host'
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points'
+      - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
+    networks:
+      - backend
+    restart: always
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    privileged: true
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    networks:
+      - backend
+    restart: always
+
+
+volumes:
+  prom_data:
+  label_discovery:
+networks:
+  gateway:
+    external: true
+  backend:
+    internal: true
--- a/coreos-config/roles/compose_project/templates/prometheus/grafana-ds.yml
+++ b/coreos-config/roles/compose_project/templates/prometheus/grafana-ds.yml
@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  url: http://prometheus:9090
+  isDefault: true
+  access: proxy
+  editable: true
--- a/coreos-config/roles/compose_project/templates/prometheus/prometheus.yml
+++ b/coreos-config/roles/compose_project/templates/prometheus/prometheus.yml
@ -0,0 +1,36 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: []
+    scheme: http
+    timeout: 10s
+    api_version: v1
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: 'service_discovery'
+  file_sd_configs:
+    - files:
+      - /label_discovery/docker-targets.json
+- job_name: minio-job
+  bearer_token: "{{ prometheus.scrape.s3.bearer_token }}"
+  metrics_path: /minio/v2/metrics/cluster
+  scheme: https
+  static_configs:
+  - targets: [s3.tobiasmanske.de]
+- job_name: drone-job
+  bearer_token: "{{ prometheus.scrape.drone.bearer_token }}"
+  scheme: https
+  static_configs:
+  - targets: [drone.tobiasmanske.de]
--- a/coreos-config/roles/compose_project/templates/traefik/docker-compose.yaml
+++ b/coreos-config/roles/compose_project/templates/traefik/docker-compose.yaml
@ -15,6 +15,9 @@ services:
      - "./traefik.yaml:/etc/traefik/traefik.yaml:Z,ro"
      - "./dynamic.yaml:/etc/traefik/dynamic.yaml:Z,ro"
      - "acme:/acme"
+    labels:
+      - "prometheus-scrape.enabled=true"
+      - "prometheus-scrape.port=9091"
    networks:
      - gateway
      - default
@ -32,6 +35,8 @@ services:
      traefik.http.routers.oauth.entrypoints: websecure
      traefik.http.routers.oauth.rule: Host(`traefik-fa.tobiasmanske.de`) || PathPrefix(`/oauth2`)
      traefik.http.services.oauth.loadbalancer.server.port: '4180'
+      prometheus-scrape.enabled: "true"
+      prometheus-scrape.port: "9091"
    environment:
      OAUTH2_PROXY_PROVIDER: 'keycloak-oidc'
      OAUTH2_PROXY_CLIENT_ID: '{{ traefik.oidc.client_id }}'
@ -44,6 +49,7 @@ services:
      OAUTH2_PROXY_EMAIL_DOMAINS: '*'
      OAUTH2_PROXY_FOOTER: '-'
      OAUTH2_PROXY_HTTP_ADDRESS: '0.0.0.0:4180'
+      OAUTH2_PROXY_METRICS_ADDRESS: "0.0.0.0:9091"
      OAUTH2_PROXY_PASS_BASIC_AUTH: 'false'
      OAUTH2_PROXY_PASS_USER_HEADERS: 'true'
      OAUTH2_PROXY_REVERSE_PROXY: 'true'
--- a/coreos-config/roles/compose_project/templates/traefik/traefik.yaml
+++ b/coreos-config/roles/compose_project/templates/traefik/traefik.yaml
@ -1,5 +1,11 @@
 log:
  level: ERROR
+metrics:
+  prometheus:
+    addEntryPointsLabels: true
+    addServicesLabels: true
+    addRoutersLabels: true
+    entryPoint: metrics
 providers:
  docker:
    network: gateway
@ -15,13 +21,13 @@ entryPoints:
          to: websecure
          scheme: https
          permanent: true
+  metrics:
+    address: ":9091"
  websecure:
    address: ":443"
    http:
      tls:
        certResolver: letsencrypt
-      middlewares:
-        - deny-metrics@file

 certificatesResolvers:
  letsencrypt: