From 1379ccb2ebbc6902e0f4014b0a37529434ae78b4 Mon Sep 17 00:00:00 2001 From: Tobias Manske Date: Tue, 9 Jan 2024 23:31:44 +0100 Subject: [PATCH] Split monitoring/logging stack so clients can federate onto the server --- ansible/group_vars/all/vault.yaml | 53 +- .../host_vars/host.nc.chaoswg.org/vault.yaml | 830 +++++++++--------- ansible/playbook.yaml | 3 + ansible/plays/grp_prometheus.yaml | 8 + ansible/plays/services/grafana/.env | 1 + .../services/grafana/docker-compose.yaml | 43 + .../grafana-dashboards/cadvisor.json | 0 .../grafana-dashboards/droneci.json | 0 .../grafana-dashboards/hedgedoc.json | 0 .../grafana-dashboards/kuma.json | 0 .../grafana-dashboards/miniflux.json | 0 .../grafana-dashboards/minio.json | 0 .../grafana-dashboards/prometheus-stats.json | 0 .../grafana-dashboards/synapse.json | 0 .../grafana-dashboards/traefik2.json | 0 .../{prometheus => grafana}/grafana-db.yml | 0 ansible/plays/services/grafana/grafana-ds.yml | 28 + ansible/plays/services/loki/.env | 1 + .../plays/services/loki/docker-compose.yaml | 27 + .../services/{prometheus => loki}/loki.yml | 0 .../matrix/synapse-config/homeserver.yaml | 4 + ansible/plays/services/metric-export/.env | 1 + .../metric-export/docker-compose.yaml | 103 +++ .../services/metric-export/prometheus.yml | 30 + .../promtail.yml | 6 +- ansible/plays/services/mimir/.env | 1 + .../{prometheus => mimir}/alertmanager.yml | 22 +- .../plays/services/mimir/docker-compose.yaml | 80 ++ .../services/{prometheus => mimir}/mimir.yml | 21 +- .../{prometheus => mimir}/rules/cadvisor.yaml | 0 .../{prometheus => mimir}/rules/node.yaml | 0 .../rules/prometheus.yaml | 16 +- ansible/plays/services/prometheus/.env | 1 - .../services/prometheus/docker-compose.yaml | 224 ----- .../plays/services/prometheus/grafana-ds.yml | 28 - .../plays/services/prometheus/prometheus.yml | 58 -- ansible/plays/vps.yaml | 4 +- 37 files changed, 822 insertions(+), 771 deletions(-) create mode 100644 ansible/plays/grp_prometheus.yaml create mode 100644 ansible/plays/services/grafana/.env create mode 100644 ansible/plays/services/grafana/docker-compose.yaml rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/cadvisor.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/droneci.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/hedgedoc.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/kuma.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/miniflux.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/minio.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/prometheus-stats.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/synapse.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-dashboards/traefik2.json (100%) rename ansible/plays/services/{prometheus => grafana}/grafana-db.yml (100%) create mode 100644 ansible/plays/services/grafana/grafana-ds.yml create mode 100644 ansible/plays/services/loki/.env create mode 100644 ansible/plays/services/loki/docker-compose.yaml rename ansible/plays/services/{prometheus => loki}/loki.yml (100%) create mode 100644 ansible/plays/services/metric-export/.env create mode 100644 ansible/plays/services/metric-export/docker-compose.yaml create mode 100644 ansible/plays/services/metric-export/prometheus.yml rename ansible/plays/services/{prometheus => metric-export}/promtail.yml (70%) create mode 100644 ansible/plays/services/mimir/.env rename ansible/plays/services/{prometheus => mimir}/alertmanager.yml (53%) create mode 100644 ansible/plays/services/mimir/docker-compose.yaml rename ansible/plays/services/{prometheus => mimir}/mimir.yml (75%) rename ansible/plays/services/{prometheus => mimir}/rules/cadvisor.yaml (100%) rename ansible/plays/services/{prometheus => mimir}/rules/node.yaml (100%) rename ansible/plays/services/{prometheus => mimir}/rules/prometheus.yaml (96%) delete mode 100644 ansible/plays/services/prometheus/.env delete mode 100644 ansible/plays/services/prometheus/docker-compose.yaml delete mode 100644 ansible/plays/services/prometheus/grafana-ds.yml delete mode 100644 ansible/plays/services/prometheus/prometheus.yml diff --git a/ansible/group_vars/all/vault.yaml b/ansible/group_vars/all/vault.yaml index 3ff5ab0..7cfdc39 100644 --- a/ansible/group_vars/all/vault.yaml +++ b/ansible/group_vars/all/vault.yaml @@ -1,18 +1,37 @@ $ANSIBLE_VAULT;1.2;AES256;secrets -33333962623165376561346162376164623463386237346462316166343537383564333232393666 -3364333738656534643236656335343938366366666635350a303466306634393662656136613734 -62393639396631653338316537356437646261356339383961643837666363616265616438313333 -6637646661653130610a303561316464383437663838363632343833306465633734653665306635 -66613038326430353639313639396531303331373765633964306564383530303564353165353835 -39343131666131363061336266646232363336636438356632393734646638383539303837633230 -64646432353935333434383030636562383566333132623732623063323530336635376238343830 -32316164656262643362313330323035336231393734383032623436336563616537303935626262 -30306634393662623337633835383330303730643133363765393237343437643433313565323266 -34613638393261633033333234313234303737363934623864306466663333393939373161626633 -65666361323535643363323539396564363966623235653637393237333162376132313266323336 -36353866373134376465643365363933636466646238386230383833626164313330346133313039 -32396537386133656264376362623430376639343062653434333834323533656166666537376431 -36326231343230613838316237366231316539373634353838346266666431666162333366313937 -33333033346563356564336631366430353633303765663139643866383064666636633262333865 -61396338306337333030666363373331326130393831393136613430643039326461333037363735 -38353436616364386237663936303735636233366330356632353463666234323230 +63613766346333616562313036363031626165663165373363366239636632623964326561306533 +6538656532313664363431376435666536643532623663380a393438363435623266393965356366 +35633966313834303836393534643537613165393363623535323261323166646338313638613465 +3161333566376538650a303063633635636239303837353630353365343266396565633566326634 +31323036333665303962336130633130643164636464396565396163303433623539363563653230 +34356130353532353331616565353463333663376633353964333366363034653633646463666432 +37326362653230343438373837313438396535366263663162363663393739656531373436343033 +33316233313065306435356366313138653462613539346537613934623236626462326636633665 +34616631613564346661336231626665333233616632376561623462303035663664393536646166 +63313866323364386534393234393162386638663037646239613231653836383964376132363936 +34623163666135623161653861353834623930646263323232373862396265643632323563663137 +31623430616661343164323362346433616462373637373961656434376432393563663762353631 +64363964326639353864643262383135396166646264643231383163363930663432623566393433 +63393233313935316238663139333964313161623435313932346631373035376564666662316130 +62393965313635393232623036353962316665346132623132353830313161646133303135653365 +30633566303430336264623934373066666439643163653330373733363863303766643939333566 +64656566653561346239343831633064346162656534373231333561353966653262626137373164 +64666633626466646538356333633433616230363837303933313839386430376438623964663166 +66343231363337663135343137363239376538346133663633353238663266393634316536316335 +62303061356535633932353263323736623935343336623234383032323235636630643964333335 +31393338386439626432626532646630613936633166313034373663333639373266363563333664 +65303632393336646563306132383265303437663233333135646261643166323331326435666132 +38386637346331653730323336386237613366383034333237306462303261396234373536393735 +34323836616462313566313136316232663839323632666532346239306561656338653030613732 +38383434623233303062353935323161653230336461636437386661633762656633323161623934 +61356535646661333365383838356465663138386263633066343662633635323262303261633032 +30316238633263346437653065356163333262323530653831356432613561653639643065346135 +66616133396632336661346437653431356365376364303561613062343036666264313263646637 +32646130346633346362316630353065633134653833393863313033373061396633643033373035 +39383034613963656631343136393230386233353563626336663831373234303836666233336234 +39363537376236313835363466363839353838373333623165633534666531656366633233646139 +33653261653563663934613462653332663065656163323038633737643636626534613663346236 +64313065663131306265306263666231613234666633326533323235393334343237653863623561 +61333532383134663935333139303362653534383937306232383133653631356462353362346463 +32373632343263343161636661643434343939656461633734353635346631373166383939316165 +32313564323438616138 diff --git a/ansible/host_vars/host.nc.chaoswg.org/vault.yaml b/ansible/host_vars/host.nc.chaoswg.org/vault.yaml index 37a0877..196ffa2 100644 --- a/ansible/host_vars/host.nc.chaoswg.org/vault.yaml +++ b/ansible/host_vars/host.nc.chaoswg.org/vault.yaml @@ -1,416 +1,416 @@ $ANSIBLE_VAULT;1.2;AES256;secrets -35353736626432633266633738663336323733636439336533613936663231363635363966363336 -6634666661343731343738613731323765396462303063660a326333633739626563343137613539 -31666534346264306639353630366434326531336463323933333131666337636262663935326263 -6163353931396563610a303538356164313031653738396139313537386632323937306239336136 -63626236366239653837386165376638313763363765666135343361353433383462306562636134 -61656262633131626135323737363335366232616335333238376661333664323332643435333266 -37393731333430623836366137363465643432633866653337326338646336626231653465353934 -30623766393361376537643038366666663033363235613064303862303235373037376235646533 -31363833643164376639363063313431656262316364383464636266666333393266373437323339 -37653263306536626334636138336465653665616238316333393363306332636262666138396539 -61383737616334383432373133646166646335363931623735346466623061326563666363336630 -63373266616634333539306363636366333865313861646563303436646434346164636631633165 -33343531356439363939386366386262646632346362316166633663613730373864323361613630 -61643034383066366664383061653338353562666439643165363931343435376363666230323963 -39303564663065373836656235613633326239626334383734343836336464633737313139623562 -64393161303039636663653137336531396166633136633166313739303432643265386235313361 -38336135656461363136656136313438353231663731323064333763613234636166386137313631 -66643237363132633734376639336365643432646135623230306563613634653730616235643439 -32623661313631636464396533666562316638316636353933306666323634366433333032623566 -37663462313239323538396661383062643562633139376334626564643066376438616563343131 -39633062373464643863346664323333666234306465363435366239333730653336333761643239 -38656164653834366163373631356637356166663530343131356637623033393934613739616130 -36666530373830636364356533323033343634323730373332376662623864383838353332336334 -33363235313439363261623862363164376531643964313765636533653731353666623632363533 -66646361363836653663616462356565633063306663353935386637383139373838396633313863 -61666531353037646265623762363931616461636235373239643035626238666464343837643932 -38346530656166313061396432646432353966633737393131636134353234313739633433663430 -32656530323762613439393738333462386564323533363738623439396236613163373564386338 -31656561363063623861306664306232343031653065323432343063616432313637613062633439 -34653139336163643361663132626463333263636365323039663736323664396561613232326235 -38393065333764383030633366383265616162633333343765346130373932623339393335383266 -61343363643839393630646661323231346234666531313663316161353239343139396632363236 -66363036393438643733643866343135303431313838333933396665623438396364363830383864 -62613266326430623062303836663562343531313735346562353930303834666639663434653065 -39646236653565346533616238396430353564313536306336383765616466633230346466373337 -32653931346537646433613064333137623865356361323161643330353365346231633866323439 -38313531643239616332353136633737646437666162306433623232343039663431653165346463 -61636466663438316364646633353963653164393634656462393235373064356234343436653965 -33356364313235386435666266656230343765376434323934666437633530333933366561643339 -35373564366632373837303936613761316463323464346333323164383234623132333163363938 -61326331656339623533363136386134343764626535646236383165623466633439343663303536 -34363938306233353866313665633139666561636364663163313235626263656432336262323565 -66353864383363613833623166336332306134653338663664646563323938323462323330306438 -33313036623564313837663939356631663564656665316137666463333239316465393264613862 -34616438383831623630646239626263626532376137316636656666346263313434336266306662 -37333937333666353963303035656164656439316462303364323534653637653231626466626538 -31353961623664303661623430663431653662346464323932653734343733666434393138343464 -64313364613861666466613834626562636235346665323534623466313236343864313863626133 -35623866613735316237633266623036353733373366386462396366353866336437366335313237 -63353631623162393434616132616266636164303666363961616462646630333765316265346137 -31343563666438316664336635333235353737313561343861376437323439366138346334653964 -36313938356638663930326131343462616164653330383763333533326438336631626538666566 -37343436383166666539373334343935303133353533643932313332373435376165633538653035 -65373032643833306161626534653534343833356364353066333363616337323835363939336131 -63663562626339636562303366326161316363383039336563383035376532316563323530323462 -63396432386533653231363837333564366231373637633964303366623132376430336231633135 -31333565616562366162636632633563326164643039323165373239383437333166633931643338 -64366335323335656630326135386334643561353236303539663265613962326536643133663532 -62333233393764633438373839366662643438376437663435333439633232353661356136396665 -34633261626163356435346130653139623866626434643665333565383331626134646365396539 -30343064383063643863333331346239353461353961313730643461346136343234396465393230 -39333236646338333238633832353162363435663666616563643262323832643034646236353062 -37666434383436376139663063646366333863653562613065636332663733663238656465626531 -64336261316261346264373235633065326131353637623834303035346535343739643037303931 -36303161363230653438626330393330326564623662363835306635646466313536343935646132 -34316636303938663461373662313961656564616562643837306163336132373434356438356534 -66636531613032323130633932646238346164386166636338643663343038376266396264623932 -39303738316166653938343633376535306439356432353963353161646234623964666136336431 -39373864323333393963343264663737343165656562303432356139313264336133396165396338 -31363838363531646339386666326437393830646534386336313561373635626462643436313131 -37326665386331656566313062303737313830383034326437326131623232333934383665376462 -32363934376137373062356236376266363664356435373065386137363730653761366535646462 -63386431643633303833303632316563373339653965643466313637393234653439633965383330 -65623437316161643434366163353962616163663066646338383766636636376634383263653630 -30303930653139326636633532643033393335326366346335336463616161626433346638323066 -64613937376230333131316135373464313736633038633166316466653161343765373835396362 -35643936343738333932303830336161373738623862633464626364393464366439333137383131 -65613530363436366665663565353661313139396338373138313137363462643966383161346231 -38303265333738363239343631336139663864323430333135623961626337376438396565396238 -33613063366165393464383230333533626438633933343137323566636537623566613662636639 -62376539346137663563663961663736626363353339633931316263393662366634373864303462 -63643361643236333130353135363333393431363933643036633337356361353665343237643130 -64303934376664653963666535343335333135363930383639613435303639343831356236373237 -31623434663032306531396531643264306532346232643331363065343464663432633230376131 -33616234663763646164633033653533373065326566343530346438313363323534623038656262 -39653633313332643336356664323664396361613964613133623231306464356630633866643961 -61386230303533653039363434386130623963323038373631346431656431386636633263303762 -36326336633333376463623930346366353133386663393063366661613636626432656339303765 -62333237363263343931313762626435653735636132383734393036653633626439623230656333 -38363832313466623430346263333665616630306563373831636165636264643739646330643334 -33323062666639303330633466623565346135333634316439303663336637373261393430306563 -31616436356264653365643537386430646536366435663038613666613263633039633738623633 -37633536633835303663353338613535653132643031323864643463346366613536653830633832 -37326131656531613133363232363838336335383961383334356239393063343630346131343162 -30333235376165356431666339323939633665323832316436666665663835343262366434616530 -63353134323933646334363566643138326263376666333162303938383861653030643935306563 -65313332643333356165376330613161373861346165346365333966343037656439306634313430 -64333138396562663431323133653233393634326361653631643835313430396533616539323663 -61326433336636386166633935646634393966653964616261356237623864376135353061393437 -65333965643434313330303237323433333661313435613532366234396330303338666361646465 -66343737653364333635366536613230346464313037643138383238633561616565393337306133 -31663064653430643435323337323239666631306139653030356537643832353138346531373634 -64343635396261616362303437626334373263633034636236303832376230336630343632396334 -66326336643037663635396539623663613065393938383230356138353366333564653534663230 -37666232343664653637393665666232316664363266333231323331626434636664366263363734 -61623638613266336164613166663466356261316637373836316364643335653233376436613937 -32373664356362313061396334633831336331613766626537346265633465343966313836363533 -38653938346637346263656163616436323562366531663432333934383733346237326336393538 -34346539343065333535633333303532393332326361333533613939613263353435333265626230 -35646438643665613937626237616637626337363238393164303633653033343361303962336464 -32643432656262666636343036306162313065366431343635626630656437396130613831653063 -63343937306662376135333261323865346438333333633139326463313764356330643137343234 -32663064623963653139333530623932303964313665626161623430633463653533386435626563 -34386330373064343930643361353531646439333430383138363735613066663531636365613232 -61623866396662613563633935366138333163386161653335393463363064343664306565636336 -30386237393331383263366335616237333437363239346530363264316434393933306263313434 -65643135653630356535353236346665653566373033373639316138303435316565663236626364 -34363862373439323036363937396633653038653134363661396138323139373131343833616262 -30343066646437303236663132656535343138316632363163393635336533383266316530346236 -38343939666665316131616265643737343265336366653966616361636633303935333563323731 -34616337346662633738313230313636313961363066386536626235353532346562626339386461 -34393762306131393739343334316563313938396235323039656233303463353437336530306465 -66633233656661636233333161653238623436323330623039306466303033366638636135663736 -64373139323932353632646234626639663034323133393537366536653066393332346431356161 -63333064303636313636326234396530376439623665386437636662383738396634393365396464 -31333235306335663636633462646236306366323437643765353865363165656261363365363130 -66666130303536646432333862636463386166333662346131306335616633336661376332316564 -63636330323765623639373134643261626236653439363866303739666663656363383630313435 -64613063666330323839396462383765323632306530613334303733613133663333323366323136 -34613934653136323234303639343338626365616534323534383337383036376639636635656237 -64366239383661616530386234323165616438356335616562653630353862396232613530653339 -34643263353933656666343435386133373437623664343634376439656266383333663134333564 -31316365623264653166633465313731383566623430363431363733623332376233353036396537 -35323564656361343732366432653131613339653566363462646232323261353034623464616633 -31343536306133353235356363323235373436633663326637656535306534336537353337663234 -65353663633064363933396365353435636666343338363034316430353361633838356430386531 -35366562393137666336363762393730383937643862626534396561623736333461373538663736 -66373363653333653132313164376564333564333166313738623131653966333561383764636635 -38346532626263393266656463646335613065336639633534306536636434663331343038323433 -30376463306163326464356266623038396533383232643739373964366362363338643130623734 -35636162383034333361333065613765663663316661393437616164353638336534306535373764 -64323533666637343132636466633439613761613637313234616130636335313065303833353966 -63366461633263623835376565636238626633616264373133633039333030616364333463323762 -63313265386563653763633761313461666632336162666432633237366361653765383833306533 -61323535393166643131646535663261366536616634313639626163646530363836386130386334 -39616238316365393430363263623539626437613732633464393632626331626137663366623135 -30336631353637333261386431336536653939646163323932663666336538373566353438383735 -65303231393334613036353638373832323736646164326436666464393366346333666138656138 -33393635326236323366393633306361656438653636353035653237373137613861393633643631 -30333564346562386332316235643939343539623963663964316165383061393131303737613835 -36333739313462666238346536333335666166373266643966643030363132353438653063373664 -37336537373966323135333062303165393961323264336130393533373533663535333933633335 -62323862303361326334393463663263363238343261323061646638616265393564336334363665 -32396130376131303865363930663764323165396639386332646333336264356530393135393463 -33333831323936643363343931633565646237613039613364333636386330613734653539313631 -62633630343535636338393034323062656336383831626438623763356333306337653536383462 -30636636326334653533616632343538613762313466666636326266333964333564393333633761 -33643132323537326434353166353432313230613939653631636233646664356163336466346238 -36653331393135663439386264646535396334373335383866623537313638343336343565616366 -35616532643334323664616431393964353635333562333538636237636136316563363230396261 -61396663373563636137343636616362383238643034313834333364326262646332666137653631 -31363764333134636431396437393732363633656534356164336439666561373237353734393461 -62336165346463363732333031316633383031356135356662383833386131653335383634393137 -61653239616134363632356131356532353336656331633738636130343638613262653630373564 -31666164373262376535393838323063303833633761336364343534396561336334333633373063 -63306461623534336138326333383437313138306464376230333064373664316466396666646436 -38343936663038313863643531636438353533326337366362336565323062316463323232666336 -62313737646633313739306361356139396562363062356234396536353862643431356334356461 -64356531376333396434333439333163626362373939666361306337346534383134623666663530 -63636634323361356463666163343662396561323438386337643437343432616231303261383666 -36396166353063653038353964353036373434323865643564396232633961663937623032636330 -32393830623139393132303036313932333436323063633438396231396630343438656637343466 -62323130346561346137386132393236356164323566306339313432303630653032633033343230 -33336535336233393835306636616161626563613231356136363430333931373061613866363833 -66613833323166326331336166313666326131663762656531633739643961613431326264316166 -62363331343837383761306565636339623361363665363933663662353430393735633431353535 -39353364633035653538396132343130386366633032616164373262666134363132663635393762 -38303833643638633365383635383961336562353038383836656431396536306635326366376162 -64623038303435663233356364373133316438623965353966666432653265663065623563656363 -65643165646336623263616466616336633238626363633762376361633065376334646131333966 -63323665643233616161343037343265303632386365373535346634633036336635623333643331 -37393838633663363463356439356631306462316133333234643538366231393731396635613061 -62666166393330346130353530333265316164313539326565356636363266356330666335386132 -66646331316466303261623331323362623930376536653032613038383935333731366566643565 -32303636656266333838393137636265366563333731373461313235333462313230313564303339 -62333234303865386663366533326633626433333437333130633761303133366239663534623761 -66343434383539333437356566313837636330393532653435396265303234363362396133353761 -62303634626232313439323939666631626534613533343438383265323364366232653865343234 -64663139313537376434366665653239393662396136656130373463396565626166333736306663 -30353065353532353633663366616239656236396330623830376332363836363565633336613765 -66346431653863376432663663636261303766616539613461653631643065323838626435393064 -31383363353530373633363736613034656338386163303637633063313534366130643435623537 -36323536306131653434626266336132333431303730336365656436616664376633393432383132 -66353837386133313366613361316563346566393936643563313432636163343831373965356233 -37383537363431613734626633356361376361663637323234643939366230633030353534366366 -35356137313932393632333732386536636462346537376138363633363831333037636532633731 -34356264646661626235363063313831326663663464343239653462633833316631313361643536 -32353630616335626431343761323233386666303431333737626163623862326631636232636236 -32616262376532393035353662633461303736643633343136333966306563313636643063323235 -38333766623335663064323033356565643432336438656164646363623361626532663162643763 -39363238633162366564373631353236336362393265656666343531343635663464383565393566 -30356137393036333862656563623563616530613861633333633335363434306461626131353735 -30366539326137346136626262346232633132626663363930653730333231663739386636383765 -65383764623435336338366566373031353437356339396238663233616461333261313866363933 -39353764303165613163306237333765346466346432666435333831626537336534633336386663 -31366661613765353934646666396432353039383935643133303735343036613564666432643737 -30633537383632346235616131353665636532313962616366303635366565386166346461306464 -64623131353166373265663566396164303735316463613661323032313131336162666466303061 -37663462316565623935346564313064663235363561393931643731646461633666356438653565 -36313864353937653266353162613865323336646361313236353263613038373766303731386534 -32306330666136383865643561386564663032623261626562653836333037326339633363363964 -35666134323562656433383238316562666139323938326535666564376464393762626461386339 -66363933316438613462653630623238386130616161343066386132633063343039323962633565 -33663133313962613235623161366561343833636666653431396633346434623035643633313664 -39346239366161383232313034383838626436333732343065323439316431376336626165363035 -34346230646631666665616138353563313138626364313262363530616633333864366531333130 -61326461393337306561373064316132363362333764303363626330313862356536356462373961 -61316263633266316436613430366334313066363834353061396239616437303135636135613736 -64613331326633616630633635663461366466613565353532386639363462363763326465623139 -32616537303435383135333239333530343330663438643065383562353635393765373639393465 -61643930366538306664333664333931643062313936376661333666623231396365646338383266 -33666338343539333565353732356633396234306466303130353836386137613835316661663638 -36616363353032653064356361636235376337363734393665636631623136666136323661353261 -37363363373633316233393132616536383132613230666636393139393536653062313030666439 -33643162396337663432356533363534616466306266626333643236623962366563623536363635 -61383639633732653530323466393730313365646562386164613763336139336130396235653130 -65336534316466353134623639363937623366353664663264623037643535353434636336643238 -39383732316233663530613032346130656162623165656132643164653735336465636231323737 -66623332656331656563393033643838656162356135363930363139393366363638623035393838 -63386663643132356566653239393339393763393433353234373838643532656664333466653061 -64626331306566613431636533343933356565363938616132623533363762376238336363636336 -61396530326466626161303766323364666335663465313238323664633264363735633165636365 -39316234613230393335643865623434356538393462643634316434643730613938646630613337 -34363364386662386132346630346262323137313435636137353262376131366161343435313934 -35313965613239353561623433353736646161643436656239326461393163333463386538643163 -62666663623665616337323832653964313838663066633537646135663630326634623466663139 -39313663383632353039393430336566393037366265363163323132306663653237653931643138 -62663939623135353161333764653864386530363833643935663061306433623931303937333032 -62656666656464626332326637653665613834383939626239303231393330333938343936623637 -64316437393434396134663538383138386438313664636264356136346437356235626538326161 -64393234346334336336336531656266386265666363386138653131376138366234386133613932 -35346661323532643766346331316562383933346561616231376562663062663836326338623336 -36323539623663386530333632613463343161396431363830356337373964333635376365343431 -32613462643835346430633732616536626338376532626530636631646131613130373064323534 -31333932303431333838303237326134643835323731363361636632356632613534373361396238 -33626462613739653461356664653737353864643632373164363663343163633365636539396230 -64326134313363383738396136656134636262666361653664353834336135303931323830303934 -32303038303831643930323565313233383364346536633132396335373239326231396630616466 -38353737353165303038393333666262383361373737373539666431613136323261383036386633 -30363263303034366566643533313965626633636566626666613865313564326639656433353134 -64616562386663336663393135666638333231346537323439363337613539383035626238393162 -32646262666435623636656362396464386435633138316534353461303662663738383035643332 -62643136646562313664623736633734623433636539636362653037343166343332626565373031 -63313064333938353732613566616561343731393230636336366535366164393939653739383032 -31383036666338646131633761663030363130313364343539303135663965663439316662623237 -32333963366238633364333232303763363963633266366534653264666662633630616435636661 -30343035333765356633623666343365393261326431626239313337653739383636333463643835 -37353865376336663561306365626266393364663465653237373065623336343938306531363136 -62356261383637633665333563383866653363303139313937356130633331343964323033646637 -36366165353864363363396130613165663964323739383764653531643335633538396239373933 -38346363656665313364646636363232386439643337663036616239313263646635326430323663 -39663731613836373562353934323564363138636637316438356332633361613730383533316261 -31643535636239323732373931376638313136616234336335633532393766623466666262373762 -64386565363366336364626462613064646166333037643937663630663633643165326265356336 -37323464303532343137626236316466363037653966303437633666633839356330353839363332 -31643132626263663639363661363433316433653137613935656461376461386364646537323862 -61353133663138373330643965313037343536356434366663303938663838366339383165623834 -62346430613539333361653734333736393233393339656631376461326638613732663335633264 -34613665626562616537333266376332663730346336353037306264323761666139303764643031 -30633565343232636338653363316464623535653135636361316364653833373031373934323931 -62393638643466393230623034623939663936343331383637313937303332323839643339653933 -64306434646463366564643763393566356136356537343133653539663934333831633733353062 -33313137623133356266333733633962346466343165386534653835343639336562323737623139 -33643066653731386637303032306532636162363339633563613334353134643038633439373565 -64613765653536363134303938336432336262316666326633396661396136303966623463613661 -31333638613732633463646363633237633432656263623933653231643630373330326661663164 -62663233623963346666356466353533323462613039356633623365396265303530303238623833 -65643136666361653864663565303433353132393730353338303161336662626237616163303262 -34363134383939313532393765613032666233643536613637376566623430616534343335353633 -66353762333364633563633239656438653265643339396437303563623034633136313839366539 -35346633646161343230663965356333303431313435633761653134373331373637393665636631 -32313937393066303439353062626338363036306131393263323136636166396230306465633563 -65656635356239356338346661633062386164373964396565633034333031376362353966326637 -35393665623562633130643132356535636230643163323437353538656661326637303430393537 -64623134656534663563383831613338323962353934353332376337333161623437303336363738 -62306635636364333539353366666631623261373832313531346138313761313433353364366265 -30303765323631383264376230643362366461656532613165313833306262326530326633336239 -32633136333262356461623031666433626233616266303834323265376338366232356366643663 -36313634613935353462613733616132356137343565623331373038633464623761343931373665 -63393962633332346331353330393166653565376466363335363734363737643735333131616365 -62313636336439343362653164653033383232656630633532396332633739613934393766343738 -32666433316430643666316437303730383530633462323733373838663361306635356338313865 -32366439373830623331626431386634386462666233653463336635656434313766616335643535 -33633233336637613862653334653437386466636431316531326130393433373035376262373635 -33633030623865386261663966366466616639666264373762626361343031353466386530313239 -66623662343238333139343565323230653265613039333135306334373637326234303130333637 -35343364303962656538343337326435333162316538393536333337653239323532326632646536 -63336430366538373464623737636536323630356135316666333463656232333763376633613730 -66386266396664653036323464363634363431373764633038383165353036313562366535316536 -30306665663930323932663038393231636533346162376564626231633437376565343263363234 -62653262613061393239363139306233343832656162653661396663386435386166656436363832 -33393938316165663434653264306530393363616334316263373133316561323536373139313262 -63636133336330393461656563386136366561623031303039356532663030653138653563343766 -64366232643336336233656264343931303535666662383936373964343139656435366334663236 -66646336363633633239386637656462323733346532353965386532313264313232636237656466 -66373737353362663762396330383366633364636631666330383337353461356264343331316335 -31613066303639303534623737636639373464383232626162613833636662333338626431626165 -37396333376435363933393338383662363166376466613030303565623636356636643536663165 -38323439396662346165633763363632646461623539633332316138326639363339356663623262 -30623035383262393934656361323635616664653164313039646536666235343634346563323761 -35633164626632623066386663663633316266663436633832313161346161373036643563373435 -35313665646539623266353464363464383665376362663161633539313635356333313561653864 -66653765623739393563333935626438353032623136356161646439366630396539306337393062 -31623337313239326338633861343530616233333139303966343739323438326261666535326165 -36663537383938363631353738643735323332393935343266353366346262393230393633653633 -39333334376333383662343663306564386461333537386264313364616466393335343065393662 -34663832343431653938643434373539333731653434356264376665333033343862613135616138 -61633132626438323963653933613266653962323264346338643435643331666332326532343235 -62643337316432363731353763633766656536336262396566356139396133303764636234326639 -65393032313064383863666533643363303935666232336536373532373839346166313134623234 -34303463646238663332366565366539363836623465356666316631613239376335343433353866 -33333131323531663831303564353133393261363133653264363938613032663034373465616464 -31383062363535663930613431613166363538663135376566363335363933316366383033646431 -32393136393862343639616135633737323463326534316339346430626536623862396363663262 -30336266333066306133343764633430303164333038306439356462396631643032383039613765 -30393833363833316336633262666538396332653637386432393961346635343333663566653763 -39376361383362393936356536666434373666366632333164313863383039346334333234646265 -38616534353965636234643061343038663938373632316338393264626465306661306566336139 -30316136616165333639643338346634366437373163376436633765313466386566366336336539 -36346632323136633565646334643833623633376435653262633936666362303933396261306637 -61323532356262393238316631633935386361653865353333383766656464613330333761383437 -66653662316237326230303665343831666335363932363135666261623636393361396164663564 -62653463613766636539373665643863633530636236363531633063313034383931363763336266 -37323035616164373762303139353732303764303530363634656561333266333733363065363935 -36326534643532636333333761366266646562663236306434393364376236643836643339613839 -62396337633230353238653966366639336531356336666631356332376365623263653936646563 -37393132303263633165643831353331633632386463616139343666386561313439383438336638 -66373833616236393932353630383133383863653039363562343837343932303130356333373866 -66323133646462623339666231643930313733646539356230613262663664653966666533653030 -65366565333763643665326233323833316530666433336164613135306161633661383131356563 -62343932396231373132336438316439616233643237303965666534663039373436626534333637 -61316139386163353633306533653330306264303633303861363361613434653866396561383334 -62616666633136356261616335376462626463376366643261303461353062356132303135336532 -34626664376138303365353030373939643566396539343231323166313661336261306461653763 -61383463343131373138636639633236633936633633306134336439346263636138303863393032 -37633336653636373331396261326632356532623331663361333236383461323832656262643433 -38386662366661383832393734323231373834323438343338326464356635313639333564386639 -61353232353363333538376161346430376566633664383733613761303437343731366663376237 -35626230383731363739383239346166396135653530303033336365303937613261363537653765 -64613962333933353564326566336365616537396638386436326161346539323133306563313438 -65666233373034393766616638633133383465613430333961636561323266623531303538303232 -62616535306436346232643336633639393262646531346564646466393831376565313534323262 -31636534343564656533336137303439323535303032326561663766666333396165643233303739 -63363931663132306132343837663631343934363764623937353230333438633132303535383763 -35666331383064393534393666323335613535626134353763383535356162626333643461666338 -33653864363034623438313661353034616637376631613265393362616537326362303133336633 -38346438666131616431336464363734666330313736353036383261373965343463396238393363 -35336437353634363633393064663236326465646430646330336161306231363834336132396630 -62346539346463356131313134653938396236633436373239616661633337626433653935366266 -35333138303166323734323263363162653965393935376531353734353932393964326164303366 -35343134626564643930346139613236303039386238633163646338663765636561663231643936 -64643966313563613039633434666639653431646130366231306166366135313537333761616566 -62646462633334373563376162353435386537626237313964653739646531386135333439333562 -35353463643065613462336530353864356564633866616438323433353032393731353739646364 -34306533336366323139383865323939373838353666663163626666626336656531356630313038 -37353838646263303237363833656438393466633935656437333065363130303965303863366635 -35333463373264376564613266653566633632623530356266663236303964303736353631653235 -62363863653035663264623864663866336563343738653435303363633735323837636432616561 -32313064613636663232656136373533353234316466663534623530633334653436323832333231 -37663537376438316330366539306439333033623831316635313934393530346166643431613764 -61353764656230666438336135666230636636376132643036303136646165376334636263323435 -62316138353630663938346664393533623137623264383164366536333766343365633764366534 -37323264346264653937363532666530663865333637333836333338306639373632656530333034 -66666534633032663230646564646261363262633031316135373139633765343735613834376235 -66346630303233383566356138393936663935643635356330633139303233303764343133366438 -64653864653261643664383736356330306235633632386531633532333738393465323137323465 -65366636376139663230396665323461363032323038343132303330393734653533613364353134 -31353538353036303531353637373337626333633635643038633639336333663265623636376266 -31363363386264366632353065663933333631643131323532623335636232636263303632653933 -34323065363563303465646239336237613963653831623865366261363635323731653036396538 -62613766356162653563303539613062643064303137356636366136346138663434663864613031 -63313335393965663433326364373431616237393934333337666330653931623938646563626461 -36343764636432333765376532636334653830303731333862356636633965373938353036396138 -37363238616465623562323639633664633833623037333233373265656131356464366633353531 -66653361353162313861313566646235663562393236383962386536386638393239656434633061 -37626266646331333366353232626635303064383037653037626333626235353630313232666165 -37666662366633323237663962653232303164313038663332653164303337356361643861353332 -35373237626533386661356365623364386331356664343166373133643166353031613538626664 -66393764643932346637313138383336623538633836616439346639393137643361393031623836 -39643166643838353461353131393661343361393232643665653431366665373039343063663033 -31393138613435303031356436363564356336663130636134336463393963346336356466323739 -38353335346239363766363031343462326638653835303539333366663333666466656663343934 -36623662616432313730343163353163303865626639343030313535393163346331643339363833 -30373839633034343438356265303066373533633131613465386339343661653462313461386665 -65656630343565643962623539663866393363663865343430303338383761396662613131666338 -34646466316563316661383034613435633562396265636631363633663537336536316530366330 -30376439323362363634306133326132383364363734656332353137643632303934353435613838 -66626430333538323336333764376130613465366462333266623538353439343231646164653465 -39613739633862366461663031343264303634336230383863383461306535346236623065323561 -34613339396233656264386136396637616230386431363865303934663161396165316263623335 -61626137353266643638646263656432393932333561396166303031346331653662373636666563 -63356236353563363530356331636333663233656536353936383838383236336438663861663731 -62623739346139323964663530613236336230666333623766346566303638363935616138313563 -33323931363331333464333739613866303039653135633637633366393437333837653530313836 -35373861353932323734396330636631613963656237396331373761333039363133653536303935 -63636362363163623263323134353335303336333638363239323765336163626237663430643434 -30363866666463646439393965623338353132306530383937666461313566613964663435373865 -64336266343533316232613865633861393739616136313835383731666566656665306265343638 -32656565333135316436303030393934616465396363346132353938376333316232306335363730 -33366437313065393237363032336334386131303431386363303435623134643433643638363638 -36313738353765303438616533303538376236653031333461633064646431326131616466303762 -32343937643663353433393134333664373437643836616432343535616433386531343238373662 -62346531316664316165313264316231666534383832656161333538646130373130306631633766 -39323864613539656337653433376165313262633261613130376535313936316435356534343263 -373835353464366363636464336562636536 +31333264623763653038306239623137343036633961383966353364313731313466646138333537 +6563363536616663653131656164313766303166643565610a363262613236343062306239616339 +64333937336262383931393564386166386332343737653132353866303730306633336432336133 +6137383237636530660a653233636462383335316231303133623832343233643839393135353238 +39376432636564636361656562656533643631373538333865376464613136623730363337623430 +31366639326536373738343130313632383238623864303264646633643162316132363061343661 +30313136326133633065323166386564656336333135383531343234613930666331306663636263 +39313763316534666632346432633637646635666561363139336539366333343532336131643135 +62393132333661346539653965613632383538653632346331333066376237643436356162373434 +30386334393063613434383137613430656461313131313036333934623665333666383361616362 +36386366383838336233633336383635323236386535663339313231616436366535313061386633 +61323565633062323966366534666437333138613264306266323562616632663866313437316664 +39333432373039636466373430653837343062336134373232313536346361666131323734383866 +65363438383533653765633663396230386138613934666166663731353964353431376136633831 +36323936663664636535363065393839393835333239303038626365346165646363666231613833 +65376565616132643031353236386332633239363735636563626437343361633537633638643132 +36373566336466366236393163303263313963353933646536326536313362336235303839373262 +34663134353137653033376233646333663939623331376234393163323266353838356237323234 +36333739386162333565633734346361636565626539616435316333386537316333326565306465 +36396630386639303966323061333733633938306437653933336331333832313639346631613162 +31326634373166346536346362663037623561373665303832306163616465383464623231363463 +37366236356136313339363265376264326335363230363561303430303561356538363563306564 +32616264336633646565363035373235366337383131313231353363303062326437613435313865 +63666165323664353362316465383566313134323264613033393763393862303666393161656234 +64666135623162613435376461333930386335326632356639396164336162386133383737653666 +32633861643565373762626637313134333463303835666566653866306432303865396534613032 +36636165653339383639333334626434396633386239323333306561393661646664363738653535 +38313336376665396538393334626630363465613064343332353037366333346339326530653261 +35373261663738653362663564623961346439613366623632623630633838623530616436373233 +63636636343932656630666432633164306666623761333764313531613235363266376439303066 +39316332623563636637376363636163613534346663666262666139383135363939623966626636 +38346362313963353365346136303863323061623739626364636461646634626363316265306536 +32333732396334343166633437383833653862663837623965356139326537383332653232653363 +66393731303965303839626634373134666536323835613364363237383133643263386235343935 +35633661626130626261396131343461313633613533353938653036383936343236653466376637 +66666565623463663937373231323938343264306631323235303434376537356235353564623666 +39633538656335316366623533303335383761653538306166303030326163623963376232366662 +32303132366639313866633866633436623532663639396332623531313836373730363565613637 +38643034316531373136666137343231653433623562643034343263653238633561346530303963 +66656439383138616463326639633230666461353234373962623839343863363236396532383466 +66633533363430336564643762326565656238366336343363373934616366363937313233393039 +62313437613735343436313266626662353065633063333864616264633833313466326664613332 +35666564303631653031356630613863313939326335363765643330356665383836656638616665 +62656464323134346533393432313061346435646166353861626531313638666136343933383831 +65633637663637376339303331663439626137396462643430326364633565393833346431353535 +37323430633632663562633138356264353538363964663335643733643734396537616465633666 +62626363386239323939656461306138636561616130376537353138376335653665353965353761 +61313130616233616433323533303738623835316638323263333566666562633763636466353935 +30356165623937663832376362313036373836336139333734386230323932376366326531643935 +30333432636432366266346334656665343662376363363439663736636139336365666239373038 +37663163623532343434313331633939643163363532336439616632373262633065396462336164 +36383463316362303138653563623130346130616534323936316461393066353861383830646233 +31346338623235303861623133353934633434393734346234386339386263376230313462346664 +61623262336134346139613863616666666265333439646632663239323036643561383239383339 +35653334616633666666396337663966343636616462623132636665313365643961653336386530 +35303036343835373331663238383138386165313538646362323937373531346434366633623231 +39646535383035623238613838643234653235356265656332346239616166306665623734626430 +34323036613064353462613236366233363065373132386331323061333237653965343936343531 +32646364643663313237623262613763633164326566663666303461373961656637306438393866 +31313132613336336265373038633032333837623737633138343538366661336231613338353064 +32396239303430353934386434376439333633666161313132323135303066346430383465326131 +37323734656631343066616563303131646661396636366238306532636136343339383362303462 +61336336653339663461656332646361653134373437373439663232623237313731613232353435 +33653063643966353561636539326661386538363738396666326535343232386130346134383262 +65313534396365313433356139623864613933323663616230366364613934386165306638386261 +63643838616462636638616239656163306637653637353861336334356235626463386636396162 +35653561336633313331643234386335633038333765393034653561323962623732346532363462 +31646136323330656264346535393732353037313766333233393536383130396564383337376634 +62386366336263646366326530313131363162646466376266383536643564663437316161663833 +34363237323435653234346562626263656234326131643663653131356139386230343138356530 +61306361663063626163663233666630383130323535616265366335313939633939336530366237 +30373438636430313064646661356535383038326539316337613735643836333962396662653931 +65313265393631366239663738333435346533613830336334376531643462356336306636643838 +66333964373966663266653835386536323861326364316139396262663638336664613033663037 +31383232313233383865343330373731623165663766663037663634306334306233356436346166 +37663630346330353163363831306537393939343036303634666634393339373264373436366433 +39363463633839653365653962663866373561393430666535303432316232366135386637393539 +39313862363864306665393538616630643730303663643463656462393861366136633832376563 +64366264636563656164346135393136646131303437393566313834656438656632626339613931 +32313062623233336233626632333862333039366137326232626165333932316534323562336332 +30343637383335323331646631393063663832333163373434386638623963656336636266396631 +33636438303665333834663664633264643335303332646437346361653330613237333163633731 +31666363653862346162613065353866323931373933333139363662613363626338386334333630 +39616364633736333838633838343239333863633730363830643765633362356138353264643238 +61313064333535653830643461326561376639663366316233623737613330343434376563343266 +38363334383539376239616665656239396231343036303730626165313633646463303565313261 +62626330323333343131356666653766666138373134643639626663313931653135303661393464 +66303263663866656536303766616266333430316130366131326461626264363665303165653136 +39303662316437656439393166383263383839336534356436623436386532396237663038663532 +32333333386536363139633362646234636334366237303730323562393064306365613465313961 +63393137356233393463623434323661643365643532323036306163613866353731303535373137 +36393636626162326132646361323562623164643230323839303035306563643761646138393130 +35396238636362313430366431633634383738366662623433613730613261666536336539666336 +37303636313964316631346561333261353739633530363064666130626130306532633163653232 +36643435646535633162613834336238663763643233613461383730393763633737666631633038 +37353265643138616535656666633131646463383031363935653934306265303130336661613562 +31336436323431346638393731616233636235366465373263393133633635663563643133333437 +64393131363266646562663835373232343738323734386539396434663133643835306166313162 +64313639396135353133343862383836383331396336346262616161383334653663366434346266 +31663161376464633836343536616136623032393062376539303964636338356130653239613232 +32323164386563633935373061396339376131393333326264376233636537623264643038656364 +35623335303131373139623066323764303335663962303331353036633233313164343337333566 +61663132363762396363323534643963373938346564303234656163353639626534376464623338 +35666434373734356366623365633034326366626136656335306662653830623936613663306130 +36633435313163623463396239323132383831613065316262636162663835366338653262323062 +39396431353765663937363732373666663165623562373032613137333963643163373235383539 +31343161656232336563383263353164623963653761313265653361333330623831643866343539 +35323537353665663830646162653235396664663130653239363038336363646434383362373233 +36613532663635333232376232613838663339363734643765373865663365393862346436336439 +36626432613933633931613138336231373564633534303264363936333831393436396361623266 +38663636313532363364383663653964336433333231393963386131653932666339626537623034 +33633661633931373866393039343262386534616633346637343032313631336630636533326132 +36626338393236326630623538333833643566623633363132616230376363306233343031366139 +65653766626138333565666661323533313734653164656539363939643966623763386465656265 +62323738323865613862333034363731343935323032666661333461323336333664353937356563 +62626565353666363563343161376636306135303364633566303466333863353963666366626530 +30386238623561643730636264643534326336393533386439663038646637316239386365623535 +38643862393562633031633661373138633231373939376334646237306662646538613039393764 +34323662336434396231343630626138373662343066653464343532356331646335336138646332 +62313338326635626132373235646532343534646366653639316564383830643265643264646234 +34373535336534396661376639393663626338373937343339343635363862623735396263326434 +63333538643564306564386265613038396238663332303364356364343835656338623662653039 +30306262333535623834326133326365303134326265326330613031343037323166336165633463 +61303335336137356438396637383739636163653230323832633839326235386362303464373063 +32363035336164626434323338383163623835396236303132313932653337663131363364396331 +62316134626238343066663130653030623064373765616262363937633438346636636334613335 +38353836386365376438363636316238633338666266323364336236613933656663616336376661 +38356265393637323261616166303361326239666264656464333338613964313337626465333134 +35393333623566333731656432663131353834363461383361396164313763386337653733626538 +33616536343238653434333436333636353437346434383363336164326366346564383835363333 +64343963363566636134333530616538666361633761363561366134313235366437653935643037 +62663335303737373130363734346532653839313131623632623463663161336263386437326337 +64636632346465616337383430313034316231626563373138356235363538363530333736346433 +34343063316637653631636664613434313863653034663662646130396137653466373536656465 +66353231613338366330333531346435653162626265366162396536316234303736323239666533 +34613565326563303030643963666633373761343439633839373135633339376230643761336661 +66383135323338653239636466623662643339373462623263356433343930323239643132636231 +38643165303239306434313733333539346530363433636264616435313534613038303562376633 +32363530313865333962343336326534303364373631643937313963313765336432633965376230 +37626437363338323564393530336539616639386631356532373236623233343665373961393064 +37663166383664333962396262373337653837363863653434363836633133636435643662353632 +63376337323462306336626534346334333431333939646333313138333032663338393665343836 +64306636356264373739366537613535353030633263373131313930663764373131636131633466 +66363035613461653762303434323661373830336662376235613662386132636339636139396365 +35373235313631663834373632316233366631646562383138343063306632373134363164383730 +32333530383262323938383331356636333735323031363564616231643333623635373534333035 +63643237366634303464336433306164656361336164333766633130386331653434656364313439 +36663461323737653562326635393037653432333537616630373039323330363161646138623034 +33333136666565666463353634623265656639303134656534373731313332316237373564623634 +63363530346233656139653136653563363833646464643730343262656530613731646239643466 +37393862626232373932313431383366383430656336343437373039346133313866613836386432 +61306630656534306638373964393266393566636233376666636335393462623131383238353531 +31306639373837343064616134666466373638376235323665383762663436333130373537633263 +62393861633465366266643536646333393064656664373866643833346137336239303462396537 +35353236336666633336633038376533383263383031316638393765663866303537373038323833 +37383133353239323966633535636462336533306236613363656561633538613232353563633638 +65653235373131363262353463656437313238386433636466633439663139306365303863393563 +63326637346235363739383936366363306162346532613136653962656564633230336537653562 +38333163663631393762333930663735636562626365306566663964633331373564373065306434 +63643536636465626662616230323336383962303365393962313834623939356666393232646265 +39343234353730356339626239316463633263626231353064323439393066656364353932643133 +31333465626332333164343862323132373833613839363563343136383539306636646266623438 +30333430636235646432396437666238333734633062336532323432366362396137343463303962 +32636130316638376530663364376532656532373737633432356536623965626536363536393033 +38643433393361326531643634383565393564636161626666323861646230366666333830336136 +34313430383634393137643736636566633566343864656131373439343335353335323333613163 +31303865623739353530633664636339303166646162636333333333393536383636636332636234 +63656535383062616663363734663636363333316665353238346366383436366537343938306238 +31386630323065383736623330663737663766306536316235663933643839653430663337633665 +32386639333839363161666133353135386533326563303038333834613664313162376535333663 +30356462643133643363376431346433626261393031613963363735366238316165306162383530 +39666563393233393962303138313234336138373564636233623266616238666139626537653933 +62326532643739326534353438666236343261396237626565626566363332353433666634653636 +33366162623438653735336165643861636536663037373862303063353436373165636232656466 +62653161633231326335343236613034323764643064616339336138623632373339616230356163 +65306134623466333131613933653966633537303365663337616663346631303830646266393235 +30326161326638393230656339663065343436363030366136356466383562363630643062363161 +30346666646632646535356464393465623230636464343538346366306238383962343065643932 +35363664363330663633343763326230363236646538393636386231616134336133613662393530 +62303534616133333030336565616432326565386330396337316466663331306335316239346363 +31636462623738356438633036393237613165353364643935373038316531316136333666376231 +32663736366239383434383435363665663639343935613461636365626430326536386538313963 +33623866376232613238336132336661303963303963626262633765663239396635326538636466 +30313963626635643730356334336133313463333662353237316362643964616662343533306262 +37623535333035373433333861366463386535616334316265316361383634383537323463373135 +64653134626634356137303133306165613532613636316434366361393339626638313364303364 +64346232333961383564353636623161383462343461333563353431386465363663663463393566 +32313032346136633532613765323033653836343562316365646436393135303233656364306163 +31623565343833336664616537393865326133376333386366396563333939623739363564386263 +63353662376434643136633833306565303130643564376563313735353734653038373438323766 +38663064373765633864636537666230333461386362663238393631353331336132366630643663 +32393134346339323538616661326463626435353865613562643462376232313531633436396231 +66333238366331316534646334656334383339383231343535313738316634386637646335666132 +38333063323833613262363062303961666234386435613765333338353932643439643038633365 +38306264353135643362616463373638333839363731356564643934316466376566626662666566 +37366537623462336539636232383835366164316462383231353238316136386261326339333431 +63353738363761663833616333633534613234663633353534366133616631353130646163643165 +39373137373939623639303731633565636232623936333264306531643866626438623738646334 +61323731383335633764383037666533393363636333626535653438303134396439346561303662 +32306162393964633636666438393531363638663265626665396430333863633961313737363139 +65613938376235303465613731633231666331653035346331636261666162653163393431323539 +63363463383131666636303839353033626265363264313537613236336262323337376365643662 +65316466326163623466373235323133363334613731653034333564356439366637663332663463 +39303037366536393935336331653330393962636532383161316334623736386637356432666233 +65313561663637633962313035363962336461363138626536383037633930353135353935346463 +35373266636135666266633563303361626530663032356230643565303135353766376138373637 +66633164326464376361303137363238346665346165393739303761383062333634643632653861 +61356538323938616437366334643761373032303662616664646434343434616136666238636164 +62353432346462653166303739306232343662343030313933623031346632616633313266346539 +32653832363266656331373564313665303664353736636434363336323263383562383335633935 +30303662343333363563353064383131326365653065373264366161306665623034616561373530 +36636239636332326465663036623437363865396136386362616163656566386332343962633639 +33303863323032303033643766306439303130363961333162333333313237326662366365636166 +65366264313132336462663534313062323463636634326636613931363962333437383536346664 +37316431363164393730666265633465393162333936383165623966396465356537363636653564 +31306262633462376164313534376637396464336461386564303539376437653037303238323066 +31383964393865343332356636626361356337303131383935316330303233653136386135663238 +37653965626638646239636235663534643766353033343836653564626133663332646131303836 +64646665633738646163616665623464303531623937366438633064336232666139366139353839 +31323531623965303139636338306139666433623866626263356562343565626264353632393437 +61623863306231383534383132366136653130636131633065346363316533316261633031626465 +37643035636466313532366463353763313665373966643931653132393961396263323461326661 +63656665623538363739333736323665633665656331303135333666653765613964646437393638 +36323638653732396163633166346161656632303666333732613961343864613131666561366332 +39313330366635663863313661623532336638626266653632656337633339363532643533326265 +37393736313838333066653134336566666164316662366232623563313339636634643133613533 +36353664326564333961306334313539393937383034303330376133376431383164396161636132 +62656261346665393339643233363433643065336362323832383265366164353531363262303138 +35636439656662306231643438346663356533646639653536636232626335383635343534663662 +33353262336565316463343837353464396462313935633338636237663433336263626662363934 +33383563363964383961656439383031306136646566343232636631653436656133343937303933 +61323534373561626463333566326266633239303861366563626637623139306330366437663434 +31383063363165663763303934313561333230393135623938336165326139333833333032316538 +32373938653663376334343266386133306565613334613766643435616165656130376239363463 +37613362636230343433613763306234383565386132333239626333623531396238623036616331 +39323931656234656564306562363236396139643361643232356132353434626633363031366639 +35636565626530316465386431333833396434363661663561663864393365613061313538616333 +31616261393635373462396465353564626466663961353238646463636134366361343030323034 +36353837663030643161313061306438666234303933313239643234306333646131356562656431 +36363062656633653363326563316664323332616165356666656132363334393130363163613662 +61316234633530656361663330313938666631646163363831653962343638363532626466643064 +61646661343262326536393630376436363934656663653037383364613435383436303539336634 +39376164373264646138343930366630373061653564663438346166343261326164373931386665 +33363734656133366561323561666334663835646464393231353762643030306635303662323836 +65613436366166303865663935643439343062656530396137623663343162323165366538643739 +36333661613864613561333463643835613839623165376261636433393637646664373266353964 +33393434323463616164366564643664613836343233653365633133633966316463386630396134 +64636565313432393164666665346136353833306534316636616234383961393131393934623161 +61343065323134633537663561653566333765393335633536373232396330666534323532353363 +35653761303630626632313239633564363230653062303062316534636437653966653931393338 +38393066663466653366653730633464653362363139626439303837633736666431646263373464 +30613563626131623038636165343334626339386233383237353265663063623731356361396636 +64356632323965323334643263356432323731353562353937393765393839653234323363633664 +33303063633463373331356136333764376663623130613662333538383561316266643564303938 +35663630383864646438336465313737386639646162623335306336666432616439336234376565 +32636532613965363933613132336266373262313932303534393564653063396531363165656437 +66393463366661653637306661316166393135373438373834663361393061353361363461366137 +63303938633231633832663063393539663833383230643965346361316230376139373839653036 +39666535663131623738313334396638636266303830306636343766313735663264356234626563 +34623031376239323330333138346136376466633031303830393631343430633363363639303639 +65376339386665393533653263343065376366313964663431383162353831333565303665313963 +37383062636230333361326539336631643936323663626564626565666134383939633739373130 +36306237613032616532303730633362363363363539396336323834383130373433386436323033 +62353834633034666139623030323836653339653461613731363039353763393132336566333737 +32356134623565383236366536326561363031623362333134346433656165333465653331343437 +32383463316238363938656331653730636163333536313834643431653536356662303337333533 +30636138663630643932303432666635393731656262313062346165333139303236393335653262 +37343830336161616465346265616666376365376334633134336362613633633337313835646437 +39653563613833656531646664623836613633636361316435303436366334666539346164336639 +36303961626434663963323333366265643734616131623463346132616165313366393261313433 +63346432613633613837316361643938313262643331323065666362643461336337633038376163 +34396331373066386337623732363566323664633239323534643733333663396464613130393163 +63333734623235663333353935656537316431393835663938353461373138653032663535333634 +61336163393538373035623338623733663261333132623536383431356265636135376163383837 +30373933306261613938376364386134333831323661366163343538366161333361363564626237 +62623231383865366461346238366262623030396133346665646362313661323361636463316336 +35343432643663396434333035343732613232616663663130656539363139306264313964643931 +32616130303663373636386533396132386135663430663936333834326333316561633636393262 +63333063336335306633366466343965363364643035636637653030396430636234326536396236 +65373130643865323031396132373938303036353966316536376130383731393338633934663738 +32636562353066653466343331313430636233313431613334646233386366663935643832646435 +38373739333434643231313336326266363335346334376462363831656332393736653365636463 +31306635303835346235316130323964313438383237633136376538363162396261333935646530 +32633136383130623763366432646563373965323964653532393339656434363231383534373336 +32393533663564636332636165653865613430343539643763373163653861356466646463663630 +32613839303636663232373534636265623363366266366638376132653639666432626637653934 +65373461313734333763666265386433613561623539333661373139316663626564343162616637 +33636461366234343264393339303932373239613036626636323939303237623038303736326134 +35643630626533346565383764626433306238646238383063343165623939306236396533623161 +34393539646438343736323536633333343133353336333936313833353732343965383338646234 +33333962353236303439313736386139336438366263366566353364353432303838336364316132 +63303539663333383431646434626235363431303337363934653062343465323337363937653665 +35323031303736663839363832323630396331313962383465323430366237386665346237643237 +36623864386431393232656561383536623130663938333730346439613134653835386139623339 +36663431616132613334386662343465616132303662653234303664646537346464326361303866 +61313361353838386638646565613032383537353138616137356266323762393035316261663534 +32666362386235363334633665386537323161653936386362326536303561376233613838316135 +62366439386136623033653730363130366139646662343333616336343530393734303136353331 +33303535343836353662636264643430353935663738653930613736636633613264613761663939 +64323830643734636131346334313961393439363732393064313162346431646332636235323936 +31396430336264373461633665373032356130313932363764383338356332626635613330353734 +30643236666565373262616438623332353433366461376330316438386432363066366533613966 +30373966336362303666646333623965633861363239633037316562653832363366376634356631 +62656461363030333934363863666661643437643866316262363263613036323232623235383562 +62346237656436623431363836363638316662323263396164303834363739633631303662343036 +66653934616433613264393062646530643536653965623330303161393035643662626333303031 +30363836376362396362346633393636623539623730663439626561303233646265393337356166 +63363833623238643738363165653037343435646436323235363765333361333936376536376161 +65353839303538383035376132666334353461656330373365646632616438316661303162393530 +64303161656234613435326338336134643337663032373931636137626235343465386166303832 +61636464623430666164373830663538383237663563366534623731656164346461383738313761 +37363466316132366366613431313536336632643766623866353439316361346630363831626636 +33656161356264626136303163326261353437323963376437613934396537363763396433623064 +62336231393735323132646436393665376164393031323433393065363339613435646536323963 +32383162653832303537366632386632373962346563396232613432633833303437323337326137 +38336335323034633032363362363461633330643463336634346364326432376339623833383237 +30356562393533333166323763646666643038346661643966336430663733386366663431646461 +36366561373533336439623530636566386562376264646231303535353030316332333564366238 +62633138333863313030636339646633356533653131383438306363346165613733346564363966 +64383861613433636334656635396233626136306563666133303765386139383362623234396562 +33626332376161303063666263646163353735393164643030396261643431383338383831316530 +65663230333035616634356263346561343664376534316239363235323134643961316165616632 +61623330366330643630303334656161636330633362646463356536313136376231363035326562 +36623266626634333762613966336432393763636363633335383835333138333264323637323662 +37663535396562303634323632626163613131366137363235383033343232646663353131396332 +65636338306539336334626166393738343864363336363831656633396562613534633432633533 +63653032633263306537653830363336316163386435383466396465613835333232383661353166 +34643962666630366130623633636362353363313533623735323031313361363563666562666137 +37653766636237633162623639643038633533653665326330633762616534653762373536656361 +64353362393539373464303736336664643933373339346631363932363031313065383131616361 +64616138306163323938633963383766336537623439653136616132666232356631373335396131 +62653538373237393735616138626466653434623732313636353930633234306236313637363832 +63633562323963643939663438633731393232343364663561653534663438323161356232646362 +66393439373661363235313464643437373833356639616561396266643761626334656331316264 +62643166613862333863353631643534326639666638353362633136363231376336663562396664 +65643831653130353933643836373062393934633132393235336361373837666264313333363633 +63303938343030323635383535313638643632386237643631633933383265343338336163393465 +66356339393261303939616537623234663563313230303830383966396436613534303536623065 +63663739613538373237616236303662353365376332363663386463353565646635653866333236 +66303533333962306336306533643132376435303838366661633865303435633737373931313363 +39613135316639653239373534663137356466656638633339643234646430666263396630636434 +63626237613239386237336263393462616365636137376165386466633266336336343434393934 +64343932303630623037303638633961333166373838626233306233333064353665636637646234 +63333733656134313462333832636330653238666664663137646236656436663535643533636362 +32393732383265383831636633376432303464366261356463636462616436396161306532313065 +39336465653861393838376639323933396639366531353835336134623864663964366130343264 +63306364343936616539626133663865646231663039623262346337363435613561333238376564 +39383332633939366138313365393037386237376437623765393731346335316537326237353835 +31366436313132363063333735363362306463356431663537346431393630306338313162633764 +32653665323234393936373138656166646534376234373261376338306264343232393963613834 +34656536653733376137363439396234666437313938353065313135383564313332336431333662 +37363233353466613962323962366235373933323230666232666533633965616132323730353964 +32363738356236643764653933356535623236623964326334666265636633666162633463326664 +38396433613738663962666334336535626439313931343937616532636135393061633863346133 +36663035653763633233386433653565333065383533373636376438643538343132633562343833 +63366464653230623866363439653066353437323139313166393038633439303834653935623239 +30623239323132656233373537633637633639393436383432363830626238363030333935663437 +32373635393963613463366362326234356262663431336265613664333566623936326531666564 +61366533653038663631643237613536336664373138666464613861333131646135326565316232 +37356664646336316437373533633935666264323765316133366538623330616632346632303963 +37386535633430643864646566336663386664356136643265326335333537396435616635333739 +38646539333365323834373364376533386561303564613934343832363161323961313537623138 +31653062336439336339386162646339616334613432346262396463393361613830366234653236 +66376663363663366230663866323537633935383432626466623432316563323631306666623862 +62383839336163626135396235396436653164343531336539383562636533633462346335316435 +36613637616461323130336665333835663633613039383932663830653836393335306631643531 +38376639623565393338633630313465323466383964303265613238316364383636653562366361 +39316630653465633961636139633335646635396366336464663535303439353133666532633030 +36323066316364323432376464373764333439633265313737636464383364353437313530336163 +37653161383833323061623535646234386564623434303932326262656262373836626263383135 +39383161393134663663303763633533333563346663656537336430333161346438336533376138 +36366363333566663238663635386362643131306435333736616433626264636332353033383331 +35323936663636646461643831653063366662306435663163383239383464356663366539363665 +66396234633038313162653533383063633164373161376237316363363235613335653433663438 +35376161346165383265336239343031383232653864666230303362646635646262636262356435 +31396632363336323630373839323536326531353430306362313130363063376164316439316162 +66356166346232643530313763366465336137303763323064393036646334383962633935393534 +62343466626335656163663465393831616330316238393235643538363064356436373065313263 +61313764613237653731316436333738633334366532373630636439376632353535363831626465 +65343233323733386536376364353636343532323362666334346234643738303135396231656365 +34323834633035623239643338643831333431393436393931393335636466393561336663626331 +36643330386365663839306134646538666236316535316232326438666236363336303563653839 +33633036633465336433313163636632666637303564393239326236326330636161363766343065 +63343664353031626334323334633330336439383464616338346165326137386439373138376230 +31323731383062643962336363636665646636313830313834363961623137643035663735663731 +31653163383733316262636561636236653164366666373133633564376666316335616364396462 +33616330386336633836356166653330376437653361313933313932383034616465353430373830 +65383766656462646665633138366437346335373362313063373330373736383963623135353335 +66373938643332633035326131353662376461613433613062396133383666646633303935663830 +64633635343832396532396666373066383066343662356533363537643534663239306362646165 +37346639383062633762376639386164386231353434343666323732343533356233326239333530 +65306565336438333735356539626639353266616230363935306464376337333665343236363763 +62346566353633323432336265646365616235313039353933323765336438643938623631363965 +39303939396433323231336630323139623466386431333034383066306265623631346133366437 +37333032653731386566356233343038306139663364303137346661396232376230386132353562 +32393130313338646364613661656536626635333037656464353864336238323263376332366231 +33373463616331626631666461383737623163346464643634336536393835303731626230663463 +32323635303531323433646632613039353837373163316637313661663263343435393663623761 +39353339326563633465663238306539373438636239666139636630656434333565303133303536 +34306431666664396665653536643862343334626433636138623664663038333134333965623466 +38353432323365306466616331363261326134343635393264393765373737633235663239326264 +39343837303835656230653631306362393630643163343539303962316630626566313761396466 +63316565323262646130663766326230346466613366323937366662663631393364326462343263 +32653562396465646337663066363839336237336138323135316435393131323130666566633661 +30636331633032343765656465336133666534616161653630393130353366623737353831353964 +31336462326533393061623030633439343134323936373165356435316662306662636337633737 +38633134383262663835313264303730643530666232353735343763653164323634393731653337 +66326539363936376665663838663435383530323137663763336164666538323935336465323838 +31323135643764356130333863333435633738303231356663393531333966643135646430396335 +66666237666663613266323131363534643034643765333639323462346661326565306638666464 +35373730393965313637383934613637656264326531393237656666623261613835346264666339 +66393465373863396265623038333863623162656335646334353635333435656430343232333862 +39656166326564616431623138383236646539626463646637663535356136653036323137363833 +37623661663763636635636437353639363063663737343131306336373630396337356466353665 +643462643039663633383739353831353066 diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index 7926323..23fa08a 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -25,4 +25,7 @@ ansible.builtin.import_playbook: plays/infra.yaml - name: backup.unruhig.eu ansible.builtin.import_playbook: plays/backup.yaml + +- name: grp_prometheus + ansible.builtin.import_playbook: plays/grp_prometheus.yaml ... diff --git a/ansible/plays/grp_prometheus.yaml b/ansible/plays/grp_prometheus.yaml new file mode 100644 index 0000000..ec6b70c --- /dev/null +++ b/ansible/plays/grp_prometheus.yaml @@ -0,0 +1,8 @@ +- name: Deploy Metrics exporter + hosts: prometheus + vars: + state: running + roles: + - {role: compose_project, service: metric-export} + +# vim: ft=yaml.ansible diff --git a/ansible/plays/services/grafana/.env b/ansible/plays/services/grafana/.env new file mode 100644 index 0000000..57421fc --- /dev/null +++ b/ansible/plays/services/grafana/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=grafana diff --git a/ansible/plays/services/grafana/docker-compose.yaml b/ansible/plays/services/grafana/docker-compose.yaml new file mode 100644 index 0000000..4362b52 --- /dev/null +++ b/ansible/plays/services/grafana/docker-compose.yaml @@ -0,0 +1,43 @@ +version: "3.4" +services: + grafana: + image: grafana/grafana:latest + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.rule=Host(`grafana.tobiasmanske.de`)" + - "traefik.http.routers.grafana.entryPoints=websecure" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + networks: + - backend + - default + environment: + - "GF_SERVER_ROOT_URL=https://grafana.tobiasmanske.de" + - "GF_SECURITY_ADMIN_USER={{ grafana.admin.user }}" + - "GF_SECURITY_ADMIN_PASSWORD={{ grafana.admin.password }}" + - "GF_AUTH_GENERIC_OAUTH_NAME=Keycloak" + - "GF_AUTH_GENERIC_OAUTH_ENABLED=true" + - "GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true" + - "GF_AUTH_GENERIC_OAUTH_CLIENT_ID={{ grafana.oidc.client_id }}" + - "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET={{ grafana.oidc.client_secret }}" + - "GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile offline_access roles" + - "GF_AUTH_GENERIC_OAUTH_GROUP_ATTRIBUTE_PATH=groups" + - "GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email" + - "GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=preferred_username" + - "GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name" + - "GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://{{ grafana.oidc.url }}/realms/{{ grafana.oidc.realm_name }}/protocol/openid-connect/auth" + - "GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://{{ grafana.oidc.url }}/realms/{{ grafana.oidc.realm_name }}/protocol/openid-connect/token" + - "GF_AUTH_GENERIC_OAUTH_API_URL=https://{{ grafana.oidc.url }}/realms/{{ grafana.oidc.realm_name }}/protocol/openid-connect/userinfo" + - "GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(resource_access.grafana.roles[*], 'serveradmin') && 'GrafanaAdmin' || contains(resource_access.grafana.roles[*], 'admin') && 'Admin' || contains(resource_access.grafana.roles[*], 'editor') && 'Editor' || 'Viewer'" + - "GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true" + volumes: + - data:/var/lib/grafana + - ./grafana-ds.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro,Z + - ./grafana-db.yml:/etc/grafana/provisioning/dashboards/datasource.yml:ro,Z + - ./grafana-dashboards:/var/lib/grafana/dashboards:ro,Z + +volumes: + data: +networks: + backend: + internal: true diff --git a/ansible/plays/services/prometheus/grafana-dashboards/cadvisor.json b/ansible/plays/services/grafana/grafana-dashboards/cadvisor.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/cadvisor.json rename to ansible/plays/services/grafana/grafana-dashboards/cadvisor.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/droneci.json b/ansible/plays/services/grafana/grafana-dashboards/droneci.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/droneci.json rename to ansible/plays/services/grafana/grafana-dashboards/droneci.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/hedgedoc.json b/ansible/plays/services/grafana/grafana-dashboards/hedgedoc.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/hedgedoc.json rename to ansible/plays/services/grafana/grafana-dashboards/hedgedoc.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/kuma.json b/ansible/plays/services/grafana/grafana-dashboards/kuma.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/kuma.json rename to ansible/plays/services/grafana/grafana-dashboards/kuma.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/miniflux.json b/ansible/plays/services/grafana/grafana-dashboards/miniflux.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/miniflux.json rename to ansible/plays/services/grafana/grafana-dashboards/miniflux.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/minio.json b/ansible/plays/services/grafana/grafana-dashboards/minio.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/minio.json rename to ansible/plays/services/grafana/grafana-dashboards/minio.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/prometheus-stats.json b/ansible/plays/services/grafana/grafana-dashboards/prometheus-stats.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/prometheus-stats.json rename to ansible/plays/services/grafana/grafana-dashboards/prometheus-stats.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/synapse.json b/ansible/plays/services/grafana/grafana-dashboards/synapse.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/synapse.json rename to ansible/plays/services/grafana/grafana-dashboards/synapse.json diff --git a/ansible/plays/services/prometheus/grafana-dashboards/traefik2.json b/ansible/plays/services/grafana/grafana-dashboards/traefik2.json similarity index 100% rename from ansible/plays/services/prometheus/grafana-dashboards/traefik2.json rename to ansible/plays/services/grafana/grafana-dashboards/traefik2.json diff --git a/ansible/plays/services/prometheus/grafana-db.yml b/ansible/plays/services/grafana/grafana-db.yml similarity index 100% rename from ansible/plays/services/prometheus/grafana-db.yml rename to ansible/plays/services/grafana/grafana-db.yml diff --git a/ansible/plays/services/grafana/grafana-ds.yml b/ansible/plays/services/grafana/grafana-ds.yml new file mode 100644 index 0000000..255dd7b --- /dev/null +++ b/ansible/plays/services/grafana/grafana-ds.yml @@ -0,0 +1,28 @@ +apiVersion: 1 + +datasources: +- name: Mimir Netcup + type: prometheus + basicAuth: true + basicAuthUser: {{ common.mimir.username }} + jsonData: + httpHeaderName1: "X-Scope-OrgID" + secureJsonData: + basicAuthPassword: {{ common.mimir.password }} + httpHeaderValue1: "{{ groups['prometheus']|map('extract', hostvars, 'inventory_hostname')|join('|')|replace('.','-') }}" + url: https://{{ common.mimir.host }}/prometheus + isDefault: false + access: proxy + editable: true +- name: Loki + type: loki + access: proxy + orgId: 1 + url: https://{{ common.loki.host }} + basicAuth: true + basicAuthUser: {{ common.loki.username }} + secureJsonData: + basicAuthPassword: {{ common.loki.password }} + isDefault: false + version: 1 + editable: true diff --git a/ansible/plays/services/loki/.env b/ansible/plays/services/loki/.env new file mode 100644 index 0000000..37b75ab --- /dev/null +++ b/ansible/plays/services/loki/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=loki diff --git a/ansible/plays/services/loki/docker-compose.yaml b/ansible/plays/services/loki/docker-compose.yaml new file mode 100644 index 0000000..7a1fff1 --- /dev/null +++ b/ansible/plays/services/loki/docker-compose.yaml @@ -0,0 +1,27 @@ +version: "3.4" +services: + loki: + image: grafana/loki:latest + restart: unless-stopped + command: -config.file=/etc/loki/loki.yaml + volumes: + - ./loki.yml:/etc/loki/loki.yaml:ro,Z + - loki_data:/loki + labels: + - "traefik.enable=true" + - "traefik.http.routers.loki.rule=Host(`loki.tobiasmanske.de`)" + - "traefik.http.middlewares.loki-auth.basicauth.users={{ common.loki.username }}:{{ common.loki.password_hash | mandatory }}" + - "traefik.http.routers.loki.entryPoints=websecure" + - "traefik.http.services.loki.loadbalancer.server.port=3100" + - "traefik.http.routers.loki.middlewares=loki-auth" + - "prometheus-scrape.enabled=true" + - "prometheus-scrape.port=3100" + networks: + - metrics + - default + +volumes: + loki_data: +networks: + metrics: + external: true diff --git a/ansible/plays/services/prometheus/loki.yml b/ansible/plays/services/loki/loki.yml similarity index 100% rename from ansible/plays/services/prometheus/loki.yml rename to ansible/plays/services/loki/loki.yml diff --git a/ansible/plays/services/matrix/synapse-config/homeserver.yaml b/ansible/plays/services/matrix/synapse-config/homeserver.yaml index e010cc3..f69585a 100644 --- a/ansible/plays/services/matrix/synapse-config/homeserver.yaml +++ b/ansible/plays/services/matrix/synapse-config/homeserver.yaml @@ -78,6 +78,10 @@ rc_joins: local: per_second: 100 burst_count: 100 +rc_login: + address: + per_second: 1000 + burst_count: 1000 server_notices: system_mxid_localpart: "server" system_mxid_display_name: "Server Notices" diff --git a/ansible/plays/services/metric-export/.env b/ansible/plays/services/metric-export/.env new file mode 100644 index 0000000..acc4a36 --- /dev/null +++ b/ansible/plays/services/metric-export/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=metrics diff --git a/ansible/plays/services/metric-export/docker-compose.yaml b/ansible/plays/services/metric-export/docker-compose.yaml new file mode 100644 index 0000000..e4df175 --- /dev/null +++ b/ansible/plays/services/metric-export/docker-compose.yaml @@ -0,0 +1,103 @@ +version: "3.4" +services: + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yml' + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z + - prom_data:/prometheus + - label_discovery:/label_discovery:ro + labels: + - "traefik.enable=false" + depends_on: + - prometheus-docker-sd + - cadvisor + - node-exporter + networks: + - default # send + - backend + - metrics + + prometheus-docker-sd: + image: registry.tobiasmanske.de/prometheus-docker-sd:latest + restart: unless-stopped + privileged: true + networks: + - backend + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro,Z + - label_discovery:/prometheus-docker-sd:rw + logging: # this service generates a HUGE amout of logs. + driver: "none" + + node-exporter: + image: quay.io/prometheus/node-exporter:latest + container_name: "{{ inventory_hostname | replace('.', '-') }}-node-exporter" + privileged: true + labels: + - "prometheus-scrape.enabled=true" + - "prometheus-scrape.port=9100" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + - /:/host:ro,rslave + - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro + command: + - '--path.rootfs=/host' + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.ignored-mount-points' + - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" + - '--collector.systemd' + networks: + - backend + restart: unless-stopped + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + privileged: true + labels: + - "prometheus-scrape.enabled=true" + - "prometheus-scrape.port=8080" + command: + - "-docker_only=true" + - "-housekeeping_interval=10s" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + networks: + - backend + restart: unless-stopped + + promtail: + image: grafana/promtail:latest + security_opt: + - label:disable + restart: unless-stopped + volumes: + - ./promtail.yml:/etc/promtail/config.yml:ro + - /var/log:/var/log:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + command: -config.file=/etc/promtail/config.yml + labels: + - "prometheus-scrape.enabled=true" + - "prometheus-scrape.port=8080" + networks: + - default # send + - backend + - metrics + +volumes: + prom_data: + label_discovery: +networks: + backend: + internal: true + metrics: + external: true diff --git a/ansible/plays/services/metric-export/prometheus.yml b/ansible/plays/services/metric-export/prometheus.yml new file mode 100644 index 0000000..66bbe30 --- /dev/null +++ b/ansible/plays/services/metric-export/prometheus.yml @@ -0,0 +1,30 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +scrape_configs: +- job_name: prometheus + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - localhost:9090 +- job_name: 'service_discovery' + metric_relabel_configs: + - source_labels: + - "container_name" + target_label: "instance" + action: replace + file_sd_configs: + - files: + - /label_discovery/docker-targets.json +remote_write: + - url: https://{{ common.mimir.host | mandatory }}/api/v1/push + headers: + X-Scope-OrgID: "{{ inventory_hostname | replace('.', '-') }}" + basic_auth: + username: "{{ common.mimir.username | mandatory }}" + password: "{{ common.mimir.password | mandatory }}" diff --git a/ansible/plays/services/prometheus/promtail.yml b/ansible/plays/services/metric-export/promtail.yml similarity index 70% rename from ansible/plays/services/prometheus/promtail.yml rename to ansible/plays/services/metric-export/promtail.yml index a9b66e3..a69b352 100644 --- a/ansible/plays/services/prometheus/promtail.yml +++ b/ansible/plays/services/metric-export/promtail.yml @@ -4,7 +4,11 @@ server: http_listen_port: 8080 clients: - - url: http://loki:3100/loki/api/v1/push + - url: https://{{ common.loki.host | mandatory }}/loki/api/v1/push + tenant_id: "{{ inventory_hostname | replace('.', '-') }}" + basic_auth: + username: "{{ common.loki.username | mandatory }}" + password: "{{ common.loki.password | mandatory }}" scrape_configs: - job_name: flog_scrape diff --git a/ansible/plays/services/mimir/.env b/ansible/plays/services/mimir/.env new file mode 100644 index 0000000..ed5d735 --- /dev/null +++ b/ansible/plays/services/mimir/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=mimir diff --git a/ansible/plays/services/prometheus/alertmanager.yml b/ansible/plays/services/mimir/alertmanager.yml similarity index 53% rename from ansible/plays/services/prometheus/alertmanager.yml rename to ansible/plays/services/mimir/alertmanager.yml index c0e3f17..b99327e 100644 --- a/ansible/plays/services/prometheus/alertmanager.yml +++ b/ansible/plays/services/mimir/alertmanager.yml @@ -31,20 +31,20 @@ route: receivers: - name: 'email' email_configs: - - to: '{{ prometheus.alertmanager.smtp.target }}' - from: '"Alertmanager" <{{ prometheus.alertmanager.smtp.username }}>' + - to: '{{ mimir.alertmanager.smtp.target }}' + from: '"Alertmanager" <{{ mimir.alertmanager.smtp.username }}>' smarthost: 'mxe8cf.netcup.net:587' - auth_username: '{{ prometheus.alertmanager.smtp.username }}' - auth_identity: '{{ prometheus.alertmanager.smtp.username }}' - auth_password: '{{ prometheus.alertmanager.smtp.password }}' + auth_username: '{{ mimir.alertmanager.smtp.username }}' + auth_identity: '{{ mimir.alertmanager.smtp.username }}' + auth_password: '{{ mimir.alertmanager.smtp.password }}' - name: 'hcio' email_configs: - - to: '{{ prometheus.alertmanager.hcio.mail }}' - from: '"Alertmanager" <{{ prometheus.alertmanager.smtp.username }}>' + - to: '{{ mimir.alertmanager.hcio.mail }}' + from: '"Alertmanager" <{{ mimir.alertmanager.smtp.username }}>' smarthost: 'mxe8cf.netcup.net:587' - auth_username: '{{ prometheus.alertmanager.smtp.username }}' - auth_identity: '{{ prometheus.alertmanager.smtp.username }}' - auth_password: '{{ prometheus.alertmanager.smtp.password }}' + auth_username: '{{ mimir.alertmanager.smtp.username }}' + auth_identity: '{{ mimir.alertmanager.smtp.username }}' + auth_password: '{{ mimir.alertmanager.smtp.password }}' - name: 'matrix-monitoring' webhook_configs: - - url: 'http://alertmanager-matrix:3000/alerts?secret={{ prometheus.alertmanager.matrix.alertmanager_token }}' + - url: 'http://alertmanager-matrix:3000/alerts?secret={{ mimir.alertmanager.matrix.alertmanager_token }}' diff --git a/ansible/plays/services/mimir/docker-compose.yaml b/ansible/plays/services/mimir/docker-compose.yaml new file mode 100644 index 0000000..032f4fa --- /dev/null +++ b/ansible/plays/services/mimir/docker-compose.yaml @@ -0,0 +1,80 @@ +services: + mimir: + image: grafana/mimir:latest + restart: unless-stopped + volumes: + - data:/mimir + - ./mimir.yml:/etc/mimir-config/mimir.yaml:ro,Z +{% for tenant in groups['prometheus']|map('extract', hostvars, 'inventory_hostname') %} + - ./rules:/data/rules/{{ tenant|replace('.', '-') }}:ro,Z +{% endfor %} + entrypoint: + - /bin/mimir + - -config.file=/etc/mimir-config/mimir.yaml + - -validation.max-label-names-per-series=60 + - -tenant-federation.enabled=true + labels: + - "prometheus-scrape.enabled=true" + - "prometheus-scrape.port=8080" + - "traefik.enable=true" + - "traefik.http.routers.mimir.rule=Host(`mimir.tobiasmanske.de`)" + - "traefik.http.middlewares.mimir-auth.basicauth.users={{ common.mimir.username }}:{{ common.mimir.password_hash | mandatory }}" + - "traefik.http.routers.mimir.entryPoints=websecure" + - "traefik.http.services.mimir.loadbalancer.server.port=8080" + - "traefik.http.routers.mimir.middlewares=mimir-auth" + networks: + - backend + - default + alertmanager: + image: prom/alertmanager:latest + labels: + - "traefik.enable=true" + - "traefik.http.routers.alertmanager.rule=Host(`alertmanager.tobiasmanske.de`)" + - "traefik.http.routers.alertmanager.entryPoints=websecure" + - "traefik.http.services.alertmanager.loadbalancer.server.port=9093" + - "traefik.http.routers.alertmanager.middlewares=oauth@file" + volumes: + - ./alertmanager.yml:/etc/alertmanager/config.yml:ro,Z + - alertmanager_data:/data + networks: + - backend + - alertmanager + - default + restart: unless-stopped + command: + - '--config.file=/etc/alertmanager/config.yml' + - '--web.external-url=https://alertmanager.tobiasmanske.de' + - '--storage.path=/data' + + alertmanager-matrix: + image: jaywink/matrix-alertmanager:latest + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.alertmanager-matrix.rule=Host(`alertmanager.tobiasmanske.de`) && PathPrefix(`/matrix/`)" + - "traefik.http.routers.alertmanager-matrix.middlewares=matrix-strip" + - "traefik.http.middlewares.matrix-strip.stripprefix.prefixes=/matrix" + - "traefik.http.middlewares.matrix-strip.stripprefix.forceslash=false" + - "traefik.http.routers.alertmanager-matrix.entryPoints=websecure" + - "traefik.http.services.alertmanager-matrix.loadbalancer.server.port=3000" + environment: + - APP_PORT=3000 + - APP_ALERTMANAGER_SECRET={{ mimir.alertmanager.matrix.alertmanager_token | mandatory }} + - MATRIX_HOMESERVER_URL=http://pantalaimon:8008 + - MATRIX_ROOMS={{ mimir.alertmanager.matrix.rooms | join('|') }} + - MATRIX_TOKEN={{ mimir.alertmanager.matrix.matrix_token }} + - MATRIX_USER=@alertmanager:{{ matrix.baseurl }} + - MENTION_ROOM=1 + networks: + - alertmanager + - pantalaimon + +volumes: + data: + alertmanager_data: +networks: + pantalaimon: + external: true + backend: + internal: true + alertmanager: diff --git a/ansible/plays/services/prometheus/mimir.yml b/ansible/plays/services/mimir/mimir.yml similarity index 75% rename from ansible/plays/services/prometheus/mimir.yml rename to ansible/plays/services/mimir/mimir.yml index 9876d82..8e20f4d 100644 --- a/ansible/plays/services/prometheus/mimir.yml +++ b/ansible/plays/services/mimir/mimir.yml @@ -30,13 +30,20 @@ store_gateway: sharding_ring: replication_factor: 1 -# ruler: -# rule_path: /data/ruler -# alertmanager_url: http://127.0.0.1:8080/alertmanager -# ring: -# # Quickly detect unhealthy rulers to speed up the tutorial. -# heartbeat_period: 2s -# heartbeat_timeout: 10s +ruler: + rule_path: /tmp/ruler + alertmanager_url: http://alertmanager:9093/ + tenant_federation: + enabled: true + ring: + # Quickly detect unhealthy rulers to speed up the tutorial. + heartbeat_period: 2s + heartbeat_timeout: 10s + +ruler_storage: + backend: local + local: + directory: /data/rules # # alertmanager: # data_dir: /data/alertmanager diff --git a/ansible/plays/services/prometheus/rules/cadvisor.yaml b/ansible/plays/services/mimir/rules/cadvisor.yaml similarity index 100% rename from ansible/plays/services/prometheus/rules/cadvisor.yaml rename to ansible/plays/services/mimir/rules/cadvisor.yaml diff --git a/ansible/plays/services/prometheus/rules/node.yaml b/ansible/plays/services/mimir/rules/node.yaml similarity index 100% rename from ansible/plays/services/prometheus/rules/node.yaml rename to ansible/plays/services/mimir/rules/node.yaml diff --git a/ansible/plays/services/prometheus/rules/prometheus.yaml b/ansible/plays/services/mimir/rules/prometheus.yaml similarity index 96% rename from ansible/plays/services/prometheus/rules/prometheus.yaml rename to ansible/plays/services/mimir/rules/prometheus.yaml index 55d1b37..1a7197e 100644 --- a/ansible/plays/services/prometheus/rules/prometheus.yaml +++ b/ansible/plays/services/mimir/rules/prometheus.yaml @@ -83,14 +83,14 @@ groups: annotations: summary: Prometheus AlertManager E2E dead man switch (instance {{ $labels.instance }}) description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: PrometheusNotConnectedToAlertmanager - expr: 'prometheus_notifications_alertmanagers_discovered < 1' - for: 0m - labels: - severity: critical - annotations: - summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }}) - description: "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: PrometheusNotConnectedToAlertmanager + # expr: 'prometheus_notifications_alertmanagers_discovered < 1' + # for: 0m + # labels: + # severity: critical + # annotations: + # summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }}) + # description: "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - alert: PrometheusRuleEvaluationFailures expr: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' for: 0m diff --git a/ansible/plays/services/prometheus/.env b/ansible/plays/services/prometheus/.env deleted file mode 100644 index 2501ed7..0000000 --- a/ansible/plays/services/prometheus/.env +++ /dev/null @@ -1 +0,0 @@ -COMPOSE_PROJECT_NAME=prometheus diff --git a/ansible/plays/services/prometheus/docker-compose.yaml b/ansible/plays/services/prometheus/docker-compose.yaml deleted file mode 100644 index f3a2ae8..0000000 --- a/ansible/plays/services/prometheus/docker-compose.yaml +++ /dev/null @@ -1,224 +0,0 @@ -version: "3.4" -services: - prometheus: - image: prom/prometheus:latest - restart: unless-stopped - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--web.external-url=https://prometheus.tobiasmanske.de' - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z - - prom_data:/prometheus - - label_discovery:/label_discovery:ro - - ./rules:/rules:ro,Z - labels: - - "traefik.enable=true" - - "traefik.http.routers.prometheus.rule=Host(`prometheus.tobiasmanske.de`)" - - "traefik.http.routers.prometheus.entryPoints=websecure" - - "traefik.http.services.prometheus.loadbalancer.server.port=9090" - - "traefik.http.routers.prometheus.middlewares=oauth@file" - depends_on: - - prometheus-docker-sd - - cadvisor - - node-exporter - networks: - - backend - - alertmanager - - metrics - - prometheus-docker-sd: - image: registry.tobiasmanske.de/prometheus-docker-sd:latest - restart: unless-stopped - privileged: true - networks: - - backend - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro,Z - - label_discovery:/prometheus-docker-sd:rw - logging: # this service generates a HUGE amout of logs. - driver: "none" - - alertmanager: - image: prom/alertmanager:latest - labels: - - "traefik.enable=true" - - "traefik.http.routers.alertmanager.rule=Host(`alertmanager.tobiasmanske.de`)" - - "traefik.http.routers.alertmanager.entryPoints=websecure" - - "traefik.http.services.alertmanager.loadbalancer.server.port=9093" - - "traefik.http.routers.alertmanager.middlewares=oauth@file" - volumes: - - ./alertmanager.yml:/etc/alertmanager/config.yml:ro,Z - - alertmanager_data:/data - networks: - - alertmanager - restart: unless-stopped - command: - - '--config.file=/etc/alertmanager/config.yml' - - '--web.external-url=https://alertmanager.tobiasmanske.de' - - '--storage.path=/data' - - alertmanager-matrix: - image: jaywink/matrix-alertmanager:latest - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.alertmanager-matrix.rule=Host(`alertmanager.tobiasmanske.de`) && PathPrefix(`/matrix/`)" - - "traefik.http.routers.alertmanager-matrix.middlewares=matrix-strip" - - "traefik.http.middlewares.matrix-strip.stripprefix.prefixes=/matrix" - - "traefik.http.middlewares.matrix-strip.stripprefix.forceslash=false" - - "traefik.http.routers.alertmanager-matrix.entryPoints=websecure" - - "traefik.http.services.alertmanager-matrix.loadbalancer.server.port=3000" - environment: - - APP_PORT=3000 - - APP_ALERTMANAGER_SECRET={{ prometheus.alertmanager.matrix.alertmanager_token }} - - MATRIX_HOMESERVER_URL=http://pantalaimon:8008 - - MATRIX_ROOMS={{ prometheus.alertmanager.matrix.rooms | join('|') }} - - MATRIX_TOKEN={{ prometheus.alertmanager.matrix.matrix_token }} - - MATRIX_USER=@alertmanager:{{ matrix.baseurl }} - - MENTION_ROOM=1 - networks: - - alertmanager - - pantalaimon - - - grafana: - image: grafana/grafana:latest - restart: unless-stopped - labels: - - "traefik.enable=true" - - "traefik.http.routers.grafana.rule=Host(`grafana.tobiasmanske.de`)" - - "traefik.http.routers.grafana.entryPoints=websecure" - - "traefik.http.services.grafana.loadbalancer.server.port=3000" - networks: - - backend - - default - environment: - - "GF_SERVER_ROOT_URL=https://grafana.tobiasmanske.de" - - "GF_SECURITY_ADMIN_USER={{ grafana.admin.user }}" - - "GF_SECURITY_ADMIN_PASSWORD={{ grafana.admin.password }}" - - "GF_AUTH_GENERIC_OAUTH_NAME=Keycloak" - - "GF_AUTH_GENERIC_OAUTH_ENABLED=true" - - "GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true" - - "GF_AUTH_GENERIC_OAUTH_CLIENT_ID={{ grafana.oidc.client_id }}" - - "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET={{ grafana.oidc.client_secret }}" - - "GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile offline_access roles" - - "GF_AUTH_GENERIC_OAUTH_GROUP_ATTRIBUTE_PATH=groups" - - "GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email" - - "GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=preferred_username" - - "GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=full_name" - - "GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://{{ grafana.oidc.url }}/realms/{{ grafana.oidc.realm_name }}/protocol/openid-connect/auth" - - "GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://{{ grafana.oidc.url }}/realms/{{ grafana.oidc.realm_name }}/protocol/openid-connect/token" - - "GF_AUTH_GENERIC_OAUTH_API_URL=https://{{ grafana.oidc.url }}/realms/{{ grafana.oidc.realm_name }}/protocol/openid-connect/userinfo" - - "GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(resource_access.grafana.roles[*], 'serveradmin') && 'GrafanaAdmin' || contains(resource_access.grafana.roles[*], 'admin') && 'Admin' || contains(resource_access.grafana.roles[*], 'editor') && 'Editor' || 'Viewer'" - - "GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true" - volumes: - - grafana_data:/var/lib/grafana - - ./grafana-ds.yml:/etc/grafana/provisioning/datasources/datasource.yml:ro,Z - - ./grafana-db.yml:/etc/grafana/provisioning/dashboards/datasource.yml:ro,Z - - ./grafana-dashboards:/var/lib/grafana/dashboards:ro,Z - - node-exporter: - image: quay.io/prometheus/node-exporter:latest - container_name: host-nc-chaoswg-org-node-exporter - privileged: true - labels: - - "prometheus-scrape.enabled=true" - - "prometheus-scrape.port=9100" - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - - /:/host:ro,rslave - - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro - command: - - '--path.rootfs=/host' - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.ignored-mount-points' - - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" - - '--collector.systemd' - networks: - - metrics - restart: unless-stopped - - cadvisor: - image: gcr.io/cadvisor/cadvisor:latest - privileged: true - labels: - - "prometheus-scrape.enabled=true" - - "prometheus-scrape.port=8080" - command: - - "-docker_only=true" - - "-housekeeping_interval=10s" - volumes: - - /:/rootfs:ro - - /var/run:/var/run:rw - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - networks: - - metrics - restart: unless-stopped - - loki: - image: grafana/loki:latest - restart: unless-stopped - command: -config.file=/etc/loki/loki.yaml - volumes: - - ./loki.yml:/etc/loki/loki.yaml:ro,Z - - loki_data:/loki - labels: - - "prometheus-scrape.enabled=true" - - "prometheus-scrape.port=3100" - networks: - - backend - - promtail: - image: grafana/promtail:latest - security_opt: - - label:disable - restart: unless-stopped - volumes: - - ./promtail.yml:/etc/promtail/config.yml:ro - - /var/log:/var/log:ro - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock - command: -config.file=/etc/promtail/config.yml - labels: - - "prometheus-scrape.enabled=true" - - "prometheus-scrape.port=8080" - networks: - - backend - - metrics - - mimir: - image: grafana/mimir:latest - restart: unless-stopped - volumes: - - mimir_data:/mimir - - ./mimir.yml:/etc/mimir-config/mimir.yaml:ro,Z - entrypoint: - - /bin/mimir - - -config.file=/etc/mimir-config/mimir.yaml - - -validation.max-label-names-per-series=60 - labels: - - "prometheus-scrape.enabled=true" - - "prometheus-scrape.port=8080" - networks: - - backend - - metrics - -volumes: - prom_data: - grafana_data: - loki_data: - label_discovery: - alertmanager_data: - mimir_data: -networks: - pantalaimon: - external: true - backend: - internal: true - alertmanager: - metrics: - external: true diff --git a/ansible/plays/services/prometheus/grafana-ds.yml b/ansible/plays/services/prometheus/grafana-ds.yml deleted file mode 100644 index 2db95f0..0000000 --- a/ansible/plays/services/prometheus/grafana-ds.yml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: 1 - -datasources: -- name: Prometheus - type: prometheus - url: http://prometheus:9090 - isDefault: true - access: proxy - editable: true -- name: Mimir Netcup - type: prometheus - jsonData: - httpHeaderName1: "X-Scope-OrgID" - secureJsonData: - httpHeaderValue1: "host-nc-chaoswg-org" - url: http://mimir:8080/prometheus - isDefault: false - access: proxy - editable: true -- name: Loki - type: loki - access: proxy - orgId: 1 - url: http://loki:3100 - basicAuth: false - isDefault: false - version: 1 - editable: true diff --git a/ansible/plays/services/prometheus/prometheus.yml b/ansible/plays/services/prometheus/prometheus.yml deleted file mode 100644 index 54a59e2..0000000 --- a/ansible/plays/services/prometheus/prometheus.yml +++ /dev/null @@ -1,58 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - scheme: http - static_configs: - - targets: [ 'alertmanager:9093' ] - - static_configs: - - targets: [] - scheme: http - timeout: 10s - api_version: v1 -rule_files: - - "/rules/*.yaml" -scrape_configs: -- job_name: prometheus - honor_timestamps: true - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - localhost:9090 -- job_name: 'service_discovery' - metric_relabel_configs: - - source_labels: - - "container_name" - target_label: "instance" - action: replace - file_sd_configs: - - files: - - /label_discovery/docker-targets.json -- job_name: minio-job - bearer_token: "{{ prometheus.scrape.s3.bearer_token }}" - metrics_path: /minio/v2/metrics/cluster - scheme: https - static_configs: - - targets: [s3.tobiasmanske.de] -- job_name: drone-job - bearer_token: "{{ prometheus.scrape.drone.bearer_token }}" - scheme: https - static_configs: - - targets: [drone.tobiasmanske.de] -- job_name: 'uptime-kuma-job' - scrape_interval: 30s - scheme: https - static_configs: - - targets: [status.tobiasmanske.de] - basic_auth: - username: "{{ prometheus.scrape.kuma.user }}" - password: "{{ prometheus.scrape.kuma.password }}" -remote_write: - - url: http://mimir:8080/api/v1/push - headers: - X-Scope-OrgID: host-nc-chaoswg-org diff --git a/ansible/plays/vps.yaml b/ansible/plays/vps.yaml index 2c77571..db57499 100644 --- a/ansible/plays/vps.yaml +++ b/ansible/plays/vps.yaml @@ -22,7 +22,9 @@ - {role: compose_project, service: traefik, with_fa: true} - {role: compose_project, service: registry} - {role: compose_project, service: pantalaimon} - - {role: compose_project, service: prometheus} + - {role: compose_project, service: mimir} + - {role: compose_project, service: grafana} + - {role: compose_project, service: loki} - {role: compose_project, service: minio} - {role: compose_project, service: repo_proxy} - {role: compose_project, service: gitea}