Prometheus: add Rules

coreos-config/host_vars/host.nc.chaoswg.org/vault.yaml

@@ -1,394 +1,398 @@
 $ANSIBLE_VAULT;1.2;AES256;secrets
-64663263656130636130653934313831343034363565666362356461306531643033396566656364
-3035373431353637313964333461356366626561383663310a613033343936356434313262616439
-38333435343662346133653737353832316336383866613138633935313936666366613032643136
-3066633338663930610a646464313133356364346463373562663864613863623138613664353263
-61303266643837396565386634383265383566613939393563383933353635626132396532333163
-33363831363232643533396636323633313037336562646536646131373332386465663130663663
-65653462373535653035303663306138636662303934333138333830666636323565326564336661
-65383863343761336663363336363066356134666330363231336439383935363034643039313931
-32636566623432373337663238666632353833336536366362393037303438613963336637396632
-61623564363564333064363537396161626135363665383434376636333964303335323034333162
-64616663316430653831663337316138633033643333343264313730363530316238316639323762
-32366535666431386132333762356465626436373131616435303538656434623637333630613530
-62613261396537353564333136326130643166333433393534633538636332343466633934643638
-30396137633138303265623139356532663635343230396662366337613064393763323831356266
-62343962643065326266363032323739363864353035386330636139343938653566303031346134
-30306664653461616534303932386131303863626436393333656139623562636332343564623563
-35376635333938303236646537366163396438636438363266643538663862313339633637363662
-37393135313461343465643735373731636134653366353132313161323535333662653064323163
-39373031623638646365613138333137313735666365353666303434323032303335656531376361
-66353264633164653232313736336130326233326531613535316462383033353332336135386433
-63396562303263633263366465376262666131393331366637323737333263643061633738616561
-62333134666332356437303734616432396633366566666133303661626261306537383736383332
-61386164613130336163663163333035356539613734326639636132323366616265623636343165
-32353535373835363836623838303639316365303334626564626562643166613431303635613338
-64663933663330656335323362333165313137666264623864623338636333323635393962373836
-34363066656464333236303439633738363364663438613039663764326338333832623338383333
-39373461323531643161353939643061383761383235393837663436363033616262306261616134
-35373466336130323939623133323035303333653630313163306537633261313263616436633766
-37613664663135633566663766353633373335366339613332326262633563343034613939356566
-39656335343936396336663235303465373666336632626464623135343638663839326566313833
-33316239363132613732653031326162383538386333343336383234623631666662386437303333
-66336137663363373937333632623730316562343531333665393864356663333034663461663361
-31376165613932663730306631616234346131316438623537363338343763313437653566663535
-32383865396336353865363031373164363032303461353738666138663834613832653965303737
-38326535343562346238303437626163643230356436663461373938386466336263373536396666
-38613061363966336331633963666131623831383735616265613533333365343335623738626132
-38326465323832376633383436613464343464343437653933343638613462623135396231663266
-30333161303464333532353333633136323764376634663432363038646361653131363533393232
-61653633633932646338306539633339313930393865356166396363366562623566643465386162
-64656237303633313439643965653739356234643536373139653637663464353962323963643831
-35656565623233663865633530663736333661643136373465363635353934323665336562373730
-63356230626264636632313232376237376566323337666438356534643934386539306465366665
-33363237336133376165363962363363636562663364643439373566663730333535313332306638
-66333037306331383532646635663339666238336534653365373334623639373962626137636538
-39343766626138323332326234386437653035626135396565303138383836333138323164383839
-39306362633065373739393135323565383632313434396562303030326632656130386135396463
-62653830663563343565326336623337633834326433623532626533323632363832666332346635
-31323138323561303665633732346165323830663736663933623536613839343033623837633835
-34316262666564656438306664323062316130326630666636623739316332373331633663626461
-62303761363535616663373238346531383638643363363666366233313664313438313032353438
-37383261616664333836663634363738323035656431386564373837323266333264346661353835
-31303639393466346336333834396639646237646262346335653038353561376538326364396164
-33316330396535313331336634363736393866393862663664646234636330623436366533616563
-38663162646639616135356333333461623932396362373733626131303638343437646462326339
-39616431646564613733646331313137623762313662303763613833353138373133633862663937
-38333064636635663939343135393966386365336366313534623236323934313764663961623734
-31303665623536353539333337346535396565653031653133613936643264653231323031333239
-63333837356334663062383331353166326639636265616662626531306330666330376638663764
-66353762383833616530313635643161306237623136616539346635653538343562376638663530
-65333264313762626365663564333137303131616232383361643163386261663232616131646530
-36366562666234666437666466343036323535626530396135666138636562303962313131613233
-38356132356239353862653338663438323037363339343137643831333731646131376439326463
-65626434353563353931323965336663393666653165363039626239353766363439313634623165
-62616338356565646234666436353537303639653863663439626561643264656432623534623534
-62623732613332656364336535616663376165323762633838366431313734623263643932323336
-38623564323731396232626362353162636139396134643433633361653230353137373133336436
-36616334333834663432316539633565343034353565396230643265323932363032376131663631
-37613464666666306532313230613363623864326636353538363166653230633135653930383139
-38356466306230363737396635306232353161383130386466666235323262393163623239373764
-66633331383731623330386631356165346161396531616233616338313034613039396366313035
-39313635633236613333353838373030353532633432643766373031393438643838306336656231
-65386462653534663733313137396665616463343366636134373531653938346134336663663336
-30303533663863653762646635636239393662373035386132643566643430666334373233656435
-32663132313530363837663161633765663239393733343034356166616362393963323362373637
-66313061346530643561356130626435393939373637323230353138376562653839643337666232
-30663733313030316238393465383965303264376433646231393066323931336637373532373731
-38616330353733313533626639393861346531363437343561363062346331306433356163663064
-65653232633435336535636136316561303865386533326231326436343165613837366138323832
-36653534653138643031306665326235316231346134333532333333303162393037633835646461
-61616130306536663732386561396137666639366437326334323931373832373935353266613766
-38643663376138613236376331323164313666653364633235656563393937383933653765653335
-63316365373166373562663162346435313330666635626331363966623865366563316334363638
-31656564653434613034373837633635313135613935393164326539363633643764343531623739
-39343034303335353434363835663432633731393532333835363665366630643430363530353730
-31396332643062616366643839303135616436393030653432316536373962343463326666323565
-31633831616336383837306562333634373333396439633266666664386233633664633464333463
-38663532346338306138616638383438343464326362303065656338336239623335336263313566
-34353038396237343562633665346665623637326132353663616430646661306339663936343635
-61373164336135326361633335643231363464383238303764623234646463306233346565633031
-61343265306635616263616231373838653462643635626138643435626566383866656566626134
-62663136633063313939383165636538373038643937396333303431313733633436313133386530
-34313963373936343862313435626237323633623539656539336363626337383830373439373036
-66333637383163623133396162656234383166316637303431336335393965666230386264303232
-38323033383837343161633538326165386639623731633436353334306266333261663839356339
-31323235616664396130666461396130653534353933653765313664323937666437666435653436
-66613038326233653535633033353661663838316234376462636638636139643961623561313861
-38396333356364316338373666333737613265663638353964643862386266333865636337343061
-64353130386338616634663266333831653666306638396565306262636431393433363939353930
-62636339333033323863333931353763613133323530666136653333353665376239643136386131
-31623531666433663231623731313464633866383934646361366137623863653833303332613531
-37346131376664376134303165633665623938363532393337353036343865353061356436656434
-32393661326165336633666563333032623365353831306134653330353136346262376364333166
-32306263343363336439616638663533366139373561643064373835353339653637303535306464
-35316336656263353637303864363239376333376134613866616433626236623737333265366230
-34666537626238346663646339333831636637303536386561303766363362633730303961663634
-30333536323634626537386163323438343734323762643531353030386463613561303233613162
-30636361643439373839363032656233363238356664326537626432326230306536356436323966
-37306338623833633936383936613366626633613066316663323933336136616131613039656639
-31386130656461643765313332366565356562383766353730366663303736626133376530366130
-64636331326439633135616666326637636438333466366637303136613431646133346633383964
-39363630633262653332386436636135386165356261613962333562616362663839643739346663
-33333164616234666437386636346138613061303565353463653138323534313766653538663231
-61346632303362643864356666346165633532643563323836633039663461633832386335323765
-35313939653466656666656337333730393938646635356237653034313061333338386632396434
-37636161623364333733643864383864323334353837303338653634373430663464323031343737
-36313933386431623461356135646339343034323533626139646530393737363561633931376131
-34323735666431366433353734373362663039616231366437626133636563353161393865303236
-31396439626430383231363938303431373061383237616531336233636335346632396364303663
-63396238663364323332616632366533346564326138323931306464633933353366633562346161
-62633432646537623532373431613832623865646234353239626635346133666436306531366665
-35383336656637663833613561333134666531353338643136643437303937653066663562376531
-31376637333861663735333334326133336530396565613138613261333166656232343438623730
-64663235616338623633656638393265366536353330663936613963623731343462633138663666
-65623137626566323636363838666336326536316331643035613137306537656433376531643964
-63386631613031316331306130316362383133613934656332306161313764313435393964656561
-39353537653861313466666339383930616237306562303837323461333933656164616436373437
-32393434346237623535333534356561643862323730626564613530633961653838666437376163
-37306165303765613766333336633736386532333931656262643561303964623630636364633932
-37666639333035623233393537656234393636653239636165306264363831393966656665333335
-39666364323039343165656532653965663233366336306439613066323837373430356238636664
-66306339386539373464636235353861663662386461323861393761376532613138373137353538
-32663238383032373432343266636165393732343066383761306434346564343136356262363465
-34663136623464656563663230303532653065613031653963363730373830643366326263313239
-33366530343936343665636135356336396434346330633230633938343833363335373935373233
-34383664623266616666303738663564656135316137333561646264383537346639386538623665
-32636438636137313434626464346239373261393630353930366565393735333365313162663436
-66663237623233653935633564666662666163373339636663326434333262643665393435343636
-33366266376330633333353232653030363561303662373030616233323263343336353662363963
-33373237653539396164313161353563656632386662623961323036653564613664613238343736
-31373539353065646333636362646631313466303565646638386634623038653764623239363537
-38323534653161386435373530666363323833373463373862663731376165636339646364636130
-37366532396332306131393962353534326337303038323335393537623033653531633366363765
-65343831383437316232393232356539303664393931343736306436623466303065616165313639
-38363261636538613731306533313164353539356330363432386630656536373032376136363630
-32386565336334653732373138326564393037373135666165323734356464633836343165326337
-63353933656166353136666263356633656264663064316635326636363333386530643732326134
-61646263626165313430353433383437393436316332643732306634623665663530653434653836
-35653435643662383033333431393834623164303461373631393166633530333038326365663963
-64366662633932316566346362646434333030373166643765626464376230393233316237363036
-66386237383035613932616463663033616638366238623830353461653664663132313134666261
-35623362633662353233363531336230383465353834376661336163643334323831323838346336
-63323532656239303730366236363930626361663333383265396338323230626139346339356262
-35336332643830386632303563303731653439386631386132343463656665646434613135656535
-65623763373164386637643737383437323837623665613563396332303061373364376434363830
-66393432636337383437353237306438373633643165633563373162613363633432636563653432
-35613264623431306131646464633163363530313962626134663238623837376633333538306136
-36663764326261303933646362333165333939366263663638643261613534366235633431353662
-32303865336435313463303964643732363730646436333137323632636636656634333735373630
-61333836616136616135323435386666373933333865353230623063306466653830346134623632
-61396238663263333136313963303838343435323466323537383764663166653861303339343738
-35643135363463616466626435386630386334363765663662373763623764353334333136353233
-39303833656638306162313365653365323836373461383431386131373461663037656138653932
-33383632306231663466643933336132313562396132343331303934353839663537643462613632
-38323937643539386136353034353532326433316334323134643832383935373332393665643339
-35396536336635373138666533343661623663623039383732393938633432636561643464373561
-63343761326537646331653864353738643439656234393132303532346634353462336436366136
-38373464663663653366383932623462303666663564376134363530633136316637313666333732
-63396164353937373764326161643236373330363563653061386661306633623862666631393563
-63323165326431303035653065393039623764616233323030333664353261663937626461303033
-63323664613236666538326332303964376266323237376332616139336239373035623330623739
-61396336316464343964393161333664303130366264356462316438326536643639663932663666
-61666438646132303739333130626561366363386163366239376463616631336630613438646364
-39653939353961646661336161336666623433366163396132323035663735333731616331373535
-64643638383765663766383836303937333735343139663631313836336465356438663962616432
-64356364373564383131656562666663396162313537633363316133336163663730333063626537
-38643563376333633561313137313039656536636339303735396562393866616235646137643934
-61623836356231393437623434303437306637633636353037393734336134633061373533376461
-66393830316566343031393937663939383737636536663362313534353664373530326163613662
-66313165346262633737613737326639396130343335336534373030633532373561663630303338
-36643766613632376662623238333530613031303535373633316134303463643137656466386633
-34363161623934626430616665663465633832663261303265643034313466323065306362316533
-35616131353934666330373266393035653830366364633966393736336235346531383365343638
-66643239306237353638323532313066663033363263396365623163366435306561336364333733
-38313737653638366261373734353361396530653731313231383765663137353735363734343164
-38323935303636643562623537313062626665343464636135326162393631353237623631323562
-34623661616639636134616266363531333438613535393331316633643765366138376335666563
-61343733663139353139623635343336346537653737646331336139623065346537663138353935
-61653631613139303463653861383237346139636237643565653230366232363665633363363737
-30666565326361626237646432333133343962313663373734643833666139303434353364646432
-61396337663866333430623235346636313236363066623431303732633336373064643935376663
-66383734646136376331633365373865613266613932346630383736303130626231656438643965
-35353633346234333364386466343464343034376636393230336166616161396538303466346365
-35616462353633636232383465656237306439313064396233376462626332366463323838653238
-64313164316630316662383535623336653933663633663030323336613537336135316534383432
-63366261623864396463616361333462313563356264376161353739353462613435373532616332
-34316465326137613933366135653863393662356361376330313534663939326161623062366361
-64353265663430343138326462356436346263326261653366633864653231626366356264353635
-30366261613633313163373136313739666532316430313262616465633531346164663636306563
-30363062393530633937663234346163326533646563336236663761373331636539616235613735
-38636165613236336537393135363131646438376361316562383131666137363638393038613432
-66313036613738366162313963633638313337613363326331646533663334386233623732313738
-31376633623037363832376233336661613333336237313730633663306439636638383863383561
-30643734396661653933373638323335323831336164663466343830336363396236363462313966
-30333663656338646139303837396138313536636366306639373732306533643531383733343462
-33353363356436623735313362353361393833663233663031623439363936343735613966333637
-65346136326462633637643438666231633833666334346664356239316166386137623862393064
-39326236643930653430313662363432393732616433653635363033396633653432666266353762
-33663862383232363539663233636366646530373866643135386435623563383931663463366361
-64386364303066616162623930323832376135643031333631393333383836383362643533346236
-33346436646435633961373133353863636433336536363863393432366135613164623138333266
-37396233376130393135383737613632306462353932356332376139363161636564343163623862
-65313739396636323633613339373266623439333739383230373131666236323565373230396463
-32336137303462326664316661306430393861643861623637316234366338643966336137313236
-63323936363038613763373638333938353765303365656535613865343162306165313934353735
-61313637366133623366366639666564396531633366356635336139663234613736366638386239
-66363032303631313164653830306234653536646136666362663436646238343939333965666438
-39633863323532303039373037396634393136356437346634613861336436646662356530363639
-36383532656239616236653331313863646236363037326162623833666538306439343736636562
-36666230393435636339343535623635663234356237623635663663363230383232323634326232
-33633531306462346563646665643463346139643433633235323338613236346236313462343062
-32663931333337346233346633353534373530636365306533396637313830336166313165383239
-63626331393265326165343931663733646139636131376639646562343235663037363861623737
-39333334363366333733313038386563393535343032353061373930653639386431636132353935
-66636431343730363262396262336230386265336261303837613837303730633663353931383137
-64356463376461356563623763323961376332623665383238653436653834316630666133616361
-62353962613033653835393238383734666465616164343539363030313232393037333732623334
-36643762376364373361656437376336383031343965393461663563303965393938646463333236
-31363038633266393330646264363937326337306337343266346364616238343037393063623130
-32383463303935323337313363666433383030643762346165626264663165316437616166306536
-30626532343137333334333333623837636139313861613931373935383461383466396336363439
-38356233373234386234643362353461643032343164663336393864643662306134326235326139
-31643230623266386663343430326565333539393361356336663832643631623962353464386133
-38306431613132353465303432616638326236393166343138393837386561643633346364666538
-36376463396366623432633635633563643365373336373538646335383865373665326130646564
-37353332653431656135343862636531323639373833316638346564653036613035323766366432
-63643733666330303437653133653364303833313737633234313835666635643763373466336637
-33383764623937663034323866306666386262633563303136353335643939653364326166336465
-39363762613865373935333236363632336636623062623736623333333331303830316439653861
-64343434386437363566306535643163623438663235376635633665336531633738613132653561
-36376137653561346335346637376331343838333038663035663065666661303130363737383961
-35353735333266346533343838383235343439656335313662356665373361356263316161333938
-66313538313466376138316462393139323562663938393032386532303463656533663734343135
-61386161666631326430626638326663643865666431336135643961323330343362653561303361
-65643563333463356535383662373361386130323539343531316531396362623666333530653338
-62303461653037663634303734636566656562316264363665656664626536313838376166623735
-66336165653739343365336663336266333065653632343131633363313735656434653336633761
-39373739386264333739613462353636343864343439376235643430663035336563333563333034
-61303030313532363232373735323361396336333130663664626564393332393761623532366664
-30333434616631613064396136333339303664326264326239366439393036353162356130613038
-37306235663036323131326634613065326263653731313132613739663533663532613631646436
-30656264323865333066313732646364636436346133333764643061366662333563303561306135
-64616164353035333430616264353662396132343064396562336538313761646666373535623561
-35343662316363613862326536646432356465633833353534366265633538613134333134343861
-33326331313734383462386363376363646438353238653031393837613561616635356364316364
-35613331343265636235343761396137633531326332656664613365663562366430373461396465
-62313562383534663638643063636630373564616431363162313461663833663430643633333334
-32386239333335626338613432643331643532353433613635316334356261626631616238373336
-38656630353163323932626632386662653831653738343935313765343463653738373233636638
-35613663376465616339363739623435613535323236653665663263313864363133373465363036
-35396363633262346635613033623361643230616362633237323133666636336264303466613462
-62626265373737633866636531646362616637613161643832613866356561393336623934393035
-36356163353362656239643338383963653664303862636239656135303063626132626563363233
-34336533346530663630613539373964666162643666613137333331343232626238326263643464
-61633930346137313165343834386663666565373039343161646636356135326337343533373032
-61663164393033303065666230653035653337396162636438666563316265663631393566386338
-33323361393330623634653664393938396162663333343334616232613237303032313963303331
-31396436303535393164613439306330393637396465653030396230313431383835636530396233
-32336161626438346633393835373263613434306361613866373762383132366230303031366135
-30613934633233343237666131383361616635613334646334323236303261373834663733383365
-63333266326663333832353636346630396265356166383731373363343632656462383535386363
-65356333646130323030373531363530643835653933333464363032663536613666396235363866
-62356365616332353335616435373162643337616634623436353762373364313234643961326434
-64363461316364643262633264333562626564373166376631373230626364333533366262303830
-31336433363337303163366564333433653464653662336536396666333536646637626562393566
-66633931353937363462636464623661353965333230306363306130393237666135363963306538
-66616636396132626464336162646239613432353863653039303962316361316133663134323033
-62633039356136386230363538613962366665616432663036336334376366353266643461353435
-66306232323863306463643537643939633261633337636231373761613761383937616434373663
-33346232303630376137383139343136613931343632643737383032393733336265646134333838
-36353561616631623964343163623661373635313539326564663631616431643136613638646633
-66333339363638333266613836633965653939666635383731363933333331636230373039316338
-30383734366230326234616266386266306138666165396639343938313732316539383766653165
-64653637343935363236303439623130346662336539393462656563646337633561636464376335
-31343138303461376563343234643136363739646566316363666531633665633963303937333966
-38393430343365306135626630353161346233343464343430626463323637666466636166356531
-38396364663262336361646565636435366166343536396630363738633039343035313337643132
-61613636333264663461316366646434353364623165333665326631323434343036613764613465
-31623239666435626165333661616462623735346661643466353264393463323630346232353135
-63333931343566623837643238313834623162353961613335643936626532636438356235346636
-35353330316533326161616565616534313530363132323530356462346262393433626533353431
-39373061396266353233316630623030653765663231373139303965383736356432393532666330
-39363437623533373439313133303239336263343533336533373232333235333739336136393438
-61316530316337386539353966663766396233323531653233363939326661643430663863383762
-39666531663765383739653438373030656135383237393939343633646563353135316661353064
-31613532306339303263323963363762363937643039313632353833353536633162383830613435
-35653637383661666463356363366334623331363831363835396266336239633936643631336131
-35326261353064383261656230396136313838386564346139333031623833396233346431653830
-64343030626333303633373062346432653332393862653632306632393361316536663239646431
-30353033663931633638383466393961343330383536376533346337346166316130633632636566
-61336437376237333030343138303030363734333938393436363737346664373662383937353538
-37623363313139383463663662313137656639323137346438326665303635633764633935363965
-32636233366536306636623835306238383137373434656564333635353632376131633231613666
-62613337623964633134663638313634663463623866376236653664613738383065396562316535
-63613036316465663464616562333865336566343632646332336430646563383232326130623231
-66306637333535633261316361653737346631373539353034386339383066313636363462386535
-39383465306266396634643062376166323165663363613633643731613133336131346435303462
-61393734306139643637303234373363613738313166633435373263353165646461396464616131
-35363362353064393430383066386135316361346430656236333430663536313832373064353461
-33313961396465333630616563623732623963323165613032616138333432343066373265396663
-65646336346365343362353432303339333964376639346666303038623333323837663931363330
-63396633303763323730383231653762323131656432346434626133643962616438346565343632
-63663363623365643830303937393834633464386439303935383337333863653835633530616335
-31353466373535333834396536613564373264376531626336653066653932333062323366383131
-33666435343862626433666639303333383632636236663636353833376266356361323861393938
-37373466336661383333643065383263626337306433633239313030616131643635356532653437
-37386633663036613165343231623635323038373763336532663330313264636263373565373434
-31393532303865663530303031613765346436346661323134373938303961373839393266333636
-36626365636134383662326235613565373231316661366139346538396334376561376566323662
-63313631626466653864356230623237376633333164363833353932373935326665383339313338
-37613335393039653635626433336635383464646362623963623039323262346138336331643038
-39623536343063303937666137333163356234333262396132303566366336303435323835316438
-36656538386336353162613066636666653165333433303034626230636466383463623837356534
-37666461663433366661623434306466373935366639363633376535393037653263386339333463
-36363039336163643038616561363937336263326464646532656338633861663830633535633837
-61653038346338663937666633303636616333666264356364663635653330643638653638613836
-62613161396537666633653533313639616639376162393230396136376134656231363436343439
-61323137666238336238656635666136656361623338386331313361303131343362353335333063
-64326236623639666465316362303538643139653637383136343764653762663161353665646637
-36323531666339663639653365353834376666316533393239316437333466343932396531383531
-39336665353563613539336339623839393230346630383336623738343565336663633032626538
-30333130383562343662323165653530306632386132393537333761306132356461626534626233
-39326366373566363466393462646166643965616636666634353466376362626437333766633530
-63336162613863613432653837383261613762366564643737623564303230626339386665366561
-33346266383563313466366663636164383764626661666363316436636463303136303062326139
-38356433616437663437316130653062373932303561663233616139363961356364333338623635
-33396434646263653561306539303462313738393232363036323662363630653135346462396361
-31393565666536653963333933343265353634623031666537356635613466306133373136646431
-30383366353536313436633838316239346434376266376333663530626263363164323565346232
-32373261663663333064626264653333653131663564383739393963613264333835336366306539
-33383264653335386134346538366231653866633463653566393766326230343637353832373765
-35366466643333666437663737363763396530636638613464666137346536383565353961333330
-63306365653035636136376338626535623262366564336662346336393365623430616630313133
-32383266613137643236323934313261316365356533633162663730306264313866326536306636
-62323237346166666235393365613539616135386133306465303033323561386164303530373930
-32366332663366306335353032383236663561633862303630616135346337306266326139306337
-34363436366431346432333637346534663137323935303734663435393235623938653136323134
-33353338393032643261666136363964383662626639396361313265623633623333666566326633
-61356265643739663065623238663165333638363365306238313235373733626632643937373962
-65396131336162316436646437656437306134616464653166653965313635376164353166613333
-37643265646265386461386365626636363236353834373335393865646639383833346330663562
-33653138626365333236353461313361656166306165643162393431616564303339313262643765
-65643036326663313835303063393836396136303831633535643366626137633731616634363433
-34373930636432313832663736646434636363303433616139353237376431616432663766643765
-36666130393363623032323664333764356431666132336431306337376231663661323835386638
-32656362383336393238666338623362316663353830623036303335386338363831623533333364
-38363263326133346666313133613764313765646464633938616366383936363365356633666230
-39343233623139363866663239636139363439363661373437626264386233366239383537383034
-31303831373334303835333238303430353762316434316630363436343737306363376532643235
-34643931626466393339633733353037393634323363663339656564643635313566346561353863
-38616230383431666662643666623664666630653831663337313331386532646230643265316433
-61646461646235303836333034626235356564626630656463396161363433623864363232613461
-61323339623762316161346433616130393537656336633333653763656461353765393764326164
-62376632336361393861623230616133323736663864383066643762346235376232623836383137
-66363364643035356131633138356661643833363765313030613533656136333562363662636137
-61303233653237636366346432323166383734313461363031303839623035623033313236643831
-36346230626331383733343762383466386534306131396463653666376635306534373034636339
-39336366643363393236633065356333346534306361303838653235666536383230336235646438
-30333036633938623762316234663366626630393539626431666562646630623938313437666133
-31383666626138646361396439626332613838396664666262643536663063613066303363623830
-34653566366562303634643030336564646631356130653766313463626131633433643761626239
-35356432616633336331333938373764366436343562363766643330346437626430383437616165
-62333162313938363138346466363939393261613634303263303862623262633763663965333030
-65613564363135303331623734313738393263633164633737363334613566623364626536663233
-36333936636237376234303362306439643437636235663032646461646337646135633765346533
-30386563316562336565636463636232316565623836636537623337373633613030373938306465
-33376635373330623266333437343962613235656332303964303337323966613266366165313731
-37323063653839623734663338373732373730633532656263383931373333346337346433653533
-32343333626438313735663364363136363331636161386666363831313437643939316335313937
-62623164663562646137396463346133323034383064633335623535363464633561643931306639
-33613739333035343431313061616362343432646435366439336264386333383865633535623663
-33343139623435336435343362396533303637623035343634343461316631376531626234373835
-62663066323461383165346664646336393733376262386431313430313364356533316436653933
-33376232386562663134363036623433393763633361636532653233623135353563653662333830
-38306435336463643837326638396663636462333761663838613564356532636534343835343465
-35656464373339303666393230326333626439373234313338343034633937626166326264303535
-31636439393836316332643264353339396533626237653537653663363264613764633135326263
-32633436336532656336346430666331343034336264363031666333376266643737343734643338
-61333337646361636366323937396664376237653932633531623430343138396432353966353330
-31366435643063393035353939643962306234333164613764393363663163396138363633613636
-36363963643531343431653465393763386638393830643739656532323036636665653564303264
-30313832346537336133346462623365613137303661653739333537633230306630343433623539
-30626662626137323536306338303339323736303832363734306565663232366538656666366331
-39303162303637363562646638356333386666333162306338393037373664343362
+62363334313439336637633463653539323063623034616132313265313765656533633632333830
+3130343733633334623837323339316536336666316563330a346664623339643761616230323337
+61386339303733363963396437656533633537396439336264633466343866303531363439643033
+6664656239343233370a306237656663373063636430313734326239326431636336666662613334
+30353661616161333565323665636266396534613634616631333639323466336232333739633065
+32646265393834636535653939393339626332623265313261613936393663326239393561303365
+62306335613338326564313565323233373862383862376265323637323561366164373237333132
+36646563336265663865396130383364373035323066626562643738306161353663633232353163
+31633237346333363337643663623330326236383137636533363131343563653063613931643166
+38356161313139396336386632623739346433396336353264626634363264623537323236333439
+33313934663134356663666161656539653739636432356264646631333466656133313635633264
+62653761363663643535363436323435633739663335303939303564303663623666666162333838
+61643532313830626631646430613230353830333935376134383632613662383931313939373036
+38646139643162663362323335323863353538373038383635643330626638323030316534393638
+62346366303436633966613861306466336130633766343630663336633235653762646533343365
+39623261653931323630613266326366663563626230353436616234366266353265623938663762
+39336561623766653434393265306338373763626334346237356431653136636237663439373663
+38393466623630333631346338303231343338343735353235643039353664646261343439313236
+64616136353738393933363761333438393838303331663732666631643730633637306431373261
+63333133626132376562383830326530366637616330616164326265326136313438623233313633
+64616234643238303232666238363966393631623864613837333536386133373336653331643665
+64636266326436653739376166363738663731656538366136303236643161383564333164303863
+38363434356433386132326263336131663663666266363962323665663739336136666664376132
+31653536323263333139613265383134616632366630336562303130643261613464393362323163
+65613637306333303162303636633762306464343330623238343336313433613633646630613034
+65303236663835373762663639633330386361626530323535333834306162363936323230376331
+37336665306161626665386135366439633966373737346339316166396561383333313436656334
+30633138313237313237623963386433366662366532303830386661626435373931346563643834
+30343365393430346139373462626230666338343335383262393137363563353033323663633935
+65363337316439356565353439326263646331393461643866373530623933376437646238663362
+36386430313739323738616162356564323839663664313865366238396638363162633632633065
+64316132663733353962633965363239306635343333666330366536623132376337376638626338
+32653933346365653136333734393161306661643739616237656631666436373231623537616163
+31323337656165333033363632313531363836386564613231323935383331353734373762366232
+38346438626332616463656433623339623237393261643930376433363764613136393362333638
+33346162393935623061316430356439323230313635306232393566313630376337333163316462
+36373733376537386162313030663832626261363764326335656437613438303534366534633937
+30353631363264313834633763643633383962346131303565343530383864626361636433633766
+36363963396633643266666538623037306363623334643765623263616436336634623035373837
+36363364383232333864373334376339343331656131613632633534363739343935636536383064
+38613136653762363830323737623265346339376131633633306430636361373934616339653363
+30396537326362623537643633363333633834383363313532396432333733313362393333323537
+37336366666130666437393966383963323363623666663333303364623733386238613133643939
+37313165323234363232313037396633376366363230333039633035323237346236363361663034
+33366163646663343666303236303239336463666364353939393233333036646531626136383732
+38313632376532626664373335383639633566613462616135343231396131623265346162623961
+38383761356335326132343336613531393765303733303931626237306463376664356563326164
+62653536353636376537653336663066313465346536396466393963363532323837373130353938
+63386132383530336131646164663330633331313535363333356438356430326132613731323262
+62326532313532313161613464316162333533646661363634333133636231636433376462623034
+62393139383735373761343731666666633732666365396162616562633164366138663362363837
+39643238316436306431623136663831653137323932333136313537376531383263353837616536
+33666239626361643835633065303733643537613634303737383034663562663031333035643163
+65633034613965393231363838353461643637346236386665333538636432633630366530653233
+33653737333464303661613534653962646430346330323933636265643230616562396666386533
+38363961313237366535326665343462363262313831343233386330653739316537653938653563
+36616339303233666662666366356335626330633162383639633034313432633764393962626531
+66663366636132366465656163616331343062393062326564376637386435396363613262363966
+39323331396339343233313333373366626635383431303861343362376632353334356332303732
+38386630633663643262333338373037336438633135396165613763643532313861633464656663
+30656265373433623033643536326436353332636663363533613033346534343163633937346466
+34633064663930613138313236653761396435373065343235656166653963666363303835313435
+39333435623538626131636564653136323061396361316663346661336234626663623165353734
+65643932333334396638633639623136656539313039323239623637396537396339343366386436
+66313864336231636263323862316432653239376431346137373431353964323963623161633938
+33363164366132653966396462356133376434376137336137326338353232616166386464363666
+30303537613237373162373833663935643263373963303031633062383731663563616239663731
+30333862663737633831623237666461376230636237383330353866643766313066626637643666
+65666639316265386463643238636463616135393635393930353864643837643937353765663934
+62313366386539616531643530303939653232363832396465393933643166666663646530346136
+31623531366561383135346337313430363439323966393334613138666164646536636234333165
+32356232613164353833616636613435656137343834383938626237303530343438356665666236
+38613665643761396635303931366236353263333338383239396564656537383165376664653336
+63666436633963633461613539356662626263373039343661303363323664396666633439646338
+63643763656663626338656163363064366661326465326530613931613064393761353938366535
+61373134636231343434663738383932663533373537393564663032653465383462323935323832
+62623533303364373139326563646334653864653461613137373061323163653831643661626431
+64323336393739663863353133623263623033366131356464393631633730356461343264363239
+35343032393762343535343663656538386532316262386538333539636363633564336632306265
+31653431333337313366383964353637313631393133346262363734336634336439336436333634
+37373736353066643236656231633330656137633334313136383064356163303530353436653032
+39623730323036376662626539396237656537373463333136383061636164633163323831313130
+66343161376436393763626261313066363966323438633237343664363166383336356537626166
+65633562393238383439313662353030386637316639383765396530383663303139633835376236
+65386163633031373733633062666336636430373966363431336662663539643963363863316630
+36396130306263396631346266393763386139336164663962656333663266366231336337356338
+33653764346535366364616236616666663931303530343930663963383037326636386263373736
+39653830643137333330343633653364626162363365623436626138343163636635633734386162
+62343439363036663332646336306639623038613039383962383234356463363836383733653939
+34653231323539343863383832386535366639626662613636666132316437666539303435366635
+38333734653638333431643165313737333262396439636465303838303637383134663432636131
+32393231343863663062626664653565313135353737323539343132633933303164656665353239
+66313166386532646530373236356263306139393239346264333461653538313332353764343236
+34326335366366356339376162633764333134313337396364313263303336356265356134643139
+31313035306337623364316462643539383039363438643563303131656161333036633031306432
+33366263643936613835643366653563623239336164396136383439643734316231613463633161
+35653937666534653661343662363761653637383464343231353863396464613233666362303730
+33343564343136316236616438376533333666663266663233366433646162663935343261393339
+66373666393035313730386662373063623261366236303661396336643736343062643064613161
+64613866383035363530336435636538323266373264393036343462643430323232346432366566
+33633663386536353034316438346238353761363139323866613331343263373837363431643038
+39333132303331623530383963323539356235303139336462653336363565306230616137373163
+34633538343664323534613130356463646363323137363062323234346262666233343530663664
+63353932376336623662623865643733613939636230313432653233303336313466306263323938
+63333464616664393037646264336435373034313565663132303737653939343562393136346335
+66343063333164623833326433653330623637333863666562373263323436393361656335653464
+63346335366338323733346336643139303037633334653933303837643965383363366436346266
+61393266636632363432356566633534613765623066653039326265376535386465656339303034
+33663665336132646135313832633337636265393536653866323266666665626535636464653562
+31613935656138313735326331366438333030613539303139313263353332666639623661626534
+62626364643965326138653532393435316461376237656133663666326636613564613364616630
+33353035303933323030376630303131343733366465313262386537613264626661623038363465
+64613934386262313563386566323066373935393533623839306439373661383865313238353830
+36323565313132613463386635666235316466333966623935633035393634623964366633353737
+39316233383839323965343633373832306336303539663437373134386633306665343032313139
+66666234393061373361393539313066353034323066623438623836333364373739613465623763
+31353535636462393832376163353437653131393737353230643537663266643564643834643833
+35323862326666326237343039343335343230646466616232613930613030643832383861613761
+33643336303964623035653433626666356330306562643930626232363939303639386334393837
+33326431653961313936383663356330376663343135663463663239386137626430623037666231
+38323665336437363432303036653865636665306662666634356537636530626262306639383531
+63376636323862383265336163313839643062313636303031386262303239353431373834613632
+35366661376438383336353339386134306137373730623461353461373532396334643834356365
+35323730633866613363643037346334376233656365646662366132653935393731383534633365
+30306266643166323631343936373536306134313839383232316365363864653333643362633538
+32646636636435643764383534383530336235366165653439623537366235353830613532613632
+34393831333036323139633966303130626130616462353231393130383162363330633530613436
+65376434316235303965383966336536663464316264653436363436616565636432633935393736
+30363535623633643930393134636561633464626439636231353230343636313137316365656565
+30396231653437386232663964663736323061303235613636663735356638363330313461626264
+65653262353163313664363364393866646236336664343163656534306361653637306362343466
+65343133623235646666326562303537353833356638633137643530383935646537323064333336
+39336262323933373333666134663261636433636639376634393531366534303461383535623363
+38666339343262623134303131623531316563336565643133643030303132653136656337626563
+61333636623263393736636134633533656365633565646638393539383131363739613637373930
+38323931373166643764393939393932383233353432333135636166613832376638623435333034
+34633766666566303963376538616138663566646338396664636230323832616535306133323434
+63306336643530326539373662633366633030656138313166623661666661663339376438333863
+32616431666435356564303134363833646566653234303934336232616361643734353239653738
+61653037376532386663333034633861303561353565613463333761393436316438383366353332
+34323939353939336261313164313830323034643661613431373335666530336231356137646133
+66356536303234323731633362373664623261366566386635356364376531393530353237313033
+62313634656263663533303539313331346361323566646330356231326339316661633639373530
+66343939623132663139666139653936343830316235353661613936313966316664353738363334
+34303534666331306465303436306237643632373630633931386431376361383061326435326337
+39356164323065653835333035363437636362393434343363633263323533343030616563613332
+61366233323036346161363338393232623136366365373663613431373564316163303632313532
+38333634333036623661656139323734383233623938326536386638376136336332373564643138
+64313030356533366430616338613434346330643661336461363033643930616535616538633430
+61643436393564626333636663303465616339646161343461663461613339313630346436323863
+61313639366638643261306334306239666465663339663838346334343662343332663864356264
+32323231623964653864653331353563373639613363663463663135623939386139303663616162
+63346334633236333562663632373733656231643038333966333035356464303237613361313438
+64343861356131646538343466646631333036633334373337393838333566633034613230303262
+36376332336537663631386162316633633738346338636361316663363562653338616262623538
+36613038613266303038313830306162626231373030353764636639353030326265366530626266
+39343862393539373162363262333031366136643135396665366631343365636239376333626531
+32386261353765333638613436656132636363633466333233363063326438613365313538346239
+31343633343237323938396537626333313832353564346161623730363636343532386262343166
+65643566316465666333313533643363323233326562663733643361346233626633656537353933
+63366131316565316363373666353036636632303637316664393863613461653330316433636663
+35643530613965663964313265636335666638313465643335353632646663653435363930333534
+65663238656336383133306535313433363934303836353962373838346237396231366539306563
+37386438346336326261326165323132383537353464346461316430396463663562373636343235
+62303261383435346163653837666561613334306139393532356665386362623633656366306232
+39626131356365663262326565353232616463336363343430333031353734323735643834306237
+62333431663035376135613965393834373335356538366563666266653332633963653538336533
+65643564613438303433623538656664363966626530343365363862636166396338626134393532
+34396663623538343261383832306161653766626434643566656330626538336565313961613230
+66323837633535303438356466303031313565616365323662666265613761303866353338656432
+35636364393065393738666431623438613433363561343066336366353531383134633531616433
+33316236303430373466313831306265623139336161393730336463663564393566643435646539
+36616637323362363537643134396630393162633735336661373763323439376435333532366538
+61363334623132313665356135353432666439613363616462663431633834613230363733313737
+36366265353066643231373830363233666130656162373166343832306638373563343633326535
+62326362363836336333356164666464643033643166396230653165646161356363386132643465
+63383364376631383635326230393235313834353932636231616366633639303364303631323561
+61626165623839356235623566646438373436303636386265366661623635333061313931313865
+39636166326564303434363061343838653161373237653865386666393530616366656165323861
+34633464616261653462636234613664363430333337646263613362346464303433316166623537
+64366635633166383533376466643437653339623662343562353430353362386164363835366438
+34316234303932326164376339343437326230363830393162313439373235343437303966333036
+63646135383663653137316239386137363663646161376266333132613265303033323037386237
+39316537646336336636353739663639646636633065663834343466353233613466346561306464
+64363664626239323238653031663164303930393832323861633536656431393064393663323031
+39633434343865363433356236333366616335346235633463363232366638653364326233346430
+35623030623230313431316234663737383461343237313566336534393866373835366263306165
+65666334623533646562313633663064333734326437316562633036616431303938653431313466
+61623832303834343464323663326161653436346463393330623566376161643565333639656464
+39336461363336373438316239343863356137663166633431363763336463623962616138303532
+62346532626663333061633365613638316339363332383535373831316331616530623532336536
+63393234623466303930333039393738353431323336656564343266633239396238303636353533
+63373133303035626437663264303532313539353035663066386135363533303537313766343265
+33396330393663393564323039336432656133623130663564313765656563393462393537666133
+62636665373539636565303536383931636636313733396630356264623038376536663766323636
+34326331313236313832663537383563643833303939363939393932393736376162646430326164
+66303539646131333064626334623663353964326661333034366236353634303439653331316365
+30623966353835323836373032666238383538646565363661353236626538396236643638663534
+32653333336433316661643161333830343362373361656531653635326239343631383961616637
+65306164396330643638313465336437393730393237363962303332396630313839646266623939
+30326535653538636135393261303564323436306262613934343463353365323433656164363066
+62363636383766666333306134323465376138653730623734383761366337623433333339353038
+64376561306531336433646562303035343566383839356661633564613861306566316630303439
+35636466373237663430306231636233333761383932633361333935306430626263366635356632
+35313037633934313532353037316462313434646539336661616538353530306439633164653235
+34346238663739303537636363336261623532646232303830666361633431393062383330373735
+38326232326635343635303864636231343739353038633138663632386264323831366539343666
+61636163666235656630323463633734303838303363343036343063623866383462326537636634
+63336235303732396332316138636331656466313734376265633865623038653163613439646166
+37313936633835343230306634356161613636643134373161343834633534363737646636666636
+35353163383064326366316332636637336262616562323461373138663137313365613035316439
+66616136636261613765373066383736666535323063336436346162376361383466653137623064
+61663930353331653534306236343964643063313865643235336631666338616363393439393164
+64326439656336313166336539653435616232386331323666366337636433666535643139633535
+63646664313635313634633463363436666531393630626439313038623764343933643634613238
+64333639353931383765613831653932323364363666343335303934323330303938636133396534
+64343534376135663033656139636231356437343666353663373839373862623238303739613166
+62356233633637613564633261363532636539396664643434306330666634616362326663656166
+35643366383437626635663331303433383630303531303164303139623432396239653166323437
+39383633643034303465376231396565366636383134646166323634373766353066353636366436
+30323638303064626239653439393961396664313630383839396561356639323436643037616239
+61363830656631316537326131333162366234663931323861666231633738613162383530353736
+32633035383631653838383838353138313034653166376536663862393038636231653235366338
+63386436336635646533626666653063343838396233643138343736356136393136353937313033
+35383433353236383138653539663737613263646430623962636230613433396331303861616436
+38313964373964666563656235646664386331336332616131643338663635356436346431323736
+31643931383765653234643834343036396661326163623164393538613236363239643539623130
+36303338666238643663633332643965656665396562663962303230633166653662653835343766
+61313034336330653365356434646237653137643038373937633434313538343535376264623231
+37326636356665313436336534613433303836386234316464343331636663393338626165626436
+33396533373863323462653461336436303538343565383261323765366334633336383932303265
+63653932313835613037616633383439396161316232363831323935383662386339386462353463
+36366538323036323533363562336663383664353839303261356661376536663736383737383830
+66333233366231376334316535633735343638373964303033626466323463323938393164616633
+63373433653465366363623338663866343838306532643863646339663932393463376137636432
+65363064623933313432653731333235626539663030353936636266613164346134653133396436
+35646431326639383030653430316661663462653466316436306536376436633633313931313231
+35666636343566643338366464666638393031646131383435613135333337623663363464336331
+32323538306262373639393236623332376265356162613732363539373835653430643434623764
+37356639643961343931356466613363343737353433613735343062666362316263323837376661
+39393730343730643137353536306239333639633439353439353439323038653337303061666331
+61303766613738316161623166313664336161323732633864616436366638633137373661326564
+34336130323230663963396639323737376437616432666137663164623961643063666662656664
+36353062646431643039373465653938316239316563386532363061643565623038313762656534
+35313031393939633265323430316361373130623330353935343666656137356332396465323132
+33663432653966353636346466393161316630393536633832613533383234626261346464643361
+62306239376235306364333662363130386365616132653031613938366165653739643130383936
+63633463353564613761303035373439333032353036656134363334373465633062623163623135
+30663764306632336664336134306136653630366662383162653863393034396132393237366438
+66373632383232303632393031343163323739366537653330353538616430306261646530623565
+30353836656432393934396537356637323263373335343436303932383762623163643835343633
+34376262306333663732333964353633373332366434353637393233363539386563333534623939
+32613037616462626530306365326466303465333930666635323737646262303636616434643966
+66363838653636643933653238363337666166333363333964336235333530623061343332643761
+32653637376133663939363237616134643137656538663936333039333236316661356430663862
+64356665666163633663613836373637623737366562333637643933626438326439656534653931
+30316237353834333434643765613462333863616136313539656461666136323663303130636137
+32613334336265663666653538643330623465333639643463376665646237643234373937393865
+38656338326666363639306464383863613339343438643137623765643461333762353933376464
+64623330346334366637323934666535626131303461663233333036373939656331383736313765
+37656664376633363130636430346535633962616433636639356666316330636462663335313535
+66353664663439353131396633656433646464323539376139643264373334333336353436636638
+62653066383731363838373731656132653137343139306138626338613864313234343866356465
+37303733363939633730373961396263323462613666636430326262303365666435323735656639
+31373962333363613133336165376339363933366339383634663031646264313161323533393331
+65633665616261353938666661646130646430396464303831616232356562656666353234373662
+62396138356333663037313533656339373564353165353534303533646465646266343461643164
+38313733396631396363363635346337343831396238316131393063366361373230336331656530
+62356564383664323636366538353866326163313132323965663232663234613430653161313030
+32616336393232643262646231353732326266633361396432353538383035633662313033396636
+30363134663733383864323561386639643236656666633966623837623637383133323331336261
+65653466313639343564623231383931303735316438613230313734656436663838383336363931
+62383530363263303635636663343434383861636137633637363739313939653065303734383937
+63333264623564333161663966643632623463313766356439363764666632316130613030356133
+62353433613133353630343733316365373965393438643766303635366130313534333437306338
+34396462343536323665313735633130393634393466396236363036396665666432346335316237
+38363061343637306134333938646531346633363064333730316535306531613066313666653335
+35636264346130313762336464323833333763646662343630633562333637316233316665336361
+30333135633766303036636138656339626130353462633066333862343832303362666265383138
+32383838303133353631346234333931373163393930313938663333396234343930663430343434
+37646561623036643136656334383139336233303737666637383438353363653764393165323538
+66323566323338356439633634306130363939363039343762346665333835353463366363343166
+34656162656366376534343737656136326562393637663266356162623033393164623963306235
+33633164633861316566656337373830636433656232383462663035613162373133316363333261
+30626365376539656230656366623463626233373133626330373564313061633839386133623436
+35346339346537393262363965373862613064373361343333343138363866353661363965303363
+31633365383030653136643837306239653262313132643238656566313930663839336236316237
+36626434366265313362366431306661303339303430633235343064643064343164323630353865
+37633664616162336161653233376565646366316438303638633833663066623730326165353336
+39353530313232356163356163356131396166653331343435313464353530356539653162376663
+37633865306332303066313561313563333832376430363963343637636166323061386533663866
+33346132643365383038393964616230356539363338373665343637666465383962333662653639
+39623733346462336565353539666438356631613233616562646637323738373761663030333764
+64633230356563326535313430663863626336613264373863326539383763353062663430343833
+34343462383030333461353633333761383430393930333831303437636334363536633635383863
+35636161323862323036623932376130356438643236303361343333633363633033636539353966
+31636338346433616232343631353837623966373433333565376538303937373932646462376461
+65643764306336363736663133616432373661303237303264343862653333646236663337623662
+65363366663837663463616236616630326366613432656365663763386532663837663862636132
+35613331373236666163666134373937353531316562626135313464613366646261653266356339
+33623866633663343838626131373930636135336431353538366438363035343532376630303561
+38353232636237633438636161656138336137343531383164346264613034343763623339343837
+61633961313166303661643633356262303461626630303466316330643763333936346632313135
+62666135333163326361313035653664623066323364623966316130346463323561333533393339
+36653461323664333635343862643530363861333436313335306165373566646130393739343332
+61653564353032636237613832323961626338376237646162303638653539333163326563643163
+35306566326362353463646132373832383236616163316264386136643865646639616330656165
+32333830656333313830656162666531376637656463316336383766333666616365666138366364
+36353531613531626433643862316439306663383034346634326631323263393237643331623463
+34333630613263643864633331333638343333373933356464613331616337626136663263373538
+66653230616137396464656536373133646631396131323965326366363630343233646430656366
+30333536623661626133653932343532333236353435383430643633373933316338313764663463
+63666536386464346231393336653532353533623930396364663730353363626332626137383930
+33656363646165313761386266626565373932613939336533333036323461346331383936346639
+65653731643633303639646130313834643136383634633463373466303166353936303166333266
+33313161626436303161343139346266396161373932663732656530623762656362313037313538
+34666436643430636561323162366236363964343663663739363737343632653836633763386533
+63616238666332376161346665323236393232643837333635396639313466326534626566313639
+30353231653231303338386137646163623935313266386239643039666431323865326131623432
+33373066313261396435303331393336393332373730376239393539393030353566653965633034
+32613739636666356162323035653064323332623433393963306161353263643861613836616532
+66323061383033313131326132373730343364363764396637313738313132663535386465613839
+65383462356265656462663638623938376536313033663431393761323134666563316630393031
+38313238336132663133336338663363633863383333613731316634656134353836343765363539
+38613465376432636639623266336563633039396638636163343537376133613233633536626431
+62333663666635636431373334306332363931323931643330323838343635636333653663663863
+36353761376535653162656335393233373833653165313836633930336433663462393131326363
+36383535306534366237393764363235323662366461353235356235333033306135316635353237
+30343362646532646661383437623939363439316662643438663161356234386461376431616337
+62663630643763373563313937623764373838393535393662336162343762373833333165316336
+36393962663036393536366634326232306531316630346238613535303130383562306432346134
+31396634323134343765343230353434376663356437306231623132366130373234333438393764
+62643366306233636232333139613963323863613565623065636536616564323236656161363463
+39343164393765376530313362356238393831303239326439313838663839353832326431663637
+30666635313065343734636131663536616562343566343766336261383062646465323762353332
+65316435396230353033386131313935376233323135346630633639353163653863353933373634
+37346539363639343364316238643634616439653033383261383637326264316538356239643036
+38646434643332346561373130633132363538353935646433313333313262613363363732386130
+66353238363364323732353862303632666139343732363330373865383030343331383134356365
+31306230303237626265366436626265313662643833626534383661373638356433656234663865
+34336266633135343165656131373538613638613566396266626534313337626366343565306161
+37653764306364393632643465333033633337613965376630343934306634623433306363663939
+66396661323537333431643231306663376636613633656430616233636566636265336264386638
+35363066363465343964306134663132303461343238396232326566646263666430633538363565
+32393235313932363238383039333238333933656634613930373362646530373861326136663935
+34386263323663646538613135636432393831346265333530653531623265396164396434623136
+65343064623963323138643062366136386531313964643037626235646132346163636566373062
+62303235653066613663383635336230396237303361333933663865616139323439663935333738
+30353436633561383133613265356432633562663937376534336633353037623265663936373732
+39353366663830366138393364396632383537323361626563363834636163353833373464666366
+39343961373561613366393639343635393763643937313432363231663462363637626237373264
+37386664373432643236613237316266623765633066333462626331393635333966663065316131
+31353238633465626437313466653361633132646537363763646362303737353362386638303130
+32643431313561656539366264346131633436313637303664313636386336613932663038353632
+30333230393036663236633033363439343239336666643364383338336635313138343537383635
+62663031383266623266343462623565393262383838653039626530393864373934396533313861
+38636162303166356238303436656666356535306362303563653331383632326438336466333939
+35303233346166643466613333343165316365623264316131363765653563646134326532643461
+36343566306662663837363462316634356230373261393336623262376330393466366239323434
+30353736323262633830326339616637356636303137363030343831373139333339323530613830
+64366137396531646566373234636463636337653261336235366639343339626266396430306662
+39623835623935306661346538386239613533333362613831313561383135343638316533613536
+66316361343336306330326630363832613830363335313166626331376630636638343637303431
+38353135323165343064646664653332636436363531326362643637653966666130386430356533
+32386331303539353834613431376265326431613161623239353038336461313732653366626137
+36353462316165303163376465363333653131653138353535366334383464346630356432663966
+66653234616238356537353737626436313132333934383438636463646463383338653135313462
+34633830353430323765636439633262363232343930653336326164393665346435396565666332
+33383239343633656562376561366332326439323234656361326130396135373733643464393462
+34323032386566346565333636396435323234333830336130623233313830363837373934373033
+61643934636335633437356535623730333032343864313835646166666337326362313265633732
+36623361313934666335346537613866656535643764366663303864663263333738356630366361
+65353433663763363063386439313736613833613933326566643366313965613136303631376538
+64653432373265393634623634666233636164366336373534333039363966363233633762366361
+65303931393362323062333663323236373434626364363866636361643832343363613132386164
+31373231326235363765363635326464366264373938336533333265303436633530393632643162
+66646163373066343539636639656662656565616136333739373466313966306165353436396261
+62326532653262316234626530613362373365623731323631333035353463303466623934616362
+66316561646430366538343030386338333861626466663464386137373837316663353361646232
+36393466333839613938643066376563333263356631353233383563376462653531383730303930
+66643132373864373935626136363234326166383233373937336532643035363036626162343030
+64343864383434366164643165333132326230653964333465313763366162353536656430333761
+30313664636536363738333230636236353931653563376634326465623334386364366337336432
+31356163663936386161303734326331326636633931343430646237366661306664623931666466
+39623838393736306466353064656166363764353365336135646333373230653636613933613132
+62363332623462666139643938656533653862643234323836353265646364316431663133313863
+36356630363037666632346134393033346131616665346534303630353238616636376438316337
+33656535323062323137643635663938646238353266623635653038656434313438613533616435
+33313634303564363261333864373730616464373335663661663766373934633531373530626163
+66653062326364373631663130316333323061373338316436326436326663383863656564363038
+66316261626363336664636266343534373133663161376438613664333939386232363361643630
+66373239643234313737633337656166323533333962666633333762343435653037396364653934
+31356430333630303037303532616132663737316562336530613635613563666561373961363739
+64626266376630366561626338643436613131653736666137363762666465613465396663613030
+33386139636432623431326635336565326434346236656466653763363232366165626535626432
+31326561303561326232663833396439393439376333653162306265346234633565653639343230
+35633935646334363633633133313339393636656664316233653938333739316433

coreos-config/roles/compose_project/templates/prometheus/alertmanager.yml

@@ -7,6 +7,11 @@ route:
   group_interval: 5m
   repeat_interval: 5m
   receiver: 'email'
+  routes:
+    - receiver: 'deadmans-switch'
+      group_wait: 0s
+      matchers:
+        - alertname="PrometheusAlertmanagerE2eDeadManSwitch"
 
 receivers:
   - name: 'email'
@@ -17,5 +22,7 @@ receivers:
         auth_username: '{{ prometheus.alertmanager.smtp.username }}'
         auth_identity: '{{ prometheus.alertmanager.smtp.username }}'
         auth_password: '{{ prometheus.alertmanager.smtp.password }}'
-
-
+  - name: 'deadmans-switch'
+    webhook_configs:
+      - send_resolved: false
+        url: "{{ prometheus.alertmanager.kumaurl }}"

coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml

@@ -9,6 +9,7 @@ services:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z
       - prom_data:/prometheus
       - label_discovery:/label_discovery:ro
+      - ./rules:/rules:ro,Z
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.prometheus.rule=Host(`prometheus.tobiasmanske.de`)"

coreos-config/roles/compose_project/templates/prometheus/prometheus.yml

@@ -12,6 +12,8 @@ alerting:
     scheme: http
     timeout: 10s
     api_version: v1
+rule_files:
+  - "/rules/*.yaml"
 scrape_configs:
 - job_name: prometheus
   honor_timestamps: true

coreos-config/roles/compose_project/templates/prometheus/rules/cadvisor.yaml

@@ -0,0 +1,54 @@
+# {% raw %}
+
+groups:
+- name: GoogleCadvisor
+  rules:
+    # - alert: ContainerKilled
+    #   expr: 'time() - container_last_seen > 60'
+    #   for: 0m
+    #   labels:
+    #     severity: warning
+    #   annotations:
+    #     summary: Container killed (instance {{ $labels.instance }})
+    #     description: "A container has disappeared\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: ContainerAbsent
+    #   expr: 'absent(container_last_seen)'
+    #   for: 5m
+    #   labels:
+    #     severity: warning
+    #   annotations:
+    #     summary: Container absent (instance {{ $labels.instance }})
+    #     description: "A container is absent for 5 min\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: ContainerCpuUsage
+      expr: '(sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 80'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Container CPU usage (instance {{ $labels.instance }})
+        description: "Container CPU usage is above 80%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: ContainerMemoryUsage
+      expr: '(sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Container Memory usage (instance {{ $labels.instance }})
+        description: "Container Memory usage is above 80%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: ContainerVolumeUsage
+    #   expr: '(1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total) BY (instance))) * 100 > 80'
+    #   for: 2m
+    #   labels:
+    #     severity: warning
+    #   annotations:
+    #     summary: Container Volume usage (instance {{ $labels.instance }})
+    #     description: "Container Volume usage is above 80%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: ContainerHighThrottleRate
+      expr: 'rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Container high throttle rate (instance {{ $labels.instance }})
+        description: "Container is being throttled\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+# {% endraw %}

coreos-config/roles/compose_project/templates/prometheus/rules/node.yaml

@@ -0,0 +1,303 @@
+# {% raw %}
+
+groups:
+- name: NodeExporter
+  rules:
+    - alert: HostOutOfMemory
+      expr: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host out of memory (instance {{ $labels.instance }})
+        description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostMemoryUnderMemoryPressure
+      expr: 'rate(node_vmstat_pgmajfault[1m]) > 1000'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host memory under memory pressure (instance {{ $labels.instance }})
+        description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: HostMemoryIsUnderUtilized
+    #   expr: '100 - (rate(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20'
+    #   for: 1w
+    #   labels:
+    #     severity: info
+    #   annotations:
+    #     summary: Host Memory is under utilized (instance {{ $labels.instance }})
+    #     description: "Node memory is < 20% for 1 week. Consider reducing memory space.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualNetworkThroughputIn
+      expr: 'sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual network throughput in (instance {{ $labels.instance }})
+        description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualNetworkThroughputOut
+      expr: 'sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual network throughput out (instance {{ $labels.instance }})
+        description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualDiskReadRate
+      expr: 'sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual disk read rate (instance {{ $labels.instance }})
+        description: "Disk is probably reading too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualDiskWriteRate
+      expr: 'sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual disk write rate (instance {{ $labels.instance }})
+        description: "Disk is probably writing too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostOutOfDiskSpace
+      expr: '(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host out of disk space (instance {{ $labels.instance }})
+        description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostDiskWillFillIn24Hours
+      expr: '(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+        description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostOutOfInodes
+      expr: 'node_filesystem_files_free / node_filesystem_files * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host out of inodes (instance {{ $labels.instance }})
+        description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostInodesWillFillIn24Hours
+      expr: 'node_filesystem_files_free / node_filesystem_files * 100 < 10 and predict_linear(node_filesystem_files_free[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }})
+        description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualDiskReadLatency
+      expr: 'rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual disk read latency (instance {{ $labels.instance }})
+        description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualDiskWriteLatency
+      expr: 'rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual disk write latency (instance {{ $labels.instance }})
+        description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostHighCpuLoad
+      expr: 'sum by (instance) (avg by (mode, instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host high CPU load (instance {{ $labels.instance }})
+        description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: HostCpuIsUnderUtilized
+    #   expr: '100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20'
+    #   for: 1w
+    #   labels:
+    #     severity: info
+    #   annotations:
+    #     summary: Host CPU is under utilized (instance {{ $labels.instance }})
+    #     description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostCpuStealNoisyNeighbor
+      expr: 'avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+        description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostCpuHighIowait
+      expr: 'avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 10'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host CPU high iowait (instance {{ $labels.instance }})
+        description: "CPU iowait > 10%. A high iowait means that you are disk or network bound.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostUnusualDiskIo
+      expr: 'rate(node_disk_io_time_seconds_total[1m]) > 0.5'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host unusual disk IO (instance {{ $labels.instance }})
+        description: "Time spent in IO is too high on {{ $labels.instance }}. Check storage for issues.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: HostContextSwitching
+    #   expr: '(rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1000'
+    #   for: 0m
+    #   labels:
+    #     severity: warning
+    #   annotations:
+    #     summary: Host context switching (instance {{ $labels.instance }})
+    #     description: "Context switching is growing on node (> 1000 / s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostSwapIsFillingUp
+      expr: '(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host swap is filling up (instance {{ $labels.instance }})
+        description: "Swap is filling up (>80%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostSystemdServiceCrashed
+      expr: 'node_systemd_unit_state{state="failed"} == 1'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host systemd service crashed (instance {{ $labels.instance }})
+        description: "systemd service crashed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostPhysicalComponentTooHot
+      expr: 'node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor) node_hwmon_sensor_label{label!="tctl"} > 75'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host physical component too hot (instance {{ $labels.instance }})
+        description: "Physical hardware component too hot\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostNodeOvertemperatureAlarm
+      expr: 'node_hwmon_temp_crit_alarm_celsius == 1'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Host node overtemperature alarm (instance {{ $labels.instance }})
+        description: "Physical node temperature alarm triggered\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostRaidArrayGotInactive
+      expr: 'node_md_state{state="inactive"} > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Host RAID array got inactive (instance {{ $labels.instance }})
+        description: "RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostRaidDiskFailure
+      expr: 'node_md_disks{state="failed"} > 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host RAID disk failure (instance {{ $labels.instance }})
+        description: "At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostKernelVersionDeviations
+      expr: 'count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1'
+      for: 6h
+      labels:
+        severity: warning
+      annotations:
+        summary: Host kernel version deviations (instance {{ $labels.instance }})
+        description: "Different kernel versions are running\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostOomKillDetected
+      expr: 'increase(node_vmstat_oom_kill[1m]) > 0'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host OOM kill detected (instance {{ $labels.instance }})
+        description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostEdacCorrectableErrorsDetected
+      expr: 'increase(node_edac_correctable_errors_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: info
+      annotations:
+        summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
+        description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostEdacUncorrectableErrorsDetected
+      expr: 'node_edac_uncorrectable_errors_total > 0'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
+        description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostNetworkReceiveErrors
+      expr: 'rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host Network Receive Errors (instance {{ $labels.instance }})
+        description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostNetworkTransmitErrors
+      expr: 'rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host Network Transmit Errors (instance {{ $labels.instance }})
+        description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostNetworkInterfaceSaturated
+      expr: '(rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000'
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+        description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostNetworkBondDegraded
+      expr: '(node_bonding_active - node_bonding_slaves) != 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host Network Bond Degraded (instance {{ $labels.instance }})
+        description: "Bond \"{{ $labels.device }}\" degraded on \"{{ $labels.instance }}\".\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostConntrackLimit
+      expr: 'node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host conntrack limit (instance {{ $labels.instance }})
+        description: "The number of conntrack is approaching limit\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostClockSkew
+      expr: '(node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host clock skew (instance {{ $labels.instance }})
+        description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostClockNotSynchronising
+      expr: 'min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Host clock not synchronising (instance {{ $labels.instance }})
+        description: "Clock not synchronising. Ensure NTP is configured on this host.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: HostRequiresReboot
+      expr: 'node_reboot_required > 0'
+      for: 4h
+      labels:
+        severity: info
+      annotations:
+        summary: Host requires reboot (instance {{ $labels.instance }})
+        description: "{{ $labels.instance }} requires a reboot.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+# {% endraw %}

coreos-config/roles/compose_project/templates/prometheus/rules/prometheus.yaml

@@ -0,0 +1,231 @@
+# {% raw %}
+
+groups:
+- name: EmbeddedExporter
+  rules:
+    - alert: PrometheusJobMissing
+      expr: 'absent(up{job="prometheus"})'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus job missing (instance {{ $labels.instance }})
+        description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTargetMissing
+      expr: 'up == 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus target missing (instance {{ $labels.instance }})
+        description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusAllTargetsMissing
+      expr: 'sum by (job) (up) == 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus all targets missing (instance {{ $labels.instance }})
+        description: "A Prometheus job does not have living target anymore.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTargetMissingWithWarmupTime
+      expr: 'sum by (instance, job) ((up == 0) * on (instance) group_right(job) (node_time_seconds - node_boot_time_seconds > 600))'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus target missing with warmup time (instance {{ $labels.instance }})
+        description: "Allow a job time to start up (10 minutes) before alerting that it's down.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusConfigurationReloadFailure
+      expr: 'prometheus_config_last_reload_successful != 1'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus configuration reload failure (instance {{ $labels.instance }})
+        description: "Prometheus configuration reload error\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTooManyRestarts
+      expr: 'changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus too many restarts (instance {{ $labels.instance }})
+        description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: PrometheusAlertmanagerJobMissing
+    #   expr: 'absent(up{job="alertmanager"})'
+    #   for: 0m
+    #   labels:
+    #     severity: warning
+    #   annotations:
+    #     summary: Prometheus AlertManager job missing (instance {{ $labels.instance }})
+    #     description: "A Prometheus AlertManager job has disappeared\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusAlertmanagerConfigurationReloadFailure
+      expr: 'alertmanager_config_last_reload_successful != 1'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus AlertManager configuration reload failure (instance {{ $labels.instance }})
+        description: "AlertManager configuration reload error\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusAlertmanagerConfigNotSynced
+      expr: 'count(count_values("config_hash", alertmanager_config_hash)) > 1'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus AlertManager config not synced (instance {{ $labels.instance }})
+        description: "Configurations of AlertManager cluster instances are out of sync\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusAlertmanagerE2eDeadManSwitch
+      expr: 'vector(1)'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus AlertManager E2E dead man switch (instance {{ $labels.instance }})
+        description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusNotConnectedToAlertmanager
+      expr: 'prometheus_notifications_alertmanagers_discovered < 1'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }})
+        description: "Prometheus cannot connect the alertmanager\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusRuleEvaluationFailures
+      expr: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus rule evaluation failures (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTemplateTextExpansionFailures
+      expr: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus template text expansion failures (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} template text expansion failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusRuleEvaluationSlow
+      expr: 'prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus rule evaluation slow (instance {{ $labels.instance }})
+        description: "Prometheus rule evaluation took more time than the scheduled interval. It indicates a slower storage backend access or too complex query.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusNotificationsBacklog
+      expr: 'min_over_time(prometheus_notifications_queue_length[10m]) > 0'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus notifications backlog (instance {{ $labels.instance }})
+        description: "The Prometheus notification queue has not been empty for 10 minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusAlertmanagerNotificationFailing
+      expr: 'rate(alertmanager_notifications_failed_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus AlertManager notification failing (instance {{ $labels.instance }})
+        description: "Alertmanager is failing sending notifications\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    # - alert: PrometheusTargetEmpty
+    #   expr: 'prometheus_sd_discovered_targets == 0'
+    #   for: 0m
+    #   labels:
+    #     severity: critical
+    #   annotations:
+    #     summary: Prometheus target empty (instance {{ $labels.instance }})
+    #     description: "Prometheus has no target in service discovery\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTargetScrapingSlow
+      expr: 'prometheus_target_interval_length_seconds{quantile="0.9"} / on (interval, instance, job) prometheus_target_interval_length_seconds{quantile="0.5"} > 1.05'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus target scraping slow (instance {{ $labels.instance }})
+        description: "Prometheus is scraping exporters slowly since it exceeded the requested interval time. Your Prometheus server is under-provisioned.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusLargeScrape
+      expr: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus large scrape (instance {{ $labels.instance }})
+        description: "Prometheus has many scrapes that exceed the sample limit\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTargetScrapeDuplicate
+      expr: 'increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus target scrape duplicate (instance {{ $labels.instance }})
+        description: "Prometheus has many samples rejected due to duplicate timestamps but different values\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbCheckpointCreationFailures
+      expr: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB checkpoint creation failures (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} checkpoint creation failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbCheckpointDeletionFailures
+      expr: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB checkpoint deletion failures (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} checkpoint deletion failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbCompactionsFailed
+      expr: 'increase(prometheus_tsdb_compactions_failed_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB compactions failed (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} TSDB compactions failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbHeadTruncationsFailed
+      expr: 'increase(prometheus_tsdb_head_truncations_failed_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB head truncations failed (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} TSDB head truncation failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbReloadFailures
+      expr: 'increase(prometheus_tsdb_reloads_failures_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB reload failures (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} TSDB reload failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbWalCorruptions
+      expr: 'increase(prometheus_tsdb_wal_corruptions_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB WAL corruptions (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} TSDB WAL corruptions\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTsdbWalTruncationsFailed
+      expr: 'increase(prometheus_tsdb_wal_truncations_failed_total[1m]) > 0'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }})
+        description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - alert: PrometheusTimeserieCardinality
+      expr: 'label_replace(count by(__name__) ({__name__=~".+"}), "name", "$1", "__name__", "(.+)") > 10000'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus timeserie cardinality (instance {{ $labels.instance }})
+        description: "The \"{{ $labels.name }}\" timeserie cardinality is getting very high: {{ $value }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+# {% endraw %}