diff --git a/coreos-config/host_vars/host.nc.chaoswg.org/vault.yaml b/coreos-config/host_vars/host.nc.chaoswg.org/vault.yaml index 9ad598d..73be1ba 100644 --- a/coreos-config/host_vars/host.nc.chaoswg.org/vault.yaml +++ b/coreos-config/host_vars/host.nc.chaoswg.org/vault.yaml @@ -1,394 +1,398 @@ $ANSIBLE_VAULT;1.2;AES256;secrets -64663263656130636130653934313831343034363565666362356461306531643033396566656364 -3035373431353637313964333461356366626561383663310a613033343936356434313262616439 -38333435343662346133653737353832316336383866613138633935313936666366613032643136 -3066633338663930610a646464313133356364346463373562663864613863623138613664353263 -61303266643837396565386634383265383566613939393563383933353635626132396532333163 -33363831363232643533396636323633313037336562646536646131373332386465663130663663 -65653462373535653035303663306138636662303934333138333830666636323565326564336661 -65383863343761336663363336363066356134666330363231336439383935363034643039313931 -32636566623432373337663238666632353833336536366362393037303438613963336637396632 -61623564363564333064363537396161626135363665383434376636333964303335323034333162 -64616663316430653831663337316138633033643333343264313730363530316238316639323762 -32366535666431386132333762356465626436373131616435303538656434623637333630613530 -62613261396537353564333136326130643166333433393534633538636332343466633934643638 -30396137633138303265623139356532663635343230396662366337613064393763323831356266 -62343962643065326266363032323739363864353035386330636139343938653566303031346134 -30306664653461616534303932386131303863626436393333656139623562636332343564623563 -35376635333938303236646537366163396438636438363266643538663862313339633637363662 -37393135313461343465643735373731636134653366353132313161323535333662653064323163 -39373031623638646365613138333137313735666365353666303434323032303335656531376361 -66353264633164653232313736336130326233326531613535316462383033353332336135386433 -63396562303263633263366465376262666131393331366637323737333263643061633738616561 -62333134666332356437303734616432396633366566666133303661626261306537383736383332 -61386164613130336163663163333035356539613734326639636132323366616265623636343165 -32353535373835363836623838303639316365303334626564626562643166613431303635613338 -64663933663330656335323362333165313137666264623864623338636333323635393962373836 -34363066656464333236303439633738363364663438613039663764326338333832623338383333 -39373461323531643161353939643061383761383235393837663436363033616262306261616134 -35373466336130323939623133323035303333653630313163306537633261313263616436633766 -37613664663135633566663766353633373335366339613332326262633563343034613939356566 -39656335343936396336663235303465373666336632626464623135343638663839326566313833 -33316239363132613732653031326162383538386333343336383234623631666662386437303333 -66336137663363373937333632623730316562343531333665393864356663333034663461663361 -31376165613932663730306631616234346131316438623537363338343763313437653566663535 -32383865396336353865363031373164363032303461353738666138663834613832653965303737 -38326535343562346238303437626163643230356436663461373938386466336263373536396666 -38613061363966336331633963666131623831383735616265613533333365343335623738626132 -38326465323832376633383436613464343464343437653933343638613462623135396231663266 -30333161303464333532353333633136323764376634663432363038646361653131363533393232 -61653633633932646338306539633339313930393865356166396363366562623566643465386162 -64656237303633313439643965653739356234643536373139653637663464353962323963643831 -35656565623233663865633530663736333661643136373465363635353934323665336562373730 -63356230626264636632313232376237376566323337666438356534643934386539306465366665 -33363237336133376165363962363363636562663364643439373566663730333535313332306638 -66333037306331383532646635663339666238336534653365373334623639373962626137636538 -39343766626138323332326234386437653035626135396565303138383836333138323164383839 -39306362633065373739393135323565383632313434396562303030326632656130386135396463 -62653830663563343565326336623337633834326433623532626533323632363832666332346635 -31323138323561303665633732346165323830663736663933623536613839343033623837633835 -34316262666564656438306664323062316130326630666636623739316332373331633663626461 -62303761363535616663373238346531383638643363363666366233313664313438313032353438 -37383261616664333836663634363738323035656431386564373837323266333264346661353835 -31303639393466346336333834396639646237646262346335653038353561376538326364396164 -33316330396535313331336634363736393866393862663664646234636330623436366533616563 -38663162646639616135356333333461623932396362373733626131303638343437646462326339 -39616431646564613733646331313137623762313662303763613833353138373133633862663937 -38333064636635663939343135393966386365336366313534623236323934313764663961623734 -31303665623536353539333337346535396565653031653133613936643264653231323031333239 -63333837356334663062383331353166326639636265616662626531306330666330376638663764 -66353762383833616530313635643161306237623136616539346635653538343562376638663530 -65333264313762626365663564333137303131616232383361643163386261663232616131646530 -36366562666234666437666466343036323535626530396135666138636562303962313131613233 -38356132356239353862653338663438323037363339343137643831333731646131376439326463 -65626434353563353931323965336663393666653165363039626239353766363439313634623165 -62616338356565646234666436353537303639653863663439626561643264656432623534623534 -62623732613332656364336535616663376165323762633838366431313734623263643932323336 -38623564323731396232626362353162636139396134643433633361653230353137373133336436 -36616334333834663432316539633565343034353565396230643265323932363032376131663631 -37613464666666306532313230613363623864326636353538363166653230633135653930383139 -38356466306230363737396635306232353161383130386466666235323262393163623239373764 -66633331383731623330386631356165346161396531616233616338313034613039396366313035 -39313635633236613333353838373030353532633432643766373031393438643838306336656231 -65386462653534663733313137396665616463343366636134373531653938346134336663663336 -30303533663863653762646635636239393662373035386132643566643430666334373233656435 -32663132313530363837663161633765663239393733343034356166616362393963323362373637 -66313061346530643561356130626435393939373637323230353138376562653839643337666232 -30663733313030316238393465383965303264376433646231393066323931336637373532373731 -38616330353733313533626639393861346531363437343561363062346331306433356163663064 -65653232633435336535636136316561303865386533326231326436343165613837366138323832 -36653534653138643031306665326235316231346134333532333333303162393037633835646461 -61616130306536663732386561396137666639366437326334323931373832373935353266613766 -38643663376138613236376331323164313666653364633235656563393937383933653765653335 -63316365373166373562663162346435313330666635626331363966623865366563316334363638 -31656564653434613034373837633635313135613935393164326539363633643764343531623739 -39343034303335353434363835663432633731393532333835363665366630643430363530353730 -31396332643062616366643839303135616436393030653432316536373962343463326666323565 -31633831616336383837306562333634373333396439633266666664386233633664633464333463 -38663532346338306138616638383438343464326362303065656338336239623335336263313566 -34353038396237343562633665346665623637326132353663616430646661306339663936343635 -61373164336135326361633335643231363464383238303764623234646463306233346565633031 -61343265306635616263616231373838653462643635626138643435626566383866656566626134 -62663136633063313939383165636538373038643937396333303431313733633436313133386530 -34313963373936343862313435626237323633623539656539336363626337383830373439373036 -66333637383163623133396162656234383166316637303431336335393965666230386264303232 -38323033383837343161633538326165386639623731633436353334306266333261663839356339 -31323235616664396130666461396130653534353933653765313664323937666437666435653436 -66613038326233653535633033353661663838316234376462636638636139643961623561313861 -38396333356364316338373666333737613265663638353964643862386266333865636337343061 -64353130386338616634663266333831653666306638396565306262636431393433363939353930 -62636339333033323863333931353763613133323530666136653333353665376239643136386131 -31623531666433663231623731313464633866383934646361366137623863653833303332613531 -37346131376664376134303165633665623938363532393337353036343865353061356436656434 -32393661326165336633666563333032623365353831306134653330353136346262376364333166 -32306263343363336439616638663533366139373561643064373835353339653637303535306464 -35316336656263353637303864363239376333376134613866616433626236623737333265366230 -34666537626238346663646339333831636637303536386561303766363362633730303961663634 -30333536323634626537386163323438343734323762643531353030386463613561303233613162 -30636361643439373839363032656233363238356664326537626432326230306536356436323966 -37306338623833633936383936613366626633613066316663323933336136616131613039656639 -31386130656461643765313332366565356562383766353730366663303736626133376530366130 -64636331326439633135616666326637636438333466366637303136613431646133346633383964 -39363630633262653332386436636135386165356261613962333562616362663839643739346663 -33333164616234666437386636346138613061303565353463653138323534313766653538663231 -61346632303362643864356666346165633532643563323836633039663461633832386335323765 -35313939653466656666656337333730393938646635356237653034313061333338386632396434 -37636161623364333733643864383864323334353837303338653634373430663464323031343737 -36313933386431623461356135646339343034323533626139646530393737363561633931376131 -34323735666431366433353734373362663039616231366437626133636563353161393865303236 -31396439626430383231363938303431373061383237616531336233636335346632396364303663 -63396238663364323332616632366533346564326138323931306464633933353366633562346161 -62633432646537623532373431613832623865646234353239626635346133666436306531366665 -35383336656637663833613561333134666531353338643136643437303937653066663562376531 -31376637333861663735333334326133336530396565613138613261333166656232343438623730 -64663235616338623633656638393265366536353330663936613963623731343462633138663666 -65623137626566323636363838666336326536316331643035613137306537656433376531643964 -63386631613031316331306130316362383133613934656332306161313764313435393964656561 -39353537653861313466666339383930616237306562303837323461333933656164616436373437 -32393434346237623535333534356561643862323730626564613530633961653838666437376163 -37306165303765613766333336633736386532333931656262643561303964623630636364633932 -37666639333035623233393537656234393636653239636165306264363831393966656665333335 -39666364323039343165656532653965663233366336306439613066323837373430356238636664 -66306339386539373464636235353861663662386461323861393761376532613138373137353538 -32663238383032373432343266636165393732343066383761306434346564343136356262363465 -34663136623464656563663230303532653065613031653963363730373830643366326263313239 -33366530343936343665636135356336396434346330633230633938343833363335373935373233 -34383664623266616666303738663564656135316137333561646264383537346639386538623665 -32636438636137313434626464346239373261393630353930366565393735333365313162663436 -66663237623233653935633564666662666163373339636663326434333262643665393435343636 -33366266376330633333353232653030363561303662373030616233323263343336353662363963 -33373237653539396164313161353563656632386662623961323036653564613664613238343736 -31373539353065646333636362646631313466303565646638386634623038653764623239363537 -38323534653161386435373530666363323833373463373862663731376165636339646364636130 -37366532396332306131393962353534326337303038323335393537623033653531633366363765 -65343831383437316232393232356539303664393931343736306436623466303065616165313639 -38363261636538613731306533313164353539356330363432386630656536373032376136363630 -32386565336334653732373138326564393037373135666165323734356464633836343165326337 -63353933656166353136666263356633656264663064316635326636363333386530643732326134 -61646263626165313430353433383437393436316332643732306634623665663530653434653836 -35653435643662383033333431393834623164303461373631393166633530333038326365663963 -64366662633932316566346362646434333030373166643765626464376230393233316237363036 -66386237383035613932616463663033616638366238623830353461653664663132313134666261 -35623362633662353233363531336230383465353834376661336163643334323831323838346336 -63323532656239303730366236363930626361663333383265396338323230626139346339356262 -35336332643830386632303563303731653439386631386132343463656665646434613135656535 -65623763373164386637643737383437323837623665613563396332303061373364376434363830 -66393432636337383437353237306438373633643165633563373162613363633432636563653432 -35613264623431306131646464633163363530313962626134663238623837376633333538306136 -36663764326261303933646362333165333939366263663638643261613534366235633431353662 -32303865336435313463303964643732363730646436333137323632636636656634333735373630 -61333836616136616135323435386666373933333865353230623063306466653830346134623632 -61396238663263333136313963303838343435323466323537383764663166653861303339343738 -35643135363463616466626435386630386334363765663662373763623764353334333136353233 -39303833656638306162313365653365323836373461383431386131373461663037656138653932 -33383632306231663466643933336132313562396132343331303934353839663537643462613632 -38323937643539386136353034353532326433316334323134643832383935373332393665643339 -35396536336635373138666533343661623663623039383732393938633432636561643464373561 -63343761326537646331653864353738643439656234393132303532346634353462336436366136 -38373464663663653366383932623462303666663564376134363530633136316637313666333732 -63396164353937373764326161643236373330363563653061386661306633623862666631393563 -63323165326431303035653065393039623764616233323030333664353261663937626461303033 -63323664613236666538326332303964376266323237376332616139336239373035623330623739 -61396336316464343964393161333664303130366264356462316438326536643639663932663666 -61666438646132303739333130626561366363386163366239376463616631336630613438646364 -39653939353961646661336161336666623433366163396132323035663735333731616331373535 -64643638383765663766383836303937333735343139663631313836336465356438663962616432 -64356364373564383131656562666663396162313537633363316133336163663730333063626537 -38643563376333633561313137313039656536636339303735396562393866616235646137643934 -61623836356231393437623434303437306637633636353037393734336134633061373533376461 -66393830316566343031393937663939383737636536663362313534353664373530326163613662 -66313165346262633737613737326639396130343335336534373030633532373561663630303338 -36643766613632376662623238333530613031303535373633316134303463643137656466386633 -34363161623934626430616665663465633832663261303265643034313466323065306362316533 -35616131353934666330373266393035653830366364633966393736336235346531383365343638 -66643239306237353638323532313066663033363263396365623163366435306561336364333733 -38313737653638366261373734353361396530653731313231383765663137353735363734343164 -38323935303636643562623537313062626665343464636135326162393631353237623631323562 -34623661616639636134616266363531333438613535393331316633643765366138376335666563 -61343733663139353139623635343336346537653737646331336139623065346537663138353935 -61653631613139303463653861383237346139636237643565653230366232363665633363363737 -30666565326361626237646432333133343962313663373734643833666139303434353364646432 -61396337663866333430623235346636313236363066623431303732633336373064643935376663 -66383734646136376331633365373865613266613932346630383736303130626231656438643965 -35353633346234333364386466343464343034376636393230336166616161396538303466346365 -35616462353633636232383465656237306439313064396233376462626332366463323838653238 -64313164316630316662383535623336653933663633663030323336613537336135316534383432 -63366261623864396463616361333462313563356264376161353739353462613435373532616332 -34316465326137613933366135653863393662356361376330313534663939326161623062366361 -64353265663430343138326462356436346263326261653366633864653231626366356264353635 -30366261613633313163373136313739666532316430313262616465633531346164663636306563 -30363062393530633937663234346163326533646563336236663761373331636539616235613735 -38636165613236336537393135363131646438376361316562383131666137363638393038613432 -66313036613738366162313963633638313337613363326331646533663334386233623732313738 -31376633623037363832376233336661613333336237313730633663306439636638383863383561 -30643734396661653933373638323335323831336164663466343830336363396236363462313966 -30333663656338646139303837396138313536636366306639373732306533643531383733343462 -33353363356436623735313362353361393833663233663031623439363936343735613966333637 -65346136326462633637643438666231633833666334346664356239316166386137623862393064 -39326236643930653430313662363432393732616433653635363033396633653432666266353762 -33663862383232363539663233636366646530373866643135386435623563383931663463366361 -64386364303066616162623930323832376135643031333631393333383836383362643533346236 -33346436646435633961373133353863636433336536363863393432366135613164623138333266 -37396233376130393135383737613632306462353932356332376139363161636564343163623862 -65313739396636323633613339373266623439333739383230373131666236323565373230396463 -32336137303462326664316661306430393861643861623637316234366338643966336137313236 -63323936363038613763373638333938353765303365656535613865343162306165313934353735 -61313637366133623366366639666564396531633366356635336139663234613736366638386239 -66363032303631313164653830306234653536646136666362663436646238343939333965666438 -39633863323532303039373037396634393136356437346634613861336436646662356530363639 -36383532656239616236653331313863646236363037326162623833666538306439343736636562 -36666230393435636339343535623635663234356237623635663663363230383232323634326232 -33633531306462346563646665643463346139643433633235323338613236346236313462343062 -32663931333337346233346633353534373530636365306533396637313830336166313165383239 -63626331393265326165343931663733646139636131376639646562343235663037363861623737 -39333334363366333733313038386563393535343032353061373930653639386431636132353935 -66636431343730363262396262336230386265336261303837613837303730633663353931383137 -64356463376461356563623763323961376332623665383238653436653834316630666133616361 -62353962613033653835393238383734666465616164343539363030313232393037333732623334 -36643762376364373361656437376336383031343965393461663563303965393938646463333236 -31363038633266393330646264363937326337306337343266346364616238343037393063623130 -32383463303935323337313363666433383030643762346165626264663165316437616166306536 -30626532343137333334333333623837636139313861613931373935383461383466396336363439 -38356233373234386234643362353461643032343164663336393864643662306134326235326139 -31643230623266386663343430326565333539393361356336663832643631623962353464386133 -38306431613132353465303432616638326236393166343138393837386561643633346364666538 -36376463396366623432633635633563643365373336373538646335383865373665326130646564 -37353332653431656135343862636531323639373833316638346564653036613035323766366432 -63643733666330303437653133653364303833313737633234313835666635643763373466336637 -33383764623937663034323866306666386262633563303136353335643939653364326166336465 -39363762613865373935333236363632336636623062623736623333333331303830316439653861 -64343434386437363566306535643163623438663235376635633665336531633738613132653561 -36376137653561346335346637376331343838333038663035663065666661303130363737383961 -35353735333266346533343838383235343439656335313662356665373361356263316161333938 -66313538313466376138316462393139323562663938393032386532303463656533663734343135 -61386161666631326430626638326663643865666431336135643961323330343362653561303361 -65643563333463356535383662373361386130323539343531316531396362623666333530653338 -62303461653037663634303734636566656562316264363665656664626536313838376166623735 -66336165653739343365336663336266333065653632343131633363313735656434653336633761 -39373739386264333739613462353636343864343439376235643430663035336563333563333034 -61303030313532363232373735323361396336333130663664626564393332393761623532366664 -30333434616631613064396136333339303664326264326239366439393036353162356130613038 -37306235663036323131326634613065326263653731313132613739663533663532613631646436 -30656264323865333066313732646364636436346133333764643061366662333563303561306135 -64616164353035333430616264353662396132343064396562336538313761646666373535623561 -35343662316363613862326536646432356465633833353534366265633538613134333134343861 -33326331313734383462386363376363646438353238653031393837613561616635356364316364 -35613331343265636235343761396137633531326332656664613365663562366430373461396465 -62313562383534663638643063636630373564616431363162313461663833663430643633333334 -32386239333335626338613432643331643532353433613635316334356261626631616238373336 -38656630353163323932626632386662653831653738343935313765343463653738373233636638 -35613663376465616339363739623435613535323236653665663263313864363133373465363036 -35396363633262346635613033623361643230616362633237323133666636336264303466613462 -62626265373737633866636531646362616637613161643832613866356561393336623934393035 -36356163353362656239643338383963653664303862636239656135303063626132626563363233 -34336533346530663630613539373964666162643666613137333331343232626238326263643464 -61633930346137313165343834386663666565373039343161646636356135326337343533373032 -61663164393033303065666230653035653337396162636438666563316265663631393566386338 -33323361393330623634653664393938396162663333343334616232613237303032313963303331 -31396436303535393164613439306330393637396465653030396230313431383835636530396233 -32336161626438346633393835373263613434306361613866373762383132366230303031366135 -30613934633233343237666131383361616635613334646334323236303261373834663733383365 -63333266326663333832353636346630396265356166383731373363343632656462383535386363 -65356333646130323030373531363530643835653933333464363032663536613666396235363866 -62356365616332353335616435373162643337616634623436353762373364313234643961326434 -64363461316364643262633264333562626564373166376631373230626364333533366262303830 -31336433363337303163366564333433653464653662336536396666333536646637626562393566 -66633931353937363462636464623661353965333230306363306130393237666135363963306538 -66616636396132626464336162646239613432353863653039303962316361316133663134323033 -62633039356136386230363538613962366665616432663036336334376366353266643461353435 -66306232323863306463643537643939633261633337636231373761613761383937616434373663 -33346232303630376137383139343136613931343632643737383032393733336265646134333838 -36353561616631623964343163623661373635313539326564663631616431643136613638646633 -66333339363638333266613836633965653939666635383731363933333331636230373039316338 -30383734366230326234616266386266306138666165396639343938313732316539383766653165 -64653637343935363236303439623130346662336539393462656563646337633561636464376335 -31343138303461376563343234643136363739646566316363666531633665633963303937333966 -38393430343365306135626630353161346233343464343430626463323637666466636166356531 -38396364663262336361646565636435366166343536396630363738633039343035313337643132 -61613636333264663461316366646434353364623165333665326631323434343036613764613465 -31623239666435626165333661616462623735346661643466353264393463323630346232353135 -63333931343566623837643238313834623162353961613335643936626532636438356235346636 -35353330316533326161616565616534313530363132323530356462346262393433626533353431 -39373061396266353233316630623030653765663231373139303965383736356432393532666330 -39363437623533373439313133303239336263343533336533373232333235333739336136393438 -61316530316337386539353966663766396233323531653233363939326661643430663863383762 -39666531663765383739653438373030656135383237393939343633646563353135316661353064 -31613532306339303263323963363762363937643039313632353833353536633162383830613435 -35653637383661666463356363366334623331363831363835396266336239633936643631336131 -35326261353064383261656230396136313838386564346139333031623833396233346431653830 -64343030626333303633373062346432653332393862653632306632393361316536663239646431 -30353033663931633638383466393961343330383536376533346337346166316130633632636566 -61336437376237333030343138303030363734333938393436363737346664373662383937353538 -37623363313139383463663662313137656639323137346438326665303635633764633935363965 -32636233366536306636623835306238383137373434656564333635353632376131633231613666 -62613337623964633134663638313634663463623866376236653664613738383065396562316535 -63613036316465663464616562333865336566343632646332336430646563383232326130623231 -66306637333535633261316361653737346631373539353034386339383066313636363462386535 -39383465306266396634643062376166323165663363613633643731613133336131346435303462 -61393734306139643637303234373363613738313166633435373263353165646461396464616131 -35363362353064393430383066386135316361346430656236333430663536313832373064353461 -33313961396465333630616563623732623963323165613032616138333432343066373265396663 -65646336346365343362353432303339333964376639346666303038623333323837663931363330 -63396633303763323730383231653762323131656432346434626133643962616438346565343632 -63663363623365643830303937393834633464386439303935383337333863653835633530616335 -31353466373535333834396536613564373264376531626336653066653932333062323366383131 -33666435343862626433666639303333383632636236663636353833376266356361323861393938 -37373466336661383333643065383263626337306433633239313030616131643635356532653437 -37386633663036613165343231623635323038373763336532663330313264636263373565373434 -31393532303865663530303031613765346436346661323134373938303961373839393266333636 -36626365636134383662326235613565373231316661366139346538396334376561376566323662 -63313631626466653864356230623237376633333164363833353932373935326665383339313338 -37613335393039653635626433336635383464646362623963623039323262346138336331643038 -39623536343063303937666137333163356234333262396132303566366336303435323835316438 -36656538386336353162613066636666653165333433303034626230636466383463623837356534 -37666461663433366661623434306466373935366639363633376535393037653263386339333463 -36363039336163643038616561363937336263326464646532656338633861663830633535633837 -61653038346338663937666633303636616333666264356364663635653330643638653638613836 -62613161396537666633653533313639616639376162393230396136376134656231363436343439 -61323137666238336238656635666136656361623338386331313361303131343362353335333063 -64326236623639666465316362303538643139653637383136343764653762663161353665646637 -36323531666339663639653365353834376666316533393239316437333466343932396531383531 -39336665353563613539336339623839393230346630383336623738343565336663633032626538 -30333130383562343662323165653530306632386132393537333761306132356461626534626233 -39326366373566363466393462646166643965616636666634353466376362626437333766633530 -63336162613863613432653837383261613762366564643737623564303230626339386665366561 -33346266383563313466366663636164383764626661666363316436636463303136303062326139 -38356433616437663437316130653062373932303561663233616139363961356364333338623635 -33396434646263653561306539303462313738393232363036323662363630653135346462396361 -31393565666536653963333933343265353634623031666537356635613466306133373136646431 -30383366353536313436633838316239346434376266376333663530626263363164323565346232 -32373261663663333064626264653333653131663564383739393963613264333835336366306539 -33383264653335386134346538366231653866633463653566393766326230343637353832373765 -35366466643333666437663737363763396530636638613464666137346536383565353961333330 -63306365653035636136376338626535623262366564336662346336393365623430616630313133 -32383266613137643236323934313261316365356533633162663730306264313866326536306636 -62323237346166666235393365613539616135386133306465303033323561386164303530373930 -32366332663366306335353032383236663561633862303630616135346337306266326139306337 -34363436366431346432333637346534663137323935303734663435393235623938653136323134 -33353338393032643261666136363964383662626639396361313265623633623333666566326633 -61356265643739663065623238663165333638363365306238313235373733626632643937373962 -65396131336162316436646437656437306134616464653166653965313635376164353166613333 -37643265646265386461386365626636363236353834373335393865646639383833346330663562 -33653138626365333236353461313361656166306165643162393431616564303339313262643765 -65643036326663313835303063393836396136303831633535643366626137633731616634363433 -34373930636432313832663736646434636363303433616139353237376431616432663766643765 -36666130393363623032323664333764356431666132336431306337376231663661323835386638 -32656362383336393238666338623362316663353830623036303335386338363831623533333364 -38363263326133346666313133613764313765646464633938616366383936363365356633666230 -39343233623139363866663239636139363439363661373437626264386233366239383537383034 -31303831373334303835333238303430353762316434316630363436343737306363376532643235 -34643931626466393339633733353037393634323363663339656564643635313566346561353863 -38616230383431666662643666623664666630653831663337313331386532646230643265316433 -61646461646235303836333034626235356564626630656463396161363433623864363232613461 -61323339623762316161346433616130393537656336633333653763656461353765393764326164 -62376632336361393861623230616133323736663864383066643762346235376232623836383137 -66363364643035356131633138356661643833363765313030613533656136333562363662636137 -61303233653237636366346432323166383734313461363031303839623035623033313236643831 -36346230626331383733343762383466386534306131396463653666376635306534373034636339 -39336366643363393236633065356333346534306361303838653235666536383230336235646438 -30333036633938623762316234663366626630393539626431666562646630623938313437666133 -31383666626138646361396439626332613838396664666262643536663063613066303363623830 -34653566366562303634643030336564646631356130653766313463626131633433643761626239 -35356432616633336331333938373764366436343562363766643330346437626430383437616165 -62333162313938363138346466363939393261613634303263303862623262633763663965333030 -65613564363135303331623734313738393263633164633737363334613566623364626536663233 -36333936636237376234303362306439643437636235663032646461646337646135633765346533 -30386563316562336565636463636232316565623836636537623337373633613030373938306465 -33376635373330623266333437343962613235656332303964303337323966613266366165313731 -37323063653839623734663338373732373730633532656263383931373333346337346433653533 -32343333626438313735663364363136363331636161386666363831313437643939316335313937 -62623164663562646137396463346133323034383064633335623535363464633561643931306639 -33613739333035343431313061616362343432646435366439336264386333383865633535623663 -33343139623435336435343362396533303637623035343634343461316631376531626234373835 -62663066323461383165346664646336393733376262386431313430313364356533316436653933 -33376232386562663134363036623433393763633361636532653233623135353563653662333830 -38306435336463643837326638396663636462333761663838613564356532636534343835343465 -35656464373339303666393230326333626439373234313338343034633937626166326264303535 -31636439393836316332643264353339396533626237653537653663363264613764633135326263 -32633436336532656336346430666331343034336264363031666333376266643737343734643338 -61333337646361636366323937396664376237653932633531623430343138396432353966353330 -31366435643063393035353939643962306234333164613764393363663163396138363633613636 -36363963643531343431653465393763386638393830643739656532323036636665653564303264 -30313832346537336133346462623365613137303661653739333537633230306630343433623539 -30626662626137323536306338303339323736303832363734306565663232366538656666366331 -39303162303637363562646638356333386666333162306338393037373664343362 +62363334313439336637633463653539323063623034616132313265313765656533633632333830 +3130343733633334623837323339316536336666316563330a346664623339643761616230323337 +61386339303733363963396437656533633537396439336264633466343866303531363439643033 +6664656239343233370a306237656663373063636430313734326239326431636336666662613334 +30353661616161333565323665636266396534613634616631333639323466336232333739633065 +32646265393834636535653939393339626332623265313261613936393663326239393561303365 +62306335613338326564313565323233373862383862376265323637323561366164373237333132 +36646563336265663865396130383364373035323066626562643738306161353663633232353163 +31633237346333363337643663623330326236383137636533363131343563653063613931643166 +38356161313139396336386632623739346433396336353264626634363264623537323236333439 +33313934663134356663666161656539653739636432356264646631333466656133313635633264 +62653761363663643535363436323435633739663335303939303564303663623666666162333838 +61643532313830626631646430613230353830333935376134383632613662383931313939373036 +38646139643162663362323335323863353538373038383635643330626638323030316534393638 +62346366303436633966613861306466336130633766343630663336633235653762646533343365 +39623261653931323630613266326366663563626230353436616234366266353265623938663762 +39336561623766653434393265306338373763626334346237356431653136636237663439373663 +38393466623630333631346338303231343338343735353235643039353664646261343439313236 +64616136353738393933363761333438393838303331663732666631643730633637306431373261 +63333133626132376562383830326530366637616330616164326265326136313438623233313633 +64616234643238303232666238363966393631623864613837333536386133373336653331643665 +64636266326436653739376166363738663731656538366136303236643161383564333164303863 +38363434356433386132326263336131663663666266363962323665663739336136666664376132 +31653536323263333139613265383134616632366630336562303130643261613464393362323163 +65613637306333303162303636633762306464343330623238343336313433613633646630613034 +65303236663835373762663639633330386361626530323535333834306162363936323230376331 +37336665306161626665386135366439633966373737346339316166396561383333313436656334 +30633138313237313237623963386433366662366532303830386661626435373931346563643834 +30343365393430346139373462626230666338343335383262393137363563353033323663633935 +65363337316439356565353439326263646331393461643866373530623933376437646238663362 +36386430313739323738616162356564323839663664313865366238396638363162633632633065 +64316132663733353962633965363239306635343333666330366536623132376337376638626338 +32653933346365653136333734393161306661643739616237656631666436373231623537616163 +31323337656165333033363632313531363836386564613231323935383331353734373762366232 +38346438626332616463656433623339623237393261643930376433363764613136393362333638 +33346162393935623061316430356439323230313635306232393566313630376337333163316462 +36373733376537386162313030663832626261363764326335656437613438303534366534633937 +30353631363264313834633763643633383962346131303565343530383864626361636433633766 +36363963396633643266666538623037306363623334643765623263616436336634623035373837 +36363364383232333864373334376339343331656131613632633534363739343935636536383064 +38613136653762363830323737623265346339376131633633306430636361373934616339653363 +30396537326362623537643633363333633834383363313532396432333733313362393333323537 +37336366666130666437393966383963323363623666663333303364623733386238613133643939 +37313165323234363232313037396633376366363230333039633035323237346236363361663034 +33366163646663343666303236303239336463666364353939393233333036646531626136383732 +38313632376532626664373335383639633566613462616135343231396131623265346162623961 +38383761356335326132343336613531393765303733303931626237306463376664356563326164 +62653536353636376537653336663066313465346536396466393963363532323837373130353938 +63386132383530336131646164663330633331313535363333356438356430326132613731323262 +62326532313532313161613464316162333533646661363634333133636231636433376462623034 +62393139383735373761343731666666633732666365396162616562633164366138663362363837 +39643238316436306431623136663831653137323932333136313537376531383263353837616536 +33666239626361643835633065303733643537613634303737383034663562663031333035643163 +65633034613965393231363838353461643637346236386665333538636432633630366530653233 +33653737333464303661613534653962646430346330323933636265643230616562396666386533 +38363961313237366535326665343462363262313831343233386330653739316537653938653563 +36616339303233666662666366356335626330633162383639633034313432633764393962626531 +66663366636132366465656163616331343062393062326564376637386435396363613262363966 +39323331396339343233313333373366626635383431303861343362376632353334356332303732 +38386630633663643262333338373037336438633135396165613763643532313861633464656663 +30656265373433623033643536326436353332636663363533613033346534343163633937346466 +34633064663930613138313236653761396435373065343235656166653963666363303835313435 +39333435623538626131636564653136323061396361316663346661336234626663623165353734 +65643932333334396638633639623136656539313039323239623637396537396339343366386436 +66313864336231636263323862316432653239376431346137373431353964323963623161633938 +33363164366132653966396462356133376434376137336137326338353232616166386464363666 +30303537613237373162373833663935643263373963303031633062383731663563616239663731 +30333862663737633831623237666461376230636237383330353866643766313066626637643666 +65666639316265386463643238636463616135393635393930353864643837643937353765663934 +62313366386539616531643530303939653232363832396465393933643166666663646530346136 +31623531366561383135346337313430363439323966393334613138666164646536636234333165 +32356232613164353833616636613435656137343834383938626237303530343438356665666236 +38613665643761396635303931366236353263333338383239396564656537383165376664653336 +63666436633963633461613539356662626263373039343661303363323664396666633439646338 +63643763656663626338656163363064366661326465326530613931613064393761353938366535 +61373134636231343434663738383932663533373537393564663032653465383462323935323832 +62623533303364373139326563646334653864653461613137373061323163653831643661626431 +64323336393739663863353133623263623033366131356464393631633730356461343264363239 +35343032393762343535343663656538386532316262386538333539636363633564336632306265 +31653431333337313366383964353637313631393133346262363734336634336439336436333634 +37373736353066643236656231633330656137633334313136383064356163303530353436653032 +39623730323036376662626539396237656537373463333136383061636164633163323831313130 +66343161376436393763626261313066363966323438633237343664363166383336356537626166 +65633562393238383439313662353030386637316639383765396530383663303139633835376236 +65386163633031373733633062666336636430373966363431336662663539643963363863316630 +36396130306263396631346266393763386139336164663962656333663266366231336337356338 +33653764346535366364616236616666663931303530343930663963383037326636386263373736 +39653830643137333330343633653364626162363365623436626138343163636635633734386162 +62343439363036663332646336306639623038613039383962383234356463363836383733653939 +34653231323539343863383832386535366639626662613636666132316437666539303435366635 +38333734653638333431643165313737333262396439636465303838303637383134663432636131 +32393231343863663062626664653565313135353737323539343132633933303164656665353239 +66313166386532646530373236356263306139393239346264333461653538313332353764343236 +34326335366366356339376162633764333134313337396364313263303336356265356134643139 +31313035306337623364316462643539383039363438643563303131656161333036633031306432 +33366263643936613835643366653563623239336164396136383439643734316231613463633161 +35653937666534653661343662363761653637383464343231353863396464613233666362303730 +33343564343136316236616438376533333666663266663233366433646162663935343261393339 +66373666393035313730386662373063623261366236303661396336643736343062643064613161 +64613866383035363530336435636538323266373264393036343462643430323232346432366566 +33633663386536353034316438346238353761363139323866613331343263373837363431643038 +39333132303331623530383963323539356235303139336462653336363565306230616137373163 +34633538343664323534613130356463646363323137363062323234346262666233343530663664 +63353932376336623662623865643733613939636230313432653233303336313466306263323938 +63333464616664393037646264336435373034313565663132303737653939343562393136346335 +66343063333164623833326433653330623637333863666562373263323436393361656335653464 +63346335366338323733346336643139303037633334653933303837643965383363366436346266 +61393266636632363432356566633534613765623066653039326265376535386465656339303034 +33663665336132646135313832633337636265393536653866323266666665626535636464653562 +31613935656138313735326331366438333030613539303139313263353332666639623661626534 +62626364643965326138653532393435316461376237656133663666326636613564613364616630 +33353035303933323030376630303131343733366465313262386537613264626661623038363465 +64613934386262313563386566323066373935393533623839306439373661383865313238353830 +36323565313132613463386635666235316466333966623935633035393634623964366633353737 +39316233383839323965343633373832306336303539663437373134386633306665343032313139 +66666234393061373361393539313066353034323066623438623836333364373739613465623763 +31353535636462393832376163353437653131393737353230643537663266643564643834643833 +35323862326666326237343039343335343230646466616232613930613030643832383861613761 +33643336303964623035653433626666356330306562643930626232363939303639386334393837 +33326431653961313936383663356330376663343135663463663239386137626430623037666231 +38323665336437363432303036653865636665306662666634356537636530626262306639383531 +63376636323862383265336163313839643062313636303031386262303239353431373834613632 +35366661376438383336353339386134306137373730623461353461373532396334643834356365 +35323730633866613363643037346334376233656365646662366132653935393731383534633365 +30306266643166323631343936373536306134313839383232316365363864653333643362633538 +32646636636435643764383534383530336235366165653439623537366235353830613532613632 +34393831333036323139633966303130626130616462353231393130383162363330633530613436 +65376434316235303965383966336536663464316264653436363436616565636432633935393736 +30363535623633643930393134636561633464626439636231353230343636313137316365656565 +30396231653437386232663964663736323061303235613636663735356638363330313461626264 +65653262353163313664363364393866646236336664343163656534306361653637306362343466 +65343133623235646666326562303537353833356638633137643530383935646537323064333336 +39336262323933373333666134663261636433636639376634393531366534303461383535623363 +38666339343262623134303131623531316563336565643133643030303132653136656337626563 +61333636623263393736636134633533656365633565646638393539383131363739613637373930 +38323931373166643764393939393932383233353432333135636166613832376638623435333034 +34633766666566303963376538616138663566646338396664636230323832616535306133323434 +63306336643530326539373662633366633030656138313166623661666661663339376438333863 +32616431666435356564303134363833646566653234303934336232616361643734353239653738 +61653037376532386663333034633861303561353565613463333761393436316438383366353332 +34323939353939336261313164313830323034643661613431373335666530336231356137646133 +66356536303234323731633362373664623261366566386635356364376531393530353237313033 +62313634656263663533303539313331346361323566646330356231326339316661633639373530 +66343939623132663139666139653936343830316235353661613936313966316664353738363334 +34303534666331306465303436306237643632373630633931386431376361383061326435326337 +39356164323065653835333035363437636362393434343363633263323533343030616563613332 +61366233323036346161363338393232623136366365373663613431373564316163303632313532 +38333634333036623661656139323734383233623938326536386638376136336332373564643138 +64313030356533366430616338613434346330643661336461363033643930616535616538633430 +61643436393564626333636663303465616339646161343461663461613339313630346436323863 +61313639366638643261306334306239666465663339663838346334343662343332663864356264 +32323231623964653864653331353563373639613363663463663135623939386139303663616162 +63346334633236333562663632373733656231643038333966333035356464303237613361313438 +64343861356131646538343466646631333036633334373337393838333566633034613230303262 +36376332336537663631386162316633633738346338636361316663363562653338616262623538 +36613038613266303038313830306162626231373030353764636639353030326265366530626266 +39343862393539373162363262333031366136643135396665366631343365636239376333626531 +32386261353765333638613436656132636363633466333233363063326438613365313538346239 +31343633343237323938396537626333313832353564346161623730363636343532386262343166 +65643566316465666333313533643363323233326562663733643361346233626633656537353933 +63366131316565316363373666353036636632303637316664393863613461653330316433636663 +35643530613965663964313265636335666638313465643335353632646663653435363930333534 +65663238656336383133306535313433363934303836353962373838346237396231366539306563 +37386438346336326261326165323132383537353464346461316430396463663562373636343235 +62303261383435346163653837666561613334306139393532356665386362623633656366306232 +39626131356365663262326565353232616463336363343430333031353734323735643834306237 +62333431663035376135613965393834373335356538366563666266653332633963653538336533 +65643564613438303433623538656664363966626530343365363862636166396338626134393532 +34396663623538343261383832306161653766626434643566656330626538336565313961613230 +66323837633535303438356466303031313565616365323662666265613761303866353338656432 +35636364393065393738666431623438613433363561343066336366353531383134633531616433 +33316236303430373466313831306265623139336161393730336463663564393566643435646539 +36616637323362363537643134396630393162633735336661373763323439376435333532366538 +61363334623132313665356135353432666439613363616462663431633834613230363733313737 +36366265353066643231373830363233666130656162373166343832306638373563343633326535 +62326362363836336333356164666464643033643166396230653165646161356363386132643465 +63383364376631383635326230393235313834353932636231616366633639303364303631323561 +61626165623839356235623566646438373436303636386265366661623635333061313931313865 +39636166326564303434363061343838653161373237653865386666393530616366656165323861 +34633464616261653462636234613664363430333337646263613362346464303433316166623537 +64366635633166383533376466643437653339623662343562353430353362386164363835366438 +34316234303932326164376339343437326230363830393162313439373235343437303966333036 +63646135383663653137316239386137363663646161376266333132613265303033323037386237 +39316537646336336636353739663639646636633065663834343466353233613466346561306464 +64363664626239323238653031663164303930393832323861633536656431393064393663323031 +39633434343865363433356236333366616335346235633463363232366638653364326233346430 +35623030623230313431316234663737383461343237313566336534393866373835366263306165 +65666334623533646562313633663064333734326437316562633036616431303938653431313466 +61623832303834343464323663326161653436346463393330623566376161643565333639656464 +39336461363336373438316239343863356137663166633431363763336463623962616138303532 +62346532626663333061633365613638316339363332383535373831316331616530623532336536 +63393234623466303930333039393738353431323336656564343266633239396238303636353533 +63373133303035626437663264303532313539353035663066386135363533303537313766343265 +33396330393663393564323039336432656133623130663564313765656563393462393537666133 +62636665373539636565303536383931636636313733396630356264623038376536663766323636 +34326331313236313832663537383563643833303939363939393932393736376162646430326164 +66303539646131333064626334623663353964326661333034366236353634303439653331316365 +30623966353835323836373032666238383538646565363661353236626538396236643638663534 +32653333336433316661643161333830343362373361656531653635326239343631383961616637 +65306164396330643638313465336437393730393237363962303332396630313839646266623939 +30326535653538636135393261303564323436306262613934343463353365323433656164363066 +62363636383766666333306134323465376138653730623734383761366337623433333339353038 +64376561306531336433646562303035343566383839356661633564613861306566316630303439 +35636466373237663430306231636233333761383932633361333935306430626263366635356632 +35313037633934313532353037316462313434646539336661616538353530306439633164653235 +34346238663739303537636363336261623532646232303830666361633431393062383330373735 +38326232326635343635303864636231343739353038633138663632386264323831366539343666 +61636163666235656630323463633734303838303363343036343063623866383462326537636634 +63336235303732396332316138636331656466313734376265633865623038653163613439646166 +37313936633835343230306634356161613636643134373161343834633534363737646636666636 +35353163383064326366316332636637336262616562323461373138663137313365613035316439 +66616136636261613765373066383736666535323063336436346162376361383466653137623064 +61663930353331653534306236343964643063313865643235336631666338616363393439393164 +64326439656336313166336539653435616232386331323666366337636433666535643139633535 +63646664313635313634633463363436666531393630626439313038623764343933643634613238 +64333639353931383765613831653932323364363666343335303934323330303938636133396534 +64343534376135663033656139636231356437343666353663373839373862623238303739613166 +62356233633637613564633261363532636539396664643434306330666634616362326663656166 +35643366383437626635663331303433383630303531303164303139623432396239653166323437 +39383633643034303465376231396565366636383134646166323634373766353066353636366436 +30323638303064626239653439393961396664313630383839396561356639323436643037616239 +61363830656631316537326131333162366234663931323861666231633738613162383530353736 +32633035383631653838383838353138313034653166376536663862393038636231653235366338 +63386436336635646533626666653063343838396233643138343736356136393136353937313033 +35383433353236383138653539663737613263646430623962636230613433396331303861616436 +38313964373964666563656235646664386331336332616131643338663635356436346431323736 +31643931383765653234643834343036396661326163623164393538613236363239643539623130 +36303338666238643663633332643965656665396562663962303230633166653662653835343766 +61313034336330653365356434646237653137643038373937633434313538343535376264623231 +37326636356665313436336534613433303836386234316464343331636663393338626165626436 +33396533373863323462653461336436303538343565383261323765366334633336383932303265 +63653932313835613037616633383439396161316232363831323935383662386339386462353463 +36366538323036323533363562336663383664353839303261356661376536663736383737383830 +66333233366231376334316535633735343638373964303033626466323463323938393164616633 +63373433653465366363623338663866343838306532643863646339663932393463376137636432 +65363064623933313432653731333235626539663030353936636266613164346134653133396436 +35646431326639383030653430316661663462653466316436306536376436633633313931313231 +35666636343566643338366464666638393031646131383435613135333337623663363464336331 +32323538306262373639393236623332376265356162613732363539373835653430643434623764 +37356639643961343931356466613363343737353433613735343062666362316263323837376661 +39393730343730643137353536306239333639633439353439353439323038653337303061666331 +61303766613738316161623166313664336161323732633864616436366638633137373661326564 +34336130323230663963396639323737376437616432666137663164623961643063666662656664 +36353062646431643039373465653938316239316563386532363061643565623038313762656534 +35313031393939633265323430316361373130623330353935343666656137356332396465323132 +33663432653966353636346466393161316630393536633832613533383234626261346464643361 +62306239376235306364333662363130386365616132653031613938366165653739643130383936 +63633463353564613761303035373439333032353036656134363334373465633062623163623135 +30663764306632336664336134306136653630366662383162653863393034396132393237366438 +66373632383232303632393031343163323739366537653330353538616430306261646530623565 +30353836656432393934396537356637323263373335343436303932383762623163643835343633 +34376262306333663732333964353633373332366434353637393233363539386563333534623939 +32613037616462626530306365326466303465333930666635323737646262303636616434643966 +66363838653636643933653238363337666166333363333964336235333530623061343332643761 +32653637376133663939363237616134643137656538663936333039333236316661356430663862 +64356665666163633663613836373637623737366562333637643933626438326439656534653931 +30316237353834333434643765613462333863616136313539656461666136323663303130636137 +32613334336265663666653538643330623465333639643463376665646237643234373937393865 +38656338326666363639306464383863613339343438643137623765643461333762353933376464 +64623330346334366637323934666535626131303461663233333036373939656331383736313765 +37656664376633363130636430346535633962616433636639356666316330636462663335313535 +66353664663439353131396633656433646464323539376139643264373334333336353436636638 +62653066383731363838373731656132653137343139306138626338613864313234343866356465 +37303733363939633730373961396263323462613666636430326262303365666435323735656639 +31373962333363613133336165376339363933366339383634663031646264313161323533393331 +65633665616261353938666661646130646430396464303831616232356562656666353234373662 +62396138356333663037313533656339373564353165353534303533646465646266343461643164 +38313733396631396363363635346337343831396238316131393063366361373230336331656530 +62356564383664323636366538353866326163313132323965663232663234613430653161313030 +32616336393232643262646231353732326266633361396432353538383035633662313033396636 +30363134663733383864323561386639643236656666633966623837623637383133323331336261 +65653466313639343564623231383931303735316438613230313734656436663838383336363931 +62383530363263303635636663343434383861636137633637363739313939653065303734383937 +63333264623564333161663966643632623463313766356439363764666632316130613030356133 +62353433613133353630343733316365373965393438643766303635366130313534333437306338 +34396462343536323665313735633130393634393466396236363036396665666432346335316237 +38363061343637306134333938646531346633363064333730316535306531613066313666653335 +35636264346130313762336464323833333763646662343630633562333637316233316665336361 +30333135633766303036636138656339626130353462633066333862343832303362666265383138 +32383838303133353631346234333931373163393930313938663333396234343930663430343434 +37646561623036643136656334383139336233303737666637383438353363653764393165323538 +66323566323338356439633634306130363939363039343762346665333835353463366363343166 +34656162656366376534343737656136326562393637663266356162623033393164623963306235 +33633164633861316566656337373830636433656232383462663035613162373133316363333261 +30626365376539656230656366623463626233373133626330373564313061633839386133623436 +35346339346537393262363965373862613064373361343333343138363866353661363965303363 +31633365383030653136643837306239653262313132643238656566313930663839336236316237 +36626434366265313362366431306661303339303430633235343064643064343164323630353865 +37633664616162336161653233376565646366316438303638633833663066623730326165353336 +39353530313232356163356163356131396166653331343435313464353530356539653162376663 +37633865306332303066313561313563333832376430363963343637636166323061386533663866 +33346132643365383038393964616230356539363338373665343637666465383962333662653639 +39623733346462336565353539666438356631613233616562646637323738373761663030333764 +64633230356563326535313430663863626336613264373863326539383763353062663430343833 +34343462383030333461353633333761383430393930333831303437636334363536633635383863 +35636161323862323036623932376130356438643236303361343333633363633033636539353966 +31636338346433616232343631353837623966373433333565376538303937373932646462376461 +65643764306336363736663133616432373661303237303264343862653333646236663337623662 +65363366663837663463616236616630326366613432656365663763386532663837663862636132 +35613331373236666163666134373937353531316562626135313464613366646261653266356339 +33623866633663343838626131373930636135336431353538366438363035343532376630303561 +38353232636237633438636161656138336137343531383164346264613034343763623339343837 +61633961313166303661643633356262303461626630303466316330643763333936346632313135 +62666135333163326361313035653664623066323364623966316130346463323561333533393339 +36653461323664333635343862643530363861333436313335306165373566646130393739343332 +61653564353032636237613832323961626338376237646162303638653539333163326563643163 +35306566326362353463646132373832383236616163316264386136643865646639616330656165 +32333830656333313830656162666531376637656463316336383766333666616365666138366364 +36353531613531626433643862316439306663383034346634326631323263393237643331623463 +34333630613263643864633331333638343333373933356464613331616337626136663263373538 +66653230616137396464656536373133646631396131323965326366363630343233646430656366 +30333536623661626133653932343532333236353435383430643633373933316338313764663463 +63666536386464346231393336653532353533623930396364663730353363626332626137383930 +33656363646165313761386266626565373932613939336533333036323461346331383936346639 +65653731643633303639646130313834643136383634633463373466303166353936303166333266 +33313161626436303161343139346266396161373932663732656530623762656362313037313538 +34666436643430636561323162366236363964343663663739363737343632653836633763386533 +63616238666332376161346665323236393232643837333635396639313466326534626566313639 +30353231653231303338386137646163623935313266386239643039666431323865326131623432 +33373066313261396435303331393336393332373730376239393539393030353566653965633034 +32613739636666356162323035653064323332623433393963306161353263643861613836616532 +66323061383033313131326132373730343364363764396637313738313132663535386465613839 +65383462356265656462663638623938376536313033663431393761323134666563316630393031 +38313238336132663133336338663363633863383333613731316634656134353836343765363539 +38613465376432636639623266336563633039396638636163343537376133613233633536626431 +62333663666635636431373334306332363931323931643330323838343635636333653663663863 +36353761376535653162656335393233373833653165313836633930336433663462393131326363 +36383535306534366237393764363235323662366461353235356235333033306135316635353237 +30343362646532646661383437623939363439316662643438663161356234386461376431616337 +62663630643763373563313937623764373838393535393662336162343762373833333165316336 +36393962663036393536366634326232306531316630346238613535303130383562306432346134 +31396634323134343765343230353434376663356437306231623132366130373234333438393764 +62643366306233636232333139613963323863613565623065636536616564323236656161363463 +39343164393765376530313362356238393831303239326439313838663839353832326431663637 +30666635313065343734636131663536616562343566343766336261383062646465323762353332 +65316435396230353033386131313935376233323135346630633639353163653863353933373634 +37346539363639343364316238643634616439653033383261383637326264316538356239643036 +38646434643332346561373130633132363538353935646433313333313262613363363732386130 +66353238363364323732353862303632666139343732363330373865383030343331383134356365 +31306230303237626265366436626265313662643833626534383661373638356433656234663865 +34336266633135343165656131373538613638613566396266626534313337626366343565306161 +37653764306364393632643465333033633337613965376630343934306634623433306363663939 +66396661323537333431643231306663376636613633656430616233636566636265336264386638 +35363066363465343964306134663132303461343238396232326566646263666430633538363565 +32393235313932363238383039333238333933656634613930373362646530373861326136663935 +34386263323663646538613135636432393831346265333530653531623265396164396434623136 +65343064623963323138643062366136386531313964643037626235646132346163636566373062 +62303235653066613663383635336230396237303361333933663865616139323439663935333738 +30353436633561383133613265356432633562663937376534336633353037623265663936373732 +39353366663830366138393364396632383537323361626563363834636163353833373464666366 +39343961373561613366393639343635393763643937313432363231663462363637626237373264 +37386664373432643236613237316266623765633066333462626331393635333966663065316131 +31353238633465626437313466653361633132646537363763646362303737353362386638303130 +32643431313561656539366264346131633436313637303664313636386336613932663038353632 +30333230393036663236633033363439343239336666643364383338336635313138343537383635 +62663031383266623266343462623565393262383838653039626530393864373934396533313861 +38636162303166356238303436656666356535306362303563653331383632326438336466333939 +35303233346166643466613333343165316365623264316131363765653563646134326532643461 +36343566306662663837363462316634356230373261393336623262376330393466366239323434 +30353736323262633830326339616637356636303137363030343831373139333339323530613830 +64366137396531646566373234636463636337653261336235366639343339626266396430306662 +39623835623935306661346538386239613533333362613831313561383135343638316533613536 +66316361343336306330326630363832613830363335313166626331376630636638343637303431 +38353135323165343064646664653332636436363531326362643637653966666130386430356533 +32386331303539353834613431376265326431613161623239353038336461313732653366626137 +36353462316165303163376465363333653131653138353535366334383464346630356432663966 +66653234616238356537353737626436313132333934383438636463646463383338653135313462 +34633830353430323765636439633262363232343930653336326164393665346435396565666332 +33383239343633656562376561366332326439323234656361326130396135373733643464393462 +34323032386566346565333636396435323234333830336130623233313830363837373934373033 +61643934636335633437356535623730333032343864313835646166666337326362313265633732 +36623361313934666335346537613866656535643764366663303864663263333738356630366361 +65353433663763363063386439313736613833613933326566643366313965613136303631376538 +64653432373265393634623634666233636164366336373534333039363966363233633762366361 +65303931393362323062333663323236373434626364363866636361643832343363613132386164 +31373231326235363765363635326464366264373938336533333265303436633530393632643162 +66646163373066343539636639656662656565616136333739373466313966306165353436396261 +62326532653262316234626530613362373365623731323631333035353463303466623934616362 +66316561646430366538343030386338333861626466663464386137373837316663353361646232 +36393466333839613938643066376563333263356631353233383563376462653531383730303930 +66643132373864373935626136363234326166383233373937336532643035363036626162343030 +64343864383434366164643165333132326230653964333465313763366162353536656430333761 +30313664636536363738333230636236353931653563376634326465623334386364366337336432 +31356163663936386161303734326331326636633931343430646237366661306664623931666466 +39623838393736306466353064656166363764353365336135646333373230653636613933613132 +62363332623462666139643938656533653862643234323836353265646364316431663133313863 +36356630363037666632346134393033346131616665346534303630353238616636376438316337 +33656535323062323137643635663938646238353266623635653038656434313438613533616435 +33313634303564363261333864373730616464373335663661663766373934633531373530626163 +66653062326364373631663130316333323061373338316436326436326663383863656564363038 +66316261626363336664636266343534373133663161376438613664333939386232363361643630 +66373239643234313737633337656166323533333962666633333762343435653037396364653934 +31356430333630303037303532616132663737316562336530613635613563666561373961363739 +64626266376630366561626338643436613131653736666137363762666465613465396663613030 +33386139636432623431326635336565326434346236656466653763363232366165626535626432 +31326561303561326232663833396439393439376333653162306265346234633565653639343230 +35633935646334363633633133313339393636656664316233653938333739316433 diff --git a/coreos-config/roles/compose_project/templates/prometheus/alertmanager.yml b/coreos-config/roles/compose_project/templates/prometheus/alertmanager.yml index 422572a..db2382d 100644 --- a/coreos-config/roles/compose_project/templates/prometheus/alertmanager.yml +++ b/coreos-config/roles/compose_project/templates/prometheus/alertmanager.yml @@ -7,6 +7,11 @@ route: group_interval: 5m repeat_interval: 5m receiver: 'email' + routes: + - receiver: 'deadmans-switch' + group_wait: 0s + matchers: + - alertname="PrometheusAlertmanagerE2eDeadManSwitch" receivers: - name: 'email' @@ -17,5 +22,7 @@ receivers: auth_username: '{{ prometheus.alertmanager.smtp.username }}' auth_identity: '{{ prometheus.alertmanager.smtp.username }}' auth_password: '{{ prometheus.alertmanager.smtp.password }}' - - + - name: 'deadmans-switch' + webhook_configs: + - send_resolved: false + url: "{{ prometheus.alertmanager.kumaurl }}" diff --git a/coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml b/coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml index 0c78cc7..7e50206 100644 --- a/coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml +++ b/coreos-config/roles/compose_project/templates/prometheus/docker-compose.yaml @@ -9,6 +9,7 @@ services: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z - prom_data:/prometheus - label_discovery:/label_discovery:ro + - ./rules:/rules:ro,Z labels: - "traefik.enable=true" - "traefik.http.routers.prometheus.rule=Host(`prometheus.tobiasmanske.de`)" diff --git a/coreos-config/roles/compose_project/templates/prometheus/prometheus.yml b/coreos-config/roles/compose_project/templates/prometheus/prometheus.yml index 70b20c7..bf4a278 100644 --- a/coreos-config/roles/compose_project/templates/prometheus/prometheus.yml +++ b/coreos-config/roles/compose_project/templates/prometheus/prometheus.yml @@ -12,6 +12,8 @@ alerting: scheme: http timeout: 10s api_version: v1 +rule_files: + - "/rules/*.yaml" scrape_configs: - job_name: prometheus honor_timestamps: true diff --git a/coreos-config/roles/compose_project/templates/prometheus/rules/cadvisor.yaml b/coreos-config/roles/compose_project/templates/prometheus/rules/cadvisor.yaml new file mode 100644 index 0000000..4e4512c --- /dev/null +++ b/coreos-config/roles/compose_project/templates/prometheus/rules/cadvisor.yaml @@ -0,0 +1,54 @@ +# {% raw %} + +groups: +- name: GoogleCadvisor + rules: + # - alert: ContainerKilled + # expr: 'time() - container_last_seen > 60' + # for: 0m + # labels: + # severity: warning + # annotations: + # summary: Container killed (instance {{ $labels.instance }}) + # description: "A container has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: ContainerAbsent + # expr: 'absent(container_last_seen)' + # for: 5m + # labels: + # severity: warning + # annotations: + # summary: Container absent (instance {{ $labels.instance }}) + # description: "A container is absent for 5 min\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: ContainerCpuUsage + expr: '(sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 80' + for: 2m + labels: + severity: warning + annotations: + summary: Container CPU usage (instance {{ $labels.instance }}) + description: "Container CPU usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: ContainerMemoryUsage + expr: '(sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80' + for: 2m + labels: + severity: warning + annotations: + summary: Container Memory usage (instance {{ $labels.instance }}) + description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: ContainerVolumeUsage + # expr: '(1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total) BY (instance))) * 100 > 80' + # for: 2m + # labels: + # severity: warning + # annotations: + # summary: Container Volume usage (instance {{ $labels.instance }}) + # description: "Container Volume usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: ContainerHighThrottleRate + expr: 'rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1' + for: 2m + labels: + severity: warning + annotations: + summary: Container high throttle rate (instance {{ $labels.instance }}) + description: "Container is being throttled\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" +# {% endraw %} diff --git a/coreos-config/roles/compose_project/templates/prometheus/rules/node.yaml b/coreos-config/roles/compose_project/templates/prometheus/rules/node.yaml new file mode 100644 index 0000000..2df76fc --- /dev/null +++ b/coreos-config/roles/compose_project/templates/prometheus/rules/node.yaml @@ -0,0 +1,303 @@ +# {% raw %} + +groups: +- name: NodeExporter + rules: + - alert: HostOutOfMemory + expr: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10' + for: 2m + labels: + severity: warning + annotations: + summary: Host out of memory (instance {{ $labels.instance }}) + description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostMemoryUnderMemoryPressure + expr: 'rate(node_vmstat_pgmajfault[1m]) > 1000' + for: 2m + labels: + severity: warning + annotations: + summary: Host memory under memory pressure (instance {{ $labels.instance }}) + description: "The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: HostMemoryIsUnderUtilized + # expr: '100 - (rate(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20' + # for: 1w + # labels: + # severity: info + # annotations: + # summary: Host Memory is under utilized (instance {{ $labels.instance }}) + # description: "Node memory is < 20% for 1 week. Consider reducing memory space.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualNetworkThroughputIn + expr: 'sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100' + for: 5m + labels: + severity: warning + annotations: + summary: Host unusual network throughput in (instance {{ $labels.instance }}) + description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualNetworkThroughputOut + expr: 'sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100' + for: 5m + labels: + severity: warning + annotations: + summary: Host unusual network throughput out (instance {{ $labels.instance }}) + description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualDiskReadRate + expr: 'sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50' + for: 5m + labels: + severity: warning + annotations: + summary: Host unusual disk read rate (instance {{ $labels.instance }}) + description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualDiskWriteRate + expr: 'sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50' + for: 2m + labels: + severity: warning + annotations: + summary: Host unusual disk write rate (instance {{ $labels.instance }}) + description: "Disk is probably writing too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOutOfDiskSpace + expr: '(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host out of disk space (instance {{ $labels.instance }}) + description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostDiskWillFillIn24Hours + expr: '(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host disk will fill in 24 hours (instance {{ $labels.instance }}) + description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOutOfInodes + expr: 'node_filesystem_files_free / node_filesystem_files * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host out of inodes (instance {{ $labels.instance }}) + description: "Disk is almost running out of available inodes (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostInodesWillFillIn24Hours + expr: 'node_filesystem_files_free / node_filesystem_files * 100 < 10 and predict_linear(node_filesystem_files_free[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }}) + description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualDiskReadLatency + expr: 'rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host unusual disk read latency (instance {{ $labels.instance }}) + description: "Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualDiskWriteLatency + expr: 'rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host unusual disk write latency (instance {{ $labels.instance }}) + description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostHighCpuLoad + expr: 'sum by (instance) (avg by (mode, instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8' + for: 0m + labels: + severity: warning + annotations: + summary: Host high CPU load (instance {{ $labels.instance }}) + description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: HostCpuIsUnderUtilized + # expr: '100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20' + # for: 1w + # labels: + # severity: info + # annotations: + # summary: Host CPU is under utilized (instance {{ $labels.instance }}) + # description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostCpuStealNoisyNeighbor + expr: 'avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10' + for: 0m + labels: + severity: warning + annotations: + summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }}) + description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostCpuHighIowait + expr: 'avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 10' + for: 0m + labels: + severity: warning + annotations: + summary: Host CPU high iowait (instance {{ $labels.instance }}) + description: "CPU iowait > 10%. A high iowait means that you are disk or network bound.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostUnusualDiskIo + expr: 'rate(node_disk_io_time_seconds_total[1m]) > 0.5' + for: 5m + labels: + severity: warning + annotations: + summary: Host unusual disk IO (instance {{ $labels.instance }}) + description: "Time spent in IO is too high on {{ $labels.instance }}. Check storage for issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: HostContextSwitching + # expr: '(rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1000' + # for: 0m + # labels: + # severity: warning + # annotations: + # summary: Host context switching (instance {{ $labels.instance }}) + # description: "Context switching is growing on node (> 1000 / s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostSwapIsFillingUp + expr: '(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80' + for: 2m + labels: + severity: warning + annotations: + summary: Host swap is filling up (instance {{ $labels.instance }}) + description: "Swap is filling up (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostSystemdServiceCrashed + expr: 'node_systemd_unit_state{state="failed"} == 1' + for: 0m + labels: + severity: warning + annotations: + summary: Host systemd service crashed (instance {{ $labels.instance }}) + description: "systemd service crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostPhysicalComponentTooHot + expr: 'node_hwmon_temp_celsius * ignoring(label) group_left(instance, job, node, sensor) node_hwmon_sensor_label{label!="tctl"} > 75' + for: 5m + labels: + severity: warning + annotations: + summary: Host physical component too hot (instance {{ $labels.instance }}) + description: "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostNodeOvertemperatureAlarm + expr: 'node_hwmon_temp_crit_alarm_celsius == 1' + for: 0m + labels: + severity: critical + annotations: + summary: Host node overtemperature alarm (instance {{ $labels.instance }}) + description: "Physical node temperature alarm triggered\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostRaidArrayGotInactive + expr: 'node_md_state{state="inactive"} > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Host RAID array got inactive (instance {{ $labels.instance }}) + description: "RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostRaidDiskFailure + expr: 'node_md_disks{state="failed"} > 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host RAID disk failure (instance {{ $labels.instance }}) + description: "At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostKernelVersionDeviations + expr: 'count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1' + for: 6h + labels: + severity: warning + annotations: + summary: Host kernel version deviations (instance {{ $labels.instance }}) + description: "Different kernel versions are running\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOomKillDetected + expr: 'increase(node_vmstat_oom_kill[1m]) > 0' + for: 0m + labels: + severity: warning + annotations: + summary: Host OOM kill detected (instance {{ $labels.instance }}) + description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostEdacCorrectableErrorsDetected + expr: 'increase(node_edac_correctable_errors_total[1m]) > 0' + for: 0m + labels: + severity: info + annotations: + summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }}) + description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostEdacUncorrectableErrorsDetected + expr: 'node_edac_uncorrectable_errors_total > 0' + for: 0m + labels: + severity: warning + annotations: + summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }}) + description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostNetworkReceiveErrors + expr: 'rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01' + for: 2m + labels: + severity: warning + annotations: + summary: Host Network Receive Errors (instance {{ $labels.instance }}) + description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostNetworkTransmitErrors + expr: 'rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01' + for: 2m + labels: + severity: warning + annotations: + summary: Host Network Transmit Errors (instance {{ $labels.instance }}) + description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostNetworkInterfaceSaturated + expr: '(rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000' + for: 1m + labels: + severity: warning + annotations: + summary: Host Network Interface Saturated (instance {{ $labels.instance }}) + description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostNetworkBondDegraded + expr: '(node_bonding_active - node_bonding_slaves) != 0' + for: 2m + labels: + severity: warning + annotations: + summary: Host Network Bond Degraded (instance {{ $labels.instance }}) + description: "Bond \"{{ $labels.device }}\" degraded on \"{{ $labels.instance }}\".\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostConntrackLimit + expr: 'node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8' + for: 5m + labels: + severity: warning + annotations: + summary: Host conntrack limit (instance {{ $labels.instance }}) + description: "The number of conntrack is approaching limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostClockSkew + expr: '(node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)' + for: 2m + labels: + severity: warning + annotations: + summary: Host clock skew (instance {{ $labels.instance }}) + description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostClockNotSynchronising + expr: 'min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16' + for: 2m + labels: + severity: warning + annotations: + summary: Host clock not synchronising (instance {{ $labels.instance }}) + description: "Clock not synchronising. Ensure NTP is configured on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostRequiresReboot + expr: 'node_reboot_required > 0' + for: 4h + labels: + severity: info + annotations: + summary: Host requires reboot (instance {{ $labels.instance }}) + description: "{{ $labels.instance }} requires a reboot.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + +# {% endraw %} diff --git a/coreos-config/roles/compose_project/templates/prometheus/rules/prometheus.yaml b/coreos-config/roles/compose_project/templates/prometheus/rules/prometheus.yaml new file mode 100644 index 0000000..55d1b37 --- /dev/null +++ b/coreos-config/roles/compose_project/templates/prometheus/rules/prometheus.yaml @@ -0,0 +1,231 @@ +# {% raw %} + +groups: +- name: EmbeddedExporter + rules: + - alert: PrometheusJobMissing + expr: 'absent(up{job="prometheus"})' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus job missing (instance {{ $labels.instance }}) + description: "A Prometheus job has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTargetMissing + expr: 'up == 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus target missing (instance {{ $labels.instance }}) + description: "A Prometheus target has disappeared. An exporter might be crashed.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusAllTargetsMissing + expr: 'sum by (job) (up) == 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus all targets missing (instance {{ $labels.instance }}) + description: "A Prometheus job does not have living target anymore.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTargetMissingWithWarmupTime + expr: 'sum by (instance, job) ((up == 0) * on (instance) group_right(job) (node_time_seconds - node_boot_time_seconds > 600))' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus target missing with warmup time (instance {{ $labels.instance }}) + description: "Allow a job time to start up (10 minutes) before alerting that it's down.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusConfigurationReloadFailure + expr: 'prometheus_config_last_reload_successful != 1' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus configuration reload failure (instance {{ $labels.instance }}) + description: "Prometheus configuration reload error\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTooManyRestarts + expr: 'changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus too many restarts (instance {{ $labels.instance }}) + description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: PrometheusAlertmanagerJobMissing + # expr: 'absent(up{job="alertmanager"})' + # for: 0m + # labels: + # severity: warning + # annotations: + # summary: Prometheus AlertManager job missing (instance {{ $labels.instance }}) + # description: "A Prometheus AlertManager job has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusAlertmanagerConfigurationReloadFailure + expr: 'alertmanager_config_last_reload_successful != 1' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus AlertManager configuration reload failure (instance {{ $labels.instance }}) + description: "AlertManager configuration reload error\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusAlertmanagerConfigNotSynced + expr: 'count(count_values("config_hash", alertmanager_config_hash)) > 1' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus AlertManager config not synced (instance {{ $labels.instance }}) + description: "Configurations of AlertManager cluster instances are out of sync\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusAlertmanagerE2eDeadManSwitch + expr: 'vector(1)' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus AlertManager E2E dead man switch (instance {{ $labels.instance }}) + description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusNotConnectedToAlertmanager + expr: 'prometheus_notifications_alertmanagers_discovered < 1' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }}) + description: "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusRuleEvaluationFailures + expr: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus rule evaluation failures (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTemplateTextExpansionFailures + expr: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus template text expansion failures (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} template text expansion failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusRuleEvaluationSlow + expr: 'prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds' + for: 5m + labels: + severity: warning + annotations: + summary: Prometheus rule evaluation slow (instance {{ $labels.instance }}) + description: "Prometheus rule evaluation took more time than the scheduled interval. It indicates a slower storage backend access or too complex query.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusNotificationsBacklog + expr: 'min_over_time(prometheus_notifications_queue_length[10m]) > 0' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus notifications backlog (instance {{ $labels.instance }}) + description: "The Prometheus notification queue has not been empty for 10 minutes\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusAlertmanagerNotificationFailing + expr: 'rate(alertmanager_notifications_failed_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus AlertManager notification failing (instance {{ $labels.instance }}) + description: "Alertmanager is failing sending notifications\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # - alert: PrometheusTargetEmpty + # expr: 'prometheus_sd_discovered_targets == 0' + # for: 0m + # labels: + # severity: critical + # annotations: + # summary: Prometheus target empty (instance {{ $labels.instance }}) + # description: "Prometheus has no target in service discovery\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTargetScrapingSlow + expr: 'prometheus_target_interval_length_seconds{quantile="0.9"} / on (interval, instance, job) prometheus_target_interval_length_seconds{quantile="0.5"} > 1.05' + for: 5m + labels: + severity: warning + annotations: + summary: Prometheus target scraping slow (instance {{ $labels.instance }}) + description: "Prometheus is scraping exporters slowly since it exceeded the requested interval time. Your Prometheus server is under-provisioned.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusLargeScrape + expr: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10' + for: 5m + labels: + severity: warning + annotations: + summary: Prometheus large scrape (instance {{ $labels.instance }}) + description: "Prometheus has many scrapes that exceed the sample limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTargetScrapeDuplicate + expr: 'increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus target scrape duplicate (instance {{ $labels.instance }}) + description: "Prometheus has many samples rejected due to duplicate timestamps but different values\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbCheckpointCreationFailures + expr: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB checkpoint creation failures (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} checkpoint creation failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbCheckpointDeletionFailures + expr: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB checkpoint deletion failures (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} checkpoint deletion failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbCompactionsFailed + expr: 'increase(prometheus_tsdb_compactions_failed_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB compactions failed (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} TSDB compactions failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbHeadTruncationsFailed + expr: 'increase(prometheus_tsdb_head_truncations_failed_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB head truncations failed (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} TSDB head truncation failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbReloadFailures + expr: 'increase(prometheus_tsdb_reloads_failures_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB reload failures (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} TSDB reload failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbWalCorruptions + expr: 'increase(prometheus_tsdb_wal_corruptions_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB WAL corruptions (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} TSDB WAL corruptions\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTsdbWalTruncationsFailed + expr: 'increase(prometheus_tsdb_wal_truncations_failed_total[1m]) > 0' + for: 0m + labels: + severity: critical + annotations: + summary: Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }}) + description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: PrometheusTimeserieCardinality + expr: 'label_replace(count by(__name__) ({__name__=~".+"}), "name", "$1", "__name__", "(.+)") > 10000' + for: 0m + labels: + severity: warning + annotations: + summary: Prometheus timeserie cardinality (instance {{ $labels.instance }}) + description: "The \"{{ $labels.name }}\" timeserie cardinality is getting very high: {{ $value }}\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + +# {% endraw %}