Refactor: Dir structure

tf-stage-0/.gitignore

@@ -0,0 +1,67 @@
+.envrc
+# Created by https://www.toptal.com/developers/gitignore/api/terraform,vim,nvim,ansible
+# Edit at https://www.toptal.com/developers/gitignore?templates=terraform,vim,ansible
+
+### Ansible ###
+*.retry
+
+### Terraform ###
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/terraform,vim,nvim,ansible
+artifacts/
+*.hcl

tf-stage-0/ansible.tf

@@ -0,0 +1,11 @@
+resource "local_file" "inventory" {
+  filename = "${var.files_dir}/inventory.yaml"
+  content = templatefile(
+    "${path.module}/templates/inventory.yaml.tpl",
+    {
+      server_under_test_ip       = hcloud_server.under_test.ipv4_address,
+      server_under_test_hostname = hcloud_server.under_test.name,
+      ssh_private_key_file       = local_sensitive_file.ssh_private_key.filename
+    }
+  )
+}

tf-stage-0/butane/host.nc.chaoswg.org.bu

@@ -0,0 +1,97 @@
+---
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    # Installing vim as a layered package with rpm-ostree
+    - name: rpm-ostree-install-pkg.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Layer packages with rpm-ostree
+        Wants=network-online.target
+        After=network-online.target
+        # We run before `zincati.service` to avoid conflicting rpm-ostree
+        # transactions.
+        Before=zincati.service
+        # Otherwise vagrant will try to run the playbook before we got python
+        Before=sshd.service
+        ConditionPathExists=!/var/lib/%N.stamp
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        # `--allow-inactive` ensures that rpm-ostree does not return an error
+        # if the package is already installed. This is useful if the package is
+        # added to the root image in a future Fedora CoreOS release as it will
+        # prevent the service from failing.
+        ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive vim python docker-compose borgbackup btop iftop iotop
+        ExecStart=/bin/touch /var/lib/%N.stamp
+
+        [Install]
+        WantedBy=multi-user.target
+    # Make sure docker is actually starting without a call to the socket.
+    - name: docker.service
+      enabled: true
+    - name: borgbackup.service
+      contents: |
+        [Unit]
+        Description=Run Backup of /var/lib/docker
+
+        [Service]
+        ExecStart=/usr/bin/bash /root/backup.sh
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: borgbackup.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Daily backup
+
+        [Timer]
+        OnCalendar=daily
+        Persistent=true
+
+        [Install]
+        WantedBy=timers.target
+storage:
+  filesystems:
+    - device: /dev/disk/by-partlabel/root
+      wipe_filesystem: true
+      format: ext4
+      label: root
+  files:
+    # Set vim as default editor
+    # We use `zz-` as prefix to make sure this is processed last in order to
+    # override any previously set defaults.
+    - path: /etc/profile.d/zz-default-editor.sh
+      overwrite: true
+      contents:
+        inline: |
+          export EDITOR=vim
+    - path: /etc/hostname
+      mode: 0644
+      contents:
+        inline: host.nc.chaoswg.org
+    - path: /etc/zincati/config.d/55-updates-strategy.toml
+      contents:
+        inline: |
+          [updates]
+          strategy = "periodic"
+          [[updates.periodic.window]]
+          days = [ "Fri", "Sat" ]
+          start_time = "23:30"
+          length_minutes = 60
+  links:
+    - path: /etc/localtime
+      target: /usr/share/zoneinfo/Europe/Berlin
+passwd:
+  users:
+    - name: core
+      groups:
+        - docker
+      ssh_authorized_keys:
+        - cert-authority,principals="rad4day,rad4day@chaoswg.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKUN/Ik3CqhsVLGEkl2rJLUhC0AXFmVp6BgETaqgVKq5 user-ca@chaoswg.org
+        - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhzs4vCOhy3yH2TF2bO5Qalt2P4WG4nDYTLarPKFrdM ansible@provisioner
+...

tf-stage-0/butane/infra.unruhig.eu

@@ -0,0 +1,115 @@
+---
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    # Installing vim as a layered package with rpm-ostree
+    - name: rpm-ostree-install-pkg.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Layer packages with rpm-ostree
+        Wants=network-online.target
+        After=network-online.target
+        # We run before `zincati.service` to avoid conflicting rpm-ostree
+        # transactions.
+        Before=zincati.service
+        # Otherwise vagrant will try to run the playbook before we got python
+        Before=sshd.service
+        ConditionPathExists=!/var/lib/%N.stamp
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        # `--allow-inactive` ensures that rpm-ostree does not return an error
+        # if the package is already installed. This is useful if the package is
+        # added to the root image in a future Fedora CoreOS release as it will
+        # prevent the service from failing.
+        ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive vim python docker-compose borgbackup btop iftop iotop
+        ExecStart=/bin/touch /var/lib/%N.stamp
+
+        [Install]
+        WantedBy=multi-user.target
+    # Make sure docker is actually starting without a call to the socket.
+    - name: docker.service
+      enabled: true
+    - name: borgbackup.service
+      contents: |
+        [Unit]
+        Description=Run Backup of /var/lib/docker
+
+        [Service]
+        ExecStart=/usr/bin/bash /root/backup.sh
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: borgbackup.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Daily backup
+
+        [Timer]
+        OnCalendar=daily
+        Persistent=true
+
+        [Install]
+        WantedBy=timers.target
+storage:
+  disks:
+    - device: /dev/disk/by-id/coreos-boot-disk
+      wipe_table: false
+      partitions:
+      - number: 4
+        label: root
+        size_mib: 8192
+        resize: true
+      - label: swap
+        size_mib: 3072
+      - label: var  # not specifying "number", so this will go after the root partition
+        size_mib: 0 # means "use the rest of the space on the disk"
+  filesystems:
+    - path: /var
+      device: /dev/disk/by-partlabel/var
+      format: xfs
+      wipe_filesystem: true # preserve /var on reinstall (this is the default, but be explicit)
+      with_mount_unit: true  # mount this filesystem in the real root
+    - device: /dev/disk/by-partlabel/swap
+      format: swap
+      wipe_filesystem: true # preserve /var on reinstall (this is the default, but be explicit)
+      with_mount_unit: true  # mount this filesystem in the real root
+  files:
+    # Set vim as default editor
+    # We use `zz-` as prefix to make sure this is processed last in order to
+    # override any previously set defaults.
+    - path: /etc/profile.d/zz-default-editor.sh
+      overwrite: true
+      contents:
+        inline: |
+          export EDITOR=vim
+    - path: /etc/hostname
+      mode: 0644
+      contents:
+        inline: infra.unruhig.eu
+    - path: /etc/zincati/config.d/55-updates-strategy.toml
+      contents:
+        inline: |
+          [updates]
+          strategy = "periodic"
+          [[updates.periodic.window]]
+          days = [ "Wed", "Mon" ]
+          start_time = "12:00"
+          length_minutes = 60
+  links:
+    - path: /etc/localtime
+      target: /usr/share/zoneinfo/Europe/Berlin
+passwd:
+  users:
+    - name: core
+      groups:
+        - docker
+      ssh_authorized_keys:
+        - cert-authority,principals="rad4day,rad4day@chaoswg.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKUN/Ik3CqhsVLGEkl2rJLUhC0AXFmVp6BgETaqgVKq5 user-ca@chaoswg.org
+        - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhzs4vCOhy3yH2TF2bO5Qalt2P4WG4nDYTLarPKFrdM ansible@provisioner
+...
+# vim: ft=yaml.butane

tf-stage-0/butane/mon1.hel1.chaoswg.org.bu

@@ -0,0 +1,114 @@
+---
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    # Installing vim as a layered package with rpm-ostree
+    - name: rpm-ostree-install-pkg.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Layer packages with rpm-ostree
+        Wants=network-online.target
+        After=network-online.target
+        # We run before `zincati.service` to avoid conflicting rpm-ostree
+        # transactions.
+        Before=zincati.service
+        # Otherwise vagrant will try to run the playbook before we got python
+        Before=sshd.service
+        ConditionPathExists=!/var/lib/%N.stamp
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        # `--allow-inactive` ensures that rpm-ostree does not return an error
+        # if the package is already installed. This is useful if the package is
+        # added to the root image in a future Fedora CoreOS release as it will
+        # prevent the service from failing.
+        ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive vim python docker-compose borgbackup btop iftop iotop
+        ExecStart=/bin/touch /var/lib/%N.stamp
+
+        [Install]
+        WantedBy=multi-user.target
+    # Make sure docker is actually starting without a call to the socket.
+    - name: docker.service
+      enabled: true
+    - name: borgbackup.service
+      contents: |
+        [Unit]
+        Description=Run Backup of /var/lib/docker
+
+        [Service]
+        ExecStart=/usr/bin/bash /root/backup.sh
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: borgbackup.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Daily backup
+
+        [Timer]
+        OnCalendar=daily
+        Persistent=true
+
+        [Install]
+        WantedBy=timers.target
+storage:
+  disks:
+    - device: /dev/disk/by-id/coreos-boot-disk
+      wipe_table: false
+      partitions:
+      - number: 4
+        label: root
+        size_mib: 8192
+        resize: true
+      - label: swap
+        size_mib: 3072
+      - label: var  # not specifying "number", so this will go after the root partition
+        size_mib: 0 # means "use the rest of the space on the disk"
+  filesystems:
+    - path: /var
+      device: /dev/disk/by-partlabel/var
+      format: xfs
+      wipe_filesystem: true # preserve /var on reinstall (this is the default, but be explicit)
+      with_mount_unit: true  # mount this filesystem in the real root
+    - device: /dev/disk/by-partlabel/swap
+      format: swap
+      wipe_filesystem: true # preserve /var on reinstall (this is the default, but be explicit)
+      with_mount_unit: true  # mount this filesystem in the real root
+  files:
+    # Set vim as default editor
+    # We use `zz-` as prefix to make sure this is processed last in order to
+    # override any previously set defaults.
+    - path: /etc/profile.d/zz-default-editor.sh
+      overwrite: true
+      contents:
+        inline: |
+          export EDITOR=vim
+    - path: /etc/hostname
+      mode: 0644
+      contents:
+        inline: mon1.hel1.chaoswg.org
+    - path: /etc/zincati/config.d/55-updates-strategy.toml
+      contents:
+        inline: |
+          [updates]
+          strategy = "periodic"
+          [[updates.periodic.window]]
+          days = [ "Wed", "Mon" ]
+          start_time = "23:30"
+          length_minutes = 60
+  links:
+    - path: /etc/localtime
+      target: /usr/share/zoneinfo/Europe/Berlin
+passwd:
+  users:
+    - name: core
+      groups:
+        - docker
+      ssh_authorized_keys:
+        - cert-authority,principals="rad4day,rad4day@chaoswg.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKUN/Ik3CqhsVLGEkl2rJLUhC0AXFmVp6BgETaqgVKq5 user-ca@chaoswg.org
+        - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhzs4vCOhy3yH2TF2bO5Qalt2P4WG4nDYTLarPKFrdM ansible@provisioner
+...

tf-stage-0/butane/thonkpad.ka.chaoswg.org.bu

@@ -0,0 +1,97 @@
+---
+variant: fcos
+version: 1.4.0
+systemd:
+  units:
+    # Installing vim as a layered package with rpm-ostree
+    - name: rpm-ostree-install-pkg.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Layer packages with rpm-ostree
+        Wants=network-online.target
+        After=network-online.target
+        # We run before `zincati.service` to avoid conflicting rpm-ostree
+        # transactions.
+        Before=zincati.service
+        # Otherwise vagrant will try to run the playbook before we got python
+        Before=sshd.service
+        ConditionPathExists=!/var/lib/%N.stamp
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        # `--allow-inactive` ensures that rpm-ostree does not return an error
+        # if the package is already installed. This is useful if the package is
+        # added to the root image in a future Fedora CoreOS release as it will
+        # prevent the service from failing.
+        ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive vim python docker-compose borgbackup btop iftop iotop
+        ExecStart=/bin/touch /var/lib/%N.stamp
+
+        [Install]
+        WantedBy=multi-user.target
+    # Make sure docker is actually starting without a call to the socket.
+    - name: docker.service
+      enabled: true
+    - name: borgbackup.service
+      contents: |
+        [Unit]
+        Description=Run Backup of /var/lib/docker
+
+        [Service]
+        ExecStart=/usr/bin/bash /root/backup.sh
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: borgbackup.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Daily backup
+
+        [Timer]
+        OnCalendar=daily
+        Persistent=true
+
+        [Install]
+        WantedBy=timers.target
+storage:
+  filesystems:
+    - device: /dev/disk/by-partlabel/root
+      wipe_filesystem: true
+      format: ext4
+      label: root
+  files:
+    # Set vim as default editor
+    # We use `zz-` as prefix to make sure this is processed last in order to
+    # override any previously set defaults.
+    - path: /etc/profile.d/zz-default-editor.sh
+      overwrite: true
+      contents:
+        inline: |
+          export EDITOR=vim
+    - path: /etc/hostname
+      mode: 0644
+      contents:
+        inline: thonkpad.ka.chaoswg.org
+    - path: /etc/zincati/config.d/55-updates-strategy.toml
+      contents:
+        inline: |
+          [updates]
+          strategy = "periodic"
+          [[updates.periodic.window]]
+          days = [ "Fri", "Sat" ]
+          start_time = "23:30"
+          length_minutes = 60
+  links:
+    - path: /etc/localtime
+      target: /usr/share/zoneinfo/Europe/Berlin
+passwd:
+  users:
+    - name: core
+      groups:
+        - docker
+      ssh_authorized_keys:
+        - cert-authority,principals="rad4day,rad4day@chaoswg.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKUN/Ik3CqhsVLGEkl2rJLUhC0AXFmVp6BgETaqgVKq5 user-ca@chaoswg.org
+        - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhzs4vCOhy3yH2TF2bO5Qalt2P4WG4nDYTLarPKFrdM ansible@provisioner
+...

tf-stage-0/configure.bu

@@ -0,0 +1,7 @@
+variant: fcos
+version: 1.4.0
+ignition:
+  config:
+    merge:
+      - local: ./artifacts/ansible.ign
+      - local: ./setup.ign

tf-stage-0/ignition.tf

@@ -0,0 +1,9 @@
+resource "local_file" "ignition" {
+  filename = "${var.files_dir}/ansible.ign"
+  content = templatefile(
+    "${path.module}/templates/ansible.ign.tpl",
+    {
+      ssh_public_key = chomp(one(tls_private_key.root[*].public_key_openssh))
+    }
+  )
+}

tf-stage-0/main.tf

@@ -0,0 +1,65 @@
+provider "hcloud" {
+  token = var.hcloud_token
+}
+
+resource "tls_private_key" "root" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "hcloud_ssh_key" "this" {
+  name       = var.ssh_key_name
+  public_key = one(tls_private_key.root[*].public_key_openssh)
+}
+
+resource "local_sensitive_file" "ssh_private_key" {
+  filename             = "${var.files_dir}/id_rsa"
+  file_permission      = "0600"
+  directory_permission = "0755"
+  content              = one(tls_private_key.root[*].private_key_pem)
+}
+
+resource "hcloud_server" "under_test" {
+  name   = var.hcloud_server_under_test_name
+  labels = { "os" = "coreos" }
+
+  server_type = var.hcloud_server_under_test_type
+  datacenter  = var.hcloud_server_datacenter
+
+  # Image is ignored, as we boot into rescue mode, but is a required field
+  image    = "fedora-36"
+  rescue   = "linux64"
+  ssh_keys = concat(hcloud_ssh_key.this[*].name, var.ssh_extra_key_names)
+
+
+  connection {
+    host        = hcloud_server.under_test.ipv4_address
+    timeout     = "5m"
+    private_key = file(local_sensitive_file.ssh_private_key.filename)
+    # Root is the available user in rescue mode
+    user = "root"
+  }
+
+  provisioner "local-exec" {
+    command = "butane --pretty --strict -d . configure.bu > ${var.files_dir}/configure.ign"
+  }
+
+
+  # Copy Ignition config to server
+  provisioner "file" {
+    content     = file("${var.files_dir}/configure.ign")
+    destination = "/root/setup.ign"
+  }
+
+  # Install Fedora CoreOS in rescue mode
+  provisioner "remote-exec" {
+    inline = [
+      "set -x",
+      "set -e",
+      "wget https://s3.tobiasmanske.de/public/tobias/coreos-installer-hetzner/main/coreos-installer -O /usr/local/bin/coreos-installer",
+      "chmod 755 /usr/local/bin/coreos-installer",
+      "coreos-installer install /dev/sda -i /root/setup.ign",
+      "shutdown -r now"
+    ]
+  }
+}

tf-stage-0/outputs.tf

@@ -0,0 +1,3 @@
+output "server_under_test_ip" {
+  value = hcloud_server.under_test.ipv4_address
+}

tf-stage-0/templates/ansible.ign.tpl

@@ -0,0 +1 @@
+{"ignition":{"version":"3.0.0"},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["${ssh_public_key}"]}]}}

tf-stage-0/templates/inventory.yaml.tpl

@@ -0,0 +1,15 @@
+---
+all:
+  hosts:
+    ${server_under_test_hostname}:
+      ansible_host: ${server_under_test_ip}
+      ansible_user: core
+      ansible_ssh_private_key_file: ${ssh_private_key_file}
+      network_interface: ens3
+  children:
+    testenv:
+      hosts:
+        ${server_under_test_hostname}: null
+    unprovisioned:
+      hosts:
+        ${server_under_test_hostname}: null

tf-stage-0/variables.tf

@@ -0,0 +1,48 @@
+variable "hcloud_token" {
+  description = "Hetzner Cloud API Token"
+  type        = string
+}
+
+variable "ssh_key_name" {
+  description = "Name of your public key to identify at Hetzner Cloud portal"
+  type        = string
+  default     = "restore_test_key"
+}
+
+variable "ssh_extra_key_names" {
+  description = "Name of additional public keys installed on the system"
+  type        = list(any)
+  default = [
+    "zahnrad"
+  ]
+}
+
+variable "files_dir" {
+  description = "Directory to store artifacts"
+  type        = string
+  default     = "artifacts/"
+}
+
+variable "hcloud_server_under_test_type" {
+  description = "vServer type name, lookup via `hcloud server-type list`"
+  type        = string
+  default     = "cx31"
+}
+
+variable "hcloud_server_tester_type" {
+  description = "vServer type name, lookup via `hcloud server-type list`"
+  type        = string
+  default     = "cx11"
+}
+
+variable "hcloud_server_under_test_name" {
+  description = "Name of the server"
+  type        = string
+  default     = "www1"
+}
+
+variable "hcloud_server_datacenter" {
+  description = "Desired datacenter location name, lookup via `hcloud datacenter list`"
+  type        = string
+  default     = "hel1-dc2"
+}

tf-stage-0/versions.tf

@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = ">= 1.32.0"
+    }
+
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.1.0"
+    }
+
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 3.1.0"
+    }
+
+  }
+}
+