From 36ece09bd20a3cd8a7b9e8e7dab0d07f4892a90e Mon Sep 17 00:00:00 2001 From: min Date: Tue, 12 Nov 2024 21:02:15 -0500 Subject: [PATCH] Move K3s to eidola --- .sops.yaml | 2 +- flake.lock | 24 ++++++------ flake.nix | 16 ++++++-- k8s/metallb/base/kustomization.yaml | 2 + .../prod/generators/secrets-generator.yaml | 7 ++++ k8s/metallb/overlays/prod/kustomization.yaml | 9 +++++ .../prod/resources/ipaddresspool.yaml | 7 ++++ .../overlays/prod/secrets/memberlist.yaml | 38 +++++++++++++++++++ nixos/hosts/eidola/disk-config.nix | 2 +- nixos/hosts/eidola/k3s.nix | 13 +++++-- nixos/hosts/eidola/nebula.nix | 6 +++ nixos/hosts/silver/configuration.nix | 1 + nixos/hosts/silver/mounts.nix | 5 --- nixos/hosts/silver/services/default.nix | 1 - nixos/hosts/silver/services/k3s.nix | 25 ------------ nixos/hosts/silver/services/nebula.nix | 6 --- nixos/keys/ssh.nix | 1 + scripts/rekey.sh | 3 ++ 18 files changed, 111 insertions(+), 57 deletions(-) create mode 100644 k8s/metallb/base/kustomization.yaml create mode 100644 k8s/metallb/overlays/prod/generators/secrets-generator.yaml create mode 100644 k8s/metallb/overlays/prod/kustomization.yaml create mode 100644 k8s/metallb/overlays/prod/resources/ipaddresspool.yaml create mode 100644 k8s/metallb/overlays/prod/secrets/memberlist.yaml delete mode 100644 nixos/hosts/silver/services/k3s.nix diff --git a/.sops.yaml b/.sops.yaml index 9ba12ef..221be7c 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -3,7 +3,7 @@ keys: - &eidola age1uqxzduupzes3tgfrrlret0n6thyldmlef60nqfzk689lmg6yayvsqpwxj6 - &silver age19yhycdgqczrvttszq97ccljh684x3r7f5dj4p0wdwqsrusqlcayse0vsh3 creation_rules: - - path_regex: k8s/apps/.*/secrets/.*\.yaml$ + - path_regex: k8s/.*/secrets/.*\.yaml$ encrypted_regex: "^(data|stringData)$" key_groups: - pgp: diff --git a/flake.lock b/flake.lock index ea8d41c..d77a1b1 100644 --- a/flake.lock +++ b/flake.lock @@ -66,11 +66,11 @@ ] }, "locked": { - "lastModified": 1730190761, - "narHash": "sha256-o5m5WzvY6cGIDupuOvjgNSS8AN6yP2iI9MtUC6q/uos=", + "lastModified": 1730751873, + "narHash": "sha256-sdY29RWz0S7VbaoTwSy6RummdHKf0wUTaBlqPxrtvmQ=", "owner": "nix-community", "repo": "disko", - "rev": "3979285062d6781525cded0f6c4ff92e71376b55", + "rev": "856a2902156ba304efebd4c1096dbf7465569454", "type": "github" }, "original": { @@ -164,11 +164,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1730327045, - "narHash": "sha256-xKel5kd1AbExymxoIfQ7pgcX6hjw9jCgbiBjiUfSVJ8=", + "lastModified": 1730602179, + "narHash": "sha256-efgLzQAWSzJuCLiCaQUCDu4NudNlHdg2NzGLX5GYaEY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "080166c15633801df010977d9d7474b4a6c549d7", + "rev": "3c2f1c4ca372622cb2f9de8016c9a0b1cbd0f37c", "type": "github" }, "original": { @@ -192,11 +192,11 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1729973466, - "narHash": "sha256-knnVBGfTCZlQgxY1SgH0vn2OyehH9ykfF8geZgS95bk=", + "lastModified": 1730602179, + "narHash": "sha256-efgLzQAWSzJuCLiCaQUCDu4NudNlHdg2NzGLX5GYaEY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd3e8833d70618c4eea8df06f95b364b016d4950", + "rev": "3c2f1c4ca372622cb2f9de8016c9a0b1cbd0f37c", "type": "github" }, "original": { @@ -226,11 +226,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1729999681, - "narHash": "sha256-qm0uCtM9bg97LeJTKQ8dqV/FvqRN+ompyW4GIJruLuw=", + "lastModified": 1730746162, + "narHash": "sha256-ZGmI+3AbT8NkDdBQujF+HIxZ+sWXuyT6X8B49etWY2g=", "owner": "Mic92", "repo": "sops-nix", - "rev": "1666d16426abe79af5c47b7c0efa82fd31bf4c56", + "rev": "59d6988329626132eaf107761643f55eb979eef1", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index c2ac07f..ec81ee9 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,18 @@ ... }: { devShells.default = pkgs.mkShell { + KUSTOMIZE_PLUGIN_HOME = pkgs.buildEnv { + name = "kustomize-plugins"; + paths = with pkgs; [ + kustomize-sops + ]; + postBuild = '' + mv $out/lib/* $out + rm -r $out/lib + ''; + pathsToLink = ["/lib"]; + }; + packages = with pkgs; [ sops ssh-to-age @@ -50,10 +62,8 @@ argocd kubectl + kustomize kubernetes-helm - minikube - cilium-cli - hubble yamllint diff --git a/k8s/metallb/base/kustomization.yaml b/k8s/metallb/base/kustomization.yaml new file mode 100644 index 0000000..3a51ea6 --- /dev/null +++ b/k8s/metallb/base/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - github.com/metallb/metallb/config/native?ref=v0.14.8 diff --git a/k8s/metallb/overlays/prod/generators/secrets-generator.yaml b/k8s/metallb/overlays/prod/generators/secrets-generator.yaml new file mode 100644 index 0000000..7b50b93 --- /dev/null +++ b/k8s/metallb/overlays/prod/generators/secrets-generator.yaml @@ -0,0 +1,7 @@ +apiVersion: viaduct.ai/v1 +kind: ksops-exec + +metadata: + name: metallb-secret-generator +files: + - ./secrets/memberlist.yaml diff --git a/k8s/metallb/overlays/prod/kustomization.yaml b/k8s/metallb/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..e255410 --- /dev/null +++ b/k8s/metallb/overlays/prod/kustomization.yaml @@ -0,0 +1,9 @@ +resources: + - ../../base +generators: + - ./generators/secrets-generator.yaml + +namespace: metallb-system + +generatorOptions: + disableNameSuffixHash: true diff --git a/k8s/metallb/overlays/prod/resources/ipaddresspool.yaml b/k8s/metallb/overlays/prod/resources/ipaddresspool.yaml new file mode 100644 index 0000000..bba6910 --- /dev/null +++ b/k8s/metallb/overlays/prod/resources/ipaddresspool.yaml @@ -0,0 +1,7 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: pool +spec: + addresses: + - 10.190.0.0/16 diff --git a/k8s/metallb/overlays/prod/secrets/memberlist.yaml b/k8s/metallb/overlays/prod/secrets/memberlist.yaml new file mode 100644 index 0000000..60a8856 --- /dev/null +++ b/k8s/metallb/overlays/prod/secrets/memberlist.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Secret +metadata: + name: memberlist + namespace: metallb-system +stringData: + secretkey: ENC[AES256_GCM,data:8nxcJ9rdL7YciYm9rhAloGFrj7vLFn70OO9t64d51W8J/Xp3S5v4bC+6IyQBkMP9aqo4MEBhPPQixD6hWtkjUw==,iv:zjv6M4tepvW5J+rt7rNwSyiOCy6nZVngB8g1bRrl3dQ=,tag:9vAehmuXBLJ4TvG6pU1Txg==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-11-05T02:07:25Z" + mac: ENC[AES256_GCM,data:nj3xo9faM/j6tlvOymQXFFrfgK3KtJxNtYa4rAFRHwFZmNk/i1luFev2wtojCoHV770EE0m6O9YUvSSi1MYYFXGV8lvgWSSOdsNb/uqMJzZ800PLczPPtK/D2SGVV503eKvRXJakadn87QSrHA/GobNPV2rF+MgebpNV+/e7+q0=,iv:0I6MB99m1Cd/9QQ+713khZoRGcAqnRAjZUjk9arfWek=,tag:K4F7ploHTgk39OpbRe9vdA==,type:str] + pgp: + - created_at: "2024-11-05T02:06:57Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAwAAAAAAAAAAAQ/7BX7Aj82Q+R0tIAsYkpdQwVAMM20afL9UTCF96E98eCXG + 5Ru16wIytCmy28jgZ4HBFEcPnBadB/kbuPzxuX2VKtL3HfBNc0akncMojcxf5fML + 7Ye37BPfUPG8AkbKf6vwfNxBZau9vK5m5a8xIZC3kenmBltVGp2oaaPIj/5kKdRX + t71fm8+xnArQQM6xH1SYXf47WiZIJjYmshG4w6kxMYQFsllyDo56ekLhTcqjv+Bf + +vn4Yznu6Aa5skkVgTrGZ0YlOcK7p3fyuMLeWxiy4VzFvS49bSlGlDEnBYL2sJqo + JPkPomj6y0BMGWczZ4va5RPyQrj86T+alLulSww2J/2gev9itu1FSpbFNoO3Yzv5 + RDOHLzXuMrJHEo/JMKwl1oMaWnNcTT0DDiSrAAdaH5hhOy9iKDbi54F+duzwZp0F + qv6jg199NrLZdviKXzOjNuNMHQHSw/tL2009Zh75WOt+1Xh+FACBW7VhlPKtC6nP + 133WhWnXROZdY6oBaCQvhMrXrf10mrsrurRhXb6bHaj9WpOdlAuPa/UYjQ5jNbno + 4e1JtV9kMT2EuTd8yhA/uT5jVEYfXtGVgwU9VrCkOSMilgltt9ASXaji+VRokaWY + bCLpdnWURQsbBVmBf2gSe+AK0kEbk1uUnwu/xdMr5e55bzKbpKvsgJqJ6i37v6zS + XgEkqgwwAQzRo9rnLLQR4bC7mu0bReqJK0Gutvsv+kR5COWak+QTmg/azxgOco2K + iMkZe1qTm85XciA22gUKrRRuoiq4bxLIyvFmIZhPvXpW2iU2y27Qdr1iMVTdE2o= + =N3f8 + -----END PGP MESSAGE----- + fp: 78795D9EBD425CBB3E850BC45DF91852CB14CEFF + encrypted_regex: ^(data|stringData)$ + version: 3.8.1 diff --git a/nixos/hosts/eidola/disk-config.nix b/nixos/hosts/eidola/disk-config.nix index 44467ec..2be70fb 100644 --- a/nixos/hosts/eidola/disk-config.nix +++ b/nixos/hosts/eidola/disk-config.nix @@ -53,7 +53,7 @@ fsType = "tmpfs"; mountOptions = [ "defaults" - "size=8G" + "size=16G" "mode=755" ]; }; diff --git a/nixos/hosts/eidola/k3s.nix b/nixos/hosts/eidola/k3s.nix index 7dd14ed..02b0354 100644 --- a/nixos/hosts/eidola/k3s.nix +++ b/nixos/hosts/eidola/k3s.nix @@ -1,4 +1,4 @@ -{config, ...}: { +{config, lib, ...}: { sops.secrets."k3s-token" = { sopsFile = ../../../secrets/k3s-token.txt; format = "binary"; @@ -6,8 +6,15 @@ services.k3s = { enable = true; - role = "agent"; - serverAddr = "https://silver.int.min.rip:6443"; + role = "server"; + clusterInit = true; tokenFile = config.sops.secrets."k3s-token".path; + + extraFlags = lib.concatStringsSep " " [ + "--disable=servicelb" + "--disable=local-storage" + "--disable=helm-controller" + "--tls-san=k8s.int.min.rip" + ]; }; } diff --git a/nixos/hosts/eidola/nebula.nix b/nixos/hosts/eidola/nebula.nix index b221a35..697b874 100644 --- a/nixos/hosts/eidola/nebula.nix +++ b/nixos/hosts/eidola/nebula.nix @@ -52,6 +52,12 @@ in { proto = "tcp"; host = "any"; } + # Allow `kube-apiserver` from anyone + { + port = 6443; + proto = "tcp"; + host = "any"; + } ]; }; } diff --git a/nixos/hosts/silver/configuration.nix b/nixos/hosts/silver/configuration.nix index 60ee786..ecbc349 100644 --- a/nixos/hosts/silver/configuration.nix +++ b/nixos/hosts/silver/configuration.nix @@ -75,6 +75,7 @@ in { vim fastfetch btop + tmux speedtest-cli ]; environment.variables.EDITOR = "vim"; diff --git a/nixos/hosts/silver/mounts.nix b/nixos/hosts/silver/mounts.nix index 514e076..81203a6 100644 --- a/nixos/hosts/silver/mounts.nix +++ b/nixos/hosts/silver/mounts.nix @@ -15,11 +15,6 @@ "/var/lib/acme" "/srv" - - "/var/lib/rancher/k3s" - "/var/lib/kubelet" - "/var/lib/cni" - "/var/lib/containerd" ]; files = [ "/etc/machine-id" diff --git a/nixos/hosts/silver/services/default.nix b/nixos/hosts/silver/services/default.nix index 55dbf35..1850d78 100644 --- a/nixos/hosts/silver/services/default.nix +++ b/nixos/hosts/silver/services/default.nix @@ -7,7 +7,6 @@ ./gitea.nix ./synapse.nix ./nebula.nix - ./k3s.nix ]; security.acme = { diff --git a/nixos/hosts/silver/services/k3s.nix b/nixos/hosts/silver/services/k3s.nix deleted file mode 100644 index fa11a15..0000000 --- a/nixos/hosts/silver/services/k3s.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ - config, - lib, - ... -}: { - sops.secrets."k3s-token" = { - sopsFile = ../../../../secrets/k3s-token.txt; - format = "binary"; - }; - - services.k3s = { - enable = true; - role = "server"; - clusterInit = true; - tokenFile = config.sops.secrets."k3s-token".path; - - extraFlags = lib.concatStringsSep " " [ - "--disable=traefik" - "--disable=servicelb" - "--disable=local-storage" - "--disable=helm-controller" - "--tls-san=silver.int.min.rip" - ]; - }; -} diff --git a/nixos/hosts/silver/services/nebula.nix b/nixos/hosts/silver/services/nebula.nix index b946a5d..a1d2b78 100644 --- a/nixos/hosts/silver/services/nebula.nix +++ b/nixos/hosts/silver/services/nebula.nix @@ -55,12 +55,6 @@ in { proto = "tcp"; host = "any"; } - # Allow `kube-apiserver` from anyone - { - port = 6443; - proto = "tcp"; - host = "any"; - } ]; }; } diff --git a/nixos/keys/ssh.nix b/nixos/keys/ssh.nix index c628a29..ac154b1 100644 --- a/nixos/keys/ssh.nix +++ b/nixos/keys/ssh.nix @@ -1,4 +1,5 @@ [ "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBCZ7P/hl8DOMyTm5vGZuMrxBeSr2bmN2tp8zeiK+y/zq/fOi4rMIbfQif8KmaZ2UDTnpWj8DNfrPhfz6li1nzU=" "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIPci/gIUGWdoiLXS8Nq8T6Fvh2Wtpxv6pnqyvbSWvzyoAAAABHNzaDo=" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINIysEjWk8jdsnfF2Ki1U1TENkRLu3ig5tGVlVUnBGTj" ] diff --git a/scripts/rekey.sh b/scripts/rekey.sh index 58329f0..45433f1 100755 --- a/scripts/rekey.sh +++ b/scripts/rekey.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +shopt -s globstar + SCRIPT_DIR="$(dirname "$0")" ROOT_DIR="$(realpath "$SCRIPT_DIR/..")" @@ -10,3 +12,4 @@ rekey_dir() { } rekey_dir "secrets/*" +rekey_dir "k8s/**/secrets/*.yaml"