feat: deploy flux with flux-operator

Signed-off-by: Devin Buhl <[email protected]>
onedr0p · Jan 8, 2025 · 63df4d3 · 63df4d3
1 parent 0f64b6d
commit 63df4d3
Show file tree

Hide file tree

Showing 59 changed files with 256 additions and 186 deletions.
diff --git a/.github/workflows/flux-diff.yaml b/.github/workflows/flux-diff.yaml
@@ -34,18 +34,34 @@ jobs:
           ref: "${{ github.event.repository.default_branch }}"
           path: default
 
+      - name: Create GitRepository
+        run: |
+          CONTENT=$(cat <<EOF
+          ---
+          apiVersion: source.toolkit.fluxcd.io/v1
+          kind: GitRepository
+          metadata:
+            name: flux-system
+            namespace: flux-system
+          EOF
+          )
+          echo "${CONTENT}" | \
+              tee -a \
+              ./pull/kubernetes/flux/config/cluster.yaml \
+              ./default/kubernetes/flux/config/cluster.yaml
+
       - name: Diff Resources
         uses: docker://ghcr.io/allenporter/flux-local:v7.0.0
         with:
           args: >-
             diff ${{ matrix.resources }}
             --unified 6
-            --path /github/workspace/pull/${{ matrix.paths }}/flux
-            --path-orig /github/workspace/default/${{ matrix.paths }}/flux
+            --path /github/workspace/pull/${{ matrix.paths }}/flux/config
+            --path-orig /github/workspace/default/${{ matrix.paths }}/flux/config
             --strip-attrs "helm.sh/chart,checksum/config,app.kubernetes.io/version,chart"
             --limit-bytes 10000
             --all-namespaces
-            --sources "home-kubernetes"
+            --sources "flux-system"
             --output-file diff.patch
 
       - name: Generate Diff

diff --git a/.taskfiles/bootstrap/Taskfile.yaml b/.taskfiles/bootstrap/Taskfile.yaml
@@ -32,32 +32,35 @@ tasks:
       - test -f {{.TALHELPER_CONFIG_FILE}}
       - which talhelper sops
 
-  apps:
-    desc: Bootstrap essential apps into the Talos cluster
+  core:
+    desc: Bootstrap core apps into the Talos cluster
     dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
     cmds:
       - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
-      - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
+      - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --selector tier=core --skip-diff-on-install --suppress-diff
       - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
     preconditions:
       - test -f {{.KUBECONFIG}}
       - test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml
-      - which helmfile kubectl talosctl
+      - which helmfile kubectl
 
-  flux:
-    desc: Bootstrap Flux into the Talos cluster
+  apps:
+    desc: Bootstrap apps into the Talos cluster
+    dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
     cmds:
+      - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply --filename -{{end}}'
       - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}sops exec-file {{.GITHUB_DEPLOY_KEY_FILE}} "kubectl apply --server-side --filename {}"{{end}}'
       - '{{if eq .SOPS_SECRET_EXISTS "false"}}cat {{.SOPS_AGE_KEY_FILE}} | kubectl --namespace flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin{{end}}'
-      - sops exec-file {{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml "kubectl apply --server-side --filename {}"
-      - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml
-      - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
+      - sops exec-file {{.KUBERNETES_DIR}}/flux/settings/cluster-secrets.sops.yaml "kubectl apply --server-side --filename {}"
+      - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/settings/cluster-settings.yaml
+      - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --selector tier=apps --skip-diff-on-install --suppress-diff
     vars:
       GITHUB_DEPLOY_KEY_EXISTS:
         sh: test -f {{.GITHUB_DEPLOY_KEY_FILE}} && echo true || echo false
       SOPS_SECRET_EXISTS:
         sh: kubectl --namespace flux-system get secret sops-age &>/dev/null && echo true || echo false
     preconditions:
       - test -f {{.KUBECONFIG}}
+      - test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml
       - test -f {{.SOPS_AGE_KEY_FILE}}
-      - which flux kubectl sops
+      - which helmfile kubectl sops
diff --git a/.taskfiles/kubernetes/Taskfile.yaml b/.taskfiles/kubernetes/Taskfile.yaml
@@ -6,7 +6,7 @@ tasks:
 
   reconcile:
     desc: Force Flux to pull in changes from your Git repository
-    cmd: flux --namespace flux-system reconcile kustomization cluster --with-source
+    cmd: flux --namespace flux-system reconcile kustomization flux-system --with-source
     preconditions:
       - test -f {{.KUBECONFIG}}
       - which flux

diff --git a/README.md b/README.md
@@ -104,31 +104,22 @@ There are **5 stages** outlined below for completing this project, make sure you
 > [!IMPORTANT]
 > After running the `task bootstrap:talos` command it might take a while for the cluster to be setup (10+ minutes is normal). During which time you will see a variety of error messages like: "couldn't get current server API group list," "error: no matching resources found", etc. **This is a normal.** If this step gets interrupted, e.g. by pressing <kbd>Ctrl</kbd> + <kbd>C</kbd>, you likely will need to [reset the cluster](#-reset) before trying again.
 
-1. Install Talos. This generates secrets, generates the Talos config files for your nodes and applies them to the nodes. After it has completed a `kubeconfig` will be placed in the root of your repository.
+1. Install Talos:
 
     ```sh
     task bootstrap:talos
     ```
 
-2. Install the essential cluster applications. This command will install the apps from the [helmfile](./templates/config/kubernetes/bootstrap/helmfile.yaml.j2) configuration file into your cluster.
+2. Install Prometheus CRDs, Cilium, Coredns and Spegel:
 
     ```sh
-    task bootstrap:apps
-    ```
-
-3. Verify the nodes are online and ready
-
-    ```sh
-    kubectl get nodes -o wide
-    # NAME           STATUS   ROLES                       AGE     VERSION
-    # k8s-0          Ready    control-plane,etcd,master   1h      v1.30.1
-    # k8s-1          Ready    worker                      1h      v1.30.1
+    task bootstrap:core
     ```
 
-4. Sync the cluster to the Git repository state
+3. Install Flux Operator, Flux Instance and sync your repository to your cluster:
 
     ```sh
-    task bootstrap:flux
+    task bootstrap:apps
     ```
 
 ### Stage 5: Cluster Verification

diff --git a/config.sample.yaml b/config.sample.yaml
@@ -131,7 +131,7 @@ cloudflare:
   #      https://dash.cloudflare.com/profile/api-tokens
   #   2. Under the `API Tokens` section click the blue `Create Token` button.
   #   3. Click the blue `Use template` button for the `Edit zone DNS` template.
-  #   4. Name your token something like `home-kubernetes`
+  #   4. Name your token something like `kubernetes`
   #   5. Under `Permissions`, click `+ Add More` and add each permission below:
   #      `Zone - DNS - Edit`
   #      `Account - Cloudflare Tunnel - Read`

diff --git a/templates/config/kubernetes/apps/cert-manager/cert-manager/ks.yaml.j2 b/templates/config/kubernetes/apps/cert-manager/cert-manager/ks.yaml.j2
@@ -14,7 +14,7 @@ spec:
   prune: true
   sourceRef:
     kind: GitRepository
-    name: home-kubernetes
+    name: flux-system
   wait: true
   interval: 30m
   timeout: 5m
@@ -37,7 +37,7 @@ spec:
   prune: true
   sourceRef:
     kind: GitRepository
-    name: home-kubernetes
+    name: flux-system
   wait: true
   interval: 30m
   timeout: 5m

diff --git a/templates/config/kubernetes/apps/flux-system/flux-operator/app/helm-values.yaml.j2 b/templates/config/kubernetes/apps/flux-system/flux-operator/app/helm-values.yaml.j2
@@ -0,0 +1,3 @@
+---
+serviceMonitor:
+  create: true
diff --git a/.../flux-system/flux/app/helmrelease.yaml.j2 → ...tem/flux-operator/app/helmrelease.yaml.j2 b/.../flux-system/flux/app/helmrelease.yaml.j2 → ...tem/flux-operator/app/helmrelease.yaml.j2
@@ -3,16 +3,16 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: flux
+  name: flux-operator
 spec:
   interval: 30m
   chart:
     spec:
-      chart: flux2
-      version: 2.14.0
+      chart: flux-operator
+      version: 0.12.0
       sourceRef:
         kind: HelmRepository
-        name: fluxcd-community
+        name: controlplaneio
         namespace: flux-system
   install:
     remediation:
@@ -24,4 +24,4 @@ spec:
       retries: 3
   valuesFrom:
     - kind: ConfigMap
-      name: flux-helm-values
+      name: flux-operator-helm-values
diff --git a/...lux-system/flux/app/kustomization.yaml.j2 → ...m/flux-operator/app/kustomization.yaml.j2 b/...lux-system/flux/app/kustomization.yaml.j2 → ...m/flux-operator/app/kustomization.yaml.j2
@@ -5,7 +5,7 @@ kind: Kustomization
 resources:
   - ./helmrelease.yaml
 configMapGenerator:
-  - name: flux-helm-values
+  - name: flux-operator-helm-values
     files:
       - values.yaml=./helm-values.yaml
 configurations:

diff --git a/...x-system/flux/app/kustomizeconfig.yaml.j2 → ...flux-operator/app/kustomizeconfig.yaml.j2 b/...x-system/flux/app/kustomizeconfig.yaml.j2 → ...flux-operator/app/kustomizeconfig.yaml.j2
diff --git a/templates/config/kubernetes/apps/flux-system/flux-operator/instance/helm-values.yaml.j2 b/templates/config/kubernetes/apps/flux-system/flux-operator/instance/helm-values.yaml.j2
@@ -0,0 +1,17 @@
+---
+instance:
+  cluster:
+    networkPolicy: false
+  components:
+    - source-controller
+    - kustomize-controller
+    - helm-controller
+    - notification-controller
+  sync:
+    kind: GitRepository
+    url: "#{ github.address }#"
+    ref: refs/heads/main
+    path: kubernetes/flux
+    #% if github.private_key %#
+    pullSecret: github-deploy-key
+    #% endif %#
diff --git a/templates/config/kubernetes/apps/flux-system/flux-operator/instance/helmrelease.yaml.j2 b/templates/config/kubernetes/apps/flux-system/flux-operator/instance/helmrelease.yaml.j2
@@ -0,0 +1,27 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: flux-instance
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: flux-instance
+      version: 0.12.0
+      sourceRef:
+        kind: HelmRepository
+        name: controlplaneio
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: flux-instance-helm-values
diff --git a/templates/config/kubernetes/apps/flux-system/flux-operator/instance/kustomization.yaml.j2 b/templates/config/kubernetes/apps/flux-system/flux-operator/instance/kustomization.yaml.j2
@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: flux-instance-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
diff --git a/templates/config/kubernetes/apps/flux-system/flux-operator/instance/kustomizeconfig.yaml.j2 b/templates/config/kubernetes/apps/flux-system/flux-operator/instance/kustomizeconfig.yaml.j2
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
diff --git a/...bernetes/apps/flux-system/flux/ks.yaml.j2 → ...apps/flux-system/flux-operator/ks.yaml.j2 b/...bernetes/apps/flux-system/flux/ks.yaml.j2 → ...apps/flux-system/flux-operator/ks.yaml.j2
@@ -3,18 +3,18 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app flux
+  name: &app flux-operator
   namespace: flux-system
 spec:
   targetNamespace: flux-system
   commonMetadata:
     labels:
       app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/flux-system/flux/app
+  path: ./kubernetes/apps/flux-system/flux-operator/app
   prune: false # never should be deleted
   sourceRef:
     kind: GitRepository
-    name: home-kubernetes
+    name: flux-system
   wait: false
   interval: 30m
   timeout: 5m
@@ -23,18 +23,20 @@ spec:
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app flux-github
+  name: &app flux-instance
   namespace: flux-system
 spec:
   targetNamespace: flux-system
   commonMetadata:
     labels:
       app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/flux-system/flux/github
-  prune: true
+  dependsOn:
+    - name: flux-operator
+  path: ./kubernetes/apps/flux-system/flux-operator/instance
+  prune: false # never should be deleted
   sourceRef:
     kind: GitRepository
-    name: home-kubernetes
+    name: flux-system
   wait: false
   interval: 30m
   timeout: 5m
diff --git a/templates/config/kubernetes/apps/flux-system/flux/app/helm-values.yaml.j2 b/templates/config/kubernetes/apps/flux-system/flux/app/helm-values.yaml.j2
diff --git a/...-system/flux/github/kustomization.yaml.j2 → ...x-system/github/app/kustomization.yaml.j2 b/...-system/flux/github/kustomization.yaml.j2 → ...x-system/github/app/kustomization.yaml.j2
diff --git a/...stem/flux/github/webhooks/ingress.yaml.j2 → ...ystem/github/app/webhooks/ingress.yaml.j2 b/...stem/flux/github/webhooks/ingress.yaml.j2 → ...ystem/github/app/webhooks/ingress.yaml.j2
diff --git a/...lux/github/webhooks/kustomization.yaml.j2 → ...github/app/webhooks/kustomization.yaml.j2 b/...lux/github/webhooks/kustomization.yaml.j2 → ...github/app/webhooks/kustomization.yaml.j2
diff --git a/...tem/flux/github/webhooks/receiver.yaml.j2 → ...stem/github/app/webhooks/receiver.yaml.j2 b/...tem/flux/github/webhooks/receiver.yaml.j2 → ...stem/github/app/webhooks/receiver.yaml.j2
@@ -14,7 +14,7 @@ spec:
   resources:
     - apiVersion: source.toolkit.fluxcd.io/v1
       kind: GitRepository
-      name: home-kubernetes
+      name: flux-system
       namespace: flux-system
     - apiVersion: kustomize.toolkit.fluxcd.io/v1
       kind: Kustomization

diff --git a/.../flux/github/webhooks/secret.sops.yaml.j2 → ...m/github/app/webhooks/secret.sops.yaml.j2 b/.../flux/github/webhooks/secret.sops.yaml.j2 → ...m/github/app/webhooks/secret.sops.yaml.j2
diff --git a/templates/config/kubernetes/apps/flux-system/github/ks.yaml.j2 b/templates/config/kubernetes/apps/flux-system/github/ks.yaml.j2
@@ -0,0 +1,20 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app flux-github
+  namespace: flux-system
+spec:
+  targetNamespace: flux-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/flux-system/github/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true
+  interval: 30m
+  timeout: 5m
diff --git a/templates/config/kubernetes/apps/flux-system/kustomization.yaml.j2 b/templates/config/kubernetes/apps/flux-system/kustomization.yaml.j2
@@ -4,4 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./flux/ks.yaml
+  - ./flux-operator/ks.yaml
+  - ./github/ks.yaml