diff --git a/module/api/v1alpha1/remoteassemblage_types.go b/module/api/v1alpha1/remoteassemblage_types.go index ee24487..03ed716 100644 --- a/module/api/v1alpha1/remoteassemblage_types.go +++ b/module/api/v1alpha1/remoteassemblage_types.go @@ -7,7 +7,6 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - asmv1 "github.com/squaremo/fleeet/assemblage/api/v1alpha1" syncapi "github.com/squaremo/fleeet/pkg/api" ) @@ -18,11 +17,42 @@ type RemoteAssemblageSpec struct { // +required KubeconfigRef LocalKubeconfigReference `json:"kubeconfigRef"` - // Assemblage gives the specification for the assemblage to create - // downstream. It will be created with the same name as this - // object. + // Syncs gives the list of sync specs, each specifying a config to apply to the remote cluster. + // +optional + Syncs []RemoteSync `json:"syncs,omitempty"` +} + +// SourceReference is a reference to supply to the Flux sync primitive created. Sources are shared +// amongst assemblages, rather than created per assemblage. +type SourceReference struct { + // Name gives the name of the source (which is assumed to be in the same namespace as the + // referrer). + // +required + Name string `json:"name"` + // APIVersion gives the API group and version of the source object, e.g., + // `source.toolkit.fluxcd.io/v1beta2` + // +required + APIVersion string `json:"apiVersion"` + // Kind gives the kind of the source object, e.g., `GitRepository` + // +required + Kind string `json:"kind"` +} + +type RemoteSync struct { + // Name gives a name to use for this sync, so that updates can be stable (changing the sync spec + // will update objects rather than replace them) + // +required + Name string `json:"name"` + // ControlPlaneBindings gives a list of variable bindings to evaluate when constructing the sync primitives + // +optional + ControlPlaneBindings []syncapi.Binding `json:"controlPlaneBindings,omitempty"` + // SourceRef gives a reference to the source to use in the sync primitive + // +required + SourceRef SourceReference `json:"sourceRef"` + // Package defines how the sources is to be applied; e.g., by kustomize // +required - Assemblage asmv1.AssemblageSpec `json:"assemblage"` + // +kubebuilder:default={"kustomize": {"path": "."}} + Package *syncapi.PackageSpec `json:"package,omitempty"` } type LocalKubeconfigReference struct { diff --git a/module/api/v1alpha1/zz_generated.deepcopy.go b/module/api/v1alpha1/zz_generated.deepcopy.go index 224d701..528439e 100644 --- a/module/api/v1alpha1/zz_generated.deepcopy.go +++ b/module/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -414,7 +415,13 @@ func (in *RemoteAssemblageList) DeepCopyObject() runtime.Object { func (in *RemoteAssemblageSpec) DeepCopyInto(out *RemoteAssemblageSpec) { *out = *in out.KubeconfigRef = in.KubeconfigRef - in.Assemblage.DeepCopyInto(&out.Assemblage) + if in.Syncs != nil { + in, out := &in.Syncs, &out.Syncs + *out = make([]RemoteSync, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteAssemblageSpec. @@ -449,6 +456,49 @@ func (in *RemoteAssemblageStatus) DeepCopy() *RemoteAssemblageStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteSync) DeepCopyInto(out *RemoteSync) { + *out = *in + if in.ControlPlaneBindings != nil { + in, out := &in.ControlPlaneBindings, &out.ControlPlaneBindings + *out = make([]api.Binding, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.SourceRef = in.SourceRef + if in.Package != nil { + in, out := &in.Package, &out.Package + *out = new(api.PackageSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteSync. +func (in *RemoteSync) DeepCopy() *RemoteSync { + if in == nil { + return nil + } + out := new(RemoteSync) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SourceReference) DeepCopyInto(out *SourceReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceReference. +func (in *SourceReference) DeepCopy() *SourceReference { + if in == nil { + return nil + } + out := new(SourceReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SyncSummary) DeepCopyInto(out *SyncSummary) { *out = *in diff --git a/module/config/crd/bases/fleet.squaremo.dev_remoteassemblages.yaml b/module/config/crd/bases/fleet.squaremo.dev_remoteassemblages.yaml index 23c820a..02442cd 100644 --- a/module/config/crd/bases/fleet.squaremo.dev_remoteassemblages.yaml +++ b/module/config/crd/bases/fleet.squaremo.dev_remoteassemblages.yaml @@ -36,130 +36,112 @@ spec: spec: description: RemoteAssemblageSpec defines the desired state of RemoteAssemblage properties: - assemblage: - description: Assemblage gives the specification for the assemblage - to create downstream. It will be created with the same name as this - object. + kubeconfigRef: + description: KubeconfigRef refers to a secret with a kubeconfig for + the remote cluster. properties: - syncs: - items: - description: NamedSync is used when there's a list of syncs, - so the name can be mentioned elsewhere to refer to the particular - sync. These always have their own bindings because they are - used in types that have bindings to evaluate in the target - cluster. - properties: - bindings: - description: Bindings gives a list of variable bindings - to use when evaluating the package spec in the sync - items: - description: Binding specifies how to obtain a value to - bind to a name. The name can then be mentioned elsewhere - in an object, and be replaced with the value as evaluated. + name: + description: Name gives the name of the secret containing a kubeconfig. + type: string + required: + - name + type: object + syncs: + description: Syncs gives the list of sync specs, each specifying a + config to apply to the remote cluster. + items: + properties: + controlPlaneBindings: + description: ControlPlaneBindings gives a list of variable bindings + to evaluate when constructing the sync primitives + items: + description: Binding specifies how to obtain a value to bind + to a name. The name can then be mentioned elsewhere in an + object, and be replaced with the value as evaluated. + properties: + name: + type: string + objectFieldRef: properties: - name: + apiVersion: + description: APIVersion gives the APIVersion (/) + for the object's type type: string - objectFieldRef: - properties: - apiVersion: - description: APIVersion gives the APIVersion (/) - for the object's type - type: string - fieldPath: - description: Path is a JSONPointer expression - for finding the value in the object identified - type: string - kind: - description: Kind gives the kind of the object's - type - type: string - name: - description: Name names the object - type: string - required: - - fieldPath - - kind - - name - type: object - value: + fieldPath: + description: Path is a JSONPointer expression for + finding the value in the object identified + type: string + kind: + description: Kind gives the kind of the object's type + type: string + name: + description: Name names the object type: string required: + - fieldPath + - kind - name type: object - type: array - name: - description: Name gives the sync a name so it can be correlated - to the status - type: string - package: - default: - kustomize: - path: . - description: Package defines how to deal with the configuration - at the source, e.g., if it's a kustomization (or YAML - files) - properties: - kustomize: - properties: - path: - default: . - description: Path gives the path within the source - to treat as the Kustomization root. - type: string - substitute: - additionalProperties: - type: string - description: Substitute gives a map of names to - values to substitute in the YAML built from the - kustomization. - type: object - type: object - type: object - source: - description: Source gives the specification for how to get - the configuration to be synced + value: + type: string + required: + - name + type: object + type: array + name: + description: Name gives a name to use for this sync, so that + updates can be stable (changing the sync spec will update + objects rather than replace them) + type: string + package: + default: + kustomize: + path: . + description: Package defines how the sources is to be applied; + e.g., by kustomize + properties: + kustomize: properties: - git: - properties: - url: - description: URL gives the URL for the git repository - type: string - version: - description: Version gives either the revision or - tag at which to get the git repo - properties: - revision: - type: string - tag: - type: string - type: object - required: - - url - - version + path: + default: . + description: Path gives the path within the source to + treat as the Kustomization root. + type: string + substitute: + additionalProperties: + type: string + description: Substitute gives a map of names to values + to substitute in the YAML built from the kustomization. type: object - required: - - git type: object + type: object + sourceRef: + description: SourceRef gives a reference to the source to use + in the sync primitive + properties: + apiVersion: + description: APIVersion gives the API group and version + of the source object, e.g., `source.toolkit.fluxcd.io/v1beta2` + type: string + kind: + description: Kind gives the kind of the source object, e.g., + `GitRepository` + type: string + name: + description: Name gives the name of the source (which is + assumed to be in the same namespace as the referrer). + type: string required: + - apiVersion + - kind - name - - source type: object - type: array - required: - - syncs - type: object - kubeconfigRef: - description: KubeconfigRef refers to a secret with a kubeconfig for - the remote cluster. - properties: - name: - description: Name gives the name of the secret containing a kubeconfig. - type: string - required: - - name - type: object + required: + - name + - sourceRef + type: object + type: array required: - - assemblage - kubeconfigRef type: object status: diff --git a/module/controllers/bootstrap_test.go b/module/controllers/bootstrap_test.go index 1227efc..696406a 100644 --- a/module/controllers/bootstrap_test.go +++ b/module/controllers/bootstrap_test.go @@ -6,7 +6,7 @@ package controllers import ( "context" - "fmt" + //"fmt" // "path/filepath" // "time" @@ -24,8 +24,8 @@ import ( // "sigs.k8s.io/controller-runtime/pkg/client/apiutil" // ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - kustomv1 "github.com/fluxcd/kustomize-controller/api/v1beta1" - meta "github.com/fluxcd/pkg/apis/meta" + //kustomv1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + //meta "github.com/fluxcd/pkg/apis/meta" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" fleetv1 "github.com/squaremo/fleeet/module/api/v1alpha1" @@ -169,8 +169,8 @@ var _ = Describe("bootstrap module controller", func() { ) BeforeEach(func() { - // Create clusters so I can check e.g., that there's a - // Kustomization per cluster, targeting the cluster. + // Create clusters so I can check e.g., that there's a RemoteAssemblage per cluster, + // targeting the cluster. clusters = make(map[string]*clusterv1.Cluster) for i := 0; i < 3; i++ { cluster := &clusterv1.Cluster{} @@ -182,7 +182,7 @@ var _ = Describe("bootstrap module controller", func() { }) - It("creates source", func() { + It("creates a source per module", func() { // Check there's a GitRepository created for the module var src sourcev1.GitRepository Eventually(func() bool { @@ -196,63 +196,29 @@ var _ = Describe("bootstrap module controller", func() { Expect(metav1.IsControlledBy(&src, &mod)).To(BeTrue()) }) - It("expands to a kustomization per cluster", func() { - var kustoms kustomv1.KustomizationList + It("creates a RemoteAssemblage per cluster", func() { + expectedClusters := make(map[string]*clusterv1.Cluster) + for n, v := range clusters { + expectedClusters[n] = v + } + + var asms fleetv1.RemoteAssemblageList Eventually(func() bool { - if err := k8sClient.List(context.TODO(), &kustoms, client.InNamespace(namespace.Name)); err != nil { + if err := k8sClient.List(context.TODO(), &asms, client.InNamespace(namespace.Name)); err != nil { return false } - return len(kustoms.Items) >= len(clusters) + return len(asms.Items) >= len(clusters) }, "5s", "1s").Should(BeTrue()) - Expect(len(kustoms.Items)).To(Equal(len(clusters))) - - // Check each Kustomization is controlled by the module, owned - // by a cluster, targets the cluster, and has the - // BootstrapModule sync with expanded bindings. - - for _, kustom := range kustoms.Items { - // the kustomization spec is what the module says - Expect(kustom.Spec.SourceRef.Kind).To(Equal("GitRepository")) - Expect(kustom.Spec.SourceRef.Name).To(Equal(mod.Name)) // == src.Name - Expect(kustom.Spec.Path).To(Equal(mod.Spec.Sync.Package.Kustomize.Path)) - - // the module owns the ksutomization - controller := metav1.GetControllerOf(&kustom) - Expect(controller).NotTo(BeNil()) - Expect(controller.Kind).To(Equal("BootstrapModule")) - Expect(controller.Name).To(Equal(mod.Name)) + Expect(len(asms.Items)).To(Equal(len(clusters))) - var clusterName string + for _, asm := range asms.Items { + clusterName := asm.Spec.KubeconfigRef.Name[:len(asm.Spec.KubeconfigRef.Name)-len("-kubeconfig")] + Expect(expectedClusters).To(HaveKey(clusterName)) + delete(expectedClusters, clusterName) - // one cluster owns the kustomization, and that cluster is - // targeted by the kubeconfig - ownersThatAreCluster := 0 - for _, owner := range kustom.GetOwnerReferences() { - if owner.Kind == "Cluster" { - Expect(clusters).To(HaveKey(owner.Name)) - Expect(kustom.Spec.KubeConfig).To(Equal(&kustomv1.KubeConfig{ - SecretRef: meta.LocalObjectReference{ - Name: fmt.Sprintf("%s-kubeconfig", owner.Name), - }, - })) - ownersThatAreCluster++ - clusterName = owner.Name - // remove from consideration - delete(clusters, owner.Name) - } - } - Expect(ownersThatAreCluster).To(Equal(1)) - Expect(clusterName).ToNot(BeEmpty()) - - // bindings are expanded - Expect(kustom.Spec.PostBuild).NotTo(BeNil()) - Expect(kustom.Spec.PostBuild.Substitute).NotTo(BeNil()) - substitutions := kustom.Spec.PostBuild.Substitute - Expect(substitutions).To(Equal(map[string]string{ - "cluster.name": clusterName, - })) + // TODO check properties of assemblage: has the expected syncs, owner } - // All the clusters were accounted for. - Expect(clusters).To(BeEmpty()) + Expect(expectedClusters).To(BeEmpty()) }) + }) diff --git a/module/controllers/bootstrapmodule_controller.go b/module/controllers/bootstrapmodule_controller.go index c2e882e..99d314c 100644 --- a/module/controllers/bootstrapmodule_controller.go +++ b/module/controllers/bootstrapmodule_controller.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Michael Bridgen . +Copyright 2021, 2022 Michael Bridgen . */ package controllers @@ -20,7 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - kustomv1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + //kustomv1 "github.com/fluxcd/kustomize-controller/api/v1beta1" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" @@ -44,8 +44,9 @@ type BootstrapModuleReconciler struct { //+kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=kustomize.toolkit.fluxcd.io,resources=kustomizations,verbs=get;list;watch;create;update;patch;delete -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. +// Reconcile moves the cluster closer to the desired state, for a particular +// BootstrapModule. Usually this means making sure each selected cluster has a remote assemblage +// containing the sync given by the module, referring to a source (GitRepository). func (r *BootstrapModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("bootstrapmodule", req.NamespacedName) @@ -56,14 +57,7 @@ func (r *BootstrapModuleReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.V(1).Info("found BootstrapModule") - // The job of this controller is to make sure each eligible - // cluster has a sync primitive targeting it. That means, in - // GitOps Toolkit terms, there is a Kustomization object using the - // cluster kubeconfig secret for each cluster, and a GitRepository - // for them to all use as a source. - - // Create (or update) a source at which to point the - // kustomizations. + // Create (or update) a source at which to point the syncs. var source sourcev1.GitRepository source.Namespace = mod.Namespace @@ -82,7 +76,9 @@ func (r *BootstrapModuleReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.Info("created/updated GitRepository", "name", source.Name, "operation", op) // TODO set a condition saying the source is created - // For each eligible cluster, create a kustomization + // For each eligible cluster, ensure there's a RemoteAssemblage + + // Find all the selected clusters var clusters clusterv1.ClusterList selector, err := metav1.LabelSelectorAsSelector(mod.Spec.Selector) if err != nil { @@ -95,81 +91,51 @@ func (r *BootstrapModuleReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, fmt.Errorf("failed to list selected clusters: %w", err) } - namespacedClient := client.NewNamespacedClient(r.Client, mod.Namespace) for _, cluster := range clusters.Items { - // start with CLUSTER_NAME available to use in bindings - memo := map[string]string{ - "CLUSTER_NAME": cluster.Name, - } - - var bindingErr error - var makeBindingFunc func(stack []string) func(string) string - makeBindingFunc = func(stack []string) func(string) string { - return func(name string) string { - for i := range stack { - if stack[i] == name { - bindingErr = fmt.Errorf("circular binding %q", name) - return "" - } - } - - if v, ok := memo[name]; ok { - return v - } - for _, b := range mod.Spec.ControlPlaneBindings { - if b.Name == name { - v, err := syncapi.ResolveBinding(ctx, namespacedClient, b, makeBindingFunc(append(stack, name))) - if err != nil { - bindingErr = err - v = "" - } - memo[name] = v - return v - } - } - memo[name] = "" - return "" - } - } - - kustomSpec, err := syncapi.KustomizationSpecFromPackage(mod.Spec.Sync.Package, source.GetName(), makeBindingFunc(nil)) - if err != nil { - return ctrl.Result{}, err - } - if bindingErr != nil { - return ctrl.Result{}, bindingErr - } - - var kustom kustomv1.Kustomization - kustom.Namespace = mod.GetNamespace() - kustom.Name = fmt.Sprintf("%s-%s", mod.GetName(), cluster.GetName()) - op, err := controllerutil.CreateOrUpdate(ctx, r.Client, &kustom, func() error { - kustom.Spec = kustomSpec - // each kustomization is controlled by the bootstrap - // module; if the module goes, so does the kustomization - if err := controllerutil.SetControllerReference(&mod, &kustom, r.Scheme); err != nil { + asm := &fleetv1.RemoteAssemblage{} + asm.Namespace = cluster.GetNamespace() + asm.Name = cluster.GetName() + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, asm, func() error { + // Each ProxyAssemblage is owned by each of the modules + // assigned to it. This is for the sake of indexing. + if err := controllerutil.SetOwnerReference(&mod, asm, r.Scheme); err != nil { return err } - // each kustomization is also owned by the cluster it - // targets, for the sake of good bookkeeping (and - // indexing) - if err := controllerutil.SetOwnerReference(&cluster, &kustom, r.Scheme); err != nil { + // Each ProxyAssemblage is _specially_ owned by the + // cluster to which it pertains. This is so that removing + // the cluster will garbage collect the remote assemblage. + if err := controllerutil.SetControllerReference(&cluster, asm, r.Scheme); err != nil { return err } - - kustom.Spec.KubeConfig = &kustomv1.KubeConfig{} - kustom.Spec.KubeConfig.SecretRef.Name = fmt.Sprintf("%s-kubeconfig", cluster.GetName()) - + asm.Spec.KubeconfigRef = fleetv1.LocalKubeconfigReference{ + Name: cluster.GetName() + "-kubeconfig", // FIXME refer to cluster instead? + } + syncs := asm.Spec.Syncs + for i, sync := range syncs { + if sync.Name == mod.Name { + // NB: CreateOrUpdate will avoid the update if the mutated object + // is deep-equal to the original. That helps this process reach a + // fixed point. + syncs[i].Package = mod.Spec.Sync.Package + syncs[i].ControlPlaneBindings = mod.Spec.ControlPlaneBindings + return nil + } + } + // not there -- add this module + asm.Spec.Syncs = append(syncs, fleetv1.RemoteSync{ + Name: mod.Name, + Package: mod.Spec.Sync.Package, + ControlPlaneBindings: mod.Spec.ControlPlaneBindings, + }) return nil }) - if err != nil { return ctrl.Result{}, err } - - log.V(1).Info("created/updated kustomization", "name", kustom.GetName(), "operation", op) + log.V(1).Info("created/updated RemoteAssemblage", "name", asm.Name, "operation", op) } - // TODO find any rogue kustomizations and delete them + // TODO find any redundant sources and assemblages and delete them return ctrl.Result{}, nil } diff --git a/module/controllers/remoteasm_test.go b/module/controllers/remoteasm_test.go new file mode 100644 index 0000000..6264be9 --- /dev/null +++ b/module/controllers/remoteasm_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2021, 2022 Michael Bridgen . +*/ + +package controllers + +import ( + "context" + // "path/filepath" + // "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + // corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + // "k8s.io/apimachinery/pkg/runtime" + //"k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + //clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + // "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + // ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kustomv1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + // meta "github.com/fluxcd/pkg/apis/meta" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" + + fleetv1 "github.com/squaremo/fleeet/module/api/v1alpha1" + syncapi "github.com/squaremo/fleeet/pkg/api" +) + +var _ = Describe("remote assemblage controller", func() { + + var ( + manager ctrl.Manager + stopManager func() + managerDone chan struct{} + ) + + BeforeEach(func() { + By("starting a controller manager") + var err error + manager, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).ToNot(HaveOccurred()) + + remoteasmReconciler := &RemoteAssemblageReconciler{ + Client: manager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("BootstrapModule"), + Scheme: manager.GetScheme(), + } + Expect(remoteasmReconciler.SetupWithManager(manager)).To(Succeed()) + + var ctx context.Context + ctx, stopManager = context.WithCancel(signalHandler) + managerDone = make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(manager.Start(ctx)).To(Succeed()) + close(managerDone) + }() + }) + + AfterEach(func() { + stopManager() + <-managerDone + }) + + var ( + namespace corev1.Namespace + clusterName string + clusterSecretName string + source1 sourcev1.GitRepository + source2 sourcev1.GitRepository + asm fleetv1.RemoteAssemblage + syncs []fleetv1.RemoteSync + ) + + BeforeEach(func() { + // this doesn't get used, it just needs to be supplied + clusterName = "cluster-" + randString(8) + clusterSecretName = clusterName + "-kubeconfig" // NB this coincides with the hack used to elide between clusters and their secrets, which itself corresponds to what ClusterAPI does + + namespace = corev1.Namespace{} + namespace.Name = randString(5) + Expect(k8sClient.Create(context.TODO(), &namespace)).To(Succeed()) + + // create a git repository for eah sync to refer to; this would usually be done by whatever + // creates the assemblage (e.g., the bootstrap module controller) + source1 = sourcev1.GitRepository{} + source1.Name = "src-" + randString(5) + source1.Namespace = namespace.Name + source1.Spec.URL = "ssh://git@github.com/cuttlefacts/cuttlefacts-app" + Expect(k8sClient.Create(context.TODO(), &source1)).To(Succeed()) + + source2 = sourcev1.GitRepository{} + source2.Name = "src-" + randString(5) + source2.Namespace = namespace.Name + source2.Spec.URL = "ssh://git@github.com/cuttlefacts/cuttlefacts-platform" + Expect(k8sClient.Create(context.TODO(), &source2)).To(Succeed()) + + syncs = []fleetv1.RemoteSync{ + { + Name: "mostly-default", + // TODO: control plane bindings + SourceRef: fleetv1.SourceReference{ + Name: source2.Name, + APIVersion: "source.toolkit.fluxcd.io/v1beta1", // <-- could be constructed from imported vars + Kind: "GitRepository", + }, + // .package left to default + }, + { + Name: "with-package-and-bindings", + ControlPlaneBindings: []syncapi.Binding{ + { + Name: "cluster.name", + BindingSource: syncapi.BindingSource{ + StringValue: &syncapi.StringValue{ + Value: "$(CLUSTER_NAME)", + }, + }, + }, + }, + SourceRef: fleetv1.SourceReference{ + Name: source2.Name, + APIVersion: "source.toolkit.fluxcd.io/v1beta1", // <-- as above + Kind: "GitRepository", + }, + Package: &syncapi.PackageSpec{ + Kustomize: &syncapi.KustomizeSpec{ + Path: "./app", + Substitute: map[string]string{ + "foo": "${cluster.name}-foo", + }, + }, + }, + }, + } + + // create an assemblage to serve as the input + asm = fleetv1.RemoteAssemblage{ + Spec: fleetv1.RemoteAssemblageSpec{ + KubeconfigRef: fleetv1.LocalKubeconfigReference{Name: clusterSecretName}, + Syncs: syncs, + }, + } + asm.Name = "asm-" + randString(5) + asm.Namespace = namespace.Name + Expect(k8sClient.Create(context.TODO(), &asm)).To(Succeed()) + }) + + It("creates a kustomization per sync", func() { + var kustoms kustomv1.KustomizationList + Eventually(func() bool { + if err := k8sClient.List(context.TODO(), &kustoms, client.InNamespace(namespace.Name)); err != nil { + return false + } + return len(kustoms.Items) >= len(syncs) + }, "5s", "1s").Should(BeTrue()) + Expect(len(kustoms.Items)).To(Equal(len(syncs))) + + // Check each Kustomization is controller-owned by the assemblage, targets the cluster, and + // has the specified sync with expanded bindings. + + for _, kustom := range kustoms.Items { + // the kustomization spec is what the module says + Expect(kustom.Spec.SourceRef.Kind).To(Equal("GitRepository")) + // the assemblage owns the kustomization + controller := metav1.GetControllerOf(&kustom) + Expect(controller).NotTo(BeNil()) + Expect(controller.Kind).To(Equal("RemoteAssemblage")) + Expect(controller.Name).To(Equal(asm.Name)) + + Expect(kustom.Spec.KubeConfig).ToNot(BeNil()) + Expect(kustom.Spec.KubeConfig.SecretRef.Name).To(Equal(clusterSecretName)) + + // bindings are expanded, where they exist in the original sync + if kustom.Spec.SourceRef.Name == source1.Name { + Expect(kustom.Spec.PostBuild).NotTo(BeNil()) + Expect(kustom.Spec.PostBuild.Substitute).NotTo(BeNil()) + substitutions := kustom.Spec.PostBuild.Substitute + Expect(substitutions).To(Equal(map[string]string{ + "cluster.name": clusterName, + })) + } + } + }) +}) diff --git a/module/controllers/remoteassemblage_controller.go b/module/controllers/remoteassemblage_controller.go index 8fd6378..1ba11ca 100644 --- a/module/controllers/remoteassemblage_controller.go +++ b/module/controllers/remoteassemblage_controller.go @@ -7,7 +7,6 @@ package controllers import ( "context" "fmt" - "strings" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -16,8 +15,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - asmv1 "github.com/squaremo/fleeet/assemblage/api/v1alpha1" + kustomv1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + fleetv1 "github.com/squaremo/fleeet/module/api/v1alpha1" + syncapi "github.com/squaremo/fleeet/pkg/api" ) // RemoteAssemblageReconciler reconciles a RemoteAssemblage object @@ -34,10 +35,9 @@ type RemoteAssemblageReconciler struct { //+kubebuilder:rbac:groups=fleet.squaremo.dev,resources=remoteassemblages/status,verbs=get;update;patch //+kubebuilder:rbac:groups=fleet.squaremo.dev,resources=remoteassemblages/finalizers,verbs=update -// FIXME: access to secrets? - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. +// Reconcile moves the cluster closer to the desired state, as specified in the named +// RemoteAssemblage. Usually this means making sure each sync in the assemblage has an up-to-date +// Flux primitive to represent it. func (r *RemoteAssemblageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("remoteassemblage", req.NamespacedName) @@ -46,44 +46,80 @@ func (r *RemoteAssemblageReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, client.IgnoreNotFound(err) } - // Let's go looking for the corresponding assemblage in the remote - // cluster. - clusterKey := client.ObjectKey{ - Namespace: asm.Namespace, - // HACK: the client cache accepts cluster keys, but we are - // ging straight to the secret; the trim gets the former from - // the latter. - Name: strings.TrimSuffix(asm.Spec.KubeconfigRef.Name, "-kubeconfig"), - } - remoteClient, err := r.cache.GetClient(ctx, clusterKey) - if err != nil { - return ctrl.Result{}, fmt.Errorf("could not get client for remote cluster: %w", err) + // TODO: go through the syncs and create a Kustomization object for each sync, pointing at the + // source it references. + clusterName := asm.Spec.KubeconfigRef.Name + clusterName = clusterName[:len(clusterName)-len("-kubeconfig")] // FIXME this is a hack, while types refer to kubeconfig secrets rather than e.g., clusters + + // Used to get any resources mentioned in controlPlaneBindings + namespacedClient := client.NewNamespacedClient(r.Client, asm.GetNamespace()) + for _, sync := range asm.Spec.Syncs { + // start with CLUSTER_NAME available to use in bindings + memo := map[string]string{ + "CLUSTER_NAME": clusterName, + } + + var bindingErr error + var makeBindingFunc func(stack []string) func(string) string + makeBindingFunc = func(stack []string) func(string) string { + return func(name string) string { + for i := range stack { + if stack[i] == name { + bindingErr = fmt.Errorf("circular binding %q", name) + return "" + } + } + + if v, ok := memo[name]; ok { + return v + } + for _, b := range sync.ControlPlaneBindings { + if b.Name == name { + v, err := syncapi.ResolveBinding(ctx, namespacedClient, b, makeBindingFunc(append(stack, name))) + if err != nil { + bindingErr = err + v = "" + } + memo[name] = v + return v + } + } + memo[name] = "" + return "" + } + } + + kustomSpec, err := syncapi.KustomizationSpecFromPackage(sync.Package, sync.SourceRef.Name, makeBindingFunc(nil)) + if err != nil { + return ctrl.Result{}, err + } + if bindingErr != nil { + return ctrl.Result{}, bindingErr + } + + var kustom kustomv1.Kustomization + kustom.Namespace = asm.GetNamespace() + kustom.Name = fmt.Sprintf("%s-%s", sync.Name, clusterName) // FIXME may need to hash one or both to limit size + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, &kustom, func() error { + kustom.Spec = kustomSpec + // each kustomization is controlled by the assemblage; if the asssemblage goes, so does the kustomization + if err := controllerutil.SetControllerReference(&asm, &kustom, r.Scheme); err != nil { + return err + } + + kustom.Spec.KubeConfig = &kustomv1.KubeConfig{} + kustom.Spec.KubeConfig.SecretRef.Name = asm.Spec.KubeconfigRef.Name + + return nil + }) + + if err != nil { + return ctrl.Result{}, err + } + log.V(1).Info("created/updated kustomization", "name", kustom.GetName(), "operation", op) } - log.V(1).Info("remote cluster connected", "cluster", clusterKey.Name) - - var counterpart asmv1.Assemblage - counterpart.Name = asm.Name - counterpart.Namespace = asm.Namespace - op, err := controllerutil.CreateOrUpdate(ctx, remoteClient, &counterpart, func() error { - counterpart.Spec = asm.Spec.Assemblage - return nil - }) - if err != nil { - return ctrl.Result{}, fmt.Errorf("while create/update counterpart in downstream: %w", err) - } - - switch op { - case controllerutil.OperationResultNone, - controllerutil.OperationResultUpdated: - asm.Status.Syncs = counterpart.Status.Syncs - case controllerutil.OperationResultCreated: - // TODO set a condition saying the downstream is created - } - - if err = r.Status().Update(ctx, &asm); err != nil { - return ctrl.Result{}, err - } + // TODO: report that status of each sync return ctrl.Result{}, nil }