From 1fbe7788eddcd6e37e4e826898660234c48b3c0c Mon Sep 17 00:00:00 2001 From: barreiro Date: Mon, 27 May 2024 19:34:56 +0100 Subject: [PATCH] Update to kubebuilder v4 --- Dockerfile | 8 +- Makefile | 150 +++++++++++------- PROJECT | 2 +- main.go => cmd/main.go | 2 +- config/crd/bases/hyperfoil.io_hyperfoils.yaml | 50 +++--- config/rbac/role.yaml | 1 - .../controller}/hyperfoil_controller.go | 0 .../controller}/suite_test.go | 2 +- 8 files changed, 130 insertions(+), 85 deletions(-) rename main.go => cmd/main.go (98%) rename {controllers => internal/controller}/hyperfoil_controller.go (100%) rename {controllers => internal/controller}/suite_test.go (96%) diff --git a/Dockerfile b/Dockerfile index 73eb124..aca26f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.19 as builder +FROM golang:1.21 AS builder ARG TARGETOS ARG TARGETARCH @@ -12,16 +12,16 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY main.go main.go +COPY cmd/main.go cmd/main.go COPY api/ api/ -COPY controllers/ controllers/ +COPY internal/controller/ internal/controller/ # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o manager main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Makefile b/Makefile index a6d697f..3853888 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,9 @@ endif # Image URL to use all building/pushing image targets IMG ?= $(CONTAINER_REPO_OVERRIDE)/hyperfoil-operator:$(VERSION) +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.29.0 + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -54,6 +57,17 @@ else GOBIN=$(shell go env GOBIN) endif +# CONTAINER_TOOL defines the container tool to be used for building images. +# Be aware that the target commands are only tested with Docker which is +# scaffolded by default. However, you might want to replace it to use other +# tools. (i.e. podman) +CONTAINER_TOOL ?= docker + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + .PHONY: all all: build @@ -67,7 +81,7 @@ clean: # The help target prints out all targets with their descriptions organized # beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the +# target descriptions by '##'. The awk command is responsible for reading the # entire set of makefiles included in this invocation, looking for lines of the # file as xyz: ## something, and then pretty-format the target and help. Then, # if there's a line with ##@ something, that gets pretty-printed as a category. @@ -98,54 +112,67 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... -ENVTEST_ASSETS_DIR=$(shell pwd)/testbin -test: manifests generate fmt vet ## Run tests. - mkdir -p ${ENVTEST_ASSETS_DIR} - test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out +.PHONY: test +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. +.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. +test-e2e: + go test ./test/e2e/ -v -ginkgo.v + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter & yamllint + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix ##@ Build .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager main.go + go build -o bin/manager cmd/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. - go run ./main.go + go run ./cmd/main.go -# If you wish built the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build -docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . - -.PHONY: docker-build-without-test -docker-build-without-test: manifests generate fmt vet ## Build docker image with the manager. - docker build -t ${IMG} . +docker-build: ## Build docker image with the manager. + $(CONTAINER_TOOL) build -t ${IMG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. - docker push ${IMG} + $(CONTAINER_TOOL) push ${IMG} -# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/ -# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=> then the export will fail) -# To properly provided solutions that supports more than one platform you should use this option. +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le .PHONY: docker-buildx -docker-buildx: test ## Build and push docker image for the manager for cross-platform support +docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - docker buildx create --driver-opt network=host --name project-v3-builder - docker buildx use project-v3-builder - - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - docker buildx rm project-v3-builder + - $(CONTAINER_TOOL) buildx create --driver-opt network=host --name project-v3-builder + $(CONTAINER_TOOL) buildx use project-v3-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm project-v3-builder rm Dockerfile.cross +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default > dist/install.yaml + ##@ Deployment ifndef ignore-not-found @@ -154,20 +181,20 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl apply -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: docker-build docker-push manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - .PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy-samples deploy-samples: ## Deploy config/samples to the K8s cluster specified in ~/.kube/config. @@ -177,7 +204,7 @@ deploy-samples: ## Deploy config/samples to the K8s cluster specified in ~/.kube undeploy-samples: ## Undeploy config/samples from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/samples | kubectl delete --ignore-not-found=$(ignore-not-found) -f - -##@ Build Dependencies +##@ Dependencies ## Location to install dependencies to LOCALBIN ?= $(shell pwd)/bin @@ -185,34 +212,32 @@ $(LOCALBIN): mkdir -p $(LOCALBIN) ## Tool Binaries -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest +KUBECTL ?= kubectl +KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) +ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) ## Tool Versions -KUSTOMIZE_VERSION ?= v5.0.1 -CONTROLLER_TOOLS_VERSION ?= v0.11.3 +KUSTOMIZE_VERSION ?= v5.3.0 +CONTROLLER_TOOLS_VERSION ?= v0.14.0 +ENVTEST_VERSION ?= release-0.17 +GOLANGCI_LINT_VERSION ?= v1.57.2 -KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) - @if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \ - echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \ - rm -rf $(LOCALBIN)/kustomize; \ - fi - test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); } + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) .PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) .PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) - test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) .PHONY: bundle ## Generate bundle manifests and metadata, then validate generated files. bundle: manifests kustomize @@ -225,7 +250,26 @@ bundle: manifests kustomize .PHONY: bundle-build ## Build the bundle image. bundle-build: - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + $(CONTAINER_TOOL) build -f bundle.Dockerfile -t $(BUNDLE_IMG) . bundle-push: - docker push $(BUNDLE_IMG) \ No newline at end of file + $(CONTAINER_TOOL) push $(BUNDLE_IMG) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION}) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary (ideally with version) +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f $(1) ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ +} +endef diff --git a/PROJECT b/PROJECT index 0e2b32f..c12c300 100644 --- a/PROJECT +++ b/PROJECT @@ -1,5 +1,5 @@ domain: hyperfoil.io -layout: go.kubebuilder.io/v3 +layout: go.kubebuilder.io/v4 plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} diff --git a/main.go b/cmd/main.go similarity index 98% rename from main.go rename to cmd/main.go index bc492bc..839ba37 100644 --- a/main.go +++ b/cmd/main.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" hyperfoiliov1alpha2 "github.com/Hyperfoil/hyperfoil-operator/api/v1alpha2" - "github.com/Hyperfoil/hyperfoil-operator/controllers" + "github.com/Hyperfoil/hyperfoil-operator/internal/controller" routev1 "github.com/openshift/api/route/v1" rbacv1 "k8s.io/api/rbac/v1" //+kubebuilder:scaffold:imports diff --git a/config/crd/bases/hyperfoil.io_hyperfoils.yaml b/config/crd/bases/hyperfoil.io_hyperfoils.yaml index b7cdb3c..c9e03fa 100644 --- a/config/crd/bases/hyperfoil.io_hyperfoils.yaml +++ b/config/crd/bases/hyperfoil.io_hyperfoils.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: hyperfoils.hyperfoil.io spec: group: hyperfoil.io @@ -39,14 +38,19 @@ spec: description: Hyperfoil is the Schema for the hyperfoils API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -115,28 +119,26 @@ spec: type: string type: object secretEnvVars: - description: List of secrets in this namespace; each entry from those - secrets will be mapped as environment variable, using the key as - variable name. + description: |- + List of secrets in this namespace; each entry from those secrets will be mapped + as environment variable, using the key as variable name. items: type: string type: array serviceType: - description: Type of the service being exposed. By default this is - ClusterIP if Openshift Route resource is available (the route will - target this service). If Openshift Routes are not available (on - vanilla Kubernetes) the default is NodePort. + description: |- + Type of the service being exposed. By default this is ClusterIP if Openshift Route resource is available (the route will target this service). + If Openshift Routes are not available (on vanilla Kubernetes) the default is NodePort. type: string triggerUrl: - description: If this is set the controller does not start benchmark - run right away after hitting /benchmark/my-benchmark/start ; instead - it responds with status 301 and header Location set to concatenation - of this string and 'BENCHMARK=my-benchmark&RUN_ID=xxxx'. CLI interprets - that response as a request to hit CI instance on this URL, assuming - that CI will trigger a new job that will eventually call /benchmark/my-benchmark/start?runId=xxxx - with header 'x-trigger-job'. This is useful if the the CI has to - synchronize Hyperfoil to other benchmarks that don't use this controller - instance. + description: |- + If this is set the controller does not start benchmark run right away after hitting + /benchmark/my-benchmark/start ; instead it responds with status 301 and header Location + set to concatenation of this string and 'BENCHMARK=my-benchmark&RUN_ID=xxxx'. + CLI interprets that response as a request to hit CI instance on this URL, assuming that + CI will trigger a new job that will eventually call /benchmark/my-benchmark/start?runId=xxxx + with header 'x-trigger-job'. This is useful if the the CI has to synchronize Hyperfoil + to other benchmarks that don't use this controller instance. type: string version: description: Tag for controller image. Defaults to version matching diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index cec857c..500f5d2 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: manager-role rules: - apiGroups: diff --git a/controllers/hyperfoil_controller.go b/internal/controller/hyperfoil_controller.go similarity index 100% rename from controllers/hyperfoil_controller.go rename to internal/controller/hyperfoil_controller.go diff --git a/controllers/suite_test.go b/internal/controller/suite_test.go similarity index 96% rename from controllers/suite_test.go rename to internal/controller/suite_test.go index 6005bf6..7bcc45c 100644 --- a/controllers/suite_test.go +++ b/internal/controller/suite_test.go @@ -54,7 +54,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, }