Add k3s integration test

.github/workflows/docker-build-push.yaml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         image:
           - imageName: examples/spin-rust-hello
-            context: images/spin
+            context: images/spin-hello-world
           - imageName: examples/spin-dotnet-hello
             context: images/spin_dotnet
           - imageName: examples/spin-outbound-redis

Makefile

@@ -67,6 +67,22 @@ integration-spin-registry-push-tests:
 tests/collect-debug-logs:
 	./scripts/collect-debug-logs.sh 2>&1
 
+# test/k3s
+
+.PHONY: install-k3s, build-and-push-images
+install-k3s:
+	./scripts/install-k3s.sh
+build-and-push-images:
+	./scripts/build-and-push-images.sh --spin
+
+.PHONY: test/k3s
+test/k3s: install-k3s build-and-push-images
+	kubectl apply -f tests/workloads-common
+	kubectl apply -f tests/k3s/workloads-pushed-using-spin-registry-push
+	make pod-terminates-test
+	SPIN_TEST_PORT=80 cargo test -p containerd-shim-spin-tests -- --nocapture
+	make teardown-workloads-k3s
+
 # fmt
 
 .PHONY: fmt fix
@@ -113,10 +129,10 @@ run-%: install load
 
 # deploy
 
-./PHONY: up move-bins deploy-workloads-pushed-using-docker-build-push deploy-workloads-pushed-using-spin-registry-push pod-terminates-test prepare-cluster-and-images
+.PHONY: up move-bins deploy-workloads-pushed-using-docker-build-push deploy-workloads-pushed-using-spin-registry-push pod-terminates-test prepare-cluster-and-images
 
 up:
-	./scripts/up.sh
+	./scripts/k3d-up.sh
 
 move-bins:
 	./scripts/move-bins.sh $(BIN_DIR)
@@ -134,9 +150,12 @@ prepare-cluster-and-images: check-bins move-bins up free-disk pod-status-check
 
 # clean
 
-./PHONY: teardown-workloads tests/clean
-teardown-workloads:
-	./scripts/teardown-workloads.sh
+.PHONY: teardown-workloads-k3d teardown-workloads-k3s tests/clean
+teardown-workloads-k3d:
+	./scripts/teardown-workloads.sh "k3d"
+teardown-workloads-k3s:
+	./scripts/teardown-workloads.sh "k3s"
+	docker container stop test-registry && docker container rm test-registry
 
 tests/clean:
 	./scripts/down.sh

deployments/k3d/DockerSetup.md

@@ -50,7 +50,7 @@ Otherwise, please append the above content to the file.
 Now you have Docker 24.0.0-beta.2 installed on your machine, you can build wasm images using the following command.
 
 ```shell
-docker buildx build --platform=wasi/wasm --load -t wasmtest_spin:latest ./images/spin
+docker buildx build --platform=wasi/wasm --load -t wasmtest_spin:latest ./images/spin-hello-world
 ```
 
 The `wasi/wasm` platform specifies that the image is a wasm image. The major benefit of using this platform is that you don't need to build each image for a different computer architecture. 

scripts/build-and-push-images.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -euo pipefail
+
+RUN_SPIN=false
+RUN_DOCKER=false
+
+cluster_name="test-cluster"
+OUT_DIRS=("test/out_spin" "test/out_spin_keyvalue" "test/out_spin_outbound_redis" "test/out_spin_multi_trigger_app" "test/out_spin_static_assets" "test/out_spin_mqtt_message_logger")
+IMAGES=("spin-hello-world" "spin-keyvalue" "spin-outbound-redis" "spin-multi-trigger-app" "spin-static-assets" "spin-mqtt-message-logger")
+
+
+spin_build_and_push() {
+  local i=$1
+  spin build -f "./images/${IMAGES[$i]}/spin.toml"
+  if [ "${IMAGES[$i]}" == "spin-static-assets" ]; then
+    export SPIN_OCI_ARCHIVE_LAYERS=1
+  fi
+  spin registry push "localhost:5000/spin-registry-push/${IMAGES[$i]}:latest" -f "./images/${IMAGES[$i]}/spin.toml" -k
+}
+
+docker_build_and_push() {
+  local image="$1"
+  local out_dir="$2"
+
+  docker buildx build -t "${image}:latest" "./images/${image}" --load
+  mkdir -p "${out_dir}"
+  docker save -o "${out_dir}/img.tar" "${image}:latest"
+  k3d image import "${out_dir}/img.tar" -c "$cluster_name"
+}
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    --spin) RUN_SPIN=true ;;
+    --docker) RUN_DOCKER=true ;;
+    --both) RUN_SPIN=true; RUN_DOCKER=true ;;
+    *) echo "Unknown option: $1"; exit 1 ;;
+  esac
+  shift
+done
+
+if ! $RUN_SPIN && ! $RUN_DOCKER; then
+  echo "Error: At least one of --spin, --docker, or --both must be specified."
+  exit 1
+fi
+
+
+if $RUN_SPIN; then
+  echo "Running Spin builds and pushes..."
+  if ! docker ps | grep -q test-registry; then
+    docker run -d -p 5000:5000 --name test-registry registry:2
+  fi
+  for i in "${!IMAGES[@]}"; do
+    spin_build_and_push "$i" &
+  done
+fi
+
+if $RUN_DOCKER; then
+  echo "Running Docker builds and pushes..."
+  for i in "${!IMAGES[@]}"; do
+    docker_build_and_push "${IMAGES[$i]}" "${OUT_DIRS[$i]}" &
+  done
+fi
+
+# Wait for all background jobs to finish
+wait
+
+sleep 5
+echo "Images are ready"

scripts/install-k3s.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Install k3s
+# Since the latest k3s already bakes in the Spin shim, we don't need to reconfigure it.
+# We will just need to make sure that containerd-shim-spin-v2 binary is in PATH
+# and that the k3s service is running.
+
+curl -sfL https://get.k3s.io | sh -s - server --write-kubeconfig-mode '0644' 
+sudo systemctl start k3s
+
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+kubectl get nodes

scripts/k3d-up.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cluster_name="test-cluster"
+dockerfile_path="deployments/k3d"
+
+docker build -t k3d-shim-test "$dockerfile_path"
+
+k3d cluster create "$cluster_name" \
+  --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 2 \
+  --registry-create test-registry:0.0.0.0:5000 \
+  --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \
+  --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*'
+
+kubectl wait --for=condition=ready node --all --timeout=120s
+
+echo "Running Spin and Docker builds and pushes..."
+./scripts/spin-build-and-push-images.sh --both
+
+echo ">>> Cluster setup and image builds/pushes complete!"

scripts/teardown-workloads.sh

@@ -2,7 +2,20 @@
 
 set -euo pipefail
 
-kubectl delete -f tests/workloads-common --wait --timeout 60s --ignore-not-found=true
-kubectl delete -f tests/workloads-pushed-using-docker-build-push --wait --timeout 60s --ignore-not-found=true
-kubectl delete -f tests/workloads-pushed-using-spin-registry-push --wait --timeout 60s --ignore-not-found=true
+case "$1" in
+  k3d)
+    TESTS_PATH="tests/k3d"
+    ;;
+  k3s)
+    TESTS_PATH="tests/k3s"
+    ;;
+  *)
+    echo "Invalid argument. Use 'k3d' or 'k3s'."
+    exit 1
+    ;;
+esac
+
+kubectl delete -f "tests/workloads-common" --wait --timeout 60s --ignore-not-found=true
+kubectl delete -f "tests/workloads-pushed-using-docker-build-push" --wait --timeout 60s --ignore-not-found=true
+kubectl delete -f "$TESTS_PATH/workloads-pushed-using-spin-registry-push" --wait --timeout 60s --ignore-not-found=true
 kubectl wait pod --for=delete -l app=wasm-spin -l app=spin-keyvalue -l app=spin-outbound-redis -l app=spin-multi-trigger-app --timeout 60s

tests/k3s/workloads-pushed-using-spin-registry-push/README.md

@@ -0,0 +1 @@
+This folder has yaml file to apply workloads which are built using `spin build` and pushed to a registry, managed with k3d in CI, using `spin registry push` command.