e2e: improve the regression test

.github/workflows/e2e_regression.yml

@@ -15,6 +15,7 @@ on:
       - .github/workflows/e2e_regression.yml
       - e2e/getdents/**
       - e2e/genpolicy/**
+      - e2e/regression/**
 
 env:
   container_registry: ghcr.io/edgelesssys
@@ -32,6 +33,7 @@ jobs:
         case:
           - getdents
           - genpolicy
+          - regression
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - uses: ./.github/actions/setup_nix
@@ -61,6 +63,10 @@ jobs:
       - name: Build and prepare deployments
         run: |
           just node-installer
+          if [[ "${{ matrix.case }}" == "regression" ]]; then
+            # build and push other dependencies for the regression test
+            just coordinator initializer port-forwarder openssl service-mesh-proxy
+          fi
       - name: Run regression test
         run: |
           nix shell -L .#contrast.e2e --command ${{ matrix.case }}.test -test.v \

e2e/internal/kubeclient/deploy.go

@@ -107,7 +107,7 @@ func (s StatefulSet) getPods(ctx context.Context, client *Kubeclient, namespace,
 // WaitForPod watches the given pod and blocks until it meets the condition Ready=True or the
 // context expires (is cancelled or times out).
 func (c *Kubeclient) WaitForPod(ctx context.Context, namespace, name string) error {
-	watcher, err := c.client.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
+	watcher, err := c.Client.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
 	if err != nil {
 		return err
 	}
@@ -149,7 +149,7 @@ func (c *Kubeclient) WaitFor(ctx context.Context, resource ResourceWaiter, names
 
 retryLoop:
 	for {
-		watcher, err := resource.watcher(ctx, c.client, namespace, name)
+		watcher, err := resource.watcher(ctx, c.Client, namespace, name)
 		if err != nil {
 			return err
 		}
@@ -217,7 +217,7 @@ retryLoop:
 
 // WaitForLoadBalancer waits until the given service is configured with an external IP and returns it.
 func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) {
-	watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
+	watcher, err := c.Client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
 	if err != nil {
 		return "", err
 	}
@@ -292,7 +292,7 @@ func isPodReady(pod *corev1.Pod) bool {
 }
 
 func (c *Kubeclient) resourceInterfaceFor(obj *unstructured.Unstructured) (dynamic.ResourceInterface, error) {
-	dyn := dynamic.New(c.client.RESTClient())
+	dyn := dynamic.New(c.Client.RESTClient())
 	gvk := obj.GroupVersionKind()
 
 	mapping, err := c.restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
@@ -354,7 +354,7 @@ func (c *Kubeclient) Restart(ctx context.Context, resource ResourceWaiter, names
 		return err
 	}
 	for _, pod := range pods {
-		err := c.client.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{
+		err := c.Client.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{
 			GracePeriodSeconds: toPtr(int64(0)),
 		})
 		if err != nil {
@@ -366,7 +366,7 @@ func (c *Kubeclient) Restart(ctx context.Context, resource ResourceWaiter, names
 
 // ScaleDeployment scales a deployment to the given number of replicas.
 func (c *Kubeclient) ScaleDeployment(ctx context.Context, namespace, name string, replicas int32) error {
-	_, err := c.client.AppsV1().Deployments(namespace).UpdateScale(ctx, name, &autoscalingv1.Scale{
+	_, err := c.Client.AppsV1().Deployments(namespace).UpdateScale(ctx, name, &autoscalingv1.Scale{
 		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
 		Spec:       autoscalingv1.ScaleSpec{Replicas: replicas},
 	}, metav1.UpdateOptions{})

e2e/internal/kubeclient/kubeclient.go

@@ -31,8 +31,8 @@ import (
 type Kubeclient struct {
 	log *slog.Logger
 
-	// client is the underlying Kubernetes client.
-	client *kubernetes.Clientset
+	// Client is the underlying Kubernetes Client.
+	Client *kubernetes.Clientset
 	// restMapper allows to look up schema information for dynamic resources
 	restMapper meta.RESTMapper
 	// config is the "Kubeconfig" for the client
@@ -53,7 +53,7 @@ func New(config *rest.Config, log *slog.Logger) (*Kubeclient, error) {
 
 	return &Kubeclient{
 		log:        log,
-		client:     client,
+		Client:     client,
 		config:     config,
 		restMapper: restmapper.NewDiscoveryRESTMapper(resources),
 	}, nil
@@ -89,11 +89,11 @@ func NewForTest(t *testing.T) *Kubeclient {
 // A pod is considered to belong to a deployment if it is owned by a ReplicaSet which is in turn
 // owned by the Deployment in question.
 func (c *Kubeclient) PodsFromDeployment(ctx context.Context, namespace, deployment string) ([]corev1.Pod, error) {
-	replicasets, err := c.client.AppsV1().ReplicaSets(namespace).List(ctx, metav1.ListOptions{})
+	replicasets, err := c.Client.AppsV1().ReplicaSets(namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("listing replicasets: %w", err)
 	}
-	pods, err := c.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	pods, err := c.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("listing pods: %w", err)
 	}
@@ -119,7 +119,7 @@ func (c *Kubeclient) PodsFromDeployment(ctx context.Context, namespace, deployme
 
 // PodsFromOwner returns the pods owned by an object in the namespace of the given kind.
 func (c *Kubeclient) PodsFromOwner(ctx context.Context, namespace, kind, name string) ([]corev1.Pod, error) {
-	pods, err := c.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	pods, err := c.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("listing pods: %w", err)
 	}
@@ -142,7 +142,7 @@ func (c *Kubeclient) Exec(ctx context.Context, namespace, pod string, argv []str
 ) {
 	buf := &bytes.Buffer{}
 	errBuf := &bytes.Buffer{}
-	request := c.client.CoreV1().RESTClient().
+	request := c.Client.CoreV1().RESTClient().
 		Post().
 		Namespace(namespace).
 		Resource("pods").
@@ -187,15 +187,15 @@ func (c *Kubeclient) ExecDeployment(ctx context.Context, namespace, deployment s
 
 // LogDebugInfo collects pod information from the cluster and writes it to the logger.
 func (c *Kubeclient) LogDebugInfo(ctx context.Context) {
-	namespaces, err := c.client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	namespaces, err := c.Client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
 	if err != nil {
 		c.log.Error("Could not get namespaces", "error", err)
 		return
 	}
 
 	for _, namespace := range namespaces.Items {
 		c.log.Debug("Collecting debug info for pods", "namespace", namespace.Name)
-		pods, err := c.client.CoreV1().Pods(namespace.Name).List(ctx, metav1.ListOptions{})
+		pods, err := c.Client.CoreV1().Pods(namespace.Name).List(ctx, metav1.ListOptions{})
 		if err != nil {
 			c.log.Error("Could not get pods", "namespace", namespace.Name, "error", err)
 			continue

e2e/internal/kubeclient/portforward.go

@@ -65,7 +65,7 @@ func (k *Kubeclient) portForwardPod(ctx context.Context, namespace, podName, rem
 	errorCh := make(chan error)
 
 	// Ports are forwarded by upgrading this POST request to a SPDY connection.
-	req := k.client.CoreV1().RESTClient().Post().
+	req := k.Client.CoreV1().RESTClient().Post().
 		Resource("pods").
 		Namespace(namespace).
 		Name(podName).

e2e/regression/README.md

@@ -0,0 +1,19 @@
+# Services Tested
+
+- [Redis](./test-data/redis.yaml)
+- [mongo db](./test-data/mongodb.yaml)
+- [apache HTTPD](./test-data/apache-httpd-fedora.yaml)
+- [MySQL](./test-data/mysql-fedora.yaml)
+- [KeyCloak](./test-data/keycloak.yaml)
+- [nginx](./test-data/nginx.yaml)
+- [Prometheus](./test-data/prometheus.yaml)
+- [varnish](./test-data/varnish.yaml)
+
+# Operating Systems Tested
+
+- [Alpine](./test-data/redis.yaml)
+- [Debian](./test-data/nginx.yaml)
+- [Ubuntu](./test-data/mongodb.yaml)
+- [Cent OS](./test-data/mysql-centos.yaml)
+- [Fedora](./test-data/apache-httpd-fedora.yaml)
+- [RedHat (ubi9)](./test-data/keycloak.yaml)

e2e/regression/regression_test.go

@@ -0,0 +1,106 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package regression
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
+	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
+	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/edgelesssys/contrast/internal/manifest"
+	"github.com/edgelesssys/contrast/internal/platforms"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	imageReplacementsFile, namespaceFile, platformStr string
+	_skipUndeploy                                     bool // just here for interoptability, ignored in this test
+)
+
+func TestRegression(t *testing.T) {
+	yamlDir := "./e2e/regression/testdata/"
+	files, err := os.ReadDir(yamlDir)
+	require.NoError(t, err)
+
+	platform, err := platforms.FromString(platformStr)
+	require.NoError(t, err)
+
+	runtimeHandler, err := manifest.RuntimeHandler(platform)
+	require.NoError(t, err)
+
+	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, platform, false)
+
+	// Initially just deploy the coordinator bundle
+
+	resources := kuberesource.CoordinatorBundle()
+	resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler)
+	resources = kuberesource.AddPortForwarders(resources)
+
+	ct.Init(t, resources)
+
+	require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")
+	require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests")
+	require.True(t, t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests")
+	require.True(t, t.Run("verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")
+
+	for _, file := range files {
+		t.Run(file.Name(), func(t *testing.T) {
+			require := require.New(t)
+
+			c := kubeclient.NewForTest(t)
+			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+			defer cancel()
+
+			yaml, err := os.ReadFile(yamlDir + file.Name())
+			require.NoError(err)
+			yaml = bytes.ReplaceAll(yaml, []byte("@@REPLACE_NAMESPACE@@"), []byte(ct.Namespace))
+
+			newResources, err := kuberesource.UnmarshalApplyConfigurations(yaml)
+			require.NoError(err)
+
+			newResources = kuberesource.PatchRuntimeHandlers(newResources, runtimeHandler)
+			newResources = kuberesource.AddPortForwarders(newResources)
+
+			// write the new resources.yaml
+			resourceBytes, err := kuberesource.EncodeResources(newResources...)
+			require.NoError(err)
+			require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yaml"), resourceBytes, 0o644))
+
+			// generate, set, deploy and verify the new policy
+			require.True(t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")
+			require.True(t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests")
+			require.True(t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests")
+			require.True(t.Run("verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")
+
+			deploymentName, _ := strings.CutSuffix(file.Name(), ".yaml")
+			require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, deploymentName))
+
+			// delete the deployment
+			require.NoError(ct.Kubeclient.Client.AppsV1().Deployments(ct.Namespace).Delete(ctx, deploymentName, metav1.DeleteOptions{}))
+		})
+	}
+}
+
+func TestMain(m *testing.M) {
+	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
+	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platformStr, "platform", "", "Deployment platform")
+
+	// ignored and just here for interoptability, we always undeploy to save resources
+	flag.BoolVar(&_skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
+	flag.Parse()
+
+	os.Exit(m.Run())
+}

e2e/regression/testdata/apache-httpd-centos.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: apache-httpd-centos
+  namespace: "@@REPLACE_NAMESPACE@@"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: apache-httpd-centos
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: apache-httpd-centos
+    spec:
+      containers:
+        - name: apache-httpd-centos
+          image: quay.io/sclorg/httpd-24-micro-c9s@sha256:80b0ca364c3bf773f5a1a85fea5df8fa303ac75693c3dd5dfaad22ddb9206e67
+          ports:
+            - containerPort: 8443
+            - containerPort: 8080
+      runtimeClassName: contrast-cc

e2e/regression/testdata/apache-httpd-fedora.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: apache-httpd-fedora
+  namespace: "@@REPLACE_NAMESPACE@@"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: apache-httpd-fedora
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: apache-httpd-fedora
+    spec:
+      containers:
+        - name: apache-httpd-fedora
+          image: quay.io/fedora/httpd-24-micro@sha256:f8f7d90feb8beace46a9f235e1a215042c7a5d04e1567e11173f7b73ab621a1d
+          ports:
+            - containerPort: 8443
+            - containerPort: 8080
+      runtimeClassName: contrast-cc

e2e/regression/testdata/keycloak.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: keycloak
+  namespace: "@@REPLACE_NAMESPACE@@"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: keycloak
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: keycloak
+    spec:
+      containers:
+        - name: keycloak
+          image: quay.io/keycloak/keycloak@sha256:b55f55ff60e905db4809ac133c6b963b87963ec1b49aae6d218fdd53646cb09e
+          ports:
+            - containerPort: 9000
+            - containerPort: 8443
+            - containerPort: 8080
+      runtimeClassName: contrast-cc

e2e/regression/testdata/mongodb.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mongodb
+  namespace: "@@REPLACE_NAMESPACE@@"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mongodb
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mongodb
+    spec:
+      containers:
+        - name: mongodb
+          securityContext:
+            runAsUser: 101
+          image: quay.io/mongodb/mongodb-community-server@sha256:8b73733842da21b6bbb6df4d7b2449229bb3135d2ec8c6880314d88205772a11
+          volumeMounts:
+            - mountPath: /data/db
+              name: db
+            - mountPath: /data/configdb
+              name: configdb
+          ports:
+            - containerPort: 27017
+      # TODO(miampf): Remove this after https://github.com/kata-containers/kata-containers/pull/10136/files is merged
+      volumes:
+        - name: db
+          emptyDir:
+            sizeLimit: 10Mi
+        - name: configdb
+          emptyDir:
+            sizeLimit: 10Mi
+      runtimeClassName: contrast-cc