Merge branch 'master' into dev-gmolto

.github/workflows/sqaaas.yaml

@@ -0,0 +1,25 @@
+name: SQAaaS OSCAR
+
+on:
+  push:
+    branches: ["sqa"]
+    tags:
+      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+jobs:
+
+  sqaaas_job:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add tox unit test step definition for a SQAaaS assessment
+        uses: eosc-synergy/sqaaas-step-action@v1
+        id: go_unit_test
+        with:
+          name: go_unit_test
+          container: "golang:1.21.4-alpine3.18"
+          tool: commands
+          commands: "go test ./... -v"
+
+      - name: SQAaaS assessment step
+        uses: eosc-synergy/sqaaas-assessment-action@v2
+        with:
+          qc_uni_steps: go_unit_test

.github/workflows/tests.yaml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Setup Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: '1.21'
 

docs/integration-egi.md

@@ -98,6 +98,7 @@ Once logged in via EGI Check-In you can obtain an Access Token with one of this
     oidc-token <account-short-name>
     ```
     where `account-short-name` is the name of your account configuration.
+
 * From the EGI Check-In Token Portal: [https://aai.egi.eu/token](https://aai.egi.eu/token)
 
 ![egi-checkin-token-portal.png](images/oidc/egi-checkin-token-portal.png)

examples/stable-diffusion/Dockerfile

@@ -0,0 +1,21 @@
+FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
+
+RUN apt update && \
+    apt install -y --no-install-recommends git wget python3-pip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/srisco/stable-diffusion-tensorflow.git
+
+WORKDIR stable-diffusion-tensorflow
+
+RUN pip install -r requirements.txt && \
+    rm -rf /root/.cache/pip/* && \
+    rm -rf /tmp/*
+
+# DOWNLOAD WEIGHTS
+RUN mkdir -p /root/.keras/datasets && \
+    wget https://huggingface.co/fchollet/stable-diffusion/resolve/main/text_encoder.h5 -O /root/.keras/datasets/text_encoder.h5 && \
+    wget https://huggingface.co/fchollet/stable-diffusion/resolve/main/diffusion_model.h5 -O /root/.keras/datasets/diffusion_model.h5 && \
+    wget https://huggingface.co/fchollet/stable-diffusion/resolve/main/decoder.h5 -O /root/.keras/datasets/decoder.h5 && \
+    wget https://huggingface.co/divamgupta/stable-diffusion-tensorflow/resolve/main/encoder_newW.h5 -O /root/.keras/datasets/encoder_newW.h5

examples/stable-diffusion/README.md

@@ -0,0 +1,31 @@
+# Stable Diffusion
+
+This example is based on a Keras / Tensorflow implementation of Stable Diffusion. The following repositories were used for the creation of the image:
+
+* [srisco/stable-diffusion-tensorflow](https://github.com/srisco/stable-diffusion-tensorflow)
+* [huggingface.co/fchollet/stable-diffusion](https://huggingface.co/fchollet/stable-diffusion)
+* [huggingface.co/divamgupta/stable-diffusion-tensorflow](https://huggingface.co/divamgupta/stable-diffusion-tensorflow/)
+
+The image if pushed to a public github registry [here](ghcr.io/grycap/stable-diffusion-tf:latest) but you can see the Dockerfile that generates it [here](Dockerfile).
+
+## Deploy an OSCAR cluster
+Follow the instructions in the documentation for your desired IaaS cloud provider.
+[See Deployment](https://docs.oscar.grycap.net/)
+
+## Create the OSCAR Service
+
+The Service can be created using the OSCAR GUI by providing the [FDL](stable-diff.yaml) and the [script.sh](script.sh) file.
+
+![OSCAR GUI Creation of a service](https://oscar.grycap.net/images/blog/post-20210803-1/create_service_gui.png)
+
+## Upload the input file to the MinIO bucket
+
+Once the service is created, you can upload the input file to the bucket. The input file should be a file containing the prompt that you want to process.
+
+For example, using the following prompt:
+
+`a chicken making a 360 with a skateboard with background flames`
+
+The following image is generated:
+
+![Cool chicken](prompt.txt.png)

examples/stable-diffusion/prompt.txt

@@ -0,0 +1 @@
+a chicken making a 360 with a skateboard with background flames

examples/stable-diffusion/script.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo "SCRIPT: Invoked stable diffusion text to image."
+FILE_NAME=`basename "$INPUT_FILE_PATH"`
+OUTPUT_FILE="$TMP_OUTPUT_DIR/$FILE_NAME.png"
+
+prompt=`cat "$INPUT_FILE_PATH"`
+echo "SCRIPT: Converting input prompt '$INPUT_FILE_PATH' to image :)"
+python3 text2image.py --prompt="$prompt" --output=$OUTPUT_FILE

examples/stable-diffusion/stable-diff.yaml

@@ -0,0 +1,17 @@
+functions:
+  oscar:
+  - oscar-intertwin:
+      name: stable-diffusion-tf
+      memory: 16Gi
+      cpu: '4'
+      image: ghcr.io/grycap/stable-diffusion-tf:latest
+      script: script.sh
+      log_level: DEBUG
+      vo: "vo.example.eu"
+      allowed_users: []
+      input:
+      - storage_provider: minio.default
+        path: stablediff/input
+      output:
+      - storage_provider: minio.default
+        path: stablediff/output

go.mod

@@ -34,6 +34,7 @@ require (
 
 require (
 	github.com/fatih/color v1.14.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/rs/xid v1.4.0 // indirect

go.sum

@@ -108,6 +108,8 @@ github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXK
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
+github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

pkg/handlers/config_test.go

@@ -43,6 +43,7 @@ func createExpectedBody(access_key string, secret_key string, cfg *types.Config)
 			"gpu_available":       false,
 			"interLink_available": false,
 			"yunikorn_enable":     false,
+			"oidc_groups":         nil,
 		},
 		"minio_provider": map[string]interface{}{
 			"endpoint":   cfg.MinIOProvider.Endpoint,

pkg/handlers/create_test.go

@@ -38,7 +38,7 @@ func TestMakeCreateHandler(t *testing.T) {
 	// Create a fake MinIO server
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, hreq *http.Request) {
 
-		if hreq.URL.Path != "/test" && hreq.URL.Path != "/test/input/" && hreq.URL.Path != "/output" && !strings.HasPrefix(hreq.URL.Path, "/minio/admin/v3/") {
+		if hreq.URL.Path != "/test" && hreq.URL.Path != "/test/input/" && hreq.URL.Path != "/test/output/" && hreq.URL.Path != "/test/mount/" && !strings.HasPrefix(hreq.URL.Path, "/minio/admin/v3/") {
 			t.Errorf("Unexpected path in request, got: %s", hreq.URL.Path)
 		}
 
@@ -88,10 +88,14 @@ func TestMakeCreateHandler(t *testing.T) {
   			],
 			"output": [
 				{
-				"storage_provider": "webdav.id",
-				"path": "/output"
+				"storage_provider": "minio",
+				"path": "/test/output"
 				}
   			],
+			"mount": {
+                "storage_provider": "minio",
+                "path": "/test/mount"
+	        },
 			"storage_providers": {
 				"webdav": {
 					"id": {

pkg/handlers/update_test.go

@@ -96,7 +96,7 @@ func TestMakeUpdateHandler(t *testing.T) {
 					}
 				}
 			},
-			"allowed_users": ["user1", "user2"]
+			"allowed_users": ["[email protected]", "[email protected]"]
 		}
 	`)
 	req, _ := http.NewRequest("PUT", "/system/services", body)

pkg/types/config.go

@@ -183,7 +183,7 @@ type Config struct {
 	// OIDCGroups OpenID comma-separated group list to grant access in the cluster.
 	// Groups defined in the "eduperson_entitlement" OIDC scope,
 	// as described here: https://docs.egi.eu/providers/check-in/sp/#10-groups
-	OIDCGroups []string `json:"-"`
+	OIDCGroups []string `json:"oidc_groups"`
 
 	//
 	IngressHost string `json:"-"`

pkg/types/expose_test.go

@@ -22,6 +22,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	testclient "k8s.io/client-go/kubernetes/fake"
@@ -229,3 +230,142 @@ func TestHortizontalAutoScaleSpec(t *testing.T) {
 		t.Errorf("Expected target cpu 40 but got %d", res.Spec.TargetCPUUtilizationPercentage)
 	}
 }
+
+func TestListIngress(t *testing.T) {
+
+	K8sObjects := []runtime.Object{
+		&netv1.Ingress{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "service-ing",
+				Namespace: "namespace",
+			},
+		},
+	}
+
+	kubeClientset := testclient.NewSimpleClientset(K8sObjects...)
+	cfg := &Config{ServicesNamespace: "namespace"}
+
+	_, err := listIngress(kubeClientset, cfg)
+
+	if err != nil {
+		t.Errorf("Error listing ingresses: %v", err)
+	}
+}
+
+func TestUpdateIngress(t *testing.T) {
+
+	K8sObjects := []runtime.Object{
+		&netv1.Ingress{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "service-ing",
+				Namespace: "namespace",
+			},
+		},
+	}
+
+	service := Service{
+		Name: "service",
+	}
+
+	kubeClientset := testclient.NewSimpleClientset(K8sObjects...)
+	cfg := &Config{ServicesNamespace: "namespace"}
+
+	err := updateIngress(service, kubeClientset, cfg)
+
+	if err != nil {
+		t.Errorf("Error updating ingress: %v", err)
+	}
+}
+
+func TestDeleteIngress(t *testing.T) {
+
+	K8sObjects := []runtime.Object{
+		&netv1.Ingress{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "service-ing",
+				Namespace: "namespace",
+			},
+		},
+	}
+
+	kubeClientset := testclient.NewSimpleClientset(K8sObjects...)
+	cfg := &Config{ServicesNamespace: "namespace"}
+
+	err := deleteIngress("service-ing", kubeClientset, cfg)
+
+	if err != nil {
+		t.Errorf("Error deleting ingress: %v", err)
+	}
+}
+
+func TestUpdateSecret(t *testing.T) {
+
+	K8sObjects := []runtime.Object{
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "service-auth-expose",
+				Namespace: "namespace",
+			},
+		},
+	}
+	service := Service{
+		Name: "service",
+	}
+
+	kubeClientset := testclient.NewSimpleClientset(K8sObjects...)
+	cfg := &Config{ServicesNamespace: "namespace"}
+
+	err := updateSecret(service, kubeClientset, cfg)
+
+	if err != nil {
+		t.Errorf("Error updating secret: %v", err)
+	}
+}
+
+func TestDeleteSecret(t *testing.T) {
+
+	K8sObjects := []runtime.Object{
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "service-auth-expose",
+				Namespace: "namespace",
+			},
+		},
+	}
+
+	kubeClientset := testclient.NewSimpleClientset(K8sObjects...)
+	cfg := &Config{ServicesNamespace: "namespace"}
+
+	err := deleteSecret("service", kubeClientset, cfg)
+
+	if err != nil {
+		t.Errorf("Error deleting secret: %v", err)
+	}
+}
+
+func TestExistsSecret(t *testing.T) {
+
+	K8sObjects := []runtime.Object{
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "service-auth-expose",
+				Namespace: "namespace",
+			},
+		},
+	}
+
+	kubeClientset := testclient.NewSimpleClientset(K8sObjects...)
+	cfg := &Config{ServicesNamespace: "namespace"}
+
+	exists := existsSecret("service", kubeClientset, cfg)
+
+	if exists != true {
+		t.Errorf("Expected secret to exist but got %v", exists)
+	}
+
+	notexists := existsSecret("service1", kubeClientset, cfg)
+
+	if notexists != false {
+		t.Errorf("Expected secret not to exist but got %v", notexists)
+	}
+}

pkg/utils/auth/auth.go

@@ -29,7 +29,7 @@ import (
 )
 
 // GetAuthMiddleware returns the appropriate gin auth middleware
-func GetAuthMiddleware(cfg *types.Config, kubeClientset *kubernetes.Clientset) gin.HandlerFunc {
+func GetAuthMiddleware(cfg *types.Config, kubeClientset kubernetes.Interface) gin.HandlerFunc {
 	if !cfg.OIDCEnable {
 		return gin.BasicAuth(gin.Accounts{
 			// Use the config's username and password for basic auth
@@ -40,7 +40,7 @@ func GetAuthMiddleware(cfg *types.Config, kubeClientset *kubernetes.Clientset) g
 }
 
 // CustomAuth returns a custom auth handler (gin middleware)
-func CustomAuth(cfg *types.Config, kubeClientset *kubernetes.Clientset) gin.HandlerFunc {
+func CustomAuth(cfg *types.Config, kubeClientset kubernetes.Interface) gin.HandlerFunc {
 	basicAuthHandler := gin.BasicAuth(gin.Accounts{
 		// Use the config's username and password for basic auth
 		cfg.Username: cfg.Password,
@@ -53,7 +53,7 @@ func CustomAuth(cfg *types.Config, kubeClientset *kubernetes.Clientset) gin.Hand
 	minIOAdminClient.CreateAllUsersGroup()
 	minIOAdminClient.UpdateUsersInGroup(oscarUser, "all_users_group", false)
 
-	oidcHandler := getOIDCMiddleware(kubeClientset, minIOAdminClient, cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups)
+	oidcHandler := getOIDCMiddleware(kubeClientset, minIOAdminClient, cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups, nil)
 	return func(c *gin.Context) {
 		authHeader := c.GetHeader("Authorization")
 		if strings.HasPrefix(authHeader, "Bearer ") {