Merge branch 'master' of github.com:grycap/oscar into devel

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+exclude = examples

.github/workflows/sqaaas.yaml

@@ -0,0 +1,25 @@
+name: SQAaaS OSCAR
+
+on:
+  push:
+    branches: ["sqa"]
+    tags:
+      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+jobs:
+
+  sqaaas_job:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add tox unit test step definition for a SQAaaS assessment
+        uses: eosc-synergy/sqaaas-step-action@v1
+        id: go_unit_test
+        with:
+          name: go_unit_test
+          container: "golang:1.21.4-alpine3.18"
+          tool: commands
+          commands: "go test ./... -v"
+
+      - name: SQAaaS assessment step
+        uses: eosc-synergy/sqaaas-assessment-action@v2
+        with:
+          qc_uni_steps: go_unit_test

.github/workflows/tests.yaml

@@ -8,16 +8,21 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Setup Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: '1.21'
 
     - name: Run tests
       run: go test ./pkg/... -cover -coverprofile=profile.cov
 
+    - name: Run Gosec Security Scanner
+      uses: securego/gosec@master
+      with:
+        args: ./...
+
     - name: Report coverage
       uses: codacy/codacy-coverage-reporter-action@v1
       with:

README.md

@@ -107,7 +107,12 @@ license text.
 
 This development is partially funded by the [EGI Strategic and Innovation Fund](https://www.egi.eu/about/egi-council/egi-strategic-and-innovation-fund/).
 
-Partially funded by the project [AI-SPRINT](https://ai-sprint-project.eu) "AI in Secure Privacy-Preserving Computing Continuum" that has received funding from the European Union’s Horizon 2020 Research and Innovation Programme under Grant 101016577.
+Partially funded by the projects:
+
+- [AI-SPRINT](https://ai-sprint-project.eu) "AI in Secure Privacy-Preserving Computing Continuum" that has received funding from the European Union’s Horizon 2020 Research and Innovation Programme under Grant 101016577.
+- [interTwin](https://intertwin.eu) "An interdisciplinary Digital Twin Engine for science" that has received funding from the European Union’s Horizon Europe Programme under Grant 101058386.
+- [AI4EOSC](https://ai4eosc.eu) "Artificial Intelligence for the European Open Science Cloud" that has received funding from the European Union’s Horizon Europe Research and Innovation Programme under Grant 101058593."
+- [iMagine](http://imagine-ai.eu) "AI-based image data analysis tools for aquatic research" that has received funding from the European Union’s Horizon Europe Research and Innovation Programme under Grant 101058625.
 
 Also, Grant PDC2021-120844-I00 funded by Ministerio de Ciencia e Innovación/Agencia Estatal de Investigación/ 10.13039/501100011033 and by “European Union NextGenerationEU/PRTR” and Grant PID2020-113126RB-I00 funded by Ministerio de Ciencia e Innovación/Agencia Estatal de Investigación/ 10.13039/501100011033.
 

docs/integration-egi.md

@@ -98,6 +98,7 @@ Once logged in via EGI Check-In you can obtain an Access Token with one of this
     oidc-token <account-short-name>
     ```
     where `account-short-name` is the name of your account configuration.
+
 * From the EGI Check-In Token Portal: [https://aai.egi.eu/token](https://aai.egi.eu/token)
 
 ![egi-checkin-token-portal.png](images/oidc/egi-checkin-token-portal.png)

examples/expose_services/deep/deep_expose.yaml

@@ -1,15 +1,19 @@
 functions:
   oscar:
   - oscar-cluster:
-      name: plant
+      name: dogs-breed
       memory: 3Gi
       cpu: '2.0'
-      image: deephdc/deep-oc-plants-classification-tf
+      image: ai4oshub/dogs-breed-detector
       log_level: CRITICAL
-      script: script.sh
+      script: deepscript.sh
+      environment:
+        Variables:
+          basePath: /system/services/dogs-breed/exposed/main
       expose:
         min_scale: 1
         max_scale: 1
         api_port: 5000
+        rewrite_target: true
         cpu_threshold: 90
         set_auth: true

examples/expose_services/deep/deepscript.sh

@@ -1 +1 @@
-deepaas-run --openwhisk-detect --listen-ip 0.0.0.0 --listen-port 5000
+deepaas-run  --listen-ip 0.0.0.0 --listen-port 5000 --base-path $basePath

examples/stable-diffusion/Dockerfile

@@ -0,0 +1,21 @@
+FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
+
+RUN apt update && \
+    apt install -y --no-install-recommends git wget python3-pip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/srisco/stable-diffusion-tensorflow.git
+
+WORKDIR stable-diffusion-tensorflow
+
+RUN pip install -r requirements.txt && \
+    rm -rf /root/.cache/pip/* && \
+    rm -rf /tmp/*
+
+# DOWNLOAD WEIGHTS
+RUN mkdir -p /root/.keras/datasets && \
+    wget https://huggingface.co/fchollet/stable-diffusion/resolve/main/text_encoder.h5 -O /root/.keras/datasets/text_encoder.h5 && \
+    wget https://huggingface.co/fchollet/stable-diffusion/resolve/main/diffusion_model.h5 -O /root/.keras/datasets/diffusion_model.h5 && \
+    wget https://huggingface.co/fchollet/stable-diffusion/resolve/main/decoder.h5 -O /root/.keras/datasets/decoder.h5 && \
+    wget https://huggingface.co/divamgupta/stable-diffusion-tensorflow/resolve/main/encoder_newW.h5 -O /root/.keras/datasets/encoder_newW.h5

examples/stable-diffusion/README.md

@@ -0,0 +1,31 @@
+# Stable Diffusion
+
+This example is based on a Keras / Tensorflow implementation of Stable Diffusion. The following repositories were used for the creation of the image:
+
+* [srisco/stable-diffusion-tensorflow](https://github.com/srisco/stable-diffusion-tensorflow)
+* [huggingface.co/fchollet/stable-diffusion](https://huggingface.co/fchollet/stable-diffusion)
+* [huggingface.co/divamgupta/stable-diffusion-tensorflow](https://huggingface.co/divamgupta/stable-diffusion-tensorflow/)
+
+The image if pushed to a public github registry [here](ghcr.io/grycap/stable-diffusion-tf:latest) but you can see the Dockerfile that generates it [here](Dockerfile).
+
+## Deploy an OSCAR cluster
+Follow the instructions in the documentation for your desired IaaS cloud provider.
+[See Deployment](https://docs.oscar.grycap.net/)
+
+## Create the OSCAR Service
+
+The Service can be created using the OSCAR GUI by providing the [FDL](stable-diff.yaml) and the [script.sh](script.sh) file.
+
+![OSCAR GUI Creation of a service](https://oscar.grycap.net/images/blog/post-20210803-1/create_service_gui.png)
+
+## Upload the input file to the MinIO bucket
+
+Once the service is created, you can upload the input file to the bucket. The input file should be a file containing the prompt that you want to process.
+
+For example, using the following prompt:
+
+`a chicken making a 360 with a skateboard with background flames`
+
+The following image is generated:
+
+![Cool chicken](prompt.txt.png)

examples/stable-diffusion/prompt.txt

@@ -0,0 +1 @@
+a chicken making a 360 with a skateboard with background flames

examples/stable-diffusion/script.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo "SCRIPT: Invoked stable diffusion text to image."
+FILE_NAME=`basename "$INPUT_FILE_PATH"`
+OUTPUT_FILE="$TMP_OUTPUT_DIR/$FILE_NAME.png"
+
+prompt=`cat "$INPUT_FILE_PATH"`
+echo "SCRIPT: Converting input prompt '$INPUT_FILE_PATH' to image :)"
+python3 text2image.py --prompt="$prompt" --output=$OUTPUT_FILE

examples/stable-diffusion/stable-diff.yaml

@@ -0,0 +1,17 @@
+functions:
+  oscar:
+  - oscar-intertwin:
+      name: stable-diffusion-tf
+      memory: 16Gi
+      cpu: '4'
+      image: ghcr.io/grycap/stable-diffusion-tf:latest
+      script: script.sh
+      log_level: DEBUG
+      vo: "vo.example.eu"
+      allowed_users: []
+      input:
+      - storage_provider: minio.default
+        path: stablediff/input
+      output:
+      - storage_provider: minio.default
+        path: stablediff/output

go.mod

@@ -34,6 +34,7 @@ require (
 
 require (
 	github.com/fatih/color v1.14.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/rs/xid v1.4.0 // indirect

go.sum

@@ -108,6 +108,8 @@ github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXK
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
+github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

pkg/backends/k8s.go

@@ -66,7 +66,7 @@ func (k *KubeBackend) ListServices() ([]*types.Service, error) {
 	services := []*types.Service{}
 
 	for _, cm := range configmaps.Items {
-		service, err := getServiceFromConfigMap(&cm)
+		service, err := getServiceFromConfigMap(&cm) // #nosec G601
 		if err != nil {
 			return nil, err
 		}

pkg/backends/knative.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"os"
 	"strconv"
 
 	"github.com/grycap/oscar/v3/pkg/imagepuller"
@@ -33,8 +32,8 @@ import (
 	knclientset "knative.dev/serving/pkg/client/clientset/versioned"
 )
 
-// Custom logger
-var knativeLogger = log.New(os.Stdout, "[KNATIVE] ", log.Flags())
+// Custom logger - uncomment if needed
+// var knativeLogger = log.New(os.Stdout, "[KNATIVE] ", log.Flags())
 
 // KnativeBackend struct to represent a Knative client
 type KnativeBackend struct {
@@ -84,7 +83,7 @@ func (kn *KnativeBackend) ListServices() ([]*types.Service, error) {
 	services := []*types.Service{}
 
 	for _, cm := range configmaps.Items {
-		service, err := getServiceFromConfigMap(&cm)
+		service, err := getServiceFromConfigMap(&cm) // #nosec G601
 		if err != nil {
 			return nil, err
 		}
@@ -129,7 +128,10 @@ func (kn *KnativeBackend) CreateService(service types.Service) error {
 
 	//Create an expose service
 	if service.Expose.APIPort != 0 {
-		types.CreateExpose(service, kn.kubeClientset, kn.config)
+		err = types.CreateExpose(service, kn.kubeClientset, kn.config)
+		if err != nil {
+			return err
+		}
 	}
 	//Create deaemonset to cache the service image on all the nodes
 	if service.ImagePrefetch {

pkg/backends/openfaas.go

@@ -92,7 +92,7 @@ func (of *OpenfaasBackend) ListServices() ([]*types.Service, error) {
 	services := []*types.Service{}
 
 	for _, cm := range configmaps.Items {
-		service, err := getServiceFromConfigMap(&cm)
+		service, err := getServiceFromConfigMap(&cm) // #nosec G601
 		if err != nil {
 			return nil, err
 		}