From e730f20cbbb9d822f3cb175b0376347a4d9e61e3 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 19 Jan 2024 10:17:48 +0100 Subject: [PATCH] First version of multitenancy implementation (testing) --- main.go | 2 +- pkg/handlers/create.go | 55 +++++++++++++------- pkg/handlers/update.go | 13 ++++- pkg/types/service.go | 4 ++ pkg/utils/auth/auth.go | 17 +++++-- pkg/utils/auth/multitenancy.go | 92 ++++++++++++++++++++++++++++++++++ pkg/utils/auth/oidc.go | 42 ++++++++++++++-- pkg/utils/minio.go | 90 +++++++++++++++++++++++++++++++++ 8 files changed, 287 insertions(+), 28 deletions(-) create mode 100644 pkg/utils/auth/multitenancy.go diff --git a/main.go b/main.go index 364ffd6f..252fa8c1 100644 --- a/main.go +++ b/main.go @@ -77,7 +77,7 @@ func main() { r := gin.Default() // Define system group with basic auth middleware - system := r.Group("/system", auth.GetAuthMiddleware(cfg)) + system := r.Group("/system", auth.GetAuthMiddleware(cfg, kubeClientset)) // Config path system.GET("/config", handlers.MakeConfigHandler(cfg)) diff --git a/pkg/handlers/create.go b/pkg/handlers/create.go index bf6e15bd..e51af010 100644 --- a/pkg/handlers/create.go +++ b/pkg/handlers/create.go @@ -57,23 +57,18 @@ func MakeCreateHandler(cfg *types.Config, back types.ServerlessBackend) gin.Hand checkValues(&service, cfg) if service.VO != "" { - oidcManager, _ := auth.NewOIDCManager(cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) - authHeader := c.GetHeader("Authorization") - rawToken := strings.TrimPrefix(authHeader, "Bearer ") - hasVO, err2 := oidcManager.UserHasVO(rawToken, service.VO) - - if err2 != nil { - c.String(http.StatusInternalServerError, err2.Error()) - return - } - - if !hasVO { - c.String(http.StatusBadRequest, fmt.Sprintf("This user isn't enrrolled on the vo: %v", service.VO)) - return + err := checkVOIdentity(&service, cfg, authHeader) + if err != nil { + c.String(http.StatusBadRequest, fmt.Sprintf("%v"), err) } } + uid_origin, _ := c.Get("uid_origin") + uid := fmt.Sprintf("%v", uid_origin) + service.Labels["uid"] = uid + service.AllowedUsers = append(service.AllowedUsers, uid) + // Create the service if err := back.CreateService(service); err != nil { // Check if error is caused because the service name provided already exists @@ -93,7 +88,7 @@ func MakeCreateHandler(cfg *types.Config, back types.ServerlessBackend) gin.Hand } // Create buckets/folders based on the Input and Output and enable notifications - if err := createBuckets(&service, cfg); err != nil { + if err := createBuckets(&service, cfg, service.AllowedUsers); err != nil { if err == errInput { c.String(http.StatusBadRequest, err.Error()) } else { @@ -140,10 +135,6 @@ func checkValues(service *types.Service, cfg *types.Config) { service.Labels[types.YunikornApplicationIDLabel] = service.Name service.Labels[types.YunikornQueueLabel] = fmt.Sprintf("%s.%s.%s", types.YunikornRootQueue, types.YunikornOscarQueue, service.Name) - if service.VO != "" { - service.Labels["vo"] = service.VO - } - // Create default annotations map if service.Annotations == nil { service.Annotations = make(map[string]string) @@ -171,7 +162,7 @@ func checkValues(service *types.Service, cfg *types.Config) { service.Token = utils.GenerateToken() } -func createBuckets(service *types.Service, cfg *types.Config) error { +func createBuckets(service *types.Service, cfg *types.Config, allowed_users []string) error { var s3Client *s3.S3 var cdmiClient *cdmi.Client var provName, provID string @@ -233,6 +224,14 @@ func createBuckets(service *types.Service, cfg *types.Config) error { return fmt.Errorf("error creating bucket %s: %v", splitPath[0], err) } } + + // Create group for the service and add users + // TODO error control + + minIOAdminClient, _ := utils.MakeMinIOAdminClient(cfg) + minIOAdminClient.CreateServiceGroup(splitPath[0]) + minIOAdminClient.AddUserToGroup(allowed_users, splitPath[0]) + // Create folder(s) if len(splitPath) == 2 { // Add "/" to the end of the key in order to create a folder @@ -347,6 +346,24 @@ func isStorageProviderDefined(storageName string, storageID string, providers *t return ok } +func checkVOIdentity(service *types.Service, cfg *types.Config, authHeader string) error { + oidcManager, _ := auth.NewOIDCManager(cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) + rawToken := strings.TrimPrefix(authHeader, "Bearer ") + hasVO, err := oidcManager.UserHasVO(rawToken, service.VO) + + if err != nil { + return err + } + + if !hasVO { + return fmt.Errorf("This user isn't enrrolled on the vo: %v", service.VO) + } + + service.Labels["vo"] = service.VO + + return nil +} + func registerMinIOWebhook(name string, token string, minIO *types.MinIOProvider, cfg *types.Config) error { minIOAdminClient, err := utils.MakeMinIOAdminClient(cfg) if err != nil { diff --git a/pkg/handlers/update.go b/pkg/handlers/update.go index e98882ca..66d384c9 100644 --- a/pkg/handlers/update.go +++ b/pkg/handlers/update.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" ) +// TODO comprobar VO como en createService // MakeUpdateHandler makes a handler for updating services func MakeUpdateHandler(cfg *types.Config, back types.ServerlessBackend) gin.HandlerFunc { return func(c *gin.Context) { @@ -53,6 +54,14 @@ func MakeUpdateHandler(cfg *types.Config, back types.ServerlessBackend) gin.Hand return } + if newService.VO != "" && newService.VO != oldService.VO { + authHeader := c.GetHeader("Authorization") + err := checkVOIdentity(&newService, cfg, authHeader) + if err != nil { + c.String(http.StatusBadRequest, fmt.Sprintf("%v"), err) + } + } + // Update the service if err := back.UpdateService(newService); err != nil { c.String(http.StatusInternalServerError, fmt.Sprintf("Error updating the service: %v", err)) @@ -68,6 +77,7 @@ func MakeUpdateHandler(cfg *types.Config, back types.ServerlessBackend) gin.Hand provName = strings.ToLower(provSlice[0]) } if provName == types.MinIOName { + // Register minio webhook and restart the server if err := registerMinIOWebhook(newService.Name, newService.Token, newService.StorageProviders.MinIO[types.DefaultProvider], cfg); err != nil { back.UpdateService(*oldService) @@ -107,5 +117,6 @@ func updateBuckets(newService, oldService *types.Service, cfg *types.Config) err } // Create the input and output buckets/folders from newService - return createBuckets(newService, cfg) + // TODO fix + return createBuckets(newService, cfg, newService.AllowedUsers) } diff --git a/pkg/types/service.go b/pkg/types/service.go index 45182cb1..fc4f04e9 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -240,6 +240,10 @@ type Service struct { // Clusters configuration for the OSCAR clusters that can be used as service's replicas // Optional Clusters map[string]Cluster `json:"clusters,omitempty"` + + // List of EGI UID's identifying the users that will have visibility of the service and its MinIO storage provider + // Optional (If the list is empty we asume the visibility is public for all cluster users) + AllowedUsers []string `json:"allowed_users"` } // ToPodSpec returns a k8s podSpec from the Service diff --git a/pkg/utils/auth/auth.go b/pkg/utils/auth/auth.go index bf589ae6..b0c22545 100644 --- a/pkg/utils/auth/auth.go +++ b/pkg/utils/auth/auth.go @@ -21,28 +21,37 @@ import ( "github.com/gin-gonic/gin" "github.com/grycap/oscar/v2/pkg/types" + "github.com/grycap/oscar/v2/pkg/utils" + "k8s.io/client-go/kubernetes" ) // GetAuthMiddleware returns the appropriate gin auth middleware -func GetAuthMiddleware(cfg *types.Config) gin.HandlerFunc { +func GetAuthMiddleware(cfg *types.Config, kubeClientset *kubernetes.Clientset) gin.HandlerFunc { if !cfg.OIDCEnable { return gin.BasicAuth(gin.Accounts{ // Use the config's username and password for basic auth cfg.Username: cfg.Password, }) } - return CustomAuth(cfg) + return CustomAuth(cfg, kubeClientset) } // CustomAuth returns a custom auth handler (gin middleware) -func CustomAuth(cfg *types.Config) gin.HandlerFunc { +func CustomAuth(cfg *types.Config, kubeClientset *kubernetes.Clientset) gin.HandlerFunc { basicAuthHandler := gin.BasicAuth(gin.Accounts{ // Use the config's username and password for basic auth cfg.Username: cfg.Password, }) - oidcHandler := getOIDCMiddleware(cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) + //TODO Initialize MinIO client and create all_users_group + minIOAdminClient, err := utils.MakeMinIOAdminClient(cfg) + if err != nil { + // TODO manage error + } + + minIOAdminClient.CreateAllUsersGroup() + oidcHandler := getOIDCMiddleware(kubeClientset, minIOAdminClient, cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) return func(c *gin.Context) { authHeader := c.GetHeader("Authorization") if strings.HasPrefix(authHeader, "Bearer ") { diff --git a/pkg/utils/auth/multitenancy.go b/pkg/utils/auth/multitenancy.go new file mode 100644 index 00000000..0fa58ef1 --- /dev/null +++ b/pkg/utils/auth/multitenancy.go @@ -0,0 +1,92 @@ +/* +Copyright (C) GRyCAP - I3M - UPV + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ServicesNamespace = "oscar-svc" + +type multitenancyConfig struct { + kubeClientset *kubernetes.Clientset + owner_uid string + usersCache []string +} + +func NewMultitenancyConfig(kubeClientset *kubernetes.Clientset, uid string) *multitenancyConfig { + return &multitenancyConfig{ + kubeClientset: kubeClientset, + owner_uid: uid, + usersCache: []string{uid}, + } +} + +// TODO periodically check that the users stored on cache still exist on MinIO (cronjob) +func (mc *multitenancyConfig) UpdateCacheStatus() { + // 1. List users on MinIO + // 2. List secrets + // 3. Compare both lists and delete from secrets the missmatchs + // 4. updateCache +} + +func (mc *multitenancyConfig) UpdateCache(uid string) { + mc.usersCache = append(mc.usersCache, uid) +} + +func (mc *multitenancyConfig) ClearCache() { + mc.usersCache = nil +} + +func (mc *multitenancyConfig) UserExists(uid string) bool { + for _, id := range mc.usersCache { + if id == uid { + return true + } + } + return false +} + +func (mc *multitenancyConfig) CreateSecretForOIDC(uid string, sk string) error { + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: uid, + Namespace: ServicesNamespace, + }, + Immutable: new(bool), + StringData: map[string]string{ + "oidc_uid": uid, + "accessKey": uid, + "secretKey": sk, + }, + } + + _, err := mc.kubeClientset.CoreV1().Secrets(ServicesNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + + if err != nil { + return err + } + + mc.UpdateCache(uid) + + return nil +} diff --git a/pkg/utils/auth/oidc.go b/pkg/utils/auth/oidc.go index b1baa65c..ac66478f 100644 --- a/pkg/utils/auth/oidc.go +++ b/pkg/utils/auth/oidc.go @@ -18,16 +18,24 @@ package auth import ( "context" + + "crypto/rand" + "encoding/base64" "net/http" "strings" "github.com/coreos/go-oidc/v3/oidc" "github.com/gin-gonic/gin" + "github.com/grycap/oscar/v2/pkg/utils" "golang.org/x/oauth2" + "k8s.io/client-go/kubernetes" ) -// EGIGroupsURNPrefix prefix to identify EGI group URNs -const EGIGroupsURNPrefix = "urn:mace:egi.eu:group" +const ( + // EGIGroupsURNPrefix prefix to identify EGI group URNs + EGIGroupsURNPrefix = "urn:mace:egi.eu:group" + SecretKeyLength = 10 +) // oidcManager struct to represent a OIDC manager, including a cache of tokens type oidcManager struct { @@ -65,7 +73,7 @@ func NewOIDCManager(issuer string, subject string, groups []string) (*oidcManage } // getIODCMiddleware returns the Gin's handler middleware to validate OIDC-based auth -func getOIDCMiddleware(issuer string, subject string, groups []string) gin.HandlerFunc { +func getOIDCMiddleware(kubeClientset *kubernetes.Clientset, minIOAdminClient *utils.MinIOAdminClient, issuer string, subject string, groups []string) gin.HandlerFunc { oidcManager, err := NewOIDCManager(issuer, subject, groups) if err != nil { return func(c *gin.Context) { @@ -73,6 +81,8 @@ func getOIDCMiddleware(issuer string, subject string, groups []string) gin.Handl } } + mc := NewMultitenancyConfig(kubeClientset, subject) + return func(c *gin.Context) { // Get token from headers authHeader := c.GetHeader("Authorization") @@ -87,6 +97,23 @@ func getOIDCMiddleware(issuer string, subject string, groups []string) gin.Handl c.AbortWithStatus(http.StatusUnauthorized) return } + + ui, _ := oidcManager.getUserInfo(rawToken) + uid := ui.subject + + // Check if exist MinIO user in cached users list + exists := mc.UserExists(uid) + if !exists { + sk, err := generateRandomKey(SecretKeyLength) + if err != nil { + //TODO manage errr + } + // Create MinIO user and k8s secret with credentials + mc.CreateSecretForOIDC(uid, sk) + minIOAdminClient.CreateMinIOUser(uid, sk) + + c.Set("uid_origin", uid) + } } } @@ -194,3 +221,12 @@ func (om *oidcManager) isAuthorised(rawToken string) bool { return false } + +func generateRandomKey(length int) (string, error) { + key := make([]byte, length) + _, err := rand.Read(key) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(key), nil +} diff --git a/pkg/utils/minio.go b/pkg/utils/minio.go index 5bc2f9ba..60a1551b 100644 --- a/pkg/utils/minio.go +++ b/pkg/utils/minio.go @@ -28,6 +28,8 @@ import ( "github.com/minio/madmin-go" ) +const ALL_USERS_GROUP = "all_users_group" + // MinIOAdminClient struct to represent a MinIO Admin client to configure webhook notifications type MinIOAdminClient struct { adminClient *madmin.AdminClient @@ -79,6 +81,51 @@ func MakeMinIOAdminClient(cfg *types.Config) (*MinIOAdminClient, error) { return minIOAdminClient, nil } +func (minIOAdminClient *MinIOAdminClient) CreateMinIOUser(ak string, sk string) error { + var users []string + err := minIOAdminClient.adminClient.AddUser(context.TODO(), ak, sk) + if err != nil { + return fmt.Errorf("error creating MinIO user: %v", err) + } + + users[0] = ak + err2 := minIOAdminClient.AddUserToGroup(users, ALL_USERS_GROUP) + if err2 != nil { + return err2 + } + return nil +} + +func (minIOAdminClient *MinIOAdminClient) CreateAllUsersGroup() { + err := createGroup(minIOAdminClient.adminClient, ALL_USERS_GROUP) + if err != nil { + //TODO manage error + } +} + +func (minIOAdminClient *MinIOAdminClient) CreateServiceGroup(bucketName string) { + err := createGroup(minIOAdminClient.adminClient, bucketName) + if err != nil { + //TODO manage error + } +} + +func (minIOAdminClient *MinIOAdminClient) AddUserToGroup(users []string, groupName string) error { + group := madmin.GroupAddRemove{ + Group: groupName, + Members: users, + Status: "enable", + IsRemove: false, + } + + err := minIOAdminClient.adminClient.UpdateGroupMembers(context.TODO(), group) + if err != nil { + return fmt.Errorf("error adding users to group: %v", err) + } + + return nil +} + // RegisterWebhook registers a new webhook in the MinIO configuration func (minIOAdminClient *MinIOAdminClient) RegisterWebhook(name string, token string) error { _, err := minIOAdminClient.adminClient.SetConfigKV(context.TODO(), fmt.Sprintf("notify_webhook:%s endpoint=%s/job/%s auth_token=%s", name, minIOAdminClient.oscarEndpoint.String(), name, token)) @@ -116,3 +163,46 @@ func (minIOAdminClient *MinIOAdminClient) RestartServer() error { return nil } + +func createPolicy(adminClient *madmin.AdminClient, groupName string) error { + policy := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::` + groupName + `*" + ] + } + ] + }` + + err := adminClient.AddCannedPolicy(context.TODO(), groupName, []byte(policy)) + if err != nil { + return fmt.Errorf("error creating MinIO policy for group %s: %v", groupName, err) + } + + err2 := adminClient.SetPolicy(context.TODO(), groupName, groupName, true) + if err2 != nil { + return fmt.Errorf("error setting MinIO policy for group %s: %v", groupName, err2) + } + return nil +} + +func createGroup(adminClient *madmin.AdminClient, groupName string) error { + group := madmin.GroupAddRemove{ + Group: groupName, + Members: []string{}, + Status: "enable", + IsRemove: false, + } + err := adminClient.UpdateGroupMembers(context.TODO(), group) + if err != nil { + return fmt.Errorf("error creating MinIO group %s: %v", groupName, err) + } + + return nil +}