diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 9d87b03ef05..21713435fc8 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -162,6 +162,33 @@ parameters: # stripeCount: <> # (optional) The object size in bytes. # objectSize: <> + + # rbd volume QoS. + # QoS provides settings for rbd volume read/write iops and read/write bandwidth. + # There are 4 base qos parameters among them, when users apply for a volume + # capacity equal to or less than BaseVolSizebytes, use base qos limit. + # For the portion of capacity exceeding BaseVolSizebytes, QoS will be increased + # in steps set per GiB. If the step size parameter per GiB is not provided, + # only base QoS limit will be used and not associated with capacity size. + # note: currently supports rbd-nbd mounter. + # (optional) the base limit of read operations per second. + # BaseReadIops: <> + # (optional) the base limit of write operations per second. + # BaseWriteIops: <> + # (optional) the base limit of read bytes per second. + # BaseReadBytesPerSecond: <> + # (optional) the base limit of write bytes per second. + # BaseWriteBytesPerSecond: <> + # (optional) the limit of read operations per GiB. + # ReadIopsPerGB: <> + # (optional) the limit of write operations per GiB. + # WriteIopsPerGB: <> + # (optional) the limit of read bytes per GiB. + # ReadBpsPerGB: <> + # (optional) the limit of write bytes per GiB. + # WriteBpsPerGB: <> + # (optional) the min size of volume waht use to calc qos beased on capacity + # BaseVolSizeBytes:<> reclaimPolicy: Delete allowVolumeExpansion: true diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 3ec067c3ecf..24de665fdf2 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest( return nil, status.Error(codes.InvalidArgument, err.Error()) } + // Get QosParameters from SC if qos configuration existing in SC + err = rbdVol.SetQOS(ctx, req.GetParameters()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + err = rbdVol.Connect(cr) if err != nil { log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err) @@ -424,6 +430,19 @@ func (cs *ControllerServer) CreateVolume( return nil, err } + // Apply Qos parameters to rbd image. + err = rbdVol.ApplyQOS(ctx) + if err != nil { + log.DebugLog(ctx, "failed apply QOS for rbd image: %v", err) + + return nil, status.Error(codes.Internal, err.Error()) + } + // Save Qos parameters from SC in Image medatate, we will use it while resize volume. + err = rbdVol.SaveQOS(ctx, req.GetParameters()) + if err != nil { + return nil, err + } + // Set Metadata on PV Create metadata := k8s.GetVolumeMetadata(req.GetParameters()) err = rbdVol.setAllMetadata(metadata) @@ -1605,6 +1624,13 @@ func (cs *ControllerServer) ControllerExpandVolume( if err != nil { log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err) + return nil, status.Error(codes.Internal, err.Error()) + } + // adjust rbd qos after resize volume. + err = rbdVol.AdjustQOS(ctx) + if err != nil { + log.DebugLog(ctx, "failed adjust QOS for rbd image") + return nil, status.Error(codes.Internal, err.Error()) } } diff --git a/internal/rbd/qos.go b/internal/rbd/qos.go new file mode 100644 index 00000000000..7ba69884825 --- /dev/null +++ b/internal/rbd/qos.go @@ -0,0 +1,299 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + "errors" + "strconv" + + "github.com/ceph/ceph-csi/internal/util/log" + + librbd "github.com/ceph/go-ceph/rbd" +) + +const ( + // Qos parameters name of StorageClass. + baseReadIops = "BaseReadIops" + baseWriteIops = "BaseWriteIops" + baseReadBytesPerSecond = "BaseReadBytesPerSecond" + baseWriteBytesPerSecond = "BaseWriteBytesPerSecond" + readIopsPerGB = "ReadIopsPerGB" + writeIopsPerGB = "WriteIopsPerGB" + readBpsPerGB = "ReadBpsPerGB" + writeBpsPerGB = "WriteBpsPerGB" + baseVolSizeBytes = "BaseVolSizeBytes" + + // Qos type name of rbd image. + readIopsLimit = "rbd_qos_read_iops_limit" + writeIopsLimit = "rbd_qos_write_iops_limit" + readBpsLimit = "rbd_qos_read_bps_limit" + writeBpsLimit = "rbd_qos_write_bps_limit" + metadataConfPrefix = "conf_" + + // The params use to calc qos based on capacity. + baseQosReadIopsLimit = "rbd_base_qos_read_iops_limit" + baseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit" + baseQosReadBpsLimit = "rbd_base_qos_read_bps_limit" + baseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit" + readIopsPerGBLimit = "rbd_read_iops_per_gb_limit" + writeIopsPerGBLimit = "rbd_write_iops_per_gb_limit" + readBpsPerGBLimit = "rbd_read_bps_per_gb_limit" + writeBpsPerGBLimit = "rbd_write_bps_per_gb_limit" + baseQosVolSize = "rbd_base_qos_vol_size" +) + +type qosSpec struct { + baseLimitType string + baseLimit string + perGBLimitType string + perGBLimit string + provide bool +} + +func parseQosParams( + params map[string]string, +) map[string]*qosSpec { + rbdQosParameters := map[string]*qosSpec{ + baseReadIops: {readIopsLimit, "", readIopsPerGB, "", false}, + baseWriteIops: {writeIopsLimit, "", writeIopsPerGB, "", false}, + baseReadBytesPerSecond: {readBpsLimit, "", readBpsPerGB, "", false}, + baseWriteBytesPerSecond: {writeBpsLimit, "", writeBpsPerGB, "", false}, + } + for k, v := range params { + if qos, ok := rbdQosParameters[k]; ok && v != "" { + qos.baseLimit = v + qos.provide = true + for p, q := range params { + if p == qos.perGBLimitType { + if q != "" { + qos.perGBLimit = q + } + } + } + } + } + + return rbdQosParameters +} + +func (rv *rbdVolume) SetQOS( + ctx context.Context, + params map[string]string, +) error { + baseVolSize := "" + if v, ok := params[baseVolSizeBytes]; ok && v != "" { + baseVolSize = v + } + + rbdQosParameters := parseQosParams(params) + for _, qos := range rbdQosParameters { + if qos.provide { + err := calcQosBasedOnCapacity(ctx, rv, *qos, baseVolSize) + if err != nil { + return err + } + } + } + + return nil +} + +func (rv *rbdVolume) ApplyQOS( + ctx context.Context, +) error { + if len(rv.QosParameters) != 0 { + err := setRbdImageQos(ctx, rv) + if err != nil { + return err + } + } + + return nil +} + +func calcQosBasedOnCapacity( + ctx context.Context, + rbdVol *rbdVolume, + qos qosSpec, + baseVolSize string, +) error { + if rbdVol.QosParameters == nil { + rbdVol.QosParameters = make(map[string]string) + } + + // Don't set qos if base qos limit empty. + if qos.baseLimit == "" { + return nil + } + baseLimit, err := strconv.ParseInt(qos.baseLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + // if provide qosPerGB and baseVolSize, we will set qos based on capacity, + // otherwise, we only set base qos limit. + if qos.perGBLimit != "" && baseVolSize != "" { + perGBLimit, err := strconv.ParseInt(qos.perGBLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + baseVolSizeInt, err := strconv.ParseInt(baseVolSize, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + if rbdVol.VolSize <= baseVolSizeInt { + rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit + } else { + capacityQos := (rbdVol.VolSize - baseVolSizeInt) / int64(oneGB) * perGBLimit + finalQosLimit := baseLimit + capacityQos + rbdVol.QosParameters[qos.baseLimitType] = strconv.FormatInt(finalQosLimit, 10) + } + } else { + rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit + } + + return nil +} + +func setRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) error { + for k, v := range rbdVol.QosParameters { + err := rbdVol.SetMetadata(metadataConfPrefix+k, v) + if err != nil { + log.ErrorLog(ctx, "failed to set rbd qos, %s: %s, %v", k, v, err) + + return err + } + } + + return nil +} + +func (rv *rbdVolume) SaveQOS( + ctx context.Context, + params map[string]string, +) error { + needSaveQosParameters := map[string]string{ + baseReadIops: baseQosReadIopsLimit, + baseWriteIops: baseQosWriteIopsLimit, + baseReadBytesPerSecond: baseQosReadBpsLimit, + baseWriteBytesPerSecond: baseQosWriteBpsLimit, + readIopsPerGB: readIopsPerGBLimit, + writeIopsPerGB: writeIopsPerGBLimit, + readBpsPerGB: readBpsPerGBLimit, + writeBpsPerGB: writeBpsPerGBLimit, + baseVolSizeBytes: baseQosVolSize, + } + for k, v := range params { + if param, ok := needSaveQosParameters[k]; ok { + if v != "" { + err := rv.SetMetadata(param, v) + if err != nil { + log.ErrorLog(ctx, "failed to save metadata, %s: %s, %v", k, v, err) + + return err + } + } + } + } + + return nil +} + +func getRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) (map[string]qosSpec, string, error) { + QosParams := map[string]struct { + rbdQosType string + rbdQosPerGBType string + }{ + baseQosReadIopsLimit: {readIopsLimit, readIopsPerGBLimit}, + baseQosWriteIopsLimit: {writeIopsLimit, writeIopsPerGBLimit}, + baseQosReadBpsLimit: {readBpsLimit, readBpsPerGBLimit}, + baseQosWriteBpsLimit: {writeBpsLimit, writeBpsPerGBLimit}, + } + rbdQosParameters := make(map[string]qosSpec) + for k, param := range QosParams { + baseLimit, err := rbdVol.GetMetadata(k) + if errors.Is(err, librbd.ErrNotFound) { + // if base qos dose not exist, skipping. + continue + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + + return nil, "", err + } + perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType) + if errors.Is(err, librbd.ErrNotFound) { + // rbdQosPerGBType does not exist, set it empty. + perGBLimit = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + + return nil, "", err + } + rbdQosParameters[k] = qosSpec{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true} + } + baseVolSize, err := rbdVol.GetMetadata(baseQosVolSize) + if errors.Is(err, librbd.ErrNotFound) { + // rbdBaseQosVolSize does not exist, set it empty. + baseVolSize = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + + return nil, "", err + } + + return rbdQosParameters, baseVolSize, nil +} + +func (rv *rbdVolume) AdjustQOS( + ctx context.Context, +) error { + rbdQosParameters, baseQosVolSize, err := getRbdImageQos(ctx, rv) + if err != nil { + log.ErrorLog(ctx, "get image metadata failed: %v", err) + + return err + } + for _, param := range rbdQosParameters { + err = calcQosBasedOnCapacity(ctx, rv, param, baseQosVolSize) + if err != nil { + return err + } + } + err = setRbdImageQos(ctx, rv) + if err != nil { + log.ErrorLog(ctx, "set image metadata failed: %v", err) + + return err + } + + return nil +} diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index fbae7f4181d..ad1ca9623ba 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -148,6 +148,9 @@ type rbdImage struct { EnableMetadata bool // ParentInTrash indicates the parent image is in trash. ParentInTrash bool + + // RBD QoS configuration + QosParameters map[string]string } // check that rbdVolume implements the types.Volume interface.