diff --git a/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role-binding.yaml b/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role-binding.yaml deleted file mode 100644 index ec284ac2b9..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-external-attacher-clusterrole-binding -subjects: - - kind: ServiceAccount - name: efs-csi-controller-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: efs-csi-external-attacher-clusterrole - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role.yaml b/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role.yaml deleted file mode 100644 index 6a946b30f5..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-external-attacher-clusterrole -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-controller-sa.yaml b/aws/aws-efs-csi-driver/base/csi-controller-sa.yaml deleted file mode 100644 index 2fffbae4bd..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-controller-sa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: efs-csi-controller-sa \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-controller-stateful-set.yaml b/aws/aws-efs-csi-driver/base/csi-controller-stateful-set.yaml deleted file mode 100644 index 6d78963b3f..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-controller-stateful-set.yaml +++ /dev/null @@ -1,51 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: efs-csi-controller -spec: - serviceName: efs-csi-controller - replicas: 1 - selector: - matchLabels: - app: efs-csi-controller - template: - metadata: - labels: - app: efs-csi-controller - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccount: efs-csi-controller-sa - #priorityClassName: system-cluster-critical - tolerations: - - key: CriticalAddonsOnly - operator: Exists - containers: - - name: efs-plugin - image: amazon/aws-efs-csi-driver:latest - imagePullPolicy: Always - args : - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=5 - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v0.4.2 - imagePullPolicy: Always - args: - - --csi-address=$(ADDRESS) - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - volumes: - - name: socket-dir - emptyDir: {} diff --git a/aws/aws-efs-csi-driver/base/csi-default-storage.yaml b/aws/aws-efs-csi-driver/base/csi-default-storage.yaml deleted file mode 100644 index 9588ee67c4..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-default-storage.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: efs-default -provisioner: efs.csi.aws.com \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-driver.yaml b/aws/aws-efs-csi-driver/base/csi-driver.yaml new file mode 100644 index 0000000000..092a69acfc --- /dev/null +++ b/aws/aws-efs-csi-driver/base/csi-driver.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: efs.csi.aws.com +spec: + attachRequired: false \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-node-cluster-role-binding.yaml b/aws/aws-efs-csi-driver/base/csi-node-cluster-role-binding.yaml deleted file mode 100644 index c99b8884bd..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-node-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-node-clusterole-binding -subjects: - - kind: ServiceAccount - name: efs-csi-node-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: efs-csi-node-clusterrole - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-node-cluster-role.yaml b/aws/aws-efs-csi-driver/base/csi-node-cluster-role.yaml deleted file mode 100644 index 77bdc5dd37..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-node-cluster-role.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-node-clusterrole -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch", "update"] \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/csi-node-daemon-set.yaml b/aws/aws-efs-csi-driver/base/csi-node-daemonset.yaml similarity index 64% rename from aws/aws-efs-csi-driver/base/csi-node-daemon-set.yaml rename to aws/aws-efs-csi-driver/base/csi-node-daemonset.yaml index a64478e1de..8a263ce5ca 100644 --- a/aws/aws-efs-csi-driver/base/csi-node-daemon-set.yaml +++ b/aws/aws-efs-csi-driver/base/csi-node-daemonset.yaml @@ -1,3 +1,5 @@ +--- +# Node Service kind: DaemonSet apiVersion: apps/v1 metadata: @@ -11,14 +13,16 @@ spec: labels: app: efs-csi-node spec: - serviceAccount: efs-csi-node-sa + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true + tolerations: + - operator: Exists containers: - name: efs-plugin securityContext: privileged: true image: amazon/aws-efs-csi-driver:latest - imagePullPolicy: Always args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr @@ -32,16 +36,24 @@ spec: mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi - - name: device-dir - mountPath: /dev + - name: efs-state-dir + mountPath: /var/run/efs + ports: + - containerPort: 9809 + name: healthz + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 - name: csi-driver-registrar - image: quay.io/k8scsi/driver-registrar:v0.4.2 - imagePullPolicy: Always + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 args: - --csi-address=$(ADDRESS) - - --mode=node-register - - --driver-requires-attachment=true - - --pod-info-mount-version="v1" - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=5 env: @@ -58,20 +70,30 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + - name: liveness-probe + imagePullPolicy: Always + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9809 + volumeMounts: + - mountPath: /csi + name: plugin-dir volumes: - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/efs.csi.aws.com/ type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins/ - type: Directory - - name: device-dir + - name: efs-state-dir hostPath: - path: /dev - type: Directory + path: /var/run/efs + type: DirectoryOrCreate + diff --git a/aws/aws-efs-csi-driver/base/csi-node-sa.yaml b/aws/aws-efs-csi-driver/base/csi-node-sa.yaml deleted file mode 100644 index 1b9165c1f0..0000000000 --- a/aws/aws-efs-csi-driver/base/csi-node-sa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: efs-csi-node-sa \ No newline at end of file diff --git a/aws/aws-efs-csi-driver/base/kustomization.yaml b/aws/aws-efs-csi-driver/base/kustomization.yaml index 7f3d5b39ec..70cefc0717 100644 --- a/aws/aws-efs-csi-driver/base/kustomization.yaml +++ b/aws/aws-efs-csi-driver/base/kustomization.yaml @@ -2,24 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: kubeflow resources: -- csi-controller-stateful-set.yaml -- csi-attacher-cluster-role.yaml -- csi-attacher-cluster-role-binding.yaml -- csi-controller-sa.yaml -- csi-node-cluster-role.yaml -- csi-node-cluster-role-binding.yaml -- csi-node-daemon-set.yaml -- csi-node-sa.yaml -- csi-default-storage.yaml +- csi-driver.yaml +- csi-node-daemonset.yaml generatorOptions: disableNameSuffixHash: true images: -- name: quay.io/k8scsi/driver-registrar - newName: quay.io/k8scsi/driver-registrar - newTag: v0.4.2 - name: amazon/aws-efs-csi-driver newName: amazon/aws-efs-csi-driver - newTag: latest -- name: quay.io/k8scsi/csi-attacher - newName: quay.io/k8scsi/csi-attacher - newTag: v0.4.2 + newTag: v0.3.0 \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role-binding.yaml b/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role-binding.yaml deleted file mode 100644 index 37740bc873..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-external-attacher-clusterrole-binding -subjects: - - kind: ServiceAccount - name: fsx-csi-controller-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: fsx-csi-external-attacher-clusterrole - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role.yaml b/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role.yaml deleted file mode 100644 index a2a898f47d..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-external-attacher-clusterrole -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role-binding.yaml b/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role-binding.yaml deleted file mode 100644 index ec55a48535..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-provisioner-binding -subjects: - - kind: ServiceAccount - name: csi-controller-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: external-provisioner-role - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role.yaml b/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role.yaml deleted file mode 100644 index f31682f695..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-provisioner-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-controller-sa.yaml b/aws/aws-fsx-csi-driver/base/csi-controller-sa.yaml index 1d6bb6c5f4..4404ec2d3e 100644 --- a/aws/aws-fsx-csi-driver/base/csi-controller-sa.yaml +++ b/aws/aws-fsx-csi-driver/base/csi-controller-sa.yaml @@ -2,3 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: fsx-csi-controller-sa + namespace: kubeflow + #Enable if EKS IAM for SA is used + #annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-controller-stateful-set.yaml b/aws/aws-fsx-csi-driver/base/csi-controller.yaml similarity index 62% rename from aws/aws-fsx-csi-driver/base/csi-controller-stateful-set.yaml rename to aws/aws-fsx-csi-driver/base/csi-controller.yaml index db76f27976..ce8fff6098 100644 --- a/aws/aws-fsx-csi-driver/base/csi-controller-stateful-set.yaml +++ b/aws/aws-fsx-csi-driver/base/csi-controller.yaml @@ -1,10 +1,10 @@ -kind: StatefulSet +--- +kind: Deployment apiVersion: apps/v1 metadata: name: fsx-csi-controller spec: - serviceName: fsx-csi-controller - replicas: 1 + replicas: 2 selector: matchLabels: app: fsx-csi-controller @@ -12,11 +12,11 @@ spec: metadata: labels: app: fsx-csi-controller - annotations: - sidecar.istio.io/inject: "false" spec: + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 serviceAccount: fsx-csi-controller-sa -# priorityClassName: system-cluster-critical tolerations: - key: CriticalAddonsOnly operator: Exists @@ -30,27 +30,29 @@ spec: env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: AWS_SECRET_ACCESS_KEY + optional: true volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v0.4.2 - args: - - --provisioner=fsx.csi.aws.com - - --csi-address=$(ADDRESS) - - --connection-timeout=5m - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v0.4.2 + image: quay.io/k8scsi/csi-provisioner:v1.3.0 args: + - --timeout=5m - --csi-address=$(ADDRESS) - --v=5 + - --enable-leader-election + - --leader-election-type=leases env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/aws/aws-fsx-csi-driver/base/csi-default-storage.yaml b/aws/aws-fsx-csi-driver/base/csi-default-storage.yaml deleted file mode 100644 index 8f9f8c7ec5..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-default-storage.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: fsx-default -provisioner: fsx.csi.aws.com \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-driver.yaml b/aws/aws-fsx-csi-driver/base/csi-driver.yaml new file mode 100644 index 0000000000..71b99a00fd --- /dev/null +++ b/aws/aws-fsx-csi-driver/base/csi-driver.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: fsx.csi.aws.com +spec: + attachRequired: false \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-node-cluster-role-binding.yaml b/aws/aws-fsx-csi-driver/base/csi-node-cluster-role-binding.yaml deleted file mode 100644 index f919a04f30..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-node-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-node-clusterrole-binding -subjects: - - kind: ServiceAccount - name: fsx-csi-node-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: fsx-csi-node-clusterrole - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-node-cluster-role.yaml b/aws/aws-fsx-csi-driver/base/csi-node-cluster-role.yaml deleted file mode 100644 index a338f44250..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-node-cluster-role.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-node-clusterrole -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch", "update"] diff --git a/aws/aws-fsx-csi-driver/base/csi-node-daemonset.yaml b/aws/aws-fsx-csi-driver/base/csi-node-daemonset.yaml index a461b85ef4..6f231d1e49 100644 --- a/aws/aws-fsx-csi-driver/base/csi-node-daemonset.yaml +++ b/aws/aws-fsx-csi-driver/base/csi-node-daemonset.yaml @@ -1,7 +1,8 @@ +--- kind: DaemonSet apiVersion: apps/v1 metadata: - name: fsx-csi-node-ds + name: fsx-csi-node spec: selector: matchLabels: @@ -11,7 +12,9 @@ spec: labels: app: fsx-csi-node spec: - serviceAccount: fsx-csi-node-sa + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 hostNetwork: true containers: - name: fsx-plugin @@ -31,15 +34,22 @@ spec: mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi - - name: device-dir - mountPath: /dev + ports: + - containerPort: 9810 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 - name: csi-driver-registrar - image: quay.io/k8scsi/driver-registrar:v0.4.2 + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 args: - --csi-address=$(ADDRESS) - - --mode=node-register - - --driver-requires-attachment=true - - --pod-info-mount-version="v1" - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=5 env: @@ -56,20 +66,25 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + - name: liveness-probe + imagePullPolicy: Always + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9810 + volumeMounts: + - mountPath: /csi + name: plugin-dir volumes: - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/fsx.csi.aws.com/ - type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins/ + path: /var/lib/kubelet/plugins_registry/ type: Directory - - name: device-dir + - name: plugin-dir hostPath: - path: /dev - type: Directory + path: /var/lib/kubelet/plugins/fsx.csi.aws.com/ + type: DirectoryOrCreate \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-node-sa.yaml b/aws/aws-fsx-csi-driver/base/csi-node-sa.yaml deleted file mode 100644 index b96326c209..0000000000 --- a/aws/aws-fsx-csi-driver/base/csi-node-sa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: fsx-csi-node-sa diff --git a/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role-binding.yaml b/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role-binding.yaml index fb9222a13f..156b19f47b 100644 --- a/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role-binding.yaml +++ b/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role-binding.yaml @@ -1,12 +1,12 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: fsx-csi-provisioner-clusterrole-binding + name: fsx-csi-external-provisioner-binding subjects: - kind: ServiceAccount name: fsx-csi-controller-sa namespace: kubeflow roleRef: kind: ClusterRole - name: fsx-external-provisioner-clusterrole + name: fsx-csi-external-provisioner-role apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role.yaml b/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role.yaml index 8b6271971b..29ab1d07ec 100644 --- a/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role.yaml +++ b/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role.yaml @@ -1,7 +1,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: fsx-external-provisioner-clusterrole + name: fsx-csi-external-provisioner-role rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -14,4 +14,13 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] \ No newline at end of file + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] \ No newline at end of file diff --git a/aws/aws-fsx-csi-driver/base/kustomization.yaml b/aws/aws-fsx-csi-driver/base/kustomization.yaml index b57f8cfb74..60c5670eb3 100644 --- a/aws/aws-fsx-csi-driver/base/kustomization.yaml +++ b/aws/aws-fsx-csi-driver/base/kustomization.yaml @@ -2,31 +2,15 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: kubeflow resources: -- csi-controller-stateful-set.yaml -- csi-attacher-cluster-role.yaml -- csi-attacher-cluster-role-binding.yaml -- csi-controller-cluster-role.yaml -- csi-controller-cluster-role-binding.yaml +- csi-driver.yaml +- csi-controller.yaml - csi-controller-sa.yaml -- csi-node-cluster-role.yaml -- csi-node-cluster-role-binding.yaml - csi-node-daemonset.yaml -- csi-node-sa.yaml - csi-provisioner-cluster-role.yaml - csi-provisioner-cluster-role-binding.yaml -- csi-default-storage.yaml generatorOptions: disableNameSuffixHash: true images: - name: amazon/aws-fsx-csi-driver newName: amazon/aws-fsx-csi-driver - newTag: latest -- name: quay.io/k8scsi/driver-registrar - newName: quay.io/k8scsi/driver-registrar - newTag: v0.4.2 -- name: quay.io/k8scsi/csi-provisioner - newName: quay.io/k8scsi/csi-provisioner - newTag: v0.4.2 -- name: quay.io/k8scsi/csi-attacher - newName: quay.io/k8scsi/csi-attacher - newTag: v0.4.2 + newTag: v0.3.0 \ No newline at end of file diff --git a/kfdef/kfctl_aws.v1.0.2.yaml b/kfdef/kfctl_aws.v1.0.2.yaml index 0e77f76137..dcedd99736 100644 --- a/kfdef/kfctl_aws.v1.0.2.yaml +++ b/kfdef/kfctl_aws.v1.0.2.yaml @@ -175,6 +175,48 @@ spec: name: manifests path: pytorch-job/pytorch-operator name: pytorch-operator + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-crds + name: knative-crds + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-install + name: knative-install + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-crds + name: kfserving-crds + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-install + name: kfserving-install + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: overlays: - application diff --git a/kfdef/kfctl_aws.yaml b/kfdef/kfctl_aws.yaml index 61cb8020b8..502150f545 100644 --- a/kfdef/kfctl_aws.yaml +++ b/kfdef/kfctl_aws.yaml @@ -175,6 +175,48 @@ spec: name: manifests path: pytorch-job/pytorch-operator name: pytorch-operator + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-crds + name: knative-crds + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-install + name: knative-install + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-crds + name: kfserving-crds + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-install + name: kfserving-install + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: overlays: - application diff --git a/kfdef/kfctl_aws_cognito.v1.0.2.yaml b/kfdef/kfctl_aws_cognito.v1.0.2.yaml index 7e90d001da..03b5b57ebc 100644 --- a/kfdef/kfctl_aws_cognito.v1.0.2.yaml +++ b/kfdef/kfctl_aws_cognito.v1.0.2.yaml @@ -170,6 +170,48 @@ spec: name: manifests path: pytorch-job/pytorch-operator name: pytorch-operator + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-crds + name: knative-crds + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-install + name: knative-install + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-crds + name: kfserving-crds + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-install + name: kfserving-install + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: overlays: - application diff --git a/kfdef/kfctl_aws_cognito.yaml b/kfdef/kfctl_aws_cognito.yaml index b0e3cee306..7550d543aa 100644 --- a/kfdef/kfctl_aws_cognito.yaml +++ b/kfdef/kfctl_aws_cognito.yaml @@ -170,6 +170,48 @@ spec: name: manifests path: pytorch-job/pytorch-operator name: pytorch-operator + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-crds + name: knative-crds + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-install + name: knative-install + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-crds + name: kfserving-crds + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-install + name: kfserving-install + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: overlays: - application diff --git a/kfdef/source/master/kfctl_aws.yaml b/kfdef/source/master/kfctl_aws.yaml index 6012ce8430..cc619595de 100644 --- a/kfdef/source/master/kfctl_aws.yaml +++ b/kfdef/source/master/kfctl_aws.yaml @@ -175,6 +175,48 @@ spec: name: manifests path: pytorch-job/pytorch-operator name: pytorch-operator + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-crds + name: knative-crds + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-install + name: knative-install + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-crds + name: kfserving-crds + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-install + name: kfserving-install + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: overlays: - application diff --git a/kfdef/source/master/kfctl_aws_cognito.yaml b/kfdef/source/master/kfctl_aws_cognito.yaml index ea333256f6..3a1ba22d79 100644 --- a/kfdef/source/master/kfctl_aws_cognito.yaml +++ b/kfdef/source/master/kfctl_aws_cognito.yaml @@ -170,6 +170,48 @@ spec: name: manifests path: pytorch-job/pytorch-operator name: pytorch-operator + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-crds + name: knative-crds + - kustomizeConfig: + overlays: + - application + parameters: + - name: namespace + value: knative-serving + repoRef: + name: manifests + path: knative/knative-serving-install + name: knative-install + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-crds + name: kfserving-crds + - kustomizeConfig: + overlays: + - application + repoRef: + name: manifests + path: kfserving/kfserving-install + name: kfserving-install + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: overlays: - application diff --git a/pipeline/pipelines-runner/base/cluster-role.yaml b/pipeline/pipelines-runner/base/cluster-role.yaml index edb248e553..ddc21aaa1b 100644 --- a/pipeline/pipelines-runner/base/cluster-role.yaml +++ b/pipeline/pipelines-runner/base/cluster-role.yaml @@ -20,11 +20,10 @@ rules: - apiGroups: - "" resources: + - persistentvolumes - persistentvolumeclaims verbs: - - create - - delete - - get + - '*' - apiGroups: - snapshot.storage.k8s.io resources: @@ -74,3 +73,15 @@ rules: - jobs verbs: - '*' +- apiGroups: + - machinelearning.seldon.io + resources: + - seldondeployments + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - virtualservices + verbs: + - '*' \ No newline at end of file diff --git a/tests/aws-aws-alb-ingress-controller-base_test.go b/tests/aws-aws-alb-ingress-controller-base_test.go index d1f8bbc955..17a36d265a 100644 --- a/tests/aws-aws-alb-ingress-controller-base_test.go +++ b/tests/aws-aws-alb-ingress-controller-base_test.go @@ -115,7 +115,8 @@ spec: # Repository location of the ALB Ingress Controller. image: docker.io/amazon/aws-alb-ingress-controller imagePullPolicy: Always - serviceAccountName: alb-ingress-controller`) + serviceAccountName: alb-ingress-controller +`) th.writeF("/manifests/aws/aws-alb-ingress-controller/base/service-account.yaml", ` apiVersion: v1 kind: ServiceAccount diff --git a/tests/aws-aws-alb-ingress-controller-overlays-application_test.go b/tests/aws-aws-alb-ingress-controller-overlays-application_test.go index 93d51b4a6c..f92c03408e 100644 --- a/tests/aws-aws-alb-ingress-controller-overlays-application_test.go +++ b/tests/aws-aws-alb-ingress-controller-overlays-application_test.go @@ -167,7 +167,8 @@ spec: # Repository location of the ALB Ingress Controller. image: docker.io/amazon/aws-alb-ingress-controller imagePullPolicy: Always - serviceAccountName: alb-ingress-controller`) + serviceAccountName: alb-ingress-controller +`) th.writeF("/manifests/aws/aws-alb-ingress-controller/base/service-account.yaml", ` apiVersion: v1 kind: ServiceAccount diff --git a/tests/aws-aws-alb-ingress-controller-overlays-vpc_test.go b/tests/aws-aws-alb-ingress-controller-overlays-vpc_test.go index dbdec14f42..28b8e406be 100644 --- a/tests/aws-aws-alb-ingress-controller-overlays-vpc_test.go +++ b/tests/aws-aws-alb-ingress-controller-overlays-vpc_test.go @@ -173,7 +173,8 @@ spec: # Repository location of the ALB Ingress Controller. image: docker.io/amazon/aws-alb-ingress-controller imagePullPolicy: Always - serviceAccountName: alb-ingress-controller`) + serviceAccountName: alb-ingress-controller +`) th.writeF("/manifests/aws/aws-alb-ingress-controller/base/service-account.yaml", ` apiVersion: v1 kind: ServiceAccount diff --git a/tests/aws-aws-efs-csi-driver-base_test.go b/tests/aws-aws-efs-csi-driver-base_test.go index fbad297b0c..a9c779c6fe 100644 --- a/tests/aws-aws-efs-csi-driver-base_test.go +++ b/tests/aws-aws-efs-csi-driver-base_test.go @@ -14,135 +14,18 @@ import ( ) func writeAwsEfsCsiDriverBase(th *KustTestHarness) { - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-controller-stateful-set.yaml", ` -kind: StatefulSet -apiVersion: apps/v1 + th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-driver.yaml", ` +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver metadata: - name: efs-csi-controller + name: efs.csi.aws.com spec: - serviceName: efs-csi-controller - replicas: 1 - selector: - matchLabels: - app: efs-csi-controller - template: - metadata: - labels: - app: efs-csi-controller - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccount: efs-csi-controller-sa - #priorityClassName: system-cluster-critical - tolerations: - - key: CriticalAddonsOnly - operator: Exists - containers: - - name: efs-plugin - image: amazon/aws-efs-csi-driver:latest - imagePullPolicy: Always - args : - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=5 - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v0.4.2 - imagePullPolicy: Always - args: - - --csi-address=$(ADDRESS) - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - volumes: - - name: socket-dir - emptyDir: {} -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role.yaml", ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-external-attacher-clusterrole -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] + attachRequired: false `) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-attacher-cluster-role-binding.yaml", ` -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-external-attacher-clusterrole-binding -subjects: - - kind: ServiceAccount - name: efs-csi-controller-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: efs-csi-external-attacher-clusterrole - apiGroup: rbac.authorization.k8s.io -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-controller-sa.yaml", ` -apiVersion: v1 -kind: ServiceAccount -metadata: - name: efs-csi-controller-sa -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-node-cluster-role.yaml", ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-node-clusterrole -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch", "update"] -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-node-cluster-role-binding.yaml", ` -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: efs-csi-node-clusterole-binding -subjects: - - kind: ServiceAccount - name: efs-csi-node-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: efs-csi-node-clusterrole - apiGroup: rbac.authorization.k8s.io -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-node-daemon-set.yaml", ` + th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-node-daemonset.yaml", ` +--- +# Node Service kind: DaemonSet apiVersion: apps/v1 metadata: @@ -156,14 +39,16 @@ spec: labels: app: efs-csi-node spec: - serviceAccount: efs-csi-node-sa + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true + tolerations: + - operator: Exists containers: - name: efs-plugin securityContext: privileged: true image: amazon/aws-efs-csi-driver:latest - imagePullPolicy: Always args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr @@ -177,16 +62,24 @@ spec: mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi - - name: device-dir - mountPath: /dev + - name: efs-state-dir + mountPath: /var/run/efs + ports: + - containerPort: 9809 + name: healthz + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 - name: csi-driver-registrar - image: quay.io/k8scsi/driver-registrar:v0.4.2 - imagePullPolicy: Always + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 args: - --csi-address=$(ADDRESS) - - --mode=node-register - - --driver-requires-attachment=true - - --pod-info-mount-version="v1" - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=5 env: @@ -203,63 +96,47 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + - name: liveness-probe + imagePullPolicy: Always + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9809 + volumeMounts: + - mountPath: /csi + name: plugin-dir volumes: - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/efs.csi.aws.com/ type: DirectoryOrCreate - - name: registration-dir + - name: efs-state-dir hostPath: - path: /var/lib/kubelet/plugins/ - type: Directory - - name: device-dir - hostPath: - path: /dev - type: Directory -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-node-sa.yaml", ` -apiVersion: v1 -kind: ServiceAccount -metadata: - name: efs-csi-node-sa -`) - th.writeF("/manifests/aws/aws-efs-csi-driver/base/csi-default-storage.yaml", ` -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: efs-default -provisioner: efs.csi.aws.com + path: /var/run/efs + type: DirectoryOrCreate + `) th.writeK("/manifests/aws/aws-efs-csi-driver/base", ` apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: kubeflow resources: -- csi-controller-stateful-set.yaml -- csi-attacher-cluster-role.yaml -- csi-attacher-cluster-role-binding.yaml -- csi-controller-sa.yaml -- csi-node-cluster-role.yaml -- csi-node-cluster-role-binding.yaml -- csi-node-daemon-set.yaml -- csi-node-sa.yaml -- csi-default-storage.yaml +- csi-driver.yaml +- csi-node-daemonset.yaml generatorOptions: disableNameSuffixHash: true images: -- name: quay.io/k8scsi/driver-registrar - newName: quay.io/k8scsi/driver-registrar - newTag: v0.4.2 - name: amazon/aws-efs-csi-driver newName: amazon/aws-efs-csi-driver - newTag: latest -- name: quay.io/k8scsi/csi-attacher - newName: quay.io/k8scsi/csi-attacher - newTag: v0.4.2 + newTag: v0.3.0 `) } diff --git a/tests/aws-aws-fsx-csi-driver-base_test.go b/tests/aws-aws-fsx-csi-driver-base_test.go index 4f6f4c7a0d..59c896e979 100644 --- a/tests/aws-aws-fsx-csi-driver-base_test.go +++ b/tests/aws-aws-fsx-csi-driver-base_test.go @@ -14,14 +14,23 @@ import ( ) func writeAwsFsxCsiDriverBase(th *KustTestHarness) { - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-controller-stateful-set.yaml", ` -kind: StatefulSet + th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-driver.yaml", ` +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: fsx.csi.aws.com +spec: + attachRequired: false +`) + th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-controller.yaml", ` +--- +kind: Deployment apiVersion: apps/v1 metadata: name: fsx-csi-controller spec: - serviceName: fsx-csi-controller - replicas: 1 + replicas: 2 selector: matchLabels: app: fsx-csi-controller @@ -29,11 +38,11 @@ spec: metadata: labels: app: fsx-csi-controller - annotations: - sidecar.istio.io/inject: "false" spec: + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 serviceAccount: fsx-csi-controller-sa -# priorityClassName: system-cluster-critical tolerations: - key: CriticalAddonsOnly operator: Exists @@ -47,27 +56,29 @@ spec: env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: AWS_SECRET_ACCESS_KEY + optional: true volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v0.4.2 - args: - - --provisioner=fsx.csi.aws.com - - --csi-address=$(ADDRESS) - - --connection-timeout=5m - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v0.4.2 + image: quay.io/k8scsi/csi-provisioner:v1.3.0 args: + - --timeout=5m - --csi-address=$(ADDRESS) - --v=5 + - --enable-leader-election + - --leader-election-type=leases env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -77,120 +88,23 @@ spec: volumes: - name: socket-dir emptyDir: {} -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role.yaml", ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-external-attacher-clusterrole -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-attacher-cluster-role-binding.yaml", ` -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-external-attacher-clusterrole-binding -subjects: - - kind: ServiceAccount - name: fsx-csi-controller-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: fsx-csi-external-attacher-clusterrole - apiGroup: rbac.authorization.k8s.io -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role.yaml", ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-provisioner-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-controller-cluster-role-binding.yaml", ` -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-provisioner-binding -subjects: - - kind: ServiceAccount - name: csi-controller-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: external-provisioner-role - apiGroup: rbac.authorization.k8s.io `) th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-controller-sa.yaml", ` apiVersion: v1 kind: ServiceAccount metadata: name: fsx-csi-controller-sa -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-node-cluster-role.yaml", ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-node-clusterrole -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch", "update"] -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-node-cluster-role-binding.yaml", ` -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: fsx-csi-node-clusterrole-binding -subjects: - - kind: ServiceAccount - name: fsx-csi-node-sa - namespace: kubeflow -roleRef: - kind: ClusterRole - name: fsx-csi-node-clusterrole - apiGroup: rbac.authorization.k8s.io + namespace: kubeflow + #Enable if EKS IAM for SA is used + #annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role `) th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-node-daemonset.yaml", ` +--- kind: DaemonSet apiVersion: apps/v1 metadata: - name: fsx-csi-node-ds + name: fsx-csi-node spec: selector: matchLabels: @@ -200,7 +114,9 @@ spec: labels: app: fsx-csi-node spec: - serviceAccount: fsx-csi-node-sa + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 hostNetwork: true containers: - name: fsx-plugin @@ -220,15 +136,22 @@ spec: mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi - - name: device-dir - mountPath: /dev + ports: + - containerPort: 9810 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 - name: csi-driver-registrar - image: quay.io/k8scsi/driver-registrar:v0.4.2 + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 args: - --csi-address=$(ADDRESS) - - --mode=node-register - - --driver-requires-attachment=true - - --pod-info-mount-version="v1" - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=5 env: @@ -245,35 +168,34 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + - name: liveness-probe + imagePullPolicy: Always + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9810 + volumeMounts: + - mountPath: /csi + name: plugin-dir volumes: - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/fsx.csi.aws.com/ - type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins/ + path: /var/lib/kubelet/plugins_registry/ type: Directory - - name: device-dir + - name: plugin-dir hostPath: - path: /dev - type: Directory -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-node-sa.yaml", ` -apiVersion: v1 -kind: ServiceAccount -metadata: - name: fsx-csi-node-sa + path: /var/lib/kubelet/plugins/fsx.csi.aws.com/ + type: DirectoryOrCreate `) th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role.yaml", ` kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: fsx-external-provisioner-clusterrole + name: fsx-csi-external-provisioner-role rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -286,62 +208,48 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] `) th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-provisioner-cluster-role-binding.yaml", ` kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: fsx-csi-provisioner-clusterrole-binding + name: fsx-csi-external-provisioner-binding subjects: - kind: ServiceAccount name: fsx-csi-controller-sa namespace: kubeflow roleRef: kind: ClusterRole - name: fsx-external-provisioner-clusterrole + name: fsx-csi-external-provisioner-role apiGroup: rbac.authorization.k8s.io -`) - th.writeF("/manifests/aws/aws-fsx-csi-driver/base/csi-default-storage.yaml", ` -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: fsx-default -provisioner: fsx.csi.aws.com `) th.writeK("/manifests/aws/aws-fsx-csi-driver/base", ` apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: kubeflow resources: -- csi-controller-stateful-set.yaml -- csi-attacher-cluster-role.yaml -- csi-attacher-cluster-role-binding.yaml -- csi-controller-cluster-role.yaml -- csi-controller-cluster-role-binding.yaml +- csi-driver.yaml +- csi-controller.yaml - csi-controller-sa.yaml -- csi-node-cluster-role.yaml -- csi-node-cluster-role-binding.yaml - csi-node-daemonset.yaml -- csi-node-sa.yaml - csi-provisioner-cluster-role.yaml - csi-provisioner-cluster-role-binding.yaml -- csi-default-storage.yaml generatorOptions: disableNameSuffixHash: true images: - name: amazon/aws-fsx-csi-driver newName: amazon/aws-fsx-csi-driver - newTag: latest -- name: quay.io/k8scsi/driver-registrar - newName: quay.io/k8scsi/driver-registrar - newTag: v0.4.2 -- name: quay.io/k8scsi/csi-provisioner - newName: quay.io/k8scsi/csi-provisioner - newTag: v0.4.2 -- name: quay.io/k8scsi/csi-attacher - newName: quay.io/k8scsi/csi-attacher - newTag: v0.4.2 + newTag: v0.3.0 `) } diff --git a/tests/pipeline-pipelines-runner-base_test.go b/tests/pipeline-pipelines-runner-base_test.go index c9cf87a67c..175593faab 100644 --- a/tests/pipeline-pipelines-runner-base_test.go +++ b/tests/pipeline-pipelines-runner-base_test.go @@ -50,11 +50,10 @@ rules: - apiGroups: - "" resources: + - persistentvolumes - persistentvolumeclaims verbs: - - create - - delete - - get + - '*' - apiGroups: - snapshot.storage.k8s.io resources: @@ -104,6 +103,18 @@ rules: - jobs verbs: - '*' +- apiGroups: + - machinelearning.seldon.io + resources: + - seldondeployments + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - virtualservices + verbs: + - '*' `) th.writeF("/manifests/pipeline/pipelines-runner/base/service-account.yaml", ` apiVersion: v1 diff --git a/tests/pipeline-pipelines-runner-overlays-application_test.go b/tests/pipeline-pipelines-runner-overlays-application_test.go index 8e3e76f6af..1fa22d6ce1 100644 --- a/tests/pipeline-pipelines-runner-overlays-application_test.go +++ b/tests/pipeline-pipelines-runner-overlays-application_test.go @@ -98,11 +98,10 @@ rules: - apiGroups: - "" resources: + - persistentvolumes - persistentvolumeclaims verbs: - - create - - delete - - get + - '*' - apiGroups: - snapshot.storage.k8s.io resources: @@ -152,6 +151,18 @@ rules: - jobs verbs: - '*' +- apiGroups: + - machinelearning.seldon.io + resources: + - seldondeployments + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - virtualservices + verbs: + - '*' `) th.writeF("/manifests/pipeline/pipelines-runner/base/service-account.yaml", ` apiVersion: v1 diff --git a/tests/pipeline-pipelines-runner-overlays-use-kf-user_test.go b/tests/pipeline-pipelines-runner-overlays-use-kf-user_test.go index 085452eb4e..d85b386211 100644 --- a/tests/pipeline-pipelines-runner-overlays-use-kf-user_test.go +++ b/tests/pipeline-pipelines-runner-overlays-use-kf-user_test.go @@ -69,11 +69,10 @@ rules: - apiGroups: - "" resources: + - persistentvolumes - persistentvolumeclaims verbs: - - create - - delete - - get + - '*' - apiGroups: - snapshot.storage.k8s.io resources: @@ -123,6 +122,18 @@ rules: - jobs verbs: - '*' +- apiGroups: + - machinelearning.seldon.io + resources: + - seldondeployments + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - virtualservices + verbs: + - '*' `) th.writeF("/manifests/pipeline/pipelines-runner/base/service-account.yaml", ` apiVersion: v1