From 6cb83f8227daaaa80846e5145521d9b7cadba1a6 Mon Sep 17 00:00:00 2001 From: Richard Liu <39319471+richardsliu@users.noreply.github.com> Date: Wed, 18 Mar 2020 12:58:42 -0700 Subject: [PATCH] Automated cherry pick of #949: Adds kfserving ingressgateway for Istio 1.1.6 Modifies Cherry pick of #949 on v1.0-branch. #949: Adds kfserving ingressgateway for Istio 1.1.6 Modifies (#1027) * Adds kfserving ingressgateway for Istio 1.1.6 Modifies KFServing and KNative Serving config-maps to use this gateway * Updates configs for Istio 1.1.6 * Updates tests for istio 1.1.6 * Adding kfserving gateway to kfdefs having istio-1-1-6 * Gateway name correction for kfserving config * Adds HPA config to gateway Co-authored-by: krishnadurai --- istio/kfserving-gateway/base/deployment.yaml | 168 ++++++++++ .../base/horizontal-pod-autoscaler.yaml | 19 ++ .../kfserving-gateway/base/kustomization.yaml | 7 + istio/kfserving-gateway/base/service.yaml | 50 +++ kfdef/kfctl_gcp_basic_auth.v1.0.0.yaml | 8 + kfdef/kfctl_gcp_basic_auth.yaml | 8 + kfdef/kfctl_gcp_iap.v1.0.0.yaml | 8 + kfdef/kfctl_gcp_iap.yaml | 8 + kfdef/kfctl_ibm.v1.0.0.yaml | 8 + kfdef/kfctl_ibm.yaml | 8 + kfdef/kfctl_k8s_istio.v1.0.0.yaml | 8 + kfdef/kfctl_k8s_istio.yaml | 8 + kfdef/source/master/kfctl_gcp_basic_auth.yaml | 8 + kfdef/source/master/kfctl_gcp_iap.yaml | 8 + kfdef/source/master/kfctl_ibm.yaml | 8 + kfdef/source/master/kfctl_k8s_istio.yaml | 8 + .../kfserving-install/base/config-map.yaml | 4 +- .../base/config-map.yaml | 2 +- .../knative-serving-install/base/gateway.yaml | 21 +- tests/istio-kfserving-gateway-base_test.go | 300 ++++++++++++++++++ .../kfserving-kfserving-install-base_test.go | 4 +- ...rving-install-overlays-application_test.go | 4 +- ...ative-knative-serving-install-base_test.go | 23 +- ...rving-install-overlays-application_test.go | 23 +- 24 files changed, 709 insertions(+), 12 deletions(-) create mode 100644 istio/kfserving-gateway/base/deployment.yaml create mode 100644 istio/kfserving-gateway/base/horizontal-pod-autoscaler.yaml create mode 100644 istio/kfserving-gateway/base/kustomization.yaml create mode 100644 istio/kfserving-gateway/base/service.yaml create mode 100644 tests/istio-kfserving-gateway-base_test.go diff --git a/istio/kfserving-gateway/base/deployment.yaml b/istio/kfserving-gateway/base/deployment.yaml new file mode 100644 index 0000000000..1faccb9b9f --- /dev/null +++ b/istio/kfserving-gateway/base/deployment.yaml @@ -0,0 +1,168 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kfserving-ingressgateway + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway +spec: + selector: + matchLabels: + app: kfserving-ingressgateway + kfserving: ingressgateway + template: + metadata: + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-ingressgateway-service-account + containers: + - name: istio-proxy + image: "docker.io/istio/proxyv2:1.1.6" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 15020 + - containerPort: 80 + - containerPort: 443 + - containerPort: 31400 + - containerPort: 15029 + - containerPort: 15030 + - containerPort: 15031 + - containerPort: 15032 + - containerPort: 15443 + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - router + - --domain + - $(POD_NAMESPACE).svc.cluster.local + - --log_output_level=default:info + - --drainDuration + - '45s' #drainDuration + - --parentShutdownDuration + - '1m0s' #parentShutdownDuration + - --connectTimeout + - '10s' #connectTimeout + - --serviceCluster + - kfserving-ingressgateway + - --zipkinAddress + - zipkin:9411 + - --proxyAdminPort + - "15000" + - --statusPort + - "15020" + - --controlPlaneAuthPolicy + - NONE + - --discoveryAddress + - istio-pilot:15010 + readinessProbe: + failureThreshold: 30 + httpGet: + path: /healthz/ready + port: 15020 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 40Mi + + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: ISTIO_META_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: ISTIO_META_CONFIG_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ISTIO_META_ROUTER_MODE + value: sni-dnat + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + - name: ingressgateway-certs + mountPath: "/etc/istio/ingressgateway-certs" + readOnly: true + - name: ingressgateway-ca-certs + mountPath: "/etc/istio/ingressgateway-ca-certs" + readOnly: true + volumes: + - name: istio-certs + secret: + secretName: istio.istio-ingressgateway-service-account + optional: true + - name: ingressgateway-certs + secret: + secretName: "istio-ingressgateway-certs" + optional: true + - name: ingressgateway-ca-certs + secret: + secretName: "istio-ingressgateway-ca-certs" + optional: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - ppc64le + - s390x + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - ppc64le + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - s390x diff --git a/istio/kfserving-gateway/base/horizontal-pod-autoscaler.yaml b/istio/kfserving-gateway/base/horizontal-pod-autoscaler.yaml new file mode 100644 index 0000000000..c0a8737dc3 --- /dev/null +++ b/istio/kfserving-gateway/base/horizontal-pod-autoscaler.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway + name: kfserving-ingressgateway +spec: + maxReplicas: 5 + metrics: + - resource: + name: cpu + targetAverageUtilization: 80 + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: kfserving-ingressgateway diff --git a/istio/kfserving-gateway/base/kustomization.yaml b/istio/kfserving-gateway/base/kustomization.yaml new file mode 100644 index 0000000000..a3a235ebc5 --- /dev/null +++ b/istio/kfserving-gateway/base/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: istio-system +resources: +- deployment.yaml +- horizontal-pod-autoscaler.yaml +- service.yaml diff --git a/istio/kfserving-gateway/base/service.yaml b/istio/kfserving-gateway/base/service.yaml new file mode 100644 index 0000000000..de83d27c6d --- /dev/null +++ b/istio/kfserving-gateway/base/service.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Service +metadata: + name: kfserving-ingressgateway + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway +spec: + type: LoadBalancer + selector: + app: kfserving-ingressgateway + kfserving: ingressgateway + ports: + - name: status-port + port: 15020 + targetPort: 15020 + - name: http2 + nodePort: 32380 + port: 80 + targetPort: 80 + - name: https + nodePort: 32390 + port: 443 + - name: tcp + nodePort: 32400 + port: 31400 + - name: tcp-pilot-grpc-tls + port: 15011 + targetPort: 15011 + - name: tcp-citadel-grpc-tls + port: 8060 + targetPort: 8060 + - name: tcp-dns-tls + port: 853 + targetPort: 853 + - name: https-kiali + port: 15029 + targetPort: 15029 + - name: http2-prometheus + port: 15030 + targetPort: 15030 + - name: http2-grafana + port: 15031 + targetPort: 15031 + - name: https-tracing + port: 15032 + targetPort: 15032 + - name: tls + port: 15443 + targetPort: 15443 diff --git a/kfdef/kfctl_gcp_basic_auth.v1.0.0.yaml b/kfdef/kfctl_gcp_basic_auth.v1.0.0.yaml index acfa9a1a3a..c601304014 100644 --- a/kfdef/kfctl_gcp_basic_auth.v1.0.0.yaml +++ b/kfdef/kfctl_gcp_basic_auth.v1.0.0.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_gcp_basic_auth.yaml b/kfdef/kfctl_gcp_basic_auth.yaml index 5043bc6642..764ae61778 100644 --- a/kfdef/kfctl_gcp_basic_auth.yaml +++ b/kfdef/kfctl_gcp_basic_auth.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_gcp_iap.v1.0.0.yaml b/kfdef/kfctl_gcp_iap.v1.0.0.yaml index a1eec6d022..fcf794c523 100644 --- a/kfdef/kfctl_gcp_iap.v1.0.0.yaml +++ b/kfdef/kfctl_gcp_iap.v1.0.0.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_gcp_iap.yaml b/kfdef/kfctl_gcp_iap.yaml index 59a720e73a..fa6b3e3ad5 100644 --- a/kfdef/kfctl_gcp_iap.yaml +++ b/kfdef/kfctl_gcp_iap.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_ibm.v1.0.0.yaml b/kfdef/kfctl_ibm.v1.0.0.yaml index 2e4944bc3a..9ea596c26c 100644 --- a/kfdef/kfctl_ibm.v1.0.0.yaml +++ b/kfdef/kfctl_ibm.v1.0.0.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_ibm.yaml b/kfdef/kfctl_ibm.yaml index e4bea87239..fd13c1978f 100644 --- a/kfdef/kfctl_ibm.yaml +++ b/kfdef/kfctl_ibm.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_k8s_istio.v1.0.0.yaml b/kfdef/kfctl_k8s_istio.v1.0.0.yaml index c72fa7be42..492c8c5502 100644 --- a/kfdef/kfctl_k8s_istio.v1.0.0.yaml +++ b/kfdef/kfctl_k8s_istio.v1.0.0.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/kfctl_k8s_istio.yaml b/kfdef/kfctl_k8s_istio.yaml index 6c8494fc72..6746e7dada 100644 --- a/kfdef/kfctl_k8s_istio.yaml +++ b/kfdef/kfctl_k8s_istio.yaml @@ -28,6 +28,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/source/master/kfctl_gcp_basic_auth.yaml b/kfdef/source/master/kfctl_gcp_basic_auth.yaml index a1218cfcda..37308edbf9 100644 --- a/kfdef/source/master/kfctl_gcp_basic_auth.yaml +++ b/kfdef/source/master/kfctl_gcp_basic_auth.yaml @@ -29,6 +29,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/source/master/kfctl_gcp_iap.yaml b/kfdef/source/master/kfctl_gcp_iap.yaml index 44a238f14e..2610cfbd46 100644 --- a/kfdef/source/master/kfctl_gcp_iap.yaml +++ b/kfdef/source/master/kfctl_gcp_iap.yaml @@ -30,6 +30,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/source/master/kfctl_ibm.yaml b/kfdef/source/master/kfctl_ibm.yaml index dbd1d6ab56..6de0b47484 100644 --- a/kfdef/source/master/kfctl_ibm.yaml +++ b/kfdef/source/master/kfctl_ibm.yaml @@ -32,6 +32,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfdef/source/master/kfctl_k8s_istio.yaml b/kfdef/source/master/kfctl_k8s_istio.yaml index 6f1762a779..0a27aa4c04 100644 --- a/kfdef/source/master/kfctl_k8s_istio.yaml +++ b/kfdef/source/master/kfctl_k8s_istio.yaml @@ -33,6 +33,14 @@ spec: name: manifests path: istio/cluster-local-gateway name: cluster-local-gateway + - kustomizeConfig: + parameters: + - name: namespace + value: istio-system + repoRef: + name: manifests + path: istio/kfserving-gateway + name: kfserving-gateway - kustomizeConfig: parameters: - name: clusterRbacConfig diff --git a/kfserving/kfserving-install/base/config-map.yaml b/kfserving/kfserving-install/base/config-map.yaml index 8ccd809ce6..8a9d600798 100644 --- a/kfserving/kfserving-install/base/config-map.yaml +++ b/kfserving/kfserving-install/base/config-map.yaml @@ -89,8 +89,8 @@ data: } ingress: |- { - "ingressGateway" : "kubeflow-gateway.kubeflow", - "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local" + "ingressGateway" : "knative-ingress-gateway.knative-serving", + "ingressService" : "kfserving-ingressgateway.istio-system.svc.cluster.local" } logger: |- { diff --git a/knative/knative-serving-install/base/config-map.yaml b/knative/knative-serving-install/base/config-map.yaml index a493e66bdf..93b18b498f 100644 --- a/knative/knative-serving-install/base/config-map.yaml +++ b/knative/knative-serving-install/base/config-map.yaml @@ -658,7 +658,7 @@ data: # {{ingress_namespace}}.svc.cluster.local"`. The {{gateway_namespace}} # is optional; when it is omitted, the system will search for # the gateway in the serving system namespace `knative-serving` - gateway.kubeflow.kubeflow-gateway: "istio-ingressgateway.istio-system.svc.cluster.local" + gateway.knative-serving.knative-ingress-gateway: "kfserving-ingressgateway.istio-system.svc.cluster.local" # A cluster local gateway to allow pods outside of the mesh to access # Services and Routes not exposing through an ingress. If the users diff --git a/knative/knative-serving-install/base/gateway.yaml b/knative/knative-serving-install/base/gateway.yaml index 67bab98319..8b939397bc 100644 --- a/knative/knative-serving-install/base/gateway.yaml +++ b/knative/knative-serving-install/base/gateway.yaml @@ -1,4 +1,3 @@ ---- apiVersion: networking.istio.io/v1alpha3 kind: Gateway metadata: @@ -17,3 +16,23 @@ spec: name: http number: 80 protocol: HTTP + +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + labels: + networking.knative.dev/ingress-provider: istio + name: knative-ingress-gateway + namespace: knative-serving +spec: + selector: + app: kfserving-ingressgateway + kfserving: ingressgateway + servers: + - hosts: + - '*' + port: + name: http + number: 80 + protocol: HTTP diff --git a/tests/istio-kfserving-gateway-base_test.go b/tests/istio-kfserving-gateway-base_test.go new file mode 100644 index 0000000000..7f8a7b466d --- /dev/null +++ b/tests/istio-kfserving-gateway-base_test.go @@ -0,0 +1,300 @@ +package tests_test + +import ( + "sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct" + "sigs.k8s.io/kustomize/v3/k8sdeps/transformer" + "sigs.k8s.io/kustomize/v3/pkg/fs" + "sigs.k8s.io/kustomize/v3/pkg/loader" + "sigs.k8s.io/kustomize/v3/pkg/plugins" + "sigs.k8s.io/kustomize/v3/pkg/resmap" + "sigs.k8s.io/kustomize/v3/pkg/resource" + "sigs.k8s.io/kustomize/v3/pkg/target" + "sigs.k8s.io/kustomize/v3/pkg/validators" + "testing" +) + +func writeKfservingGatewayBase(th *KustTestHarness) { + th.writeF("/manifests/istio/kfserving-gateway/base/deployment.yaml", ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kfserving-ingressgateway + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway +spec: + selector: + matchLabels: + app: kfserving-ingressgateway + kfserving: ingressgateway + template: + metadata: + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-ingressgateway-service-account + containers: + - name: istio-proxy + image: "docker.io/istio/proxyv2:1.1.6" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 15020 + - containerPort: 80 + - containerPort: 443 + - containerPort: 31400 + - containerPort: 15029 + - containerPort: 15030 + - containerPort: 15031 + - containerPort: 15032 + - containerPort: 15443 + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - router + - --domain + - $(POD_NAMESPACE).svc.cluster.local + - --log_output_level=default:info + - --drainDuration + - '45s' #drainDuration + - --parentShutdownDuration + - '1m0s' #parentShutdownDuration + - --connectTimeout + - '10s' #connectTimeout + - --serviceCluster + - kfserving-ingressgateway + - --zipkinAddress + - zipkin:9411 + - --proxyAdminPort + - "15000" + - --statusPort + - "15020" + - --controlPlaneAuthPolicy + - NONE + - --discoveryAddress + - istio-pilot:15010 + readinessProbe: + failureThreshold: 30 + httpGet: + path: /healthz/ready + port: 15020 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 40Mi + + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: ISTIO_META_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: ISTIO_META_CONFIG_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ISTIO_META_ROUTER_MODE + value: sni-dnat + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + - name: ingressgateway-certs + mountPath: "/etc/istio/ingressgateway-certs" + readOnly: true + - name: ingressgateway-ca-certs + mountPath: "/etc/istio/ingressgateway-ca-certs" + readOnly: true + volumes: + - name: istio-certs + secret: + secretName: istio.istio-ingressgateway-service-account + optional: true + - name: ingressgateway-certs + secret: + secretName: "istio-ingressgateway-certs" + optional: true + - name: ingressgateway-ca-certs + secret: + secretName: "istio-ingressgateway-ca-certs" + optional: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - ppc64le + - s390x + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - ppc64le + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - s390x +`) + th.writeF("/manifests/istio/kfserving-gateway/base/horizontal-pod-autoscaler.yaml", ` +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway + name: kfserving-ingressgateway +spec: + maxReplicas: 5 + metrics: + - resource: + name: cpu + targetAverageUtilization: 80 + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: kfserving-ingressgateway +`) + th.writeF("/manifests/istio/kfserving-gateway/base/service.yaml", ` +apiVersion: v1 +kind: Service +metadata: + name: kfserving-ingressgateway + labels: + app: kfserving-ingressgateway + kfserving: ingressgateway +spec: + type: LoadBalancer + selector: + app: kfserving-ingressgateway + kfserving: ingressgateway + ports: + - name: status-port + port: 15020 + targetPort: 15020 + - name: http2 + nodePort: 32380 + port: 80 + targetPort: 80 + - name: https + nodePort: 32390 + port: 443 + - name: tcp + nodePort: 32400 + port: 31400 + - name: tcp-pilot-grpc-tls + port: 15011 + targetPort: 15011 + - name: tcp-citadel-grpc-tls + port: 8060 + targetPort: 8060 + - name: tcp-dns-tls + port: 853 + targetPort: 853 + - name: https-kiali + port: 15029 + targetPort: 15029 + - name: http2-prometheus + port: 15030 + targetPort: 15030 + - name: http2-grafana + port: 15031 + targetPort: 15031 + - name: https-tracing + port: 15032 + targetPort: 15032 + - name: tls + port: 15443 + targetPort: 15443 +`) + th.writeK("/manifests/istio/kfserving-gateway/base", ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: istio-system +resources: +- deployment.yaml +- horizontal-pod-autoscaler.yaml +- service.yaml +`) +} + +func TestKfservingGatewayBase(t *testing.T) { + th := NewKustTestHarness(t, "/manifests/istio/kfserving-gateway/base") + writeKfservingGatewayBase(th) + m, err := th.makeKustTarget().MakeCustomizedResMap() + if err != nil { + t.Fatalf("Err: %v", err) + } + expected, err := m.AsYaml() + if err != nil { + t.Fatalf("Err: %v", err) + } + targetPath := "../istio/kfserving-gateway/base" + fsys := fs.MakeRealFS() + lrc := loader.RestrictionRootOnly + _loader, loaderErr := loader.NewLoader(lrc, validators.MakeFakeValidator(), targetPath, fsys) + if loaderErr != nil { + t.Fatalf("could not load kustomize loader: %v", loaderErr) + } + rf := resmap.NewFactory(resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()), transformer.NewFactoryImpl()) + pc := plugins.DefaultPluginConfig() + kt, err := target.NewKustTarget(_loader, rf, transformer.NewFactoryImpl(), plugins.NewLoader(pc, rf)) + if err != nil { + th.t.Fatalf("Unexpected construction error %v", err) + } + actual, err := kt.MakeCustomizedResMap() + if err != nil { + t.Fatalf("Err: %v", err) + } + th.assertActualEqualsExpected(actual, string(expected)) +} diff --git a/tests/kfserving-kfserving-install-base_test.go b/tests/kfserving-kfserving-install-base_test.go index f12ae28101..4046c5409b 100644 --- a/tests/kfserving-kfserving-install-base_test.go +++ b/tests/kfserving-kfserving-install-base_test.go @@ -344,8 +344,8 @@ data: } ingress: |- { - "ingressGateway" : "kubeflow-gateway.kubeflow", - "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local" + "ingressGateway" : "knative-ingress-gateway.knative-serving", + "ingressService" : "kfserving-ingressgateway.istio-system.svc.cluster.local" } logger: |- { diff --git a/tests/kfserving-kfserving-install-overlays-application_test.go b/tests/kfserving-kfserving-install-overlays-application_test.go index 682b39de07..a8f5115171 100644 --- a/tests/kfserving-kfserving-install-overlays-application_test.go +++ b/tests/kfserving-kfserving-install-overlays-application_test.go @@ -402,8 +402,8 @@ data: } ingress: |- { - "ingressGateway" : "kubeflow-gateway.kubeflow", - "ingressService" : "istio-ingressgateway.istio-system.svc.cluster.local" + "ingressGateway" : "knative-ingress-gateway.knative-serving", + "ingressService" : "kfserving-ingressgateway.istio-system.svc.cluster.local" } logger: |- { diff --git a/tests/knative-knative-serving-install-base_test.go b/tests/knative-knative-serving-install-base_test.go index 2d2e10319f..1f44fd0ab2 100644 --- a/tests/knative-knative-serving-install-base_test.go +++ b/tests/knative-knative-serving-install-base_test.go @@ -15,7 +15,6 @@ import ( func writeKnativeServingInstallBase(th *KustTestHarness) { th.writeF("/manifests/knative/knative-serving-install/base/gateway.yaml", ` ---- apiVersion: networking.istio.io/v1alpha3 kind: Gateway metadata: @@ -34,6 +33,26 @@ spec: name: http number: 80 protocol: HTTP + +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + labels: + networking.knative.dev/ingress-provider: istio + name: knative-ingress-gateway + namespace: knative-serving +spec: + selector: + app: kfserving-ingressgateway + kfserving: ingressgateway + servers: + - hosts: + - '*' + port: + name: http + number: 80 + protocol: HTTP `) th.writeF("/manifests/knative/knative-serving-install/base/cluster-role.yaml", ` --- @@ -1062,7 +1081,7 @@ data: # {{ingress_namespace}}.svc.cluster.local"`+"`"+`. The {{gateway_namespace}} # is optional; when it is omitted, the system will search for # the gateway in the serving system namespace `+"`"+`knative-serving`+"`"+` - gateway.kubeflow.kubeflow-gateway: "istio-ingressgateway.istio-system.svc.cluster.local" + gateway.knative-serving.knative-ingress-gateway: "kfserving-ingressgateway.istio-system.svc.cluster.local" # A cluster local gateway to allow pods outside of the mesh to access # Services and Routes not exposing through an ingress. If the users diff --git a/tests/knative-knative-serving-install-overlays-application_test.go b/tests/knative-knative-serving-install-overlays-application_test.go index 717eda154f..8050504728 100644 --- a/tests/knative-knative-serving-install-overlays-application_test.go +++ b/tests/knative-knative-serving-install-overlays-application_test.go @@ -63,7 +63,6 @@ commonLabels: app.kubernetes.io/version: v0.11.1 `) th.writeF("/manifests/knative/knative-serving-install/base/gateway.yaml", ` ---- apiVersion: networking.istio.io/v1alpha3 kind: Gateway metadata: @@ -82,6 +81,26 @@ spec: name: http number: 80 protocol: HTTP + +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + labels: + networking.knative.dev/ingress-provider: istio + name: knative-ingress-gateway + namespace: knative-serving +spec: + selector: + app: kfserving-ingressgateway + kfserving: ingressgateway + servers: + - hosts: + - '*' + port: + name: http + number: 80 + protocol: HTTP `) th.writeF("/manifests/knative/knative-serving-install/base/cluster-role.yaml", ` --- @@ -1110,7 +1129,7 @@ data: # {{ingress_namespace}}.svc.cluster.local"`+"`"+`. The {{gateway_namespace}} # is optional; when it is omitted, the system will search for # the gateway in the serving system namespace `+"`"+`knative-serving`+"`"+` - gateway.kubeflow.kubeflow-gateway: "istio-ingressgateway.istio-system.svc.cluster.local" + gateway.knative-serving.knative-ingress-gateway: "kfserving-ingressgateway.istio-system.svc.cluster.local" # A cluster local gateway to allow pods outside of the mesh to access # Services and Routes not exposing through an ingress. If the users