Merge pull request #1470 from mapellidario/20240424_crabhelm

crabserver - improve deployment with helm charts

helm/crabserver/README.md

@@ -0,0 +1,33 @@
+## Deploy CRAB
+
+At the moment CRAB team is transitioning to using helm. However, we do noy like
+to use helm for deployment. See this 
+[comment](https://github.com/dmwm/CRABServer/issues/7843#issuecomment-2025085120)
+for some context.
+
+We like to use helm for templating, using the helm charts to generate the 
+manifest file, then applying it with `kubectl apply`.
+
+We therefore ask cmsweb operators to deploy crab with the following procedure:
+
+Generate the manifest with
+
+```bash
+# testbed
+helm template crabserver . -f values.yaml -f values-testbed.yaml > ../../kubernetes/cmsweb/services/crabserver.yaml
+
+# prod
+helm template crabserver . -f values.yaml -f values-prod.yaml > ../../kubernetes/cmsweb/services/crabserver.yaml
+```
+
+Then to deploy it with the usual `deploy-srv.sh` script
+
+```bash
+# testbed
+./scripts/deploy-srv.sh crabserver v3.231006 preprod
+
+# prod
+./scripts/deploy-srv.sh crabserver v3.231006 prod
+```
+
+Changes to `../../kubernetes/cmsweb/services/crabserver.yaml` should not be committed.

helm/crabserver/templates/deployment.yaml

@@ -8,8 +8,10 @@ metadata:
   namespace: crab
 spec:
   {{- if not .Values.autoscaling.enabled }}
-  {{- if or (eq (toString $environment) "prod") (eq (toString $environment) "preprod") }}
+  {{- if (eq (toString $environment) "prod") }}
   replicas: {{ .Values.replicaCount.prod }}
+  {{- else if (eq (toString $environment) "preprod")}}
+  replicas: {{ .Values.replicaCount.preprod }}
   {{- else }}
   replicas: {{ .Values.replicaCount.test }}
   {{- end }}

helm/crabserver/values-prod.yaml

@@ -0,0 +1,5 @@
+environment: "prod"
+
+image:
+  tag: "v3.240508"
+

helm/crabserver/values-test11.yaml

@@ -0,0 +1,14 @@
+environment: "test"
+
+image:
+  tag: "v3.240325"
+
+livenessProbeTest:
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 600
+  timeoutSeconds: 60
+
+readinessProbeTest:
+  periodSeconds: 600
+  timeoutSeconds: 60

helm/crabserver/values-test2.yaml

@@ -1,179 +1,4 @@
-# Default values for crabserver.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-replicaCount: 
-  prod: 8
-  test: 1
+environment: "test"
 
 image:
-  path: registry.cern.ch/cmsweb/crabserver
-  pullPolicy: Always
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: "v3.240219"
-  command:
-  - /bin/bash
-  - /opt/setup-certs-and-run/setup-certs-and-run.sh
-  env:
-  - name: CRABSERVER_LOGSTDOUT
-    value: "t"
-
-environment:
-
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: false
-  # Annotations to add to the service account
-  annotations: {}
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name: ""
-
-podAnnotations: 
-  prometheus.io/scrape: 'true'
-  prometheus.io/port: "18270"
-
-podSecurityContext:
-  runAsUser: 1000
-  runAsGroup: 1000
-  fsGroup: 2000
-
-imageFilebeatSecurityContext:
-  allowPrivilegeEscalation: false
-
-securityContext: 
-  privileged: true
-
-serviceMon:
-  name: crabserver-mon
-  port: 18720
-  protocol: TCP
-
-ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-
-resources: 
-  limits:
-    cpu: 1500m
-    memory: 3Gi
-  requests:
-    cpu: 200m
-    memory: 256Mi
-
-
-deamonset:
-  name: filebeat
-  path: docker.elastic.co/beats/filebeat:8.5.1
-  policy: IfNotPresent
-  args:
-  - bash
-  - -c
-  - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e
-  env: 
-  - name: MY_NODE_NAME
-    valueFrom:
-      fieldRef:
-        fieldPath: spec.nodeName
-  - name: NODE_NAME
-    valueFrom:
-      fieldRef:
-        fieldPath: spec.nodeName
-  - name: MY_NAMESPACE
-    valueFrom:
-      fieldRef:
-        apiVersion: v1
-        fieldPath: metadata.namespace
-  resources:
-    requests:
-      memory: "128Mi"
-      cpu: "200m"
-    limits: 
-      cpu: "1"
-      memory: "1Gi"
-
-
-livenessProbe:
-  exec:
-    command:
-    - /bin/sh
-    - -c
-    - |
-      cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  failureThreshold: 3
-  initialDelaySeconds: 120
-  periodSeconds: 60
-  timeoutSeconds: 60
-
-readinessProbe:
-  exec:
-    command:
-    - /bin/sh
-    - -c
-    - |
-      cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  periodSeconds: 60  
-  timeoutSeconds: 60
-
-livenessProbePreProd:
-  exec:
-    command:
-    - /bin/sh
-    - -c
-    - |
-      cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  failureThreshold: 3
-  initialDelaySeconds: 120
-  periodSeconds: 60
-  timeoutSeconds: 60
-
-readinessProbePreProd:
-  exec:
-    command:
-    - /bin/sh
-    - -c
-    - |
-      cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  periodSeconds: 60  
-  timeoutSeconds: 60
-
-livenessProbeTest:
-  exec:
-    command:
-    - /bin/sh
-    - -c
-    - |
-      cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  failureThreshold: 3
-  initialDelaySeconds: 120
-  periodSeconds: 60
-  timeoutSeconds: 60
-
-readinessProbeTest:
-  exec:
-    command:
-    - /bin/sh
-    - -c
-    - |
-      cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  periodSeconds: 60  
-  timeoutSeconds: 60
-
-autoscaling:
-  enabled: false
-  minReplicas: 1
-  maxReplicas: 100
-  targetCPUUtilizationPercentage: 80
-  # targetMemoryUtilizationPercentage: 80
-
-nodeSelector: {}
-
-tolerations: []
-
-affinity: {}
+  tag: "v3.240325"

helm/crabserver/values-testbed.yaml

@@ -0,0 +1,4 @@
+environment: "preprod"
+
+image:
+  tag: "v3.240508"

helm/crabserver/values.yaml

@@ -3,7 +3,8 @@
 # Declare variables to be passed into your templates.
 
 replicaCount: 
-  prod: 8
+  prod: 15
+  preprod: 1
   test: 1
 
 image: