Merge branch 'main' into fix-prepare-env-for-simcore

YuryHrytsuk · Jan 3, 2025 · c7d6d70 · c7d6d70
2 parents a04bc41 + 8c0bae8
commit c7d6d70
Show file tree

Hide file tree

Showing 18 changed files with 56 additions and 61 deletions.
diff --git a/charts/aws-ebs-csi-driver/values.yaml.gotmpl b/charts/aws-ebs-csi-driver/values.yaml.gotmpl
@@ -0,0 +1,21 @@
+# Relies on ec2 instance profile attached to K8s EC2.
+# It lets ebs csi driver to manage EBS volumes
+
+image:
+    tag: "v1.38.1"
+
+storageClasses:
+  - name: "ebs-sc"
+    parameters:
+        type: "gp3"
+    allowVolumeExpansion: true
+    reclaimPolicy: Retain  # EBS Volume will not be deleted when PV is deleted
+
+controller:
+    extraVolumeTags:
+        Deployment: {{ requiredEnv "MACHINE_FQDN" }}
+    volumeModificationFeature:
+        enabled: true
+    resources:
+        limits:
+            cpu: 1  # no cpu limits by default
diff --git a/charts/portainer/values.ebs-pv.yaml.gotmpl b/charts/portainer/values.ebs-pv.yaml.gotmpl
@@ -0,0 +1,4 @@
+persistence:
+  enabled: true
+  size: "1Gi"  # minimal size for gp3 is 1Gi
+  storageClass: "ebs-sc"
diff --git a/charts/portainer/values.s3-pv.yaml.gotmpl b/charts/portainer/values.s3-pv.yaml.gotmpl
@@ -0,0 +1,4 @@
+persistence:
+  enabled: true
+  size: "1Gi"
+  storageClass: "csi-s3"
diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl
@@ -18,12 +18,8 @@ serviceAccount:
   # The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: portainer-sa-clusteradmin
-persistence:
-  enabled: true
-  size: "1Gi"
-  annotations: {}
-  storageClass: "csi-s3"
-  existingClaim:
+
+persistence: {}
 
 podAnnotations: {}
 podLabels: {}
@@ -61,7 +57,6 @@ ingress:
               port:
                 number: 9000
 
-
 resources:
   limits:
     cpu: 2

diff --git a/scripts/common.Makefile b/scripts/common.Makefile
@@ -253,13 +253,13 @@ venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environmen
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) --customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3)
 endef
 
 endif

diff --git a/scripts/deployments/deploy_everything_locally.bash b/scripts/deployments/deploy_everything_locally.bash
@@ -254,5 +254,8 @@ if [ "$start_opsstack" -eq 0 ]; then
 fi
 if [ "$start_simcore" -eq 0 ]; then
     log_info "starting simcore..."
-    "${repo_basedir}"/scripts/deployments/start_simcore_locally.bash
+    service_dir="${repo_basedir}"/services/simcore
+    pushd "${service_dir}"
+    call_make "." up-"$stack_target"
+    popd
 fi
diff --git a/services/maintenance-page/Makefile b/services/maintenance-page/Makefile
@@ -33,7 +33,7 @@ up-master: up
 
 .PHONY: docker-compose.yml
 docker-compose.yml: .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 

diff --git a/services/monitoring/Makefile b/services/monitoring/Makefile
@@ -74,7 +74,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml con
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
 
 docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
-	$(call jinja,$<,$@)
+	$(call jinja,$<,.env,$@)
 
 .PHONY: update.grafana.pwd
 update.grafana.pwd: .env ## Change grafana pwd
@@ -101,7 +101,7 @@ grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafan
 
 .PHONY: config.grafana.dashboards
 config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
-	$(call jinja, $<, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
+	$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
 
 .PHONY: config.monitoring
 config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
@@ -133,7 +133,7 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
-	$(call jinja, prometheus/prometheus-ceph.yml.j2, prometheus/prometheus-ceph.yml); \
+	$(call jinja, prometheus/prometheus-ceph.yml.j2, .env, prometheus/prometheus-ceph.yml); \
 	cat prometheus/prometheus-base.yml | $(_yq) '. *+ load("prometheus/prometheus-simcore.yml")' | cat | \
 	$(_yq) '. *+ load("prometheus/prometheus-ceph.yml")' > prometheus/prometheus.yml; \
 	envsubst < prometheus/prometheus.yml > prometheus/prometheus.temp.yml; \
@@ -148,10 +148,10 @@ config.prometheus: ${REPO_CONFIG_LOCATION} .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 .PHONY: grafana/assets
 grafana/assets: ${REPO_CONFIG_LOCATION}

diff --git a/services/monitoring/docker-compose.yml.j2 b/services/monitoring/docker-compose.yml.j2
@@ -25,8 +25,6 @@ configs:
     file: ./prometheus/prometheus-federation.yml
   prometheus_rules:
     file: ./prometheus/prometheus.rules.yml
-  grafana_image_renderer_config:
-    file: ./grafana-image-renderer/config.json
   pgsql_query_exporter_config:
     file: ./pgsql_query_exporter_config.yaml
   smokeping_prober_config:
@@ -249,7 +247,7 @@ services:
           cpus: "0.1"
 
   grafana:
-    image: grafana/grafana-oss:11.1.3
+    image: grafana/grafana-oss:11.2.4
     volumes:
       - grafana_data:/var/lib/grafana
     env_file:

diff --git a/services/monitoring/grafana-image-renderer/README.md b/services/monitoring/grafana-image-renderer/README.md
diff --git a/services/monitoring/grafana-image-renderer/config.json b/services/monitoring/grafana-image-renderer/config.json
diff --git a/services/monitoring/grafana/template-config.monitoring b/services/monitoring/grafana/template-config.monitoring
@@ -3,8 +3,6 @@ GF_SECURITY_ADMIN_USER=${SERVICES_USER}
 GF_USERS_ALLOW_SIGN_UP=false
 GF_SERVER_ROOT_URL=https://${MONITORING_DOMAIN}/grafana
 GF_INSTALL_PLUGINS=grafana-piechart-panel, grafana-worldmap-panel, grafana-polystat-panel
-# NOTE: grafana image rendering plugin: https://grafana.com/blog/2020/05/07/grafana-7.0-preview-new-image-renderer-plugin-to-replace-phantomjs/
-GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
 GF_RENDERING_CALLBACK_URL=http://grafana:3000/
 GF_LOG_FILTERS=rendering:debug
 GF_SERVER_DOMAIN=${MONITORING_DOMAIN}

diff --git a/services/redis-commander/Makefile b/services/redis-commander/Makefile
@@ -52,6 +52,6 @@ ${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.d
 
 config.json: config.json.j2 .env .venv
 	# generate $@
-	@$(call jinja, $<, $@)
+	@$(call jinja, $<, .env, $@)
 	# validate and format $@
 	@cat $@ | jq > /dev/null
diff --git a/services/registry/Makefile b/services/registry/Makefile
@@ -52,7 +52,7 @@ up-public: up-dalco
 # Helpers -------------------------------------------------
 
 docker-compose.yml:  docker-compose.yml.j2 .venv .env
-	@$(call jinja, $<, $@)
+	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}-local
 ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.self-signed.yml .env

diff --git a/services/simcore/Makefile b/services/simcore/Makefile
@@ -11,6 +11,10 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
+.PHONY: up-local
+up-local:
+	@${REPO_BASE_DIR}/scripts/deployments/start_simcore_locally.bash
+
 .PHONY: compose-local
 compose-local: .env ${TEMP_COMPOSE}-local ## Create docker-compose.deploy for local deployment
 

diff --git a/services/simcore/docker-compose.yml b/services/simcore/docker-compose.yml
@@ -887,12 +887,12 @@ services:
         - traefik.enable=true
         - traefik.docker.network=${PUBLIC_NETWORK}
         - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.server.port=8000
+        - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.sticky.cookie=true
+        - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.sticky.cookie.name=sticky_session
         - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/dynamic-scheduler`)
         - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.entrypoints=https
         - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.tls=true
-        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.regex=^/dynamic-scheduler/(.*)$$
-        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.replacement=/$${1}
-        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.middlewares=${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex@swarm, ops_gzip@swarm, ops_auth@swarm
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.middlewares=ops_gzip@swarm, ops_auth@swarm
 
 volumes:
   rabbit_data:

diff --git a/services/traefik/Makefile b/services/traefik/Makefile
@@ -101,13 +101,13 @@ ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml .env
 
 .PHONY: docker-compose.letsencrypt.dns.yml
 docker-compose.letsencrypt.dns.yml: .venv .env
-	@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, docker-compose.letsencrypt.dns.yml.unlinted) && \
+	@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, .env, docker-compose.letsencrypt.dns.yml.unlinted) && \
 	$(_yq) docker-compose.letsencrypt.dns.yml.unlinted > docker-compose.letsencrypt.dns.yml; \
 	rm docker-compose.letsencrypt.dns.yml.unlinted >/dev/null 2>&1;
 
 .PHONY: docker-compose.yml
 docker-compose.yml: traefik_dynamic_config.yml .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 

diff --git a/services/vendors/Makefile b/services/vendors/Makefile
@@ -32,7 +32,7 @@ up-local: up
 
 .PHONY: docker-compose.yml
 docker-compose.yml: .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;