From 6492a26ad35525b209f0b9c0f16a008c0813e965 Mon Sep 17 00:00:00 2001 From: Yury Hrytsuk <50014626+YuryHrytsuk@users.noreply.github.com> Date: Thu, 16 Nov 2023 08:50:18 +0100 Subject: [PATCH 1/4] Introduce PR template (#442) * Return removed staging CA for letsenrypt * Fix missing $ * Remove PAYMENTS_AUTORECHARGE_DEFAULT_MIN_BALANCE * add pr template --- .github/workflows/PULL_REQUEST_TEMPLATE.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/PULL_REQUEST_TEMPLATE.md diff --git a/.github/workflows/PULL_REQUEST_TEMPLATE.md b/.github/workflows/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..0ea60e13 --- /dev/null +++ b/.github/workflows/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## What do these changes do? + +## Related issue/s + + + +## Related PR/s + +## Checklist + +- [ ] I tested and it works From 5b1f1641d849de03c56128eac9dab91a5c13c7ee Mon Sep 17 00:00:00 2001 From: Yury Hrytsuk <50014626+YuryHrytsuk@users.noreply.github.com> Date: Thu, 16 Nov 2023 08:53:50 +0100 Subject: [PATCH 2/4] Fix PR template (#443) * Return removed staging CA for letsenrypt * Fix missing $ * Remove PAYMENTS_AUTORECHARGE_DEFAULT_MIN_BALANCE * add pr template * minor fixes * Remove file --- .github/PULL_REQUEST_TEMPLATE.md | 9 +++++++++ .github/workflows/PULL_REQUEST_TEMPLATE.md | 16 ---------------- 2 files changed, 9 insertions(+), 16 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .github/workflows/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..2130585f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +## What do these changes do? + +## Related issue/s + +## Related PR/s + +## Checklist + +- [ ] I tested and it works diff --git a/.github/workflows/PULL_REQUEST_TEMPLATE.md b/.github/workflows/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 0ea60e13..00000000 --- a/.github/workflows/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,16 +0,0 @@ -## What do these changes do? - -## Related issue/s - - - -## Related PR/s - -## Checklist - -- [ ] I tested and it works From eb8968e900dee89747890d81beccdbd1ecc87de3 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:00:23 +0100 Subject: [PATCH 3/4] removed all envs (#438) --- services/simcore/docker-compose.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/services/simcore/docker-compose.yml b/services/simcore/docker-compose.yml index f33e8c1f..4f281650 100644 --- a/services/simcore/docker-compose.yml +++ b/services/simcore/docker-compose.yml @@ -29,34 +29,6 @@ services: - node.labels.simcore==true autoscaling: - environment: - - EC2_ACCESS_KEY_ID=${AUTOSCALING_EC2_ACCESS_KEY_ID} # defines the access key to EC2 - - EC2_SECRET_ACCESS_KEY=${AUTOSCALING_EC2_SECRET_ACCESS_KEY} # defines the secret key to EC2 - - EC2_REGION_NAME=${AUTOSCALING_EC2_REGION_NAME} - - EC2_INSTANCES_ALLOWED_TYPES=${AUTOSCALING_EC2_INSTANCES_ALLOWED_TYPES} # defines the allowed EC2 instance types (["t2.nano", ...]) - - EC2_INSTANCES_AMI_ID=${AUTOSCALING_EC2_INSTANCES_AMI_ID} # defines the AMI ID to use - - EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS=${AUTOSCALING_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS} - - EC2_INSTANCES_MACHINES_BUFFER=${AUTOSCALING_EC2_INSTANCES_MACHINES_BUFFER} # defines the number of reserved machines - - EC2_INSTANCES_MAX_INSTANCES=${AUTOSCALING_EC2_INSTANCES_MAX_INSTANCES} # maximum number of instances - - EC2_INSTANCES_PRE_PULL_IMAGES=${AUTOSCALING_EC2_INSTANCES_PRE_PULL_IMAGES} - - EC2_INSTANCES_SECURITY_GROUP_IDS=${AUTOSCALING_EC2_INSTANCES_SECURITY_GROUP_IDS} # defines the security groups for starting EC2s - - EC2_INSTANCES_SUBNET_ID=${AUTOSCALING_EC2_INSTANCES_SUBNET_ID} # defines the subnet ID to use for the EC2 instances - - EC2_INSTANCES_KEY_NAME=${AUTOSCALING_EC2_INSTANCES_KEY_NAME} # defines the used Key name for the EC2 instances - - NODES_MONITORING_NODE_LABELS=${AUTOSCALING_NODES_MONITORING_NODE_LABELS} # defines the labels to check for on the nodes (i.e. ["dynamicsidecar"]) - - NODES_MONITORING_SERVICE_LABELS=${AUTOSCALING_NODES_MONITORING_SERVICE_LABELS} # optional defines the label to check for on the monitored services - - NODES_MONITORING_NEW_NODES_LABELS=${NODES_MONITORING_NEW_NODES_LABELS} # optional defines the name of the monitored service - - RABBIT_HOST=${RABBIT_HOST} - - RABBIT_PASSWORD=${RABBIT_PASSWORD} - - RABBIT_PORT=${RABBIT_PORT} - - RABBIT_SECURE=${RABBIT_SECURE} - - RABBIT_USER=${RABBIT_USER} - - REDIS_HOST=${REDIS_HOST} - - REDIS_PORT=${REDIS_PORT} - - REGISTRY_USER=${REGISTRY_USER} - - REGISTRY_PW=${REGISTRY_PW} - - REGISTRY_URL=${REGISTRY_URL} - - REGISTRY_SSL=${REGISTRY_SSL} - - REGISTRY_AUTH=${REGISTRY_AUTH} deploy: update_config: parallelism: 2 From bc812587f63ba6f50fcef9e42dfea02a5273a94d Mon Sep 17 00:00:00 2001 From: Dustin Kaiser <8209087+mrnicegyu11@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:33:47 +0100 Subject: [PATCH 4/4] Add: validate simcore stack yml bash script - to be used in CI/CD (#441) * Add the file * fixes * Add version-pinned yq * minor fixes * Add comment * Rename venv folder to .venv --------- Co-authored-by: kaiser --- .../validate_simcore_stack_yml.bash | 71 +++++++++++++++++++ services/graylog/Makefile | 6 +- 2 files changed, 74 insertions(+), 3 deletions(-) create mode 100755 scripts/deployments/validate_simcore_stack_yml.bash diff --git a/scripts/deployments/validate_simcore_stack_yml.bash b/scripts/deployments/validate_simcore_stack_yml.bash new file mode 100755 index 00000000..8f5cccf2 --- /dev/null +++ b/scripts/deployments/validate_simcore_stack_yml.bash @@ -0,0 +1,71 @@ +#!/bin/bash +set -o nounset +set -o pipefail +# +# This bash script expects the following environment variables to be set: +# - PREFIX_STACK_NAME: prefix of the stack name +# +export COMPOSE_FILE=simcore_stack.yml +export SETTINGS_BINARY_PATH=/home/scu/.venv/bin +# +export SERVICES_PREFIX=${PREFIX_STACK_NAME} +exit_code=0 +# Download version-pinned yq binary +python -c "import urllib.request,os,sys,urllib; f = open(os.path.basename(sys.argv[1]), 'wb'); f.write(urllib.request.urlopen(sys.argv[1]).read()); f.close();" https://github.com/mikefarah/yq/releases/download/v4.29.2/yq_linux_amd64 +mv yq_linux_amd64 yq +chmod +x yq +_yq=$(realpath ./yq) +export _yq +# +# start +# +for service in $($_yq e '.services | keys | .[]' ${COMPOSE_FILE}) +do + export TARGETNAME=${service#"${SERVICES_PREFIX}"_} + # continue if the service == director since it doesnt have settings + if [ "${TARGETNAME}" == "director" ]; then + continue + fi + if [ "${TARGETNAME}" == "migration" ]; then + continue + fi + if [ "${TARGETNAME}" == "postgres" ]; then + continue + fi + if [ "${TARGETNAME}" == "rabbit" ]; then + continue + fi + if [ "${TARGETNAME}" == "redis" ]; then + continue + fi + if [ "${TARGETNAME}" == "traefik" ]; then + continue + fi + if [ "${TARGETNAME}" == "traefik_api" ]; then + continue + fi + if [ "${TARGETNAME}" == "whoami" ]; then + continue + fi + export TARGETFILE="simcore-service-${TARGETNAME}" + echo TARGETFILE="${TARGETFILE}" + echo "Assuming targetfile in ${SETTINGS_BINARY_PATH}/${TARGETFILE}" + echo "Checking ${SETTINGS_BINARY_PATH}/${TARGETFILE}" + if docker compose --file ${COMPOSE_FILE} run --rm "${service}" test -f "${SETTINGS_BINARY_PATH}"/"${TARGETFILE}" > /dev/null 2>&1; then + echo "FOUND_EXECUTABLE=${SETTINGS_BINARY_PATH}/$TARGETFILE" + export FOUND_EXECUTABLE="${SETTINGS_BINARY_PATH}/$TARGETFILE" + if docker compose --file ${COMPOSE_FILE} run --entrypoint "${FOUND_EXECUTABLE}" --rm "${service}" settings --as-json > /dev/null 2>&1; then + echo "SUCCESS: Validation of environment variables for ${service}" + else + echo "ERROR: Validation of environment variables for ${service} failed" + docker compose --file ${COMPOSE_FILE} run --entrypoint "${FOUND_EXECUTABLE}" --rm "${service}" settings --as-json + exit_code=1 + fi + else + echo "WARN: Settings executable not found for ${service}" + fi + + + echo "--------------------------" +done +exit $exit_code diff --git a/services/graylog/Makefile b/services/graylog/Makefile index c7bbceb8..8c2487fc 100644 --- a/services/graylog/Makefile +++ b/services/graylog/Makefile @@ -87,11 +87,11 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.letsencrypt.dns.yml docke .PHONY: configure configure: .env ## Test is Graylog is online and configure Graylog inputs @cd scripts;\ - if [ ! -d "venv" ]; \ + if [ ! -d ".venv" ]; \ then\ - python3 -m venv venv;\ + python3 -m venv .venv;\ fi;\ - source venv/bin/activate;\ + source .venv/bin/activate;\ pip install -r requirements.txt > /dev/null 2>&1;\ set -o allexport; \ source ../$<;\