From 5b1c3fb3caa0c51ddb2919251daf1774b3f5c4c2 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Mon, 16 Dec 2024 10:29:39 +0100 Subject: [PATCH] Add grafana terrform tooling --- .../tf_helper_list_json_files_in_folder.sh | 16 +++++++++ scripts/tf_helper_list_subfolders.sh | 16 +++++++++ services/monitoring/Makefile | 32 ++++++++++++++--- .../grafana/terraform/.terraform-version | 1 + .../grafana/terraform/dashboards.tf | 35 +++++++++++++++++++ .../grafana/terraform/datasources.tf | 25 +++++++++++++ services/monitoring/grafana/terraform/main.tf | 21 +++++++++++ .../monitoring/grafana/terraform/variables.tf | 20 +++++++++++ 8 files changed, 161 insertions(+), 5 deletions(-) create mode 100755 scripts/tf_helper_list_json_files_in_folder.sh create mode 100755 scripts/tf_helper_list_subfolders.sh create mode 100644 services/monitoring/grafana/terraform/.terraform-version create mode 100644 services/monitoring/grafana/terraform/dashboards.tf create mode 100644 services/monitoring/grafana/terraform/datasources.tf create mode 100644 services/monitoring/grafana/terraform/main.tf create mode 100644 services/monitoring/grafana/terraform/variables.tf diff --git a/scripts/tf_helper_list_json_files_in_folder.sh b/scripts/tf_helper_list_json_files_in_folder.sh new file mode 100755 index 00000000..65f0267b --- /dev/null +++ b/scripts/tf_helper_list_json_files_in_folder.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +DIRECTORY=$1 + +# Find all JSON files within the directory +FILES=$(find "$DIRECTORY" -mindepth 1 -maxdepth 1 -type f -name '*.json') + +# Create a JSON object where each file's basename is the key, with full paths as values +JSON_OBJECT=$(echo "$FILES" | while read -r FILE; do + BASENAME=$(basename "$FILE" .json) + echo "{\"$BASENAME\": \"$FILE\"}" +done | jq -s 'add') + +# Output the JSON map +jq -n --argjson files "$JSON_OBJECT" '$files' diff --git a/scripts/tf_helper_list_subfolders.sh b/scripts/tf_helper_list_subfolders.sh new file mode 100755 index 00000000..2b61270a --- /dev/null +++ b/scripts/tf_helper_list_subfolders.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +DIRECTORY=$1 + +# Use `find` to get the directories' base names +SUBFOLDERS=$(find "$DIRECTORY" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;) + +# Convert the subfolder names into a JSON object with jq, where each is paired with itself +JSON_OBJECT=$(echo "$SUBFOLDERS" | tr ' ' '\n' | jq -Rn ' + [inputs] | + map(select(. != "")) | + map({key: ., value: .}) | + from_entries') +# Output the JSON map +jq -n --argjson subfolders "$JSON_OBJECT" '$subfolders' diff --git a/services/monitoring/Makefile b/services/monitoring/Makefile index 1827363a..501f9c73 100644 --- a/services/monitoring/Makefile +++ b/services/monitoring/Makefile @@ -84,6 +84,23 @@ update.grafana.pwd: .env ## Change grafana pwd grafanacontainerid=$$(docker ps | grep grafana | awk '{print $$1;}');\ docker exec -ti $$grafanacontainerid grafana-cli admin reset-admin-password $$TRAEFIK_PASSWORD +.PHONY: ensure-grafana-online +ensure-grafana-online: + @set -o allexport; \ + source $(REPO_CONFIG_LOCATION); \ + set +o allexport; \ + url=$${TF_VAR_grafana_url}; \ + echo "Waiting for grafana at $$url to become reachable."; \ + while true; do \ + status_code=$$(curl -k -o /dev/null -s -w "%{http_code}" $$url); \ + if [ "$$status_code" -ge 200 ] && [ "$$status_code" -lt 400 ]; then \ + echo "Grafana is online"; \ + break; \ + else \ + echo "Grafana still unreachable, waiting 5s for grafana to become reachable..."; \ + sleep 5; \ + fi; \ +done; .PHONY: grafana-export grafana-export: .venv## Export the remote grafana dashboards and datasources TO YOUR LOCAL MACHINE @@ -93,11 +110,16 @@ grafana-export: .venv## Export the remote grafana dashboards and datasources TO python3 export.py; .PHONY: grafana-import -grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE - @cd grafana/scripts;\ - source ${REPO_BASE_DIR}/.venv/bin/activate;\ - pip install -r requirements.txt > /dev/null 2>&1;\ - python3 import.py +grafana-import: grafana/assets ensure-grafana-online ## Imports AND OVERWRITES the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE + @set -o allexport; \ + source $(REPO_CONFIG_LOCATION); \ + set +o allexport; \ + pushd ${REPO_BASE_DIR}/services/monitoring/grafana/terraform && \ + terraform init && \ + terraform destroy -auto-approve && \ + terraform apply -auto-approve; \ + popd > /dev/null + .PHONY: config.grafana.dashboards config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters diff --git a/services/monitoring/grafana/terraform/.terraform-version b/services/monitoring/grafana/terraform/.terraform-version new file mode 100644 index 00000000..26ca5946 --- /dev/null +++ b/services/monitoring/grafana/terraform/.terraform-version @@ -0,0 +1 @@ +1.5.1 diff --git a/services/monitoring/grafana/terraform/dashboards.tf b/services/monitoring/grafana/terraform/dashboards.tf new file mode 100644 index 00000000..d0867b26 --- /dev/null +++ b/services/monitoring/grafana/terraform/dashboards.tf @@ -0,0 +1,35 @@ +// Import subfolders using an external script +data "external" "subfolders" { + program = ["bash", "${path.module}/../../../../scripts/tf_helper_list_subfolders.sh", "${path.module}/../assets/shared/dashboards"] +} + +// Local mappings of folder names to their paths +locals { + folder_map = data.external.subfolders.result +} + +// Create Grafana folders for each subfolder +resource "grafana_folder" "subfolders" { + for_each = local.folder_map + + title = each.key // Use each.key to access each folder's name +} + +// Function to list all JSON files within a directory +data "external" "dashboard_files" { + for_each = local.folder_map + + program = ["bash", "${path.module}/../../../../scripts/tf_helper_list_json_files_in_folder.sh", "${path.module}/../assets/shared/dashboards/${each.key}"] +} + +// Create Grafana dashboards from JSON files +resource "grafana_dashboard" "dashboards" { + for_each = toset(flatten([ + for folder_name in keys(local.folder_map) : [ + for file in values(data.external.dashboard_files[folder_name].result) : "${folder_name},${file}" + ]] + )) + # CSV approach + config_json = jsonencode(jsondecode(file(split(",", each.value)[1])).dashboard) + folder = grafana_folder.subfolders[split(",", each.value)[0]].id +} diff --git a/services/monitoring/grafana/terraform/datasources.tf b/services/monitoring/grafana/terraform/datasources.tf new file mode 100644 index 00000000..f8a996eb --- /dev/null +++ b/services/monitoring/grafana/terraform/datasources.tf @@ -0,0 +1,25 @@ + +resource "grafana_data_source" "prometheusfederation" { + type = "prometheus" + name = "prometheus-federation" + url = var.prometheus_federation_url + basic_auth_enabled = false + is_default = true +} + +resource "grafana_data_source" "prometheuscatchall" { + type = "prometheus" + name = "prometheus-catchall" + url = var.prometheus_catchall_url + basic_auth_enabled = false + is_default = false + uid = "RmZEr52nz" +} + +resource "grafana_data_source" "tempo" { + type = "tempo" + name = "tempo" + url = var.tempo_url + basic_auth_enabled = false + is_default = false +} diff --git a/services/monitoring/grafana/terraform/main.tf b/services/monitoring/grafana/terraform/main.tf new file mode 100644 index 00000000..efe82fa6 --- /dev/null +++ b/services/monitoring/grafana/terraform/main.tf @@ -0,0 +1,21 @@ +terraform { + required_version = "~> 1.5.1" + backend "local" { + path = "terraform.tfstate" # Specify the path for the state file, can be a different path if needed + } + required_providers { + grafana = { + source = "grafana/grafana" + version = "~> 3.13" + } + random = { + source = "hashicorp/random" + version = "~> 3.1" + } + } +} + +provider "grafana" { + url = var.grafana_url + auth = var.grafana_auth +} diff --git a/services/monitoring/grafana/terraform/variables.tf b/services/monitoring/grafana/terraform/variables.tf new file mode 100644 index 00000000..113d4d54 --- /dev/null +++ b/services/monitoring/grafana/terraform/variables.tf @@ -0,0 +1,20 @@ +variable "grafana_url" { + description = "grafana_url" + sensitive = false +} +variable "grafana_auth" { + description = "Username:Password" + sensitive = true +} +variable "prometheus_federation_url" { + description = "Prometheus Federation URL" + sensitive = false +} +variable "prometheus_catchall_url" { + description = "Prometheus Catchall URL" + sensitive = false +} +variable "tempo_url" { + description = "Tempo URL" + sensitive = false +}