From 4ed4818ddf7985abee6104c533048eb953e30d9a Mon Sep 17 00:00:00 2001
From: Bruce Becker <brucellino@protonmail.com>
Date: Thu, 4 Jan 2024 08:39:05 +0100
Subject: [PATCH] feat/grafana 10 (#83)

feat(node-exporter): add a node exporter job
fix(github-exporter): bump version
fix(github-exporter): use the variable in the artifact string instead of hard-coding it
fix(github-exporter): template the secret to be read into the environment
fix(jenkins): use vault secret in template
fix(grafana): use podman driver
BREAKING CHANGE: no longer use the exec driver
chore(monitoring): bump versions
refactor(monitoring): lookup mysql service in the consul catalog
fix(prometheus): fix path to metrics
fix(promtail): add nomad job log scraping

Signed-off-by: Bruce Becker <brucellino@protonmail.com>

---------

Signed-off-by: Bruce Becker <brucellino@protonmail.com>
---
 jenkins/jenkins.nomad                   | 247 ++++++++++++------------
 jenkins/jenkins.yml.tmpl                |  71 +++++++
 monitoring/github-exporter.nomad        |  44 +++--
 monitoring/grafana.nomad                |  52 +++--
 monitoring/monitoring.nomad             | 178 ++++++++---------
 monitoring/templates/grafana.ini.tpl    |  10 +-
 monitoring/templates/prometheus.yml.tpl |   2 +-
 monitoring/templates/promtail.yml.tpl   |  20 ++
 node-exporter.nomad                     |  72 +++++++
 9 files changed, 434 insertions(+), 262 deletions(-)
 create mode 100644 jenkins/jenkins.yml.tmpl
 create mode 100644 node-exporter.nomad

diff --git a/jenkins/jenkins.nomad b/jenkins/jenkins.nomad
index fc84bc6..0923903 100644
--- a/jenkins/jenkins.nomad
+++ b/jenkins/jenkins.nomad
@@ -1,28 +1,40 @@
+variable "jenkins_war" {
+  type = map(string)
+  default = {
+    war_version = "2.426.2"
+    war_sha256  = "3731b9f44973fbbf3e535f98a80c21aad9719cb4eea8a1e59e974c11fe846848" #pragma: allowlist secret
+  }
+  description = "Version of the Jenkins release to deploy"
+}
+
+variable "plugin_manager" {
+  type = map(string)
+  default = {
+    version = "2.12.8"
+  }
+  description = "Map of configuration entries for the java plugin manager."
+}
+
 job "jenkins" {
 
   update {
     max_parallel = 1
     health_check = "checks"
-    auto_revert = true
+    auto_revert  = true
     auto_promote = true
-    canary = 1
-  }
-
-  constraint {
-    attribute  = "${attr.unique.hostname}"
-    operator = "regexp"
-    value = "^turing.*"
+    canary       = 1
   }
 
   datacenters = ["dc1"]
+
   type = "service"
-  group "main" {
+
+  group "controller" {
     count = 1
     network {
-      port "ui" {
-        static = "8080"
-      }
-      mode = "host"
+      port "ui" {}
+      // mode = "host"
+      port "agent" {}
     }
 
     volume "casc" {
@@ -31,157 +43,150 @@ job "jenkins" {
       source    = "jenkins_casc"
     }
 
-    task "controller" {
+    task "plugins" {
       driver = "exec"
+      env {
+        CASC_JENKINS_CONFIG = "alloc/data/jenkins.yml"
+        JENKINS_HOME        = "alloc/jenkins"
+        CACHE_DIR           = "local/"
+      }
+      lifecycle {
+        hook    = "prestart"
+        sidecar = false
+      }
       volume_mount {
         volume      = "casc"
-        destination = "/usr/share/jenkins"
+        destination = "/usr/share/jenkins/"
         read_only   = false
       }
 
       artifact {
-        source = "https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/2.12.8/jenkins-plugin-manager-2.12.8.jar"
+        source      = "https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/${var.plugin_manager.version}/jenkins-plugin-manager-${var.plugin_manager.version}.jar"
         destination = "alloc/data/jenkins-plugin-manager.jar"
-        mode = "file"
+        mode        = "file"
       }
 
       artifact {
-        source = "https://get.jenkins.io/war-stable/2.346.1/jenkins.war"
+        source      = "https://get.jenkins.io/war-stable/${var.jenkins_war.war_version}/jenkins.war"
         destination = "alloc/data/jenkins.war"
-        mode = "file"
+        mode        = "file"
         options {
-          checksum = "sha256:176e2ce5c23d3c0b439befe0461e7ed1f53ac3091db05980198c23c7fde53b27"
+          checksum = "sha256:${var.jenkins_war.war_sha256}"
         }
       }
+
       template {
-        data = "{{ key \"jenkins/plugins\" }}"
-        destination = "local/plugins.txt"
+        data        = "{{ key \"nomad/jenkins/plugins\" }}"
+        destination = "alloc/data/plugins.txt"
         change_mode = "restart"
       }
-      env {
-        CASC_JENKINS_CONFIG = "alloc/data/jenkins.yml"
-        JENKINS_HOME        = "/usr/share/jenkins"
-        CACHE_DIR = "local/"
-      }
+
       template {
-        data = <<EOF
+        data        = <<EOF
 #!/bin/bash
-pwd
 set -eou pipefail
-mkdir -vp /usr/share/jenkins/plugins
+mkdir -vp /alloc/jenkins/plugins
 java -jar alloc/data/jenkins-plugin-manager.jar \
      --war alloc/data/jenkins.war \
-     --plugin-file local/plugins.txt \
+     --plugin-file alloc/data/plugins.txt \
      --skip-failed-plugins \
      --verbose \
-     -d /usr/share/jenkins/plugins/
+     -d /alloc/jenkins/plugins/
+ls -lht /alloc/jenkins/plugins
 echo "plugins installed"
-
-java \
-  -Xmx1024m \
-  -Xms256m \
-  -Dhudson.footerURL=https://hashiatho.me \
-  -Dhudson.model.WorkspaceCleanupThread.disabled=true \
-  -Dhudson.slaves.ConnectionActivityMonitor.timeToPing=30000 \
-  -Djenkins.install.runSetupWizard=false \
-  -Djenkins.security.SystemReadPermission=true \
-  -Djenkins.ui.refresh=true \
-  -jar alloc/data/jenkins.war \
-  --httpPort=${NOMAD_PORT_ui}
 EOF
-        destination = "local/script.sh"
-        perms = "0777"
+        destination = "local/plugins.sh"
+        perms       = "0777"
       }
 
       config {
         command = "/bin/bash"
-        args = ["local/script.sh"]
+        args    = ["local/plugins.sh"]
+      }
+
+      resources {
+        cpu    = "128"
+        memory = "64"
+        // memory_max = "256"
+      } // plugin task resources
+    }   // task
+
+    task "launch" {
+      env {
+        CASC_JENKINS_CONFIG = "alloc/data/jenkins.yml"
+        JENKINS_HOME        = "alloc/jenkins"
+        CACHE_DIR           = "local/"
+      }
+      driver = "java"
+      config {
+        jvm_options = [
+          "-Xmx2048m",
+          "-Xms256m",
+          "-Dhudson.footerURL=https://hashiatho.me",
+          "-Dhudson.model.WorkspaceCleanupThread.disabled=true",
+          "-Dhudson.slaves.ConnectionActivityMonitor.timeToPing=30000",
+          "-Djenkins.install.runSetupWizard=false",
+          "-Djenkins.security.SystemReadPermission=true",
+          "-Djenkins.ui.refresh=true"
+        ]
+        jar_path = "alloc/jenkins/jenkins.war"
+        args = [
+          "--httpPort=${NOMAD_PORT_ui}",
+          "--httpListenAddress=${NOMAD_IP_ui}"
+        ]
       }
 
+      volume_mount {
+        volume      = "casc"
+        destination = "/usr/share/jenkins"
+        read_only   = false
+      }
+      resources {
+        cpu    = "2048"
+        memory = "2048"
+        // memory_max = "2048"
+      } // launch task resources
       service {
         port = "ui"
-
+        name = "jenkins-controller"
         check {
-          type = "http"
-          port = "ui"
-          path = "/prometheus/"
+          type     = "http"
+          port     = "ui"
+          path     = "/prometheus/"
           interval = "10s"
-          timeout = "5s"
+          timeout  = "5s"
         }
-
         on_update = "require_healthy"
+        tags      = ["urlprefix-/jenkins"]
+      } // jenkins ui service
 
-        tags = ["urlprefix-/jenkins"]
-      }
-      template {
-        data = <<EOH
----
-jenkins:
-  agentprotocols:
-    - "JNLP4-connect"
-    - "Ping"
-  numExecutors: 0
-  authorizationStrategy:
-    globalMatrix:
-      permissions:
-        - "Overall/Administer:admin"
-  crumbIssuer:
-    strict:
-      checkSessionMatch: false
-  remotingSecurity:
-    enabled: true
-  systemMessage: "This the best ever message"
-  securityRealm:
-    local:
-      allowsSignup: false
-      users:
-        - id: "admin"
-          password: "1234"
-  clouds:
-    - nomad:
-        clientPassword: ""
-        name: "nomad"
-        nomadUrl: "http://nomad.service.consul:4646"
-        prune: false
-        serverPassword: ""
-        tlsEnabled: false
-        workerTimeout: 1
-
-security:
-  apiToken:
-    creationOfLegacyTokenEnabled: false
-    tokenGenerationOnCreationEnabled: false
-    usageStatisticsEnabled: true
-  globalJobDslSecurityConfiguration:
-    useScriptSecurity: true
-  sSHD:
-    port: -1
-unclassified:
-  prometheusConfiguration:
-    appendParamLabel: true
-    appendStatusLabel: true
-    collectDiskUsage: true
-    collectingMetricsPeriodInSeconds: 60
-    countAbortedBuilds: true
-    countFailedBuilds: true
-    countNotBuiltBuilds: true
-    countSuccessfulBuilds: true
-    countUnstableBuilds: true
-    defaultNamespace: "default"
-    fetchTestResults: true
-    jobAttributeName: "jenkins_job"
-    path: "prometheus"
-    processingDisabledBuilds: false
-    useAuthenticatedEndpoint: false
-EOH
-        destination = "alloc/data/jenkins.yml"
+      service {
+        port = "agent"
+        name = "jenkins-controller-inbound-agent"
+        check {
+          type      = "http"
+          port      = "agent"
+          path      = "/"
+          on_update = "require_healthy"
+          interval  = "30s"
+          timeout   = "5s"
+        }
       }
 
-      resources {
-        cores = 3
-        memory = "500"
-        memory_max = "1000"
+      artifact {
+        source      = "https://get.jenkins.io/war-stable/${var.jenkins_war.war_version}/jenkins.war"
+        destination = "alloc/jenkins/jenkins.war"
+        mode        = "file"
+        options {
+          checksum = "sha256:${var.jenkins_war.war_sha256}"
+        }
       }
+
+      template {
+        data        = file("jenkins.yml.tmpl")
+        destination = "alloc/data/jenkins.yml"
+        change_mode = "restart"
+      } // jenkins.yml template
     }
   }
 }
diff --git a/jenkins/jenkins.yml.tmpl b/jenkins/jenkins.yml.tmpl
new file mode 100644
index 0000000..135d54d
--- /dev/null
+++ b/jenkins/jenkins.yml.tmpl
@@ -0,0 +1,71 @@
+{{ with secret "hashiatho.me-v2/jenkins" }}
+---
+jenkins:
+  agentprotocols:
+    - "JNLP4-connect"
+    - "Ping"
+  numExecutors: 0
+  authorizationStrategy:
+    globalMatrix:
+      permissions:
+        - "Overall/Administer:{{ .Data.data.admin_username }}"
+
+  crumbIssuer:
+    strict:
+      checkSessionMatch: false
+  disableRememberMe: false
+  markdownFormatter: "plainText"
+  mode: NORMAL
+  myViewsTabBar: "standard"
+  nodeMonitors:
+    - diskSpaceMonitor:
+        freeSpaceThreshold: "1GB"
+    - tmpSpace:
+        freeSpaceThreshold: "1GB"
+  remotingSecurity:
+    enabled: true
+  systemMessage: "This the best ever message"
+  slaveAgentPort: ${NOMAD_PORT_agent}
+  securityRealm:
+    local:
+      allowsSignup: false
+      users:
+        - id: "{{ .Data.data.admin_username }}"
+          password: "{{ .Data.data.admin_password }}"
+  clouds:
+    - nomad:
+        clientPassword: ""
+        name: "nomad"
+        nomadUrl: "http://nomad.service.consul:4646"
+        prune: true
+        serverPassword: ""
+        tlsEnabled: false
+        workerTimeout: 1
+
+security:
+  apiToken:
+    creationOfLegacyTokenEnabled: false
+    tokenGenerationOnCreationEnabled: false
+    usageStatisticsEnabled: true
+  globalJobDslSecurityConfiguration:
+    useScriptSecurity: true
+  sSHD:
+    port: -1
+unclassified:
+  prometheusConfiguration:
+    appendParamLabel: true
+    appendStatusLabel: true
+    collectDiskUsage: true
+    collectingMetricsPeriodInSeconds: 60
+    countAbortedBuilds: true
+    countFailedBuilds: true
+    countNotBuiltBuilds: true
+    countSuccessfulBuilds: true
+    countUnstableBuilds: true
+    defaultNamespace: "default"
+    fetchTestResults: true
+    jobAttributeName: "jenkins_job"
+    path: "prometheus"
+    processingDisabledBuilds: false
+    useAuthenticatedEndpoint: false
+{{ end }}
diff --git a/monitoring/github-exporter.nomad b/monitoring/github-exporter.nomad
index 788016c..3cf193f 100644
--- a/monitoring/github-exporter.nomad
+++ b/monitoring/github-exporter.nomad
@@ -12,13 +12,13 @@ variable "go_url" {
 variable "exporter_version" {
   type        = string
   description = "Version of the github exporter to use"
-  default     = "1.0.3"
+  default     = "1.0.6"
 }
 
 variable "artifact_url" {
   type        = string
   description = "URL for the artifact -- a GitHub repo"
-  default     = "github.com/infinityworks/github-exporter"
+  default     = "github.com/githubexporter/github-exporter"
 }
 
 job "github-exporter" {
@@ -48,7 +48,7 @@ job "github-exporter" {
       stagger           = "30s"
     }
     task "build" {
-      driver = "exec"
+      driver = "raw_exec"
       lifecycle {
         hook    = "prestart"
         sidecar = false
@@ -65,7 +65,7 @@ job "github-exporter" {
       }
       artifact {
         // source = "${var.go_url}/go${var.go_version}.${attr.kernel.name}-${attr.cpu.arch}.tar.gz"
-        source      = "https://go.dev/dl/go1.20.3.linux-arm64.tar.gz"
+        source      = "https://go.dev/dl/go${var.go_version}.linux-arm64.tar.gz"
         destination = "${NOMAD_ALLOC_DIR}/data"
       }
       config {
@@ -85,7 +85,7 @@ ${NOMAD_ALLOC_DIR}/data/go/bin/go build -buildvcs=false -o ${NOMAD_ALLOC_DIR}/da
       }
     }
     task "main" {
-      driver = "exec"
+      driver = "raw_exec"
       env {
         ORGS              = "AAROC"
         LISTEN_PORT       = "${NOMAD_PORT_exporter}"
@@ -93,12 +93,14 @@ ${NOMAD_ALLOC_DIR}/data/go/bin/go build -buildvcs=false -o ${NOMAD_ALLOC_DIR}/da
       }
       template {
         data        = <<EOH
-{{ with secret "kv/data/github" }}{{ .Data.data.exporter_token }}{{ end }}
+{{ with secret "kv/data/github" }}GITHUB_TOKEN={{ .Data.data.exporter_token }}{{ end }}
         EOH
-        destination = "${NOMAD_SECRETS_DIR}/gh_token"
+        env         = true
+        destination = "secrets/.env"
       }
       config {
         command = "${NOMAD_ALLOC_DIR}/data/github_exporter"
+
       }
       service {
         port = "exporter"
@@ -132,7 +134,7 @@ ${NOMAD_ALLOC_DIR}/data/go/bin/go build -buildvcs=false -o ${NOMAD_ALLOC_DIR}/da
       stagger           = "30s"
     }
     task "build" {
-      driver = "exec"
+      driver = "raw_exec"
       lifecycle {
         hook    = "prestart"
         sidecar = false
@@ -170,17 +172,16 @@ ${NOMAD_ALLOC_DIR}/data/go/bin/go build -buildvcs=false -o ${NOMAD_ALLOC_DIR}/da
     }
 
     task "main" {
-      driver = "exec"
+      driver = "raw_exec"
       env {
-        ORGS              = "hashi-at-home"
-        LISTEN_PORT       = "${NOMAD_PORT_exporter}"
-        GITHUB_TOKEN_FILE = "${NOMAD_SECRETS_DIR}/gh_token"
+        ORGS        = "hashi-at-home"
+        LISTEN_PORT = "${NOMAD_PORT_exporter}"
+        // GITHUB_TOKEN_FILE = "${NOMAD_SECRETS_DIR}/.env"
       }
       template {
-        data        = <<EOH
-{{ with secret "kv/data/github" }}{{ .Data.data.exporter_token }}{{ end }}
-        EOH
-        destination = "${NOMAD_SECRETS_DIR}/gh_token"
+        data        = "{{ with secret \"kv/data/github\" }}GITHUB_TOKEN={{ .Data.data.exporter_token }}{{ end }}"
+        env         = true
+        destination = "secrets/.env"
       }
       config {
         command = "${NOMAD_ALLOC_DIR}/data/github_exporter"
@@ -217,7 +218,7 @@ ${NOMAD_ALLOC_DIR}/data/go/bin/go build -buildvcs=false -o ${NOMAD_ALLOC_DIR}/da
       stagger           = "30s"
     }
     task "build" {
-      driver = "exec"
+      driver = "raw_exec"
       lifecycle {
         hook    = "prestart"
         sidecar = false
@@ -254,17 +255,18 @@ ${NOMAD_ALLOC_DIR}/data/go/bin/go build -buildvcs=false -o ${NOMAD_ALLOC_DIR}/da
       }
     }
     task "main" {
-      driver = "exec"
+      driver = "raw_exec"
       env {
-        USERS              = "brucellino"
+        USERS             = "brucellino"
         LISTEN_PORT       = "${NOMAD_PORT_exporter}"
         GITHUB_TOKEN_FILE = "${NOMAD_SECRETS_DIR}/gh_token"
       }
       template {
         data        = <<EOH
-{{ with secret "kv/data/github" }}{{ .Data.data.exporter_token }}{{ end }}
+{{ with secret "kv/data/github" }}GITHUB_TOKEN={{ .Data.data.exporter_token }}{{ end }}
         EOH
-        destination = "${NOMAD_SECRETS_DIR}/gh_token"
+        env         = true
+        destination = "secrets/.env"
       }
       config {
         command = "${NOMAD_ALLOC_DIR}/data/github_exporter"
diff --git a/monitoring/grafana.nomad b/monitoring/grafana.nomad
index 2671615..8da31f7 100644
--- a/monitoring/grafana.nomad
+++ b/monitoring/grafana.nomad
@@ -1,6 +1,6 @@
 variable "grafana_version" {
-  type = string
-  default = "9.4.7"
+  type        = string
+  default     = "9.4.7"
   description = "Grafana version"
 }
 
@@ -19,7 +19,7 @@ variable "grafana_version" {
 job "dashboard" {
 
   datacenters = ["dc1"]
-  type = "service"
+  type        = "service"
 
   # Select ARMv7 machines
   constraint {
@@ -39,8 +39,8 @@ job "dashboard" {
   }
 
   migrate {
-    max_parallel = 1
-    health_check = "checks"
+    max_parallel     = 1
+    health_check     = "checks"
     min_healthy_time = "15s"
     healthy_deadline = "5m"
   }
@@ -50,7 +50,7 @@ job "dashboard" {
     network {
       port "mysql_server" {
         static = 3306
-        to = 3306
+        to     = 3306
       }
     }
     service {
@@ -70,8 +70,8 @@ job "dashboard" {
     restart {
       attempts = 1
       interval = "2m"
-      delay = "15s"
-      mode = "fail"
+      delay    = "15s"
+      mode     = "fail"
     }
     task "mysql" {
       leader = true
@@ -82,9 +82,9 @@ job "dashboard" {
       }
       env {
         MYSQL_ROOT_PASSWORD = "password" # pragma: allowlist secret
-        MYSQL_USER = "mysql"
-        MYSQL_PASSWORD = "password" # pragma: allowlist secret
-        MYSQL_DATABASE = "grafana"
+        MYSQL_USER          = "mysql"
+        MYSQL_PASSWORD      = "password" # pragma: allowlist secret
+        MYSQL_DATABASE      = "grafana"
       }
       resources {
         cpu    = 1000
@@ -96,7 +96,7 @@ job "dashboard" {
 
   group "grafana" {
     network {
-      port "grafana_server" { }
+      port "grafana_server" {}
     }
     # select machines with more than 4GB of RAM
     constraint {
@@ -110,7 +110,7 @@ job "dashboard" {
       port = "grafana_server"
 
       check {
-        port = "grafana_server"
+        port     = "grafana_server"
         name     = "grafana-api"
         path     = "/api/health"
         type     = "http"
@@ -122,8 +122,8 @@ job "dashboard" {
     restart {
       attempts = 1
       interval = "2m"
-      delay = "15s"
-      mode = "fail"
+      delay    = "15s"
+      mode     = "fail"
     }
 
     ephemeral_disk {
@@ -137,38 +137,34 @@ job "dashboard" {
       driver = "raw_exec"
       config {
         command = "sh"
-        args = ["-c", "while ! nc -z mysql.service.consul 3306 ; do sleep 1 ; done"]
+        args    = ["-c", "while ! nc -z mysql.service.consul 3306 ; do sleep 1 ; done"]
       }
     }
 
     task "grafana" {
-      driver = "exec"
+      driver = "podman"
       logs {
-        max_files     = 10
+        max_files     = 2
         max_file_size = 15
       }
-      artifact {
-        // source = local.grafana_url
-        source = "https://dl.grafana.com/oss/release/grafana-${var.grafana_version}.linux-arm64.tar.gz"
-        destination = "${NOMAD_ALLOC_DIR}"
-      }
       resources {
         cpu    = 1000
-        memory = 512
+        memory = 1024
       }
 
       config {
-        command = "${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/bin/grafana-server"
+        image = "docker://grafana/grafana:${var.grafana_version}"
         args = [
           "-homepath=${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}",
           "--config=${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/conf/conf.ini"
         ]
+        ports = ["grafana_server"]
       }
 
       template {
-        data = file("templates/grafana.ini.tpl")
+        data        = file("templates/grafana.ini.tpl")
         destination = "${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/conf/conf.ini"
       } // Configuration template
-    } // Grafana server task
-  } // grafana server group
+    }   // Grafana server task
+  }     // grafana server group
 }
diff --git a/monitoring/monitoring.nomad b/monitoring/monitoring.nomad
index 1c94306..8f1aacc 100644
--- a/monitoring/monitoring.nomad
+++ b/monitoring/monitoring.nomad
@@ -1,30 +1,36 @@
 variable "prom_version" {
-  default     = "2.43.0"
+  default     = "2.48.0"
   type        = string
   description = "Version of prometheus to use"
 }
 
 variable "prom_sha2" {
-  type        = string
-  default     = "79c4262a27495e5dff45a2ce85495be2394d3eecd51f0366c706f6c9c729f672" #pragma: allowlist secret
+  type = map(string)
+  default = {
+    arm64 : "79c4262a27495e5dff45a2ce85495be2394d3eecd51f0366c706f6c9c729f672" #pragma: allowlist secret
+    amd64 : "5871ca9e01ae35bb7ab7a129a845a7a80f0e1453f00f776ac564dd41ff4d754e" #pragma: allowlist secret
+  }
   description = "https://prometheus.io/download/"
 }
 
 variable "mimir_version" {
-  default     = "2.8.0"
+  default     = "2.9.3"
   type        = string
   description = "Version of mimir to use"
 }
 
 variable "mimir_sha2" {
-  type        = string
-  default     = "e7d2d401f616b185bded25cfe84f7b6543e169f4d0d8a36e19f7ba124848b712" #pragma: allowlist secret
-  description = "https://prometheus.io/download/"
+  type = map(string)
+  default = {
+    arm64 : "e7d2d401f616b185bded25cfe84f7b6543e169f4d0d8a36e19f7ba124848b712" #pragma: allowlist secret
+    amd64 : "3a1aa0ccb97692989433c9087bfc478e21cc6d879637f19f091a4c13778ce441" #pragma: allowlist secret
+  }
+  description = "See https://github.com/grafana/mimir/releases"
 }
 
 variable "grafana_version" {
   type        = string
-  default     = "9.4.7"
+  default     = "10.2.2"
   description = "Grafana version"
 }
 
@@ -53,16 +59,11 @@ job "monitoring" {
   }
   constraint {
     attribute = attr.cpu.arch
-    value     = "arm64"
+    value     = "amd64"
   }
 
   group "prometheus" {
     count = 1
-    // volume "data" {
-    //   type      = "host"
-    //   read_only = false
-    //   source    = "scratch"
-    // }
     network {
       port "prometheus_ui" {}
     }
@@ -86,12 +87,12 @@ job "monitoring" {
 
     task "prometheus" {
       artifact {
-        source      = "https://github.com/prometheus/prometheus/releases/download/v${var.prom_version}/prometheus-${var.prom_version}.linux-arm64.tar.gz"
+        source      = "https://github.com/prometheus/prometheus/releases/download/v${var.prom_version}/prometheus-${var.prom_version}.linux-amd64.tar.gz"
         destination = "local"
 
-        options {
-          checksum = "sha256:${var.prom_sha2}"
-        }
+        // options {
+        //   checksum = "sha256:${var.prom_sha2}"
+        // }
       }
       template {
         change_mode   = "signal"
@@ -118,7 +119,7 @@ job "monitoring" {
       driver = "exec"
 
       config {
-        command = "local/prometheus-${var.prom_version}.linux-arm64/prometheus"
+        command = "local/prometheus-${var.prom_version}.linux-amd64/prometheus"
         args = [
           "--config.file=local/prometheus.yml",
           "--storage.tsdb.retention.size=1GB",
@@ -128,14 +129,10 @@ job "monitoring" {
           "--storage.tsdb.path=data"
         ]
       }
-      // volume_mount {
-      //   volume      = "data"
-      //   destination = "data"
-      //   read_only   = false
-      // }
+
       resources {
-        cpu    = 250
-        memory = 400
+        cpu    = 1000
+        memory = 1024
       }
 
       service {
@@ -163,11 +160,7 @@ job "monitoring" {
 
   group "mimir" {
     count = 1
-    // volume "data" {
-    //   type      = "host"
-    //   read_only = false
-    //   source    = "scratch"
-    // }
+
     network {
       port "mimir_ui" {}
     }
@@ -176,7 +169,7 @@ job "monitoring" {
       attempts = 1
       interval = "7m"
       delay    = "1m"
-      mode     = "fail"
+      mode     = "delay"
     }
 
     reschedule {
@@ -185,22 +178,40 @@ job "monitoring" {
       unlimited      = true
     }
 
+    update {
+      max_parallel      = 1
+      min_healthy_time  = "20s"
+      healthy_deadline  = "5m"
+      progress_deadline = "15m"
+      auto_revert       = true
+      auto_promote      = true
+      canary            = 1
+    }
+
+    migrate {
+      max_parallel     = 1
+      health_check     = "checks"
+      min_healthy_time = "30s"
+      healthy_deadline = "10m"
+    }
+
     ephemeral_disk {
       size = 300
     }
 
     task "mimir" {
       vault {
-        policies      = ["read-only"]
+        // policies      = ["read-only"]
         change_mode   = "restart"
         change_signal = "SIGHUP"
       }
       artifact {
-        source      = "https://github.com/grafana/mimir/releases/download/mimir-${var.mimir_version}/mimir-linux-arm64"
+        source      = "https://github.com/grafana/mimir/releases/download/mimir-${var.mimir_version}/mimir-linux-amd64"
         destination = "local"
-        options {
-          checksum = "sha256:${var.mimir_sha2}"
-        }
+
+        // options {
+        //   checksum = "sha256:${var.mimir_sha2}"
+        // }
       }
       template {
         change_mode   = "signal"
@@ -216,17 +227,13 @@ job "monitoring" {
       driver = "exec"
 
       config {
-        command = "local/mimir-linux-arm64"
+        command = "local/mimir-linux"
         args = [
           "-server.http-listen-port=${NOMAD_PORT_mimir_ui}",
           "--config.file=local/mimir.yml"
         ]
       }
-      // volume_mount {
-      //   volume      = "data"
-      //   destination = "data"
-      //   read_only   = false
-      // }
+
       resources {
         cpu    = 250
         memory = 400
@@ -263,23 +270,13 @@ job "monitoring" {
     constraint {
       attribute = "${attr.cpu.arch}"
       operator  = "="
-      value     = "arm64"
+      value     = "amd64"
     }
     service {
       name = "mysql"
       tags = ["db", "urlprefix-/mysql:3306 proto=tcp"]
       port = "mysql_server"
 
-      // check {
-      //   type = "script"
-      //   command = "/bin/bash"
-      //   args = ["-c", "mysqld status"]
-      //   name = "mysql_ready"
-      //   interval = "5s"
-      //   timeout = "2s"
-      //   task = "mysql"
-      // }
-
       check {
         type     = "tcp"
         name     = "mysql_alive"
@@ -314,9 +311,9 @@ job "monitoring" {
     }
 
     task "mysql" {
-      driver = "podman"
+      driver = "docker"
       config {
-        image        = "docker://arm64v8/mysql:oracle"
+        image        = "mysql:oracle"
         ports        = ["mysql_server"]
         network_mode = "host"
       }
@@ -327,8 +324,8 @@ job "monitoring" {
         MYSQL_DATABASE      = "grafana"
       }
       resources {
-        cpu    = 125
-        memory = 1000
+        cpu    = 500
+        memory = 512
       }
     }
   }
@@ -339,6 +336,20 @@ job "monitoring" {
       port "grafana_server" {}
     }
 
+    affinity {
+      attribute = "${memory.totalbytes}"
+      weight    = 50
+      operator  = ">="
+      value     = "307863731"
+    }
+
+    affinity {
+      attribute = "${cpu.frequency}"
+      weight    = 100
+      operator  = ">="
+      value     = "2000"
+    }
+
     // volume "grafana_data" {
     //   type = "csi"
     //   source = "grafana"
@@ -373,15 +384,9 @@ job "monitoring" {
     constraint {
       attribute = "${attr.cpu.arch}"
       operator  = "="
-      value     = "arm64"
+      value     = "amd64"
     }
 
-    # select machines with more than 4GB of RAM
-    // constraint {
-    //   attribute = "${attr.memory.totalbytes}"
-    //   value     = "500MB"
-    //   operator  = ">="
-    // }
     update {
       max_parallel      = 1
       min_healthy_time  = "20s"
@@ -399,12 +404,12 @@ job "monitoring" {
       healthy_deadline = "10m"
     }
 
-    ephemeral_disk {
-      size = 200
-    }
+    // ephemeral_disk {
+    //   size = 200
+    // }
 
     vault {
-      policies      = ["read-only"]
+      // policies      = ["read-only"]
       change_mode   = "restart"
       change_signal = "SIGHUP"
     }
@@ -413,7 +418,7 @@ job "monitoring" {
       lifecycle {
         hook = "prestart"
       }
-      driver = "exec"
+      driver = "raw_exec"
       config {
         command = "sh"
         args    = ["-c", "while ! nc -z mysql.service.consul 3306 ; do sleep 1 ; done"]
@@ -425,33 +430,32 @@ job "monitoring" {
     }
 
     task "grafana" {
-      driver = "exec"
+      driver = "docker"
       logs {
-        max_files     = 10
+        max_files     = 1
         max_file_size = 15
       }
-      artifact {
-        // source = local.grafana_url
-        source      = "https://dl.grafana.com/oss/release/grafana-${var.grafana_version}.linux-arm64.tar.gz"
-        destination = "${NOMAD_ALLOC_DIR}"
-      }
+
       resources {
-        cpu    = 1000
-        memory = 512
+        cores  = 1
+        memory = 2048
       }
 
+      env {
+        GF_PATHS_CONFIG = "/local/conf.ini"
+        GF_PATHS_HOME   = "/home/grafana"
+      }
       config {
-        command = "${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/bin/grafana-server"
-        args = [
-          "-homepath=${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}",
-          "--config=${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/conf/conf.ini"
-        ]
+        image        = "grafana/grafana:${var.grafana_version}"
+        ports        = ["grafana_server"]
+        network_mode = "bridge"
       }
 
       template {
-        data = file("templates/grafana.ini.tpl")
-
-        destination = "${NOMAD_ALLOC_DIR}/grafana-${var.grafana_version}/conf/conf.ini"
+        data        = file("templates/grafana.ini.tpl")
+        destination = "local/conf.ini"
+        perms       = "0666"
+        uid         = "472"
       } // Configuration template
     }   // Grafana server task
   }     // grafana server group
diff --git a/monitoring/templates/grafana.ini.tpl b/monitoring/templates/grafana.ini.tpl
index ff90fbf..9309755 100644
--- a/monitoring/templates/grafana.ini.tpl
+++ b/monitoring/templates/grafana.ini.tpl
@@ -10,7 +10,8 @@ http_port = ${NOMAD_HOST_PORT_grafana_server}
 
 [database]
 type = mysql
-host = mysql.service.consul:3306
+{% comment %} host = mysql.service.consul:3306 {% endcomment %}
+{{- range service "mysql" }}host = {{ .Address }}:{{ .Port }}{{- end }}
 user = root
 password = """{{ .Data.data.root_password }}"""
 ssl_mode = disable
@@ -20,9 +21,9 @@ ssl_mode = disable
 # server_cert_name = none
 
 [paths]
-data = ${NOMAD_ALLOC_DIR}/data/
-logs = ${NOMAD_ALLOC_DIR}/log/
-plugins = ${NOMAD_ALLOC_DIR}/plugins
+data = /local/data/
+logs = /local/log/
+plugins = /local/plugins
 
 [analytics]
 reporting_enabled = false
@@ -34,6 +35,7 @@ external_enabled = false
 admin_user = admin
 admin_password = {{ .Data.data.grafana_admin_password }}
 disable_gravatar = true
+
 [dashboards]
 versions_to_keep = 10
 
diff --git a/monitoring/templates/prometheus.yml.tpl b/monitoring/templates/prometheus.yml.tpl
index 6ac952f..f721db2 100644
--- a/monitoring/templates/prometheus.yml.tpl
+++ b/monitoring/templates/prometheus.yml.tpl
@@ -37,7 +37,7 @@ scrape_configs:
         replacement: $1
         action: replace
     scrape_interval: 5s
-    metrics_path: /v1/metrics
+    metrics_path: /metrics
     params:
       format: ['prometheus']
   - job_name: nomad_metrics
diff --git a/monitoring/templates/promtail.yml.tpl b/monitoring/templates/promtail.yml.tpl
index c92103e..ec6ca10 100644
--- a/monitoring/templates/promtail.yml.tpl
+++ b/monitoring/templates/promtail.yml.tpl
@@ -10,6 +10,18 @@ clients:
   - url: http://localhost:9999/loki/loki/api/v1/push
 
 scrape_configs:
+  - job_name: vault
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: vault-server
+          __path__: /var/log/vault.log
+      - targets:
+          - localhost
+        labels:
+          job: vault-agent
+          __path__: /var/log/vault-agent.log
   - job_name: consul
     static_configs:
     - targets:
@@ -24,6 +36,13 @@ scrape_configs:
       labels:
         job: nomad
         __path__: /var/log/nomad*.log
+  - job_name: Nomad Jobs
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nomad_allocations
+        __path__: /opt/nomad/alloc/*/*/alloc/logs
   - job_name: journal
     journal:
       json: false
@@ -32,6 +51,7 @@ scrape_configs:
       matches: _TRANSPORT=kernel
       labels:
         job: systemd-journal
+
     relabel_configs:
       - source_labels: ['__journal__systemd_unit']
         target_label: 'unit'
diff --git a/node-exporter.nomad b/node-exporter.nomad
new file mode 100644
index 0000000..225a469
--- /dev/null
+++ b/node-exporter.nomad
@@ -0,0 +1,72 @@
+variable "node_exporter_version" {
+  description = "Version of node exporter"
+  type        = string
+  default     = "1.5.0"
+}
+
+job "node-exporter" {
+  priority = 100
+  meta {
+    auto-backup      = true
+    backup-schedule  = "@daily"
+    backup-target-db = "postgres"
+  }
+  datacenters = ["dc1"]
+  type        = "system"
+
+  group "node-exporter" {
+    count = 1
+    network {
+      port "http" {}
+    }
+
+    service {
+      name = "node-exporter"
+      tags = ["http"]
+      port = "http"
+
+      check {
+        name     = "node-exporter-alive"
+        type     = "tcp"
+        interval = "20s"
+        timeout  = "15s"
+      }
+
+      check {
+        name     = "Node-exporter healthy"
+        type     = "http"
+        path     = "/metrics"
+        interval = "10s"
+        timeout  = "5s"
+      }
+    }
+
+    ephemeral_disk {
+      size = 300
+    }
+
+    task "node-exporter" {
+      # The "driver" parameter specifies the task driver that should be used to
+      # run the task.
+      driver = "raw_exec"
+
+      config {
+        command = "node_exporter/node_exporter-${var.node_exporter_version}.linux-arm64/node_exporter"
+        args = [
+          "--web.listen-address=:${NOMAD_PORT_http}"
+        ]
+      }
+
+      artifact {
+        source      = "https://github.com/prometheus/node_exporter/releases/download/v${var.node_exporter_version}/node_exporter-${var.node_exporter_version}.linux-arm64.tar.gz"
+        destination = "local/node_exporter"
+        mode        = "any"
+      }
+
+      resources {
+        cpu    = 60
+        memory = 125
+      }
+    }
+  }
+}