fix(loki): add migrate and update stanzas (#75)

fix(loki): add migrate and update stanzas fix(loki): set ingester to use the nomad port -------- Signed-off-by: Bruce Becker <[email protected]>
brucellino · Oct 27, 2023 · b8dd7ae · b8dd7ae
1 parent 33ada11
commit b8dd7ae
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 31 deletions.
diff --git a/loki.nomad b/loki.nomad
@@ -1,29 +1,45 @@
 variable "loki_version" {
-  type = string
+  type    = string
   default = "v2.7.5"
 }
 
 job "loki" {
   datacenters = ["dc1"]
-  type = "service"
-  name = "loki"
-
+  type        = "service"
+  name        = "loki"
+  priority    = 80
   meta {
-    auto-backup = true
-    backup-schedule = "@hourly"
+    auto-backup      = true
+    backup-schedule  = "@hourly"
     backup-target-db = "postgres"
   }
+
   update {
-    max_parallel = 2
-    health_check = "checks"
-    min_healthy_time = "5s"
-    healthy_deadline = "300s"
+    max_parallel      = 2
+    health_check      = "checks"
+    min_healthy_time  = "5s"
+    healthy_deadline  = "300s"
     progress_deadline = "10m"
-    auto_revert = true
-    auto_promote = true
-    canary = 1
+    auto_revert       = true
+    auto_promote      = true
+    canary            = 1
+  }
+
+  reschedule {
+    interval       = "30m"
+    delay          = "30s"
+    delay_function = "exponential"
+    max_delay      = "1200s"
+    unlimited      = true
   }
-  priority = 80
+
+  migrate {
+    max_parallel     = 2
+    health_check     = "checks"
+    min_healthy_time = "10s"
+    healthy_deadline = "5m"
+  }
+
   group "log-server" {
     count = 1
 
@@ -32,18 +48,18 @@ job "loki" {
       port "grpc" {}
     }
     service {
-      name = "loki-http-server"
-      tags = ["urlprefix-/loki strip=/loki"]
-      port = "http"
+      name      = "loki-http-server"
+      tags      = ["urlprefix-/loki strip=/loki"]
+      port      = "http"
       on_update = "require_healthy"
 
       check {
-        name = "loki_ready"
-        type = "http"
-        path = "/ready"
-        port = "http"
+        name     = "loki_ready"
+        type     = "http"
+        path     = "/ready"
+        port     = "http"
         interval = "10s"
-        timeout = "3s"
+        timeout  = "3s"
       }
     }
 
@@ -55,8 +71,8 @@ job "loki" {
     task "server" {
       driver = "exec"
       vault {
-        policies = ["read-only"]
-        change_mode = "signal"
+        policies      = ["read-only"]
+        change_mode   = "signal"
         change_signal = "SIGHUP"
       }
       config {
@@ -66,11 +82,11 @@ job "loki" {
         ]
       }
       resources {
-        cpu = 128
+        cpu    = 1000
         memory = 200
       }
       template {
-        data = file("loki.yml.tpl")
+        data        = file("loki.yml.tpl")
         destination = "local/loki.yml"
         change_mode = "restart"
       }
@@ -79,7 +95,7 @@ job "loki" {
         options { # checksum depends on the cpu arch
         }
         destination = "local/loki"
-        mode = "file"
+        mode        = "file"
       }
     }
   }

diff --git a/loki.yml.tpl b/loki.yml.tpl
@@ -1,3 +1,7 @@
+{{/*
+  Read the Loki Logs bucket secret. This will have scope throughout the template.
+  See the end of the template for the closing statement.
+*/}}
 {{ with secret "hashiatho.me-v2/loki_logs_bucket" }}
 auth_enabled: false
 server:
@@ -15,7 +19,8 @@ distributor:
 
 ingester:
   lifecycler:
-    address: 127.0.0.1
+    address: {{ env "NOMAD_IP_http" }}
+    port: {{ env "NOMAD_PORT_http" }}
     ring:
       kvstore:
         store: consul
@@ -27,8 +32,10 @@ ingester:
   flush_op_timeout: 20m
 schema_config:
   configs:
-{{/*  store: boltdb-shipper
-      object_store: filesystem*/}}
+{{/*
+    store: boltdb-shipper
+    object_store: filesystem
+*/}}
     - from: 2022-01-01
       store: boltdb-shipper
       object_store: aws
@@ -60,7 +67,7 @@ storage_config:
 
 limits_config:
   enforce_metric_name: false
-  reject_old_samples: true
+  reject_old_samples: false
   reject_old_samples_max_age: 168h
 
 compactor: