Merge branch 'main' of https://github.com/brucellino/nomad-jobs into …

…main

fabio.nomad

@@ -1,3 +1,8 @@
+variable "fabio_version" {
+  type = string
+  default = "1.6.3"
+  description = "Version of Fabio to use"
+}
 job "fabio" {
   datacenters = ["dc1"]
   type = "system"
@@ -17,15 +22,15 @@ job "fabio" {
       }
     }
     restart {
-      attempts = 3
-      interval = "10m"
+      attempts = 1
+      interval = "2m"
       delay = "15s"
       mode = "delay"
     }
 
     task "fabio" {
       artifact {
-        source      = "https://github.com/fabiolb/fabio/releases/download/v1.6.0/fabio-1.6.0-linux_${attr.cpu.arch}"
+        source      = "https://github.com/fabiolb/fabio/releases/download/v${var.fabio_version}/fabio-${var.fabio_version}-linux_${attr.cpu.arch}"
         destination = "local/fabio"
         mode = "file"
       }

loki/loki-local.yml.tpl

@@ -0,0 +1,37 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: /tmp/loki/index
+
+  filesystem:
+    directory: /tmp/loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h

loki/loki.nomad

@@ -8,21 +8,21 @@ variable "secret_key" {
 
 variable "loki_version" {
   type = string
-  default = "v2.6.0"
+  default = "v2.7.1"
 }
 
 job "loki" {
   datacenters = ["dc1"]
   type = "service"
   name = "loki"
-  // migrate {}
+
   meta {
     auto-backup = true
     backup-schedule = "@hourly"
     backup-target-db = "postgres"
   }
   update {
-    max_parallel = 1
+    max_parallel = 2
     health_check = "checks"
     min_healthy_time = "5s"
     healthy_deadline = "300s"
@@ -45,18 +45,10 @@ job "loki" {
     }
     service {
       name = "loki-http-server"
-      tags = ["logs", "loki", "observability", "urlprefix-/loki"]
+      tags = ["urlprefix-/loki strip=/loki"]
       port = "http"
       on_update = "require_healthy"
 
-      check {
-        name = "loki_alive"
-        type = "grpc"
-        port = "grpc"
-        interval = "10s"
-        timeout = "3s"
-      }
-
       check {
         name = "loki_ready"
         type = "http"
@@ -66,6 +58,12 @@ job "loki" {
         timeout = "3s"
       }
     }
+
+    service {
+      name = "loki-grpc"
+      port = "grpc"
+    }
+
     task "server" {
       driver = "exec"
       env {

loki/loki.yml.tpl

@@ -3,31 +3,49 @@ auth_enabled: false
 server:
   http_listen_port: {{ env "NOMAD_PORT_http" }}
   grpc_listen_port: {{ env "NOMAD_PORT_grpc" }}
-memberlist:
-  join_members:
-    - loki-http-server
+  register_instrumentation: true
+  http_server_read_timeout: "40s"
+  http_server_write_timeout: "50s"
+distributor:
+  ring:
+    kvstore:
+      store: consul
+      prefix: loki/collectors
+ingester:
+  lifecycler:
+    address: loki-grpc.service.consul
+    ring:
+      kvstore:
+        store: consul
+        prefix: loki/collectors
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1m
+  chunk_retain_period: 30s
 schema_config:
   configs:
-    - from: 2022-01-01
-      store: boltdb-shipper
+    - from: 2020-01-01
+      store: aws
       object_store: s3
       schema: v11
       index:
-        prefix: index_
-        period: 24h
-common:
-  path_prefix: local/
-  replication_factor: 1
-  storage:
-    s3:
-      endpoint:  {{ key "jobs/loki/s3_endpoint" }}
-      bucketnames: {{ key "jobs/loki/logs_bucket" }}
-      access_key_id: {{ env "access_key" }}
-      secret_access_key: {{ env "secret_key" }}
-      s3forcepathstyle: true
-  ring:
-    kvstore:
-      store: consul
+        prefix: loki_
+
+storage_config:
+  aws:
+    region: ams3
+    endpoint:  https://{{ key "jobs/loki/s3_endpoint" }}
+    bucketnames: {{ key "jobs/loki/logs_bucket" }}
+    access_key_id: {{ env "access_key" }}
+    secret_access_key: {{ env "secret_key" }}
+    s3forcepathstyle: true
+    insecure: false
+    dynamodb:
+      dynamodb_url: inmemory:///index
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+    shared_store: s3
 ruler:
   storage:
     s3:

loki/main.tf

@@ -38,13 +38,9 @@ provider "digitalocean" {
   spaces_secret_key = jsondecode(data.vault_kv_secret_v2.digitalocean.data_json)["spaces_secret"]
 }
 
-provider "nomad" {
+provider "nomad" {}
 
-}
-
-provider "consul" {
-
-}
+provider "consul" {}
 
 resource "digitalocean_spaces_bucket" "logs" {
   region = var.doregion
@@ -79,7 +75,7 @@ resource "consul_keys" "endpoint" {
 
   key {
     path  = "jobs/loki/s3_endpoint"
-    value = "https://${digitalocean_spaces_bucket.logs.region}.digitaloceanspaces.com"
+    value = "${digitalocean_spaces_bucket.logs.region}.digitaloceanspaces.com"
   }
 }
 
@@ -90,10 +86,11 @@ resource "nomad_job" "loki" {
     enabled  = true
     allow_fs = true
     vars = {
-      "access_key" = jsondecode(data.vault_kv_secret_v2.digitalocean.data_json)["spaces_key"]
-      "secret_key" = jsondecode(data.vault_kv_secret_v2.digitalocean.data_json)["spaces_secret"]
+      "access_key" = jsondecode(data.vault_kv_secret_v2.digitalocean.data_json)["loki_spaces_key"]
+      "secret_key" = jsondecode(data.vault_kv_secret_v2.digitalocean.data_json)["loki_spaces_secret"]
     }
   }
-  purge_on_destroy = true
-  detach           = false
+  purge_on_destroy      = false
+  detach                = true
+  deregister_on_destroy = false
 }

loki/variables.tf

@@ -3,3 +3,9 @@ variable "doregion" {
   default     = "ams3"
   type        = string
 }
+
+variable "loki_version" {
+  description = "Version of Grafana Loki to deploy. See "
+  type = string
+  default = "v2.7.1"
+}

prometheus.nomad

@@ -20,7 +20,7 @@ job "prometheus" {
   }
 
   group "monitoring" {
-    count = 1
+    count = 2
 
     network {
       port "prometheus_ui" {
@@ -90,7 +90,7 @@ scrape_configs:
       format: ['prometheus']
   - job_name: 'nomad_metrics'
     nomad_sd_configs:
-      - server: nomad.service.consul:4646
+      - server: http://nomad.service.consul:4646
 EOH
       }
 

promtail.nomad

@@ -1,16 +1,9 @@
-# There can only be a single job definition per file. This job is named
-# "example" so it will create a job with the ID and Name "example".
-
-# The "job" stanza is the top-most configuration option in the job
-# specification. A job is a declarative specification of tasks that Nomad
-# should run. Jobs have a globally unique name, one or many task groups, which
-# are themselves collections of one or many tasks.
-#
-# For more information and examples on the "job" stanza, please see
-# the online documentation at:
-#
-#     https://www.nomadproject.io/docs/job-specification/job
-#
+variable "promtail_version" {
+  description = "Version of Promtail to deploy"
+  type = string
+  default = "v2.5.0"
+}
+
 job "promtail" {
 
   meta {
@@ -61,11 +54,26 @@ job "promtail" {
           ignore_warnings = false
         }
       }
+    }
+
+    service {
+      name = "grpc"
+      tags = ["logs", "promtail", "observability", "grpc"]
+      port = "grpc"
 
+      check {
+        name = "promtail-grpc"
+        grpc_service = ""
+        type = "grpc"
+        interval = "15s"
+        timeout = "5s"
+        grpc_use_tls = false
+        tls_skip_verify = true
+      }
     }
 
     restart {
-      attempts = 2
+      attempts = 3
       interval = "10m"
       delay = "15s"
       mode = "delay"
@@ -91,7 +99,7 @@ job "promtail" {
       //    mode = "file"
       // }
       artifact {
-        source = "https://github.com/grafana/loki/releases/download/v2.5.0/promtail-linux-arm64.zip"
+        source = "https://github.com/grafana/loki/releases/download/v2.5.0/promtail-linux-${attr.cpu.arch}.zip"
         destination = "local/promtail"
         mode = "file"
       }

promtail.yml.tpl

@@ -1,9 +1,10 @@
 server:
+  log_level: info
   http_listen_port: 9080
-  grpc_listen_port: 0
+  grpc_listen_port: 9095
 
 positions:
-  filename: /tmp/positions.yaml
+  filename: /data/positions.yaml
 
 clients:
   - url: http://loki-http-server.service.consul:3100/loki/api/v1/push
@@ -23,3 +24,11 @@ scrape_configs:
     labels:
       job: nomad
       __path__: /var/log/nomad*.log
+- job_name: journal
+  journal:
+    max_age: 12h
+    labels:
+      job: systemd-journal
+    relabel_configs:
+      - source_labsl: ['__journal__systemd_unit']
+        target_label: 'unit'