diff --git a/manifests/profile/prometheus/exporter/webserver/base.pp b/manifests/profile/prometheus/exporter/webserver/base.pp new file mode 100644 index 000000000..65460c58e --- /dev/null +++ b/manifests/profile/prometheus/exporter/webserver/base.pp @@ -0,0 +1,31 @@ +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. + +class nebula::profile::prometheus::exporter::webserver::base ( + Hash[String, Hash[String, String]] $mariadb_connect = {}, + Hash[String, Array[String]] $nfs_mounts = {}, + Hash[String, Array[String]] $solr_instances = {}, + Hash[String, Boolean] $check_shibd = {}, + String $target, +) { + include stdlib + $target_mariadb = pick_default($mariadb_connect[$target], {}) + $target_nfs = pick_default($nfs_mounts[$target], []) + $target_solr = pick_default($solr_instances[$target], []) + $target_shibd = pick_default($check_shibd[$target], false) + + file { "/usr/local/lib/prom_web_exporter/metrics": + mode => "0755", + content => template("nebula/profile/prometheus/exporter/webserver/metrics.sh.erb"), + require => File["/usr/local/lib/prom_web_exporter"], + } + + file { "/usr/local/lib/prom_web_exporter": + ensure => "directory", + } + + if $target_mariadb != {} { + package { "mariadb-client": } + } +} diff --git a/manifests/profile/prometheus/exporter/webserver/fulcrum.pp b/manifests/profile/prometheus/exporter/webserver/fulcrum.pp new file mode 100644 index 000000000..5cb4ba23f --- /dev/null +++ b/manifests/profile/prometheus/exporter/webserver/fulcrum.pp @@ -0,0 +1,9 @@ +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. + +class nebula::profile::prometheus::exporter::webserver::fulcrum { + class { "nebula::profile::prometheus::exporter::webserver::base": + target => "fulcrum" + } +} diff --git a/manifests/profile/prometheus/exporter/webserver/hathitrust.pp b/manifests/profile/prometheus/exporter/webserver/hathitrust.pp new file mode 100644 index 000000000..88a7bc741 --- /dev/null +++ b/manifests/profile/prometheus/exporter/webserver/hathitrust.pp @@ -0,0 +1,9 @@ +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. + +class nebula::profile::prometheus::exporter::webserver::hathitrust { + class { "nebula::profile::prometheus::exporter::webserver::base": + target => "hathitrust" + } +} diff --git a/manifests/profile/prometheus/exporter/webserver/quod.pp b/manifests/profile/prometheus/exporter/webserver/quod.pp new file mode 100644 index 000000000..c88420dff --- /dev/null +++ b/manifests/profile/prometheus/exporter/webserver/quod.pp @@ -0,0 +1,9 @@ +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. + +class nebula::profile::prometheus::exporter::webserver::quod { + class { "nebula::profile::prometheus::exporter::webserver::base": + target => "quod" + } +} diff --git a/manifests/profile/prometheus/exporter/webserver/vhost.pp b/manifests/profile/prometheus/exporter/webserver/vhost.pp new file mode 100644 index 000000000..377bf4079 --- /dev/null +++ b/manifests/profile/prometheus/exporter/webserver/vhost.pp @@ -0,0 +1,18 @@ +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. + +class nebula::profile::prometheus::exporter::webserver::vhost ( + Boolean $testing = false +) { + if $testing { + include apache + } + + apache::vhost { "prometheus-webserver-exporter": + port => "9180", + docroot => "/usr/local/lib/prom_web_exporter", + aliases => [{ scriptalias => "/", path => "/usr/local/lib/prom_web_exporter/" }], + rewrites => [{ rewrite_rule => ["^/$ /metrics [last,redirect=permanent]"]}], + } +} diff --git a/manifests/profile/prometheus/exporter/webserver/www_lib.pp b/manifests/profile/prometheus/exporter/webserver/www_lib.pp new file mode 100644 index 000000000..4f9cdde71 --- /dev/null +++ b/manifests/profile/prometheus/exporter/webserver/www_lib.pp @@ -0,0 +1,9 @@ +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. + +class nebula::profile::prometheus::exporter::webserver::www_lib { + class { "nebula::profile::prometheus::exporter::webserver::base": + target => "www_lib" + } +} diff --git a/manifests/role/app_host/quod_dev.pp b/manifests/role/app_host/quod_dev.pp index 7c1d277a3..005438ed7 100644 --- a/manifests/role/app_host/quod_dev.pp +++ b/manifests/role/app_host/quod_dev.pp @@ -19,4 +19,5 @@ include nebula::profile::quod::dev::perl include nebula::profile::prometheus::exporter::mysql include nebula::profile::tesseract + include nebula::profile::prometheus::exporter::webserver::quod } diff --git a/manifests/role/app_host/quod_prod.pp b/manifests/role/app_host/quod_prod.pp index cd6ac1a3c..9aa519880 100644 --- a/manifests/role/app_host/quod_prod.pp +++ b/manifests/role/app_host/quod_prod.pp @@ -17,4 +17,5 @@ include nebula::profile::quod::prod::perl include nebula::profile::quod::prod::haproxy include nebula::profile::networking::firewall::http + include nebula::profile::prometheus::exporter::webserver::quod } diff --git a/manifests/role/webhost/fulcrum_www_and_app.pp b/manifests/role/webhost/fulcrum_www_and_app.pp index 2be6131fe..891e4bc3a 100644 --- a/manifests/role/webhost/fulcrum_www_and_app.pp +++ b/manifests/role/webhost/fulcrum_www_and_app.pp @@ -60,4 +60,7 @@ } ensure_packages(['pigz']) + + include nebula::profile::prometheus::exporter::webserver::vhost + include nebula::profile::prometheus::exporter::webserver::fulcrum } diff --git a/manifests/role/webhost/htvm.pp b/manifests/role/webhost/htvm.pp index d30b32100..a0104e462 100644 --- a/manifests/role/webhost/htvm.pp +++ b/manifests/role/webhost/htvm.pp @@ -39,4 +39,7 @@ # for HathiTrust deployment scripts package { 'rdist': } + + include nebula::profile::prometheus::exporter::webserver::vhost + include nebula::profile::prometheus::exporter::webserver::hathitrust } diff --git a/manifests/role/webhost/www_lib_vm.pp b/manifests/role/webhost/www_lib_vm.pp index 9a14e513c..5df9ec3db 100644 --- a/manifests/role/webhost/www_lib_vm.pp +++ b/manifests/role/webhost/www_lib_vm.pp @@ -45,4 +45,7 @@ config_source => $shibboleth_config_source, watchdog_minutes => '*/30', } + + include nebula::profile::prometheus::exporter::webserver::vhost + include nebula::profile::prometheus::exporter::webserver::www_lib } diff --git a/spec/classes/profile/prometheus/exporter/webserver/base_spec.rb b/spec/classes/profile/prometheus/exporter/webserver/base_spec.rb new file mode 100644 index 000000000..37fe1d8b9 --- /dev/null +++ b/spec/classes/profile/prometheus/exporter/webserver/base_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. +require 'spec_helper' + +describe 'nebula::profile::prometheus::exporter::webserver::base' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + it { is_expected.not_to compile } + + context "with target set to abcd" do + let(:params) { { target: "abcd" } } + + def have_script + contain_file("/usr/local/lib/prom_web_exporter/metrics") + end + + it { is_expected.to compile } + it { is_expected.not_to contain_package("mariadb-client") } + it { is_expected.to contain_file("/usr/local/lib/prom_web_exporter").with_ensure("directory") } + it { is_expected.to have_script.with_mode("0755") } + it { is_expected.to have_script.with_content(/^#!\/usr\/bin\/env bash$/) } + it { is_expected.to have_script.with_content(/^ENABLE_MARIADB_CHECK="false"$/) } + it { is_expected.to have_script.with_content(/^NFS_MOUNTS=\(\)$/) } + it { is_expected.to have_script.with_content(/^SOLR_INSTANCES=\(\)$/) } + it { is_expected.to have_script.with_content(/^ENABLE_SHIBD_CHECK="false"$/) } + + context "with mariadb credentials set" do + let(:params) do + super().merge({ mariadb_connect: { "abcd" => { + "hostname" => "abcd-db-host", + "database" => "abcd-database", + "username" => "abcd-db-monitor", + "password" => "random-password", + }}}) + end + + it { is_expected.to contain_package("mariadb-client") } + it { is_expected.to have_script.with_content(/^ENABLE_MARIADB_CHECK="true"$/) } + end + + context "with nfs mounts set" do + let(:params) do + super().merge({ nfs_mounts: { "abcd" => %w[/abc /def /ghi] } }) + end + + it { is_expected.to have_script.with_content(/^NFS_MOUNTS=\("\/abc" "\/def" "\/ghi"\)$/) } + end + + context "with solr instances set" do + let(:params) do + super().merge({ solr_instances: { "abcd" => ["http://solr/solr/core"] } }) + end + + it { is_expected.to have_script.with_content(/^SOLR_INSTANCES=\("http:\/\/solr\/solr\/core"\)$/) } + end + + context "with shibd check enabled" do + let(:params) do + super().merge({ check_shibd: { "abcd" => true } }) + end + + it { is_expected.to have_script.with_content(/^ENABLE_SHIBD_CHECK="true"$/) } + end + end + end + end +end diff --git a/spec/classes/profile/prometheus/exporter/webserver/fulcrum_spec.rb b/spec/classes/profile/prometheus/exporter/webserver/fulcrum_spec.rb new file mode 100644 index 000000000..0b7eb30f5 --- /dev/null +++ b/spec/classes/profile/prometheus/exporter/webserver/fulcrum_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. +require 'spec_helper' + +describe 'nebula::profile::prometheus::exporter::webserver::fulcrum' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:base) { "nebula::profile::prometheus::exporter::webserver::base" } + + it { is_expected.to compile } + it { is_expected.to contain_class(base).with_target("fulcrum") } + end + end +end diff --git a/spec/classes/profile/prometheus/exporter/webserver/hathitrust_spec.rb b/spec/classes/profile/prometheus/exporter/webserver/hathitrust_spec.rb new file mode 100644 index 000000000..d0b8b6afe --- /dev/null +++ b/spec/classes/profile/prometheus/exporter/webserver/hathitrust_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. +require 'spec_helper' + +describe 'nebula::profile::prometheus::exporter::webserver::hathitrust' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:base) { "nebula::profile::prometheus::exporter::webserver::base" } + + it { is_expected.to compile } + it { is_expected.to contain_class(base).with_target("hathitrust") } + end + end +end diff --git a/spec/classes/profile/prometheus/exporter/webserver/quod_spec.rb b/spec/classes/profile/prometheus/exporter/webserver/quod_spec.rb new file mode 100644 index 000000000..6e4d5df97 --- /dev/null +++ b/spec/classes/profile/prometheus/exporter/webserver/quod_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. +require 'spec_helper' + +describe 'nebula::profile::prometheus::exporter::webserver::quod' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:base) { "nebula::profile::prometheus::exporter::webserver::base" } + + it { is_expected.to compile } + it { is_expected.to contain_class(base).with_target("quod") } + end + end +end diff --git a/spec/classes/profile/prometheus/exporter/webserver/vhost_spec.rb b/spec/classes/profile/prometheus/exporter/webserver/vhost_spec.rb new file mode 100644 index 000000000..583215623 --- /dev/null +++ b/spec/classes/profile/prometheus/exporter/webserver/vhost_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. +require 'spec_helper' + +describe 'nebula::profile::prometheus::exporter::webserver::vhost' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + it { is_expected.not_to compile } + + context "with apache configured for testing" do + let(:params) { { testing: true } } + + def have_vhost + contain_apache__vhost("prometheus-webserver-exporter") + end + + it { is_expected.to compile } + it { is_expected.to have_vhost.with_port(9180) } + end + end + end +end diff --git a/spec/classes/profile/prometheus/exporter/webserver/www_lib_spec.rb b/spec/classes/profile/prometheus/exporter/webserver/www_lib_spec.rb new file mode 100644 index 000000000..1407ec9ea --- /dev/null +++ b/spec/classes/profile/prometheus/exporter/webserver/www_lib_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 The Regents of the University of Michigan. +# All Rights Reserved. Licensed according to the terms of the Revised +# BSD License. See LICENSE.txt for details. +require 'spec_helper' + +describe 'nebula::profile::prometheus::exporter::webserver::www_lib' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:base) { "nebula::profile::prometheus::exporter::webserver::base" } + + it { is_expected.to compile } + it { is_expected.to contain_class(base).with_target("www_lib") } + end + end +end diff --git a/templates/profile/prometheus/exporter/webserver/metrics.sh.erb b/templates/profile/prometheus/exporter/webserver/metrics.sh.erb new file mode 100644 index 000000000..3e3a1ead8 --- /dev/null +++ b/templates/profile/prometheus/exporter/webserver/metrics.sh.erb @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Managed by puppet (nebula/profile/prometheus/exporter/webserver/metrics.sh.erb) +set -eo pipefail +IFS=$'\n\t' +PROG="$0" +USAGE="[-h]" +<% if @target_mariadb.empty? -%> +ENABLE_MARIADB_CHECK="false" +<% else -%> +ENABLE_MARIADB_CHECK="true" +DB_HOST="<%= @target_mariadb["hostname"] %>" +DB_NAME="<%= @target_mariadb["database"] %>" +DB_USER="<%= @target_mariadb["username"] %>" +DB_PASS="<%= @target_mariadb["password"] %>" +<% end -%> +<% if @target_nfs.empty? -%> +NFS_MOUNTS=() +<% else -%> +NFS_MOUNTS=("<%= @target_nfs.join('" "') %>") +<% end -%> +<% if @target_solr.empty? -%> +SOLR_INSTANCES=() +<% else -%> +SOLR_INSTANCES=("<%= @target_solr.join('" "') %>") +<% end -%> +ENABLE_SHIBD_CHECK="<%= @target_shibd %>" + +errorout() { + echo "usage: $PROG $USAGE" >&2 + [ -n "$1" ] && echo "${PROG}: error: $@" >&2 + exit 1 +} + +printhelp() { + cat < /dev/null 2> /dev/null +} + +check_nfs_mounts() { + echo "# HELP mlibrary_webserver_nfs_mount_up Check whether NFS mount is up" + echo "# TYPE mlibrary_webserver_nfs_mount_up gauge" + for mount in "${NFS_MOUNTS[@]}"; do + if nfs_mount_is_ok "$mount"; then + echo "mlibrary_webserver_nfs_mount_up{mountpoint=\"$mount\"} 1" + else + echo "mlibrary_webserver_nfs_mount_up{mountpoint=\"$mount\"} 0" + fi + done +} + +nfs_mount_is_ok() { + grep -qs " $1 " "/proc/mounts" && timeout 1 ls "$1" > /dev/null 2> /dev/null +} + +check_solr_instances() { + echo "# HELP mlibrary_webserver_solr_instance_up Check whether Solr instance is up" + echo "# TYPE mlibrary_webserver_solr_instance_up gauge" + for solr in "${SOLR_INSTANCES[@]}"; do + if solr_instance_is_ok "$solr"; then + echo "mlibrary_webserver_solr_instance_up{url=\"$solr\"} 1" + else + echo "mlibrary_webserver_solr_instance_up{url=\"$solr\"} 0" + fi + done +} + +solr_instance_is_ok() { + wget -q -T 1 -t 5 -O - "$1/admin/ping" > /dev/null 2> /dev/null +} + +check_shibd() { + echo "# HELP mlibrary_webserver_shibd_up Check whether shibd is up" + echo "# TYPE mlibrary_webserver_shibd_up gauge" + if shibd_is_ok; then + echo "mlibrary_webserver_shibd_up 1" + else + echo "mlibrary_webserver_shibd_up 0" + fi +} + +shibd_is_ok() { + timeout 1 pgrep -x "shibd" > /dev/null 2> /dev/null +} + +main "$@"