Merge pull request #1567 from nikodemas/add_grafana_backup

Add grafana production copy cronjob

kubernetes/monitoring/crons/grafana-dashboard-copy.yaml

@@ -0,0 +1,31 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: grafana-dashboard-copy
+  namespace: default
+spec:
+  schedule: "1 5 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: grafana-dashboard-copy
+            image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.5.0.3
+            command: [ "bash", "-c"]
+            args:
+            - date; echo "run dashboard-copy.py through run.sh";
+              cd /data/CMSMonitoring/grafana-backup/production-copy/;
+              ./run.sh $GRAFANA_URL /etc/secrets/token.json
+            env:
+              - name: GRAFANA_URL
+                value: "https://monit-grafana.cern.ch"
+            volumeMounts:
+              - name: grafana-backup-secrets
+                mountPath: /etc/secrets
+                readOnly: true
+          restartPolicy: OnFailure
+          volumes:
+          - name: grafana-backup-secrets
+            secret:
+              secretName: grafana-backup-secrets

kubernetes/monitoring/crons/grafana-backup.yaml → kubernetes/monitoring/crons/grafana-dashboard-exporter.yaml

@@ -1,22 +1,22 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: grafana-backup
-  namespace: hdfs
+  name: grafana-dashboard-exporter
+  namespace: default
 spec:
   schedule: "1 4 * * *"
   jobTemplate:
     spec:
       template:
         spec:
           containers:
-          - name: grafana-backup
-            image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.5.0.0
+          - name: grafana-dashboard-exporter
+            image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.5.0.3
             command: [ "bash", "-c"]
             args:
             - date; echo "run dashboard-exporter.py through run.sh";
-              cd /data/CMSMonitoring/grafana-backup/;
-              ./run.sh /etc/secrets/keytab /etc/secrets/token_backup.json $FILESYSTEM_PATH
+              cd /data/CMSMonitoring/grafana-backup/dashboard-exporter/;
+              ./run.sh /etc/secrets/keytab /etc/secrets/token.json $FILESYSTEM_PATH
             env:
               - name: FILESYSTEM_PATH
                 value: "/eos/cms/store/group/offcomp_monit/grafana_backup/"

kubernetes/monitoring/deploy-cron.sh

@@ -97,6 +97,8 @@ function rm_temp_deploy_secrets_sh() {
 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 function deploy_secrets() {
     create_temp_deploy_secrets_sh
+    # default
+    "$deploy_secrets_sh" default grafana-backup-secrets
     # hdfs
     "$deploy_secrets_sh" hdfs proxy-secrets
     "$deploy_secrets_sh" hdfs robot-secrets
@@ -105,13 +107,14 @@ function deploy_secrets() {
     "$deploy_secrets_sh" hdfs cron-size-quotas-secrets
     "$deploy_secrets_sh" hdfs cms-eos-mon-secrets
     "$deploy_secrets_sh" hdfs cron-spark-jobs-secrets
-    "$deploy_secrets_sh" hdfs grafana-backup-secrets
     # sqoop
     "$deploy_secrets_sh" sqoop sqoop-secrets
     #
     rm_temp_deploy_secrets_sh
 }
 function clean_secrets() {
+    # default
+    kubectl -n default --ignore-not-found=true delete secret grafana-backup-secrets
     # hdfs
     kubectl -n hdfs --ignore-not-found=true delete secret proxy-secrets
     kubectl -n hdfs --ignore-not-found=true delete secret robot-secrets
@@ -120,16 +123,16 @@ function clean_secrets() {
     kubectl -n hdfs --ignore-not-found=true delete secret cron-size-quotas-secrets
     kubectl -n hdfs --ignore-not-found=true delete secret cms-eos-mon-secrets
     kubectl -n hdfs --ignore-not-found=true delete secret cron-spark-jobs-secrets
-    kubectl -n hdfs --ignore-not-found=true delete secret grafana-backup-secrets
     # sqoop
     kubectl -n sqoop --ignore-not-found=true delete secret sqoop-secrets
 }
 function deploy_services() {
     # default
     kubectl -n default apply -f services/pushgateway.yaml
+    kubectl -n default apply -f crons/grafana-dashboard-exporter.yaml
+    kubectl -n default apply -f crons/grafana-dashboard-copy.yaml
     # hdfs
     kubectl -n hdfs apply -f crons/cron-proxy.yaml
-    kubectl -n hdfs apply -f crons/grafana-backup.yaml
     kubectl -n hdfs apply -f services/cmsmon-hpc-usage.yaml
     kubectl -n hdfs apply -f services/cmsmon-rucio-ds.yaml
     kubectl -n hdfs apply -f services/cron-size-quotas.yaml
@@ -140,9 +143,10 @@ function deploy_services() {
 function clean_services() {
     # default
     kubectl -n default --ignore-not-found=true delete -f services/pushgateway.yaml
+    kubectl -n default --ignore-not-found=true delete -f crons/grafana-dashboard-exporter.yaml
+    kubectl -n default --ignore-not-found=true delete -f crons/grafana-dashboard-copy.yaml
     # hdfs
     kubectl -n hdfs --ignore-not-found=true delete -f crons/cron-proxy.yaml
-    kubectl -n hdfs --ignore-not-found=true delete -f crons/grafana-backup.yaml
     kubectl -n hdfs --ignore-not-found=true delete -f services/cmsmon-hpc-usage.yaml
     kubectl -n hdfs --ignore-not-found=true delete -f services/cmsmon-rucio-ds.yaml
     kubectl -n hdfs --ignore-not-found=true delete -f services/cron-size-quotas.yaml

kubernetes/monitoring/deploy-secrets.sh

@@ -80,7 +80,7 @@ elif [ "$secret" == "promxy-secrets" ]; then
 elif [ "$secret" == "vmalert-secrets" ]; then
     files=`ls $cdir/vmalert/ | awk '{ORS=" " ; print "--from-file="D"/"$1""}' D=$cdir/vmalert | sed "s, $,,g"`
 elif [ "$secret" == "grafana-backup-secrets" ]; then
-    token="--from-file=${sdir}/grafana-backup/token_backup.json"
+    token="--from-file=${sdir}/grafana-backup/token.json"
     cmsmonit_k="--from-file=${sdir}/cmsmonit-keytab/keytab"
     files="${token} ${cmsmonit_k}"
 elif [ "$secret" == "s3-keys-secrets" ]; then