diff --git a/kubernetes/monitoring/crons/grafana-dashboard-copy.yaml b/kubernetes/monitoring/crons/grafana-dashboard-copy.yaml new file mode 100644 index 000000000..c0ac26a79 --- /dev/null +++ b/kubernetes/monitoring/crons/grafana-dashboard-copy.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: grafana-dashboard-copy + namespace: default +spec: + schedule: "1 5 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: grafana-dashboard-copy + image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.5.0.3 + command: [ "bash", "-c"] + args: + - date; echo "run dashboard-copy.py through run.sh"; + cd /data/CMSMonitoring/grafana-backup/production-copy/; + ./run.sh $GRAFANA_URL /etc/secrets/token.json + env: + - name: GRAFANA_URL + value: "https://monit-grafana.cern.ch" + volumeMounts: + - name: grafana-backup-secrets + mountPath: /etc/secrets + readOnly: true + restartPolicy: OnFailure + volumes: + - name: grafana-backup-secrets + secret: + secretName: grafana-backup-secrets \ No newline at end of file diff --git a/kubernetes/monitoring/crons/grafana-backup.yaml b/kubernetes/monitoring/crons/grafana-dashboard-exporter.yaml similarity index 78% rename from kubernetes/monitoring/crons/grafana-backup.yaml rename to kubernetes/monitoring/crons/grafana-dashboard-exporter.yaml index 68e6703ed..9147d1667 100644 --- a/kubernetes/monitoring/crons/grafana-backup.yaml +++ b/kubernetes/monitoring/crons/grafana-dashboard-exporter.yaml @@ -1,8 +1,8 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: grafana-backup - namespace: hdfs + name: grafana-dashboard-exporter + namespace: default spec: schedule: "1 4 * * *" jobTemplate: @@ -10,13 +10,13 @@ spec: template: spec: containers: - - name: grafana-backup - image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.5.0.0 + - name: grafana-dashboard-exporter + image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.5.0.3 command: [ "bash", "-c"] args: - date; echo "run dashboard-exporter.py through run.sh"; - cd /data/CMSMonitoring/grafana-backup/; - ./run.sh /etc/secrets/keytab /etc/secrets/token_backup.json $FILESYSTEM_PATH + cd /data/CMSMonitoring/grafana-backup/dashboard-exporter/; + ./run.sh /etc/secrets/keytab /etc/secrets/token.json $FILESYSTEM_PATH env: - name: FILESYSTEM_PATH value: "/eos/cms/store/group/offcomp_monit/grafana_backup/" diff --git a/kubernetes/monitoring/deploy-cron.sh b/kubernetes/monitoring/deploy-cron.sh index 3cc9c93b9..99da2e8e1 100755 --- a/kubernetes/monitoring/deploy-cron.sh +++ b/kubernetes/monitoring/deploy-cron.sh @@ -97,6 +97,8 @@ function rm_temp_deploy_secrets_sh() { #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ function deploy_secrets() { create_temp_deploy_secrets_sh + # default + "$deploy_secrets_sh" default grafana-backup-secrets # hdfs "$deploy_secrets_sh" hdfs proxy-secrets "$deploy_secrets_sh" hdfs robot-secrets @@ -105,13 +107,14 @@ function deploy_secrets() { "$deploy_secrets_sh" hdfs cron-size-quotas-secrets "$deploy_secrets_sh" hdfs cms-eos-mon-secrets "$deploy_secrets_sh" hdfs cron-spark-jobs-secrets - "$deploy_secrets_sh" hdfs grafana-backup-secrets # sqoop "$deploy_secrets_sh" sqoop sqoop-secrets # rm_temp_deploy_secrets_sh } function clean_secrets() { + # default + kubectl -n default --ignore-not-found=true delete secret grafana-backup-secrets # hdfs kubectl -n hdfs --ignore-not-found=true delete secret proxy-secrets kubectl -n hdfs --ignore-not-found=true delete secret robot-secrets @@ -120,16 +123,16 @@ function clean_secrets() { kubectl -n hdfs --ignore-not-found=true delete secret cron-size-quotas-secrets kubectl -n hdfs --ignore-not-found=true delete secret cms-eos-mon-secrets kubectl -n hdfs --ignore-not-found=true delete secret cron-spark-jobs-secrets - kubectl -n hdfs --ignore-not-found=true delete secret grafana-backup-secrets # sqoop kubectl -n sqoop --ignore-not-found=true delete secret sqoop-secrets } function deploy_services() { # default kubectl -n default apply -f services/pushgateway.yaml + kubectl -n default apply -f crons/grafana-dashboard-exporter.yaml + kubectl -n default apply -f crons/grafana-dashboard-copy.yaml # hdfs kubectl -n hdfs apply -f crons/cron-proxy.yaml - kubectl -n hdfs apply -f crons/grafana-backup.yaml kubectl -n hdfs apply -f services/cmsmon-hpc-usage.yaml kubectl -n hdfs apply -f services/cmsmon-rucio-ds.yaml kubectl -n hdfs apply -f services/cron-size-quotas.yaml @@ -140,9 +143,10 @@ function deploy_services() { function clean_services() { # default kubectl -n default --ignore-not-found=true delete -f services/pushgateway.yaml + kubectl -n default --ignore-not-found=true delete -f crons/grafana-dashboard-exporter.yaml + kubectl -n default --ignore-not-found=true delete -f crons/grafana-dashboard-copy.yaml # hdfs kubectl -n hdfs --ignore-not-found=true delete -f crons/cron-proxy.yaml - kubectl -n hdfs --ignore-not-found=true delete -f crons/grafana-backup.yaml kubectl -n hdfs --ignore-not-found=true delete -f services/cmsmon-hpc-usage.yaml kubectl -n hdfs --ignore-not-found=true delete -f services/cmsmon-rucio-ds.yaml kubectl -n hdfs --ignore-not-found=true delete -f services/cron-size-quotas.yaml diff --git a/kubernetes/monitoring/deploy-secrets.sh b/kubernetes/monitoring/deploy-secrets.sh index a03c17ff8..4b2b9f241 100755 --- a/kubernetes/monitoring/deploy-secrets.sh +++ b/kubernetes/monitoring/deploy-secrets.sh @@ -80,7 +80,7 @@ elif [ "$secret" == "promxy-secrets" ]; then elif [ "$secret" == "vmalert-secrets" ]; then files=`ls $cdir/vmalert/ | awk '{ORS=" " ; print "--from-file="D"/"$1""}' D=$cdir/vmalert | sed "s, $,,g"` elif [ "$secret" == "grafana-backup-secrets" ]; then - token="--from-file=${sdir}/grafana-backup/token_backup.json" + token="--from-file=${sdir}/grafana-backup/token.json" cmsmonit_k="--from-file=${sdir}/cmsmonit-keytab/keytab" files="${token} ${cmsmonit_k}" elif [ "$secret" == "s3-keys-secrets" ]; then