diff --git a/docker/pypi/wmagent-couchdb/Dockerfile b/docker/pypi/wmagent-couchdb/Dockerfile new file mode 100644 index 000000000..c1e78dd2a --- /dev/null +++ b/docker/pypi/wmagent-couchdb/Dockerfile @@ -0,0 +1,70 @@ +ARG TAG=3.2.2 +FROM couchdb:${TAG} +MAINTAINER Valentin Kuznetsov vkuznet@gmail.com + +ARG TAG +ENV TAG=${TAG} +RUN echo TAG=$TAG + +RUN apt-get update && apt-get install -y vim less sudo wget unzip python pip + +# # Install some debugging tools +RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean + +RUN pip install CMSCouchapp + +ENV COUCH_PORT=5984 +ENV COUCH_ROOT_DIR=/data + +ENV COUCH_BASE_DIR=$COUCH_ROOT_DIR/srv/couchdb +ENV COUCH_ADMIN_DIR=$COUCH_ROOT_DIR/admin/couchdb +ENV WMA_ADMIN_DIR=$COUCH_ROOT_DIR/admin/wmagent +ENV COUCH_CERTS_DIR=$COUCH_ROOT_DIR/certs + +ENV COUCH_CURRENT_DIR=$COUCH_BASE_DIR/$TAG +ENV COUCH_MANAGE_DIR=$COUCH_CURRENT_DIR +ENV COUCH_AUTH_DIR=$COUCH_BASE_DIR/auth/ +ENV COUCH_INSTALL_DIR=$COUCH_CURRENT_DIR/install +ENV COUCH_STATE_DIR=$COUCH_CURRENT_DIR/state +ENV COUCH_DATABASE_DIR=$COUCH_INSTALL_DIR/database +ENV COUCH_CONFIG_DIR=$COUCH_CURRENT_DIR/config +ENV COUCH_LOG_DIR=$COUCH_CURRENT_DIR/logs +ENV COUCH_DEPLOY_DIR=/usr/local +ENV COUCH_ENV_FILE=$COUCH_DEPLOY_DIR/deploy/env.sh +ENV COUCH_SECRETS_FILE=$COUCH_ADMIN_DIR/CouchDB.secrets +ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets + + +# RUN mkdir -p /etc/grid-security + +# start the setup +RUN mkdir -p $COUCH_ROOT_DIR + +ENV PATH="${COUCH_ROOT_DIR}:${PATH}" + +RUN mkdir -p $COUCH_CURRENT_DIR $COUCH_CONFIG_DIR $COUCH_MANAGE_DIR $COUCH_LOG_DIR $COUCH_DATABASE_DIR $COUCH_STATE_DIR $COUCH_AUTH_DIR +RUN ln -s $COUCH_CURRENT_DIR $COUCH_BASE_DIR/current + +# add necessary scripts +ADD run.sh ${COUCH_ROOT_DIR}/ +ADD manage ${COUCH_MANAGE_DIR}/manage +RUN ln -s ${COUCH_MANAGE_DIR}/manage ${COUCH_ROOT_DIR}/manage + +# The $COUCH_CONFIG_DIR is to be mounted from the host and locla.ini read from there +ADD local.ini ${COUCH_DEPLOY_DIR}/local.ini +RUN ln -s ${COUCH_CONFIG_DIR}/local.ini /opt/couchdb/etc/local.d/ + +ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}" +ENV CRYPTOGRAPHY_ALLOW_OPENSSL_102=true + +# Set command prompt for root +RUN <> /root/.bashrc +export PS1="(CouchDB-$TAG) [\u@\h:\W]# " +EOF + +# allow dynamic users to create homefolders and .bashrc +RUN chmod 777 /home + +# setup final environment +WORKDIR $COUCH_ROOT_DIR +ENTRYPOINT ["./run.sh"] diff --git a/docker/pypi/wmagent-couchdb/README.md b/docker/pypi/wmagent-couchdb/README.md new file mode 100644 index 000000000..17269fb79 --- /dev/null +++ b/docker/pypi/wmagent-couchdb/README.md @@ -0,0 +1,257 @@ +# Couchdb default image for running wmagent + +## Prerequisites + +This image inherits from the mainstream `couchdb` one, and follows the same +tagging schema. On top of the base `couchdb` image we add all the structure +needed for running the WMAgent with CouchDB and two main scripts: + +* `couchdb-docker-run.sh` +* `couchdb-docker-build.sh` + +For building the containers, and for creating the mount area at the host and the +the bind mounts inside the container, respectively. Those are as follows: + +* At the host: + +``` +/data/dockerMount/{admin|srv}/couchdb +``` + +* At the container: + +``` +/data/{admin|srv}/couchdb +``` + +Upon starting the container we try to initialize the default user and system +databases, which if previously created should exist in the host mount area. + +There are no other external dependencies. + +We fetch all the passwords from two secrets files (giving the container-based paths below, which map to `/data/dockerMount/{wmagent|couchdb}` in the host): + +* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials of the + user to be used by the WMAgent to connect to the database + +## Usage + +### Building CouchDB image + +We can build everything locally and upload it at the CERN registry: https://registry.cern.ch + +* Using the wrapper script to build CouchDB locally: +``` +$ ssh vocms**** +user@vocms0290:wmagent-couchdb $ cd /data +user@vocms0290:wmagent-couchdb $ git clone https://github.com/dmwm/CMSKubernetes.git +user@vocms0290:wmagent-couchdb $ cd /data/CMSKubernetes/docker/pypi/wmagent-couchdb/ +user@vocms0290:wmagent-couchdb $ ./couchdb-docker-build.sh -t 3.2.2 + +user@vocms0290:wmagent-couchdb $ docker image ls +REPOSITORY TAG IMAGE ID CREATED SIZE +local/couchdb 3.2.2 2ec9e59aa0e9 1 minute ago 819MB +``` +* Using the wrapper script to build and upload CouchDB to registry.cern.ch: +``` +./couchdb-docker-build.sh -t 3.2.2 -p +``` + +### Running a CouchDB container + +We can run from the local repository or from upstream CERN registry. + +* Running from a local build: + +``` +cmst1@vocms0290:wmagent-couchdb $ ./couchdb-docker-run.sh -t 3.2.2 +Starting the couchdb:3.2.2 docker container with the following parameters: --user cmst1 +8decd12e153d74c9de48764b8d3faf975d7d52897c5b0fc1032e6fef7a7c74dd + +cmst1@vocms0290:wmagent-couchdb $ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +8decd12e153d local/couchdb:3.2.2 "./run.sh" 12 seconds ago Up 11 seconds couchdb +``` + +* Running from CERN registry: +``` +cmst1@vocms0290:wmagent-couchdb $ ./couchdb-docker-run.sh -t 3.2.2 -p +Pulling Docker image: registry.cern.ch/cmsweb/cdb:3.2.2 +3.2.2: Pulling from cmsweb/couchdb +Digest: sha256:61f798b55a1c743686e1568509975308dc07b5b24486894053d6a312983c4af6 +Status: Downloaded newer image for registry.cern.ch/cmsweb/couchdb:3.2.2 +registry.cern.ch/cmsweb/couchdb:3.2.2 +Starting the couchdb:3.2.2 docker container with the following parameters: --user cmst1 +21d9c6598f35e627834d1b796460047605d6255cebc746d572289c7b418053ed + +cmst1@vocms0290:wmagent-couchdb $ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +21d9c6598f35 registry.cern.ch/couchdb:3.2.2 "./run.sh" 7 seconds ago Up 6 seconds couchdb + +``` + +* Killing the container directly from the host: +``` +cmst1@vocms0290:wmagent-couchdb $ docker kill couchdb +couchdb + +``` + +* Connecting to a running container: +``` +cmst1@vocms0290:wmagent-couchdb $ docker exec -it couchdb bash +(CouchDB-3.2.2) [cmst1@vocms0290:data]$ + +``` + +* Fetching startup logs: +``` +cmst1@vocms0265:wmagent-couchdb $ docker logs couchdb + + +Installing WorkQueue app into database: http://localhost:5984/workqueue +Installing WorkQueue app into database: http://localhost:5984/workqueue_inbox +start sleeping....zzz +``` + +### Managing the database service: + +All of the commands bellow must be run from inside the container. + +* General options: +``` +(CouchDB-3.2.2) [cmst1@vocms0290:data]$ manage help + +Usage: manage ACTION [ARG] [SECURITY-STRING] + +Available actions: + help show this help + version get current version of the service + status show current service's status + sysboot start server from crond if not running + restart (re)start the service + start (re)start the service + stop stop the service + pushapps push couch applications + pushreps push couch replications + updatecouchapps pull new couch applications from WMCore repo + compact compact database ARG + compactviews compact database views for design doc ARG ARG + cleanviews clean view named ARG + backup rsync databases to ARG (i.e. [user@]host:path) + archive archive backups to ARG area in castor + print-settings print a basic set of environment variables + +For more details please refer to operations page: + https://twiki.cern.ch/twiki/bin/view/CMS/CouchDB + +``` + +* Start/Stop the database server: +``` +(CouchDB-3.2.2) [cmst1@vocms0290:data]$ manage start + +Which couchdb: /opt/couchdb/bin/couchdb + With configuration directory: /data/srv/couchdb/3.2.2/config + With logdir: /data/srv/couchdb/3.2.2/logs + nohup couchdb -couch_ini /data/srv/couchdb/3.2.2/config >> /data/srv/couchdb/3.2.2/logs/couch.log 2>&1 & + +``` + +* Printing the basic set of environment variables for the current server: + +``` +(CouchDB-3.2.2) [cmst1@vocms0290:data]$ manage print-settings + +ME : 3.2.2 +TOP : /data +ROOT : /data/srv +CFGDIR : /data/srv/couchdb/3.2.2/config +LOGDIR : /data/srv/couchdb/3.2.2/logs +STATEDIR : /data/srv/couchdb/3.2.2/state +KEYFILE : /data/srv/couchdb/auth//hmackey.ini + +COUCH_ROOT_DIR : /data +COUCH_BASE_DIR : /data/srv/couchdb +COUCH_STATE_DIR : /data/srv/couchdb/3.2.2/state +COUCH_INSTALL_DIR : /data/srv/couchdb/3.2.2/install +COUCH_CONFIG_DIR : /data/srv/couchdb/3.2.2/config + +``` + +* Pushing new couch applications: + +``` +(CouchDB-3.2.2) [cmst1@vocms0265:data]$ manage pushapps + +Installing ACDC app into database: http://localhost:5984/acdcserver +Installing GroupUser app into database: http://localhost:5984/acdcserver +Installing ReqMgrAux app into database: http://localhost:5984/reqmgr_auxiliary +Installing ReqMgr app into database: http://localhost:5984/reqmgr_workload_cache +Installing ConfigCache app into database: http://localhost:5984/reqmgr_config_cache +Installing WorkloadSummary app into database: http://localhost:5984/workloadsummary +Installing LogDB app into database: http://localhost:5984/wmstats_logdb +Installing WMStats app into database: http://localhost:5984/wmstats +Installing WMStatsErl app into database: http://localhost:5984/wmstats +Installing WMStatsErl1 app into database: http://localhost:5984/wmstats +Installing WMStatsErl2 app into database: http://localhost:5984/wmstats +Installing WMStatsErl3 app into database: http://localhost:5984/wmstats +Installing WMStatsErl4 app into database: http://localhost:5984/wmstats +Installing WMStatsErl5 app into database: http://localhost:5984/wmstats +Installing WMStatsErl6 app into database: http://localhost:5984/wmstats +Installing WMStatsErl7 app into database: http://localhost:5984/wmstats +Installing T0Request app into database: http://localhost:5984/t0_request +Installing WorkloadSummary app into database: http://localhost:5984/t0_workloadsummary +Installing LogDB app into database: http://localhost:5984/t0_logdb +Installing WMStats app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl1 app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl2 app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl3 app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl4 app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl5 app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl6 app into database: http://localhost:5984/tier0_wmstats +Installing WMStatsErl7 app into database: http://localhost:5984/tier0_wmstats +Installing WorkQueue app into database: http://localhost:5984/workqueue +Installing WorkQueue app into database: http://localhost:5984/workqueue_inbox + +``` + +* Pushing new couch replications + +``` +(CouchDB-3.2.2) [cmst1@vocms0265:data]$ manage pushreps +``` + +* Pulling new applications from WMCore repo + +``` +(CouchDB-3.2.2) [cmst1@vocms0290:data]$ manage updatecouchapps 2.3.0 + +/data/srv/couchdb/3.2.2/install/stagingarea/tmp /data + +Pulling couchapps version 2.3.0 from Github... +2024-03-13 12:45:01 URL:https://codeload.github.com/dmwm/WMCore/tar.gz/refs/tags/2.3.0 [11592963] -> "2.3.0.tar.gz" [1] + +Pulling additional reqmon and t0_reqmon dependencies... + +Downloading jquery-ui.min.js... +2024-03-13 12:45:02 URL:https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.18/jquery-ui.min.js [201842/201842] -> "jquery-ui.min.js" [1] + +Downloading jquery.min.js... +2024-03-13 12:45:02 URL:http://code.jquery.com/jquery-1.7.2.min.js [94840/94840] -> "jquery-1.7.2.min.js" [1] + +Downloading Datatables... +2024-03-13 12:45:02 URL:https://datatables.net/releases/DataTables-1.9.1.zip [2415658/2415658] -> "DataTables-1.9.1.zip" [1] + +Downloading YUI... +2024-03-13 12:45:03 URL:https://yui.github.io/yui2/archives/yui_2.9.0.zip [14294111/14294111] -> "yui_2.9.0.zip" [1] +/data/srv/couchdb/3.2.2/install/stagingarea/tmp/yui /data/srv/couchdb/3.2.2/install/stagingarea/tmp /data +/data/srv/couchdb/3.2.2/install/stagingarea/tmp /data +Removing old couchapps... +Installing new couchapps... +/data +Cleaning up! +``` + + diff --git a/docker/pypi/wmagent-couchdb/couchdb-docker-build.sh b/docker/pypi/wmagent-couchdb/couchdb-docker-build.sh new file mode 100755 index 000000000..bb6c9b889 --- /dev/null +++ b/docker/pypi/wmagent-couchdb/couchdb-docker-build.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +### This script is to be used for building a CouchDB docker image based on pypi +### It depends on a single parameter WMA_TAG, which is to be passed to the basic +### CouchDB deployment script install.sh at build time through `docker --build-arg` + + +help(){ + echo -e $* + cat < [-p] + + -t The Couchdb (upstream tag) tag to base this this image on + -p Push the CouchDB image to registry.cern.ch + -l Push the current tag also as latest to registry.cern.ch + -h Provides this help + +Example: ./couchdb-docker-build.sh -t 3.2.2 + +EOF +} + +usage(){ + help $* + exit 1 +} + +TAG=3.2.2 +PUSH=false +LATEST=false + +### Argument parsing: +while getopts ":t:hpl" opt; do + case ${opt} in + t) TAG=$OPTARG ;; + p) PUSH=true ;; + l) LATEST=true ;; + h) help; exit $? ;; + \? ) + msg="Invalid Option: -$OPTARG" + usage "$msg" ;; + : ) + msg="Invalid Option: -$OPTARG requires an argument" + usage "$msg" ;; + esac +done + +# NOTE: The COUCH_USER may refer to both the couch user to run the CouchDB +# service inside the container and to the database admin user as well +# dockerOpts=" --network=host --progress=plain --build-arg COUCH_USER=$COUCH_USER --build-arg COUCH_PASS=$COUCH_PASS --build-arg TAG=$TAG" +dockerOpts=" --network=host --progress=plain --build-arg TAG=$TAG" + +registry=local +repository=wmagent-couchdb + +docker build $dockerOpts -t $registry/$repository:$TAG -t $registry/$repository:latest . + +$PUSH && { + # For security reasons we check if the login name and the current user match. + # If they do not, abort the execution and push nothing to registry.cern.ch. + loginUser=`logname` + currUser=`id -un` + localReg=$registry + registry=registry.cern.ch + project=cmsweb + [[ $loginUser == $currUser ]] || { + echo "ERROR: The CURRENT and the LOGIN users do not match!" + echo "ERROR: You MUST connect to $registry with your login user rather than with $currUser" + exit 1 + } + echo "Testing for existing login session to $registry with Username: $loginUser" + docker login $registry < /dev/null >/dev/null 2>&1 || { + echo "ERROR: A valid login session to $registry is required in order to be able to upload any docker image" + echo "ERROR: Please consider running 'docker login $registry' with USER:$currUser and retry again." + exit 1 + } + docker tag $localReg/$repository:$TAG $registry/$project/$repository:$TAG + echo "Uploading image $registry/$project/$repository:$TAG" + docker push $registry/$project/$repository:$TAG + $LATEST && { + docker tag $localReg/$repository:$TAG $registry/$project/$repository:latest + echo "Uploading image $registry/$project/$repository:latest" + docker push $registry/$project/$repository:latest + } +} diff --git a/docker/pypi/wmagent-couchdb/couchdb-docker-run.sh b/docker/pypi/wmagent-couchdb/couchdb-docker-run.sh new file mode 100755 index 000000000..04e738997 --- /dev/null +++ b/docker/pypi/wmagent-couchdb/couchdb-docker-run.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +### This script is to be used for running the Couchdb docker container at a VM +### Its sole purpose is to set all the needed mount points from the Host VM and +### forward all Couchdb runtime parameters to the Couchdb container entrypoint run.sh +### It accepts only the set of parameters relevant to Couchdb's container run.sh +### and no build dependent ones. The docker image tag to be searched for execution is +### always `latest`. + + +# NOTE: In the help call to the current script we only repeat the help and usage +# information for all the parameters accepted by run.sh. +help(){ + echo -e $* + cat <] [-p] [-h] + + -p Pull the image from registry.cern.ch + -t The Couchdb version/tag to be downloaded from registry.cern.ch [Default:latest] + -h + +Example: ./couchdb-docker-run.sh -t 3.2.2 + +EOF +} + +usage(){ + help $* + exit 1 +} + +PULL=false +COUCH_TAG=3.2.2 + + +### Argument parsing: +while getopts ":t:hp" opt; do + case ${opt} in + t) COUCH_TAG=$OPTARG ;; + p) PULL=true ;; + h) help; exit $? ;; + : ) + msg="Invalid Option: -$OPTARG requires an argument" + usage "$msg" ;; + esac +done + +# This is the root at the host only, it may differ from the root inside the container. +# NOTE: this may be parameterized, so that the container can run on a different mount point. +HOST_MOUNT_DIR=/data/dockerMount + +thisUser=$(id -un) +thisGroup=$(id -gn) + +# create the passwd and group mount point dynamically at runtime +passwdEntry=$(getent passwd $thisUser | awk -F : -v thisHome="/home/$thisUser" '{print $1 ":" $2 ":" $3 ":" $4 ":" $5 ":" thisHome ":" $7}') +groupEntry=$(getent group $thisGroup) + +# workaround case where Unix account is not in the local system (e.g. sssd) +[[ -d $HOST_MOUNT_DIR/admin/etc/ ]] || (mkdir -p $HOST_MOUNT_DIR/admin/etc) || exit $? +[[ -f $HOST_MOUNT_DIR/admin/etc/passwd ]] || { + echo "Creating passwd file" + getent passwd > $HOST_MOUNT_DIR/admin/etc/passwd + echo $passwdEntry >> $HOST_MOUNT_DIR/admin/etc/passwd +} +[[ -f $HOST_MOUNT_DIR/admin/etc/group ]] || { + echo "Creating group file" + getent group > $HOST_MOUNT_DIR/admin/etc/group + echo $groupEntry >> $HOST_MOUNT_DIR/admin/etc/group +} + +[[ -d $HOST_MOUNT_DIR/certs ]] || mkdir -p $HOST_MOUNT_DIR/certs || exit $? +[[ -d $HOST_MOUNT_DIR/admin/couchdb ]] || mkdir -p $HOST_MOUNT_DIR/admin/couchdb || exit $? +[[ -d $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/config ]] || mkdir -p $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/config || exit $? +[[ -d $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/install ]] || mkdir -p $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/install || exit $? +[[ -d $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/logs ]] || mkdir -p $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/logs || exit $? +[[ -d $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/state ]] || mkdir -p $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/state || exit $? + + +dockerOpts=" +--detach \ +--network=host \ +--rm \ +--hostname=$(hostname -f) \ +--user $(id -u):$(id -g) \ +--name=couchdb \ +--mount type=bind,source=/tmp,target=/tmp \ +--mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/install,target=/data/srv/couchdb/current/install \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/logs,target=/data/srv/couchdb/current/logs \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/state,target=/data/srv/couchdb/current/state \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/config,target=/data/srv/couchdb/current/config \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/couchdb,target=/data/admin/couchdb/ \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/passwd,target=/etc/passwd,readonly \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/group,target=/etc/group,readonly \ +--mount type=bind,source=/etc/sudoers,target=/etc/sudoers,readonly \ +--mount type=bind,source=/etc/sudoers.d,target=/etc/sudoers.d,readonly \ +" + +# --mount type=bind,source=$HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/config,target=/data/srv/couchdb/current/config \ +# --mount type=bind,source=$HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG/install/database,target=/data/srv/couchdb/current/install/database \ + +registry=local +repository=wmagent-couchdb + +$PULL && { + registry=registry.cern.ch + project=cmsweb + echo "Pulling Docker image: $registry/$project/$repository:$COUCH_TAG" + docker pull $registry/$project/$repository:$COUCH_TAG + docker tag $registry/$project/$repository:$COUCH_TAG $registry/$repository:$COUCH_TAG + docker tag $registry/$project/$repository:$COUCH_TAG $registry/$repository:latest +} + +echo "Starting couchdb:$COUCH_TAG docker container with user: $thisUser:$thisGroup" +docker run $dockerOpts $registry/$repository:$COUCH_TAG && ( + [[ -h $HOST_MOUNT_DIR/srv/couchdb/current ]] && rm -f $HOST_MOUNT_DIR/srv/couchdb/current + ln -s $HOST_MOUNT_DIR/srv/couchdb/$COUCH_TAG $HOST_MOUNT_DIR/srv/couchdb/current ) diff --git a/docker/pypi/wmagent-couchdb/local.ini b/docker/pypi/wmagent-couchdb/local.ini new file mode 100644 index 000000000..ea5122f93 --- /dev/null +++ b/docker/pypi/wmagent-couchdb/local.ini @@ -0,0 +1,72 @@ +; WMAgent CouchDB configuration settings + +[chttpd] +; port = 6994 +port = 5984 +bind_address = 0.0.0.0 +; Maximum period in milliseconds to wait for a change before the response is sent +changes_timeout = 300000 + +[couchdb] +;max_document_size = 4294967296 ; bytes +database_dir = /data/srv/couchdb/current/install/database +view_index_dir = /data/srv/couchdb/current/install/database +uri_file = /data/srv/couchdb/current/logs/couch.uri +os_process_timeout = 1000000 +; single node is only for test purposes, otherwise it will work just like CouchDB 1.x +; for now, define it in the default config such that _users, _replicator and _global_changes databases are automatically created +single_node=true +max_dbs_open = 500 +uuid = 18f53118737ed74893055db0ffa972e2 + +[log] +level = info +file = /data/srv/couchdb/current/logs/couch.log +; include_sasl = true + +[ssl] +enable = true +cert_file = /data/certs/servicecert.pem +key_file = /data/certs/servicekey.pem +cacert_file = /data/certs/servicecert.pem +ssl_certificate_max_depth = 10 +verify_ssl_certificates = false +; fail_if_no_peer_cert = false +; tls_versions = ['tlsv1.3'] +; log_level = debug + +[replicator] +cert_file = /data/certs/servicecert.pem +key_file = /data/certs/servicekey.pem +cacert_file = /data/certs/servicecert.pem +ssl_certificate_max_depth = 10 +verify_ssl_certificates = false +; fail_if_no_peer_cert = false +; tls_versions = ['tlsv1.3'] + +; checkpoint setup: 10 minutes interval +use_checkpoints = true +checkpoint_interval = 120000 +; performance setup (still to be evaluated in the production nodes) +worker_processes = 4 +http_connections = 10 +worker_batch_size = 2000 +socket_options = [{keepalive, true}, {nodelay, true}] +; don't give up if replication fails; set timeout to 200secs +max_replication_retry_count = infinity +; wait for 300 seconds before timing out (actual timeout is 1/3 of it, since it's used in other parts of the code) +connection_timeout = 900000 + +[compactions] +_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "20:00"}, {to, "05:00"}] + +[compaction_daemon] +; check for databases and views every hour +check_interval = 3600 +; ~200 MB +min_file_size = 209715200 + +[query_server_config] +os_process_limit = 50 + +[admins] diff --git a/docker/pypi/wmagent-couchdb/manage b/docker/pypi/wmagent-couchdb/manage new file mode 100755 index 000000000..00ab4382d --- /dev/null +++ b/docker/pypi/wmagent-couchdb/manage @@ -0,0 +1,812 @@ +#!/bin/bash + +##H Usage: manage ACTION [ARG] [SECURITY-STRING] +##H +##H Available actions: +##H help show this help +##H version get current version of the service +##H status show current service's status +##H sysboot start server from crond if not running +##H restart (re)start the service +##H start (re)start the service +##H stop stop the service +##H pushapps push couch applications +##H pushreps push couch replications +##H updatecouchapps pull new couch applications from WMCore repo +##H compact compact database ARG +##H compactviews compact database views for design doc ARG ARG +##H cleanviews clean view named ARG +##H backup rsync databases to ARG (i.e. [user@]host:path) +##H archive archive backups to ARG area in castor +##H print-settings print a basic set of environment variables +##H +##H For more details please refer to operations page: +##H https://twiki.cern.ch/twiki/bin/view/CMS/CouchDB + +[ $(id -un) != cmsweb -o "$1" = "backup" -o "$1" = "archive" ] || + { echo "ERROR: please use another account" 1>&2; exit 1; } + +case $(uname) in Darwin ) + md5sum() { md5 -r ${1+"$@"}; } + ;; +esac + +ME=$(basename $(dirname $0)) +TOP=$(cd $(dirname $0)/../../.. && pwd) +ROOT=$(cd $(dirname $0)/../.. && pwd) +CFGDIR=$COUCH_CONFIG_DIR +LOGDIR=$COUCH_LOG_DIR +STATEDIR=$COUCH_STATE_DIR +KEYFILE=$COUCH_AUTH_DIR/hmackey.ini + +# if [ -f /etc/secrets/couch_creds ]; then +# # k8s setup +# COUCH_CREDS=/etc/secrets/couch_creds +# else +# COUCH_CREDS=$ROOT/auth/$ME/couch_creds +# fi +# EXCEPTIONS="wmstats" + +print_settings(){ + echo ME : $ME + echo TOP : $TOP + echo ROOT : $ROOT + echo CFGDIR : $CFGDIR + echo LOGDIR : $LOGDIR + echo STATEDIR : $STATEDIR + echo KEYFILE : $KEYFILE + echo + echo COUCH_ROOT_DIR : $COUCH_ROOT_DIR + echo COUCH_BASE_DIR : $COUCH_BASE_DIR + echo COUCH_STATE_DIR : $COUCH_STATE_DIR + echo COUCH_INSTALL_DIR : $COUCH_INSTALL_DIR + echo COUCH_CONFIG_DIR : $COUCH_CONFIG_DIR + echo +} + + +COLOR_OK="\\033[0;32m" +COLOR_WARN="\\033[0;31m" +COLOR_NORMAL="\\033[0;39m" +PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/kerberos/bin:/usr/local/bin:/opt/couchdb/bin + + +_load_secrets(){ + # Auxiliary function to parse WMAgent.secrets or CouchDB.secrets or local.ini files + # and load a set of variables from them + # :param $1: Path to WMAgent.secrets or file + # :param $2: String with variable names to be checked + # :retrun: Error value if one or more values have been left unset in the secrets file + local errVal=0 + local value="" + local secretsFile=$1 + local varsToLoad=$2 + + [[ -f $secretsFile ]] || { + echo "$FUNCNAME: ERROR: Password file: $secretsFile does not exist" + echo "$FUNCNAME: ERROR: Either set WMA_SECRETS_FILE environment variable to a valid file or check that $HOME/WMAgent.secrets exists" + return $(false) + } + + # All variables need to be fetched in lowercase through: ${varName,,} + local badValuesReg="(update-me|updateme|||fix-me|fixme|||^$)" + for varName in $varsToLoad + do + value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | awk -F\= '{print $2}'` + [[ ${value,,} =~ $badValuesReg ]] && { echo "$FUNCNAME: Bad value for: $varName=$value"; let errVal+=1 ;} + done + + for varName in $varsToLoad + do + value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | sed "s/ *$varName=//"` + eval $varName=$value + [[ -n $varName ]] || { echo "$FUNCNAME: Empty value for: $varName=$value"; let errVal+=1 ;} + done + return $errVal +} + + +_load_secrets $WMA_SECRETS_FILE "COUCH_USER COUCH_PASS" +COUCH_CREDS_URL="http://$COUCH_USER:$COUCH_PASS@localhost:$COUCH_PORT" + + +# Specific files required for WorkQueue, from the YUI JavaScript/CSS library +files_needed_from_yui=('build/animation/animation-min.js' + 'build/assets/skins/sam/ajax-loader.gif' + 'build/assets/skins/sam/asc.gif' + 'build/assets/skins/sam/autocomplete.css' + 'build/assets/skins/sam/back-h.png' + 'build/assets/skins/sam/back-v.png' + 'build/assets/skins/sam/bar-h.png' + 'build/assets/skins/sam/bar-v.png' + 'build/assets/skins/sam/bg-h.gif' + 'build/assets/skins/sam/bg-v.gif' + 'build/assets/skins/sam/blankimage.png' + 'build/assets/skins/sam/button.css' + 'build/assets/skins/sam/calendar.css' + 'build/assets/skins/sam/carousel.css' + 'build/assets/skins/sam/check0.gif' + 'build/assets/skins/sam/check1.gif' + 'build/assets/skins/sam/check2.gif' + 'build/assets/skins/sam/colorpicker.css' + 'build/assets/skins/sam/container.css' + 'build/assets/skins/sam/datatable.css' + 'build/assets/skins/sam/desc.gif' + 'build/assets/skins/sam/dt-arrow-dn.png' + 'build/assets/skins/sam/dt-arrow-up.png' + 'build/assets/skins/sam/editor-knob.gif' + 'build/assets/skins/sam/editor-sprite-active.gif' + 'build/assets/skins/sam/editor-sprite.gif' + 'build/assets/skins/sam/editor.css' + 'build/assets/skins/sam/header_background.png' + 'build/assets/skins/sam/hue_bg.png' + 'build/assets/skins/sam/imagecropper.css' + 'build/assets/skins/sam/layout.css' + 'build/assets/skins/sam/layout_sprite.png' + 'build/assets/skins/sam/loading.gif' + 'build/assets/skins/sam/logger.css' + 'build/assets/skins/sam/menu-button-arrow-disabled.png' + 'build/assets/skins/sam/menu-button-arrow.png' + 'build/assets/skins/sam/menu.css' + 'build/assets/skins/sam/menubaritem_submenuindicator.png' + 'build/assets/skins/sam/menubaritem_submenuindicator_disabled.png' + 'build/assets/skins/sam/menuitem_checkbox.png' + 'build/assets/skins/sam/menuitem_checkbox_disabled.png' + 'build/assets/skins/sam/menuitem_submenuindicator.png' + 'build/assets/skins/sam/menuitem_submenuindicator_disabled.png' + 'build/assets/skins/sam/paginator.css' + 'build/assets/skins/sam/picker_mask.png' + 'build/assets/skins/sam/profilerviewer.css' + 'build/assets/skins/sam/progressbar.css' + 'build/assets/skins/sam/resize.css' + 'build/assets/skins/sam/simpleeditor.css' + 'build/assets/skins/sam/skin.css' + 'build/assets/skins/sam/slider.css' + 'build/assets/skins/sam/split-button-arrow-active.png' + 'build/assets/skins/sam/split-button-arrow-disabled.png' + 'build/assets/skins/sam/split-button-arrow-focus.png' + 'build/assets/skins/sam/split-button-arrow-hover.png' + 'build/assets/skins/sam/split-button-arrow.png' + 'build/assets/skins/sam/sprite.png' + 'build/assets/skins/sam/sprite.psd' + 'build/assets/skins/sam/tabview.css' + 'build/assets/skins/sam/treeview-loading.gif' + 'build/assets/skins/sam/treeview-sprite.gif' + 'build/assets/skins/sam/treeview.css' + 'build/assets/skins/sam/wait.gif' + 'build/assets/skins/sam/yuitest.css' + 'build/connection/connection-min.js' + 'build/connection/connection_core-min.js' + 'build/container/assets/alrt16_1.gif' + 'build/container/assets/blck16_1.gif' + 'build/container/assets/close12_1.gif' + 'build/container/assets/container-core.css' + 'build/container/assets/container.css' + 'build/container/assets/hlp16_1.gif' + 'build/container/assets/info16_1.gif' + 'build/container/assets/skins/sam/container-skin.css' + 'build/container/assets/skins/sam/container.css' + 'build/container/assets/tip16_1.gif' + 'build/container/assets/warn16_1.gif' + 'build/container/container-min.js' + 'build/container/container_core-min.js' + 'build/datasource/datasource-min.js' + 'build/datatable/assets/datatable-core.css' + 'build/datatable/assets/datatable.css' + 'build/datatable/assets/skins/sam/datatable-skin.css' + 'build/datatable/assets/skins/sam/datatable.css' + 'build/datatable/assets/skins/sam/dt-arrow-dn.png' + 'build/datatable/assets/skins/sam/dt-arrow-up.png' + 'build/datatable/datatable-min.js' + 'build/dragdrop/dragdrop-min.js' + 'build/element/element-min.js' + 'build/fonts/fonts-min.css' + 'build/fonts/fonts.css' + 'build/json/json-min.js' + 'build/layout/assets/layout-core.css' + 'build/layout/assets/skins/sam/layout-skin.css' + 'build/layout/assets/skins/sam/layout.css' + 'build/layout/assets/skins/sam/layout_sprite.png' + 'build/layout/layout-min.js' + 'build/menu/assets/menu-core.css' + 'build/menu/assets/menu.css' + 'build/menu/assets/menu_down_arrow.png' + 'build/menu/assets/menu_down_arrow_disabled.png' + 'build/menu/assets/menu_up_arrow.png' + 'build/menu/assets/menu_up_arrow_disabled.png' + 'build/menu/assets/menubaritem_submenuindicator.png' + 'build/menu/assets/menubaritem_submenuindicator_disabled.png' + 'build/menu/assets/menubaritem_submenuindicator_selected.png' + 'build/menu/assets/menuitem_checkbox.png' + 'build/menu/assets/menuitem_checkbox_disabled.png' + 'build/menu/assets/menuitem_checkbox_selected.png' + 'build/menu/assets/menuitem_submenuindicator.png' + 'build/menu/assets/menuitem_submenuindicator_disabled.png' + 'build/menu/assets/menuitem_submenuindicator_selected.png' + 'build/menu/assets/skins/sam/menu-skin.css' + 'build/menu/assets/skins/sam/menu.css' + 'build/menu/assets/skins/sam/menubaritem_submenuindicator.png' + 'build/menu/assets/skins/sam/menubaritem_submenuindicator_disabled.png' + 'build/menu/assets/skins/sam/menuitem_checkbox.png' + 'build/menu/assets/skins/sam/menuitem_checkbox_disabled.png' + 'build/menu/assets/skins/sam/menuitem_submenuindicator.png' + 'build/menu/assets/skins/sam/menuitem_submenuindicator_disabled.png' + 'build/menu/menu-min.js' + 'build/paginator/assets/paginator-core.css' + 'build/paginator/assets/skins/sam/paginator-skin.css' + 'build/paginator/assets/skins/sam/paginator.css' + 'build/paginator/paginator-min.js' + 'build/progressbar/assets/progressbar-core.css' + 'build/progressbar/assets/skins/sam/back-h.png' + 'build/progressbar/assets/skins/sam/back-v.png' + 'build/progressbar/assets/skins/sam/bar-h.png' + 'build/progressbar/assets/skins/sam/bar-v.png' + 'build/progressbar/assets/skins/sam/progressbar-skin.css' + 'build/progressbar/assets/skins/sam/progressbar.css' + 'build/progressbar/progressbar-min.js' + 'build/reset-fonts-grids/reset-fonts-grids.css' + 'build/resize/assets/resize-core.css' + 'build/resize/assets/skins/sam/layout_sprite.png' + 'build/resize/assets/skins/sam/resize-skin.css' + 'build/resize/assets/skins/sam/resize.css' + 'build/resize/resize-min.js' + 'build/utilities/utilities.js' + 'build/yahoo-dom-event/yahoo-dom-event.js') + + +# Start service conditionally on crond restart. +sysboot() +{ + num_procs=$(get_num_couch_process) + if [ -z "$num_procs" ]; then + start + fi +} + +# Start the service. +start() +{ + _parse_localini $COUCH_CONFIG_DIR/local.ini || { echo "$FUNCNAME: ERROR: Not configured database. Try manage init first"; return 1 ;} + cd $STATEDIR + echo -n "Which couchdb: " + which couchdb + echo " With configuration directory: $CFGDIR" + echo " With logdir: $LOGDIR" + echo " nohup couchdb -couch_ini $CFGDIR >> ${LOGDIR}/couch.log 2>&1 &" + nohup couchdb -couch_ini $CFGDIR >> ${LOGDIR}/couch.log 2>&1 & + + # push_apps + # replications push +} + +# Stop the service. +stop() +{ + echo "Stopping CouchDB service..." + for couch_pid in $(ps aux | grep '_couchdb' | egrep -v 'grep|couchdb\/manage|ps aux' | awk '{print $2}'); do + echo " killing CouchDB process... ${couch_pid}" + kill -9 $couch_pid || true + done +} + +# check the number of beam/couchdb processes +get_num_couch_process() +{ + local num_proc=`ps aux | grep beam | grep -v grep` + echo $num_proc +} + +# Check if the server is running. +status() +{ + curl -s localhost:$COUCH_PORT/_up | grep -q '"status":"ok"' || + { echo -e "$ME is ${COLOR_WARN}NOT RUNNING${COLOR_NORMAL}."; return; } + + echo -e "$ME is ${COLOR_OK}RUNNING${COLOR_NORMAL}" + local TASKS=$(curl -s $COUCH_CREDS_URL/_active_tasks) + [ "$TASKS" = '[]' ] && TASKS="No active tasks (e.g. compactions)" + echo $TASKS + + replications status +} + +# When a view is changed, such as a new app version is deployed, +# invoke this to clean up the views in that database. +clean_views() +{ + local database=$1 + [ -n "$database" ] || + { echo "You must specify the database you wish to clean the views "; exit 1; } + + curl -s -H "Content-Type: application/json" -X POST $COUCH_CREDS_URL/$database/_view_cleanup | \ + grep -q '{"ok":true}' || + { echo "An error occured while cleaning the views. Please look in the CouchDB logs."; exit 3; } +} + +# Push applications from staging area into couchdb. +push_apps() +{ + local couchapps_path=$COUCH_INSTALL_DIR/stagingarea/couchapps + + if [[ ! -d $couchapps_path ]]; then + echo "Couchapps not found. Installing from latest WMCore tag." + update_couchapps "latest" + fi + + n=0 started=false + while [ $n -le 100 ]; do + curl -s localhost:$COUCH_PORT/_up | grep -q '"status":"ok"' && + started=true && break + echo "waiting for couchdb..." + sleep 1 + n=$(expr $n + 1) + done + + if $started; then + # acdc server + for ddoc in "ACDC" "GroupUser"; do + couchapp push -p $couchapps_path/$ddoc -c $COUCH_CREDS_URL/acdcserver + done + + # reqmgr2 + couchapp push -p $couchapps_path/ReqMgrAux -c $COUCH_CREDS_URL/reqmgr_auxiliary + couchapp push -p $couchapps_path/ReqMgr -c $COUCH_CREDS_URL/reqmgr_workload_cache + couchapp push -p $couchapps_path/ConfigCache -c $COUCH_CREDS_URL/reqmgr_config_cache + + # reqmon + couchapp push -p $couchapps_path/WorkloadSummary -c $COUCH_CREDS_URL/workloadsummary + couchapp push -p $couchapps_path/LogDB -c $COUCH_CREDS_URL/wmstats_logdb + for ddoc in "WMStats" "WMStatsErl" "WMStatsErl1" "WMStatsErl2" "WMStatsErl3" \ + "WMStatsErl4" "WMStatsErl5" "WMStatsErl6" "WMStatsErl7"; do + couchapp push -p $couchapps_path/$ddoc -c $COUCH_CREDS_URL/wmstats + done + + # t0_reqmon + couchapp push -p $couchapps_path/T0Request -c $COUCH_CREDS_URL/t0_request + couchapp push -p $couchapps_path/WorkloadSummary -c $COUCH_CREDS_URL/t0_workloadsummary + couchapp push -p $couchapps_path/LogDB -c $COUCH_CREDS_URL/t0_logdb + for ddoc in "WMStats" "WMStatsErl" "WMStatsErl1" "WMStatsErl2" "WMStatsErl3" \ + "WMStatsErl4" "WMStatsErl5" "WMStatsErl6" "WMStatsErl7"; do + couchapp push -p $couchapps_path/$ddoc -c $COUCH_CREDS_URL/tier0_wmstats + done + + # workqueue + couchapp push -p $couchapps_path/WorkQueue -c $COUCH_CREDS_URL/workqueue + couchapp push -p $couchapps_path/WorkQueue -c $COUCH_CREDS_URL/workqueue_inbox + + # clean views + local databases="acdcserver reqmgr_auxiliary reqmgr_workload_cache reqmgr_config_cache workloadsummary + wmstats_logdb wmstats t0_request t0_workloadsummary t0_logdb tier0_wmstats" + + for DB in $databases; do + clean_views $DB + done + else + echo "couchdb did not start, not pushing application" + exit 1 + fi +} + +update_couchapps() +{ + local WMCORE_TAG=$1 + [ -n $WMCORE_TAG ] || + { echo "WMCore tag not provided. Please provide a WMCore tag."; exit 1; } + + local couchapps_dir=$COUCH_INSTALL_DIR/stagingarea + local tmp_dir=$couchapps_dir/tmp + + # clean up and recreate $tmp_dir + rm -rf $tmp_dir + mkdir -p $tmp_dir + + pushd $tmp_dir + + if [[ $WMCORE_TAG == "latest" ]]; then + WMCORE_TAG=$(curl https://api.github.com/repos/dmwm/WMCore/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') + fi + + echo -e "\nPulling couchapps version $1 from Github..." + wget -nv https://github.com/dmwm/WMCore/archive/refs/tags/${WMCORE_TAG}.tar.gz || + { echo "Error pulling couchapps version $1 from Github"; exit 3; } + + tar --strip-components=2 -xzf ${WMCORE_TAG}.tar.gz WMCore-$WMCORE_TAG/src/couchapps || + { echo "Error extracting couchapps tarball"; exit 3; } + + # grab external dependencies for reqmon/t0_reqmon + echo -e "\nPulling additional reqmon and t0_reqmon dependencies..." + cp -R $tmp_dir/couchapps/couchskel/vendor $tmp_dir/couchapps/WMStats + mkdir -p $tmp_dir/couchapps/WMStats/vendor/{jquery,datatables}/_attachments + + # jquery-ui.min.js + echo -e "\nDownloading jquery-ui.min.js..." + wget -nv https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.18/jquery-ui.min.js || + { echo "Error downloading jquery-ui.min.js"; exit 3; } + cp jquery-ui.min.js $tmp_dir/couchapps/WMStats/vendor/jquery/_attachments/jquery-ui.min.js + + # jquery.min.js + echo -e "\nDownloading jquery.min.js..." + wget -nv http://code.jquery.com/jquery-1.7.2.min.js || + { echo "Error downloading jquery-1.7.2.min.js"; exit 3; } + cp jquery-1.7.2.min.js $tmp_dir/couchapps/WMStats/vendor/jquery/_attachments/jquery.min.js + + # Datatables + echo -e "\nDownloading Datatables..." + wget -nv http://datatables.net/releases/DataTables-1.9.1.zip || + { echo "Error downloading Datatables-1.9.1.zip"; exit 3; } + unzip DataTables-1.9.1.zip &> /dev/null + cp DataTables*/{media/js/jquery.dataTables.min,extras/ColVis/media/js/ColVis.min}.js $tmp_dir/couchapps/WMStats/vendor/datatables/_attachments + + # YUI library, required by WorkQueue + # List of required files at: https://github.com/dmwm/WMCore/blob/master/bin/wmagent-couchapp-init#L135 + # TODO: can we copy the whole library instead of specific files?!? + echo -e "\nDownloading YUI..." + wget -nv https://yui.github.io/yui2/archives/yui_2.9.0.zip || + { echo "Error downloading yui_2.9.0.zip"; exit 3; } + unzip yui_2.9.0.zip &> /dev/null + mkdir -p $tmp_dir/couchapps/WorkQueue/vendor/yui/_attachments + pushd yui + for yui_file in "${files_needed_from_yui[@]}" + do + # copy the file itself and the parent directories + cp --parents ${yui_file} $tmp_dir/couchapps/WorkQueue/vendor/yui/_attachments/ + done + popd + + echo "Removing old couchapps..." + rm -rf $couchapps_dir/couchapps &> /dev/null + + echo "Installing new couchapps..." + mv couchapps $couchapps_dir + + popd + echo "Cleaning up!" + rm -rf $tmp_dir +} + +replications() +{ + [ "$1" = "push" ] && local status=false || local status=true + local all_reps=$(curl -s $COUCH_CREDS_URL/_replicator/_all_docs?include_docs=true \ + | grep '^{"id":"[^_]' | awk -F\" '{print $4,$14,$28,$32,$38,$42}') + local req_reps=$(cat $STATEDIR/replication/* 2>/dev/null || echo "") + + if [ -n "$all_reps" ]; then + echo "$all_reps" | while read ID REV SRC DST FILTER STATE; do + if echo "$req_reps" | grep -q "$SRC $DST $FILTER"; then + echo "Replication 'id=$ID source=$SRC target=$DST filter=$FILTER' $STATE." + else + echo "Replication 'id=$ID source=$SRC target=$DST filter=$FILTER' unknown." + if ! $status; then + echo -n "Removing it... " + curl -s -X DELETE $COUCH_CREDS_URL/_replicator/$ID?rev=$REV + fi + fi + done + fi + + if [ -n "$req_reps" ]; then + local IDS=$(echo "$all_reps" | awk '{print $1}') + echo "$req_reps" | while read SRC DST FILTER; do + local ID=$(echo "$SRC $DST $FILTER" | md5sum | awk '{print $1}') + if ! echo "$IDS" | grep -q "$ID"; then + echo "Replication 'id=$ID source=$SRC target=$DST filter=$FILTER' not pushed." + if ! $status; then + echo -n "Pushing it... " + curl -s -X PUT $COUCH_CREDS_URL/_replicator/$ID \ + -d "{\"source\":\"$SRC\", \"target\":\"$DST\", \"continuous\":true, \"filter\":\"$FILTER\"}" + fi + fi + done + fi +} + +# Trigger database compaction. +# - If "all", then compact ALL databases. +# - If "all_but_exceptions", then compact all views but the ones listed in EXCEPTIONS +# - Otherwise, the database must be provided +compact() +{ + local database=$1 + [ -n "$database" ] || + { echo "You must specify a database to compact"; exit 3; } + + if [ "$database" = all -o "$database" = all_but_exceptions ]; then + for db in $STATEDIR/database/[^_]*.couch; do + [ -f $db ] || continue + db=${db##*/} + db=${db%.couch} + if [[ $EXCEPTIONS == *$db* && "$database" = all_but_exceptions ]]; then + continue + fi + compact_database $db + done + else + compact_database $database + fi +} + +compact_database() +{ + local database=$1 + curl -s $COUCH_CREDS_URL/$database | grep -q '"compact_running":true' && + { echo "$database is already compacting"; exit 5; } + + curl -s -H "Content-Type: application/json" -X POST $COUCH_CREDS_URL/$database/_compact | \ + grep -q '{"ok":true}' || + { echo "An error occured triggering compaction. Please look in the CouchDB logs."; exit 7; } +} + +# Trigger view compaction. +# - If "all", then compact ALL views +# - If "all_but_exceptions", then compact all views but the ones listed in EXCEPTIONS +# - Otherwise, the database and designdoc must be provided +compact_views() +{ + local database=$1 + local designdoc=$2 + [ -n "$database" ] || + { echo "You must specify a database to compact its views"; exit 3; } + [ -n "$designdoc" ] || + { echo "You must specify a design doc to compact its views"; exit 3; } + + if [ "$database" = all -o "$database" = all_but_exceptions ]; then + for db in $STATEDIR/database/[^_]*.couch; do + [ -f $db ] || continue + db=${db##*/} + db=${db%.couch} + if [[ $EXCEPTIONS == *$db* && "$database" = all_but_exceptions ]]; then + continue + fi + curl -s "$COUCH_CREDS_URL/$db/_all_docs?startkey=%22_design%22&endkey=%22_design/zzzzzzzz%22" | + awk -F\" '/"id":"_design/ {print $4}' | + while read dbview; do + compact_view $db ${dbview#*/} + done + done + elif [ "$designdoc" = all ]; then + curl -s "$COUCH_CREDS_URL/$database/_all_docs?startkey=%22_design%22&endkey=%22_design/zzzzzzzz%22" | + awk -F\" '/"id":"_design/ {print $4}' | + while read dbview; do + compact_view $database ${dbview#*/} + done + else + compact_view $database $designdoc + fi +} + +# Trigger single view compaction +compact_view() +{ + local database=$1 + local designdoc=$2 + + curl -s $COUCH_CREDS_URL/$database/_design/$designdoc/_info | grep -q '"compact_running":true' && + { echo "$database/$designdoc is already compacting"; exit 5; } + + curl -s -H "Content-Type: application/json" -X POST $COUCH_CREDS_URL/$database/_compact/$designdoc | \ + grep -q '{"ok":true}' || + { echo "An error occured triggering view compaction. Please look in the CouchDB logs."; exit 7; } +} + +# Rsync couchdb databases to elsewhere (can be used to restore a backup too) +backup() +{ + local hostdir=$1 + [ -n "$hostdir" ] || + { echo "You must specify a destination [user@]host:path."; exit 9; } + #ionice -c3 rsync --delete --delete-excluded --exclude "*.compact*" -au -e 'ssh -c arcfour' $STATEDIR/database/ $hostdir/ || + ionice -c3 rsync --delete --delete-excluded --exclude "*.compact*" -au -e 'ssh -c aes128-ctr' $STATEDIR/database/ $hostdir/ || + + { echo "failed to synchronise databases to $hostdir: $?" 1>&2; exit 11; } +} + +# Archive couchdb backups to castor +archive() +{ + local archdir=$1 + klist -s || + { echo "You must have a valid kerberos token to run the archive." 1>&2; exit 20; } + rfstat $archdir >/dev/null || + { echo "Could not stat $archdir in the castor archive" 1>&2; exit 19; } + + echo "Checking for couchdb archives not yet staged out to tape:" + stager_qry -M $archdir | grep couchdb | grep -v STAGED || echo "all staged out." + + echo + echo "Starting to archive couchdb backups on $(date)" + for bkp in $STATEDIR/backup/*; do + [ -d $bkp ] || continue; + tarfile=couchdb_${bkp##*/}_$(date +%Y%m%d-%H%M).tar + echo "$archdir/$tarfile" + tar cf - -C $bkp . | rfcp - $archdir/$tarfile & + done + wait; + [ -n "$tarfile" ] && echo "Archive copy to castor completed on $(date)." || + echo "No backups found on $STATEDIR/backup. Nothing to archive." + + #FIXME: do some sanity checks on castor to make sure new archives are genuine + + # Cleanup old archives + for bkp in $STATEDIR/backup/*; do + [ -d $bkp ] || continue; + tarprefix=couchdb_${bkp##*/} + since=$(date --date='3 months ago' +%Y%m%d) + echo + echo "Cleaning old archives for $tarprefix:" + for f in $(nsls $archdir|grep $tarprefix); do + if [ $(echo -n $f | cut -d_ -f3| cut -d- -f1) -lt $since ]; then + echo $archdir/$f + rfrm $archdir/$f + fi + done + done + + echo + echo "Archive procedure done." +} + +# Verify the security string. +check() +{ + CHECK=$(echo "$1" | md5sum | awk '{print $1}') + if [ $CHECK != 94e261a5a70785552d34a65068819993 ]; then + echo "$0: cannot complete operation, please check documentation." 1>&2 + exit 2; + fi +} + +_parse_localini (){ + # Auxiliary function to provide basic parsing of the local.ini file at the container + # :param $1: path to local.ini file + local errVal=0 + local value="" + local secretsFile=$1 + # All variables need to be fetched in lowercase through: ${var,,} + local badValuesReg="(update-me|updateme|||fix-me|fixme|||^$)" + local varsToCheck=`grep -vE "^[[:blank:]]*(\;|\#|\[).*$" $secretsFile |awk -F\= '{print $1}'` + # `awk -F\= '{print $1}' $secretsFile | grep -vE "^[[:blank:]]*(\#|\;|\[).*$;"` + for var in $varsToCheck + do + value=`grep -E "^[[:blank:]]*$var" $secretsFile | awk -F\= '{print $2}'` + [[ ${value,,} =~ $badValuesReg ]] && { echo "$FUNCNAME: WARNING: Bad value for: $var=$value"; let errVal+=1 ;} + done + return $errVal +} + + +init_couchdb() { + # Auxiliary function to check for initial setting of couchdb. + # i.e. check if COUCHDB_SECRETS_FILE and WMAGENT_SECRETS_FILE from the host + # have been parsed correctly and the passwords propagated to local.ini + + [[ -f $COUCH_CONFIG_DIR/local.ini ]] || cp -v $COUCH_DEPLOY_DIR/local.ini $COUCH_CONFIG_DIR/local.ini + + # First check if all variables in the local.ini file are properly set + local parseOk=true + _parse_localini $COUCH_CONFIG_DIR/local.ini || parseOk=false + + # Separate only the [admins] section of the local.ini file: + local adminSection=`grep -vE "^(\;|\#)" ${COUCH_CONFIG_DIR}/local.ini | sed -n '/^ *\[admins\].*/,/^ *\[.*\].*/p' |sed -e 's/$/\\\\n/'` + + # Load all accounts already configurred in the [admins] section of the local.ini file: + local accountList="" + for var in `echo -e $adminSection |sed 's/ *= */=/g'|grep \= |grep -vE "^\;"` + do + [[ $var =~ .*=.* ]] && { + local $var + accountList="$accountList ${var%%=*}" + } + done + + # Check if we local.ini format is ok and if we have any admin accounts properly configurred + if $parseOk && [[ -n $accountList ]]; then + echo "$FUNCNAME: The local.ini file at the container is already properly configured." + echo "$FUNCNAME: NOT adding any new admin accounts from the secrets files!" + return + else + _load_secrets $WMA_SECRETS_FILE "COUCH_USER COUCH_PASS" || { + err=$? + echo "$FUNCNAME: ERROR: Could not properly load WMAgent User password for CouchDB" + exit $err + } + + # Finally, Compare and add the missing ones at local.ini inside the container. + # NOTE: those will be increpted and replaced by CouchDB immediately after the first run + if [[ $accountList =~ $COUCH_USER ]] ;then + echo "$FUNCNAME: Substitute already existing account: $COUCH_USER" + sed -i "s/$COUCH_USER *=.*/$COUCH_USER = $COUCH_PASS/g" $COUCH_CONFIG_DIR/local.ini + else + echo "$FUNCNAME: Adding a new account: $COUCH_USER" + sed -i "/\[admins\]/a $COUCH_USER = $COUCH_PASS" $COUCH_CONFIG_DIR/local.ini + fi + fi +} + +# Main routine, perform action requested on command line. +case ${1:-status} in + sysboot ) + sysboot + ;; + + init ) + init_couchdb + ;; + + restart ) + # check "$2" + stop + sleep 1 + start + ;; + + status ) + status + ;; + + stop ) + # check "$2" + stop + ;; + + pushapps ) + # check "$2" + push_apps + ;; + + pushreps ) + # check "$2" + replications push + ;; + + updatecouchapps ) + # check "$3" + update_couchapps $2 + ;; + + compact ) + # check "$3" + compact $2 + ;; + + compactviews ) + # check "$4" + compact_views $2 $3 + ;; + + cleanviews ) + # check "$3" + clean_views $2 + ;; + + backup ) + # check "$3" + backup $2 + ;; + + archive ) + # check "$3" + archive $2 + ;; + + help ) + perl -ne '/^##H/ && do { s/^##H ?//; print }' < $0 + ;; + + version ) + echo "$COUCHDB_VERSION" + ;; + + start ) + start + ;; + + print-settings ) + print_settings + ;; + + * ) + echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2 + exit 1 + ;; +esac diff --git a/docker/pypi/wmagent-couchdb/run.sh b/docker/pypi/wmagent-couchdb/run.sh new file mode 100755 index 000000000..8c3f85e76 --- /dev/null +++ b/docker/pypi/wmagent-couchdb/run.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Basic initialization for CouchDB +thisUser=$(id -un) +thisGroup=$(id -gn) +thisUserID=$(id -u) +thisGroupID=$(id -g) +echo "Running CouchDB container with user: $thisUser (ID: $thisUserID) and group: $thisGroup (ID: $thisGroupID)" + +export USER=$thisUser +[[ -d ${HOME} ]] || mkdir -p ${HOME} + +<> ~/.bashrc +export USER=$thisUser + +alias lll="ls -lathr" +alias ls="ls --color=auto" +alias ll='ls -la --color=auto' +alias scurl='curl -k --cert ${COUCH_CERTS_DIR}/servicecert.pem --key ${COUCH_CERTS_DIR}/servicekey.pem' + +alias manage=$COUCH_MANAGE_DIR/manage + +# Set command prompt for the running user inside the container +export PS1="(CouchDB-$TAG) [\u@\h:\W]\$ " +EOF +source ${HOME}/.bashrc + +manage init | tee -a $COUCH_LOG_DIR/run.log +manage start | tee -a $COUCH_LOG_DIR/run.log +manage pushapps | tee -a $COUCH_LOG_DIR/run.log + +echo "start sleeping....zzz" +sleep infinity + +# # start the service +# manage start diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile index da0e79052..c11928a9f 100644 --- a/docker/pypi/wmagent-mariadb/Dockerfile +++ b/docker/pypi/wmagent-mariadb/Dockerfile @@ -32,10 +32,6 @@ ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets ENV WMA_DATABASE=wmagent -# create the system user to run the database -RUN groupadd -g 1399 zh -RUN useradd -u 31961 -g 1399 -G 999 -m cmst1 - # start the setup RUN mkdir -p $MDB_ROOT_DIR $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR \ $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR @@ -52,20 +48,14 @@ ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}" -# set MariaDB docker specific bash prompt and manage alias for all users: -RUN <>/root/.bashrc -alias manage=$MDB_MANAGE_DIR/manage -export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$([[ \$(id -u) -eq 0 ]] && echo \# || echo \$) " -EOF - -RUN <>/home/cmst1/.bashrc -alias manage=$MDB_MANAGE_DIR/manage -export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$([[ \$(id -u) -eq 0 ]] && echo \# || echo \$) " +# Set command prompt for root +RUN <> /root/.bashrc +export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]# " EOF -# RUN chown -R ${USER} ${MDB_ROOT_DIR} +# allow dynamic users to create homefolders and .bashrc +RUN chmod 777 /home # setup final environment -# USER $USER WORKDIR $MDB_ROOT_DIR ENTRYPOINT ["./run.sh", "2>&1"] diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh index bdf63fcdc..ce2c1961c 100755 --- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh +++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh @@ -49,27 +49,43 @@ while getopts ":t:hp" opt; do esac done - -mariadbUser=`id -un` -mariadbOpts=" --user $mariadbUser -e USER=$mariadbUser" - # This is the root at the host only, it may differ from the root inside the container. # NOTE: this may be parametriesed, so that the container can run on a different mount point. HOST_MOUNT_DIR=/data/dockerMount +thisUser=$(id -un) +thisGroup=$(id -gn) + +# create the passwd and group mount point dynamically at runtime +passwdEntry=$(getent passwd $thisUser | awk -F : -v thisHome="/home/$thisUser" '{print $1 ":" $2 ":" $3 ":" $4 ":" $5 ":" thisHome ":" $7}') +groupEntry=$(getent group $thisGroup) + +# workaround case where Unix account is not in the local system (e.g. sssd) +[[ -d $HOST_MOUNT_DIR/admin/etc/ ]] || (mkdir -p $HOST_MOUNT_DIR/admin/etc) || exit $? +[[ -f $HOST_MOUNT_DIR/admin/etc/passwd ]] || { + echo "Creating passwd file" + getent passwd > $HOST_MOUNT_DIR/admin/etc/passwd + echo $passwdEntry >> $HOST_MOUNT_DIR/admin/etc/passwd +} +[[ -f $HOST_MOUNT_DIR/admin/etc/group ]] || { + echo "Creating group file" + getent group > $HOST_MOUNT_DIR/admin/etc/group + echo $groupEntry >> $HOST_MOUNT_DIR/admin/etc/group +} + [[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $? [[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $? # [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config) || exit $? [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ;} || exit $? [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ;} || exit $? -# sudo chown -R $mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG dockerOpts=" --detach \ --network=host \ --rm \ ---hostname=`hostname -f` \ +--hostname=$(hostname -f) \ +--user $(id -u):$(id -g) \ --name=mariadb \ --mount type=bind,source=/tmp,target=/tmp \ --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \ @@ -77,6 +93,10 @@ dockerOpts=" --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs,target=/data/srv/mariadb/current/logs \ --mount type=bind,source=$HOST_MOUNT_DIR/admin/mariadb,target=/data/admin/mariadb/ \ --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/passwd,target=/etc/passwd,readonly \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/group,target=/etc/group,readonly \ +--mount type=bind,source=/etc/sudoers,target=/etc/sudoers,readonly \ +--mount type=bind,source=/etc/sudoers.d,target=/etc/sudoers.d,readonly \ " # --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config,target=/data/srv/mariadb/current/config \ @@ -94,7 +114,7 @@ $PULL && { docker tag $registry/$project/$repository:$MDB_TAG $registry/$repository:latest } -echo "Starting the $registry/$repository:$MDB_TAG docker container with the following parameters: $mariadbOpts" -docker run $dockerOpts $mariadbOpts $registry/$repository:$MDB_TAG && ( +echo "Starting $repository:$MDB_TAG docker container with user: $thisUser:$thisGroup" +docker run $dockerOpts $registry/$repository:$MDB_TAG && ( [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current ) diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh index d9ce91b8f..3dc16e08e 100755 --- a/docker/pypi/wmagent-mariadb/run.sh +++ b/docker/pypi/wmagent-mariadb/run.sh @@ -1,7 +1,31 @@ #!/bin/bash +# Basic initialization for MariaDB +thisUser=$(id -un) +thisGroup=$(id -gn) +thisUserID=$(id -u) +thisGroupID=$(id -g) +echo "Running MariaDB container with user: $thisUser (ID: $thisUserID) and group: $thisGroup (ID: $thisGroupID)" + +export USER=$thisUser +[[ -d ${HOME} ]] || mkdir -p ${HOME} + +<> ~/.bashrc +export USER=$thisUser + +alias lll="ls -lathr" +alias ls="ls --color=auto" +alias ll='ls -la --color=auto' + +alias manage=$MDB_MANAGE_DIR/manage + +# Set command prompt for the running user inside the container +export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ " +EOF +source ${HOME}/.bashrc + manage init-mariadb 2>&1 | tee -a $MDB_LOG_DIR/run.log manage start-mariadb 2>&1 | tee -a $MDB_LOG_DIR/run.log echo "Start sleeping....zzz" -while true; do sleep 10; done +sleep infinity \ No newline at end of file diff --git a/docker/pypi/wmagent/Dockerfile b/docker/pypi/wmagent/Dockerfile index 9d0310d2e..982ccac3b 100644 --- a/docker/pypi/wmagent/Dockerfile +++ b/docker/pypi/wmagent/Dockerfile @@ -5,20 +5,14 @@ MAINTAINER Valentin Kuznetsov vkuznet@gmail.com ARG TAG=None ARG WMA_TAG=$TAG ENV WMA_TAG=$WMA_TAG -ENV WMA_USER=cmst1 -ENV WMA_GROUP=zh -ENV WMA_UID=31961 -ENV WMA_GID=1399 ENV WMA_ROOT_DIR=/data - # Basic WMAgent directory structure passed to all scripts through env variables: # NOTE: Those should be static and depend only on $WMA_BASE_DIR ENV WMA_BASE_DIR=$WMA_ROOT_DIR/srv/wmagent ENV WMA_ADMIN_DIR=$WMA_ROOT_DIR/admin/wmagent ENV WMA_CERTS_DIR=$WMA_ROOT_DIR/certs - # ENV WMA_HOSTADMIN_DIR=$WMA_ADMIN_DIR/hostadmin ENV WMA_CURRENT_DIR=$WMA_BASE_DIR/$WMA_TAG ENV WMA_AUTH_DIR=$WMA_CURRENT_DIR/auth/ @@ -32,17 +26,6 @@ ENV WMA_ENV_FILE=$WMA_DEPLOY_DIR/deploy/env.sh ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets ENV ORACLE_PATH=$WMA_DEPLOY_DIR/etc/oracle - -# Setting up users and previleges -RUN groupadd -g ${WMA_GID} ${WMA_GROUP} -RUN useradd -u ${WMA_UID} -g ${WMA_GID} -m ${WMA_USER} -RUN install -o ${WMA_USER} -g ${WMA_GID} -d ${WMA_ROOT_DIR} -RUN usermod -aG mysql ${WMA_USER} -RUN rm -f /etc/mysql/mariadb.conf.d/50-server.cnf - -# Add WMA_USER to sudoers -RUN echo "${WMA_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - # Add all deployment needed directories ADD bin $WMA_DEPLOY_DIR/bin ADD etc $WMA_DEPLOY_DIR/etc @@ -56,16 +39,17 @@ ADD init.sh ${WMA_ROOT_DIR}/init.sh # Install the requested WMA_TAG. RUN ${WMA_ROOT_DIR}/install.sh -t ${WMA_TAG} -RUN chown -R ${WMA_USER}:${WMA_GID} ${WMA_ROOT_DIR} -# Switch to the runtime directory and user +# Switch to the runtime directory WORKDIR ${WMA_ROOT_DIR} -USER ${WMA_USER} -ENV USER=$WMA_USER -# Define the entrypoint (Using exec Form): -ENTRYPOINT ["./run.sh", "2>&1"] +# Set command prompt for root +RUN <> /root/.bashrc +export PS1="(WMAgent-\$WMA_TAG) [\u@\h:\W]# " +EOF -# # Define the entrypoint (Using shell Form): -# SHELL ["/bin/bash", "-c"] -# ENTRYPOINT /data/run.sh 2>&1 \ No newline at end of file +# allow dynamic users to create homefolders and .bashrc +RUN chmod 777 /home + +# Define the entrypoint (Using exec form): +ENTRYPOINT ["./run.sh", "2>&1"] diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage index 7ee39c408..e876f92c6 100755 --- a/docker/pypi/wmagent/bin/manage +++ b/docker/pypi/wmagent/bin/manage @@ -136,6 +136,7 @@ init_wmagent(){ --reqmgr2_url=$REQMGR2_URL \ --acdc_url=$ACDC_URL \ --dbs3_url=$DBS3_URL \ + --dbs3_reader_url=$DBS3_READER_URL \ --dqm_url=$DQM_URL \ --requestcouch_url=$REQUESTCOUCH_URL \ --central_logdb_url=$CENTRAL_LOGDB_URL \ diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh index 44a03c40a..028c948f1 100644 --- a/docker/pypi/wmagent/bin/manage-common.sh +++ b/docker/pypi/wmagent/bin/manage-common.sh @@ -20,9 +20,12 @@ wmaInitSqlDB=$WMA_CONFIG_DIR/.initSqlDB # set once the metad wmaInitCouchDB=$WMA_CONFIG_DIR/.initCouchDB # set immediately after agent initialization wmaInitConfig=$WMA_CONFIG_DIR/.initConfig # set upon final WMAgent config file tweaks have been applied wmaInitResourceControl=$WMA_CONFIG_DIR/.initResourceControl # set once the resource control of the agent has been populated +wmaInitResourceOpp=$WMA_CONFIG_DIR/.initResourceOpp # set once the resource control of the agent has been populated for opportunistic resources wmaInitUpload=$WMA_CONFIG_DIR/.initUpload # set once the agent config has been uploaded to central CouchDB +wmaInitRuntime=$WMA_CONFIG_DIR/.initRuntime # set once the runtime scripts needed for HTCondor are copied at the host wmaInitUsing=$WMA_CONFIG_DIR/.initUsing # Final init flag to mark that the agent is fully activated, initialized, and already in use by the system + # Setting database name and schema dump location. wmaSchemaFile=$WMA_CONFIG_DIR/.wmaSchemaFile.sql wmaDBName=wmagent @@ -314,7 +317,7 @@ _renew_proxy(){ echo "$FUNCNAME: ERROR: Failed to renew expired myproxy"; return $(false) ;} # Stay safe and always change the service {cert,key} and myproxy mode here: - sudo chmod 400 $WMA_CERTS_DIR/* + chmod 400 $WMA_CERTS_DIR/* echo "$FUNCNAME: OK" else echo "$FUNCNAME: ERROR: We found no service certificate installed at $WMA_CERTS_DIR!" @@ -326,17 +329,26 @@ _renew_proxy(){ _parse_wmasecrets(){ # Auxiliary function to provide basic parsing of the WMAgent.secrets file - # :param $1: path to WMAgent.secrets file + # :param $1: path to WMAgent.secrets file (Default: $WMA_SECRETS_FILE) + # :param $2: a particular value to check (Default: *) local errVal=0 local value="" - local secretsFile=$1 + local secretsFile=${1:-$WMA_SECRETS_FILE} + local varsToCheck=${2:-""} + # All variables need to be fetched in lowercase through: ${var,,} local badValuesReg="(update-me|updateme|||fix-me|fixme|||^$)" - local varsToCheck=`awk -F\= '{print $1}' $secretsFile | grep -vE "^[[:blank:]]*#.*$"` + # local varsToCheck=`awk -F\= '{print $1}' $secretsFile | grep -vE "^[[:blank:]]*#.*$"` + + # Building the list by parsing the secrets file itself. + [[ -n $varsToCheck ]] || { + varsToCheck=`grep -v "^[[:blank:]]#" $secretsFile |grep \= | awk -F\= '{print $1}'` + } + for var in $varsToCheck do value=`grep -E "^[[:blank:]]*$var" $secretsFile | awk -F\= '{print $2}'` - [[ ${value,,} =~ $badValuesReg ]] && { echo "$FUNCNAME: Bad value for: $var=$value"; let errVal+=1 ;} + [[ ${value,,} =~ $badValuesReg ]] && { echo "$FUNCNAME: ERROR: Bad value for: $var=$value"; let errVal+=1 ;} done return $errVal } @@ -346,150 +358,94 @@ _parse_wmasecrets(){ # Passwords/Secrets handling # _load_wmasecrets(){ - if [ "x$WMA_SECRETS_FILE" == "x" ]; then - WMA_SECRETS_FILE=$HOME/WMAgent.secrets; - fi - if [ ! -e $WMA_SECRETS_FILE ]; then - echo "$FUNCNAME: Password file: $WMA_SECRETS_FILE does not exist" - echo "$FUNCNAME: Either set WMA_SECRETS_FILE environment variable to a valid file or check that $HOME/WMAgent.secrets exists" - return 1; - fi - _parse_wmasecrets $WMA_SECRETS_FILE || { echo "$FUNCNAME: WARNING: Not loading raw or not updated secrets file at $WMA_SECRETS_FILE"; return $(false) ;} - - local MATCH_ORACLE_USER=`cat $WMA_SECRETS_FILE | grep ORACLE_USER | sed s/ORACLE_USER=//` - local MATCH_ORACLE_PASS=`cat $WMA_SECRETS_FILE | grep ORACLE_PASS | sed s/ORACLE_PASS=//` - local MATCH_ORACLE_TNS=`cat $WMA_SECRETS_FILE | grep ORACLE_TNS | sed s/ORACLE_TNS=//` - local MATCH_GRAFANA_TOKEN=`cat $WMA_SECRETS_FILE | grep GRAFANA_TOKEN | sed s/GRAFANA_TOKEN=//` - local MATCH_MDB_USER=`cat $WMA_SECRETS_FILE | grep MDB_USER | sed s/MDB_USER=//` - local MATCH_MDB_PASS=`cat $WMA_SECRETS_FILE | grep MDB_PASS | sed s/MDB_PASS=//` - local MATCH_MDB_HOST=`cat $WMA_SECRETS_FILE | grep MDB_HOST | sed s/MDB_HOST=//` - local MATCH_COUCH_USER=`cat $WMA_SECRETS_FILE | grep COUCH_USER | sed s/COUCH_USER=//` - local MATCH_COUCH_PASS=`cat $WMA_SECRETS_FILE | grep COUCH_PASS | sed s/COUCH_PASS=//` - local MATCH_COUCH_PORT=`cat $WMA_SECRETS_FILE | grep COUCH_PORT | sed s/COUCH_PORT=//` - local MATCH_COUCH_HOST=`cat $WMA_SECRETS_FILE | grep COUCH_HOST | sed s/COUCH_HOST=//` - local MATCH_COUCH_CERT_FILE=`cat $WMA_SECRETS_FILE | grep COUCH_CERT_FILE | sed s/COUCH_CERT_FILE=//` - local MATCH_COUCH_KEY_FILE=`cat $WMA_SECRETS_FILE | grep COUCH_KEY_FILE | sed s/COUCH_KEY_FILE=//` - local MATCH_GLOBAL_WORKQUEUE_URL=`cat $WMA_SECRETS_FILE | grep GLOBAL_WORKQUEUE_URL | sed s/GLOBAL_WORKQUEUE_URL=//` - local MATCH_LOCAL_WORKQUEUE_DBNAME=`cat $WMA_SECRETS_FILE | grep LOCAL_WORKQUEUE_DBNAME | sed s/LOCAL_WORKQUEUE_DBNAME=//` - local MATCH_WORKLOAD_SUMMARY_URL=`cat $WMA_SECRETS_FILE | grep WORKLOAD_SUMMARY_URL | sed s/WORKLOAD_SUMMARY_URL=//` - local MATCH_WORKLOAD_SUMMARY_DBNAME=`cat $WMA_SECRETS_FILE | grep WORKLOAD_SUMMARY_DBNAME | sed s/WORKLOAD_SUMMARY_DBNAME=//` - local MATCH_WMSTATS_URL=`cat $WMA_SECRETS_FILE | grep WMSTATS_URL | sed s/WMSTATS_URL=//` - local MATCH_REQMGR2_URL=`cat $WMA_SECRETS_FILE | grep REQMGR2_URL | sed s/REQMGR2_URL=//` - local MATCH_ACDC_URL=`cat $WMA_SECRETS_FILE | grep ACDC_URL | sed s/ACDC_URL=//` - local MATCH_DBS3_URL=`cat $WMA_SECRETS_FILE | grep DBS3_URL | sed s/DBS3_URL=//` - local MATCH_DQM_URL=`cat $WMA_SECRETS_FILE | grep DQM_URL | sed s/DQM_URL=//` - local MATCH_REQUESTCOUCH_URL=`cat $WMA_SECRETS_FILE | grep REQUESTCOUCH_URL | sed s/REQUESTCOUCH_URL=//` - local MATCH_CENTRAL_LOGDB_URL=`cat $WMA_SECRETS_FILE | grep CENTRAL_LOGDB_URL | sed s/CENTRAL_LOGDB_URL=//` - local MATCH_WMARCHIVE_URL=`cat $WMA_SECRETS_FILE | grep WMARCHIVE_URL | sed s/WMARCHIVE_URL=//` - local MATCH_AMQ_CREDENTIALS=`cat $WMA_SECRETS_FILE | grep AMQ_CREDENTIALS | sed s/AMQ_CREDENTIALS=//` - local MATCH_RUCIO_HOST=`cat $WMA_SECRETS_FILE | grep RUCIO_HOST | sed s/RUCIO_HOST=//` - local MATCH_RUCIO_AUTH=`cat $WMA_SECRETS_FILE | grep RUCIO_AUTH | sed s/RUCIO_AUTH=//` - local MATCH_RUCIO_ACCOUNT=`cat $WMA_SECRETS_FILE | grep RUCIO_ACCOUNT | sed s/RUCIO_ACCOUNT=//` - local MATCH_TEAMNAME=`cat $WMA_SECRETS_FILE | grep TEAMNAME | sed s/TEAMNAME=//` - local MATCH_AGENT_NUMBER=`cat $WMA_SECRETS_FILE | grep AGENT_NUMBER | sed s/AGENT_NUMBER=//` - - - # database settings (mysql or oracle) - if [ "x$MATCH_ORACLE_USER" == "x" ]; then + # Auxiliary function to parse WMAgent.secrets or MariaDB.secrets files + # and load a set of variables from them + # :param $1: Path to WMAgent.secrets or file (Default: $WMA_SECRETS_FILE) + # :param $2: String with variable names to be checked (Default: *) + # :return: Error value if one or more values have been left unset in the secrets file + local errVal=0 + local value="" + local secretsFile=${1:-$WMA_SECRETS_FILE} + local varsToLoad=${2:-""} + + [[ -f $secretsFile ]] || { + echo "$FUNCNAME: ERROR: Secrets file $secretsFile does not exist" + echo "$FUNCNAME: ERROR: Either set WMA_SECRETS_FILE environment variable to a valid file or check that $HOME/WMAgent.secrets exists" + return $(false) + } + + # If no list of variables to be loaded was given assume all of them. + # Building the list by parsing the secrets file itself. + [[ -n $varsToLoad ]] || { + varsToLoad=`grep -v "^[[:blank:]]#" $secretsFile |grep \= | awk -F\= '{print $1}'` + } + + # Here we validate every variable for itself before loading it + for varName in $varsToLoad + do + _parse_wmasecrets $secretsFile $varName || { + let errVal+=1 + echo "$FUNCNAME: ERROR: Bad value found for $varName" + return $errVal + } + done + + # Now load them all + for varName in $varsToLoad + do + value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | sed "s/ *$varName=//"` + [[ $varName =~ ^RESOURCE_ ]] && declare -g -A $varName + eval $varName=$value + [[ -n $varName ]] || { echo "$FUNCNAME: ERROR: Empty value for: $varName=$value"; let errVal+=1 ;} + done + + # Finaly check and set defaults: + + # Relational database settings (mariaDB or oracle) + if [[ -z $ORACLE_USER ]]; then AGENT_FLAVOR=mysql - MDB_USER=${MATCH_MDB_USER:-$USER}; - MDB_PASS=${MATCH_MDB_PASS:-$MDB_PASS}; - MDB_HOST=${MATCH_MDB_HOST:-127.0.0.1}; + MDB_USER=${MDB_USER:-$USER}; + MDB_HOST=${MDB_HOST:-127.0.0.1}; else AGENT_FLAVOR=oracle - ORACLE_USER=$MATCH_ORACLE_USER; - ORACLE_PASS=$MATCH_ORACLE_PASS; - ORACLE_TNS=$MATCH_ORACLE_TNS; - if [ "x$ORACLE_PASS" == "x" ] || [ "x$ORACLE_TNS" == "x" ]; then - echo "$FUNCNAME: Secrets file doesnt contain ORACLE_PASS or ORACLE_TNS"; - exit 1 + if [[ -z $ORACLE_PASS ]] || [[ -z $ORACLE_TNS ]]; then + echo "$FUNCNAME: ERROR: Secrets file doesnt contain ORACLE_PASS or ORACLE_TNS"; let errVal+=1 fi fi - GRAFANA_TOKEN=${MATCH_GRAFANA_TOKEN:-$GRAFANA_TOKEN}; - if [ "x$GRAFANA_TOKEN" == "x" ]; then - echo "$FUNCNAME: Secrets file doesnt contain GRAFANA_TOKEN"; - exit 1 - fi - - # basic couch settings - COUCH_USER=${MATCH_COUCH_USER:-wmagentcouch}; - COUCH_PASS=${MATCH_COUCH_PASS:-$COUCH_PASS}; - if [ "x$COUCH_PASS" == "x" ]; then - echo "$FUNCNAME: Secrets file doesnt contain COUCH_PASS"; - exit 1 + if [[ -z $GRAFANA_TOKEN ]]; then + echo "$FUNCNAME: ERROR: Secrets file doesnt contain GRAFANA_TOKEN"; let errVal+=1 fi - COUCH_PORT=${MATCH_COUCH_PORT:-$COUCH_PORT}; - COUCH_HOST=${MATCH_COUCH_HOST:-127.0.0.1}; + # CouchDB settings # if couch ssl certificate not specified check X509_USER_CERT and X509_USER_PROXY - COUCH_CERT_FILE=${MATCH_COUCH_CERT_FILE:-${X509_USER_CERT:-$X509_USER_PROXY}}; - # if couch ssl key not specified check X509_USER_KEY and X509_USER_PROXY - COUCH_KEY_FILE=${MATCH_COUCH_KEY_FILE:-${X509_USER_KEY:-$X509_USER_PROXY}}; - - GLOBAL_WORKQUEUE_URL=${MATCH_GLOBAL_WORKQUEUE_URL:-$GLOBAL_WORKQUEUE_URL}; - - LOCAL_WORKQUEUE_DBNAME=${MATCH_LOCAL_WORKQUEUE_DBNAME:-$LOCAL_WORKQUEUE_DBNAME}; - - WORKLOAD_SUMMARY_URL=${MATCH_WORKLOAD_SUMMARY_URL:-$WORKLOAD_SUMMARY_URL}; - - WMSTATS_URL=${MATCH_WMSTATS_URL:-$WMSTATS_URL} - - REQMGR2_URL=${MATCH_REQMGR2_URL:-$REQMGR2_URL} - - ACDC_URL=${MATCH_ACDC_URL:-$ACDC_URL} - - DBS3_URL=${MATCH_DBS3_URL:-$DBS3_URL} - - DQM_URL=${MATCH_DQM_URL:-$DQM_URL} - - REQUESTCOUCH_URL=${MATCH_REQUESTCOUCH_URL:-$REQUESTCOUCH_URL} - - CENTRAL_LOGDB_URL=${MATCH_CENTRAL_LOGDB_URL:-$CENTRAL_LOGDB_URL} - - WMARCHIVE_URL=${MATCH_WMARCHIVE_URL:-$WMARCHIVE_URL} + COUCH_USER=${COUCH_USER:-wmagentcouch}; + COUCH_HOST=${COUCH_HOST:-127.0.0.1}; + COUCH_CERT_FILE=${COUCH_CERT_FILE:-${X509_USER_CERT:-$X509_USER_PROXY}}; + COUCH_KEY_FILE=${COUCH_KEY_FILE:-${X509_USER_KEY:-$X509_USER_PROXY}}; + if [[ -z $COUCH_PASS ]]; then + echo "$FUNCNAME: ERROR: Secrets file doesnt contain COUCH_PASS"; let errVal+=1 + fi - AMQ_CREDENTIALS=${MATCH_AMQ_CREDENTIALS:-$AMQ_CREDENTIALS} - RUCIO_HOST=${MATCH_RUCIO_HOST:-$RUCIO_HOST} - RUCIO_AUTH=${MATCH_RUCIO_AUTH:-$RUCIO_AUTH} - RUCIO_ACCOUNT=${MATCH_RUCIO_ACCOUNT:-$RUCIO_ACCOUNT} - TEAMNAME=${MATCH_TEMANAME:-$TEAMNAME} - AGENT_NUMBER=${MATCH_AGENT_NUMBER:-$AGENT_NUMBER} + return $errVal } _print_settings(){ + echo "-------------- WMA_* environment variables: --------------" env |grep ^WMA| sort - echo "ORACLE_USER= $ORACLE_USER " - echo "ORACLE_PASS= $ORACLE_PASS " - echo "ORACLE_TNS= $ORACLE_TNS " - echo "GRAFANA_TOKEN= $GRAFANA_TOKEN " - echo "MDB_USER= $MDB_USER " - echo "MDB_PASS= $MDB_PASS " - echo "MDB_HOST= $MDB_HOST " - echo "COUCH_USER= $COUCH_USER " - echo "COUCH_PASS= $COUCH_PASS " - echo "COUCH_PORT= $COUCH_PORT " - echo "COUCH_HOST= $COUCH_HOST " - echo "COUCH_CERT_FILE= $COUCH_CERT_FILE " - echo "COUCH_KEY_FILE= $COUCH_KEY_FILE " - echo "GLOBAL_WORKQUEUE_URL= $GLOBAL_WORKQUEUE_URL " - echo "LOCAL_WORKQUEUE_DBNAME= $LOCAL_WORKQUEUE_DBNAME " - echo "WORKLOAD_SUMMARY_URL= $WORKLOAD_SUMMARY_URL " - echo "WORKLOAD_SUMMARY_DBNAME= $WORKLOAD_SUMMARY_DBNAME " - echo "WMSTATS_URL= $WMSTATS_URL " - echo "REQMGR2_URL= $REQMGR2_URL " - echo "ACDC_URL= $ACDC_URL " - echo "DBS3_URL= $DBS3_URL " - echo "DQM_URL= $DQM_URL " - echo "REQUESTCOUCH_URL= $REQUESTCOUCH_URL " - echo "CENTRAL_LOGDB_URL= $CENTRAL_LOGDB_URL " - echo "WMARCHIVE_URL= $WMARCHIVE_URL " - echo "AMQ_CREDENTIALS= $AMQ_CREDENTIALS " - echo "RUCIO_HOST= $RUCIO_HOST " - echo "RUCIO_AUTH= $RUCIO_AUTH " - echo "RUCIO_ACCOUNT= $RUCIO_ACCOUNT " - echo "TEAMNAME= $TEAMNAME " - echo "AGENT_NUMBER= $AGENT_NUMBER " + + echo "-------------- WMA_SECRETS_FILE variables: --------------" + varsToPrint=`grep -v "^[[:blank:]]#" $WMA_SECRETS_FILE |grep \= | awk -F\= '{print $1}'` + + for varName in $varsToPrint + do + if [[ $varName =~ ^RESOURCE_ ]]; then + declare -p $varName + else + echo $varName=${!varName} + fi + done + echo "---------------------------------------------------------" } diff --git a/docker/pypi/wmagent/etc/wmagent_bashrc b/docker/pypi/wmagent/etc/wmagent_bashrc new file mode 100644 index 000000000..6d552b3c3 --- /dev/null +++ b/docker/pypi/wmagent/etc/wmagent_bashrc @@ -0,0 +1,29 @@ +### Template .bashrc file to be used for the user running WMAgent +alias agentenv='source $WMA_ENV_FILE' +alias manage=\$WMA_MANAGE_DIR/manage + +alias lll="ls -lathr" +alias ls="ls --color=auto" +alias ll='ls -la --color=auto' + +alias condorq='condor_q -format "%i." ClusterID -format "%s " ProcId -format " %i " JobStatus -format " %d " ServerTime-EnteredCurrentStatus -format "%s" UserLog -format " %s\n" DESIRED_Sites' +alias condorqrunning='condor_q -constraint JobStatus==2 -format "%i." ClusterID -format "%s " ProcId -format " %i " JobStatus -format " %d " ServerTime-EnteredCurrentStatus -format "%s" UserLog -format " %s\n" DESIRED_Sites' + +alias runningagent="ps aux | egrep 'couch|wmcore|mysql|beam'" +alias foldersize="du -h --max-depth=1 | sort -hr" + +# Better curl command +alias scurl='curl -k --cert ${WMA_CERTS_DIR}/servicecert.pem --key ${WMA_CERTS_DIR}/servicekey.pem' + +# Set command prompt for the running user inside the container +export PS1="(WMAgent-\$WMA_TAG) [\u@\h:\W]\$ " + +# debugging tool +unpkl () +{ + python3 -c 'import pickle,sys,pprint;d=pickle.load(open(sys.argv[1],"rb"));print(d);pprint.pprint(d)' "\$1" +} + +# load the agent environment and utilitarian functions +source $WMA_ENV_FILE +source $WMA_DEPLOY_DIR/bin/manage-common.sh \ No newline at end of file diff --git a/docker/pypi/wmagent/init.sh b/docker/pypi/wmagent/init.sh index ad42f4827..c05abebe9 100755 --- a/docker/pypi/wmagent/init.sh +++ b/docker/pypi/wmagent/init.sh @@ -19,7 +19,7 @@ HOSTIP=`hostname -i` # Setup defaults: [[ $WMA_TAG == $WMCoreVersion ]] || { - echo "WARNING: Container WMA_TAG: $WAM_TAG and actual WMCoreVersion: $WMCoreVersion mismatch." + echo "WARNING: Container WMA_TAG: $WMA_TAG and actual WMCoreVersion: $WMCoreVersion mismatch." echo "WARNING: Assuming WMA_TAG=$WMCoreVersion" WMA_TAG=$WMCoreVersion } @@ -119,6 +119,21 @@ deploy_to_host(){ echo "$FUNCNAME: Copy the proper manage file" cp -fv $WMA_DEPLOY_DIR/bin/manage $WMA_MANAGE_DIR/manage && chmod 755 $WMA_MANAGE_DIR/manage + echo "$FUNCNAME: Copy the Runtime scripts" + _init_valid $wmaInitRuntime || { + # checking if $WMA_DEPLOY_DIR is root path for $pythonLib: + if [[ $pythonLib =~ ^$WMA_DEPLOY_DIR ]]; then + mkdir -p $WMA_INSTALL_DIR/Docker/ + cp -rav $pythonLib/WMCore/WMRuntime $WMA_INSTALL_DIR/Docker/ + cp -rav $WMA_DEPLOY_DIR/etc/ $WMA_CONFIG_DIR/ + echo $WMA_BUILD_ID > $wmaInitRuntime + else + echo "$FUNCNAME: ERROR: \$WMA_DEPLOY_DIR: $WMA_DEPLOY_DIR is not a root path for \$pythonLib: $pithonLib" + echo "$FUNCNAME: ERROR: We cannot find the correct WMCore/WMRuntime source to copy at the current host!" + return $(false) + fi + } + # Check if the host has a basic WMAgent.secrets file and copy a template if missing # NOTE: Here we never overwrite any existing WMAGent.secrets file: We follow: # * Check if there is any at the host, and if so, is it a blank template or a fully configured one @@ -280,6 +295,23 @@ check_databases() { _check_couch } +set_cronjob() { + stepMsg="Populating cronjob with utilitarian scripts for the $WMA_USER" + echo "-----------------------------------------------------------------------" + echo "Start: $stepMsg" + + chmod +x $WMA_DEPLOY_DIR/deploy/renew_proxy.sh $WMA_DEPLOY_DIR/deploy/restartComponent.sh + + crontab -u $WMA_USER - < /dev/null +EOF + + echo "Done: $stepMsg!" && echo + echo "-----------------------------------------------------------------------" +} + check_docker_init() { # A function to check all previously populated */.dockerInit files # from all previous steps and compare them with the /data/.dockerBuildId @@ -291,8 +323,10 @@ check_docker_init() { $wmaInitActive $wmaInitAgent $wmaInitConfig + $wmaInitRuntime $wmaInitUpload $wmaInitResourceControl + $wmaInitResourceOpp $wmaInitCouchDB $wmaInitSqlDB $wmaInitRucio @@ -368,7 +402,7 @@ agent_tweakconfig() { # make this a docker agent sed -i "s+Agent.isDocker = False+Agent.isDocker = True+" $WMA_CONFIG_DIR/config.py # update the location of submit.sh for docker - sed -i "s+config.JobSubmitter.submitScript.*+config.JobSubmitter.submitScript = '$WMA_DEPLOY_DIR/etc/submit.sh'+" $WMA_CONFIG_DIR/config.py + sed -i "s+config.JobSubmitter.submitScript.*+config.JobSubmitter.submitScript = '$WMA_CONFIG_DIR/etc/submit_py3.sh'+" $WMA_CONFIG_DIR/config.py # replace all tags with current sed -i "s+$WMA_TAG+current+" $WMA_CONFIG_DIR/config.py @@ -425,6 +459,45 @@ agent_resource_control() { echo "-------------------------------------------------------" } +agent_resource_opp() { + ## In this function, the list of available opportunistic resources is added to the resource control + ## This is done fetching the env variables starting with RESOURCE_ + ## that are associative arrays defined in WMAgent.secrets with the following schema: + ## RESOURCE_OPP=([name]= [run]= [pend]= [state]=) + local stepMsg="Performing $FUNCNAME" + echo "-------------------------------------------------------" + echo "Start: $stepMsg" + if _init_valid $wmaInitResourceOpp && \ + _init_valid $wmaInitSqlDB + then + echo "$FUNCNAME: Agent Opportunistic Resource control has been populated already." + else + echo "$FUNCNAME: triggered." + local errVal=0 + ## Populating opportunistic resource-control + echo "$FUNCNAME: Populating opportunistic resource-control" + ## Loop over the env variables starting with RESOURCE_* + for res in ${!RESOURCE_*} + do + ## Parsing of the information stored in RESOURCE_* env variable, according to the schema reported above + eval `declare -p $res | sed -e "s/$res/site/g"` + if [[ ${site[name]} =~ .*_US_.* ]] && [[ $HOSTNAME =~ .*cern\.ch ]]; then + echo "I am based at CERN, so I cannot use US opportunistic resources, moving to the next site" + continue + else + manage execute-agent wmagent-resource-control --plugin=SimpleCondorPlugin --opportunistic --pending-slots=${site[pend]} --running-slots=${site[run]} --add-one-site=${site[name]} ; let errVal+=$? + fi + done + [[ $errVal -eq 0 ]] || { echo "ERROR: Failed to populate WMAgent's opportunistic resource control!"; return $(false) ;} + echo $WMA_BUILD_ID > $wmaInitResourceOpp + fi + + echo "Done: $stepMsg" + echo "-------------------------------------------------------" +} + + + agent_upload_config(){ # A function to be used for uploading WMAgentConfig to AuxDB at Central CouchDB # NOTE: The final config/.dockerInit is to be set here after full agent initialisation. @@ -482,6 +555,7 @@ main(){ (init_agent) || { err=$?; echo "ERROR: init_agent"; exit $err ;} (agent_tweakconfig) || { err=$?; echo "ERROR: agent_tweakconfig"; exit $err ;} (agent_resource_control) || { err=$?; echo "ERROR: agent_resource_control"; exit $err ;} + (agent_resource_opp) || { err=$?; echo "ERROR: agent_resource_opp"; exit $err ;} (agent_upload_config) || { err=$?; echo "ERROR: agent_upload_config"; exit $err ;} echo $WMA_BUILD_ID > $wmaInitUsing (check_docker_init) || { err=$?; echo "ERROR: DockerBuild vs. HostConfiguration version missmatch"; exit $err ; } @@ -502,6 +576,7 @@ main(){ echo "Have a nice day!" && echo return $(true) } + (set_cronjob) || { err=$?; echo "ERROR: set_cronjob"; exit $err ;} (check_databases) || { err=$?; echo "ERROR: check_databases"; exit $err ;} (_renew_proxy) || { err=$?; echo "ERROR: _renew_proxy"; exit $err ;} (start_agent) || { err=$?; echo "ERROR: start_agent"; exit $err ;} diff --git a/docker/pypi/wmagent/install.sh b/docker/pypi/wmagent/install.sh index a169d220c..682926d27 100755 --- a/docker/pypi/wmagent/install.sh +++ b/docker/pypi/wmagent/install.sh @@ -120,11 +120,9 @@ echo "-----------------------------------------------------------------------" tweakEnv(){ # A function to apply environment tweaks for the docker image - echo "-------------------------------------------------------" echo "Edit \$WMA_ENV_FILE script to point to \$WMA_ROOT_DIR" sed -i "s|/data/|\$WMA_ROOT_DIR/|g" $WMA_ENV_FILE - echo "-------------------------------------------------------" echo "Edit \$WMA_ENV_FILE script to point to the correct install, config and manage" sed -i "s|install=.*|install=\$WMA_INSTALL_DIR|g" $WMA_ENV_FILE sed -i "s|config=.*|config=\$WMA_CONFIG_DIR|g" $WMA_ENV_FILE @@ -134,7 +132,6 @@ tweakEnv(){ echo "Edit $WMA_DEPLOY_DIR/deploy/renew_proxy.sh script to point to \$WMA_ROOT_DIR" sed -i "s|/data/|\$WMA_ROOT_DIR/|g" $WMA_DEPLOY_DIR/deploy/renew_proxy.sh sed -i "s|source.*env\.sh|source \$WMA_ENV_FILE|g" $WMA_DEPLOY_DIR/deploy/renew_proxy.sh - echo "-------------------------------------------------------" cat <> $WMA_ENV_FILE @@ -154,52 +151,14 @@ stepMsg="Tweaking runtime environment for user: $WMA_USER" echo "-----------------------------------------------------------------------" echo "Start $stepMsg" tweakEnv || { err=$?; echo ""; exit $err ; } -cat <> /home/${WMA_USER}/.bashrc - -alias lll="ls -lathr" -alias ls="ls --color=auto" -alias ll='ls -la --color=auto' - -alias condorq='condor_q -format "%i." ClusterID -format "%s " ProcId -format " %i " JobStatus -format " %d " ServerTime-EnteredCurrentStatus -format "%s" UserLog -format " %s\n" DESIRED_Sites' -alias condorqrunning='condor_q -constraint JobStatus==2 -format "%i." ClusterID -format "%s " ProcId -format " %i " JobStatus -format " %d " ServerTime-EnteredCurrentStatus -format "%s" UserLog -format " %s\n" DESIRED_Sites' -alias agentenv='source $WMA_ENV_FILE' -alias manage=\$WMA_MANAGE_DIR/manage - -# Aliases for Tier0-Ops. -alias runningagent="ps aux | egrep 'couch|wmcore|mysql|beam'" -alias foldersize="du -h --max-depth=1 | sort -hr" - -# Better curl command -alias scurl='curl -k --cert ${CERT_DIR}/servicecert.pem --key ${CERT_DIR}/servicekey.pem' - -# set WMAgent docker specific bash prompt: -export PS1="(WMAgent-\$WMA_TAG) [\u@\h:\W]\$ " source $WMA_ENV_FILE -EOF -echo "Done $stepMsg!" && echo -echo "-----------------------------------------------------------------------" - -stepMsg="Populating cronjob with utilitarian scripts for the WMA_USER" -echo "-----------------------------------------------------------------------" -echo "Start $stepMsg" - -# TODO: These executable flags we should consider fixing them for all *.sh -# scripts under the /deploy top level area in the WMCore github repository -chmod +x $WMA_DEPLOY_DIR/deploy/renew_proxy.sh $WMA_DEPLOY_DIR/deploy/restartComponent.sh - -crontab -u $WMA_USER - < /dev/null -EOF - +source $WMA_DEPLOY_DIR/bin/manage-common.sh echo "Done $stepMsg!" && echo echo "-----------------------------------------------------------------------" - echo "-----------------------------------------------------------------------" -echo "WMAgent contaner build finished!!" && echo +echo "WMAgent image build finished!!" && echo echo "Have a nice day!" && echo echo "=======================================================================" diff --git a/docker/pypi/wmagent/run.sh b/docker/pypi/wmagent/run.sh index c0bf8e0f0..28fc3e0d1 100755 --- a/docker/pypi/wmagent/run.sh +++ b/docker/pypi/wmagent/run.sh @@ -1,11 +1,23 @@ #!/bin/bash ### Basic initialization wrapper for WMAgent to serve as the main entry point for the WMAgent Docker container +wmaUser=$(id -un) +wmaGroup=$(id -gn) +wmaUserID=$(id -u) +wmaGroupID=$(id -g) +echo "Running WMAgent container with user: $wmaUser (ID: $wmaUserID) and group: $wmaGroup (ID: $wmaGroupID)" + +echo "Setting up bashrc for user: $wmaUser under home directory: $HOME" +export WMA_USER=$wmaUser +export USER=$wmaUser +[[ -d ${HOME} ]] || mkdir -p ${HOME} + +mv ${WMA_CONFIG_DIR}/etc/wmagent_bashrc $HOME/.bashrc +source $HOME/.bashrc echo "Start initialization" -./init.sh | tee -a $WMA_LOG_DIR/init.log || true +$WMA_ROOT_DIR/init.sh | tee -a $WMA_LOG_DIR/init.log || true echo "Start sleeping now ...zzz..." - -while true; do sleep 10; done +sleep infinity diff --git a/docker/pypi/wmagent/wmagent-docker-run.sh b/docker/pypi/wmagent/wmagent-docker-run.sh index 1c698123b..97b9aa999 100755 --- a/docker/pypi/wmagent/wmagent-docker-run.sh +++ b/docker/pypi/wmagent/wmagent-docker-run.sh @@ -43,9 +43,8 @@ while getopts ":t:hp" opt; do esac done - -wmaUser=cmst1 -wmaOpts=" --user $wmaUser" +wmaUser=$(id -un) +wmaGroup=$(id -gn) # This is the root at the host only, it may differ from the root inside the container. # NOTE: This is parametriesed, so that the container can run on a different mount point. @@ -55,37 +54,55 @@ HOST_MOUNT_DIR=/data/dockerMount [[ -h /data/srv/wmagent ]] && rm -f /data/srv/wmagent ln -s $HOST_MOUNT_DIR/srv/wmagent /data/srv/wmagent - +# create the passwd and group mount point dynamically at runtime +passwdEntry=$(getent passwd $wmaUser | awk -F : -v wmaHome="/home/$wmaUser" '{print $1 ":" $2 ":" $3 ":" $4 ":" $5 ":" wmaHome ":" $7}') +groupEntry=$(getent group $wmaGroup) + +# workaround case where Unix account is not in the local system (e.g. sssd) +[[ -d $HOST_MOUNT_DIR/admin/etc/ ]] || (mkdir -p $HOST_MOUNT_DIR/admin/etc) || exit $? +if ! [ -f $HOST_MOUNT_DIR/admin/etc/passwd ]; then + echo "Creating passwd file" + getent passwd > $HOST_MOUNT_DIR/admin/etc/passwd + echo $passwdEntry >> $HOST_MOUNT_DIR/admin/etc/passwd +fi +if ! [ -f $HOST_MOUNT_DIR/admin/etc/group ]; then + echo "Creating group file" + getent group > $HOST_MOUNT_DIR/admin/etc/group + echo $groupEntry >> $HOST_MOUNT_DIR/admin/etc/group +fi + +# create regular mount points at runtime [[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $? [[ -d $HOST_MOUNT_DIR/admin/wmagent ]] || (mkdir -p $HOST_MOUNT_DIR/admin/wmagent) || exit $? [[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install) || exit $? [[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config) || exit $? [[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ;} || exit $? -chown -R $wmaUser $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG || exit $? - # NOTE: Before mounting /etc/tnsnames.ora we should check it exists, otherwise the run will fail on the FNAL agents tnsMount="" [[ -f /etc/tnsnames.ora ]] && tnsMount="--mount type=bind,source=/etc/tnsnames.ora,target=/etc/tnsnames.ora,readonly " dockerOpts=" \ ---detach +--detach \ --network=host \ --rm \ ---hostname=`hostname -f` \ +--hostname=$(hostname -f) \ +--user $(id -u):$(id -g) \ --name=wmagent \ -$tnsMount +$tnsMount \ --mount type=bind,source=/etc/condor,target=/etc/condor,readonly \ --mount type=bind,source=/tmp,target=/tmp \ --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \ --mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install,target=/data/srv/wmagent/current/install \ --mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config,target=/data/srv/wmagent/current/config \ --mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs,target=/data/srv/wmagent/current/logs \ ---mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/passwd,target=/etc/passwd,readonly \ +--mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/group,target=/etc/group,readonly \ +--mount type=bind,source=/etc/sudoers,target=/etc/sudoers,readonly \ +--mount type=bind,source=/etc/sudoers.d,target=/etc/sudoers.d,readonly \ " -wmaOpts="$wmaOpt $*" - $PULL && { echo "Pulling Docker image: registry.cern.ch/cmsweb/wmagent:$WMA_TAG" docker login registry.cern.ch @@ -95,9 +112,9 @@ $PULL && { } echo "Checking if there is no other wmagent container running and creating a link to the $WMA_TAG in the host mount area." -[[ `docker container inspect -f '{{.State.Status}}' wmagent 2>/dev/null ` == 'running' ]] || ( +[[ $(docker container inspect -f '{{.State.Status}}' wmagent 2>/dev/null) == 'running' ]] || ( [[ -h $HOST_MOUNT_DIR/srv/wmagent/current ]] && rm -f $HOST_MOUNT_DIR/srv/wmagent/current ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG $HOST_MOUNT_DIR/srv/wmagent/current ) -echo "Starting the wmagent:$WMA_TAG docker container with the following parameters: $wmaOpts" -docker run $dockerOpts local/wmagent:$WMA_TAG $wmaOpts +echo "Starting wmagent:$WMA_TAG docker container with user: $wmaUser:$wmaGroup" +docker run $dockerOpts local/wmagent:$WMA_TAG