From a40e76e350a2971495f4aeb990f0948617773f9c Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Sep 2023 23:41:01 +0200
Subject: [PATCH 01/27] Add minimal MariaDB container for WMAgent

---
 docker/pypi/wmagent-mariadb/Dockerfile        |  80 ++++++++
 docker/pypi/wmagent-mariadb/manage            | 178 ++++++++++++++++++
 .../wmagent-mariadb/mariadb-docker-run.sh     |  99 ++++++++++
 docker/pypi/wmagent-mariadb/my.cnf            |  76 ++++++++
 docker/pypi/wmagent-mariadb/run.sh            |   7 +
 docker/pypi/wmagent-mariadb/start-mysql.sh    |  63 +++++++
 6 files changed, 503 insertions(+)
 create mode 100644 docker/pypi/wmagent-mariadb/Dockerfile
 create mode 100755 docker/pypi/wmagent-mariadb/manage
 create mode 100755 docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
 create mode 100644 docker/pypi/wmagent-mariadb/my.cnf
 create mode 100755 docker/pypi/wmagent-mariadb/run.sh
 create mode 100755 docker/pypi/wmagent-mariadb/start-mysql.sh

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
new file mode 100644
index 000000000..3a6125e7e
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -0,0 +1,80 @@
+ARG TAG=11.1
+FROM mariadb:${TAG}
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+
+ARG TAG
+ENV TAG=${TAG}
+RUN echo TAG=$TAG
+
+RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
+
+# # Install some debugging tools
+RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
+
+ENV USER=cmst1
+# ENV MARIADB_PORT=
+ENV UID=31961
+ENV MARIADB_ROOT_DIR=/data
+
+ENV MARIADB_BASE_DIR=$MARIADB_ROOT_DIR/srv/mariadb
+ENV MARIADB_ADMIN_DIR=$MARIADB_ROOT_DIR/admin/wmagent
+ENV MARIADB_CERTS_DIR=$MARIADB_ROOT_DIR/certs
+
+ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$TAG
+ENV MARIADB_MANAGE_DIR=$MARIADB_CURRENT_DIR
+ENV MARIADB_AUTH_DIR=$MARIADB_CURRENT_DIR/auth/
+ENV MARIADB_INSTALL_DIR=$MARIADB_CURRENT_DIR/install
+ENV MARIADB_STATE_DIR=$MARIADB_CURRENT_DIR/state
+ENV MARIADB_DATABASE_DIR=$MARIADB_INSTALL_DIR/database
+ENV MARIADB_CONFIG_DIR=$MARIADB_CURRENT_DIR/config
+ENV MARIADB_LOG_DIR=$MARIADB_CURRENT_DIR/logs
+ENV MARIADB_DEPLOY_DIR=/usr/local
+ENV MARIADB_ENV_FILE=$MARIADB_DEPLOY_DIR/deploy/env.sh
+ENV MARIADB_SECRETS_FILE=$MARIADB_ADMIN_DIR/MariaDB.secrets
+
+
+RUN useradd -u $UID -m $USER
+
+# add user to sudoers file
+RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# start the setup
+RUN mkdir -p $MARIADB_ROOT_DIR
+
+ENV PATH="${MARIADB_ROOT_DIR}:${PATH}"
+
+RUN mkdir -p $MARIADB_CURRENT_DIR $MARIADB_CONFIG_DIR $MARIADB_MANAGE_DIR $MARIADB_LOG_DIR $MARIADB_DATABASE_DIR $MARIADB_STATE_DIR $MARIADB_AUTH_DIR
+RUN ln -s $MARIADB_CURRENT_DIR $MARIADB_BASE_DIR/current
+
+# add necessary scripts
+ADD run.sh ${MARIADB_ROOT_DIR}/
+ADD start-mysql.sh ${MARIADB_ROOT_DIR}/
+ADD manage ${MARIADB_MANAGE_DIR}/manage
+RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
+
+# The $MARIADB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
+# ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
+# RUN ln -s ${MARIADB_CONFIG_DIR}/my.cnf /opt/mariadb/etc/local.d/
+ADD my.cnf /etc/mysql/my.cnf
+
+
+ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
+
+RUN <<EOF cat >> /home/${USER}/.bashrc
+
+alias lll="ls -lathr"
+alias ls="ls --color=auto"
+alias ll='ls -la --color=auto'
+
+alias manage=$MARIADB_MANAGE_DIR/manage
+
+# set MariaDB docker specific bash prompt:
+export PS1="(MariaDB-$TAG) [\u@\h:\W]\$ "
+EOF
+
+RUN chown -R ${USER} ${MARIADB_ROOT_DIR}
+
+# setup final environment
+USER $USER
+WORKDIR $MARIADB_ROOT_DIR
+ENTRYPOINT ["./run.sh"]
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
new file mode 100755
index 000000000..372e8dd93
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -0,0 +1,178 @@
+#!/bin/bash
+
+#### These are old mysql actions previously defined in the wmagent manage script
+#### Some of them need to be rewritten some of them must go away
+
+
+
+# #########################
+# #  MySQL                #
+# #########################
+
+# #
+# # first time startup routines for mysql
+# # pre gets called before startup, post called after it
+# init_mysql_db_pre(){
+#     echo "Installing the mysql database area..."
+#     mkdir -p $INSTALL_MYSQL/database
+#     mkdir -p $INSTALL_MYSQL/logs
+#     mysql_install_db --datadir=$INSTALL_MYSQL/database
+# }
+# init_mysql_db_post(){
+#     #install the WMAgent stuff
+#     echo "Installing the mysql schema..."
+#     load_secrets_file;
+#     local TIMEOUT=0;
+#     while [ ! -e $MYSQL_SOCK ]
+#     do
+#         sleep 2;
+#         TIMEOUT=$(($TIMEOUT+2))
+#         if [ $TIMEOUT -ge 300 ]; then
+#             echo "ERROR: Timeout waiting for mysqld to start."
+#             exit 1;
+#         fi
+#     done
+#     echo "Socket file exists, proceeding with schema install..."
+
+#     inited_mysql;
+
+#     # create a user - different than root and current unix user - and grant privileges
+#     if [ "$MYSQL_USER" != "$USER" ]; then
+#         mysql -u $USER --socket=$MYSQL_SOCK --execute "CREATE USER '${MYSQL_USER}'@'localhost'"
+#         mysql -u $USER --socket=$MYSQL_SOCK --execute "GRANT ALL ON *.* TO $MYSQL_USER@localhost WITH GRANT OPTION"
+#     fi
+
+#     # create databases for agent
+#     if [ $USING_AG -eq 1 ]; then
+#         echo "Installing WMAgent Database: ${MYSQL_DATABASE_AG}"
+#         mysql -u $USER --socket=$MYSQL_SOCK --execute "create database ${MYSQL_DATABASE_AG}"
+#     fi
+# }
+
+# status_of_mysql(){
+#     load_secrets_file;
+#     if [ "x$MYSQL_USER" == "x" ]; then
+# 	echo "Not using MySQL..."
+# 	exit 1;
+#     fi
+
+#     echo "+ Status of MySQL"
+#     if [ ! -e $INSTALL_MYSQL/logs/mysqld.pid ]; then
+#         echo "++ MySQL process file not found"
+#         return
+#     fi
+#     local MYSQL_PID=`cat $INSTALL_MYSQL/logs/mysqld.pid`
+#     kill -0 $MYSQL_PID;
+#     local MYSQL_STATUS=$?
+#     if [ $MYSQL_STATUS -eq 0 ]; then
+#         echo "++ MYSQL running with process: $MYSQL_PID";
+#     else
+#         echo "++ MYSQL process not running"
+#     fi
+
+#     echo "++" `mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK status`
+# }
+
+# #
+# # Main startup method for MySQL.
+# # Checks for initialisation
+# start_mysql(){
+#     load_secrets_file;
+#     if [ "x$MYSQL_USER" == "x" ]; then
+# 	    echo "Not using MySQL..."
+# 	    exit 1;
+#     fi
+
+#     echo "Starting mysql..."
+
+#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
+# 	    echo "MySQL has not been initialised... running pre initialisation";
+# 	    init_mysql_db_pre;
+#     fi
+
+#     # Start mysqld to install the database schemas
+#     #
+#     echo "starting mysqld_safe..."
+#     nohup mysqld_safe --defaults-extra-file=$CONFIG_MYSQL/my.cnf \
+#                       --datadir=$INSTALL_MYSQL/database \
+#                       --log-bin \
+#                       --socket=$MYSQL_SOCK \
+#                       --log-error=$INSTALL_MYSQL/logs/error.log \
+#                       --pid-file=$INSTALL_MYSQL/logs/mysqld.pid > /dev/null 2>&1 < /dev/null &
+#     local TIMEOUT=0;
+#     echo "Checking MySQL Socket file exists..."
+#     while [ ! -e $MYSQL_SOCK ]
+#       do
+#       sleep 2;
+#       TIMEOUT=$(($TIMEOUT+2))
+#       if [ $TIMEOUT -ge 300 ]; then
+# 	    echo "ERROR: Timeout waiting for mysqld to start."
+# 	    exit 1;
+#       fi
+#     done
+#     echo "Socket file exists: $MYSQL_SOCK"
+
+#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
+#         echo "MySQL has not been initialised... running post initialisation";
+#         init_mysql_db_post;
+#     fi
+#     echo "Checking Server connection..."
+#     mysql -u $USER --socket=$MYSQL_SOCK --execute "SHOW GLOBAL STATUS" > /dev/null;
+#     if [ $? -ne 0 ]; then
+# 	    echo "ERROR: checking mysql database is running, failed to execute SHOW GLOBAL STATUS"
+# 	    exit 1
+#     fi
+#    echo "Connection OK"
+# }
+
+# #
+# # stop MySQL
+# #
+# stop_mysql(){
+#     load_secrets_file;
+#     if [ "x$MYSQL_USER" == "x" ]; then
+# 	echo "Not using MySQL..."
+# 	exit 1;
+#     fi
+
+#     echo "stopping mysql..."
+#     mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK shutdown &
+#     wait $!
+#     echo "Making sure the MySQL socket file is removed..."
+#     local TIMEOUT=0;
+#     while [ -e $MYSQL_SOCK ]
+#       do
+#       sleep 2;
+#       TIMEOUT=$(($TIMEOUT+2))
+#       if [ $TIMEOUT -ge 300 ]; then
+# 	  echo "ERROR: Timeout waiting for mysqld to shutdown."
+# 	  echo "ERROR: Socket file still exists: $MYSQL_SOCK"
+# 	  exit 1;
+#       fi
+#     done
+#     echo "MySQL is shutdown."
+# }
+
+# #
+
+# case $1 in
+#   status)
+#     status ;;
+#   start-mysql)
+#     start_mysql;;
+#   stop-mysql)
+#     stop_mysql;;
+#   clean-mysql)
+#      clean_mysql;;
+#   db-prompt)
+#      db_prompt $@;;
+#   mysql-prompt)
+#      db_prompt $@;;
+#   help)
+#     help ;;
+#   version)
+#     echo "Management script for WMAgent. No idea what version, at least 2 though" ;;
+#   * )
+#     echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
+#     exit 1 ;;
+# esac
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
new file mode 100755
index 000000000..cf086e8b6
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+### This script is to be used for running the Mariadb docker container at a VM
+### Its sole purpose is to set all the needed mount points from the Host VM and
+### forward all Mariadb runtime parameters to the Mariadb container entrypoint run.sh
+### It accepts only the set of parameters relevant to Mariadb's container run.sh
+### and no build dependent ones. The docker image tag to be searched for execution is
+### always `latest`.
+
+
+# NOTE: In the help call to the current scrit we only repeat the help and usage
+#       information for all the parameters accepted by run.sh.
+help(){
+    echo -e $*
+    cat <<EOF
+
+The script to be used for running a Mariadb docker container at a VM. The full set of arguments
+passed to the current script are to be forwarded to the Mariadb container entrypoint 'run.sh'
+
+Usage: mariadb-docker-run.sh [-t <team_name>] [-n <agent_number>] [-f <db_flavour>]
+
+    -p <pull_image>   Pull the image from registry.cern.ch
+    -t <mariadb_tag>  The Mariadb version/tag to be downloaded from registry.cern.ch [Default:latest]
+    -h <help>
+
+Example: ./mariadb-docker-run.sh -t 3.2.2
+
+EOF
+}
+
+usage(){
+    help $*
+    exit 1
+}
+
+PULL=false
+MARIADB_TAG=11.1
+
+
+### Argument parsing:
+while getopts ":t:hp" opt; do
+    case ${opt} in
+        t) MARIADB_TAG=$OPTARG ;;
+        p) PULL=true ;;
+        h) help; exit $? ;;
+        : )
+            msg="Invalid Option: -$OPTARG requires an argument"
+            usage "$msg" ;;
+    esac
+done
+
+
+mariadbUser=cmst1
+mariadbOpts=" --user $mariadbUser"
+
+# This is the root at the host only, it may differ from the root inside the container.
+# NOTE: this may be parametriesed, so that the container can run on a different mount point.
+HOST_MOUNT_DIR=/data/dockerMount
+
+[[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
+[[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
+
+sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
+
+
+dockerOpts="
+--detach \
+--network=host \
+--rm \
+--hostname=`hostname -f` \
+--name=mariadb \
+--mount type=bind,source=/tmp,target=/tmp \
+--mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs,target=/data/srv/mariadb/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
+"
+
+
+
+# mariadbOpts=$*
+# mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
+
+$PULL && {
+    echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG"
+    docker login registry.cern.ch
+    docker pull registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:$MARIADB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:latest
+}
+
+echo "Starting the mariadb:$MARIADB_TAG docker container with the following parameters: $mariadbOpts"
+docker run $dockerOpts $mariadbOpts local/mariadb:$MARIADB_TAG && (
+    [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && sudo rm -f $HOST_MOUNT_DIR/srv/mariadb/current
+    sudo ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/my.cnf b/docker/pypi/wmagent-mariadb/my.cnf
new file mode 100644
index 000000000..2d8466311
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/my.cnf
@@ -0,0 +1,76 @@
+[mysqld]
+# this is the default setting in >= 10.2.4
+sql_mode="NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES"
+# default: REPEATABLE-READ
+transaction-isolation=READ-COMMITTED
+bind-address = 127.0.0.1
+
+max_heap_table_size=2048M
+max_allowed_packet=128M
+max_connections = 200
+connect_timeout = 60
+
+# default: MIXED
+binlog_format=ROW
+# default: 16MB
+tmp_table_size=2048M
+# default: 10
+long_query_time=5
+
+# default: 134217728
+key_buffer_size=4000M
+
+# default: 0
+# disabling the query cache for now
+# unittests do not work with this enabled
+#query_cache_type=1
+#query_cache_limit=10M
+#query_cache_size=128M
+
+# threading
+# thread_cache_size defaults to 256, if > than max_connections, it is
+# set to max_connections
+thread_cache_size = 64
+thread_cache_size = 16
+thread_stack = 192K
+
+# innodb
+# default: O_DIRECT
+innodb_flush_method=O_DIRECT
+# default: 4
+innodb_read_io_threads = 4
+# default: 4
+innodb_write_io_threads = 4
+# default: full_crc23
+# Commented out due  to old mariadb version
+# innodb_checksum_algorithm=full_crc32
+# default: 1
+innodb_doublewrite=0
+
+innodb_log_file_size=512M
+innodb_log_buffer_size=8M
+# Changed for small testing machines
+# innodb_buffer_pool_size=2G
+innodb_buffer_pool_size=50M
+# default: 30
+innodb_sync_spin_loops=60
+# default: 0
+innodb_force_recovery = 0
+# default: 50
+innodb_lock_wait_timeout = 100
+
+# Changes to support DYNAMIC / COMPRESSED row format
+# default: Barracuda
+innodb_file_format=Barracuda
+# default: ON
+innodb_file_per_table=ON
+# supports prefix index larger than 767 bytes (might be already implicit in the DYNAMIC mode?)
+# default: ON
+innodb_large_prefix=ON
+# default: ON
+innodb_strict_mode=ON
+#innodb_page_size=32k  # default is 16k
+
+# Introduced in MariaDB 10.1.32, COMP is currently using 10.1.21
+# If COMPRESSED is used, it cannot be set globally and has to be passed in the CREATE TABLE statement
+#innodb_default_row_format=DYNAMIC
diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh
new file mode 100755
index 000000000..62920288d
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/run.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+echo "start sleeping....zzz"
+while true; do sleep 10; done
+
+
+# # start the service
+# manage start
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
new file mode 100755
index 000000000..9064f6b0e
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -0,0 +1,63 @@
+#/bin/bash
+
+### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
+###       !!!! NOTHING MUST STAY HERE !!!!
+###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
+###       NEED TO BE PERFORMED AT THE MYSQL DOCKER IMAGE
+mysqlRoot=
+mysqlRootPass=
+mysqlUser=cmst1
+mysqlUserPass=
+
+configDir=/data/srv/mysql/current/config
+dataDir=/data/srv/mysql/current/install/database
+logDir=/data/srv/mysql/current/logs
+socket=/data/srv/mysql/current/logs/mysql.sock
+agentDb=wmagent
+
+echo -------------------------------------------------------------------------
+echo Stopping any previously running mysql server
+mysqladmin -u $mysqlRoot --password=$mysqlRootPass -h 127.0.0.1 shutdown
+# mysqladmin -u $mysqlRoot --password=$mysqlRootPass --socket=$socket shutdown
+echo
+
+
+echo -------------------------------------------------------------------------
+echo Installing system database
+mysql_install_db --datadir=$dataDir
+echo
+
+
+echo -------------------------------------------------------------------------
+echo starting the server
+mysqld_safe --defaults-extra-file=$configDir/my.cnf \
+      --datadir=$dataDir \
+      --log-bin \
+      --socket=$socket \
+      --log-error=$logDir/error.log \
+      --pid-file=$logDir/mysqld.pid  & # > /dev/null 2>&1 < /dev/null &
+echo ...
+sleep 10
+echo
+
+echo -------------------------------------------------------------------------
+echo Securing mysqlRoot and removing temp databases
+mysqladmin -u $mysqlRoot password $mysqlRootPass --socket=$socket
+mysqladmin -u $mysqlRoot --password=$mysqlRootPass  -h 127.0.0.1 password $mysqlRootPass
+# mysql_secure_installation
+echo
+
+echo -------------------------------------------------------------------------
+echo creating new users
+# create a user - different than root and current unix user - and grant privileges
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@localhost WITH GRANT OPTION"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@127.0.0.1 WITH GRANT OPTION"
+
+echo -------------------------------------------------------------------------
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database $agentDb"
+
+echo -------------------------------------------------------------------------

From 70cd175fd602665209332e3d4094e18595d77e4d Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 8 Sep 2023 01:36:21 +0200
Subject: [PATCH 02/27] Fix broken server startup procedure

---
 docker/pypi/wmagent-mariadb/start-mysql.sh | 23 +++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 9064f6b0e..97c181af5 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -4,15 +4,15 @@
 ###       !!!! NOTHING MUST STAY HERE !!!!
 ###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
 ###       NEED TO BE PERFORMED AT THE MYSQL DOCKER IMAGE
-mysqlRoot=
+mysqlRoot=root
 mysqlRootPass=
 mysqlUser=cmst1
 mysqlUserPass=
 
-configDir=/data/srv/mysql/current/config
-dataDir=/data/srv/mysql/current/install/database
-logDir=/data/srv/mysql/current/logs
-socket=/data/srv/mysql/current/logs/mysql.sock
+configDir=/data/dockerMount/srv/mysql/current/config
+dataDir=/data/dockerMount/srv/mysql/current/install/database
+logDir=/data/dockerMount/srv/mysql/current/logs
+socket=/data/dockerMount/srv/mysql/current/logs/mysql.sock
 agentDb=wmagent
 
 echo -------------------------------------------------------------------------
@@ -47,17 +47,18 @@ mysqladmin -u $mysqlRoot --password=$mysqlRootPass  -h 127.0.0.1 password $mysql
 # mysql_secure_installation
 echo
 
+echo -------------------------------------------------------------------------
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database '$agentDb'"
+
 echo -------------------------------------------------------------------------
 echo creating new users
 # create a user - different than root and current unix user - and grant privileges
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost' IDENTIFIED BY '$mysqlUserPass'"
 mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@localhost WITH GRANT OPTION"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1' IDENTIFIED BY '$mysqlUserPass'"
 mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@127.0.0.1 WITH GRANT OPTION"
 
-echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database $agentDb"
 
 echo -------------------------------------------------------------------------

From a5353869629cef47a0517f58443cdd341eb10ae6 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 31 Jan 2024 17:43:57 +0100
Subject: [PATCH 03/27] Update values.yaml

---
 helm/mongodb/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/mongodb/values.yaml b/helm/mongodb/values.yaml
index 248392fd8..1d5475002 100644
--- a/helm/mongodb/values.yaml
+++ b/helm/mongodb/values.yaml
@@ -3,7 +3,7 @@ quickSetting:
   namespace: default
   env: test-rs
   image:
-    mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2310f-test
+    mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2311a
   pvc:
     storageClassName: cpio1
     accessMode: ReadWriteMany

From 5599b84d58d155794050dcf58905c84277905300 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 31 Jan 2024 17:48:08 +0100
Subject: [PATCH 04/27] Update Chart.yaml

---
 helm/mongodb/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/mongodb/Chart.yaml b/helm/mongodb/Chart.yaml
index 7fb69effd..b201143c1 100644
--- a/helm/mongodb/Chart.yaml
+++ b/helm/mongodb/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: mongodb
 description: A Helm chart for launching a MongoDB ReplicaSet with 3 instances
 type: application
-version: 1.0.10
-appVersion: 1.0.10
+version: 1.0.11
+appVersion: 1.0.11

From 992166e3a966253a342c3215aa2626d9281980d8 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Mon, 12 Feb 2024 15:30:19 +0100
Subject: [PATCH 05/27] Update ing-dqm.yaml

---
 kubernetes/cmsweb/ingress/ing-dqm.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubernetes/cmsweb/ingress/ing-dqm.yaml b/kubernetes/cmsweb/ingress/ing-dqm.yaml
index def044e64..59f125365 100644
--- a/kubernetes/cmsweb/ingress/ing-dqm.yaml
+++ b/kubernetes/cmsweb/ingress/ing-dqm.yaml
@@ -17,7 +17,7 @@ spec:
   - host: cmsweb-srv.cern.ch
     http:
       paths:
-      - path: /dqm/dqm-square-k8
+      - path: /dqm/dqm-square
         pathType: Prefix
         backend:
           service:

From 6ac9fe2606f508c7f667e8beb29cd71c941b0b28 Mon Sep 17 00:00:00 2001
From: Stefano Belforte <stefano.belforte@cern.ch>
Date: Wed, 21 Feb 2024 18:00:35 +0100
Subject: [PATCH 06/27] add values for cmsweb-test2

---
 helm/crabserver/Chart.yaml        |   2 +-
 helm/crabserver/values-test2.yaml | 179 ++++++++++++++++++++++++++++++
 2 files changed, 180 insertions(+), 1 deletion(-)
 create mode 100644 helm/crabserver/values-test2.yaml

diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index 4a1245c0c..fe64ef13c 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -20,4 +20,4 @@ version: 0.1.6
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: "v3.230913-stable"
+appVersion: "v3.240219.1"
diff --git a/helm/crabserver/values-test2.yaml b/helm/crabserver/values-test2.yaml
new file mode 100644
index 000000000..5a4206f71
--- /dev/null
+++ b/helm/crabserver/values-test2.yaml
@@ -0,0 +1,179 @@
+# Default values for crabserver.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 
+  prod: 8
+  test: 1
+
+image:
+  path: registry.cern.ch/cmsweb/crabserver
+  pullPolicy: Always
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "v3.240219"
+  command:
+  - /bin/bash
+  - /opt/setup-certs-and-run/setup-certs-and-run.sh
+  env:
+  - name: CRABSERVER_LOGSTDOUT
+    value: "t"
+
+environment:
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: false
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: 
+  prometheus.io/scrape: 'true'
+  prometheus.io/port: "18270"
+
+podSecurityContext:
+  runAsUser: 1000
+  runAsGroup: 1000
+  fsGroup: 2000
+
+imageFilebeatSecurityContext:
+  allowPrivilegeEscalation: false
+
+securityContext: 
+  privileged: true
+
+serviceMon:
+  name: crabserver-mon
+  port: 18720
+  protocol: TCP
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+
+resources: 
+  limits:
+    cpu: 1500m
+    memory: 3Gi
+  requests:
+    cpu: 200m
+    memory: 256Mi
+
+
+deamonset:
+  name: filebeat
+  path: docker.elastic.co/beats/filebeat:8.5.1
+  policy: IfNotPresent
+  args:
+  - bash
+  - -c
+  - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e
+  env: 
+  - name: MY_NODE_NAME
+    valueFrom:
+      fieldRef:
+        fieldPath: spec.nodeName
+  - name: NODE_NAME
+    valueFrom:
+      fieldRef:
+        fieldPath: spec.nodeName
+  - name: MY_NAMESPACE
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.namespace
+  resources:
+    requests:
+      memory: "128Mi"
+      cpu: "200m"
+    limits: 
+      cpu: "1"
+      memory: "1Gi"
+
+
+livenessProbe:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 60
+  timeoutSeconds: 60
+
+readinessProbe:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  periodSeconds: 60  
+  timeoutSeconds: 60
+
+livenessProbePreProd:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 60
+  timeoutSeconds: 60
+
+readinessProbePreProd:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  periodSeconds: 60  
+  timeoutSeconds: 60
+
+livenessProbeTest:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 60
+  timeoutSeconds: 60
+
+readinessProbeTest:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  periodSeconds: 60  
+  timeoutSeconds: 60
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

From edd265552c865ab92f503a5168b01abd04d44cc9 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 21 Feb 2024 18:47:44 +0100
Subject: [PATCH 07/27] Update Chart.yaml

---
 helm/crabserver/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index fe64ef13c..0f0558217 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.6
+version: 0.1.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

From 8edb94c3b7009d41e2000fcff4a19fd660fc9940 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:04:03 +0100
Subject: [PATCH 08/27] Update Chart.yaml

---
 helm/crabserver/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index 0f0558217..d507f2f76 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.7
+version: 0.1.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

From 4f163c0d262197452f9ce97171fe1c1b1e009560 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 12:43:28 +0100
Subject: [PATCH 09/27] Fix broken mount  points and paths

---
 docker/pypi/wmagent-mariadb/Dockerfile        | 19 ++++++++++---------
 .../wmagent-mariadb/mariadb-docker-run.sh     | 17 ++++++++---------
 docker/pypi/wmagent-mariadb/start-mysql.sh    |  8 ++++----
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 3a6125e7e..ae36bb893 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,10 +1,11 @@
-ARG TAG=11.1
-FROM mariadb:${TAG}
+ARG TAG=10.11
+ARG MDB_TAG=$TAG
+FROM mariadb:${MDB_TAG}
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
 
-ARG TAG
-ENV TAG=${TAG}
-RUN echo TAG=$TAG
+ARG MDB_TAG=$TAG
+ENV MDB_TAG=${MDB_TAG}
+RUN echo MDB_TAG=$MDB_TAG
 
 RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
 
@@ -20,7 +21,7 @@ ENV MARIADB_BASE_DIR=$MARIADB_ROOT_DIR/srv/mariadb
 ENV MARIADB_ADMIN_DIR=$MARIADB_ROOT_DIR/admin/wmagent
 ENV MARIADB_CERTS_DIR=$MARIADB_ROOT_DIR/certs
 
-ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$TAG
+ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$MDB_TAG
 ENV MARIADB_MANAGE_DIR=$MARIADB_CURRENT_DIR
 ENV MARIADB_AUTH_DIR=$MARIADB_CURRENT_DIR/auth/
 ENV MARIADB_INSTALL_DIR=$MARIADB_CURRENT_DIR/install
@@ -53,9 +54,9 @@ ADD manage ${MARIADB_MANAGE_DIR}/manage
 RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
 
 # The $MARIADB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
-# ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
+ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
 # RUN ln -s ${MARIADB_CONFIG_DIR}/my.cnf /opt/mariadb/etc/local.d/
-ADD my.cnf /etc/mysql/my.cnf
+# ADD my.cnf /etc/mysql/my.cnf
 
 
 ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
@@ -69,7 +70,7 @@ alias ll='ls -la --color=auto'
 alias manage=$MARIADB_MANAGE_DIR/manage
 
 # set MariaDB docker specific bash prompt:
-export PS1="(MariaDB-$TAG) [\u@\h:\W]\$ "
+export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ "
 EOF
 
 RUN chown -R ${USER} ${MARIADB_ROOT_DIR}
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index cf086e8b6..7449654ea 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -34,7 +34,7 @@ usage(){
 }
 
 PULL=false
-MARIADB_TAG=11.1
+MARIADB_TAG=latest
 
 
 ### Argument parsing:
@@ -59,11 +59,11 @@ HOST_MOUNT_DIR=/data/dockerMount
 
 [[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
 [[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
+# [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
 
-sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
+# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
 
 
 dockerOpts="
@@ -75,12 +75,11 @@ dockerOpts="
 --mount type=bind,source=/tmp,target=/tmp \
 --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs,target=/data/srv/mariadb/current/logs \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
 "
 
-
+# --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
 
 # mariadbOpts=$*
 # mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
@@ -95,5 +94,5 @@ $PULL && {
 
 echo "Starting the mariadb:$MARIADB_TAG docker container with the following parameters: $mariadbOpts"
 docker run $dockerOpts $mariadbOpts local/mariadb:$MARIADB_TAG && (
-    [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && sudo rm -f $HOST_MOUNT_DIR/srv/mariadb/current
-    sudo ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
+    [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
+    ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 97c181af5..8e307a6f9 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -9,10 +9,10 @@ mysqlRootPass=
 mysqlUser=cmst1
 mysqlUserPass=
 
-configDir=/data/dockerMount/srv/mysql/current/config
-dataDir=/data/dockerMount/srv/mysql/current/install/database
-logDir=/data/dockerMount/srv/mysql/current/logs
-socket=/data/dockerMount/srv/mysql/current/logs/mysql.sock
+configDir=/data/srv/mariadb/current/config
+dataDir=/data/srv/mariadb/current/install/database
+logDir=/data/srv/mariadb/current/logs
+socket=/data/srv/mariadb/current/logs/mysql.sock
 agentDb=wmagent
 
 echo -------------------------------------------------------------------------

From 8c2310a8dfdd3f27de1a6d381a2877834cee4fcc Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 12:44:54 +0100
Subject: [PATCH 10/27] Add mariadb-docker-build.sh

---
 .../wmagent-mariadb/mariadb-docker-build.sh   | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100755 docker/pypi/wmagent-mariadb/mariadb-docker-build.sh

diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
new file mode 100755
index 000000000..9e59c1d94
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+### This script is to be used for building a MariaDB docker imagge based on pypi
+### It depends on a single parameter MDB_TAG
+
+
+help(){
+    echo -e $*
+    cat <<EOF
+
+The MariaDB docker build script for Docker image creation based on pypi:
+
+Usage: mariadb-docker-build.sh -v <mariadb_tag>
+
+    -t <mariadb_tag>                The MariaDB version/tag to be used for the Docker image creation
+    -p <push_image>                 Push the image to registry.cern.ch
+    -l <latest_tag_toregistry>      Push the curernt tag also as latest to registry.cern.ch
+
+Example: ./mariadb-docker-build.sh -v 2.2.0.2
+
+EOF
+}
+
+usage(){
+    help $*
+    exit 1
+}
+
+MDB_TAG=None
+PUSH=false
+LATEST=false
+
+### Argument parsing:
+while getopts ":t:hpl" opt; do
+    case ${opt} in
+        t) MDB_TAG=$OPTARG ;;
+        p) PUSH=true ;;
+        l) LATEST=true ;;
+        h) help; exit $? ;;
+        \? )
+            msg="Invalid Option: -$OPTARG"
+            usage "$msg" ;;
+        : )
+            msg="Invalid Option: -$OPTARG requires an argument"
+            usage "$msg" ;;
+    esac
+done
+
+
+# NOTE: NO MDB_TAG validation is done in the current script. It is implemented at the install.sh
+
+dockerOpts=" --network=host --progress=plain --build-arg MDB_TAG=$MDB_TAG "
+
+docker build $dockerOpts -t local/mariadb:$MDB_TAG -t local/mariadb:latest  .
+
+$PUSH && {
+    docker login registry.cern.ch
+    docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    echo "Uploading image registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
+    docker push registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    $LATEST &&  {
+        docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:latest
+        echo "Uploading image registry.cern.ch/cmsweb/mariadb:latest"
+        docker push registry.cern.ch/cmsweb/mariadb:latest
+    }
+}

From 76f8cf5f0a4883f598625c13b1530a25da88ea3d Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 15:18:10 +0100
Subject: [PATCH 11/27] Switch to mariadb commands && Fix user initialiazation
 procedure

---
 docker/pypi/wmagent-mariadb/start-mysql.sh | 65 ++++++++++++----------
 1 file changed, 37 insertions(+), 28 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 8e307a6f9..3f37c50ec 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -3,62 +3,71 @@
 ### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
 ###       !!!! NOTHING MUST STAY HERE !!!!
 ###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
-###       NEED TO BE PERFORMED AT THE MYSQL DOCKER IMAGE
-mysqlRoot=root
-mysqlRootPass=
-mysqlUser=cmst1
-mysqlUserPass=
+###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
+mariadbRoot=root
+mariadbRootPass=FIXME
+mariadbUser=cmst1
+mariadbUserPass=FIXME
 
 configDir=/data/srv/mariadb/current/config
 dataDir=/data/srv/mariadb/current/install/database
 logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/logs/mysql.sock
+socket=/data/srv/mariadb/current/logs/mariadb.sock
 agentDb=wmagent
 
 echo -------------------------------------------------------------------------
-echo Stopping any previously running mysql server
-mysqladmin -u $mysqlRoot --password=$mysqlRootPass -h 127.0.0.1 shutdown
-# mysqladmin -u $mysqlRoot --password=$mysqlRootPass --socket=$socket shutdown
+echo Stopping any previously running mariadb server
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
+mariadb-admin -u $mariadbUser --socket=$socket shutdown
 echo
 
 
-echo -------------------------------------------------------------------------
-echo Installing system database
-mysql_install_db --datadir=$dataDir
-echo
+# echo -------------------------------------------------------------------------
+# echo Installing system database
+mariadb-install-db --datadir=$dataDir
+# echo
 
 
 echo -------------------------------------------------------------------------
 echo starting the server
-mysqld_safe --defaults-extra-file=$configDir/my.cnf \
+mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
       --datadir=$dataDir \
       --log-bin \
       --socket=$socket \
       --log-error=$logDir/error.log \
-      --pid-file=$logDir/mysqld.pid  & # > /dev/null 2>&1 < /dev/null &
+      --pid-file=$logDir/mariadbd.pid  & # > /dev/null 2>&1 < /dev/null &
 echo ...
 sleep 10
 echo
 
+
+
 echo -------------------------------------------------------------------------
-echo Securing mysqlRoot and removing temp databases
-mysqladmin -u $mysqlRoot password $mysqlRootPass --socket=$socket
-mysqladmin -u $mysqlRoot --password=$mysqlRootPass  -h 127.0.0.1 password $mysqlRootPass
-# mysql_secure_installation
+echo Securing $mariadbRoot and removing temp databases
+sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
 echo
 
 echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database '$agentDb'"
+echo Securing $mariadbUser and removing temp databases
+mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
+echo
 
 echo -------------------------------------------------------------------------
-echo creating new users
-# create a user - different than root and current unix user - and grant privileges
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost' IDENTIFIED BY '$mysqlUserPass'"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@localhost WITH GRANT OPTION"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1' IDENTIFIED BY '$mysqlUserPass'"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@127.0.0.1 WITH GRANT OPTION"
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
 
+# echo -------------------------------------------------------------------------
+# echo creating new users
+# # create a user - different than root and current unix user - and grant privileges
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
 
 echo -------------------------------------------------------------------------

From b0f528cccb6b74979b9d90fda34b8c4ce4848b1e Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 15:54:22 +0100
Subject: [PATCH 12/27] Fix 127.0.0.1 Access denied && Ramp up to v11.0 && Fix
 tag variable name.

Typo
---
 docker/pypi/wmagent-mariadb/Dockerfile        |  2 +-
 .../wmagent-mariadb/mariadb-docker-run.sh     | 32 +++++++++----------
 docker/pypi/wmagent-mariadb/start-mysql.sh    | 21 ++++++------
 3 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index ae36bb893..1f021470c 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,4 +1,4 @@
-ARG TAG=10.11
+ARG TAG=11.0
 ARG MDB_TAG=$TAG
 FROM mariadb:${MDB_TAG}
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index 7449654ea..b3b888c1f 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -34,13 +34,13 @@ usage(){
 }
 
 PULL=false
-MARIADB_TAG=latest
+MDB_TAG=latest
 
 
 ### Argument parsing:
 while getopts ":t:hp" opt; do
     case ${opt} in
-        t) MARIADB_TAG=$OPTARG ;;
+        t) MDB_TAG=$OPTARG ;;
         p) PULL=true ;;
         h) help; exit $? ;;
         : )
@@ -59,11 +59,11 @@ HOST_MOUNT_DIR=/data/dockerMount
 
 [[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
 [[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $?
-# [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
+# [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ;} || exit $?
 
-# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
+# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
 
 
 dockerOpts="
@@ -74,25 +74,25 @@ dockerOpts="
 --name=mariadb \
 --mount type=bind,source=/tmp,target=/tmp \
 --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs,target=/data/srv/mariadb/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs,target=/data/srv/mariadb/current/logs \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
 "
 
-# --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
+# --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config,target=/data/srv/mariadb/current/config \
 
 # mariadbOpts=$*
 # mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
 
 $PULL && {
-    echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG"
+    echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
     docker login registry.cern.ch
-    docker pull registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:$MARIADB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:latest
+    docker pull registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:$MDB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:latest
 }
 
-echo "Starting the mariadb:$MARIADB_TAG docker container with the following parameters: $mariadbOpts"
-docker run $dockerOpts $mariadbOpts local/mariadb:$MARIADB_TAG && (
+echo "Starting the mariadb:$MDB_TAG docker container with the following parameters: $mariadbOpts"
+docker run $dockerOpts $mariadbOpts local/mariadb:$MDB_TAG && (
     [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
-    ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
+    ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 3f37c50ec..8cb0f6eda 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -5,9 +5,9 @@
 ###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
 ###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
 mariadbRoot=root
-mariadbRootPass=FIXME
+mariadbRootPass=fixme
 mariadbUser=cmst1
-mariadbUserPass=FIXME
+mariadbUserPass=fixme
 
 configDir=/data/srv/mariadb/current/config
 dataDir=/data/srv/mariadb/current/install/database
@@ -62,12 +62,15 @@ echo creating agent databases
 echo "Installing WMAgent Database: $agentDb"
 mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
 
-# echo -------------------------------------------------------------------------
-# echo creating new users
-# # create a user - different than root and current unix user - and grant privileges
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
+echo
+
+echo -------------------------------------------------------------------------
+echo creating new users and setting grants
+# try to create a user different than root (if it does not already exist), and grant privileges
+# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
 
 echo -------------------------------------------------------------------------

From 36d74c638f6e04bb1409bcaac46168bd4a466760 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 18:21:27 +0100
Subject: [PATCH 13/27] Add manage

---
 docker/pypi/wmagent-mariadb/Dockerfile       |   2 +-
 docker/pypi/wmagent-mariadb/manage           | 255 ++++++-------------
 docker/pypi/wmagent-mariadb/start-mariadb.sh |  79 ++++++
 3 files changed, 159 insertions(+), 177 deletions(-)
 create mode 100755 docker/pypi/wmagent-mariadb/start-mariadb.sh

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 1f021470c..dc295ad5d 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -49,7 +49,7 @@ RUN ln -s $MARIADB_CURRENT_DIR $MARIADB_BASE_DIR/current
 
 # add necessary scripts
 ADD run.sh ${MARIADB_ROOT_DIR}/
-ADD start-mysql.sh ${MARIADB_ROOT_DIR}/
+ADD start-mariadb.sh ${MARIADB_ROOT_DIR}/
 ADD manage ${MARIADB_MANAGE_DIR}/manage
 RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
 
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index 372e8dd93..ca34b812e 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -1,178 +1,81 @@
 #!/bin/bash
 
-#### These are old mysql actions previously defined in the wmagent manage script
-#### Some of them need to be rewritten some of them must go away
-
-
-
-# #########################
-# #  MySQL                #
-# #########################
-
-# #
-# # first time startup routines for mysql
-# # pre gets called before startup, post called after it
-# init_mysql_db_pre(){
-#     echo "Installing the mysql database area..."
-#     mkdir -p $INSTALL_MYSQL/database
-#     mkdir -p $INSTALL_MYSQL/logs
-#     mysql_install_db --datadir=$INSTALL_MYSQL/database
-# }
-# init_mysql_db_post(){
-#     #install the WMAgent stuff
-#     echo "Installing the mysql schema..."
-#     load_secrets_file;
-#     local TIMEOUT=0;
-#     while [ ! -e $MYSQL_SOCK ]
-#     do
-#         sleep 2;
-#         TIMEOUT=$(($TIMEOUT+2))
-#         if [ $TIMEOUT -ge 300 ]; then
-#             echo "ERROR: Timeout waiting for mysqld to start."
-#             exit 1;
-#         fi
-#     done
-#     echo "Socket file exists, proceeding with schema install..."
-
-#     inited_mysql;
-
-#     # create a user - different than root and current unix user - and grant privileges
-#     if [ "$MYSQL_USER" != "$USER" ]; then
-#         mysql -u $USER --socket=$MYSQL_SOCK --execute "CREATE USER '${MYSQL_USER}'@'localhost'"
-#         mysql -u $USER --socket=$MYSQL_SOCK --execute "GRANT ALL ON *.* TO $MYSQL_USER@localhost WITH GRANT OPTION"
-#     fi
-
-#     # create databases for agent
-#     if [ $USING_AG -eq 1 ]; then
-#         echo "Installing WMAgent Database: ${MYSQL_DATABASE_AG}"
-#         mysql -u $USER --socket=$MYSQL_SOCK --execute "create database ${MYSQL_DATABASE_AG}"
-#     fi
-# }
-
-# status_of_mysql(){
-#     load_secrets_file;
-#     if [ "x$MYSQL_USER" == "x" ]; then
-# 	echo "Not using MySQL..."
-# 	exit 1;
-#     fi
-
-#     echo "+ Status of MySQL"
-#     if [ ! -e $INSTALL_MYSQL/logs/mysqld.pid ]; then
-#         echo "++ MySQL process file not found"
-#         return
-#     fi
-#     local MYSQL_PID=`cat $INSTALL_MYSQL/logs/mysqld.pid`
-#     kill -0 $MYSQL_PID;
-#     local MYSQL_STATUS=$?
-#     if [ $MYSQL_STATUS -eq 0 ]; then
-#         echo "++ MYSQL running with process: $MYSQL_PID";
-#     else
-#         echo "++ MYSQL process not running"
-#     fi
-
-#     echo "++" `mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK status`
-# }
-
-# #
-# # Main startup method for MySQL.
-# # Checks for initialisation
-# start_mysql(){
-#     load_secrets_file;
-#     if [ "x$MYSQL_USER" == "x" ]; then
-# 	    echo "Not using MySQL..."
-# 	    exit 1;
-#     fi
-
-#     echo "Starting mysql..."
-
-#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
-# 	    echo "MySQL has not been initialised... running pre initialisation";
-# 	    init_mysql_db_pre;
-#     fi
-
-#     # Start mysqld to install the database schemas
-#     #
-#     echo "starting mysqld_safe..."
-#     nohup mysqld_safe --defaults-extra-file=$CONFIG_MYSQL/my.cnf \
-#                       --datadir=$INSTALL_MYSQL/database \
-#                       --log-bin \
-#                       --socket=$MYSQL_SOCK \
-#                       --log-error=$INSTALL_MYSQL/logs/error.log \
-#                       --pid-file=$INSTALL_MYSQL/logs/mysqld.pid > /dev/null 2>&1 < /dev/null &
-#     local TIMEOUT=0;
-#     echo "Checking MySQL Socket file exists..."
-#     while [ ! -e $MYSQL_SOCK ]
-#       do
-#       sleep 2;
-#       TIMEOUT=$(($TIMEOUT+2))
-#       if [ $TIMEOUT -ge 300 ]; then
-# 	    echo "ERROR: Timeout waiting for mysqld to start."
-# 	    exit 1;
-#       fi
-#     done
-#     echo "Socket file exists: $MYSQL_SOCK"
-
-#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
-#         echo "MySQL has not been initialised... running post initialisation";
-#         init_mysql_db_post;
-#     fi
-#     echo "Checking Server connection..."
-#     mysql -u $USER --socket=$MYSQL_SOCK --execute "SHOW GLOBAL STATUS" > /dev/null;
-#     if [ $? -ne 0 ]; then
-# 	    echo "ERROR: checking mysql database is running, failed to execute SHOW GLOBAL STATUS"
-# 	    exit 1
-#     fi
-#    echo "Connection OK"
-# }
-
-# #
-# # stop MySQL
-# #
-# stop_mysql(){
-#     load_secrets_file;
-#     if [ "x$MYSQL_USER" == "x" ]; then
-# 	echo "Not using MySQL..."
-# 	exit 1;
-#     fi
-
-#     echo "stopping mysql..."
-#     mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK shutdown &
-#     wait $!
-#     echo "Making sure the MySQL socket file is removed..."
-#     local TIMEOUT=0;
-#     while [ -e $MYSQL_SOCK ]
-#       do
-#       sleep 2;
-#       TIMEOUT=$(($TIMEOUT+2))
-#       if [ $TIMEOUT -ge 300 ]; then
-# 	  echo "ERROR: Timeout waiting for mysqld to shutdown."
-# 	  echo "ERROR: Socket file still exists: $MYSQL_SOCK"
-# 	  exit 1;
-#       fi
-#     done
-#     echo "MySQL is shutdown."
-# }
-
-# #
-
-# case $1 in
-#   status)
-#     status ;;
-#   start-mysql)
-#     start_mysql;;
-#   stop-mysql)
-#     stop_mysql;;
-#   clean-mysql)
-#      clean_mysql;;
-#   db-prompt)
-#      db_prompt $@;;
-#   mysql-prompt)
-#      db_prompt $@;;
-#   help)
-#     help ;;
-#   version)
-#     echo "Management script for WMAgent. No idea what version, at least 2 though" ;;
-#   * )
-#     echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
-#     exit 1 ;;
-# esac
+mariadbRoot=root
+mariadbRootPass=fixme
+mariadbUser=cmst1
+mariadbUserPass=fixme
+
+configDir=/data/srv/mariadb/current/config
+dataDir=/data/srv/mariadb/current/install/database
+logDir=/data/srv/mariadb/current/logs
+socket=/data/srv/mariadb/current/mariadb.sock
+agentDb=wmagent
+
+help(){
+    echo -e $*
+    cat <<EOF
+
+The manage script of the MariaDB docker image for WMAgent
+
+Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt | version
+
+EOF
+}
+
+usage(){
+    help $*
+    exit 1
+}
+
+status(){
+    mariadb-admin --socket=$socket status
+}
+
+start_mariadb(){
+    echo starting MariaDB server
+    mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
+                  --datadir=$dataDir \
+                  --log-bin \
+                  --socket=$socket \
+                  --log-error=$logDir/error.log \
+                  --pid-file=$logDir/mariadbd.pid  &
+    echo ...
+    sleep 10
+    echo status
+}
+
+stop_mariadb(){
+    mariadb-admin --socket=$socket shutdown
+}
+
+db_prompt(){
+    mariadb --socket=$socket --database=$wmaDBName --pager='less -SFX'
+}
+
+clean_mariadb(){
+    db_prompt "drop database $wmaDBName"
+}
+
+version(){
+    mariadb-admin --socket=$socket version
+}
+
+case $1 in
+  status)
+      status ;;
+  start-mariadb)
+      start_mariadb;;
+  stop-mariadb)
+      stop_mariadb;;
+  clean-mariadb)
+      clean_mariadb;;
+  db-prompt)
+      shift
+      db_prompt $@;;
+  version)
+      version ;;
+  help)
+      help ;;
+  *)
+      usage ;;
+esac
diff --git a/docker/pypi/wmagent-mariadb/start-mariadb.sh b/docker/pypi/wmagent-mariadb/start-mariadb.sh
new file mode 100755
index 000000000..b60a6ac9e
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/start-mariadb.sh
@@ -0,0 +1,79 @@
+#/bin/bash
+
+### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
+###       !!!! NOTHING MUST STAY HERE !!!!
+###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
+###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
+mariadbRoot=root
+mariadbRootPass=fixme
+mariadbUser=cmst1
+mariadbUserPass=fixme
+
+configDir=/data/srv/mariadb/current/config
+dataDir=/data/srv/mariadb/current/install/database
+logDir=/data/srv/mariadb/current/logs
+socket=/data/srv/mariadb/current/mariadb.sock
+agentDb=wmagent
+
+echo -------------------------------------------------------------------------
+echo Stopping any previously running mariadb server
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
+mariadb-admin -u $mariadbUser --socket=$socket shutdown
+echo
+
+
+# echo -------------------------------------------------------------------------
+# echo Installing system database
+mariadb-install-db --datadir=$dataDir
+# echo
+
+
+
+manage start-mariadb
+
+# echo -------------------------------------------------------------------------
+# echo starting the server
+# mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
+#       --datadir=$dataDir \
+#       --log-bin \
+#       --socket=$socket \
+#       --log-error=$logDir/error.log \
+#       --pid-file=$logDir/mariadbd.pid  &
+# echo ...
+# sleep 10
+# echo
+
+
+
+echo -------------------------------------------------------------------------
+echo Securing $mariadbRoot and removing temp databases
+sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
+echo
+
+echo -------------------------------------------------------------------------
+echo Securing $mariadbUser and removing temp databases
+mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
+echo
+
+echo -------------------------------------------------------------------------
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
+
+echo
+
+echo -------------------------------------------------------------------------
+echo creating new users and setting grants
+# try to create a user different than root (if it does not already exist), and grant privileges
+# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
+
+echo -------------------------------------------------------------------------

From 85b8a76f6fbde584fd546268804f6551ac17bcd1 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Sat, 24 Feb 2024 12:59:49 +0100
Subject: [PATCH 14/27] Properly load WMAgent.secrets and MariaDB.secrets files
 && Delete mariadb-start.sh and mysql-start.sh

---
 docker/pypi/wmagent-mariadb/Dockerfile        |  70 +++----
 docker/pypi/wmagent-mariadb/manage            | 182 ++++++++++++++----
 .../wmagent-mariadb/mariadb-docker-run.sh     |   2 +-
 docker/pypi/wmagent-mariadb/run.sh            |   8 +-
 docker/pypi/wmagent-mariadb/start-mariadb.sh  |  79 --------
 docker/pypi/wmagent-mariadb/start-mysql.sh    |  76 --------
 6 files changed, 184 insertions(+), 233 deletions(-)
 delete mode 100755 docker/pypi/wmagent-mariadb/start-mariadb.sh
 delete mode 100755 docker/pypi/wmagent-mariadb/start-mysql.sh

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index dc295ad5d..7a946a56d 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -9,29 +9,33 @@ RUN echo MDB_TAG=$MDB_TAG
 
 RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
 
-# # Install some debugging tools
+# Install some debugging tools
 RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
 
 ENV USER=cmst1
-# ENV MARIADB_PORT=
+# ENV MDB_PORT=
 ENV UID=31961
-ENV MARIADB_ROOT_DIR=/data
-
-ENV MARIADB_BASE_DIR=$MARIADB_ROOT_DIR/srv/mariadb
-ENV MARIADB_ADMIN_DIR=$MARIADB_ROOT_DIR/admin/wmagent
-ENV MARIADB_CERTS_DIR=$MARIADB_ROOT_DIR/certs
-
-ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$MDB_TAG
-ENV MARIADB_MANAGE_DIR=$MARIADB_CURRENT_DIR
-ENV MARIADB_AUTH_DIR=$MARIADB_CURRENT_DIR/auth/
-ENV MARIADB_INSTALL_DIR=$MARIADB_CURRENT_DIR/install
-ENV MARIADB_STATE_DIR=$MARIADB_CURRENT_DIR/state
-ENV MARIADB_DATABASE_DIR=$MARIADB_INSTALL_DIR/database
-ENV MARIADB_CONFIG_DIR=$MARIADB_CURRENT_DIR/config
-ENV MARIADB_LOG_DIR=$MARIADB_CURRENT_DIR/logs
-ENV MARIADB_DEPLOY_DIR=/usr/local
-ENV MARIADB_ENV_FILE=$MARIADB_DEPLOY_DIR/deploy/env.sh
-ENV MARIADB_SECRETS_FILE=$MARIADB_ADMIN_DIR/MariaDB.secrets
+ENV MDB_ROOT_DIR=/data
+
+ENV MDB_BASE_DIR=$MDB_ROOT_DIR/srv/mariadb
+ENV MDB_ADMIN_DIR=$MDB_ROOT_DIR/admin/mariadb
+ENV WMA_ADMIN_DIR=$MDB_ROOT_DIR/admin/wmagent
+ENV MDB_CERTS_DIR=$MDB_ROOT_DIR/certs
+
+ENV MDB_CURRENT_DIR=$MDB_BASE_DIR/$MDB_TAG
+ENV MDB_MANAGE_DIR=$MDB_CURRENT_DIR
+ENV MDB_AUTH_DIR=$MDB_CURRENT_DIR/auth/
+ENV MDB_INSTALL_DIR=$MDB_CURRENT_DIR/install
+ENV MDB_STATE_DIR=$MDB_CURRENT_DIR/state
+ENV MDB_DATABASE_DIR=$MDB_INSTALL_DIR/database
+ENV MDB_CONFIG_DIR=$MDB_CURRENT_DIR/config
+ENV MDB_LOG_DIR=$MDB_CURRENT_DIR/logs
+ENV MDB_DEPLOY_DIR=/usr/local
+ENV MDB_ENV_FILE=$MDB_DEPLOY_DIR/deploy/env.sh
+ENV MDB_SOCKET_FILE=$MDB_CURRENT_DIR/mariadb.sock
+ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
+ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
+ENV WMA_DATABASE=wmagent
 
 
 RUN useradd -u $UID -m $USER
@@ -40,24 +44,20 @@ RUN useradd -u $UID -m $USER
 RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 # start the setup
-RUN mkdir -p $MARIADB_ROOT_DIR
+RUN mkdir -p $MDB_ROOT_DIR
 
-ENV PATH="${MARIADB_ROOT_DIR}:${PATH}"
+ENV PATH="${MDB_ROOT_DIR}:${PATH}"
 
-RUN mkdir -p $MARIADB_CURRENT_DIR $MARIADB_CONFIG_DIR $MARIADB_MANAGE_DIR $MARIADB_LOG_DIR $MARIADB_DATABASE_DIR $MARIADB_STATE_DIR $MARIADB_AUTH_DIR
-RUN ln -s $MARIADB_CURRENT_DIR $MARIADB_BASE_DIR/current
+RUN mkdir -p $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
+RUN ln -s $MDB_CURRENT_DIR $MDB_BASE_DIR/current
 
 # add necessary scripts
-ADD run.sh ${MARIADB_ROOT_DIR}/
-ADD start-mariadb.sh ${MARIADB_ROOT_DIR}/
-ADD manage ${MARIADB_MANAGE_DIR}/manage
-RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
-
-# The $MARIADB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
-ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
-# RUN ln -s ${MARIADB_CONFIG_DIR}/my.cnf /opt/mariadb/etc/local.d/
-# ADD my.cnf /etc/mysql/my.cnf
+ADD run.sh ${MDB_ROOT_DIR}/
+ADD manage ${MDB_MANAGE_DIR}/manage
+RUN ln -s ${MDB_MANAGE_DIR}/manage ${MDB_ROOT_DIR}/manage
 
+# The $MDB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
+ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
 ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
 
@@ -67,15 +67,15 @@ alias lll="ls -lathr"
 alias ls="ls --color=auto"
 alias ll='ls -la --color=auto'
 
-alias manage=$MARIADB_MANAGE_DIR/manage
+alias manage=$MDB_MANAGE_DIR/manage
 
 # set MariaDB docker specific bash prompt:
 export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ "
 EOF
 
-RUN chown -R ${USER} ${MARIADB_ROOT_DIR}
+RUN chown -R ${USER} ${MDB_ROOT_DIR}
 
 # setup final environment
 USER $USER
-WORKDIR $MARIADB_ROOT_DIR
+WORKDIR $MDB_ROOT_DIR
 ENTRYPOINT ["./run.sh"]
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index ca34b812e..25281a923 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -1,16 +1,5 @@
 #!/bin/bash
 
-mariadbRoot=root
-mariadbRootPass=fixme
-mariadbUser=cmst1
-mariadbUserPass=fixme
-
-configDir=/data/srv/mariadb/current/config
-dataDir=/data/srv/mariadb/current/install/database
-logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/mariadb.sock
-agentDb=wmagent
-
 help(){
     echo -e $*
     cat <<EOF
@@ -27,29 +16,67 @@ usage(){
     exit 1
 }
 
+_load_secrets(){
+    # Auxiliary function to parse WMAgent.secrets or MariaDB.secrets files
+    # and load a set of variables from them
+    # :param $1: Path to WMAgent.secrets or file
+    # :param $2: String with variable names to be checked
+    # :retrun:   Error value if one or more values have been left unset in the secrets file
+    local errVal=0
+    local value=""
+    local secretsFile=$1
+    local varsToLoad=$2
+
+    [[ -f $secretsFile ]] || {
+        echo "$FUNCNAME: ERROR: Password file: $secretsFile does not exist"
+        echo "$FUNCNAME: ERROR: Either set WMA_SECRETS_FILE environment variable to a valid file or check that $HOME/WMAgent.secrets exists"
+        return $(false)
+    }
+
+    # All variables need to be fetched in lowercase through: ${varName,,}
+    local badValuesReg="(update-me|updateme|<update-me>|<updateme>|fix-me|fixme|<fix-me>|<fixme>|^$)"
+    for varName in $varsToLoad
+    do
+        value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | awk -F\= '{print $2}'`
+        [[ ${value,,} =~ $badValuesReg ]] && { echo "$FUNCNAME: Bad value for: $varName=$value"; let errVal+=1 ;}
+    done
+
+    for varName in $varsToLoad
+    do
+        value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | sed "s/ *$varName=//"`
+        eval $varName=$value
+        [[ -n $varName ]] || { echo "$FUNCNAME: Empty value for: $varName=$value"; let errVal+=1 ;}
+    done
+    return $errVal
+}
+
+
 status(){
-    mariadb-admin --socket=$socket status
+    mariadb-admin --socket=$MDB_SOCKET_FILE version
+    echo
+    mariadb-admin --socket=$MDB_SOCKET_FILE status
+    echo
 }
 
 start_mariadb(){
     echo starting MariaDB server
-    mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
-                  --datadir=$dataDir \
+    mariadbd-safe --defaults-extra-file=$MDB_CONFIG_DIR/my.cnf \
+                  --datadir=$MDB_DATABASE_DIR \
                   --log-bin \
-                  --socket=$socket \
-                  --log-error=$logDir/error.log \
-                  --pid-file=$logDir/mariadbd.pid  &
+                  --socket=$MDB_SOCKET_FILE \
+                  --log-error=$MDB_LOG_DIR/error.log \
+                  --pid-file=$MDB_LOG_DIR/mariadbd.pid  &
     echo ...
     sleep 10
-    echo status
+    status
 }
 
 stop_mariadb(){
-    mariadb-admin --socket=$socket shutdown
+    mariadb-admin --socket=$MDB_SOCKET_FILE shutdown
 }
 
 db_prompt(){
-    mariadb --socket=$socket --database=$wmaDBName --pager='less -SFX'
+    mariadb --socket=$MDB_SOCKET_FILE --database=$wmaDBName --pager='less -SFX'
 }
 
 clean_mariadb(){
@@ -57,25 +84,104 @@ clean_mariadb(){
 }
 
 version(){
-    mariadb-admin --socket=$socket version
+    mariadb-admin --socket=$MDB_SOCKET_FILE version
+}
+
+
+init_mariadb(){
+    # The function to set/check initial database configurations and user preveleges
+    [[ $USER == $MDB_ROOT ]] || {
+        echo "ERROR: The current user does not match the MariaDB root user from $MDB_SECRETS_FILE."
+        echo "ERROR: Canot continue. Exit..."
+        exit 1
+    }
+
+    echo -------------------------------------------------------------------------
+    echo Stopping any previously running mariadb server
+    mariadb-admin -u $MDB_USER --socket=$MDB_SOCKET_FILE shutdown
+    echo
+
+    echo -------------------------------------------------------------------------
+    echo Trying to install system database if it is not present already
+
+    errMsg=$(mariadb-install-db --skip-test-db --datadir=$MDB_DATABASE_DIR)
+    err=$?
+
+    if [[ $err -ne 0 ]]; then
+        echo "ERROR: Could not create system and user databases."
+        echo "ERROR: $errMsg"
+        exit $err
+    elif echo $errMsg|grep -i "exists" ; then
+        echo "WARNING: System and user databases already exist. NOT trying to create them."
+        return $err
+    fi
+
+    echo -------------------------------------------------------------------------
+    echo Starting MariaDB server
+    start_mariadb
+
+    echo -------------------------------------------------------------------------
+    echo Securing MariaDB Root users
+    sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
+    mariadb-admin -u $MDB_ROOT password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
+    echo
+
+    echo -------------------------------------------------------------------------
+    echo Creating WMAgent databases
+    echo "Installing WMAgent Database: $WMA_DATABASE"
+    mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "create database $WMA_DATABASE"
+    echo
+
+    echo -------------------------------------------------------------------------
+    echo Creating WMAgent user and setting grants
+    # try to create a user different than root (if it does not already exist), and grant privileges
+    # we need ${MDB_USER}'@'127.0.0.1 user in paralel to ${MDB_USER}'@'localhost
+
+    if [[ $MDB_USER == $MDB_ROOT ]]; then
+        echo "WARNING: WMAgent user set is the same as the MariaDB Root user. You must configure a different one!"
+        echo "WARNING: NOT creating WMAgent users and NOT granting priveleges to $WMA_DATABASE database"
+        return 1
+    else
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "CREATE USER '${MDB_USER}'@'localhost' IDENTIFIED BY '$MDB_PASS'"
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "GRANT ALL ON $WMA_DATABASE.* TO $MDB_USER@localhost"
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "CREATE USER '${MDB_USER}'@'127.0.0.1' IDENTIFIED BY '$MDB_PASS'"
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "GRANT ALL ON $WMA_DATABASE.* TO $MDB_USER@127.0.0.1"
+    fi
+    echo -------------------------------------------------------------------------
 }
 
+
+_load_secrets $MDB_SECRETS_FILE "MDB_ROOT MDB_ROOTPASS" || {
+    err=$?
+    echo "ERROR: Could not properly load root password for MariaDB"
+    exit $err
+}
+
+_load_secrets $WMA_SECRETS_FILE "MDB_USER MDB_PASS" || {
+    err=$?
+    echo "ERROR: Could not properly load WMAgent User password for MariaDB"
+    exit $err
+}
+
+
 case $1 in
-  status)
-      status ;;
-  start-mariadb)
-      start_mariadb;;
-  stop-mariadb)
-      stop_mariadb;;
-  clean-mariadb)
-      clean_mariadb;;
-  db-prompt)
-      shift
-      db_prompt $@;;
-  version)
-      version ;;
-  help)
-      help ;;
-  *)
-      usage ;;
+    status)
+        status ;;
+    init-mariadb)
+        init_mariadb;;
+    start-mariadb)
+        start_mariadb;;
+    stop-mariadb)
+        stop_mariadb;;
+    clean-mariadb)
+        clean_mariadb;;
+    db-prompt)
+        shift
+        db_prompt $@;;
+    version)
+        version ;;
+    help)
+        help ;;
+    *)
+        usage ;;
 esac
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index b3b888c1f..57052bc29 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -65,7 +65,6 @@ HOST_MOUNT_DIR=/data/dockerMount
 
 # sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
 
-
 dockerOpts="
 --detach \
 --network=host \
@@ -76,6 +75,7 @@ dockerOpts="
 --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs,target=/data/srv/mariadb/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/admin/mariadb,target=/data/admin/mariadb/ \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
 "
 
diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh
index 62920288d..82965cc9f 100755
--- a/docker/pypi/wmagent-mariadb/run.sh
+++ b/docker/pypi/wmagent-mariadb/run.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-echo "start sleeping....zzz"
-while true; do sleep 10; done
 
+manage init-mariadb 2>&1 | tee -a run.log
+manage start-mariadb 2>&1 | tee -a run.log
 
-# # start the service
-# manage start
+echo "Start sleeping....zzz"
+while true; do sleep 10; done
diff --git a/docker/pypi/wmagent-mariadb/start-mariadb.sh b/docker/pypi/wmagent-mariadb/start-mariadb.sh
deleted file mode 100755
index b60a6ac9e..000000000
--- a/docker/pypi/wmagent-mariadb/start-mariadb.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#/bin/bash
-
-### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
-###       !!!! NOTHING MUST STAY HERE !!!!
-###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
-###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
-mariadbRoot=root
-mariadbRootPass=fixme
-mariadbUser=cmst1
-mariadbUserPass=fixme
-
-configDir=/data/srv/mariadb/current/config
-dataDir=/data/srv/mariadb/current/install/database
-logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/mariadb.sock
-agentDb=wmagent
-
-echo -------------------------------------------------------------------------
-echo Stopping any previously running mariadb server
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
-mariadb-admin -u $mariadbUser --socket=$socket shutdown
-echo
-
-
-# echo -------------------------------------------------------------------------
-# echo Installing system database
-mariadb-install-db --datadir=$dataDir
-# echo
-
-
-
-manage start-mariadb
-
-# echo -------------------------------------------------------------------------
-# echo starting the server
-# mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
-#       --datadir=$dataDir \
-#       --log-bin \
-#       --socket=$socket \
-#       --log-error=$logDir/error.log \
-#       --pid-file=$logDir/mariadbd.pid  &
-# echo ...
-# sleep 10
-# echo
-
-
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbRoot and removing temp databases
-sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbUser and removing temp databases
-mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
-
-echo
-
-echo -------------------------------------------------------------------------
-echo creating new users and setting grants
-# try to create a user different than root (if it does not already exist), and grant privileges
-# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
-
-echo -------------------------------------------------------------------------
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
deleted file mode 100755
index 8cb0f6eda..000000000
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#/bin/bash
-
-### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
-###       !!!! NOTHING MUST STAY HERE !!!!
-###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
-###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
-mariadbRoot=root
-mariadbRootPass=fixme
-mariadbUser=cmst1
-mariadbUserPass=fixme
-
-configDir=/data/srv/mariadb/current/config
-dataDir=/data/srv/mariadb/current/install/database
-logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/logs/mariadb.sock
-agentDb=wmagent
-
-echo -------------------------------------------------------------------------
-echo Stopping any previously running mariadb server
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
-mariadb-admin -u $mariadbUser --socket=$socket shutdown
-echo
-
-
-# echo -------------------------------------------------------------------------
-# echo Installing system database
-mariadb-install-db --datadir=$dataDir
-# echo
-
-
-echo -------------------------------------------------------------------------
-echo starting the server
-mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
-      --datadir=$dataDir \
-      --log-bin \
-      --socket=$socket \
-      --log-error=$logDir/error.log \
-      --pid-file=$logDir/mariadbd.pid  & # > /dev/null 2>&1 < /dev/null &
-echo ...
-sleep 10
-echo
-
-
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbRoot and removing temp databases
-sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbUser and removing temp databases
-mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
-
-echo
-
-echo -------------------------------------------------------------------------
-echo creating new users and setting grants
-# try to create a user different than root (if it does not already exist), and grant privileges
-# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
-
-echo -------------------------------------------------------------------------

From 5f3ab241562d403d1d2da1c9ba963a3a079b4736 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Mon, 26 Feb 2024 19:25:22 +0100
Subject: [PATCH 15/27] Review comments

---
 docker/pypi/wmagent-mariadb/Dockerfile | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 7a946a56d..583d90b8a 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,16 +1,13 @@
-ARG TAG=11.0
-ARG MDB_TAG=$TAG
-FROM mariadb:${MDB_TAG}
+ARG MDB_TAG=10.0
+FROM mariadb:$MDB_TAG
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
 
-ARG MDB_TAG=$TAG
-ENV MDB_TAG=${MDB_TAG}
+ARG MDB_TAG
+ENV MDB_TAG=$MDB_TAG
 RUN echo MDB_TAG=$MDB_TAG
 
-RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
-
-# Install some debugging tools
-RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
+RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip \
+    hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
 
 ENV USER=cmst1
 # ENV MDB_PORT=
@@ -37,7 +34,6 @@ ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
 ENV WMA_DATABASE=wmagent
 
-
 RUN useradd -u $UID -m $USER
 
 # add user to sudoers file
@@ -46,8 +42,6 @@ RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 # start the setup
 RUN mkdir -p $MDB_ROOT_DIR
 
-ENV PATH="${MDB_ROOT_DIR}:${PATH}"
-
 RUN mkdir -p $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
 RUN ln -s $MDB_CURRENT_DIR $MDB_BASE_DIR/current
 
@@ -59,7 +53,7 @@ RUN ln -s ${MDB_MANAGE_DIR}/manage ${MDB_ROOT_DIR}/manage
 # The $MDB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
 ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
-ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
+ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}"
 
 RUN <<EOF cat >> /home/${USER}/.bashrc
 

From a0a404da78bf7babbcc525c89e38893299e35930 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Tue, 27 Feb 2024 11:28:19 +0100
Subject: [PATCH 16/27] Add extra login name checks for
 mariadb-docker-push/run.sh && Review comments.

---
 docker/pypi/wmagent-mariadb/Dockerfile        | 14 ++++----
 .../wmagent-mariadb/mariadb-docker-build.sh   | 32 +++++++++++++------
 .../wmagent-mariadb/mariadb-docker-run.sh     | 15 ++++++---
 3 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 583d90b8a..01720cedc 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -34,15 +34,12 @@ ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
 ENV WMA_DATABASE=wmagent
 
-RUN useradd -u $UID -m $USER
-
-# add user to sudoers file
-RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+# create the system user to run the database and add it to the sudoers file
+RUN useradd -u $UID -m $USER && echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 # start the setup
-RUN mkdir -p $MDB_ROOT_DIR
-
-RUN mkdir -p $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
+RUN mkdir -p $MDB_ROOT_DIR $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR \
+    $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
 RUN ln -s $MDB_CURRENT_DIR $MDB_BASE_DIR/current
 
 # add necessary scripts
@@ -50,7 +47,8 @@ ADD run.sh ${MDB_ROOT_DIR}/
 ADD manage ${MDB_MANAGE_DIR}/manage
 RUN ln -s ${MDB_MANAGE_DIR}/manage ${MDB_ROOT_DIR}/manage
 
-# The $MDB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
+# The $MDB_CONFIG_DIR is NOT to be mounted from the host
+# and the my.cnf file is going to be accessible only from the container
 ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
 ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}"
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
index 9e59c1d94..50710fc3e 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -47,20 +47,32 @@ while getopts ":t:hpl" opt; do
 done
 
 
-# NOTE: NO MDB_TAG validation is done in the current script. It is implemented at the install.sh
-
-dockerOpts=" --network=host --progress=plain --build-arg MDB_TAG=$MDB_TAG "
+# NOTE: NO MDB_TAG validation is done in the current script. The proper tag
+#       to be used should be taken from: https://mariadb.org/mariadb/all-releases/
+dockerOpts=" --network host --progress=plain --build-arg MDB_TAG=$MDB_TAG "
 
 docker build $dockerOpts -t local/mariadb:$MDB_TAG -t local/mariadb:latest  .
 
 $PUSH && {
-    docker login registry.cern.ch
-    docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:$MDB_TAG
-    echo "Uploading image registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
-    docker push registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    # For security reasons we check if the login name and the current user match.
+    # If they do not, abort the execution and push nothing to registry.cern.ch.
+    loginUser=`logname`
+    currUser=`id -un`
+    registry=registry.cern.ch
+    [[ $loginUser == $currUser ]] || {
+        echo "ERROR: The CURRENT and the LOGIN users do not match!"
+        echo "ERROR: You MUST connect to $registry with your login user rather than with $currUser"
+        exit 1
+    }
+    echo "Connecting to $registry with Username: $loginUser"
+    docker login -u $loginUser $registry
+    docker tag local/mariadb:$MDB_TAG $registry/cmsweb/mariadb:$MDB_TAG
+    echo "Uploading image $registry/cmsweb/mariadb:$MDB_TAG"
+    docker push $registry/cmsweb/mariadb:$MDB_TAG
     $LATEST &&  {
-        docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:latest
-        echo "Uploading image registry.cern.ch/cmsweb/mariadb:latest"
-        docker push registry.cern.ch/cmsweb/mariadb:latest
+        docker tag local/mariadb:$MDB_TAG $registry/cmsweb/mariadb:latest
+        echo "Uploading image $registry/cmsweb/mariadb:latest"
+        docker push $registry/cmsweb/mariadb:latest
     }
+    docker logout $registry
 }
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index 57052bc29..79f8c0d2f 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -84,15 +84,20 @@ dockerOpts="
 # mariadbOpts=$*
 # mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
 
+registry=local
+repository=mariadb
+
 $PULL && {
+    registry=registry.cern.ch
+    project=cmsweb
+    repository=mariadb
     echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
-    docker login registry.cern.ch
-    docker pull registry.cern.ch/cmsweb/mariadb:$MDB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:$MDB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:latest
+    docker pull $registry/$project/$repository:$MDB_TAG
+    docker tag  $registry/$project/$repository:$MDB_TAG $registry/$repository:$MDB_TAG
+    docker tag  $registry/$project/$repository:$MDB_TAG $registry/$repository:latest
 }
 
 echo "Starting the mariadb:$MDB_TAG docker container with the following parameters: $mariadbOpts"
-docker run $dockerOpts $mariadbOpts local/mariadb:$MDB_TAG && (
+docker run $dockerOpts $mariadbOpts $registry/$repository:$MDB_TAG && (
     [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
     ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )

From 681a5fab12d3d61aede95da99b4d80807fd64419 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 28 Feb 2024 14:13:08 +0100
Subject: [PATCH 17/27] Resolve the user to run the database server at runtime

Set Default tag to 10.6.5
---
 docker/pypi/wmagent-mariadb/Dockerfile        | 32 +++++++++----------
 docker/pypi/wmagent-mariadb/manage            | 13 ++++----
 .../wmagent-mariadb/mariadb-docker-run.sh     | 11 +++----
 docker/pypi/wmagent-mariadb/run.sh            |  4 +--
 4 files changed, 27 insertions(+), 33 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 01720cedc..a390796fe 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,4 +1,4 @@
-ARG MDB_TAG=10.0
+ARG MDB_TAG=10.6.5
 FROM mariadb:$MDB_TAG
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
 
@@ -9,9 +9,7 @@ RUN echo MDB_TAG=$MDB_TAG
 RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip \
     hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
 
-ENV USER=cmst1
 # ENV MDB_PORT=
-ENV UID=31961
 ENV MDB_ROOT_DIR=/data
 
 ENV MDB_BASE_DIR=$MDB_ROOT_DIR/srv/mariadb
@@ -29,13 +27,14 @@ ENV MDB_CONFIG_DIR=$MDB_CURRENT_DIR/config
 ENV MDB_LOG_DIR=$MDB_CURRENT_DIR/logs
 ENV MDB_DEPLOY_DIR=/usr/local
 ENV MDB_ENV_FILE=$MDB_DEPLOY_DIR/deploy/env.sh
-ENV MDB_SOCKET_FILE=$MDB_CURRENT_DIR/mariadb.sock
+ENV MDB_SOCKET_FILE=/var/run/mysqld/mariadb.sock
 ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
 ENV WMA_DATABASE=wmagent
 
-# create the system user to run the database and add it to the sudoers file
-RUN useradd -u $UID -m $USER && echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+# create the system user to run the database
+RUN groupadd -g 1399 zh
+RUN useradd -u 31961  -g 1399 -G 999 -m cmst1
 
 # start the setup
 RUN mkdir -p $MDB_ROOT_DIR $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR \
@@ -53,21 +52,20 @@ ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
 ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}"
 
-RUN <<EOF cat >> /home/${USER}/.bashrc
-
-alias lll="ls -lathr"
-alias ls="ls --color=auto"
-alias ll='ls -la --color=auto'
-
+# set MariaDB docker specific bash prompt and manage alias for all users:
+RUN <<EOF cat >>/root/.bashrc
 alias manage=$MDB_MANAGE_DIR/manage
+export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$([[ \$(id -u) -eq 0 ]] && echo \# || echo \$) "
+EOF
 
-# set MariaDB docker specific bash prompt:
-export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ "
+RUN <<EOF cat >>/home/cmst1/.bashrc
+alias manage=$MDB_MANAGE_DIR/manage
+export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$([[ \$(id -u) -eq 0 ]] && echo \# || echo \$) "
 EOF
 
-RUN chown -R ${USER} ${MDB_ROOT_DIR}
+# RUN chown -R ${USER} ${MDB_ROOT_DIR}
 
 # setup final environment
-USER $USER
+# USER $USER
 WORKDIR $MDB_ROOT_DIR
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["./run.sh", "2>&1"]
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index 25281a923..3fc0ac8d3 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -54,8 +54,6 @@ _load_secrets(){
 status(){
     mariadb-admin --socket=$MDB_SOCKET_FILE version
     echo
-    mariadb-admin --socket=$MDB_SOCKET_FILE status
-    echo
 }
 
 start_mariadb(){
@@ -98,14 +96,15 @@ init_mariadb(){
 
     echo -------------------------------------------------------------------------
     echo Stopping any previously running mariadb server
-    mariadb-admin -u $MDB_USER --socket=$MDB_SOCKET_FILE shutdown
+    mariadb-admin -u $MDB_ROOT --socket=$MDB_SOCKET_FILE shutdown
     echo
 
     echo -------------------------------------------------------------------------
-    echo Trying to install system database if it is not present already
+    echo "Trying to install system database with user: $USER (if it is not already present)"
 
-    errMsg=$(mariadb-install-db --skip-test-db --datadir=$MDB_DATABASE_DIR)
+    errMsg=$(mariadb-install-db --skip-test-db --user=$USER --datadir=$MDB_DATABASE_DIR)
     err=$?
+    echo $errMsg
 
     if [[ $err -ne 0 ]]; then
         echo "ERROR: Could not create system and user databases."
@@ -121,8 +120,8 @@ init_mariadb(){
     start_mariadb
 
     echo -------------------------------------------------------------------------
-    echo Securing MariaDB Root users
-    sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
+    echo Securing MariaDB Root user
+    # sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     mariadb-admin -u $MDB_ROOT password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     echo
 
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index 79f8c0d2f..f512ea114 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -50,8 +50,8 @@ while getopts ":t:hp" opt; do
 done
 
 
-mariadbUser=cmst1
-mariadbOpts=" --user $mariadbUser"
+mariadbUser=`id -un`
+mariadbOpts=" --user $mariadbUser -e USER=$mariadbUser"
 
 # This is the root at the host only, it may differ from the root inside the container.
 # NOTE: this may be parametriesed, so that the container can run on a different mount point.
@@ -63,7 +63,7 @@ HOST_MOUNT_DIR=/data/dockerMount
 [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ;} || exit $?
 [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ;} || exit $?
 
-# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
+# sudo chown -R $mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
 
 dockerOpts="
 --detach \
@@ -81,9 +81,6 @@ dockerOpts="
 
 # --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config,target=/data/srv/mariadb/current/config \
 
-# mariadbOpts=$*
-# mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
-
 registry=local
 repository=mariadb
 
@@ -97,7 +94,7 @@ $PULL && {
     docker tag  $registry/$project/$repository:$MDB_TAG $registry/$repository:latest
 }
 
-echo "Starting the mariadb:$MDB_TAG docker container with the following parameters: $mariadbOpts"
+echo "Starting the $registry/$repository:$MDB_TAG docker container with the following parameters: $mariadbOpts"
 docker run $dockerOpts $mariadbOpts $registry/$repository:$MDB_TAG && (
     [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
     ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh
index 82965cc9f..d9ce91b8f 100755
--- a/docker/pypi/wmagent-mariadb/run.sh
+++ b/docker/pypi/wmagent-mariadb/run.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-manage init-mariadb 2>&1 | tee -a run.log
-manage start-mariadb 2>&1 | tee -a run.log
+manage init-mariadb  2>&1 | tee -a $MDB_LOG_DIR/run.log
+manage start-mariadb 2>&1 | tee -a $MDB_LOG_DIR/run.log
 
 echo "Start sleeping....zzz"
 while true; do sleep 10; done

From 7f716374e2b5d9a5364b39a95706fd43f2f1ac0c Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 28 Feb 2024 14:28:31 +0100
Subject: [PATCH 18/27] Review comments

---
 docker/pypi/wmagent-mariadb/Dockerfile              | 2 +-
 docker/pypi/wmagent-mariadb/mariadb-docker-build.sh | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index a390796fe..da0e79052 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,6 +1,6 @@
 ARG MDB_TAG=10.6.5
 FROM mariadb:$MDB_TAG
-MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+MAINTAINER Todor Ivanov todor.ivanov@cern.ch
 
 ARG MDB_TAG
 ENV MDB_TAG=$MDB_TAG
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
index 50710fc3e..d5054e014 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -64,8 +64,12 @@ $PUSH && {
         echo "ERROR: You MUST connect to $registry with your login user rather than with $currUser"
         exit 1
     }
-    echo "Connecting to $registry with Username: $loginUser"
-    docker login -u $loginUser $registry
+    echo "Testing for existing login session to $registry with Username: $loginUser"
+    docker login $registry < /dev/null >/dev/null 2>&1 || {
+        echo "ERROR: A valid login session to $registry is required in order to be able to upload any docker image"
+        echo "ERROR: Please consider running 'docker login $registry' with USER:$currUser and retry again."
+        exit 1
+    }
     docker tag local/mariadb:$MDB_TAG $registry/cmsweb/mariadb:$MDB_TAG
     echo "Uploading image $registry/cmsweb/mariadb:$MDB_TAG"
     docker push $registry/cmsweb/mariadb:$MDB_TAG
@@ -74,5 +78,4 @@ $PUSH && {
         echo "Uploading image $registry/cmsweb/mariadb:latest"
         docker push $registry/cmsweb/mariadb:latest
     }
-    docker logout $registry
 }

From 56d44f651b18fdf2f481d7600d6e9b5b810f95e7 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 10:34:26 +0100
Subject: [PATCH 19/27] Alan's review comments && Add README

---
 docker/pypi/wmagent-mariadb/manage            | 52 ++++++++++++-------
 .../wmagent-mariadb/mariadb-docker-build.sh   | 10 ++--
 .../wmagent-mariadb/mariadb-docker-run.sh     |  6 +--
 docker/pypi/wmagent-mariadb/my.cnf            |  7 +--
 4 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index 3fc0ac8d3..dc4a7cf73 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -57,7 +57,11 @@ status(){
 }
 
 start_mariadb(){
-    echo starting MariaDB server
+    mariadb-admin --socket=$MDB_SOCKET_FILE status >/dev/null 2>&1 && {
+        echo "$FUNCNAME: WARNING: MariaDB Server already running on --socket=$MDB_SOCKET_FILE"
+        return
+    }
+    echo "$FUNCNAME: Starting MariaDB server"
     mariadbd-safe --defaults-extra-file=$MDB_CONFIG_DIR/my.cnf \
                   --datadir=$MDB_DATABASE_DIR \
                   --log-bin \
@@ -74,11 +78,21 @@ stop_mariadb(){
 }
 
 db_prompt(){
-    mariadb --socket=$MDB_SOCKET_FILE --database=$wmaDBName --pager='less -SFX'
+    mariadb --socket=$MDB_SOCKET_FILE --database=$WMA_DATABASE --pager='less -SFX'
 }
 
 clean_mariadb(){
-    db_prompt "drop database $wmaDBName"
+    echo
+    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $WMA_DATABASE DATABASE."
+    echo -n "$FUNCNAME: Continue? [n]: "
+    read x && [[ $x =~ (y|yes|yeS|yEs|Yes|yES|YEs|YeS|YES|Y) ]] || return 102
+    echo "$FUNCNAME: ..."
+    echo "$FUNCNAME: You still have 5 sec. to cancel before we proceed."
+    echo
+    sleep 5
+    echo "$FUNCNAME: DROPPING $WMA_DATABASE DATABASE!"
+    mariadb --socket=$MDB_SOCKET_FILE -e "drop database $WMA_DATABASE"
+    mariadb --socket=$MDB_SOCKET_FILE -e "create database $WMA_DATABASE"
 }
 
 version(){
@@ -89,56 +103,56 @@ version(){
 init_mariadb(){
     # The function to set/check initial database configurations and user preveleges
     [[ $USER == $MDB_ROOT ]] || {
-        echo "ERROR: The current user does not match the MariaDB root user from $MDB_SECRETS_FILE."
-        echo "ERROR: Canot continue. Exit..."
+        echo "$FUNCNAME: ERROR: The current user does not match the MariaDB root user from $MDB_SECRETS_FILE."
+        echo "$FUNCNAME: ERROR: Cannot continue. Exit..."
         exit 1
     }
 
     echo -------------------------------------------------------------------------
-    echo Stopping any previously running mariadb server
+    echo "$FUNCNAME: Stopping any previously running mariadb server"
     mariadb-admin -u $MDB_ROOT --socket=$MDB_SOCKET_FILE shutdown
     echo
 
     echo -------------------------------------------------------------------------
-    echo "Trying to install system database with user: $USER (if it is not already present)"
+    echo "$FUNCNAME: Trying to install system database with user: $USER (if it is not already present)"
 
     errMsg=$(mariadb-install-db --skip-test-db --user=$USER --datadir=$MDB_DATABASE_DIR)
     err=$?
     echo $errMsg
 
     if [[ $err -ne 0 ]]; then
-        echo "ERROR: Could not create system and user databases."
-        echo "ERROR: $errMsg"
+        echo "$FUNCNAME: ERROR: Could not create system and user databases."
+        echo "$FUNCNAME: ERROR: $errMsg"
         exit $err
     elif echo $errMsg|grep -i "exists" ; then
-        echo "WARNING: System and user databases already exist. NOT trying to create them."
+        echo "$FUNCNAME: WARNING: System and user databases already exist. NOT trying to create them."
         return $err
     fi
 
     echo -------------------------------------------------------------------------
-    echo Starting MariaDB server
+    echo "$FUNCNAME: Starting MariaDB server"
     start_mariadb
 
     echo -------------------------------------------------------------------------
-    echo Securing MariaDB Root user
+    echo "$FUNCNAME: Securing MariaDB Root user"
     # sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     mariadb-admin -u $MDB_ROOT password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     echo
 
     echo -------------------------------------------------------------------------
-    echo Creating WMAgent databases
-    echo "Installing WMAgent Database: $WMA_DATABASE"
+    echo "$FUNCNAME: Creating WMAgent databases"
+    echo "$FUNCNAME: Installing WMAgent Database: $WMA_DATABASE"
     mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "create database $WMA_DATABASE"
     echo
 
     echo -------------------------------------------------------------------------
-    echo Creating WMAgent user and setting grants
+    echo "$FUNCNAME: Creating WMAgent user and setting grants"
     # try to create a user different than root (if it does not already exist), and grant privileges
     # we need ${MDB_USER}'@'127.0.0.1 user in paralel to ${MDB_USER}'@'localhost
 
     if [[ $MDB_USER == $MDB_ROOT ]]; then
-        echo "WARNING: WMAgent user set is the same as the MariaDB Root user. You must configure a different one!"
-        echo "WARNING: NOT creating WMAgent users and NOT granting priveleges to $WMA_DATABASE database"
+        echo "$FUNCNAME: WARNING: WMAgent user set is the same as the MariaDB Root user. You must configure a different one!"
+        echo "$FUNCNAME: WARNING: NOT creating WMAgent users and NOT granting priveleges to $WMA_DATABASE database"
         return 1
     else
         mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "CREATE USER '${MDB_USER}'@'localhost' IDENTIFIED BY '$MDB_PASS'"
@@ -152,13 +166,13 @@ init_mariadb(){
 
 _load_secrets $MDB_SECRETS_FILE "MDB_ROOT MDB_ROOTPASS" || {
     err=$?
-    echo "ERROR: Could not properly load root password for MariaDB"
+    echo "$FUNCNAME: ERROR: Could not properly load root password for MariaDB"
     exit $err
 }
 
 _load_secrets $WMA_SECRETS_FILE "MDB_USER MDB_PASS" || {
     err=$?
-    echo "ERROR: Could not properly load WMAgent User password for MariaDB"
+    echo "$FUNCNAME: ERROR: Could not properly load WMAgent User password for MariaDB"
     exit $err
 }
 
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
index d5054e014..729e90431 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -10,13 +10,13 @@ help(){
 
 The MariaDB docker build script for Docker image creation based on pypi:
 
-Usage: mariadb-docker-build.sh -v <mariadb_tag>
+Usage: mariadb-docker-build.sh -t <mariadb_tag> [-p] [-l]
 
-    -t <mariadb_tag>                The MariaDB version/tag to be used for the Docker image creation
-    -p <push_image>                 Push the image to registry.cern.ch
-    -l <latest_tag_toregistry>      Push the curernt tag also as latest to registry.cern.ch
+    -t <mariadb_tag>  The MariaDB version/tag to be used for the Docker image creation
+    -p                Push the image to registry.cern.ch
+    -l                Push the curernt tag also as latest to registry.cern.ch
 
-Example: ./mariadb-docker-build.sh -v 2.2.0.2
+Example: ./mariadb-docker-build.sh -t 2.2.0.2
 
 EOF
 }
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index f512ea114..bdf63fcdc 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -17,11 +17,11 @@ help(){
 The script to be used for running a Mariadb docker container at a VM. The full set of arguments
 passed to the current script are to be forwarded to the Mariadb container entrypoint 'run.sh'
 
-Usage: mariadb-docker-run.sh [-t <team_name>] [-n <agent_number>] [-f <db_flavour>]
+Usage: mariadb-docker-run.sh [-t <mariadb_tag>] [-p]
 
-    -p <pull_image>   Pull the image from registry.cern.ch
     -t <mariadb_tag>  The Mariadb version/tag to be downloaded from registry.cern.ch [Default:latest]
-    -h <help>
+    -p                Pull the image from registry.cern.ch
+    -h                Help
 
 Example: ./mariadb-docker-run.sh -t 3.2.2
 
diff --git a/docker/pypi/wmagent-mariadb/my.cnf b/docker/pypi/wmagent-mariadb/my.cnf
index 2d8466311..cbcf065fa 100644
--- a/docker/pypi/wmagent-mariadb/my.cnf
+++ b/docker/pypi/wmagent-mariadb/my.cnf
@@ -42,16 +42,13 @@ innodb_read_io_threads = 4
 # default: 4
 innodb_write_io_threads = 4
 # default: full_crc23
-# Commented out due  to old mariadb version
-# innodb_checksum_algorithm=full_crc32
+innodb_checksum_algorithm=full_crc32
 # default: 1
 innodb_doublewrite=0
 
 innodb_log_file_size=512M
 innodb_log_buffer_size=8M
-# Changed for small testing machines
-# innodb_buffer_pool_size=2G
-innodb_buffer_pool_size=50M
+innodb_buffer_pool_size=2G
 # default: 30
 innodb_sync_spin_loops=60
 # default: 0

From 8250d427d9a5c5914071c3201c37b9a5e0265b3c Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 10:35:40 +0100
Subject: [PATCH 20/27] Add README

---
 docker/pypi/wmagent-mariadb/README | 221 +++++++++++++++++++++++++++++
 1 file changed, 221 insertions(+)
 create mode 100644 docker/pypi/wmagent-mariadb/README

diff --git a/docker/pypi/wmagent-mariadb/README b/docker/pypi/wmagent-mariadb/README
new file mode 100644
index 000000000..9316f9993
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/README
@@ -0,0 +1,221 @@
+# MariaDB default image for running WMAgent
+
+## Prerequisites
+
+This image inherits from the mainstream `mariadb` one, and follows the same
+tagging schema. On top of the base `mariadb` image we add all the structure
+needed for running the WMAgent with MariaDB and two main scripts:
+
+* `mariadb-docker-run.sh`
+* `mariadb-docker-build.sh`
+
+For building the containers, and for creating the mount area at the host and the
+the bind mounts inside the container, respectively. Those are as follows:
+
+* At the host:
+```
+/data/dockerMount/{admin|srv}/mariadb
+```
+* At the container:
+
+```
+/data/{admin|srv}/mariadb
+```
+
+Upon starting the container we try to initialize the default user and system
+databases, which if previously created should exist in the host mount area. And
+the last steps are creating the `wmagent` database.
+
+There are no other external dependencies.
+
+We fetch all the passwords from two secrets files:
+
+* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials for the
+  user to be used by the WMAgent to connect to the datbase
+* `/data/admin/mariadb/MariaDB.secrets` - for reading the the credentials for
+    the root user who is about to have full administrative rights on the MariaDB
+    server
+    **NOTE:** The server admin user configured at the `MariaDB.secrets` file,
+        must match the username of the one who is to run the server inside the
+        container. And the later is resolved at runtime, depending on where we
+        run the container, it could be on of the three:
+   * CERN - production agent
+   * CERN - T0 agent
+   * FNAL
+
+## Usage
+
+### Building MariaDB image
+
+We can build everything locally and upload it at the CERN registry: https://registry.cern.ch
+
+* Using the wrapper script to build MariaDB locally:
+```
+$ ssh vocms****
+user@vocms0290:wmagent-mariadb $ cd /data
+user@vocms0290:wmagent-mariadb $ git clone https://github.com/dmwm/CMSKubernetes.git
+user@vocms0290:wmagent-mariadb $ cd /data/CMSKubernetes/docker/pypi/wmagent-mariadb/
+user@vocms0290:wmagent-mariadb $ ./mariadb-docker-build.sh -t 10.6.5
+
+user@vocms0290:wmagent-mariadb $ docker image  ls
+REPOSITORY                 TAG       IMAGE ID       CREATED          SIZE
+local/mariadb              10.6.5    4efa646aea3e   6 minutes ago    950MB
+local/mariadb              latest    4efa646aea3e   6 minutes ago    950MB
+```
+* Using the wrapper script to build and upload MariaDB to registry.cern.ch:
+```
+./mariadb-docker-build.sh -t 10.6.5 -p
+```
+
+### Running a MariaDB container
+
+We can run from local repository or from upstream CERN registry. The set of
+images one may end up working may look like:
+
+```
+cmst1@vocms0290:wmagent-mariadb $ docker image  ls
+REPOSITORY                 TAG       IMAGE ID       CREATED          SIZE
+local/mariadb              10.6.5    4efa646aea3e   6 minutes ago    950MB
+local/mariadb              latest    4efa646aea3e   6 minutes ago    950MB
+registry.cern.ch/mariadb   10.6.5    8539e03b7a1d   21 minutes ago   950MB
+registry.cern.ch/mariadb   latest    8539e03b7a1d   21 minutes ago   950MB
+```
+
+* Running from a local build:
+
+```
+cmst1@vocms0290:wmagent-mariadb $ ./mariadb-docker-run.sh -t 10.6.5
+Starting the mariadb:10.6.5 docker container with the following parameters:  --user cmst1
+eb7e0d879d4d7fa597587c734837c5289886a6aaf6a82c072187371fdf312b90
+
+cmst1@vocms0290:wmagent-mariadb $ docker ps
+CONTAINER ID   IMAGE                  COMMAND      CREATED         STATUS         PORTS     NAMES
+eb7e0d879d4d   local/mariadb:10.6.5   "./run.sh"   3 seconds ago   Up 2 seconds             mariadb
+```
+
+* Running from CERN registry:
+```
+cmst1@vocms0290:wmagent-mariadb $ ./mariadb-docker-run.sh -t 10.6.5 -p
+Pulling Docker image: registry.cern.ch/cmsweb/mariadb:10.6.5
+10.6.5: Pulling from cmsweb/mariadb
+Digest: sha256:61f798b55a1c743686e1568509975308dc07b5b24486894053d6a312983c4af6
+Status: Downloaded newer image for registry.cern.ch/cmsweb/mariadb:10.6.5
+registry.cern.ch/cmsweb/mariadb:10.6.5
+Starting the mariadb:10.6.5 docker container with the following parameters:  --user cmst1
+21d9c6598f35e627834d1b796460047605d6255cebc746d572289c7b418053ed
+
+cmst1@vocms0290:wmagent-mariadb $ docker ps
+CONTAINER ID   IMAGE                             COMMAND      CREATED         STATUS         PORTS     NAMES
+21d9c6598f35   registry.cern.ch/mariadb:10.6.5   "./run.sh"   7 seconds ago   Up 6 seconds             mariadb
+
+```
+
+* Killing the container directly from the host:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker kill mariadb
+mariadb
+
+```
+
+* Connecting to a running container:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker exec -it mariadb bash
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$
+
+```
+
+* Managing the databse service:
+    * General options:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage --help
+
+The manage script of the MariaDB docker image for WMAgent
+
+Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt | version
+
+```
+    * Stat/Stop the database:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
+start_mariadb: Starting MariaDB server
+...
+240301 09:25:54 mysqld_safe Can't log to error log and syslog at the same time.  Remove all --log-error configuration options for --syslog to take effect.
+240301 09:25:54 mysqld_safe Logging to '/data/srv/mariadb/10.6.5/logs/error.log'.
+240301 09:25:54 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.6.5/install/database
+mariadb-admin  Ver 9.1 Distrib 10.6.5-MariaDB, for debian-linux-gnu on x86_64
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Server version		10.6.5-MariaDB-1:10.6.5+maria~focal-log
+Protocol version	10
+Connection		Localhost via UNIX socket
+UNIX socket		/var/run/mysqld/mariadb.sock
+Uptime:			10 sec
+
+Threads: 2  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
+
+```
+If one tries to start a second server on the same socket:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
+start_mariadb: WARNING: MariaDB Server already running on --socket=/var/run/mysqld/mariadb.sock
+
+```
+    * Cleaning the WMAgent database:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage clean-mariadb
+
+clean_mariadb: THE CURRENT OPERATIONS WILL WIPE OUT THE wmagent DATABASE.
+clean_mariadb: Continue? [n]: y
+clean_mariadb: ...
+clean_mariadb: You still have 5 sec. to cancel before we proceed.
+
+clean_mariadb: DROPPING wmagent DATABASE!
+
+```
+
+    * Connecting to the database with the admin user locally from inside the container:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage db-prompt
+Welcome to the MariaDB monitor.  Commands end with ; or \g.
+Your MariaDB connection id is 5
+Server version: 10.6.5-MariaDB-1:10.6.5+maria~focal-log mariadb.org binary distribution
+
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
+
+MariaDB [wmagent]>
+```
+
+    * Fetching startup logs:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
+-------------------------------------------------------------------------
+Stopping any previously running mariadb server
+mariadb-admin: connect to server at 'localhost' failed
+error: 'Can't connect to local MySQL server through socket '/data/srv/mariadb/10.5/mariadb.sock' (2)'
+Check that mysqld is running and that the socket: '/data/srv/mariadb/10.5/mariadb.sock' exists!
+
+-------------------------------------------------------------------------
+Trying to install system database if it is not present already
+mysql.user table already exists! Run mysql_upgrade, not mysql_install_db
+WARNING: System and user databases already exist. NOT trying to create them.
+starting MariaDB server
+...
+240226 18:24:13 mysqld_safe Logging to '/data/srv/mariadb/10.5/logs/error.log'.
+240226 18:24:13 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.5/install/database
+mariadb-admin  Ver 9.1 Distrib 10.5.24-MariaDB, for debian-linux-gnu on x86_64
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Server version		10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log
+Protocol version	10
+Connection		Localhost via UNIX socket
+UNIX socket		/data/srv/mariadb/10.5/mariadb.sock
+Uptime:			10 sec
+
+Threads: 1  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
+
+Uptime: 10  Threads: 1  Questions: 2  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.200
+
+Start sleeping....zzz
+```
\ No newline at end of file

From 74e15bdf5dee199319edced47b8b8e3fee01de31 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 12:43:08 +0100
Subject: [PATCH 21/27] Rename README to README.md

---
 docker/pypi/wmagent-mariadb/{README => README.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docker/pypi/wmagent-mariadb/{README => README.md} (100%)

diff --git a/docker/pypi/wmagent-mariadb/README b/docker/pypi/wmagent-mariadb/README.md
similarity index 100%
rename from docker/pypi/wmagent-mariadb/README
rename to docker/pypi/wmagent-mariadb/README.md

From 997d6d730e255be7f150c384de502d4ae95842d1 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 12:49:21 +0100
Subject: [PATCH 22/27] Fixing README formating

---
 docker/pypi/wmagent-mariadb/README.md | 77 ++++++++++++++-------------
 1 file changed, 40 insertions(+), 37 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/README.md b/docker/pypi/wmagent-mariadb/README.md
index 9316f9993..d8328c8e5 100644
--- a/docker/pypi/wmagent-mariadb/README.md
+++ b/docker/pypi/wmagent-mariadb/README.md
@@ -124,8 +124,42 @@ cmst1@vocms0290:wmagent-mariadb $ docker exec -it mariadb bash
 
 ```
 
-* Managing the databse service:
-    * General options:
+* Fetching startup logs:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
+-------------------------------------------------------------------------
+Stopping any previously running mariadb server
+mariadb-admin: connect to server at 'localhost' failed
+error: 'Can't connect to local MySQL server through socket '/data/srv/mariadb/10.5/mariadb.sock' (2)'
+Check that mysqld is running and that the socket: '/data/srv/mariadb/10.5/mariadb.sock' exists!
+
+-------------------------------------------------------------------------
+Trying to install system database if it is not present already
+mysql.user table already exists! Run mysql_upgrade, not mysql_install_db
+WARNING: System and user databases already exist. NOT trying to create them.
+starting MariaDB server
+...
+240226 18:24:13 mysqld_safe Logging to '/data/srv/mariadb/10.5/logs/error.log'.
+240226 18:24:13 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.5/install/database
+mariadb-admin  Ver 9.1 Distrib 10.5.24-MariaDB, for debian-linux-gnu on x86_64
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Server version		10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log
+Protocol version	10
+Connection		Localhost via UNIX socket
+UNIX socket		/data/srv/mariadb/10.5/mariadb.sock
+Uptime:			10 sec
+
+Threads: 1  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
+
+Uptime: 10  Threads: 1  Questions: 2  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.200
+
+Start sleeping....zzz
+```
+
+### Managing the databse service:
+
+* General options:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage --help
 
@@ -134,7 +168,8 @@ The manage script of the MariaDB docker image for WMAgent
 Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt | version
 
 ```
-    * Stat/Stop the database:
+
+* Stat/Stop the database:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
 start_mariadb: Starting MariaDB server
@@ -160,7 +195,8 @@ If one tries to start a second server on the same socket:
 start_mariadb: WARNING: MariaDB Server already running on --socket=/var/run/mysqld/mariadb.sock
 
 ```
-    * Cleaning the WMAgent database:
+
+* Cleaning the WMAgent database:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage clean-mariadb
 
@@ -186,36 +222,3 @@ Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
 MariaDB [wmagent]>
 ```
-
-    * Fetching startup logs:
-```
-cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
--------------------------------------------------------------------------
-Stopping any previously running mariadb server
-mariadb-admin: connect to server at 'localhost' failed
-error: 'Can't connect to local MySQL server through socket '/data/srv/mariadb/10.5/mariadb.sock' (2)'
-Check that mysqld is running and that the socket: '/data/srv/mariadb/10.5/mariadb.sock' exists!
-
--------------------------------------------------------------------------
-Trying to install system database if it is not present already
-mysql.user table already exists! Run mysql_upgrade, not mysql_install_db
-WARNING: System and user databases already exist. NOT trying to create them.
-starting MariaDB server
-...
-240226 18:24:13 mysqld_safe Logging to '/data/srv/mariadb/10.5/logs/error.log'.
-240226 18:24:13 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.5/install/database
-mariadb-admin  Ver 9.1 Distrib 10.5.24-MariaDB, for debian-linux-gnu on x86_64
-Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
-
-Server version		10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log
-Protocol version	10
-Connection		Localhost via UNIX socket
-UNIX socket		/data/srv/mariadb/10.5/mariadb.sock
-Uptime:			10 sec
-
-Threads: 1  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
-
-Uptime: 10  Threads: 1  Questions: 2  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.200
-
-Start sleeping....zzz
-```
\ No newline at end of file

From 1c3bd606435ff85a53294dadcdb60c1a9c067d39 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 12:57:56 +0100
Subject: [PATCH 23/27] Refine README

Refine README
---
 docker/pypi/wmagent-mariadb/README.md | 32 ++++++++++++++-------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/README.md b/docker/pypi/wmagent-mariadb/README.md
index d8328c8e5..14ca5f79e 100644
--- a/docker/pypi/wmagent-mariadb/README.md
+++ b/docker/pypi/wmagent-mariadb/README.md
@@ -30,18 +30,18 @@ There are no other external dependencies.
 
 We fetch all the passwords from two secrets files:
 
-* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials for the
+* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials of the
   user to be used by the WMAgent to connect to the datbase
-* `/data/admin/mariadb/MariaDB.secrets` - for reading the the credentials for
-    the root user who is about to have full administrative rights on the MariaDB
-    server
-    **NOTE:** The server admin user configured at the `MariaDB.secrets` file,
-        must match the username of the one who is to run the server inside the
-        container. And the later is resolved at runtime, depending on where we
-        run the container, it could be on of the three:
-   * CERN - production agent
+* `/data/admin/mariadb/MariaDB.secrets` - for reading the credentials of the
+  root user who is about to have full administrative rights on the MariaDB server
+
+  **NOTE:** The server admin user configured at the `MariaDB.secrets` file,
+    must match the username of the one who is to run the server inside the
+    container. And the later is resolved at runtime, depending on where we
+    run the container, it could be on of the three:
+   * CERN - WM agent
    * CERN - T0 agent
-   * FNAL
+   * FNAL - WM agent
 
 ## Usage
 
@@ -69,8 +69,8 @@ local/mariadb              latest    4efa646aea3e   6 minutes ago    950MB
 
 ### Running a MariaDB container
 
-We can run from local repository or from upstream CERN registry. The set of
-images one may end up working may look like:
+We can run from the local repository or from upstream CERN registry. The typical
+set of images one could end up working with, may look like this:
 
 ```
 cmst1@vocms0290:wmagent-mariadb $ docker image  ls
@@ -126,7 +126,7 @@ cmst1@vocms0290:wmagent-mariadb $ docker exec -it mariadb bash
 
 * Fetching startup logs:
 ```
-cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
+cmst1@vocms0290:wmagent-mariadb $ docker logs mariadb
 -------------------------------------------------------------------------
 Stopping any previously running mariadb server
 mariadb-admin: connect to server at 'localhost' failed
@@ -159,6 +159,8 @@ Start sleeping....zzz
 
 ### Managing the databse service:
 
+All of the commands bellow must be run from inside the container.
+
 * General options:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage --help
@@ -169,7 +171,7 @@ Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt
 
 ```
 
-* Stat/Stop the database:
+* Start/Stop the database server:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
 start_mariadb: Starting MariaDB server
@@ -209,7 +211,7 @@ clean_mariadb: DROPPING wmagent DATABASE!
 
 ```
 
-    * Connecting to the database with the admin user locally from inside the container:
+* Connecting to the database with the admin user locally from inside the container:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage db-prompt
 Welcome to the MariaDB monitor.  Commands end with ; or \g.

From c6ae2d9aa23d7dc2c0bd927eab301b2fdd2a2870 Mon Sep 17 00:00:00 2001
From: Valentin <vkuznet@gmail.com>
Date: Mon, 4 Mar 2024 19:35:19 +0100
Subject: [PATCH 24/27] New repos for base images from various Linux
 distributions

---
 docker/pypi/alma-base/Dockerfile |  4 ++++
 docker/pypi/alma-base/errors.txt | 14 ++++++++++++++
 docker/pypi/deb-base/Dockerfile  |  4 ++++
 docker/pypi/rh-base/Dockerfile   |  4 ++++
 4 files changed, 26 insertions(+)
 create mode 100644 docker/pypi/alma-base/Dockerfile
 create mode 100644 docker/pypi/alma-base/errors.txt
 create mode 100644 docker/pypi/deb-base/Dockerfile
 create mode 100644 docker/pypi/rh-base/Dockerfile

diff --git a/docker/pypi/alma-base/Dockerfile b/docker/pypi/alma-base/Dockerfile
new file mode 100644
index 000000000..1b2148e9b
--- /dev/null
+++ b/docker/pypi/alma-base/Dockerfile
@@ -0,0 +1,4 @@
+FROM almalinux:latest
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+RUN yum install -y curl-minimal libcurl-minimal vim python3-pycurl pip sudo less \
+    && yum clean all && rm -rf /var/cache/yum
diff --git a/docker/pypi/alma-base/errors.txt b/docker/pypi/alma-base/errors.txt
new file mode 100644
index 000000000..70e1e2973
--- /dev/null
+++ b/docker/pypi/alma-base/errors.txt
@@ -0,0 +1,14 @@
+Step 3/3 : RUN yum install -y curl vim python3 pip sudo less     && yum clean all && rm -rf /var/cache/yum
+ ---> Running in 77dbadded671
+AlmaLinux 9 - AppStream                          12 MB/s | 9.1 MB     00:00
+AlmaLinux 9 - BaseOS                             12 MB/s | 4.7 MB     00:00
+AlmaLinux 9 - Extras                             47 kB/s |  17 kB     00:00
+Package python3-3.9.18-1.el9_3.x86_64 is already installed.
+Package less-590-2.el9_2.x86_64 is already installed.
+Error:
+ Problem: problem with installed package curl-minimal-7.76.1-26.el9_3.2.x86_64
+  - package curl-minimal-7.76.1-26.el9_3.2.x86_64 from @System conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos
+  - package curl-minimal-7.76.1-26.el9.x86_64 from baseos conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos
+  - package curl-minimal-7.76.1-26.el9_3.2.x86_64 from baseos conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos
+  - cannot install the best candidate for the job
+(try to add '--allowerasing' to command line to replace conflicting packages or '--skip-broken' to skip uninstallable packages or '--nobest' to use not only best candidate packages)
diff --git a/docker/pypi/deb-base/Dockerfile b/docker/pypi/deb-base/Dockerfile
new file mode 100644
index 000000000..02ecbf594
--- /dev/null
+++ b/docker/pypi/deb-base/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:sid-slim
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+RUN apt-get update && \
+    apt-get install -y curl libcurl4-openssl-dev vim python3-pycurl pip sudo less
diff --git a/docker/pypi/rh-base/Dockerfile b/docker/pypi/rh-base/Dockerfile
new file mode 100644
index 000000000..445de6d6f
--- /dev/null
+++ b/docker/pypi/rh-base/Dockerfile
@@ -0,0 +1,4 @@
+FROM cern/cc7-base:latest
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+RUN yum install -y curl lbcurl ibcurl-openssl-devel vim python3 pip python36-pycurl sudo less \
+    && yum clean all && rm -rf /var/cache/yum

From 044372bb16906ab9dd965578128cac0ff86b40c3 Mon Sep 17 00:00:00 2001
From: Nikodemas Tuckus <ntuckus@gmail.com>
Date: Thu, 7 Mar 2024 14:14:48 +0100
Subject: [PATCH 25/27] Update image versions

---
 kubernetes/monitoring/services/cmsmon-hpc-usage.yaml    | 2 +-
 kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml | 2 +-
 kubernetes/monitoring/services/cpueff/cpueff-spark.yaml | 2 +-
 kubernetes/monitoring/services/cron-spark-jobs.yaml     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml b/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml
index f7dd46217..8ac25f15e 100644
--- a/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml
+++ b/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml
@@ -57,7 +57,7 @@ spec:
       hostname: hpc-usage
       containers:
         - name: hpc-usage
-          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.1.10
+          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.2.7
           env:
             - name: MY_NODE_NAME
               valueFrom:
diff --git a/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml b/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml
index 705ab2507..921713f03 100644
--- a/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml
+++ b/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml
@@ -32,7 +32,7 @@ spec:
     spec:
       containers:
       - name: cpueff-goweb
-        image: registry.cern.ch/cmsmonitoring/cpueff-goweb:cpueff-0.0.20
+        image: registry.cern.ch/cmsmonitoring/cpueff-goweb:cpueff-0.0.22
         # image: golang
         # command: [ "sleep" ]
         # args: [ "infinity" ]
diff --git a/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml b/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml
index e347f843d..3fafb0fe1 100644
--- a/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml
+++ b/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml
@@ -60,7 +60,7 @@ spec:
           hostname: cpueff-spark
           containers:
             - name: cpueff-spark
-              image: registry.cern.ch/cmsmonitoring/cpueff-spark:cpueff-0.0.20
+              image: registry.cern.ch/cmsmonitoring/cpueff-spark:cpueff-0.0.22
               command: [ "/bin/bash", "-c" ]
               args:
                 - source /etc/environment;
diff --git a/kubernetes/monitoring/services/cron-spark-jobs.yaml b/kubernetes/monitoring/services/cron-spark-jobs.yaml
index 48aa8b834..4c4ac3570 100644
--- a/kubernetes/monitoring/services/cron-spark-jobs.yaml
+++ b/kubernetes/monitoring/services/cron-spark-jobs.yaml
@@ -73,7 +73,7 @@ spec:
       hostname: cron-spark-jobs
       containers:
         - name: cron-spark-jobs
-          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.2.4
+          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.2.7
           env:
             - name: MY_NODE_NAME
               valueFrom:

From 0760f1b4bbef3943e9fbb2b1262dc91d5f9a084d Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Thu, 7 Mar 2024 18:53:29 +0100
Subject: [PATCH 26/27] Update monitor.sh

---
 docker/frontend/monitor.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/frontend/monitor.sh b/docker/frontend/monitor.sh
index 77acca9be..18b3f30d1 100755
--- a/docker/frontend/monitor.sh
+++ b/docker/frontend/monitor.sh
@@ -49,6 +49,8 @@ if [ -f /data/filebeat.yaml ] && [ -f /usr/bin/filebeat ]; then
     fi
     ldir=/data/filebeat/${NAME}
     mkdir -p $ldir/data
+    sudo mkdir -p /var/log/filebeat/
+    sudo chown _frontend:_frontend /var/log/filebeat/
     nohup /usr/bin/filebeat \
         -c /data/filebeat.yaml \
         --path.data $ldir/data --path.logs $ldir -e 2>&1 1>& $ldir/log < /dev/null &

From 58fe713ca437f0fa6cc66aabdba6a974f2499131 Mon Sep 17 00:00:00 2001
From: Aroosha Pervaiz <aroosha.pervaiz@cern.ch>
Date: Mon, 18 Mar 2024 16:15:10 +0100
Subject: [PATCH 27/27] Pemission issue with copy_Cron

---
 docker/frontend/copy_cron.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 docker/frontend/copy_cron.sh

diff --git a/docker/frontend/copy_cron.sh b/docker/frontend/copy_cron.sh
old mode 100644
new mode 100755