From 4f949b351e9b85f5408acbd69cd66ae8ce5cf701 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Sep 2023 23:41:01 +0200
Subject: [PATCH 01/45] Add minimal MariaDB container for WMAgent

---
 docker/pypi/wmagent-mariadb/Dockerfile        |  80 ++++++++
 docker/pypi/wmagent-mariadb/manage            | 178 ++++++++++++++++++
 .../wmagent-mariadb/mariadb-docker-run.sh     |  99 ++++++++++
 docker/pypi/wmagent-mariadb/my.cnf            |  76 ++++++++
 docker/pypi/wmagent-mariadb/run.sh            |   7 +
 docker/pypi/wmagent-mariadb/start-mysql.sh    |  63 +++++++
 6 files changed, 503 insertions(+)
 create mode 100644 docker/pypi/wmagent-mariadb/Dockerfile
 create mode 100755 docker/pypi/wmagent-mariadb/manage
 create mode 100755 docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
 create mode 100644 docker/pypi/wmagent-mariadb/my.cnf
 create mode 100755 docker/pypi/wmagent-mariadb/run.sh
 create mode 100755 docker/pypi/wmagent-mariadb/start-mysql.sh

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
new file mode 100644
index 000000000..3a6125e7e
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -0,0 +1,80 @@
+ARG TAG=11.1
+FROM mariadb:${TAG}
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+
+ARG TAG
+ENV TAG=${TAG}
+RUN echo TAG=$TAG
+
+RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
+
+# # Install some debugging tools
+RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
+
+ENV USER=cmst1
+# ENV MARIADB_PORT=
+ENV UID=31961
+ENV MARIADB_ROOT_DIR=/data
+
+ENV MARIADB_BASE_DIR=$MARIADB_ROOT_DIR/srv/mariadb
+ENV MARIADB_ADMIN_DIR=$MARIADB_ROOT_DIR/admin/wmagent
+ENV MARIADB_CERTS_DIR=$MARIADB_ROOT_DIR/certs
+
+ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$TAG
+ENV MARIADB_MANAGE_DIR=$MARIADB_CURRENT_DIR
+ENV MARIADB_AUTH_DIR=$MARIADB_CURRENT_DIR/auth/
+ENV MARIADB_INSTALL_DIR=$MARIADB_CURRENT_DIR/install
+ENV MARIADB_STATE_DIR=$MARIADB_CURRENT_DIR/state
+ENV MARIADB_DATABASE_DIR=$MARIADB_INSTALL_DIR/database
+ENV MARIADB_CONFIG_DIR=$MARIADB_CURRENT_DIR/config
+ENV MARIADB_LOG_DIR=$MARIADB_CURRENT_DIR/logs
+ENV MARIADB_DEPLOY_DIR=/usr/local
+ENV MARIADB_ENV_FILE=$MARIADB_DEPLOY_DIR/deploy/env.sh
+ENV MARIADB_SECRETS_FILE=$MARIADB_ADMIN_DIR/MariaDB.secrets
+
+
+RUN useradd -u $UID -m $USER
+
+# add user to sudoers file
+RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# start the setup
+RUN mkdir -p $MARIADB_ROOT_DIR
+
+ENV PATH="${MARIADB_ROOT_DIR}:${PATH}"
+
+RUN mkdir -p $MARIADB_CURRENT_DIR $MARIADB_CONFIG_DIR $MARIADB_MANAGE_DIR $MARIADB_LOG_DIR $MARIADB_DATABASE_DIR $MARIADB_STATE_DIR $MARIADB_AUTH_DIR
+RUN ln -s $MARIADB_CURRENT_DIR $MARIADB_BASE_DIR/current
+
+# add necessary scripts
+ADD run.sh ${MARIADB_ROOT_DIR}/
+ADD start-mysql.sh ${MARIADB_ROOT_DIR}/
+ADD manage ${MARIADB_MANAGE_DIR}/manage
+RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
+
+# The $MARIADB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
+# ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
+# RUN ln -s ${MARIADB_CONFIG_DIR}/my.cnf /opt/mariadb/etc/local.d/
+ADD my.cnf /etc/mysql/my.cnf
+
+
+ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
+
+RUN <<EOF cat >> /home/${USER}/.bashrc
+
+alias lll="ls -lathr"
+alias ls="ls --color=auto"
+alias ll='ls -la --color=auto'
+
+alias manage=$MARIADB_MANAGE_DIR/manage
+
+# set MariaDB docker specific bash prompt:
+export PS1="(MariaDB-$TAG) [\u@\h:\W]\$ "
+EOF
+
+RUN chown -R ${USER} ${MARIADB_ROOT_DIR}
+
+# setup final environment
+USER $USER
+WORKDIR $MARIADB_ROOT_DIR
+ENTRYPOINT ["./run.sh"]
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
new file mode 100755
index 000000000..372e8dd93
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -0,0 +1,178 @@
+#!/bin/bash
+
+#### These are old mysql actions previously defined in the wmagent manage script
+#### Some of them need to be rewritten some of them must go away
+
+
+
+# #########################
+# #  MySQL                #
+# #########################
+
+# #
+# # first time startup routines for mysql
+# # pre gets called before startup, post called after it
+# init_mysql_db_pre(){
+#     echo "Installing the mysql database area..."
+#     mkdir -p $INSTALL_MYSQL/database
+#     mkdir -p $INSTALL_MYSQL/logs
+#     mysql_install_db --datadir=$INSTALL_MYSQL/database
+# }
+# init_mysql_db_post(){
+#     #install the WMAgent stuff
+#     echo "Installing the mysql schema..."
+#     load_secrets_file;
+#     local TIMEOUT=0;
+#     while [ ! -e $MYSQL_SOCK ]
+#     do
+#         sleep 2;
+#         TIMEOUT=$(($TIMEOUT+2))
+#         if [ $TIMEOUT -ge 300 ]; then
+#             echo "ERROR: Timeout waiting for mysqld to start."
+#             exit 1;
+#         fi
+#     done
+#     echo "Socket file exists, proceeding with schema install..."
+
+#     inited_mysql;
+
+#     # create a user - different than root and current unix user - and grant privileges
+#     if [ "$MYSQL_USER" != "$USER" ]; then
+#         mysql -u $USER --socket=$MYSQL_SOCK --execute "CREATE USER '${MYSQL_USER}'@'localhost'"
+#         mysql -u $USER --socket=$MYSQL_SOCK --execute "GRANT ALL ON *.* TO $MYSQL_USER@localhost WITH GRANT OPTION"
+#     fi
+
+#     # create databases for agent
+#     if [ $USING_AG -eq 1 ]; then
+#         echo "Installing WMAgent Database: ${MYSQL_DATABASE_AG}"
+#         mysql -u $USER --socket=$MYSQL_SOCK --execute "create database ${MYSQL_DATABASE_AG}"
+#     fi
+# }
+
+# status_of_mysql(){
+#     load_secrets_file;
+#     if [ "x$MYSQL_USER" == "x" ]; then
+# 	echo "Not using MySQL..."
+# 	exit 1;
+#     fi
+
+#     echo "+ Status of MySQL"
+#     if [ ! -e $INSTALL_MYSQL/logs/mysqld.pid ]; then
+#         echo "++ MySQL process file not found"
+#         return
+#     fi
+#     local MYSQL_PID=`cat $INSTALL_MYSQL/logs/mysqld.pid`
+#     kill -0 $MYSQL_PID;
+#     local MYSQL_STATUS=$?
+#     if [ $MYSQL_STATUS -eq 0 ]; then
+#         echo "++ MYSQL running with process: $MYSQL_PID";
+#     else
+#         echo "++ MYSQL process not running"
+#     fi
+
+#     echo "++" `mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK status`
+# }
+
+# #
+# # Main startup method for MySQL.
+# # Checks for initialisation
+# start_mysql(){
+#     load_secrets_file;
+#     if [ "x$MYSQL_USER" == "x" ]; then
+# 	    echo "Not using MySQL..."
+# 	    exit 1;
+#     fi
+
+#     echo "Starting mysql..."
+
+#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
+# 	    echo "MySQL has not been initialised... running pre initialisation";
+# 	    init_mysql_db_pre;
+#     fi
+
+#     # Start mysqld to install the database schemas
+#     #
+#     echo "starting mysqld_safe..."
+#     nohup mysqld_safe --defaults-extra-file=$CONFIG_MYSQL/my.cnf \
+#                       --datadir=$INSTALL_MYSQL/database \
+#                       --log-bin \
+#                       --socket=$MYSQL_SOCK \
+#                       --log-error=$INSTALL_MYSQL/logs/error.log \
+#                       --pid-file=$INSTALL_MYSQL/logs/mysqld.pid > /dev/null 2>&1 < /dev/null &
+#     local TIMEOUT=0;
+#     echo "Checking MySQL Socket file exists..."
+#     while [ ! -e $MYSQL_SOCK ]
+#       do
+#       sleep 2;
+#       TIMEOUT=$(($TIMEOUT+2))
+#       if [ $TIMEOUT -ge 300 ]; then
+# 	    echo "ERROR: Timeout waiting for mysqld to start."
+# 	    exit 1;
+#       fi
+#     done
+#     echo "Socket file exists: $MYSQL_SOCK"
+
+#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
+#         echo "MySQL has not been initialised... running post initialisation";
+#         init_mysql_db_post;
+#     fi
+#     echo "Checking Server connection..."
+#     mysql -u $USER --socket=$MYSQL_SOCK --execute "SHOW GLOBAL STATUS" > /dev/null;
+#     if [ $? -ne 0 ]; then
+# 	    echo "ERROR: checking mysql database is running, failed to execute SHOW GLOBAL STATUS"
+# 	    exit 1
+#     fi
+#    echo "Connection OK"
+# }
+
+# #
+# # stop MySQL
+# #
+# stop_mysql(){
+#     load_secrets_file;
+#     if [ "x$MYSQL_USER" == "x" ]; then
+# 	echo "Not using MySQL..."
+# 	exit 1;
+#     fi
+
+#     echo "stopping mysql..."
+#     mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK shutdown &
+#     wait $!
+#     echo "Making sure the MySQL socket file is removed..."
+#     local TIMEOUT=0;
+#     while [ -e $MYSQL_SOCK ]
+#       do
+#       sleep 2;
+#       TIMEOUT=$(($TIMEOUT+2))
+#       if [ $TIMEOUT -ge 300 ]; then
+# 	  echo "ERROR: Timeout waiting for mysqld to shutdown."
+# 	  echo "ERROR: Socket file still exists: $MYSQL_SOCK"
+# 	  exit 1;
+#       fi
+#     done
+#     echo "MySQL is shutdown."
+# }
+
+# #
+
+# case $1 in
+#   status)
+#     status ;;
+#   start-mysql)
+#     start_mysql;;
+#   stop-mysql)
+#     stop_mysql;;
+#   clean-mysql)
+#      clean_mysql;;
+#   db-prompt)
+#      db_prompt $@;;
+#   mysql-prompt)
+#      db_prompt $@;;
+#   help)
+#     help ;;
+#   version)
+#     echo "Management script for WMAgent. No idea what version, at least 2 though" ;;
+#   * )
+#     echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
+#     exit 1 ;;
+# esac
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
new file mode 100755
index 000000000..cf086e8b6
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+### This script is to be used for running the Mariadb docker container at a VM
+### Its sole purpose is to set all the needed mount points from the Host VM and
+### forward all Mariadb runtime parameters to the Mariadb container entrypoint run.sh
+### It accepts only the set of parameters relevant to Mariadb's container run.sh
+### and no build dependent ones. The docker image tag to be searched for execution is
+### always `latest`.
+
+
+# NOTE: In the help call to the current scrit we only repeat the help and usage
+#       information for all the parameters accepted by run.sh.
+help(){
+    echo -e $*
+    cat <<EOF
+
+The script to be used for running a Mariadb docker container at a VM. The full set of arguments
+passed to the current script are to be forwarded to the Mariadb container entrypoint 'run.sh'
+
+Usage: mariadb-docker-run.sh [-t <team_name>] [-n <agent_number>] [-f <db_flavour>]
+
+    -p <pull_image>   Pull the image from registry.cern.ch
+    -t <mariadb_tag>  The Mariadb version/tag to be downloaded from registry.cern.ch [Default:latest]
+    -h <help>
+
+Example: ./mariadb-docker-run.sh -t 3.2.2
+
+EOF
+}
+
+usage(){
+    help $*
+    exit 1
+}
+
+PULL=false
+MARIADB_TAG=11.1
+
+
+### Argument parsing:
+while getopts ":t:hp" opt; do
+    case ${opt} in
+        t) MARIADB_TAG=$OPTARG ;;
+        p) PULL=true ;;
+        h) help; exit $? ;;
+        : )
+            msg="Invalid Option: -$OPTARG requires an argument"
+            usage "$msg" ;;
+    esac
+done
+
+
+mariadbUser=cmst1
+mariadbOpts=" --user $mariadbUser"
+
+# This is the root at the host only, it may differ from the root inside the container.
+# NOTE: this may be parametriesed, so that the container can run on a different mount point.
+HOST_MOUNT_DIR=/data/dockerMount
+
+[[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
+[[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
+
+sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
+
+
+dockerOpts="
+--detach \
+--network=host \
+--rm \
+--hostname=`hostname -f` \
+--name=mariadb \
+--mount type=bind,source=/tmp,target=/tmp \
+--mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs,target=/data/srv/mariadb/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
+"
+
+
+
+# mariadbOpts=$*
+# mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
+
+$PULL && {
+    echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG"
+    docker login registry.cern.ch
+    docker pull registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:$MARIADB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:latest
+}
+
+echo "Starting the mariadb:$MARIADB_TAG docker container with the following parameters: $mariadbOpts"
+docker run $dockerOpts $mariadbOpts local/mariadb:$MARIADB_TAG && (
+    [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && sudo rm -f $HOST_MOUNT_DIR/srv/mariadb/current
+    sudo ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/my.cnf b/docker/pypi/wmagent-mariadb/my.cnf
new file mode 100644
index 000000000..2d8466311
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/my.cnf
@@ -0,0 +1,76 @@
+[mysqld]
+# this is the default setting in >= 10.2.4
+sql_mode="NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES"
+# default: REPEATABLE-READ
+transaction-isolation=READ-COMMITTED
+bind-address = 127.0.0.1
+
+max_heap_table_size=2048M
+max_allowed_packet=128M
+max_connections = 200
+connect_timeout = 60
+
+# default: MIXED
+binlog_format=ROW
+# default: 16MB
+tmp_table_size=2048M
+# default: 10
+long_query_time=5
+
+# default: 134217728
+key_buffer_size=4000M
+
+# default: 0
+# disabling the query cache for now
+# unittests do not work with this enabled
+#query_cache_type=1
+#query_cache_limit=10M
+#query_cache_size=128M
+
+# threading
+# thread_cache_size defaults to 256, if > than max_connections, it is
+# set to max_connections
+thread_cache_size = 64
+thread_cache_size = 16
+thread_stack = 192K
+
+# innodb
+# default: O_DIRECT
+innodb_flush_method=O_DIRECT
+# default: 4
+innodb_read_io_threads = 4
+# default: 4
+innodb_write_io_threads = 4
+# default: full_crc23
+# Commented out due  to old mariadb version
+# innodb_checksum_algorithm=full_crc32
+# default: 1
+innodb_doublewrite=0
+
+innodb_log_file_size=512M
+innodb_log_buffer_size=8M
+# Changed for small testing machines
+# innodb_buffer_pool_size=2G
+innodb_buffer_pool_size=50M
+# default: 30
+innodb_sync_spin_loops=60
+# default: 0
+innodb_force_recovery = 0
+# default: 50
+innodb_lock_wait_timeout = 100
+
+# Changes to support DYNAMIC / COMPRESSED row format
+# default: Barracuda
+innodb_file_format=Barracuda
+# default: ON
+innodb_file_per_table=ON
+# supports prefix index larger than 767 bytes (might be already implicit in the DYNAMIC mode?)
+# default: ON
+innodb_large_prefix=ON
+# default: ON
+innodb_strict_mode=ON
+#innodb_page_size=32k  # default is 16k
+
+# Introduced in MariaDB 10.1.32, COMP is currently using 10.1.21
+# If COMPRESSED is used, it cannot be set globally and has to be passed in the CREATE TABLE statement
+#innodb_default_row_format=DYNAMIC
diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh
new file mode 100755
index 000000000..62920288d
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/run.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+echo "start sleeping....zzz"
+while true; do sleep 10; done
+
+
+# # start the service
+# manage start
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
new file mode 100755
index 000000000..9064f6b0e
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -0,0 +1,63 @@
+#/bin/bash
+
+### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
+###       !!!! NOTHING MUST STAY HERE !!!!
+###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
+###       NEED TO BE PERFORMED AT THE MYSQL DOCKER IMAGE
+mysqlRoot=
+mysqlRootPass=
+mysqlUser=cmst1
+mysqlUserPass=
+
+configDir=/data/srv/mysql/current/config
+dataDir=/data/srv/mysql/current/install/database
+logDir=/data/srv/mysql/current/logs
+socket=/data/srv/mysql/current/logs/mysql.sock
+agentDb=wmagent
+
+echo -------------------------------------------------------------------------
+echo Stopping any previously running mysql server
+mysqladmin -u $mysqlRoot --password=$mysqlRootPass -h 127.0.0.1 shutdown
+# mysqladmin -u $mysqlRoot --password=$mysqlRootPass --socket=$socket shutdown
+echo
+
+
+echo -------------------------------------------------------------------------
+echo Installing system database
+mysql_install_db --datadir=$dataDir
+echo
+
+
+echo -------------------------------------------------------------------------
+echo starting the server
+mysqld_safe --defaults-extra-file=$configDir/my.cnf \
+      --datadir=$dataDir \
+      --log-bin \
+      --socket=$socket \
+      --log-error=$logDir/error.log \
+      --pid-file=$logDir/mysqld.pid  & # > /dev/null 2>&1 < /dev/null &
+echo ...
+sleep 10
+echo
+
+echo -------------------------------------------------------------------------
+echo Securing mysqlRoot and removing temp databases
+mysqladmin -u $mysqlRoot password $mysqlRootPass --socket=$socket
+mysqladmin -u $mysqlRoot --password=$mysqlRootPass  -h 127.0.0.1 password $mysqlRootPass
+# mysql_secure_installation
+echo
+
+echo -------------------------------------------------------------------------
+echo creating new users
+# create a user - different than root and current unix user - and grant privileges
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@localhost WITH GRANT OPTION"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@127.0.0.1 WITH GRANT OPTION"
+
+echo -------------------------------------------------------------------------
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database $agentDb"
+
+echo -------------------------------------------------------------------------

From e62c478aa854f341178fe450ce6c8ad46d44d046 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 8 Sep 2023 01:36:21 +0200
Subject: [PATCH 02/45] Fix broken server startup procedure

---
 docker/pypi/wmagent-mariadb/start-mysql.sh | 23 +++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 9064f6b0e..97c181af5 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -4,15 +4,15 @@
 ###       !!!! NOTHING MUST STAY HERE !!!!
 ###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
 ###       NEED TO BE PERFORMED AT THE MYSQL DOCKER IMAGE
-mysqlRoot=
+mysqlRoot=root
 mysqlRootPass=
 mysqlUser=cmst1
 mysqlUserPass=
 
-configDir=/data/srv/mysql/current/config
-dataDir=/data/srv/mysql/current/install/database
-logDir=/data/srv/mysql/current/logs
-socket=/data/srv/mysql/current/logs/mysql.sock
+configDir=/data/dockerMount/srv/mysql/current/config
+dataDir=/data/dockerMount/srv/mysql/current/install/database
+logDir=/data/dockerMount/srv/mysql/current/logs
+socket=/data/dockerMount/srv/mysql/current/logs/mysql.sock
 agentDb=wmagent
 
 echo -------------------------------------------------------------------------
@@ -47,17 +47,18 @@ mysqladmin -u $mysqlRoot --password=$mysqlRootPass  -h 127.0.0.1 password $mysql
 # mysql_secure_installation
 echo
 
+echo -------------------------------------------------------------------------
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database '$agentDb'"
+
 echo -------------------------------------------------------------------------
 echo creating new users
 # create a user - different than root and current unix user - and grant privileges
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost' IDENTIFIED BY '$mysqlUserPass'"
 mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@localhost WITH GRANT OPTION"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1'"
+mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1' IDENTIFIED BY '$mysqlUserPass'"
 mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@127.0.0.1 WITH GRANT OPTION"
 
-echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database $agentDb"
 
 echo -------------------------------------------------------------------------

From 365394a35bf2c94df041451db5c3cf1936a37a89 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 31 Jan 2024 17:43:57 +0100
Subject: [PATCH 03/45] Update values.yaml

---
 helm/mongodb/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/mongodb/values.yaml b/helm/mongodb/values.yaml
index 248392fd8..1d5475002 100644
--- a/helm/mongodb/values.yaml
+++ b/helm/mongodb/values.yaml
@@ -3,7 +3,7 @@ quickSetting:
   namespace: default
   env: test-rs
   image:
-    mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2310f-test
+    mongodb: registry.cern.ch/cmsweb/cmsmongo:HG2311a
   pvc:
     storageClassName: cpio1
     accessMode: ReadWriteMany

From 22ed62a9012a2b1daae1a5b104b19fb825027cae Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 31 Jan 2024 17:48:08 +0100
Subject: [PATCH 04/45] Update Chart.yaml

---
 helm/mongodb/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/mongodb/Chart.yaml b/helm/mongodb/Chart.yaml
index 7fb69effd..b201143c1 100644
--- a/helm/mongodb/Chart.yaml
+++ b/helm/mongodb/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: mongodb
 description: A Helm chart for launching a MongoDB ReplicaSet with 3 instances
 type: application
-version: 1.0.10
-appVersion: 1.0.10
+version: 1.0.11
+appVersion: 1.0.11

From a6612ad45ea6ca79915e71e343cc29a97418973a Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Mon, 12 Feb 2024 15:30:19 +0100
Subject: [PATCH 05/45] Update ing-dqm.yaml

---
 kubernetes/cmsweb/ingress/ing-dqm.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubernetes/cmsweb/ingress/ing-dqm.yaml b/kubernetes/cmsweb/ingress/ing-dqm.yaml
index def044e64..59f125365 100644
--- a/kubernetes/cmsweb/ingress/ing-dqm.yaml
+++ b/kubernetes/cmsweb/ingress/ing-dqm.yaml
@@ -17,7 +17,7 @@ spec:
   - host: cmsweb-srv.cern.ch
     http:
       paths:
-      - path: /dqm/dqm-square-k8
+      - path: /dqm/dqm-square
         pathType: Prefix
         backend:
           service:

From 14d8a10ced5d49cc9f0b3eb39554a50d3767a289 Mon Sep 17 00:00:00 2001
From: Stefano Belforte <stefano.belforte@cern.ch>
Date: Wed, 21 Feb 2024 18:00:35 +0100
Subject: [PATCH 06/45] add values for cmsweb-test2

---
 helm/crabserver/Chart.yaml        |   2 +-
 helm/crabserver/values-test2.yaml | 179 ++++++++++++++++++++++++++++++
 2 files changed, 180 insertions(+), 1 deletion(-)
 create mode 100644 helm/crabserver/values-test2.yaml

diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index 4a1245c0c..fe64ef13c 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -20,4 +20,4 @@ version: 0.1.6
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: "v3.230913-stable"
+appVersion: "v3.240219.1"
diff --git a/helm/crabserver/values-test2.yaml b/helm/crabserver/values-test2.yaml
new file mode 100644
index 000000000..5a4206f71
--- /dev/null
+++ b/helm/crabserver/values-test2.yaml
@@ -0,0 +1,179 @@
+# Default values for crabserver.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 
+  prod: 8
+  test: 1
+
+image:
+  path: registry.cern.ch/cmsweb/crabserver
+  pullPolicy: Always
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "v3.240219"
+  command:
+  - /bin/bash
+  - /opt/setup-certs-and-run/setup-certs-and-run.sh
+  env:
+  - name: CRABSERVER_LOGSTDOUT
+    value: "t"
+
+environment:
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: false
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: 
+  prometheus.io/scrape: 'true'
+  prometheus.io/port: "18270"
+
+podSecurityContext:
+  runAsUser: 1000
+  runAsGroup: 1000
+  fsGroup: 2000
+
+imageFilebeatSecurityContext:
+  allowPrivilegeEscalation: false
+
+securityContext: 
+  privileged: true
+
+serviceMon:
+  name: crabserver-mon
+  port: 18720
+  protocol: TCP
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+
+resources: 
+  limits:
+    cpu: 1500m
+    memory: 3Gi
+  requests:
+    cpu: 200m
+    memory: 256Mi
+
+
+deamonset:
+  name: filebeat
+  path: docker.elastic.co/beats/filebeat:8.5.1
+  policy: IfNotPresent
+  args:
+  - bash
+  - -c
+  - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e
+  env: 
+  - name: MY_NODE_NAME
+    valueFrom:
+      fieldRef:
+        fieldPath: spec.nodeName
+  - name: NODE_NAME
+    valueFrom:
+      fieldRef:
+        fieldPath: spec.nodeName
+  - name: MY_NAMESPACE
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.namespace
+  resources:
+    requests:
+      memory: "128Mi"
+      cpu: "200m"
+    limits: 
+      cpu: "1"
+      memory: "1Gi"
+
+
+livenessProbe:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 60
+  timeoutSeconds: 60
+
+readinessProbe:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  periodSeconds: 60  
+  timeoutSeconds: 60
+
+livenessProbePreProd:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 60
+  timeoutSeconds: 60
+
+readinessProbePreProd:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  periodSeconds: 60  
+  timeoutSeconds: 60
+
+livenessProbeTest:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  failureThreshold: 3
+  initialDelaySeconds: 120
+  periodSeconds: 60
+  timeoutSeconds: 60
+
+readinessProbeTest:
+  exec:
+    command:
+    - /bin/sh
+    - -c
+    - |
+      cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
+  periodSeconds: 60  
+  timeoutSeconds: 60
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

From cab17e026596920304af74a8e7d515f88800163c Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 21 Feb 2024 18:47:44 +0100
Subject: [PATCH 07/45] Update Chart.yaml

---
 helm/crabserver/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index fe64ef13c..0f0558217 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.6
+version: 0.1.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

From d1c38db00af1d64b891a33aabe402d4161b86eb3 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:04:03 +0100
Subject: [PATCH 08/45] Update Chart.yaml

---
 helm/crabserver/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index 0f0558217..d507f2f76 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.7
+version: 0.1.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

From 0ef54ad12dc9b579302a64e00d5ba186443174a5 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 12:43:28 +0100
Subject: [PATCH 09/45] Fix broken mount  points and paths

---
 docker/pypi/wmagent-mariadb/Dockerfile        | 19 ++++++++++---------
 .../wmagent-mariadb/mariadb-docker-run.sh     | 17 ++++++++---------
 docker/pypi/wmagent-mariadb/start-mysql.sh    |  8 ++++----
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 3a6125e7e..ae36bb893 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,10 +1,11 @@
-ARG TAG=11.1
-FROM mariadb:${TAG}
+ARG TAG=10.11
+ARG MDB_TAG=$TAG
+FROM mariadb:${MDB_TAG}
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
 
-ARG TAG
-ENV TAG=${TAG}
-RUN echo TAG=$TAG
+ARG MDB_TAG=$TAG
+ENV MDB_TAG=${MDB_TAG}
+RUN echo MDB_TAG=$MDB_TAG
 
 RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
 
@@ -20,7 +21,7 @@ ENV MARIADB_BASE_DIR=$MARIADB_ROOT_DIR/srv/mariadb
 ENV MARIADB_ADMIN_DIR=$MARIADB_ROOT_DIR/admin/wmagent
 ENV MARIADB_CERTS_DIR=$MARIADB_ROOT_DIR/certs
 
-ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$TAG
+ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$MDB_TAG
 ENV MARIADB_MANAGE_DIR=$MARIADB_CURRENT_DIR
 ENV MARIADB_AUTH_DIR=$MARIADB_CURRENT_DIR/auth/
 ENV MARIADB_INSTALL_DIR=$MARIADB_CURRENT_DIR/install
@@ -53,9 +54,9 @@ ADD manage ${MARIADB_MANAGE_DIR}/manage
 RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
 
 # The $MARIADB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
-# ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
+ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
 # RUN ln -s ${MARIADB_CONFIG_DIR}/my.cnf /opt/mariadb/etc/local.d/
-ADD my.cnf /etc/mysql/my.cnf
+# ADD my.cnf /etc/mysql/my.cnf
 
 
 ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
@@ -69,7 +70,7 @@ alias ll='ls -la --color=auto'
 alias manage=$MARIADB_MANAGE_DIR/manage
 
 # set MariaDB docker specific bash prompt:
-export PS1="(MariaDB-$TAG) [\u@\h:\W]\$ "
+export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ "
 EOF
 
 RUN chown -R ${USER} ${MARIADB_ROOT_DIR}
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index cf086e8b6..7449654ea 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -34,7 +34,7 @@ usage(){
 }
 
 PULL=false
-MARIADB_TAG=11.1
+MARIADB_TAG=latest
 
 
 ### Argument parsing:
@@ -59,11 +59,11 @@ HOST_MOUNT_DIR=/data/dockerMount
 
 [[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
 [[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
+# [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
 
-sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
+# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
 
 
 dockerOpts="
@@ -75,12 +75,11 @@ dockerOpts="
 --mount type=bind,source=/tmp,target=/tmp \
 --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs,target=/data/srv/mariadb/current/logs \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
 "
 
-
+# --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
 
 # mariadbOpts=$*
 # mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
@@ -95,5 +94,5 @@ $PULL && {
 
 echo "Starting the mariadb:$MARIADB_TAG docker container with the following parameters: $mariadbOpts"
 docker run $dockerOpts $mariadbOpts local/mariadb:$MARIADB_TAG && (
-    [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && sudo rm -f $HOST_MOUNT_DIR/srv/mariadb/current
-    sudo ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
+    [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
+    ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 97c181af5..8e307a6f9 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -9,10 +9,10 @@ mysqlRootPass=
 mysqlUser=cmst1
 mysqlUserPass=
 
-configDir=/data/dockerMount/srv/mysql/current/config
-dataDir=/data/dockerMount/srv/mysql/current/install/database
-logDir=/data/dockerMount/srv/mysql/current/logs
-socket=/data/dockerMount/srv/mysql/current/logs/mysql.sock
+configDir=/data/srv/mariadb/current/config
+dataDir=/data/srv/mariadb/current/install/database
+logDir=/data/srv/mariadb/current/logs
+socket=/data/srv/mariadb/current/logs/mysql.sock
 agentDb=wmagent
 
 echo -------------------------------------------------------------------------

From 653c329d738db1c8cfeecddb5cabfe96a5bf6d8e Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 12:44:54 +0100
Subject: [PATCH 10/45] Add mariadb-docker-build.sh

---
 .../wmagent-mariadb/mariadb-docker-build.sh   | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100755 docker/pypi/wmagent-mariadb/mariadb-docker-build.sh

diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
new file mode 100755
index 000000000..9e59c1d94
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+### This script is to be used for building a MariaDB docker imagge based on pypi
+### It depends on a single parameter MDB_TAG
+
+
+help(){
+    echo -e $*
+    cat <<EOF
+
+The MariaDB docker build script for Docker image creation based on pypi:
+
+Usage: mariadb-docker-build.sh -v <mariadb_tag>
+
+    -t <mariadb_tag>                The MariaDB version/tag to be used for the Docker image creation
+    -p <push_image>                 Push the image to registry.cern.ch
+    -l <latest_tag_toregistry>      Push the curernt tag also as latest to registry.cern.ch
+
+Example: ./mariadb-docker-build.sh -v 2.2.0.2
+
+EOF
+}
+
+usage(){
+    help $*
+    exit 1
+}
+
+MDB_TAG=None
+PUSH=false
+LATEST=false
+
+### Argument parsing:
+while getopts ":t:hpl" opt; do
+    case ${opt} in
+        t) MDB_TAG=$OPTARG ;;
+        p) PUSH=true ;;
+        l) LATEST=true ;;
+        h) help; exit $? ;;
+        \? )
+            msg="Invalid Option: -$OPTARG"
+            usage "$msg" ;;
+        : )
+            msg="Invalid Option: -$OPTARG requires an argument"
+            usage "$msg" ;;
+    esac
+done
+
+
+# NOTE: NO MDB_TAG validation is done in the current script. It is implemented at the install.sh
+
+dockerOpts=" --network=host --progress=plain --build-arg MDB_TAG=$MDB_TAG "
+
+docker build $dockerOpts -t local/mariadb:$MDB_TAG -t local/mariadb:latest  .
+
+$PUSH && {
+    docker login registry.cern.ch
+    docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    echo "Uploading image registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
+    docker push registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    $LATEST &&  {
+        docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:latest
+        echo "Uploading image registry.cern.ch/cmsweb/mariadb:latest"
+        docker push registry.cern.ch/cmsweb/mariadb:latest
+    }
+}

From 1d9af52c164fad7d7f17b7f3da28772edcea7f6b Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 15:18:10 +0100
Subject: [PATCH 11/45] Switch to mariadb commands && Fix user initialiazation
 procedure

---
 docker/pypi/wmagent-mariadb/start-mysql.sh | 65 ++++++++++++----------
 1 file changed, 37 insertions(+), 28 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 8e307a6f9..3f37c50ec 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -3,62 +3,71 @@
 ### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
 ###       !!!! NOTHING MUST STAY HERE !!!!
 ###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
-###       NEED TO BE PERFORMED AT THE MYSQL DOCKER IMAGE
-mysqlRoot=root
-mysqlRootPass=
-mysqlUser=cmst1
-mysqlUserPass=
+###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
+mariadbRoot=root
+mariadbRootPass=FIXME
+mariadbUser=cmst1
+mariadbUserPass=FIXME
 
 configDir=/data/srv/mariadb/current/config
 dataDir=/data/srv/mariadb/current/install/database
 logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/logs/mysql.sock
+socket=/data/srv/mariadb/current/logs/mariadb.sock
 agentDb=wmagent
 
 echo -------------------------------------------------------------------------
-echo Stopping any previously running mysql server
-mysqladmin -u $mysqlRoot --password=$mysqlRootPass -h 127.0.0.1 shutdown
-# mysqladmin -u $mysqlRoot --password=$mysqlRootPass --socket=$socket shutdown
+echo Stopping any previously running mariadb server
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
+mariadb-admin -u $mariadbUser --socket=$socket shutdown
 echo
 
 
-echo -------------------------------------------------------------------------
-echo Installing system database
-mysql_install_db --datadir=$dataDir
-echo
+# echo -------------------------------------------------------------------------
+# echo Installing system database
+mariadb-install-db --datadir=$dataDir
+# echo
 
 
 echo -------------------------------------------------------------------------
 echo starting the server
-mysqld_safe --defaults-extra-file=$configDir/my.cnf \
+mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
       --datadir=$dataDir \
       --log-bin \
       --socket=$socket \
       --log-error=$logDir/error.log \
-      --pid-file=$logDir/mysqld.pid  & # > /dev/null 2>&1 < /dev/null &
+      --pid-file=$logDir/mariadbd.pid  & # > /dev/null 2>&1 < /dev/null &
 echo ...
 sleep 10
 echo
 
+
+
 echo -------------------------------------------------------------------------
-echo Securing mysqlRoot and removing temp databases
-mysqladmin -u $mysqlRoot password $mysqlRootPass --socket=$socket
-mysqladmin -u $mysqlRoot --password=$mysqlRootPass  -h 127.0.0.1 password $mysqlRootPass
-# mysql_secure_installation
+echo Securing $mariadbRoot and removing temp databases
+sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
 echo
 
 echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "create database '$agentDb'"
+echo Securing $mariadbUser and removing temp databases
+mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
+echo
 
 echo -------------------------------------------------------------------------
-echo creating new users
-# create a user - different than root and current unix user - and grant privileges
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'localhost' IDENTIFIED BY '$mysqlUserPass'"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@localhost WITH GRANT OPTION"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "CREATE USER '${mysqlUser}'@'127.0.0.1' IDENTIFIED BY '$mysqlUserPass'"
-mysql -u $mysqlRoot --password=$mysqlRootPass --socket=$socket --execute "GRANT ALL ON *.* TO $mysqlUser@127.0.0.1 WITH GRANT OPTION"
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
 
+# echo -------------------------------------------------------------------------
+# echo creating new users
+# # create a user - different than root and current unix user - and grant privileges
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
+# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
 
 echo -------------------------------------------------------------------------

From 991b26e4fc5ec3d1ac0cc538b4b627cf458957c9 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 15:54:22 +0100
Subject: [PATCH 12/45] Fix 127.0.0.1 Access denied && Ramp up to v11.0 && Fix
 tag variable name.

Typo
---
 docker/pypi/wmagent-mariadb/Dockerfile        |  2 +-
 .../wmagent-mariadb/mariadb-docker-run.sh     | 32 +++++++++----------
 docker/pypi/wmagent-mariadb/start-mysql.sh    | 21 ++++++------
 3 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index ae36bb893..1f021470c 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,4 +1,4 @@
-ARG TAG=10.11
+ARG TAG=11.0
 ARG MDB_TAG=$TAG
 FROM mariadb:${MDB_TAG}
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index 7449654ea..b3b888c1f 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -34,13 +34,13 @@ usage(){
 }
 
 PULL=false
-MARIADB_TAG=latest
+MDB_TAG=latest
 
 
 ### Argument parsing:
 while getopts ":t:hp" opt; do
     case ${opt} in
-        t) MARIADB_TAG=$OPTARG ;;
+        t) MDB_TAG=$OPTARG ;;
         p) PULL=true ;;
         h) help; exit $? ;;
         : )
@@ -59,11 +59,11 @@ HOST_MOUNT_DIR=/data/dockerMount
 
 [[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
 [[ -d $HOST_MOUNT_DIR/admin/mariadb ]] || (mkdir -p $HOST_MOUNT_DIR/admin/mariadb) || exit $?
-# [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config)  || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database ;} || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs ;} || exit $?
+# [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ;} || exit $?
 
-# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG
+# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
 
 
 dockerOpts="
@@ -74,25 +74,25 @@ dockerOpts="
 --name=mariadb \
 --mount type=bind,source=/tmp,target=/tmp \
 --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/logs,target=/data/srv/mariadb/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs,target=/data/srv/mariadb/current/logs \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
 "
 
-# --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG/config,target=/data/srv/mariadb/current/config \
+# --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config,target=/data/srv/mariadb/current/config \
 
 # mariadbOpts=$*
 # mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
 
 $PULL && {
-    echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG"
+    echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
     docker login registry.cern.ch
-    docker pull registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:$MARIADB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MARIADB_TAG local/mariadb:latest
+    docker pull registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:$MDB_TAG
+    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:latest
 }
 
-echo "Starting the mariadb:$MARIADB_TAG docker container with the following parameters: $mariadbOpts"
-docker run $dockerOpts $mariadbOpts local/mariadb:$MARIADB_TAG && (
+echo "Starting the mariadb:$MDB_TAG docker container with the following parameters: $mariadbOpts"
+docker run $dockerOpts $mariadbOpts local/mariadb:$MDB_TAG && (
     [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
-    ln -s $HOST_MOUNT_DIR/srv/mariadb/$MARIADB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
+    ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
index 3f37c50ec..8cb0f6eda 100755
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ b/docker/pypi/wmagent-mariadb/start-mysql.sh
@@ -5,9 +5,9 @@
 ###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
 ###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
 mariadbRoot=root
-mariadbRootPass=FIXME
+mariadbRootPass=fixme
 mariadbUser=cmst1
-mariadbUserPass=FIXME
+mariadbUserPass=fixme
 
 configDir=/data/srv/mariadb/current/config
 dataDir=/data/srv/mariadb/current/install/database
@@ -62,12 +62,15 @@ echo creating agent databases
 echo "Installing WMAgent Database: $agentDb"
 mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
 
-# echo -------------------------------------------------------------------------
-# echo creating new users
-# # create a user - different than root and current unix user - and grant privileges
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
-# mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
+echo
+
+echo -------------------------------------------------------------------------
+echo creating new users and setting grants
+# try to create a user different than root (if it does not already exist), and grant privileges
+# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
 
 echo -------------------------------------------------------------------------

From 5acd2194ea92796aae6d55257d9c6259a7cf3dd3 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 23 Feb 2024 18:21:27 +0100
Subject: [PATCH 13/45] Add manage

---
 docker/pypi/wmagent-mariadb/Dockerfile       |   2 +-
 docker/pypi/wmagent-mariadb/manage           | 255 ++++++-------------
 docker/pypi/wmagent-mariadb/start-mariadb.sh |  79 ++++++
 3 files changed, 159 insertions(+), 177 deletions(-)
 create mode 100755 docker/pypi/wmagent-mariadb/start-mariadb.sh

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 1f021470c..dc295ad5d 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -49,7 +49,7 @@ RUN ln -s $MARIADB_CURRENT_DIR $MARIADB_BASE_DIR/current
 
 # add necessary scripts
 ADD run.sh ${MARIADB_ROOT_DIR}/
-ADD start-mysql.sh ${MARIADB_ROOT_DIR}/
+ADD start-mariadb.sh ${MARIADB_ROOT_DIR}/
 ADD manage ${MARIADB_MANAGE_DIR}/manage
 RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
 
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index 372e8dd93..ca34b812e 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -1,178 +1,81 @@
 #!/bin/bash
 
-#### These are old mysql actions previously defined in the wmagent manage script
-#### Some of them need to be rewritten some of them must go away
-
-
-
-# #########################
-# #  MySQL                #
-# #########################
-
-# #
-# # first time startup routines for mysql
-# # pre gets called before startup, post called after it
-# init_mysql_db_pre(){
-#     echo "Installing the mysql database area..."
-#     mkdir -p $INSTALL_MYSQL/database
-#     mkdir -p $INSTALL_MYSQL/logs
-#     mysql_install_db --datadir=$INSTALL_MYSQL/database
-# }
-# init_mysql_db_post(){
-#     #install the WMAgent stuff
-#     echo "Installing the mysql schema..."
-#     load_secrets_file;
-#     local TIMEOUT=0;
-#     while [ ! -e $MYSQL_SOCK ]
-#     do
-#         sleep 2;
-#         TIMEOUT=$(($TIMEOUT+2))
-#         if [ $TIMEOUT -ge 300 ]; then
-#             echo "ERROR: Timeout waiting for mysqld to start."
-#             exit 1;
-#         fi
-#     done
-#     echo "Socket file exists, proceeding with schema install..."
-
-#     inited_mysql;
-
-#     # create a user - different than root and current unix user - and grant privileges
-#     if [ "$MYSQL_USER" != "$USER" ]; then
-#         mysql -u $USER --socket=$MYSQL_SOCK --execute "CREATE USER '${MYSQL_USER}'@'localhost'"
-#         mysql -u $USER --socket=$MYSQL_SOCK --execute "GRANT ALL ON *.* TO $MYSQL_USER@localhost WITH GRANT OPTION"
-#     fi
-
-#     # create databases for agent
-#     if [ $USING_AG -eq 1 ]; then
-#         echo "Installing WMAgent Database: ${MYSQL_DATABASE_AG}"
-#         mysql -u $USER --socket=$MYSQL_SOCK --execute "create database ${MYSQL_DATABASE_AG}"
-#     fi
-# }
-
-# status_of_mysql(){
-#     load_secrets_file;
-#     if [ "x$MYSQL_USER" == "x" ]; then
-# 	echo "Not using MySQL..."
-# 	exit 1;
-#     fi
-
-#     echo "+ Status of MySQL"
-#     if [ ! -e $INSTALL_MYSQL/logs/mysqld.pid ]; then
-#         echo "++ MySQL process file not found"
-#         return
-#     fi
-#     local MYSQL_PID=`cat $INSTALL_MYSQL/logs/mysqld.pid`
-#     kill -0 $MYSQL_PID;
-#     local MYSQL_STATUS=$?
-#     if [ $MYSQL_STATUS -eq 0 ]; then
-#         echo "++ MYSQL running with process: $MYSQL_PID";
-#     else
-#         echo "++ MYSQL process not running"
-#     fi
-
-#     echo "++" `mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK status`
-# }
-
-# #
-# # Main startup method for MySQL.
-# # Checks for initialisation
-# start_mysql(){
-#     load_secrets_file;
-#     if [ "x$MYSQL_USER" == "x" ]; then
-# 	    echo "Not using MySQL..."
-# 	    exit 1;
-#     fi
-
-#     echo "Starting mysql..."
-
-#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
-# 	    echo "MySQL has not been initialised... running pre initialisation";
-# 	    init_mysql_db_pre;
-#     fi
-
-#     # Start mysqld to install the database schemas
-#     #
-#     echo "starting mysqld_safe..."
-#     nohup mysqld_safe --defaults-extra-file=$CONFIG_MYSQL/my.cnf \
-#                       --datadir=$INSTALL_MYSQL/database \
-#                       --log-bin \
-#                       --socket=$MYSQL_SOCK \
-#                       --log-error=$INSTALL_MYSQL/logs/error.log \
-#                       --pid-file=$INSTALL_MYSQL/logs/mysqld.pid > /dev/null 2>&1 < /dev/null &
-#     local TIMEOUT=0;
-#     echo "Checking MySQL Socket file exists..."
-#     while [ ! -e $MYSQL_SOCK ]
-#       do
-#       sleep 2;
-#       TIMEOUT=$(($TIMEOUT+2))
-#       if [ $TIMEOUT -ge 300 ]; then
-# 	    echo "ERROR: Timeout waiting for mysqld to start."
-# 	    exit 1;
-#       fi
-#     done
-#     echo "Socket file exists: $MYSQL_SOCK"
-
-#     if [ $MYSQL_INIT_DONE -eq 0 ]; then
-#         echo "MySQL has not been initialised... running post initialisation";
-#         init_mysql_db_post;
-#     fi
-#     echo "Checking Server connection..."
-#     mysql -u $USER --socket=$MYSQL_SOCK --execute "SHOW GLOBAL STATUS" > /dev/null;
-#     if [ $? -ne 0 ]; then
-# 	    echo "ERROR: checking mysql database is running, failed to execute SHOW GLOBAL STATUS"
-# 	    exit 1
-#     fi
-#    echo "Connection OK"
-# }
-
-# #
-# # stop MySQL
-# #
-# stop_mysql(){
-#     load_secrets_file;
-#     if [ "x$MYSQL_USER" == "x" ]; then
-# 	echo "Not using MySQL..."
-# 	exit 1;
-#     fi
-
-#     echo "stopping mysql..."
-#     mysqladmin -u $MYSQL_USER --socket=$MYSQL_SOCK shutdown &
-#     wait $!
-#     echo "Making sure the MySQL socket file is removed..."
-#     local TIMEOUT=0;
-#     while [ -e $MYSQL_SOCK ]
-#       do
-#       sleep 2;
-#       TIMEOUT=$(($TIMEOUT+2))
-#       if [ $TIMEOUT -ge 300 ]; then
-# 	  echo "ERROR: Timeout waiting for mysqld to shutdown."
-# 	  echo "ERROR: Socket file still exists: $MYSQL_SOCK"
-# 	  exit 1;
-#       fi
-#     done
-#     echo "MySQL is shutdown."
-# }
-
-# #
-
-# case $1 in
-#   status)
-#     status ;;
-#   start-mysql)
-#     start_mysql;;
-#   stop-mysql)
-#     stop_mysql;;
-#   clean-mysql)
-#      clean_mysql;;
-#   db-prompt)
-#      db_prompt $@;;
-#   mysql-prompt)
-#      db_prompt $@;;
-#   help)
-#     help ;;
-#   version)
-#     echo "Management script for WMAgent. No idea what version, at least 2 though" ;;
-#   * )
-#     echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
-#     exit 1 ;;
-# esac
+mariadbRoot=root
+mariadbRootPass=fixme
+mariadbUser=cmst1
+mariadbUserPass=fixme
+
+configDir=/data/srv/mariadb/current/config
+dataDir=/data/srv/mariadb/current/install/database
+logDir=/data/srv/mariadb/current/logs
+socket=/data/srv/mariadb/current/mariadb.sock
+agentDb=wmagent
+
+help(){
+    echo -e $*
+    cat <<EOF
+
+The manage script of the MariaDB docker image for WMAgent
+
+Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt | version
+
+EOF
+}
+
+usage(){
+    help $*
+    exit 1
+}
+
+status(){
+    mariadb-admin --socket=$socket status
+}
+
+start_mariadb(){
+    echo starting MariaDB server
+    mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
+                  --datadir=$dataDir \
+                  --log-bin \
+                  --socket=$socket \
+                  --log-error=$logDir/error.log \
+                  --pid-file=$logDir/mariadbd.pid  &
+    echo ...
+    sleep 10
+    echo status
+}
+
+stop_mariadb(){
+    mariadb-admin --socket=$socket shutdown
+}
+
+db_prompt(){
+    mariadb --socket=$socket --database=$wmaDBName --pager='less -SFX'
+}
+
+clean_mariadb(){
+    db_prompt "drop database $wmaDBName"
+}
+
+version(){
+    mariadb-admin --socket=$socket version
+}
+
+case $1 in
+  status)
+      status ;;
+  start-mariadb)
+      start_mariadb;;
+  stop-mariadb)
+      stop_mariadb;;
+  clean-mariadb)
+      clean_mariadb;;
+  db-prompt)
+      shift
+      db_prompt $@;;
+  version)
+      version ;;
+  help)
+      help ;;
+  *)
+      usage ;;
+esac
diff --git a/docker/pypi/wmagent-mariadb/start-mariadb.sh b/docker/pypi/wmagent-mariadb/start-mariadb.sh
new file mode 100755
index 000000000..b60a6ac9e
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/start-mariadb.sh
@@ -0,0 +1,79 @@
+#/bin/bash
+
+### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
+###       !!!! NOTHING MUST STAY HERE !!!!
+###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
+###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
+mariadbRoot=root
+mariadbRootPass=fixme
+mariadbUser=cmst1
+mariadbUserPass=fixme
+
+configDir=/data/srv/mariadb/current/config
+dataDir=/data/srv/mariadb/current/install/database
+logDir=/data/srv/mariadb/current/logs
+socket=/data/srv/mariadb/current/mariadb.sock
+agentDb=wmagent
+
+echo -------------------------------------------------------------------------
+echo Stopping any previously running mariadb server
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
+mariadb-admin -u $mariadbUser --socket=$socket shutdown
+echo
+
+
+# echo -------------------------------------------------------------------------
+# echo Installing system database
+mariadb-install-db --datadir=$dataDir
+# echo
+
+
+
+manage start-mariadb
+
+# echo -------------------------------------------------------------------------
+# echo starting the server
+# mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
+#       --datadir=$dataDir \
+#       --log-bin \
+#       --socket=$socket \
+#       --log-error=$logDir/error.log \
+#       --pid-file=$logDir/mariadbd.pid  &
+# echo ...
+# sleep 10
+# echo
+
+
+
+echo -------------------------------------------------------------------------
+echo Securing $mariadbRoot and removing temp databases
+sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
+echo
+
+echo -------------------------------------------------------------------------
+echo Securing $mariadbUser and removing temp databases
+mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
+# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
+# mariadb-secure-installation --socket=$socket
+echo
+
+echo -------------------------------------------------------------------------
+echo creating agent databases
+echo "Installing WMAgent Database: $agentDb"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
+
+echo
+
+echo -------------------------------------------------------------------------
+echo creating new users and setting grants
+# try to create a user different than root (if it does not already exist), and grant privileges
+# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
+mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
+
+echo -------------------------------------------------------------------------

From 1dc74de847c58bce4c638c9307e972ac4ba919a9 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Sat, 24 Feb 2024 12:59:49 +0100
Subject: [PATCH 14/45] Properly load WMAgent.secrets and MariaDB.secrets files
 && Delete mariadb-start.sh and mysql-start.sh

---
 docker/pypi/wmagent-mariadb/Dockerfile        |  70 +++----
 docker/pypi/wmagent-mariadb/manage            | 182 ++++++++++++++----
 .../wmagent-mariadb/mariadb-docker-run.sh     |   2 +-
 docker/pypi/wmagent-mariadb/run.sh            |   8 +-
 docker/pypi/wmagent-mariadb/start-mariadb.sh  |  79 --------
 docker/pypi/wmagent-mariadb/start-mysql.sh    |  76 --------
 6 files changed, 184 insertions(+), 233 deletions(-)
 delete mode 100755 docker/pypi/wmagent-mariadb/start-mariadb.sh
 delete mode 100755 docker/pypi/wmagent-mariadb/start-mysql.sh

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index dc295ad5d..7a946a56d 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -9,29 +9,33 @@ RUN echo MDB_TAG=$MDB_TAG
 
 RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
 
-# # Install some debugging tools
+# Install some debugging tools
 RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
 
 ENV USER=cmst1
-# ENV MARIADB_PORT=
+# ENV MDB_PORT=
 ENV UID=31961
-ENV MARIADB_ROOT_DIR=/data
-
-ENV MARIADB_BASE_DIR=$MARIADB_ROOT_DIR/srv/mariadb
-ENV MARIADB_ADMIN_DIR=$MARIADB_ROOT_DIR/admin/wmagent
-ENV MARIADB_CERTS_DIR=$MARIADB_ROOT_DIR/certs
-
-ENV MARIADB_CURRENT_DIR=$MARIADB_BASE_DIR/$MDB_TAG
-ENV MARIADB_MANAGE_DIR=$MARIADB_CURRENT_DIR
-ENV MARIADB_AUTH_DIR=$MARIADB_CURRENT_DIR/auth/
-ENV MARIADB_INSTALL_DIR=$MARIADB_CURRENT_DIR/install
-ENV MARIADB_STATE_DIR=$MARIADB_CURRENT_DIR/state
-ENV MARIADB_DATABASE_DIR=$MARIADB_INSTALL_DIR/database
-ENV MARIADB_CONFIG_DIR=$MARIADB_CURRENT_DIR/config
-ENV MARIADB_LOG_DIR=$MARIADB_CURRENT_DIR/logs
-ENV MARIADB_DEPLOY_DIR=/usr/local
-ENV MARIADB_ENV_FILE=$MARIADB_DEPLOY_DIR/deploy/env.sh
-ENV MARIADB_SECRETS_FILE=$MARIADB_ADMIN_DIR/MariaDB.secrets
+ENV MDB_ROOT_DIR=/data
+
+ENV MDB_BASE_DIR=$MDB_ROOT_DIR/srv/mariadb
+ENV MDB_ADMIN_DIR=$MDB_ROOT_DIR/admin/mariadb
+ENV WMA_ADMIN_DIR=$MDB_ROOT_DIR/admin/wmagent
+ENV MDB_CERTS_DIR=$MDB_ROOT_DIR/certs
+
+ENV MDB_CURRENT_DIR=$MDB_BASE_DIR/$MDB_TAG
+ENV MDB_MANAGE_DIR=$MDB_CURRENT_DIR
+ENV MDB_AUTH_DIR=$MDB_CURRENT_DIR/auth/
+ENV MDB_INSTALL_DIR=$MDB_CURRENT_DIR/install
+ENV MDB_STATE_DIR=$MDB_CURRENT_DIR/state
+ENV MDB_DATABASE_DIR=$MDB_INSTALL_DIR/database
+ENV MDB_CONFIG_DIR=$MDB_CURRENT_DIR/config
+ENV MDB_LOG_DIR=$MDB_CURRENT_DIR/logs
+ENV MDB_DEPLOY_DIR=/usr/local
+ENV MDB_ENV_FILE=$MDB_DEPLOY_DIR/deploy/env.sh
+ENV MDB_SOCKET_FILE=$MDB_CURRENT_DIR/mariadb.sock
+ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
+ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
+ENV WMA_DATABASE=wmagent
 
 
 RUN useradd -u $UID -m $USER
@@ -40,24 +44,20 @@ RUN useradd -u $UID -m $USER
 RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 # start the setup
-RUN mkdir -p $MARIADB_ROOT_DIR
+RUN mkdir -p $MDB_ROOT_DIR
 
-ENV PATH="${MARIADB_ROOT_DIR}:${PATH}"
+ENV PATH="${MDB_ROOT_DIR}:${PATH}"
 
-RUN mkdir -p $MARIADB_CURRENT_DIR $MARIADB_CONFIG_DIR $MARIADB_MANAGE_DIR $MARIADB_LOG_DIR $MARIADB_DATABASE_DIR $MARIADB_STATE_DIR $MARIADB_AUTH_DIR
-RUN ln -s $MARIADB_CURRENT_DIR $MARIADB_BASE_DIR/current
+RUN mkdir -p $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
+RUN ln -s $MDB_CURRENT_DIR $MDB_BASE_DIR/current
 
 # add necessary scripts
-ADD run.sh ${MARIADB_ROOT_DIR}/
-ADD start-mariadb.sh ${MARIADB_ROOT_DIR}/
-ADD manage ${MARIADB_MANAGE_DIR}/manage
-RUN ln -s ${MARIADB_MANAGE_DIR}/manage ${MARIADB_ROOT_DIR}/manage
-
-# The $MARIADB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
-ADD my.cnf ${MARIADB_CONFIG_DIR}/my.cnf
-# RUN ln -s ${MARIADB_CONFIG_DIR}/my.cnf /opt/mariadb/etc/local.d/
-# ADD my.cnf /etc/mysql/my.cnf
+ADD run.sh ${MDB_ROOT_DIR}/
+ADD manage ${MDB_MANAGE_DIR}/manage
+RUN ln -s ${MDB_MANAGE_DIR}/manage ${MDB_ROOT_DIR}/manage
 
+# The $MDB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
+ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
 ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
 
@@ -67,15 +67,15 @@ alias lll="ls -lathr"
 alias ls="ls --color=auto"
 alias ll='ls -la --color=auto'
 
-alias manage=$MARIADB_MANAGE_DIR/manage
+alias manage=$MDB_MANAGE_DIR/manage
 
 # set MariaDB docker specific bash prompt:
 export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ "
 EOF
 
-RUN chown -R ${USER} ${MARIADB_ROOT_DIR}
+RUN chown -R ${USER} ${MDB_ROOT_DIR}
 
 # setup final environment
 USER $USER
-WORKDIR $MARIADB_ROOT_DIR
+WORKDIR $MDB_ROOT_DIR
 ENTRYPOINT ["./run.sh"]
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index ca34b812e..25281a923 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -1,16 +1,5 @@
 #!/bin/bash
 
-mariadbRoot=root
-mariadbRootPass=fixme
-mariadbUser=cmst1
-mariadbUserPass=fixme
-
-configDir=/data/srv/mariadb/current/config
-dataDir=/data/srv/mariadb/current/install/database
-logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/mariadb.sock
-agentDb=wmagent
-
 help(){
     echo -e $*
     cat <<EOF
@@ -27,29 +16,67 @@ usage(){
     exit 1
 }
 
+_load_secrets(){
+    # Auxiliary function to parse WMAgent.secrets or MariaDB.secrets files
+    # and load a set of variables from them
+    # :param $1: Path to WMAgent.secrets or file
+    # :param $2: String with variable names to be checked
+    # :retrun:   Error value if one or more values have been left unset in the secrets file
+    local errVal=0
+    local value=""
+    local secretsFile=$1
+    local varsToLoad=$2
+
+    [[ -f $secretsFile ]] || {
+        echo "$FUNCNAME: ERROR: Password file: $secretsFile does not exist"
+        echo "$FUNCNAME: ERROR: Either set WMA_SECRETS_FILE environment variable to a valid file or check that $HOME/WMAgent.secrets exists"
+        return $(false)
+    }
+
+    # All variables need to be fetched in lowercase through: ${varName,,}
+    local badValuesReg="(update-me|updateme|<update-me>|<updateme>|fix-me|fixme|<fix-me>|<fixme>|^$)"
+    for varName in $varsToLoad
+    do
+        value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | awk -F\= '{print $2}'`
+        [[ ${value,,} =~ $badValuesReg ]] && { echo "$FUNCNAME: Bad value for: $varName=$value"; let errVal+=1 ;}
+    done
+
+    for varName in $varsToLoad
+    do
+        value=`grep -E "^[[:blank:]]*$varName=" $secretsFile | sed "s/ *$varName=//"`
+        eval $varName=$value
+        [[ -n $varName ]] || { echo "$FUNCNAME: Empty value for: $varName=$value"; let errVal+=1 ;}
+    done
+    return $errVal
+}
+
+
 status(){
-    mariadb-admin --socket=$socket status
+    mariadb-admin --socket=$MDB_SOCKET_FILE version
+    echo
+    mariadb-admin --socket=$MDB_SOCKET_FILE status
+    echo
 }
 
 start_mariadb(){
     echo starting MariaDB server
-    mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
-                  --datadir=$dataDir \
+    mariadbd-safe --defaults-extra-file=$MDB_CONFIG_DIR/my.cnf \
+                  --datadir=$MDB_DATABASE_DIR \
                   --log-bin \
-                  --socket=$socket \
-                  --log-error=$logDir/error.log \
-                  --pid-file=$logDir/mariadbd.pid  &
+                  --socket=$MDB_SOCKET_FILE \
+                  --log-error=$MDB_LOG_DIR/error.log \
+                  --pid-file=$MDB_LOG_DIR/mariadbd.pid  &
     echo ...
     sleep 10
-    echo status
+    status
 }
 
 stop_mariadb(){
-    mariadb-admin --socket=$socket shutdown
+    mariadb-admin --socket=$MDB_SOCKET_FILE shutdown
 }
 
 db_prompt(){
-    mariadb --socket=$socket --database=$wmaDBName --pager='less -SFX'
+    mariadb --socket=$MDB_SOCKET_FILE --database=$wmaDBName --pager='less -SFX'
 }
 
 clean_mariadb(){
@@ -57,25 +84,104 @@ clean_mariadb(){
 }
 
 version(){
-    mariadb-admin --socket=$socket version
+    mariadb-admin --socket=$MDB_SOCKET_FILE version
+}
+
+
+init_mariadb(){
+    # The function to set/check initial database configurations and user preveleges
+    [[ $USER == $MDB_ROOT ]] || {
+        echo "ERROR: The current user does not match the MariaDB root user from $MDB_SECRETS_FILE."
+        echo "ERROR: Canot continue. Exit..."
+        exit 1
+    }
+
+    echo -------------------------------------------------------------------------
+    echo Stopping any previously running mariadb server
+    mariadb-admin -u $MDB_USER --socket=$MDB_SOCKET_FILE shutdown
+    echo
+
+    echo -------------------------------------------------------------------------
+    echo Trying to install system database if it is not present already
+
+    errMsg=$(mariadb-install-db --skip-test-db --datadir=$MDB_DATABASE_DIR)
+    err=$?
+
+    if [[ $err -ne 0 ]]; then
+        echo "ERROR: Could not create system and user databases."
+        echo "ERROR: $errMsg"
+        exit $err
+    elif echo $errMsg|grep -i "exists" ; then
+        echo "WARNING: System and user databases already exist. NOT trying to create them."
+        return $err
+    fi
+
+    echo -------------------------------------------------------------------------
+    echo Starting MariaDB server
+    start_mariadb
+
+    echo -------------------------------------------------------------------------
+    echo Securing MariaDB Root users
+    sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
+    mariadb-admin -u $MDB_ROOT password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
+    echo
+
+    echo -------------------------------------------------------------------------
+    echo Creating WMAgent databases
+    echo "Installing WMAgent Database: $WMA_DATABASE"
+    mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "create database $WMA_DATABASE"
+    echo
+
+    echo -------------------------------------------------------------------------
+    echo Creating WMAgent user and setting grants
+    # try to create a user different than root (if it does not already exist), and grant privileges
+    # we need ${MDB_USER}'@'127.0.0.1 user in paralel to ${MDB_USER}'@'localhost
+
+    if [[ $MDB_USER == $MDB_ROOT ]]; then
+        echo "WARNING: WMAgent user set is the same as the MariaDB Root user. You must configure a different one!"
+        echo "WARNING: NOT creating WMAgent users and NOT granting priveleges to $WMA_DATABASE database"
+        return 1
+    else
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "CREATE USER '${MDB_USER}'@'localhost' IDENTIFIED BY '$MDB_PASS'"
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "GRANT ALL ON $WMA_DATABASE.* TO $MDB_USER@localhost"
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "CREATE USER '${MDB_USER}'@'127.0.0.1' IDENTIFIED BY '$MDB_PASS'"
+        mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "GRANT ALL ON $WMA_DATABASE.* TO $MDB_USER@127.0.0.1"
+    fi
+    echo -------------------------------------------------------------------------
 }
 
+
+_load_secrets $MDB_SECRETS_FILE "MDB_ROOT MDB_ROOTPASS" || {
+    err=$?
+    echo "ERROR: Could not properly load root password for MariaDB"
+    exit $err
+}
+
+_load_secrets $WMA_SECRETS_FILE "MDB_USER MDB_PASS" || {
+    err=$?
+    echo "ERROR: Could not properly load WMAgent User password for MariaDB"
+    exit $err
+}
+
+
 case $1 in
-  status)
-      status ;;
-  start-mariadb)
-      start_mariadb;;
-  stop-mariadb)
-      stop_mariadb;;
-  clean-mariadb)
-      clean_mariadb;;
-  db-prompt)
-      shift
-      db_prompt $@;;
-  version)
-      version ;;
-  help)
-      help ;;
-  *)
-      usage ;;
+    status)
+        status ;;
+    init-mariadb)
+        init_mariadb;;
+    start-mariadb)
+        start_mariadb;;
+    stop-mariadb)
+        stop_mariadb;;
+    clean-mariadb)
+        clean_mariadb;;
+    db-prompt)
+        shift
+        db_prompt $@;;
+    version)
+        version ;;
+    help)
+        help ;;
+    *)
+        usage ;;
 esac
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index b3b888c1f..57052bc29 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -65,7 +65,6 @@ HOST_MOUNT_DIR=/data/dockerMount
 
 # sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
 
-
 dockerOpts="
 --detach \
 --network=host \
@@ -76,6 +75,7 @@ dockerOpts="
 --mount type=bind,source=$HOST_MOUNT_DIR/certs,target=/data/certs \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database,target=/data/srv/mariadb/current/install/database \
 --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs,target=/data/srv/mariadb/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/admin/mariadb,target=/data/admin/mariadb/ \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent/ \
 "
 
diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh
index 62920288d..82965cc9f 100755
--- a/docker/pypi/wmagent-mariadb/run.sh
+++ b/docker/pypi/wmagent-mariadb/run.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-echo "start sleeping....zzz"
-while true; do sleep 10; done
 
+manage init-mariadb 2>&1 | tee -a run.log
+manage start-mariadb 2>&1 | tee -a run.log
 
-# # start the service
-# manage start
+echo "Start sleeping....zzz"
+while true; do sleep 10; done
diff --git a/docker/pypi/wmagent-mariadb/start-mariadb.sh b/docker/pypi/wmagent-mariadb/start-mariadb.sh
deleted file mode 100755
index b60a6ac9e..000000000
--- a/docker/pypi/wmagent-mariadb/start-mariadb.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#/bin/bash
-
-### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
-###       !!!! NOTHING MUST STAY HERE !!!!
-###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
-###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
-mariadbRoot=root
-mariadbRootPass=fixme
-mariadbUser=cmst1
-mariadbUserPass=fixme
-
-configDir=/data/srv/mariadb/current/config
-dataDir=/data/srv/mariadb/current/install/database
-logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/mariadb.sock
-agentDb=wmagent
-
-echo -------------------------------------------------------------------------
-echo Stopping any previously running mariadb server
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
-mariadb-admin -u $mariadbUser --socket=$socket shutdown
-echo
-
-
-# echo -------------------------------------------------------------------------
-# echo Installing system database
-mariadb-install-db --datadir=$dataDir
-# echo
-
-
-
-manage start-mariadb
-
-# echo -------------------------------------------------------------------------
-# echo starting the server
-# mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
-#       --datadir=$dataDir \
-#       --log-bin \
-#       --socket=$socket \
-#       --log-error=$logDir/error.log \
-#       --pid-file=$logDir/mariadbd.pid  &
-# echo ...
-# sleep 10
-# echo
-
-
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbRoot and removing temp databases
-sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbUser and removing temp databases
-mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
-
-echo
-
-echo -------------------------------------------------------------------------
-echo creating new users and setting grants
-# try to create a user different than root (if it does not already exist), and grant privileges
-# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
-
-echo -------------------------------------------------------------------------
diff --git a/docker/pypi/wmagent-mariadb/start-mysql.sh b/docker/pypi/wmagent-mariadb/start-mysql.sh
deleted file mode 100755
index 8cb0f6eda..000000000
--- a/docker/pypi/wmagent-mariadb/start-mysql.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#/bin/bash
-
-### NOTE: !!!! All OF THIS IS TO BE REMOVED !!!!!
-###       !!!! NOTHING MUST STAY HERE !!!!
-###       THIS IS JUST A PLACEHOLDER OF ALL THE STEPS THAT
-###       NEED TO BE PERFORMED AT THE MARIADB DOCKER IMAGE
-mariadbRoot=root
-mariadbRootPass=fixme
-mariadbUser=cmst1
-mariadbUserPass=fixme
-
-configDir=/data/srv/mariadb/current/config
-dataDir=/data/srv/mariadb/current/install/database
-logDir=/data/srv/mariadb/current/logs
-socket=/data/srv/mariadb/current/logs/mariadb.sock
-agentDb=wmagent
-
-echo -------------------------------------------------------------------------
-echo Stopping any previously running mariadb server
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass -h 127.0.0.1 shutdown
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass --socket=$socket shutdown
-mariadb-admin -u $mariadbUser --socket=$socket shutdown
-echo
-
-
-# echo -------------------------------------------------------------------------
-# echo Installing system database
-mariadb-install-db --datadir=$dataDir
-# echo
-
-
-echo -------------------------------------------------------------------------
-echo starting the server
-mariadbd-safe --defaults-extra-file=$configDir/my.cnf \
-      --datadir=$dataDir \
-      --log-bin \
-      --socket=$socket \
-      --log-error=$logDir/error.log \
-      --pid-file=$logDir/mariadbd.pid  & # > /dev/null 2>&1 < /dev/null &
-echo ...
-sleep 10
-echo
-
-
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbRoot and removing temp databases
-sudo mariadb-admin -u $mariadbRoot password $mariadbRootPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo Securing $mariadbUser and removing temp databases
-mariadb-admin -u $mariadbUser password $mariadbUserPass --socket=$socket
-# mariadb-admin -u $mariadbRoot --password=$mariadbRootPass  -h 127.0.0.1 password $mariadbRootPass
-# mariadb-secure-installation --socket=$socket
-echo
-
-echo -------------------------------------------------------------------------
-echo creating agent databases
-echo "Installing WMAgent Database: $agentDb"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "create database $agentDb"
-
-echo
-
-echo -------------------------------------------------------------------------
-echo creating new users and setting grants
-# try to create a user different than root (if it does not already exist), and grant privileges
-# we need ${mariadbUser}'@'127.0.0.1 user in paralel to ${mariadbUser}'@'localhost
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'localhost' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@localhost WITH GRANT OPTION"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "CREATE USER '${mariadbUser}'@'127.0.0.1' IDENTIFIED BY '$mariadbUserPass'"
-mariadb -u $mariadbUser --password=$mariadbUserPass --socket=$socket --execute "GRANT ALL ON *.* TO $mariadbUser@127.0.0.1 WITH GRANT OPTION"
-
-echo -------------------------------------------------------------------------

From 40ebe3b3d31e7c29ed9e68cb7bdacc793c717f09 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Mon, 26 Feb 2024 19:25:22 +0100
Subject: [PATCH 15/45] Review comments

---
 docker/pypi/wmagent-mariadb/Dockerfile | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 7a946a56d..583d90b8a 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,16 +1,13 @@
-ARG TAG=11.0
-ARG MDB_TAG=$TAG
-FROM mariadb:${MDB_TAG}
+ARG MDB_TAG=10.0
+FROM mariadb:$MDB_TAG
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
 
-ARG MDB_TAG=$TAG
-ENV MDB_TAG=${MDB_TAG}
+ARG MDB_TAG
+ENV MDB_TAG=$MDB_TAG
 RUN echo MDB_TAG=$MDB_TAG
 
-RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip
-
-# Install some debugging tools
-RUN apt-get install -y hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
+RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip \
+    hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
 
 ENV USER=cmst1
 # ENV MDB_PORT=
@@ -37,7 +34,6 @@ ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
 ENV WMA_DATABASE=wmagent
 
-
 RUN useradd -u $UID -m $USER
 
 # add user to sudoers file
@@ -46,8 +42,6 @@ RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 # start the setup
 RUN mkdir -p $MDB_ROOT_DIR
 
-ENV PATH="${MDB_ROOT_DIR}:${PATH}"
-
 RUN mkdir -p $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
 RUN ln -s $MDB_CURRENT_DIR $MDB_BASE_DIR/current
 
@@ -59,7 +53,7 @@ RUN ln -s ${MDB_MANAGE_DIR}/manage ${MDB_ROOT_DIR}/manage
 # The $MDB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
 ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
-ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"
+ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}"
 
 RUN <<EOF cat >> /home/${USER}/.bashrc
 

From be369bb94193dc6bf3619eee5f4748b08b152487 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Tue, 27 Feb 2024 11:28:19 +0100
Subject: [PATCH 16/45] Add extra login name checks for
 mariadb-docker-push/run.sh && Review comments.

---
 docker/pypi/wmagent-mariadb/Dockerfile        | 14 ++++----
 .../wmagent-mariadb/mariadb-docker-build.sh   | 32 +++++++++++++------
 .../wmagent-mariadb/mariadb-docker-run.sh     | 15 ++++++---
 3 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 583d90b8a..01720cedc 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -34,15 +34,12 @@ ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
 ENV WMA_DATABASE=wmagent
 
-RUN useradd -u $UID -m $USER
-
-# add user to sudoers file
-RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+# create the system user to run the database and add it to the sudoers file
+RUN useradd -u $UID -m $USER && echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 # start the setup
-RUN mkdir -p $MDB_ROOT_DIR
-
-RUN mkdir -p $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
+RUN mkdir -p $MDB_ROOT_DIR $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR \
+    $MDB_LOG_DIR $MDB_DATABASE_DIR $MDB_STATE_DIR $MDB_AUTH_DIR
 RUN ln -s $MDB_CURRENT_DIR $MDB_BASE_DIR/current
 
 # add necessary scripts
@@ -50,7 +47,8 @@ ADD run.sh ${MDB_ROOT_DIR}/
 ADD manage ${MDB_MANAGE_DIR}/manage
 RUN ln -s ${MDB_MANAGE_DIR}/manage ${MDB_ROOT_DIR}/manage
 
-# The $MDB_CONFIG_DIR is to be mounted from the host and my.cnf read from there
+# The $MDB_CONFIG_DIR is NOT to be mounted from the host
+# and the my.cnf file is going to be accessible only from the container
 ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
 ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}"
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
index 9e59c1d94..50710fc3e 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -47,20 +47,32 @@ while getopts ":t:hpl" opt; do
 done
 
 
-# NOTE: NO MDB_TAG validation is done in the current script. It is implemented at the install.sh
-
-dockerOpts=" --network=host --progress=plain --build-arg MDB_TAG=$MDB_TAG "
+# NOTE: NO MDB_TAG validation is done in the current script. The proper tag
+#       to be used should be taken from: https://mariadb.org/mariadb/all-releases/
+dockerOpts=" --network host --progress=plain --build-arg MDB_TAG=$MDB_TAG "
 
 docker build $dockerOpts -t local/mariadb:$MDB_TAG -t local/mariadb:latest  .
 
 $PUSH && {
-    docker login registry.cern.ch
-    docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:$MDB_TAG
-    echo "Uploading image registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
-    docker push registry.cern.ch/cmsweb/mariadb:$MDB_TAG
+    # For security reasons we check if the login name and the current user match.
+    # If they do not, abort the execution and push nothing to registry.cern.ch.
+    loginUser=`logname`
+    currUser=`id -un`
+    registry=registry.cern.ch
+    [[ $loginUser == $currUser ]] || {
+        echo "ERROR: The CURRENT and the LOGIN users do not match!"
+        echo "ERROR: You MUST connect to $registry with your login user rather than with $currUser"
+        exit 1
+    }
+    echo "Connecting to $registry with Username: $loginUser"
+    docker login -u $loginUser $registry
+    docker tag local/mariadb:$MDB_TAG $registry/cmsweb/mariadb:$MDB_TAG
+    echo "Uploading image $registry/cmsweb/mariadb:$MDB_TAG"
+    docker push $registry/cmsweb/mariadb:$MDB_TAG
     $LATEST &&  {
-        docker tag mariadb:$MDB_TAG registry.cern.ch/cmsweb/mariadb:latest
-        echo "Uploading image registry.cern.ch/cmsweb/mariadb:latest"
-        docker push registry.cern.ch/cmsweb/mariadb:latest
+        docker tag local/mariadb:$MDB_TAG $registry/cmsweb/mariadb:latest
+        echo "Uploading image $registry/cmsweb/mariadb:latest"
+        docker push $registry/cmsweb/mariadb:latest
     }
+    docker logout $registry
 }
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index 57052bc29..79f8c0d2f 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -84,15 +84,20 @@ dockerOpts="
 # mariadbOpts=$*
 # mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
 
+registry=local
+repository=mariadb
+
 $PULL && {
+    registry=registry.cern.ch
+    project=cmsweb
+    repository=mariadb
     echo "Pulling Docker image: registry.cern.ch/cmsweb/mariadb:$MDB_TAG"
-    docker login registry.cern.ch
-    docker pull registry.cern.ch/cmsweb/mariadb:$MDB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:$MDB_TAG
-    docker tag registry.cern.ch/cmsweb/mariadb:$MDB_TAG local/mariadb:latest
+    docker pull $registry/$project/$repository:$MDB_TAG
+    docker tag  $registry/$project/$repository:$MDB_TAG $registry/$repository:$MDB_TAG
+    docker tag  $registry/$project/$repository:$MDB_TAG $registry/$repository:latest
 }
 
 echo "Starting the mariadb:$MDB_TAG docker container with the following parameters: $mariadbOpts"
-docker run $dockerOpts $mariadbOpts local/mariadb:$MDB_TAG && (
+docker run $dockerOpts $mariadbOpts $registry/$repository:$MDB_TAG && (
     [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
     ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )

From 8fd215554c2b708dc17129184f321744132db54e Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 28 Feb 2024 14:13:08 +0100
Subject: [PATCH 17/45] Resolve the user to run the database server at runtime

Set Default tag to 10.6.5
---
 docker/pypi/wmagent-mariadb/Dockerfile        | 32 +++++++++----------
 docker/pypi/wmagent-mariadb/manage            | 13 ++++----
 .../wmagent-mariadb/mariadb-docker-run.sh     | 11 +++----
 docker/pypi/wmagent-mariadb/run.sh            |  4 +--
 4 files changed, 27 insertions(+), 33 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index 01720cedc..a390796fe 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,4 +1,4 @@
-ARG MDB_TAG=10.0
+ARG MDB_TAG=10.6.5
 FROM mariadb:$MDB_TAG
 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
 
@@ -9,9 +9,7 @@ RUN echo MDB_TAG=$MDB_TAG
 RUN apt-get update && apt-get install -y vim less sudo wget unzip python3 pip \
     hostname net-tools iputils-ping procps emacs-nox tcpdump && apt-get clean
 
-ENV USER=cmst1
 # ENV MDB_PORT=
-ENV UID=31961
 ENV MDB_ROOT_DIR=/data
 
 ENV MDB_BASE_DIR=$MDB_ROOT_DIR/srv/mariadb
@@ -29,13 +27,14 @@ ENV MDB_CONFIG_DIR=$MDB_CURRENT_DIR/config
 ENV MDB_LOG_DIR=$MDB_CURRENT_DIR/logs
 ENV MDB_DEPLOY_DIR=/usr/local
 ENV MDB_ENV_FILE=$MDB_DEPLOY_DIR/deploy/env.sh
-ENV MDB_SOCKET_FILE=$MDB_CURRENT_DIR/mariadb.sock
+ENV MDB_SOCKET_FILE=/var/run/mysqld/mariadb.sock
 ENV MDB_SECRETS_FILE=$MDB_ADMIN_DIR/MariaDB.secrets
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
 ENV WMA_DATABASE=wmagent
 
-# create the system user to run the database and add it to the sudoers file
-RUN useradd -u $UID -m $USER && echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+# create the system user to run the database
+RUN groupadd -g 1399 zh
+RUN useradd -u 31961  -g 1399 -G 999 -m cmst1
 
 # start the setup
 RUN mkdir -p $MDB_ROOT_DIR $MDB_CURRENT_DIR $MDB_CONFIG_DIR $MDB_MANAGE_DIR \
@@ -53,21 +52,20 @@ ADD my.cnf ${MDB_CONFIG_DIR}/my.cnf
 
 ENV PATH="/usr/local/bin/:${MDB_ROOT_DIR}:${PATH}"
 
-RUN <<EOF cat >> /home/${USER}/.bashrc
-
-alias lll="ls -lathr"
-alias ls="ls --color=auto"
-alias ll='ls -la --color=auto'
-
+# set MariaDB docker specific bash prompt and manage alias for all users:
+RUN <<EOF cat >>/root/.bashrc
 alias manage=$MDB_MANAGE_DIR/manage
+export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$([[ \$(id -u) -eq 0 ]] && echo \# || echo \$) "
+EOF
 
-# set MariaDB docker specific bash prompt:
-export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$ "
+RUN <<EOF cat >>/home/cmst1/.bashrc
+alias manage=$MDB_MANAGE_DIR/manage
+export PS1="(MariaDB-$MDB_TAG) [\u@\h:\W]\$([[ \$(id -u) -eq 0 ]] && echo \# || echo \$) "
 EOF
 
-RUN chown -R ${USER} ${MDB_ROOT_DIR}
+# RUN chown -R ${USER} ${MDB_ROOT_DIR}
 
 # setup final environment
-USER $USER
+# USER $USER
 WORKDIR $MDB_ROOT_DIR
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["./run.sh", "2>&1"]
diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index 25281a923..3fc0ac8d3 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -54,8 +54,6 @@ _load_secrets(){
 status(){
     mariadb-admin --socket=$MDB_SOCKET_FILE version
     echo
-    mariadb-admin --socket=$MDB_SOCKET_FILE status
-    echo
 }
 
 start_mariadb(){
@@ -98,14 +96,15 @@ init_mariadb(){
 
     echo -------------------------------------------------------------------------
     echo Stopping any previously running mariadb server
-    mariadb-admin -u $MDB_USER --socket=$MDB_SOCKET_FILE shutdown
+    mariadb-admin -u $MDB_ROOT --socket=$MDB_SOCKET_FILE shutdown
     echo
 
     echo -------------------------------------------------------------------------
-    echo Trying to install system database if it is not present already
+    echo "Trying to install system database with user: $USER (if it is not already present)"
 
-    errMsg=$(mariadb-install-db --skip-test-db --datadir=$MDB_DATABASE_DIR)
+    errMsg=$(mariadb-install-db --skip-test-db --user=$USER --datadir=$MDB_DATABASE_DIR)
     err=$?
+    echo $errMsg
 
     if [[ $err -ne 0 ]]; then
         echo "ERROR: Could not create system and user databases."
@@ -121,8 +120,8 @@ init_mariadb(){
     start_mariadb
 
     echo -------------------------------------------------------------------------
-    echo Securing MariaDB Root users
-    sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
+    echo Securing MariaDB Root user
+    # sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     mariadb-admin -u $MDB_ROOT password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     echo
 
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index 79f8c0d2f..f512ea114 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -50,8 +50,8 @@ while getopts ":t:hp" opt; do
 done
 
 
-mariadbUser=cmst1
-mariadbOpts=" --user $mariadbUser"
+mariadbUser=`id -un`
+mariadbOpts=" --user $mariadbUser -e USER=$mariadbUser"
 
 # This is the root at the host only, it may differ from the root inside the container.
 # NOTE: this may be parametriesed, so that the container can run on a different mount point.
@@ -63,7 +63,7 @@ HOST_MOUNT_DIR=/data/dockerMount
 [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/install/database ;} || exit $?
 [[ -d $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/logs ;} || exit $?
 
-# sudo chown -R $mariadbUser:$mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
+# sudo chown -R $mariadbUser $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG
 
 dockerOpts="
 --detach \
@@ -81,9 +81,6 @@ dockerOpts="
 
 # --mount type=bind,source=$HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG/config,target=/data/srv/mariadb/current/config \
 
-# mariadbOpts=$*
-# mariadbOpts="$mariadbOpts --user mariadb -e MARIADB_USER=TestAdmin -e MARIADB_PASSWORD=TestPass"
-
 registry=local
 repository=mariadb
 
@@ -97,7 +94,7 @@ $PULL && {
     docker tag  $registry/$project/$repository:$MDB_TAG $registry/$repository:latest
 }
 
-echo "Starting the mariadb:$MDB_TAG docker container with the following parameters: $mariadbOpts"
+echo "Starting the $registry/$repository:$MDB_TAG docker container with the following parameters: $mariadbOpts"
 docker run $dockerOpts $mariadbOpts $registry/$repository:$MDB_TAG && (
     [[ -h $HOST_MOUNT_DIR/srv/mariadb/current ]] && rm -f $HOST_MOUNT_DIR/srv/mariadb/current
     ln -s $HOST_MOUNT_DIR/srv/mariadb/$MDB_TAG $HOST_MOUNT_DIR/srv/mariadb/current )
diff --git a/docker/pypi/wmagent-mariadb/run.sh b/docker/pypi/wmagent-mariadb/run.sh
index 82965cc9f..d9ce91b8f 100755
--- a/docker/pypi/wmagent-mariadb/run.sh
+++ b/docker/pypi/wmagent-mariadb/run.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-manage init-mariadb 2>&1 | tee -a run.log
-manage start-mariadb 2>&1 | tee -a run.log
+manage init-mariadb  2>&1 | tee -a $MDB_LOG_DIR/run.log
+manage start-mariadb 2>&1 | tee -a $MDB_LOG_DIR/run.log
 
 echo "Start sleeping....zzz"
 while true; do sleep 10; done

From 2b6572b39be8501b8a977fb346e233f1cb3285e3 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 28 Feb 2024 14:28:31 +0100
Subject: [PATCH 18/45] Review comments

---
 docker/pypi/wmagent-mariadb/Dockerfile              | 2 +-
 docker/pypi/wmagent-mariadb/mariadb-docker-build.sh | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/Dockerfile b/docker/pypi/wmagent-mariadb/Dockerfile
index a390796fe..da0e79052 100644
--- a/docker/pypi/wmagent-mariadb/Dockerfile
+++ b/docker/pypi/wmagent-mariadb/Dockerfile
@@ -1,6 +1,6 @@
 ARG MDB_TAG=10.6.5
 FROM mariadb:$MDB_TAG
-MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+MAINTAINER Todor Ivanov todor.ivanov@cern.ch
 
 ARG MDB_TAG
 ENV MDB_TAG=$MDB_TAG
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
index 50710fc3e..d5054e014 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -64,8 +64,12 @@ $PUSH && {
         echo "ERROR: You MUST connect to $registry with your login user rather than with $currUser"
         exit 1
     }
-    echo "Connecting to $registry with Username: $loginUser"
-    docker login -u $loginUser $registry
+    echo "Testing for existing login session to $registry with Username: $loginUser"
+    docker login $registry < /dev/null >/dev/null 2>&1 || {
+        echo "ERROR: A valid login session to $registry is required in order to be able to upload any docker image"
+        echo "ERROR: Please consider running 'docker login $registry' with USER:$currUser and retry again."
+        exit 1
+    }
     docker tag local/mariadb:$MDB_TAG $registry/cmsweb/mariadb:$MDB_TAG
     echo "Uploading image $registry/cmsweb/mariadb:$MDB_TAG"
     docker push $registry/cmsweb/mariadb:$MDB_TAG
@@ -74,5 +78,4 @@ $PUSH && {
         echo "Uploading image $registry/cmsweb/mariadb:latest"
         docker push $registry/cmsweb/mariadb:latest
     }
-    docker logout $registry
 }

From 9fa3b84470fc8e14cc56a56a6e86ec1cd8b43649 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 10:34:26 +0100
Subject: [PATCH 19/45] Alan's review comments && Add README

---
 docker/pypi/wmagent-mariadb/manage            | 52 ++++++++++++-------
 .../wmagent-mariadb/mariadb-docker-build.sh   | 10 ++--
 .../wmagent-mariadb/mariadb-docker-run.sh     |  6 +--
 docker/pypi/wmagent-mariadb/my.cnf            |  7 +--
 4 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/manage b/docker/pypi/wmagent-mariadb/manage
index 3fc0ac8d3..dc4a7cf73 100755
--- a/docker/pypi/wmagent-mariadb/manage
+++ b/docker/pypi/wmagent-mariadb/manage
@@ -57,7 +57,11 @@ status(){
 }
 
 start_mariadb(){
-    echo starting MariaDB server
+    mariadb-admin --socket=$MDB_SOCKET_FILE status >/dev/null 2>&1 && {
+        echo "$FUNCNAME: WARNING: MariaDB Server already running on --socket=$MDB_SOCKET_FILE"
+        return
+    }
+    echo "$FUNCNAME: Starting MariaDB server"
     mariadbd-safe --defaults-extra-file=$MDB_CONFIG_DIR/my.cnf \
                   --datadir=$MDB_DATABASE_DIR \
                   --log-bin \
@@ -74,11 +78,21 @@ stop_mariadb(){
 }
 
 db_prompt(){
-    mariadb --socket=$MDB_SOCKET_FILE --database=$wmaDBName --pager='less -SFX'
+    mariadb --socket=$MDB_SOCKET_FILE --database=$WMA_DATABASE --pager='less -SFX'
 }
 
 clean_mariadb(){
-    db_prompt "drop database $wmaDBName"
+    echo
+    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $WMA_DATABASE DATABASE."
+    echo -n "$FUNCNAME: Continue? [n]: "
+    read x && [[ $x =~ (y|yes|yeS|yEs|Yes|yES|YEs|YeS|YES|Y) ]] || return 102
+    echo "$FUNCNAME: ..."
+    echo "$FUNCNAME: You still have 5 sec. to cancel before we proceed."
+    echo
+    sleep 5
+    echo "$FUNCNAME: DROPPING $WMA_DATABASE DATABASE!"
+    mariadb --socket=$MDB_SOCKET_FILE -e "drop database $WMA_DATABASE"
+    mariadb --socket=$MDB_SOCKET_FILE -e "create database $WMA_DATABASE"
 }
 
 version(){
@@ -89,56 +103,56 @@ version(){
 init_mariadb(){
     # The function to set/check initial database configurations and user preveleges
     [[ $USER == $MDB_ROOT ]] || {
-        echo "ERROR: The current user does not match the MariaDB root user from $MDB_SECRETS_FILE."
-        echo "ERROR: Canot continue. Exit..."
+        echo "$FUNCNAME: ERROR: The current user does not match the MariaDB root user from $MDB_SECRETS_FILE."
+        echo "$FUNCNAME: ERROR: Cannot continue. Exit..."
         exit 1
     }
 
     echo -------------------------------------------------------------------------
-    echo Stopping any previously running mariadb server
+    echo "$FUNCNAME: Stopping any previously running mariadb server"
     mariadb-admin -u $MDB_ROOT --socket=$MDB_SOCKET_FILE shutdown
     echo
 
     echo -------------------------------------------------------------------------
-    echo "Trying to install system database with user: $USER (if it is not already present)"
+    echo "$FUNCNAME: Trying to install system database with user: $USER (if it is not already present)"
 
     errMsg=$(mariadb-install-db --skip-test-db --user=$USER --datadir=$MDB_DATABASE_DIR)
     err=$?
     echo $errMsg
 
     if [[ $err -ne 0 ]]; then
-        echo "ERROR: Could not create system and user databases."
-        echo "ERROR: $errMsg"
+        echo "$FUNCNAME: ERROR: Could not create system and user databases."
+        echo "$FUNCNAME: ERROR: $errMsg"
         exit $err
     elif echo $errMsg|grep -i "exists" ; then
-        echo "WARNING: System and user databases already exist. NOT trying to create them."
+        echo "$FUNCNAME: WARNING: System and user databases already exist. NOT trying to create them."
         return $err
     fi
 
     echo -------------------------------------------------------------------------
-    echo Starting MariaDB server
+    echo "$FUNCNAME: Starting MariaDB server"
     start_mariadb
 
     echo -------------------------------------------------------------------------
-    echo Securing MariaDB Root user
+    echo "$FUNCNAME: Securing MariaDB Root user"
     # sudo mariadb-admin -u root password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     mariadb-admin -u $MDB_ROOT password $MDB_ROOTPASS --socket=$MDB_SOCKET_FILE
     echo
 
     echo -------------------------------------------------------------------------
-    echo Creating WMAgent databases
-    echo "Installing WMAgent Database: $WMA_DATABASE"
+    echo "$FUNCNAME: Creating WMAgent databases"
+    echo "$FUNCNAME: Installing WMAgent Database: $WMA_DATABASE"
     mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "create database $WMA_DATABASE"
     echo
 
     echo -------------------------------------------------------------------------
-    echo Creating WMAgent user and setting grants
+    echo "$FUNCNAME: Creating WMAgent user and setting grants"
     # try to create a user different than root (if it does not already exist), and grant privileges
     # we need ${MDB_USER}'@'127.0.0.1 user in paralel to ${MDB_USER}'@'localhost
 
     if [[ $MDB_USER == $MDB_ROOT ]]; then
-        echo "WARNING: WMAgent user set is the same as the MariaDB Root user. You must configure a different one!"
-        echo "WARNING: NOT creating WMAgent users and NOT granting priveleges to $WMA_DATABASE database"
+        echo "$FUNCNAME: WARNING: WMAgent user set is the same as the MariaDB Root user. You must configure a different one!"
+        echo "$FUNCNAME: WARNING: NOT creating WMAgent users and NOT granting priveleges to $WMA_DATABASE database"
         return 1
     else
         mariadb -u $MDB_ROOT --password=$MDB_ROOTPASS --socket=$MDB_SOCKET_FILE --execute "CREATE USER '${MDB_USER}'@'localhost' IDENTIFIED BY '$MDB_PASS'"
@@ -152,13 +166,13 @@ init_mariadb(){
 
 _load_secrets $MDB_SECRETS_FILE "MDB_ROOT MDB_ROOTPASS" || {
     err=$?
-    echo "ERROR: Could not properly load root password for MariaDB"
+    echo "$FUNCNAME: ERROR: Could not properly load root password for MariaDB"
     exit $err
 }
 
 _load_secrets $WMA_SECRETS_FILE "MDB_USER MDB_PASS" || {
     err=$?
-    echo "ERROR: Could not properly load WMAgent User password for MariaDB"
+    echo "$FUNCNAME: ERROR: Could not properly load WMAgent User password for MariaDB"
     exit $err
 }
 
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
index d5054e014..729e90431 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-build.sh
@@ -10,13 +10,13 @@ help(){
 
 The MariaDB docker build script for Docker image creation based on pypi:
 
-Usage: mariadb-docker-build.sh -v <mariadb_tag>
+Usage: mariadb-docker-build.sh -t <mariadb_tag> [-p] [-l]
 
-    -t <mariadb_tag>                The MariaDB version/tag to be used for the Docker image creation
-    -p <push_image>                 Push the image to registry.cern.ch
-    -l <latest_tag_toregistry>      Push the curernt tag also as latest to registry.cern.ch
+    -t <mariadb_tag>  The MariaDB version/tag to be used for the Docker image creation
+    -p                Push the image to registry.cern.ch
+    -l                Push the curernt tag also as latest to registry.cern.ch
 
-Example: ./mariadb-docker-build.sh -v 2.2.0.2
+Example: ./mariadb-docker-build.sh -t 2.2.0.2
 
 EOF
 }
diff --git a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
index f512ea114..bdf63fcdc 100755
--- a/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
+++ b/docker/pypi/wmagent-mariadb/mariadb-docker-run.sh
@@ -17,11 +17,11 @@ help(){
 The script to be used for running a Mariadb docker container at a VM. The full set of arguments
 passed to the current script are to be forwarded to the Mariadb container entrypoint 'run.sh'
 
-Usage: mariadb-docker-run.sh [-t <team_name>] [-n <agent_number>] [-f <db_flavour>]
+Usage: mariadb-docker-run.sh [-t <mariadb_tag>] [-p]
 
-    -p <pull_image>   Pull the image from registry.cern.ch
     -t <mariadb_tag>  The Mariadb version/tag to be downloaded from registry.cern.ch [Default:latest]
-    -h <help>
+    -p                Pull the image from registry.cern.ch
+    -h                Help
 
 Example: ./mariadb-docker-run.sh -t 3.2.2
 
diff --git a/docker/pypi/wmagent-mariadb/my.cnf b/docker/pypi/wmagent-mariadb/my.cnf
index 2d8466311..cbcf065fa 100644
--- a/docker/pypi/wmagent-mariadb/my.cnf
+++ b/docker/pypi/wmagent-mariadb/my.cnf
@@ -42,16 +42,13 @@ innodb_read_io_threads = 4
 # default: 4
 innodb_write_io_threads = 4
 # default: full_crc23
-# Commented out due  to old mariadb version
-# innodb_checksum_algorithm=full_crc32
+innodb_checksum_algorithm=full_crc32
 # default: 1
 innodb_doublewrite=0
 
 innodb_log_file_size=512M
 innodb_log_buffer_size=8M
-# Changed for small testing machines
-# innodb_buffer_pool_size=2G
-innodb_buffer_pool_size=50M
+innodb_buffer_pool_size=2G
 # default: 30
 innodb_sync_spin_loops=60
 # default: 0

From 5aa9131302359c5744aa25f1d43176623b9506a5 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 10:35:40 +0100
Subject: [PATCH 20/45] Add README

---
 docker/pypi/wmagent-mariadb/README | 221 +++++++++++++++++++++++++++++
 1 file changed, 221 insertions(+)
 create mode 100644 docker/pypi/wmagent-mariadb/README

diff --git a/docker/pypi/wmagent-mariadb/README b/docker/pypi/wmagent-mariadb/README
new file mode 100644
index 000000000..9316f9993
--- /dev/null
+++ b/docker/pypi/wmagent-mariadb/README
@@ -0,0 +1,221 @@
+# MariaDB default image for running WMAgent
+
+## Prerequisites
+
+This image inherits from the mainstream `mariadb` one, and follows the same
+tagging schema. On top of the base `mariadb` image we add all the structure
+needed for running the WMAgent with MariaDB and two main scripts:
+
+* `mariadb-docker-run.sh`
+* `mariadb-docker-build.sh`
+
+For building the containers, and for creating the mount area at the host and the
+the bind mounts inside the container, respectively. Those are as follows:
+
+* At the host:
+```
+/data/dockerMount/{admin|srv}/mariadb
+```
+* At the container:
+
+```
+/data/{admin|srv}/mariadb
+```
+
+Upon starting the container we try to initialize the default user and system
+databases, which if previously created should exist in the host mount area. And
+the last steps are creating the `wmagent` database.
+
+There are no other external dependencies.
+
+We fetch all the passwords from two secrets files:
+
+* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials for the
+  user to be used by the WMAgent to connect to the datbase
+* `/data/admin/mariadb/MariaDB.secrets` - for reading the the credentials for
+    the root user who is about to have full administrative rights on the MariaDB
+    server
+    **NOTE:** The server admin user configured at the `MariaDB.secrets` file,
+        must match the username of the one who is to run the server inside the
+        container. And the later is resolved at runtime, depending on where we
+        run the container, it could be on of the three:
+   * CERN - production agent
+   * CERN - T0 agent
+   * FNAL
+
+## Usage
+
+### Building MariaDB image
+
+We can build everything locally and upload it at the CERN registry: https://registry.cern.ch
+
+* Using the wrapper script to build MariaDB locally:
+```
+$ ssh vocms****
+user@vocms0290:wmagent-mariadb $ cd /data
+user@vocms0290:wmagent-mariadb $ git clone https://github.com/dmwm/CMSKubernetes.git
+user@vocms0290:wmagent-mariadb $ cd /data/CMSKubernetes/docker/pypi/wmagent-mariadb/
+user@vocms0290:wmagent-mariadb $ ./mariadb-docker-build.sh -t 10.6.5
+
+user@vocms0290:wmagent-mariadb $ docker image  ls
+REPOSITORY                 TAG       IMAGE ID       CREATED          SIZE
+local/mariadb              10.6.5    4efa646aea3e   6 minutes ago    950MB
+local/mariadb              latest    4efa646aea3e   6 minutes ago    950MB
+```
+* Using the wrapper script to build and upload MariaDB to registry.cern.ch:
+```
+./mariadb-docker-build.sh -t 10.6.5 -p
+```
+
+### Running a MariaDB container
+
+We can run from local repository or from upstream CERN registry. The set of
+images one may end up working may look like:
+
+```
+cmst1@vocms0290:wmagent-mariadb $ docker image  ls
+REPOSITORY                 TAG       IMAGE ID       CREATED          SIZE
+local/mariadb              10.6.5    4efa646aea3e   6 minutes ago    950MB
+local/mariadb              latest    4efa646aea3e   6 minutes ago    950MB
+registry.cern.ch/mariadb   10.6.5    8539e03b7a1d   21 minutes ago   950MB
+registry.cern.ch/mariadb   latest    8539e03b7a1d   21 minutes ago   950MB
+```
+
+* Running from a local build:
+
+```
+cmst1@vocms0290:wmagent-mariadb $ ./mariadb-docker-run.sh -t 10.6.5
+Starting the mariadb:10.6.5 docker container with the following parameters:  --user cmst1
+eb7e0d879d4d7fa597587c734837c5289886a6aaf6a82c072187371fdf312b90
+
+cmst1@vocms0290:wmagent-mariadb $ docker ps
+CONTAINER ID   IMAGE                  COMMAND      CREATED         STATUS         PORTS     NAMES
+eb7e0d879d4d   local/mariadb:10.6.5   "./run.sh"   3 seconds ago   Up 2 seconds             mariadb
+```
+
+* Running from CERN registry:
+```
+cmst1@vocms0290:wmagent-mariadb $ ./mariadb-docker-run.sh -t 10.6.5 -p
+Pulling Docker image: registry.cern.ch/cmsweb/mariadb:10.6.5
+10.6.5: Pulling from cmsweb/mariadb
+Digest: sha256:61f798b55a1c743686e1568509975308dc07b5b24486894053d6a312983c4af6
+Status: Downloaded newer image for registry.cern.ch/cmsweb/mariadb:10.6.5
+registry.cern.ch/cmsweb/mariadb:10.6.5
+Starting the mariadb:10.6.5 docker container with the following parameters:  --user cmst1
+21d9c6598f35e627834d1b796460047605d6255cebc746d572289c7b418053ed
+
+cmst1@vocms0290:wmagent-mariadb $ docker ps
+CONTAINER ID   IMAGE                             COMMAND      CREATED         STATUS         PORTS     NAMES
+21d9c6598f35   registry.cern.ch/mariadb:10.6.5   "./run.sh"   7 seconds ago   Up 6 seconds             mariadb
+
+```
+
+* Killing the container directly from the host:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker kill mariadb
+mariadb
+
+```
+
+* Connecting to a running container:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker exec -it mariadb bash
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$
+
+```
+
+* Managing the databse service:
+    * General options:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage --help
+
+The manage script of the MariaDB docker image for WMAgent
+
+Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt | version
+
+```
+    * Stat/Stop the database:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
+start_mariadb: Starting MariaDB server
+...
+240301 09:25:54 mysqld_safe Can't log to error log and syslog at the same time.  Remove all --log-error configuration options for --syslog to take effect.
+240301 09:25:54 mysqld_safe Logging to '/data/srv/mariadb/10.6.5/logs/error.log'.
+240301 09:25:54 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.6.5/install/database
+mariadb-admin  Ver 9.1 Distrib 10.6.5-MariaDB, for debian-linux-gnu on x86_64
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Server version		10.6.5-MariaDB-1:10.6.5+maria~focal-log
+Protocol version	10
+Connection		Localhost via UNIX socket
+UNIX socket		/var/run/mysqld/mariadb.sock
+Uptime:			10 sec
+
+Threads: 2  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
+
+```
+If one tries to start a second server on the same socket:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
+start_mariadb: WARNING: MariaDB Server already running on --socket=/var/run/mysqld/mariadb.sock
+
+```
+    * Cleaning the WMAgent database:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage clean-mariadb
+
+clean_mariadb: THE CURRENT OPERATIONS WILL WIPE OUT THE wmagent DATABASE.
+clean_mariadb: Continue? [n]: y
+clean_mariadb: ...
+clean_mariadb: You still have 5 sec. to cancel before we proceed.
+
+clean_mariadb: DROPPING wmagent DATABASE!
+
+```
+
+    * Connecting to the database with the admin user locally from inside the container:
+```
+(MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage db-prompt
+Welcome to the MariaDB monitor.  Commands end with ; or \g.
+Your MariaDB connection id is 5
+Server version: 10.6.5-MariaDB-1:10.6.5+maria~focal-log mariadb.org binary distribution
+
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
+
+MariaDB [wmagent]>
+```
+
+    * Fetching startup logs:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
+-------------------------------------------------------------------------
+Stopping any previously running mariadb server
+mariadb-admin: connect to server at 'localhost' failed
+error: 'Can't connect to local MySQL server through socket '/data/srv/mariadb/10.5/mariadb.sock' (2)'
+Check that mysqld is running and that the socket: '/data/srv/mariadb/10.5/mariadb.sock' exists!
+
+-------------------------------------------------------------------------
+Trying to install system database if it is not present already
+mysql.user table already exists! Run mysql_upgrade, not mysql_install_db
+WARNING: System and user databases already exist. NOT trying to create them.
+starting MariaDB server
+...
+240226 18:24:13 mysqld_safe Logging to '/data/srv/mariadb/10.5/logs/error.log'.
+240226 18:24:13 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.5/install/database
+mariadb-admin  Ver 9.1 Distrib 10.5.24-MariaDB, for debian-linux-gnu on x86_64
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Server version		10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log
+Protocol version	10
+Connection		Localhost via UNIX socket
+UNIX socket		/data/srv/mariadb/10.5/mariadb.sock
+Uptime:			10 sec
+
+Threads: 1  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
+
+Uptime: 10  Threads: 1  Questions: 2  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.200
+
+Start sleeping....zzz
+```
\ No newline at end of file

From 0a9632cbf36d4b714e997772f014ae1c2652e201 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 12:43:08 +0100
Subject: [PATCH 21/45] Rename README to README.md

---
 docker/pypi/wmagent-mariadb/{README => README.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docker/pypi/wmagent-mariadb/{README => README.md} (100%)

diff --git a/docker/pypi/wmagent-mariadb/README b/docker/pypi/wmagent-mariadb/README.md
similarity index 100%
rename from docker/pypi/wmagent-mariadb/README
rename to docker/pypi/wmagent-mariadb/README.md

From 9727aa7bda21c83275ac42ed010c5083034e619f Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 12:49:21 +0100
Subject: [PATCH 22/45] Fixing README formating

---
 docker/pypi/wmagent-mariadb/README.md | 77 ++++++++++++++-------------
 1 file changed, 40 insertions(+), 37 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/README.md b/docker/pypi/wmagent-mariadb/README.md
index 9316f9993..d8328c8e5 100644
--- a/docker/pypi/wmagent-mariadb/README.md
+++ b/docker/pypi/wmagent-mariadb/README.md
@@ -124,8 +124,42 @@ cmst1@vocms0290:wmagent-mariadb $ docker exec -it mariadb bash
 
 ```
 
-* Managing the databse service:
-    * General options:
+* Fetching startup logs:
+```
+cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
+-------------------------------------------------------------------------
+Stopping any previously running mariadb server
+mariadb-admin: connect to server at 'localhost' failed
+error: 'Can't connect to local MySQL server through socket '/data/srv/mariadb/10.5/mariadb.sock' (2)'
+Check that mysqld is running and that the socket: '/data/srv/mariadb/10.5/mariadb.sock' exists!
+
+-------------------------------------------------------------------------
+Trying to install system database if it is not present already
+mysql.user table already exists! Run mysql_upgrade, not mysql_install_db
+WARNING: System and user databases already exist. NOT trying to create them.
+starting MariaDB server
+...
+240226 18:24:13 mysqld_safe Logging to '/data/srv/mariadb/10.5/logs/error.log'.
+240226 18:24:13 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.5/install/database
+mariadb-admin  Ver 9.1 Distrib 10.5.24-MariaDB, for debian-linux-gnu on x86_64
+Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
+
+Server version		10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log
+Protocol version	10
+Connection		Localhost via UNIX socket
+UNIX socket		/data/srv/mariadb/10.5/mariadb.sock
+Uptime:			10 sec
+
+Threads: 1  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
+
+Uptime: 10  Threads: 1  Questions: 2  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.200
+
+Start sleeping....zzz
+```
+
+### Managing the databse service:
+
+* General options:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage --help
 
@@ -134,7 +168,8 @@ The manage script of the MariaDB docker image for WMAgent
 Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt | version
 
 ```
-    * Stat/Stop the database:
+
+* Stat/Stop the database:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
 start_mariadb: Starting MariaDB server
@@ -160,7 +195,8 @@ If one tries to start a second server on the same socket:
 start_mariadb: WARNING: MariaDB Server already running on --socket=/var/run/mysqld/mariadb.sock
 
 ```
-    * Cleaning the WMAgent database:
+
+* Cleaning the WMAgent database:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage clean-mariadb
 
@@ -186,36 +222,3 @@ Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
 MariaDB [wmagent]>
 ```
-
-    * Fetching startup logs:
-```
-cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
--------------------------------------------------------------------------
-Stopping any previously running mariadb server
-mariadb-admin: connect to server at 'localhost' failed
-error: 'Can't connect to local MySQL server through socket '/data/srv/mariadb/10.5/mariadb.sock' (2)'
-Check that mysqld is running and that the socket: '/data/srv/mariadb/10.5/mariadb.sock' exists!
-
--------------------------------------------------------------------------
-Trying to install system database if it is not present already
-mysql.user table already exists! Run mysql_upgrade, not mysql_install_db
-WARNING: System and user databases already exist. NOT trying to create them.
-starting MariaDB server
-...
-240226 18:24:13 mysqld_safe Logging to '/data/srv/mariadb/10.5/logs/error.log'.
-240226 18:24:13 mysqld_safe Starting mariadbd daemon with databases from /data/srv/mariadb/10.5/install/database
-mariadb-admin  Ver 9.1 Distrib 10.5.24-MariaDB, for debian-linux-gnu on x86_64
-Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
-
-Server version		10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log
-Protocol version	10
-Connection		Localhost via UNIX socket
-UNIX socket		/data/srv/mariadb/10.5/mariadb.sock
-Uptime:			10 sec
-
-Threads: 1  Questions: 1  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.100
-
-Uptime: 10  Threads: 1  Questions: 2  Slow queries: 0  Opens: 16  Open tables: 10  Queries per second avg: 0.200
-
-Start sleeping....zzz
-```
\ No newline at end of file

From 731e1bb5d85a58bffee195e3bb4577d3da754d9d Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Fri, 1 Mar 2024 12:57:56 +0100
Subject: [PATCH 23/45] Refine README

Refine README
---
 docker/pypi/wmagent-mariadb/README.md | 32 ++++++++++++++-------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/docker/pypi/wmagent-mariadb/README.md b/docker/pypi/wmagent-mariadb/README.md
index d8328c8e5..14ca5f79e 100644
--- a/docker/pypi/wmagent-mariadb/README.md
+++ b/docker/pypi/wmagent-mariadb/README.md
@@ -30,18 +30,18 @@ There are no other external dependencies.
 
 We fetch all the passwords from two secrets files:
 
-* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials for the
+* `/data/admin/wmagent/WMAgent.secrets` - for reading the credentials of the
   user to be used by the WMAgent to connect to the datbase
-* `/data/admin/mariadb/MariaDB.secrets` - for reading the the credentials for
-    the root user who is about to have full administrative rights on the MariaDB
-    server
-    **NOTE:** The server admin user configured at the `MariaDB.secrets` file,
-        must match the username of the one who is to run the server inside the
-        container. And the later is resolved at runtime, depending on where we
-        run the container, it could be on of the three:
-   * CERN - production agent
+* `/data/admin/mariadb/MariaDB.secrets` - for reading the credentials of the
+  root user who is about to have full administrative rights on the MariaDB server
+
+  **NOTE:** The server admin user configured at the `MariaDB.secrets` file,
+    must match the username of the one who is to run the server inside the
+    container. And the later is resolved at runtime, depending on where we
+    run the container, it could be on of the three:
+   * CERN - WM agent
    * CERN - T0 agent
-   * FNAL
+   * FNAL - WM agent
 
 ## Usage
 
@@ -69,8 +69,8 @@ local/mariadb              latest    4efa646aea3e   6 minutes ago    950MB
 
 ### Running a MariaDB container
 
-We can run from local repository or from upstream CERN registry. The set of
-images one may end up working may look like:
+We can run from the local repository or from upstream CERN registry. The typical
+set of images one could end up working with, may look like this:
 
 ```
 cmst1@vocms0290:wmagent-mariadb $ docker image  ls
@@ -126,7 +126,7 @@ cmst1@vocms0290:wmagent-mariadb $ docker exec -it mariadb bash
 
 * Fetching startup logs:
 ```
-cmst1@vocms0290:wmagent-mariadb $ docker  logs mariadb
+cmst1@vocms0290:wmagent-mariadb $ docker logs mariadb
 -------------------------------------------------------------------------
 Stopping any previously running mariadb server
 mariadb-admin: connect to server at 'localhost' failed
@@ -159,6 +159,8 @@ Start sleeping....zzz
 
 ### Managing the databse service:
 
+All of the commands bellow must be run from inside the container.
+
 * General options:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage --help
@@ -169,7 +171,7 @@ Usage: manage  status | start-mariadb | stop-mariadb | clean-mariadb | db-prompt
 
 ```
 
-* Stat/Stop the database:
+* Start/Stop the database server:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage start-mariadb
 start_mariadb: Starting MariaDB server
@@ -209,7 +211,7 @@ clean_mariadb: DROPPING wmagent DATABASE!
 
 ```
 
-    * Connecting to the database with the admin user locally from inside the container:
+* Connecting to the database with the admin user locally from inside the container:
 ```
 (MariaDB-10.6.5) [cmst1@vocms0290:data]$ manage db-prompt
 Welcome to the MariaDB monitor.  Commands end with ; or \g.

From caf1c889a23f68921e6c84d1a5ecd6b93bfeb907 Mon Sep 17 00:00:00 2001
From: Valentin <vkuznet@gmail.com>
Date: Mon, 4 Mar 2024 19:35:19 +0100
Subject: [PATCH 24/45] New repos for base images from various Linux
 distributions

---
 docker/pypi/alma-base/Dockerfile |  4 ++++
 docker/pypi/alma-base/errors.txt | 14 ++++++++++++++
 docker/pypi/deb-base/Dockerfile  |  4 ++++
 docker/pypi/rh-base/Dockerfile   |  4 ++++
 4 files changed, 26 insertions(+)
 create mode 100644 docker/pypi/alma-base/Dockerfile
 create mode 100644 docker/pypi/alma-base/errors.txt
 create mode 100644 docker/pypi/deb-base/Dockerfile
 create mode 100644 docker/pypi/rh-base/Dockerfile

diff --git a/docker/pypi/alma-base/Dockerfile b/docker/pypi/alma-base/Dockerfile
new file mode 100644
index 000000000..1b2148e9b
--- /dev/null
+++ b/docker/pypi/alma-base/Dockerfile
@@ -0,0 +1,4 @@
+FROM almalinux:latest
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+RUN yum install -y curl-minimal libcurl-minimal vim python3-pycurl pip sudo less \
+    && yum clean all && rm -rf /var/cache/yum
diff --git a/docker/pypi/alma-base/errors.txt b/docker/pypi/alma-base/errors.txt
new file mode 100644
index 000000000..70e1e2973
--- /dev/null
+++ b/docker/pypi/alma-base/errors.txt
@@ -0,0 +1,14 @@
+Step 3/3 : RUN yum install -y curl vim python3 pip sudo less     && yum clean all && rm -rf /var/cache/yum
+ ---> Running in 77dbadded671
+AlmaLinux 9 - AppStream                          12 MB/s | 9.1 MB     00:00
+AlmaLinux 9 - BaseOS                             12 MB/s | 4.7 MB     00:00
+AlmaLinux 9 - Extras                             47 kB/s |  17 kB     00:00
+Package python3-3.9.18-1.el9_3.x86_64 is already installed.
+Package less-590-2.el9_2.x86_64 is already installed.
+Error:
+ Problem: problem with installed package curl-minimal-7.76.1-26.el9_3.2.x86_64
+  - package curl-minimal-7.76.1-26.el9_3.2.x86_64 from @System conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos
+  - package curl-minimal-7.76.1-26.el9.x86_64 from baseos conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos
+  - package curl-minimal-7.76.1-26.el9_3.2.x86_64 from baseos conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos
+  - cannot install the best candidate for the job
+(try to add '--allowerasing' to command line to replace conflicting packages or '--skip-broken' to skip uninstallable packages or '--nobest' to use not only best candidate packages)
diff --git a/docker/pypi/deb-base/Dockerfile b/docker/pypi/deb-base/Dockerfile
new file mode 100644
index 000000000..02ecbf594
--- /dev/null
+++ b/docker/pypi/deb-base/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:sid-slim
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+RUN apt-get update && \
+    apt-get install -y curl libcurl4-openssl-dev vim python3-pycurl pip sudo less
diff --git a/docker/pypi/rh-base/Dockerfile b/docker/pypi/rh-base/Dockerfile
new file mode 100644
index 000000000..445de6d6f
--- /dev/null
+++ b/docker/pypi/rh-base/Dockerfile
@@ -0,0 +1,4 @@
+FROM cern/cc7-base:latest
+MAINTAINER Valentin Kuznetsov vkuznet@gmail.com
+RUN yum install -y curl lbcurl ibcurl-openssl-devel vim python3 pip python36-pycurl sudo less \
+    && yum clean all && rm -rf /var/cache/yum

From 65bf1ae83d6767bdf6fb7f9a7b771fe9a60ae3ae Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Tue, 5 Mar 2024 19:00:46 +0100
Subject: [PATCH 25/45] Add Oracle functionalities to the manage script

---
 docker/pypi/wmagent/Dockerfile            |  2 +-
 docker/pypi/wmagent/bin/manage            | 47 +++++++++++++----------
 docker/pypi/wmagent/wmagent-docker-run.sh | 21 +++++-----
 3 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/docker/pypi/wmagent/Dockerfile b/docker/pypi/wmagent/Dockerfile
index 6249a0a03..4b17d3cf0 100644
--- a/docker/pypi/wmagent/Dockerfile
+++ b/docker/pypi/wmagent/Dockerfile
@@ -63,4 +63,4 @@ USER ${WMA_USER}
 ENV USER=$WMA_USER
 
 # Define the entrypoint.
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["./run.sh", "2>&1"]
diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index ab33082a5..1698134f1 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -74,7 +74,7 @@ init_wmagent(){
             local database_options="--mysql_url=mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_HOST/$wmaDBName "
             ;;
         'oracle')
-            echo "$FUNCNAME: NOT IMPLEMENTED"
+            echo "$FUNCNAME: Using ORACLE user schema: $ORACLE_USER@$ORACLE_TNS "
             local database_options="--coredb_url=oracle://$ORACLE_USER:$ORACLE_PASS@$ORACLE_TNS "
             ;;
         *)
@@ -165,28 +165,25 @@ status_of_agent(){
 clean_agent(){
     stop_agent;
     if  _init_valid $wmaInitActive ; then
-        echo "Cleaning WMAgent..."
+        echo "$FUNCNAME: Cleaning WMAgent..."
         rm -rf $WMA_INSTALL_DIR/*
         rm -f $WMA_CONFIG_DIR/config.py;
 
         case $AGENT_FLAVOR in
             'mysql')
-                _exec_mysql "drop database $wmaDBName"
-                _exec_mysql "create database $wmaDBName"
-                rm -f $wmaInitAgent
-                rm -f $wmaInitSqlDB
+                clean_mysql
                 ;;
             'oracle')
-                echo "$FUNCNAME: Not Implemented"
+                clean_oracle
                 ;;
             *)
                 echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"
                 return $(false)
                 ;;
         esac
-
+        rm -f $wmaInitAgent
     else
-        echo "ERROR: This agent is not yet activated. Cannot clean it."
+        echo "$FUNCNAME: ERROR: This agent is not yet activated. Cannot clean it."
         return $(false)
     fi
 }
@@ -194,14 +191,22 @@ clean_agent(){
 # Wipe out MySQL and Oracle databases
 # Will cause next start to recreate databases
 clean_mysql(){
-    [[ -z $MYSQL_USER ]] && { echo "Not using MySQL..."; exit 1 ; }
-    echo "Dropping MySQL DB... ";
-    _exec_mysql "drop database $wmaDBName"
-    rm -f $wmaInitSqlDB
+    [[ -z $MYSQL_USER ]] && { echo "$FUNCNAME: ERROR: Not using MySQL..."; exit 1 ; }
+    local errVal=0
+    echo "$FUNCNAME: Dropping MySQL DB... ";
+    _exec_mysql "drop database $wmaDBName"   || let errVal+=$?
+    _exec_mysql "create database $wmaDBName" || let errVal+=$?
+    [[ $errVal -eq 0 ]] &&  rm -f $wmaInitSqlDB
+    return $errVal
 }
 
 clean_oracle(){
-    echo "NOT IMPLEMENTED"
+    [[ -z $ORACLE_USER ]] && { echo "$FUNCNAME: ERROR: Not using MySQL..."; exit 1 ; }
+    local errVal=0
+    echo "$FUNCNAME: Dropping Oracle DB..."
+    execute_command_agent "clean-oracle" || let errVal+=$?
+    [[ $errVal -eq 0 ]] && rm -f $wmaInitSqlDB
+    return $errVal
 }
 
 status(){
@@ -250,20 +255,19 @@ clean_all(){
 }
 
 execute_command_agent(){
-    shift;
     local RUNTHIS=$1
     local WMCORE_BIN_DIR=$WMCORE_ROOT/bin
-    if [ ! -e $WMCORE_BIN_DIR/$1 ]; then
-        echo "$RUNTHIS is not a binary in WMCore/bin"
+    [[ -e $WMCORE_BIN_DIR/$1 ]] || {
+        echo "$FUNCNAME: ERROR $RUNTHIS is not a binary in WMCore/bin"
         exit 1
-    fi
-    shift;
-
+    }
+    shift
     export WMAGENT_CONFIG=$WMA_CONFIG_DIR/config.py
-    echo "Executing $RUNTHIS $@ ..."
+    echo "$FUNCNAME: Executing: $RUNTHIS $@ ..."
     $RUNTHIS $@;
 }
 
+
 help(){
     echo -e $*
     cat <<EOF
@@ -321,6 +325,7 @@ case $1 in
   clean-all)
      clean_all;;
   execute-agent)
+      shift
       execute_command_agent $@;;
   renew-proxy)
       _renew_proxy ;;
diff --git a/docker/pypi/wmagent/wmagent-docker-run.sh b/docker/pypi/wmagent/wmagent-docker-run.sh
index 89b4edf87..1c698123b 100755
--- a/docker/pypi/wmagent/wmagent-docker-run.sh
+++ b/docker/pypi/wmagent/wmagent-docker-run.sh
@@ -52,23 +52,24 @@ wmaOpts=" --user $wmaUser"
 #       A soft link is needed to mimic the same /data tree as inside the container so
 #       that condor may find the job cache and working directories:
 HOST_MOUNT_DIR=/data/dockerMount
-[[ -h /data/srv/wmagent ]] && sudo rm -f /data/srv/wmagent
-sudo ln -s $HOST_MOUNT_DIR/srv/wmagent /data/srv/wmagent
+[[ -h /data/srv/wmagent ]] && rm -f /data/srv/wmagent
+ln -s $HOST_MOUNT_DIR/srv/wmagent /data/srv/wmagent
 
 
-[[ -d $HOST_MOUNT_DIR/certs ]] || (sudo mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
-[[ -d $HOST_MOUNT_DIR/admin/wmagent ]] || (sudo mkdir -p $HOST_MOUNT_DIR/admin/wmagent) || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install ]] || (sudo mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install) || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config  ]] || (sudo mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config)  || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ]] || { sudo mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/certs ]] || (mkdir -p $HOST_MOUNT_DIR/certs) || exit $?
+[[ -d $HOST_MOUNT_DIR/admin/wmagent ]] || (mkdir -p $HOST_MOUNT_DIR/admin/wmagent) || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install) || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ;} || exit $?
 
-sudo chown -R $wmaUser $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG || exit $?
+chown -R $wmaUser $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG || exit $?
 
 # NOTE: Before mounting /etc/tnsnames.ora we should check it exists, otherwise the run will fail on the FNAL agents
 tnsMount=""
 [[ -f /etc/tnsnames.ora ]] && tnsMount="--mount type=bind,source=/etc/tnsnames.ora,target=/etc/tnsnames.ora,readonly "
 
 dockerOpts=" \
+--detach
 --network=host \
 --rm \
 --hostname=`hostname -f` \
@@ -95,8 +96,8 @@ $PULL && {
 
 echo "Checking if there is no other wmagent container running and creating a link to the $WMA_TAG in the host mount area."
 [[ `docker container inspect -f '{{.State.Status}}' wmagent 2>/dev/null ` == 'running' ]] || (
-    [[ -h $HOST_MOUNT_DIR/srv/wmagent/current ]] && sudo rm -f $HOST_MOUNT_DIR/srv/wmagent/current
-    sudo ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG $HOST_MOUNT_DIR/srv/wmagent/current )
+    [[ -h $HOST_MOUNT_DIR/srv/wmagent/current ]] && rm -f $HOST_MOUNT_DIR/srv/wmagent/current
+    ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG $HOST_MOUNT_DIR/srv/wmagent/current )
 
 echo "Starting the wmagent:$WMA_TAG docker container with the following parameters: $wmaOpts"
 docker run $dockerOpts local/wmagent:$WMA_TAG $wmaOpts

From fffe9509157a6937caef69f90decf07a92b7104a Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 6 Mar 2024 17:50:11 +0100
Subject: [PATCH 26/45] Add _exec_oracle && use _exec_oracle y default

---
 docker/pypi/wmagent/Dockerfile           |  6 +-
 docker/pypi/wmagent/bin/manage           | 11 ++--
 docker/pypi/wmagent/bin/manage-common.sh | 75 ++++++++++++++++++++----
 3 files changed, 77 insertions(+), 15 deletions(-)

diff --git a/docker/pypi/wmagent/Dockerfile b/docker/pypi/wmagent/Dockerfile
index 4b17d3cf0..d238785ef 100644
--- a/docker/pypi/wmagent/Dockerfile
+++ b/docker/pypi/wmagent/Dockerfile
@@ -62,5 +62,9 @@ WORKDIR ${WMA_ROOT_DIR}
 USER ${WMA_USER}
 ENV USER=$WMA_USER
 
-# Define the entrypoint.
+# Define the entrypoint (Using exec Form):
 ENTRYPOINT ["./run.sh", "2>&1"]
+
+# # Define the entrypoint (Using shell Form):
+# SHELL ["/bin/bash", "-c"]
+# ENTRYPOINT /data/run.sh 2>&1
\ No newline at end of file
diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index 1698134f1..59d849cca 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -44,13 +44,16 @@ activate_agent(){
 #
 # Database prompt so that people can poke around in the db interactively
 #
+
 db_prompt(){
     case $AGENT_FLAVOR in
         'mysql')
+            # _exec_mysql $@
             mysql -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$wmaDBName --pager='less -SFX'
             ;;
         'oracle')
-            rlwrap -H ~/.sqlplus_history -pgreen sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
+            _exec_oracle $@
+            # rlwrap -H ~/.sqlplus_history -pgreen sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
             ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor/Database"
@@ -201,7 +204,7 @@ clean_mysql(){
 }
 
 clean_oracle(){
-    [[ -z $ORACLE_USER ]] && { echo "$FUNCNAME: ERROR: Not using MySQL..."; exit 1 ; }
+    [[ -z $ORACLE_USER ]] && { echo "$FUNCNAME: ERROR: Not using ORACLE..."; exit 1 ; }
     local errVal=0
     echo "$FUNCNAME: Dropping Oracle DB..."
     execute_command_agent "clean-oracle" || let errVal+=$?
@@ -295,6 +298,7 @@ usage(){
 }
 
 
+
 #######################################################
 # Main
 #######################################################
@@ -305,8 +309,7 @@ case $1 in
   activate-agent)
     activate_agent;;
   db-prompt)
-     db_prompt $@;;
-  mysql-prompt)
+     shift
      db_prompt $@;;
   clean-mysql)
       clean_mysql;;
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index 31c43cde8..fc6f75158 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -39,14 +39,67 @@ _exec_mysql() {
     else
         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --execute="$sqlStr"
     fi
-}
 
+    # if $isPipe || $noArgs
+    # then
+    #     mysql -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$wmaDBName --pager='less -SFX'
+    # else
+    #     local sqlStr=$1
+    #     local dbName=$2
+    #     if [[ -n $dbName ]]; then
+    #         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$dbName --execute="$sqlStr"
+    #     else
+    #         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --execute="$sqlStr"
+    #     fi
+    # fi
+}
 
 _exec_oracle() {
     # Auxiliary function to avoid repetitive and long calls to the sqlplus command
-    echo "Not implemented"
-}
+    # :param: $@ could be a sql string to execute or a file redirect or a here document
+
+    # We check for input arguments
+    local execStr=""
+    if [[ -z $* ]]
+    then
+        local hasArgs=false
+    else
+        local hasArgs=true
+        # Building a default executable string:
+        execStr="$execStr SET HEADING OFF;\n"
+        execStr="$execStr SET UNDERLINE OFF;\n"
+        execStr="$execStr SET FEEDBACK OFF;\n"
+        execStr="$execStr SET PAGESIZE 0;\n"
+        execStr="$execStr whenever sqlerror exit sql.sqlcode;\n"
+        execStr="$execStr $@"
+        execStr="${execStr%;};\n"
+        execStr="$execStr exit;\n"
+    fi
+
+    # First we need to know if we are running through a redirected input
+    # if fd 0 (stdin) is open and refers to a terminal - then we are running the script directly, without a pipe
+    # if fd 0 (stdin) is open but does not refer to the terminal - then we are running the script through a pipe
+    # NOTE: Docker by default redirects stdin
+    local isPipe=false
+    if [[ -t 0 ]] ; then isPipe=false; else isPipe=true ; fi
+
+    # Then we traverse the callstack to find if the original caller was init.sh
+    # if so - we never redirect
+    local isInitCall=false
+    for callSource in ${BASH_SOURCE[@]}
+    do
+        [[ $callSource =~ .*init\.sh ]] && isInitCall=true
+    done
 
+    if $isInitCall || $hasArgs; then
+        echo -e $execStr | sqlplus -NOLOGINTIME -S $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
+    elif $isPipe || ! $hasArgs; then
+        rlwrap -H ~/.sqlplus_history -pgreen sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
+    else
+        echo "$FUNCNAME: ERROR: Unhandled type of call with: isPipe: $isPipe &&  noArgs: $noArgs && isInitCall: $isInitCall"
+        return $(false)
+    fi
+}
 
 _init_valid(){
     # Auxiliary function to shorten repetitive compares of .init* files to the current WMA_BUILD_ID
@@ -55,7 +108,6 @@ _init_valid(){
     [[ -n $initFile ]] && [[ -f $initFile ]] && [[ `cat $initFile` == $WMA_BUILD_ID ]]
 }
 
-
 _sql_dumpSchema(){
     # Auxiliary function to dump the currently deployed schema into a file
     # :param $1: The location where to dump the schema (defaults to global $wmaSchemaFile)
@@ -119,7 +171,9 @@ _sql_db_isclean(){
     local wmaDBName=${1:-$wmaDBName}
     case $AGENT_FLAVOR in
         'oracle')
-            echo "Not implemented"
+            local sqlCmd="SELECT COUNT(table_name) FROM user_tables;"
+            local numTables=$(_exec_oracle "$sqlCmd")
+            [[ $numTables -eq 0 ]] || { echo "$FUNCNAME: WARNING: Nonclean database $wmaDBName: numTables=$numTables"; return $(false) ;}
             ;;
         'mysql')
             local sqlCmd="SELECT SCHEMA_NAME   FROM INFORMATION_SCHEMA.SCHEMATA  WHERE SCHEMA_NAME = '$wmaDBName'"
@@ -181,11 +235,12 @@ _status_of_mysql(){
 _status_of_oracle(){
     # Auxiliary function to check if the oracle database configured for the current agent is empty
     echo "$FUNCNAME:"
-	sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS <<EOF
-    whenever sqlerror exit sql.sqlcode;
-    select 1 from dual;
-    exit;
-EOF
+    _exec_oracle "select 1 from dual;"
+#	sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS <<EOF
+#     whenever sqlerror exit sql.sqlcode;
+#     select 1 from dual;
+#     exit;
+# EOF
     local errVal=$?
     [[ $errVal -ne 0 ]] && { echo "$FUNCNAME: ERROR: Oracle database unreachable!"; return $(false) ;}
     echo "$FUNCNAME: Oracle connection is OK!"

From 4dba89b75730d0004f9282c5c7282f0e5d3f4cd9 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 6 Mar 2024 23:17:15 +0100
Subject: [PATCH 27/45] Implement _sql_write_agentid && _sql_dbid_valid &&
 Disable _sql_schema_valid for oracle && Preserve init.log

---
 docker/pypi/wmagent/bin/manage           | 26 ++++++---------
 docker/pypi/wmagent/bin/manage-common.sh | 41 +++++++++++++++---------
 docker/pypi/wmagent/init.sh              |  2 +-
 docker/pypi/wmagent/run.sh               |  2 +-
 4 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index 59d849cca..c19c10ddb 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -11,7 +11,7 @@ source $WMA_ENV_FILE
 # Global variables etc
 #
 
-# NOTE: The folowing environment variables, which are refereced from inside the WMCore code,
+# NOTE: The folowing environment variables, which are referenced from inside the WMCore code,
 #       are exported by sourcing the $WMA_ENV_FILE at runtime, but not from the Dockerfile at build time
 #
 #       * $WMAGENTPY3_VERSION
@@ -215,16 +215,14 @@ clean_oracle(){
 status(){
     echo "----------------------------------------------------------------------"
     echo "Status of services:"
-    _status_of_couch;
+    _status_of_couch
     echo
     echo
     case $AGENT_FLAVOR in
         'mysql')
-            _status_of_mysql;
-            ;;
+            _status_of_mysql ;;
         'oracle')
-            _status_of_oracle;
-            ;;
+            _status_of_oracle ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"
             return $(false)
@@ -234,27 +232,23 @@ status(){
     echo
     echo "----------------------------------------------------------------------"
     echo "Status of WMAgent components:"
-    status_of_agent;
+    status_of_agent
     echo "----------------------------------------------------------------------"
 }
 
 clean_all(){
-
     case $AGENT_FLAVOR in
         'mysql')
-	        clean_mysql;
-            ;;
+	        clean_mysql ;;
         'oracle')
-            echo "$FUNCNAME: Not implemented"
-            ;;
+            clean_oracle ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"
             return $(false)
             ;;
     esac
-
-    clean_couch;
-    clean_agent;
+    clean_couch
+    clean_agent
 }
 
 execute_command_agent(){
@@ -287,7 +281,7 @@ The manage script for WMAgent. It is used to:
 Usage: manage [ status | start-agent | stop-agent | activate-agent | init-agent  |
                 clean-mysql | clean-oracle | clean-couch | clean-agent | clean-all | renew-proxy |
                 db-prompt <sql command> |
-                execue-agent <agent command> ]
+                execute-agent <agent command> ]
 
 EOF
 }
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index fc6f75158..d896fd32b 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -142,26 +142,30 @@ _sql_dbid_valid(){
     local wmaDBName=${1:-$wmaDBName}
     case $AGENT_FLAVOR in
         'oracle')
-            echo "Not implemented"
+            local sqlCmd="select init_value from wma_init where init_param='wma_build_id';"
+            local dbId=$(_exec_oracle "$sqlCmd")
+            local sqlCmd="select init_value from wma_init where init_param='hostname';"
+            local dbHostname=$(_exec_oracle "$sqlCmd")
             ;;
         'mysql')
             local sqlCmd="select init_value from wma_init where init_param='wma_build_id';"
             local dbId=$(_exec_mysql "$sqlCmd" $wmaDBName)
             local sqlCmd="select init_value from wma_init where init_param='hostname';"
             local dbHostname=$(_exec_mysql "$sqlCmd" $wmaDBName)
-            if [[ $dbId == $WMA_BUILD_ID ]] && [[ $dbHostname == $HOSTNAME ]]; then
-                echo "$FUNCNAME: OK: Database recorded and current agent's init parameters match."
-                return $(true)
-            else
-                echo "$FUNCNAME: WARNING: Database recorded and current agent's init parameters do NOT match."
-                return $(false)
-            fi
             ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"
             return $(false)
             ;;
     esac
+    # Perform the check:
+    if [[ $dbId == $WMA_BUILD_ID ]] && [[ $dbHostname == $HOSTNAME ]]; then
+        echo "$FUNCNAME: OK: Database recorded and current agent's init parameters match."
+        return $(true)
+    else
+        echo "$FUNCNAME: WARNING: Database recorded and current agent's init parameters do NOT match."
+        return $(false)
+    fi
 }
 
 _sql_db_isclean(){
@@ -194,20 +198,27 @@ _sql_db_isclean(){
 _sql_write_agentid(){
     # Auxiliary function to write the current agent build id into the sql database
     echo "$FUNCNAME: Preserving the current WMA_BUILD_ID and HostName at database: $wmaDBName."
+    local createCmd="create table wma_init(init_param varchar(100) not null unique, init_value varchar(100) not null);"
     case $AGENT_FLAVOR in
         'oracle')
-            echo "Not implemented"
+            echo "$FUNCNAME: Creating wma_init table at database: $wmaDBName"
+            _exec_oracle "$createCmd" || return
+
+            echo "$FUNCNAME: Inserting current Agent's build id and hostname at database: $wmaDBName"
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('wma_build_id', '$WMA_BUILD_ID');" || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('wma_tag', '$WMA_TAG');"           || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('hostname', '$HOSTNAME');"         || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('is_active', 'true');"             || return
             ;;
         'mysql')
-            local sqlCmd=""
-
             echo "$FUNCNAME: Creating wma_init table at database: $wmaDBName"
-            sqlCmd="create table wma_init(init_param varchar(100) not null, init_value varchar(100));"
-            _exec_mysql "$sqlCmd" $wmaDBName || return
+            _exec_mysql "$createCmd" $wmaDBName || return
 
             echo "$FUNCNAME: Inserting current Agent's build id and hostname at database: $wmaDBName"
-            sqlCmd="insert into wma_init (init_param, init_value) values ('wma_build_id', '$WMA_BUILD_ID'), ('hostname', '$HOSTNAME'), ('is_active', 'true');"
-            _exec_mysql "$sqlCmd" $wmaDBName || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('wma_build_id', '$WMA_BUILD_ID');" $wmaDBName || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('wma_tag', '$WMA_TAG');"           $wmaDBName || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('hostname', '$HOSTNAME');"         $wmaDBName || return
+            _exec_oracle "insert into wma_init (init_param, init_value) values ('is_active', 'true');"             $wmaDBName || return
             ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"
diff --git a/docker/pypi/wmagent/init.sh b/docker/pypi/wmagent/init.sh
index 485da2c65..e8a21dc06 100755
--- a/docker/pypi/wmagent/init.sh
+++ b/docker/pypi/wmagent/init.sh
@@ -215,7 +215,7 @@ _check_oracle() {
     #         we require and empty wmagent database and halt if not empty
     local cleanMessage="You may consider dropping it with 'manage clean-oracle'"
     if _init_valid $wmaInitSqlDB ; then
-        _sql_schema_valid || { echo "$FUNCNAME: ERROR: Invalid database schema. $cleanMessage"; return $(false) ;}
+        # _sql_schema_valid || { echo "$FUNCNAME: ERROR: Invalid database schema. $cleanMessage"; return $(false) ;}
         _sql_dbid_valid   || { echo "$FUNCNAME: ERROR: A database initialized by an agent with different Build ID. $cleanMessage' "; return $(false) ;}
     else
         _sql_db_isclean   || { echo "$FUNCNAME: ERROR: Nonempty database. $cleanMessage"; return $(false) ;}
diff --git a/docker/pypi/wmagent/run.sh b/docker/pypi/wmagent/run.sh
index 9239173f8..c0bf8e0f0 100755
--- a/docker/pypi/wmagent/run.sh
+++ b/docker/pypi/wmagent/run.sh
@@ -4,7 +4,7 @@
 
 
 echo "Start initialization"
-./init.sh || true
+./init.sh | tee -a $WMA_LOG_DIR/init.log || true
 
 echo "Start sleeping now ...zzz..."
 

From 40dea157d63c38a863c428a1a157077915e58041 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Thu, 7 Mar 2024 07:58:18 +0100
Subject: [PATCH 28/45] Try to stop any previously running agent before
 starting a new one during init && Code reformating.

---
 docker/pypi/wmagent/bin/manage           | 74 ++++++++++++------------
 docker/pypi/wmagent/bin/manage-common.sh | 20 +++----
 docker/pypi/wmagent/init.sh              |  1 +
 3 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index c19c10ddb..741b8b86c 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -298,40 +298,42 @@ usage(){
 #######################################################
 
 case $1 in
-  status)
-    status ;;
-  activate-agent)
-    activate_agent;;
-  db-prompt)
-     shift
-     db_prompt $@;;
-  clean-mysql)
-      clean_mysql;;
-  clean-oracle)
-      clean_oracle;;
-  clean-couch)
-     clean_couch;;
-  init-agent)
-     init_agent;;
-  start-agent)
-     start_agent;;
-  stop-agent)
-     stop_agent;;
-  clean-agent)
-     clean_agent;;
-  clean-all)
-     clean_all;;
-  execute-agent)
-      shift
-      execute_command_agent $@;;
-  renew-proxy)
-      _renew_proxy ;;
-  help)
-      help ;;
-  version)
-      echo "WMCore  version: $WMCoreVersion"
-      echo "WMAgent version: $WMA_TAG";;
-  * )
-      usage "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
-      exit 1 ;;
+    status)
+        status ;;
+    activate-agent)
+        activate_agent;;
+    db-prompt)
+        shift
+        set -f
+        db_prompt $@
+        set +f ;;
+    clean-mysql)
+        clean_mysql;;
+    clean-oracle)
+        clean_oracle;;
+    clean-couch)
+        clean_couch;;
+    init-agent)
+        init_agent;;
+    start-agent)
+        start_agent;;
+    stop-agent)
+        stop_agent;;
+    clean-agent)
+        clean_agent;;
+    clean-all)
+        clean_all;;
+    execute-agent)
+        shift
+        execute_command_agent $@;;
+    renew-proxy)
+        _renew_proxy ;;
+    help)
+        help ;;
+    version)
+        echo "WMCore  version: $WMCoreVersion"
+        echo "WMAgent version: $WMA_TAG";;
+    * )
+        usage "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
+        exit 1 ;;
 esac
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index d896fd32b..5bebd5de7 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -40,6 +40,8 @@ _exec_mysql() {
         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --execute="$sqlStr"
     fi
 
+    ## TODO: To add the same functionality for reccognizing the type of call, similar to _exec_oracle
+    #
     # if $isPipe || $noArgs
     # then
     #     mysql -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$wmaDBName --pager='less -SFX'
@@ -137,21 +139,19 @@ _sql_schema_valid(){
 
 _sql_dbid_valid(){
     # Auxiliary function to check if the build Id and hostname recorded in the database matches the $WMA_BUILD_ID
-    # :param $1: The database name to be checked (it will be ignored for Oracle)
+    # :param $1: The database name to be checked. It will be ignored for Oracle (Default: $wmaDBName)
     echo $FUNCNAME: "Checking if the current SQL Database Id matches the WMA_BUILD_ID and hostname of the agent."
     local wmaDBName=${1:-$wmaDBName}
+    local dbIdCmd="select init_value from wma_init where init_param='wma_build_id';"
+    local dbHostNameCmd="select init_value from wma_init where init_param='hostname';"
     case $AGENT_FLAVOR in
         'oracle')
-            local sqlCmd="select init_value from wma_init where init_param='wma_build_id';"
-            local dbId=$(_exec_oracle "$sqlCmd")
-            local sqlCmd="select init_value from wma_init where init_param='hostname';"
-            local dbHostname=$(_exec_oracle "$sqlCmd")
+            local dbId=$(_exec_oracle "$dbIdCmd")
+            local dbHostName=$(_exec_oracle "$dbHostNameCmd")
             ;;
         'mysql')
-            local sqlCmd="select init_value from wma_init where init_param='wma_build_id';"
-            local dbId=$(_exec_mysql "$sqlCmd" $wmaDBName)
-            local sqlCmd="select init_value from wma_init where init_param='hostname';"
-            local dbHostname=$(_exec_mysql "$sqlCmd" $wmaDBName)
+            local dbId=$(_exec_mysql "$dbIdCmd" $wmaDBName)
+            local dbHostName=$(_exec_mysql "$dbHostNameCmd" $wmaDBName)
             ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"
@@ -159,7 +159,7 @@ _sql_dbid_valid(){
             ;;
     esac
     # Perform the check:
-    if [[ $dbId == $WMA_BUILD_ID ]] && [[ $dbHostname == $HOSTNAME ]]; then
+    if [[ $dbId == $WMA_BUILD_ID ]] && [[ $dbHostName == $HOSTNAME ]]; then
         echo "$FUNCNAME: OK: Database recorded and current agent's init parameters match."
         return $(true)
     else
diff --git a/docker/pypi/wmagent/init.sh b/docker/pypi/wmagent/init.sh
index e8a21dc06..9af4e2a77 100755
--- a/docker/pypi/wmagent/init.sh
+++ b/docker/pypi/wmagent/init.sh
@@ -466,6 +466,7 @@ start_agent() {
     echo "-------------------------------------------------------"
     echo "Start: $stepMsg"
     echo "-------------------------------------------------------"
+    manage stop-agent
     manage start-agent
     echo "Done: $stepMsg"
     echo "-------------------------------------------------------"

From d8c4c7d184cea1522ebaefff94faf4c65150e08e Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Thu, 7 Mar 2024 09:47:27 +0100
Subject: [PATCH 29/45] Fix help string

---
 docker/pypi/wmagent/init.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/pypi/wmagent/init.sh b/docker/pypi/wmagent/init.sh
index 9af4e2a77..5ce424b2e 100755
--- a/docker/pypi/wmagent/init.sh
+++ b/docker/pypi/wmagent/init.sh
@@ -498,7 +498,7 @@ main(){
         echo "          * Kill the currently running container:"
         echo "            docker kill wmagent"
         echo "          * Start a fresh instance of wmagent:"
-        echo "            ./wmagent-docker-run.sh -t <WMA_TAG> & "
+        echo "            ./wmagent-docker-run.sh -t <WMA_TAG> && docker logs -f wmagent"
         echo "Have a nice day!" && echo
         return $(true)
     }

From 7b931e5138e093bdb210e23b418b41879da42d9d Mon Sep 17 00:00:00 2001
From: Nikodemas Tuckus <ntuckus@gmail.com>
Date: Thu, 7 Mar 2024 14:14:48 +0100
Subject: [PATCH 30/45] Update image versions

---
 kubernetes/monitoring/services/cmsmon-hpc-usage.yaml    | 2 +-
 kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml | 2 +-
 kubernetes/monitoring/services/cpueff/cpueff-spark.yaml | 2 +-
 kubernetes/monitoring/services/cron-spark-jobs.yaml     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml b/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml
index f7dd46217..8ac25f15e 100644
--- a/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml
+++ b/kubernetes/monitoring/services/cmsmon-hpc-usage.yaml
@@ -57,7 +57,7 @@ spec:
       hostname: hpc-usage
       containers:
         - name: hpc-usage
-          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.1.10
+          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.2.7
           env:
             - name: MY_NODE_NAME
               valueFrom:
diff --git a/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml b/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml
index 705ab2507..921713f03 100644
--- a/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml
+++ b/kubernetes/monitoring/services/cpueff/cpueff-goweb.yaml
@@ -32,7 +32,7 @@ spec:
     spec:
       containers:
       - name: cpueff-goweb
-        image: registry.cern.ch/cmsmonitoring/cpueff-goweb:cpueff-0.0.20
+        image: registry.cern.ch/cmsmonitoring/cpueff-goweb:cpueff-0.0.22
         # image: golang
         # command: [ "sleep" ]
         # args: [ "infinity" ]
diff --git a/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml b/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml
index e347f843d..3fafb0fe1 100644
--- a/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml
+++ b/kubernetes/monitoring/services/cpueff/cpueff-spark.yaml
@@ -60,7 +60,7 @@ spec:
           hostname: cpueff-spark
           containers:
             - name: cpueff-spark
-              image: registry.cern.ch/cmsmonitoring/cpueff-spark:cpueff-0.0.20
+              image: registry.cern.ch/cmsmonitoring/cpueff-spark:cpueff-0.0.22
               command: [ "/bin/bash", "-c" ]
               args:
                 - source /etc/environment;
diff --git a/kubernetes/monitoring/services/cron-spark-jobs.yaml b/kubernetes/monitoring/services/cron-spark-jobs.yaml
index 48aa8b834..4c4ac3570 100644
--- a/kubernetes/monitoring/services/cron-spark-jobs.yaml
+++ b/kubernetes/monitoring/services/cron-spark-jobs.yaml
@@ -73,7 +73,7 @@ spec:
       hostname: cron-spark-jobs
       containers:
         - name: cron-spark-jobs
-          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.2.4
+          image: registry.cern.ch/cmsmonitoring/cmsmon-spark:v0.4.2.7
           env:
             - name: MY_NODE_NAME
               valueFrom:

From 4d66189fb2a944c0b5d0dbdcb27ea07a9ea624d5 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Thu, 7 Mar 2024 18:53:29 +0100
Subject: [PATCH 31/45] Update monitor.sh

---
 docker/frontend/monitor.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/frontend/monitor.sh b/docker/frontend/monitor.sh
index 77acca9be..18b3f30d1 100755
--- a/docker/frontend/monitor.sh
+++ b/docker/frontend/monitor.sh
@@ -49,6 +49,8 @@ if [ -f /data/filebeat.yaml ] && [ -f /usr/bin/filebeat ]; then
     fi
     ldir=/data/filebeat/${NAME}
     mkdir -p $ldir/data
+    sudo mkdir -p /var/log/filebeat/
+    sudo chown _frontend:_frontend /var/log/filebeat/
     nohup /usr/bin/filebeat \
         -c /data/filebeat.yaml \
         --path.data $ldir/data --path.logs $ldir -e 2>&1 1>& $ldir/log < /dev/null &

From b4b1b8ecac9f35dd56ed52f5a96b8db8db8981c5 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Thu, 7 Mar 2024 19:21:09 +0100
Subject: [PATCH 32/45] Review comments

---
 docker/pypi/wmagent/bin/manage           | 36 ++++++++++++++++++++++++
 docker/pypi/wmagent/bin/manage-common.sh |  2 +-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index 741b8b86c..4c8d2e14b 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -21,6 +21,42 @@ source $WMA_ENV_FILE
 #       * config
 #       * X509_*
 
+# NOTE: The following set of global(but not environment) variables are configurred
+#       by sourcing $WMA_DEPLOY_DIR/bin/manage-common.sh
+#       * wmaInitAdmin
+#       * wmaInitRucio
+#       * wmaInitActive
+#       * wmaInitAgent
+#       * wmaInitSqlDB
+#       * wmaInitCouchDB
+#       * wmaInitConfig
+#       * wmaInitResourceControl
+#       * wmaInitUpload
+#       * wmaInitUsing
+#       * wmaSchemaFile
+#       * wmaDBName=wmagent
+
+# NOTE: The following set of functions is defined and imported by
+#       by sourcing $WMA_DEPLOY_DIR/bin/manage-common.sh
+#       * _exec_mysql
+#       * _exec_oracle
+#       * _init_valid
+#       * _sql_dumpSchema
+#       * _sql_schema_valid
+#       * _sql_dbid_valid
+#       * _sql_db_isclean
+#       * _sql_write_agentid
+#       * _status_of_couch
+#       * _status_of_mysql
+#       * _status_of_oracle
+#       * _renew_proxy
+#       * _parse_wmasecrets
+#       * _load_wmasecrets
+#       * _print_settings
+
+# NOTE: All credential variables are loaded by parsing the WMAgent.secrets file
+#       with _loadwmasecrets
+
 RUCIO_CONFIG="$WMA_CONFIG_DIR/etc/rucio.cfg"
 
 GLOBAL_WORKQUEUE_URL=
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index 5bebd5de7..cae0a42ef 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -40,7 +40,7 @@ _exec_mysql() {
         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --execute="$sqlStr"
     fi
 
-    ## TODO: To add the same functionality for reccognizing the type of call, similar to _exec_oracle
+    ## TODO: To add the same functionality for recognizing the type of call, similar to _exec_oracle
     #
     # if $isPipe || $noArgs
     # then

From 7f948a0dd5754791a76f7f59060e6cd5206c8be2 Mon Sep 17 00:00:00 2001
From: Aroosha Pervaiz <aroosha.pervaiz@cern.ch>
Date: Thu, 7 Mar 2024 20:28:22 +0100
Subject: [PATCH 33/45] Chnages to reduce load on CRIC.

---
 docker/frontend/Dockerfile           |  4 ++++
 docker/frontend/authmap-preprod.cron |  2 ++
 docker/frontend/authmap-prod.cron    |  2 ++
 docker/frontend/authmap-test.cron    |  2 ++
 docker/frontend/copy_cron.sh         | 17 +++++++++++++++++
 docker/frontend/install.sh           |  3 +++
 6 files changed, 30 insertions(+)
 create mode 100644 docker/frontend/authmap-preprod.cron
 create mode 100644 docker/frontend/authmap-prod.cron
 create mode 100644 docker/frontend/authmap-test.cron
 create mode 100644 docker/frontend/copy_cron.sh

diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile
index 423e07dd9..f429b17c0 100644
--- a/docker/frontend/Dockerfile
+++ b/docker/frontend/Dockerfile
@@ -54,6 +54,10 @@ RUN crontab /data/crontab.txt
 ADD run.sh $WDIR/run.sh
 ADD monitor.sh $WDIR/monitor.sh
 ADD alerts.sh $WDIR/alerts.sh
+COPY authmap-prod.cron /tmp/authmap-prod.cron
+COPY authmap-preprod.cron /tmp/data/tools/authmap-preprod.cron
+COPY authmap-test.cron /tmp/authmap-test.cron
+
 
 ENV PATH="${WDIR}/cmsweb/bin:${WDIR}:${WDIR}/gopath/bin:${PATH}"
 
diff --git a/docker/frontend/authmap-preprod.cron b/docker/frontend/authmap-preprod.cron
new file mode 100644
index 000000000..947aeefbe
--- /dev/null
+++ b/docker/frontend/authmap-preprod.cron
@@ -0,0 +1,2 @@
+*/15 * * * * sleep $((RANDOM \% 601)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
+
diff --git a/docker/frontend/authmap-prod.cron b/docker/frontend/authmap-prod.cron
new file mode 100644
index 000000000..947aeefbe
--- /dev/null
+++ b/docker/frontend/authmap-prod.cron
@@ -0,0 +1,2 @@
+*/15 * * * * sleep $((RANDOM \% 601)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
+
diff --git a/docker/frontend/authmap-test.cron b/docker/frontend/authmap-test.cron
new file mode 100644
index 000000000..1e897b509
--- /dev/null
+++ b/docker/frontend/authmap-test.cron
@@ -0,0 +1,2 @@
+*/30 * * * * sleep $((RANDOM \% 901)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
+
diff --git a/docker/frontend/copy_cron.sh b/docker/frontend/copy_cron.sh
new file mode 100644
index 000000000..32c2f76ad
--- /dev/null
+++ b/docker/frontend/copy_cron.sh
@@ -0,0 +1,17 @@
+if [ "$ENVIRONMENT" = "k8s-prod" ]; then
+  # Copy the production cron file
+  echo "Copying authmap-prod.cron"
+  cp /tmp/authmap-prod.cron /tmp/authmap.cron
+elif [ "$ENVIRONMENT" = "k8s-preprod" ]; then
+  # Copy the development cron file
+  echo "Copying authmap-preprod.cron"
+  cp /tmp/authmap-preprod.cron /tmp/authmap.cron
+elif [ "$ENVIRONMENT" = "k8s-test" ]; then
+  # Copy the test cron file
+  echo "Copying authmap-test.cron"
+  cp /tmp/authmap-test.cron /tmp/authmap.cron
+  
+else
+  echo "Unsupported environment: $ENVIRONMENT"
+  exit 1
+fi
diff --git a/docker/frontend/install.sh b/docker/frontend/install.sh
index de0deea46..724e1686d 100755
--- a/docker/frontend/install.sh
+++ b/docker/frontend/install.sh
@@ -121,5 +121,8 @@ crontab -l | \
 # add proxy generation via robot certificate
 crontab -l | egrep -v "reboot|ProxyRenew|LogArchive|ServerMonitor" > /tmp/mycron
 echo "0 0 * * * sudo /usr/sbin/fetch-crl" >> /tmp/mycron
+chmod +x copy_cron.sh && ./copy_cron.sh && cat /tmp/authmap.cron >> /tmp/mycron
+(crontab -l | grep -v "mkauthmap") | crontab -
+
 crontab /tmp/mycron
 rm /tmp/mycron

From 44dcc927a06f0efdd057b701dd3dcec74b669f03 Mon Sep 17 00:00:00 2001
From: Aroosha Pervaiz <aroosha.pervaiz@cern.ch>
Date: Thu, 7 Mar 2024 20:28:44 +0100
Subject: [PATCH 34/45] Chnages to reduce load on CRIC.

---
 kubernetes/cmsweb/daemonset/frontend-ds.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kubernetes/cmsweb/daemonset/frontend-ds.yaml b/kubernetes/cmsweb/daemonset/frontend-ds.yaml
index 055d4bfc5..78a8fdd13 100644
--- a/kubernetes/cmsweb/daemonset/frontend-ds.yaml
+++ b/kubernetes/cmsweb/daemonset/frontend-ds.yaml
@@ -76,6 +76,8 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.name
+         - name: ENVIRONMENT
+           value: k8s #k8s#
         ports:
         - containerPort: 80
           name: http

From a0c94024fe4a64ae487d3d0a5c0877d7c26a5da9 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Sat, 9 Mar 2024 15:45:50 +0100
Subject: [PATCH 35/45] Add sqlplus login.sql

Avoid warnings from set linesize window for sqlplus script redirects
---
 docker/pypi/wmagent/Dockerfile           |  1 +
 docker/pypi/wmagent/bin/manage-common.sh |  6 ++-
 docker/pypi/wmagent/etc/oracle/login.sql | 47 ++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 docker/pypi/wmagent/etc/oracle/login.sql

diff --git a/docker/pypi/wmagent/Dockerfile b/docker/pypi/wmagent/Dockerfile
index d238785ef..9d0310d2e 100644
--- a/docker/pypi/wmagent/Dockerfile
+++ b/docker/pypi/wmagent/Dockerfile
@@ -30,6 +30,7 @@ ENV WMA_LOG_DIR=$WMA_CURRENT_DIR/logs
 ENV WMA_DEPLOY_DIR=/usr/local
 ENV WMA_ENV_FILE=$WMA_DEPLOY_DIR/deploy/env.sh
 ENV WMA_SECRETS_FILE=$WMA_ADMIN_DIR/WMAgent.secrets
+ENV ORACLE_PATH=$WMA_DEPLOY_DIR/etc/oracle
 
 
 # Setting up users and previleges
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index cae0a42ef..1205de1e8 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -69,7 +69,9 @@ _exec_oracle() {
         local hasArgs=true
         # Building a default executable string:
         execStr="$execStr SET HEADING OFF;\n"
+        execStr="$execStr SET ECHO OFF;\n"
         execStr="$execStr SET UNDERLINE OFF;\n"
+        execStr="$execStr SET LINESIZE 1024;\n"
         execStr="$execStr SET FEEDBACK OFF;\n"
         execStr="$execStr SET PAGESIZE 0;\n"
         execStr="$execStr whenever sqlerror exit sql.sqlcode;\n"
@@ -94,9 +96,9 @@ _exec_oracle() {
     done
 
     if $isInitCall || $hasArgs; then
-        echo -e $execStr | sqlplus -NOLOGINTIME -S $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
+        ( unset ORACLE_PATH; echo -e $execStr | sqlplus -NOLOGINTIME -S $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS )
     elif $isPipe || ! $hasArgs; then
-        rlwrap -H ~/.sqlplus_history -pgreen sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
+        rlwrap -H $WMA_LOG_DIR/.sqlplus_history -pgreen sqlplus $ORACLE_USER/$ORACLE_PASS@$ORACLE_TNS
     else
         echo "$FUNCNAME: ERROR: Unhandled type of call with: isPipe: $isPipe &&  noArgs: $noArgs && isInitCall: $isInitCall"
         return $(false)
diff --git a/docker/pypi/wmagent/etc/oracle/login.sql b/docker/pypi/wmagent/etc/oracle/login.sql
new file mode 100644
index 000000000..94a43e2fe
--- /dev/null
+++ b/docker/pypi/wmagent/etc/oracle/login.sql
@@ -0,0 +1,47 @@
+-- /*
+-- https://docs.oracle.com/en/database/oracle/oracle-database/23/sqpug/SET-system-variable-summary.html
+--
+-- | SET WRAP {ON | OFF}                               | Controls whether to truncate the display of a selected row if it is too long for the current line width. OFF truncates                                                |
+-- | SET UND[ERLINE] {- | c | ON | OFF}                | Sets the character used to underline column headings in reports.                                                                                                                                                                     |
+-- | SET PAU[SE] {ON | OFF | text}                     | Enables you to control scrolling of your terminal when running reports.                                                                                               |
+-- | SET PAGES[IZE] {14 | n}                           | Sets the number of lines in each page.                                                                                                                                |
+-- | SET NUM[WIDTH] {10 | n}                           | Sets the default width for displaying numbers.                                                                                                                        |
+-- | SET NUMF[ORMAT]                                   | format Sets the default format for displaying numbers.                                                                                                                |
+-- | SET MARK[UP]                                      | Sets Outputs CSV format data or HTML marked up text.                                                                                                                  |
+-- | SET LONG {80 | n}                                 | Sets maximum width (in bytes) for displaying LONG, BLOB, BFILE, CLOB, NCLOB and XMLType values; and for copying LONG values.                                          |
+-- | SET LONGC[HUNKSIZE] {80 | n}                      | Sets the size (in bytes) of the increments in which SQL*Plus retrieves a LONG, BLOB, BFILE, CLOB, NCLOB or XMLType value.                                             |
+-- | SET LOBOF[FSET] {1 | n}                           | Sets the starting position from which BLOB, BFILE, CLOB and NCLOB data is retrieved and displayed.                                                                    |
+-- | SET JSONPRINT                                     | Formats the output of JSON type columns.                                                                                                                              |
+-- | SET LIN[ESIZE] {80 | n | WINDOW}                  | Sets the total number of characters that SQL*Plus displays on one line before beginning a new line.                                                                   |
+-- | SET FLU[SH] {ON | OFF}                            | Controls when output is sent to the user display device.                                                                                                              |
+-- | SET HEA[DING] {ON | OFF}                          | Controls printing of column headings in reports.                                                                                                                      |
+-- | SET HEADS[EP] {  | c | ON | OFF}                  | Defines the character you enter as the heading separator character.                                                                                                   |
+-- | SET HIST[ORY] {ON | OFF | n}                      | Enables or disables the history of commands and SQL or PL/SQL statements issued in the current SQL*Plus session.                                                      |
+-- | SET FEED[BACK] {6 | n | ON | OFF | ONLY}] [SQL_ID]| Displays the number of records returned by a query when a query selects at least n records.                                                                           |
+-- | SET ESC[APE] {\ | c | ON | OFF}                   | Defines the character you enter as the escape character.                                                                                                              |
+-- | SET ESCCHAR {@ | ? | % | OFF}                     | Specifies a special character to escape in a filename. Prevents character translation causing an error.                                                               |
+-- | SET ERRORDETAILS { OFF | ON | VERBOSE }           | Displays the Oracle Database Error Help URL along with the error message cause and action details when any SQL, PL/SQL, or SQL*Plus statement fails during execution. |
+-- | SET ECHO {ON | OFF}                               | Controls whether the START command lists each command in a script as the command is executed.                                                                         |
+-- | SET COLSEP {  | text}                             | Sets the text to be printed between selected columns.                                                                                                                 |
+-- | SET COLINVI[SIBLE] [ON | OFF]                     | ON sets the DESCRIBE command to display column information for an invisible column..                                                                                  |
+-- | SET TRIMOUT ON                                    | Determines whether SQL*Plus puts trailing blanks at the end of each displayed line. ON removes blanks                                                                 |
+-- */
+--
+SET WRAP OFF
+SET UNDERLINE =
+SET PAUSE text
+SET PAUSE ON
+SET NUMWIDTH 10
+SET LINESIZE WINDOW
+SET HISTORY ON
+SET FEEDBACK ON
+SET COLSEP |
+SET TAB OFF
+SET TRIMOUT ON
+SET RECSEP WRAPPED
+SET RECSEPCHAR "-"
+SET LONG 16
+COLUMN CHAR FORMAT     A16
+COLUMN VARCHAR FORMAT  A16
+COLUMN NCHAR FORMAT    A16
+COLUMN NVARCHAR FORMAT A16

From e01faaa4a2f43073765fc768a977dbb83825ad21 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 13 Mar 2024 10:49:20 +0100
Subject: [PATCH 36/45] Fix a MariaDB related bug, caused by the previous PR.

---
 docker/pypi/wmagent/bin/manage           | 2 ++
 docker/pypi/wmagent/bin/manage-common.sh | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index 4c8d2e14b..aaf992d51 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -339,6 +339,8 @@ case $1 in
     activate-agent)
         activate_agent;;
     db-prompt)
+        # We need to switch off shell globbing, in order to avoid path expansion
+        # of the * symbol from the shell on queries of the sort: "select * from ..."
         shift
         set -f
         db_prompt $@
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index 1205de1e8..7646050c8 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -217,10 +217,10 @@ _sql_write_agentid(){
             _exec_mysql "$createCmd" $wmaDBName || return
 
             echo "$FUNCNAME: Inserting current Agent's build id and hostname at database: $wmaDBName"
-            _exec_oracle "insert into wma_init (init_param, init_value) values ('wma_build_id', '$WMA_BUILD_ID');" $wmaDBName || return
-            _exec_oracle "insert into wma_init (init_param, init_value) values ('wma_tag', '$WMA_TAG');"           $wmaDBName || return
-            _exec_oracle "insert into wma_init (init_param, init_value) values ('hostname', '$HOSTNAME');"         $wmaDBName || return
-            _exec_oracle "insert into wma_init (init_param, init_value) values ('is_active', 'true');"             $wmaDBName || return
+            _exec_mysql "insert into wma_init (init_param, init_value) values ('wma_build_id', '$WMA_BUILD_ID');" $wmaDBName || return
+            _exec_mysql "insert into wma_init (init_param, init_value) values ('wma_tag', '$WMA_TAG');"           $wmaDBName || return
+            _exec_mysql "insert into wma_init (init_param, init_value) values ('hostname', '$HOSTNAME');"         $wmaDBName || return
+            _exec_mysql "insert into wma_init (init_param, init_value) values ('is_active', 'true');"             $wmaDBName || return
             ;;
         *)
             echo "$FUNCNAME: ERROR: Unknown or not set Agent Flavor"

From 3e8ed41b438128ffdf4b9c9d86c35f421c2779cc Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 13 Mar 2024 14:27:47 +0100
Subject: [PATCH 37/45] fix bad indentation

---
 docker/pypi/wmagent/bin/manage | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index aaf992d51..63436ade2 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -275,7 +275,7 @@ status(){
 clean_all(){
     case $AGENT_FLAVOR in
         'mysql')
-	        clean_mysql ;;
+            clean_mysql ;;
         'oracle')
             clean_oracle ;;
         *)

From f28e5ba7bad56ae608aff9b254b465f8eaafae44 Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Wed, 13 Mar 2024 15:26:01 +0100
Subject: [PATCH 38/45] Remove database recreation line from clean_mysql && Add
 interactive user confirmation for cleaning the databases

---
 docker/pypi/wmagent/bin/manage | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index 63436ade2..3389545d5 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -232,9 +232,16 @@ clean_agent(){
 clean_mysql(){
     [[ -z $MYSQL_USER ]] && { echo "$FUNCNAME: ERROR: Not using MySQL..."; exit 1 ; }
     local errVal=0
+    echo
+    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $WMA_DATABASE DATABASE."
+    echo -n "$FUNCNAME: Continue? [n]: "
+    read x && [[ $x =~ (y|yes|yeS|yEs|Yes|yES|YEs|YeS|YES|Y) ]] || return 102
+    echo "$FUNCNAME: ..."
+    echo "$FUNCNAME: You still have 5 sec. to cancel before we proceed."
+    echo
+    sleep 5
     echo "$FUNCNAME: Dropping MySQL DB... ";
     _exec_mysql "drop database $wmaDBName"   || let errVal+=$?
-    _exec_mysql "create database $wmaDBName" || let errVal+=$?
     [[ $errVal -eq 0 ]] &&  rm -f $wmaInitSqlDB
     return $errVal
 }
@@ -242,6 +249,14 @@ clean_mysql(){
 clean_oracle(){
     [[ -z $ORACLE_USER ]] && { echo "$FUNCNAME: ERROR: Not using ORACLE..."; exit 1 ; }
     local errVal=0
+    echo
+    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $WMA_DATABASE DATABASE."
+    echo -n "$FUNCNAME: Continue? [n]: "
+    read x && [[ $x =~ (y|yes|yeS|yEs|Yes|yES|YEs|YeS|YES|Y) ]] || return 102
+    echo "$FUNCNAME: ..."
+    echo "$FUNCNAME: You still have 5 sec. to cancel before we proceed."
+    echo
+    sleep 5
     echo "$FUNCNAME: Dropping Oracle DB..."
     execute_command_agent "clean-oracle" || let errVal+=$?
     [[ $errVal -eq 0 ]] && rm -f $wmaInitSqlDB

From f4e0835f17261e3cbf3907c28c2a90c15f7effc3 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:23:04 +0100
Subject: [PATCH 39/45] Update Dockerfile

---
 docker/frontend/Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile
index f429b17c0..4190eb404 100644
--- a/docker/frontend/Dockerfile
+++ b/docker/frontend/Dockerfile
@@ -57,6 +57,8 @@ ADD alerts.sh $WDIR/alerts.sh
 COPY authmap-prod.cron /tmp/authmap-prod.cron
 COPY authmap-preprod.cron /tmp/data/tools/authmap-preprod.cron
 COPY authmap-test.cron /tmp/authmap-test.cron
+COPY copy_cron.sh $WDIR/copy_cron.sh
+
 
 
 ENV PATH="${WDIR}/cmsweb/bin:${WDIR}:${WDIR}/gopath/bin:${PATH}"

From 22993ffbeaf9fbb1fc9ac366e9f2e517b5418894 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:50:46 +0100
Subject: [PATCH 40/45] Update install.sh

---
 docker/frontend/install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/frontend/install.sh b/docker/frontend/install.sh
index 724e1686d..cb51df534 100755
--- a/docker/frontend/install.sh
+++ b/docker/frontend/install.sh
@@ -121,8 +121,8 @@ crontab -l | \
 # add proxy generation via robot certificate
 crontab -l | egrep -v "reboot|ProxyRenew|LogArchive|ServerMonitor" > /tmp/mycron
 echo "0 0 * * * sudo /usr/sbin/fetch-crl" >> /tmp/mycron
+sed -i '/mkauthmap/d' /tmp/mycron
 chmod +x copy_cron.sh && ./copy_cron.sh && cat /tmp/authmap.cron >> /tmp/mycron
-(crontab -l | grep -v "mkauthmap") | crontab -
 
 crontab /tmp/mycron
 rm /tmp/mycron

From 6972f1360098eb812cca9367013e1677266b27fd Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:54:33 +0100
Subject: [PATCH 41/45] Update authmap-test.cron

---
 docker/frontend/authmap-test.cron | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/frontend/authmap-test.cron b/docker/frontend/authmap-test.cron
index 1e897b509..2575d1f78 100644
--- a/docker/frontend/authmap-test.cron
+++ b/docker/frontend/authmap-test.cron
@@ -1,2 +1,2 @@
-*/30 * * * * sleep $((RANDOM \% 901)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
+*/30 * * * * sleep $((RANDOM % 901)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
 

From c15b06672601fe45bebfdd842ba7b04e48873d21 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:54:55 +0100
Subject: [PATCH 42/45] Update authmap-preprod.cron

---
 docker/frontend/authmap-preprod.cron | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/frontend/authmap-preprod.cron b/docker/frontend/authmap-preprod.cron
index 947aeefbe..b76717e13 100644
--- a/docker/frontend/authmap-preprod.cron
+++ b/docker/frontend/authmap-preprod.cron
@@ -1,2 +1,2 @@
-*/15 * * * * sleep $((RANDOM \% 601)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
+*/15 * * * * sleep $((RANDOM % 601)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
 

From 83e81c187a842e32bbbff506ac0b468020d5019d Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:55:17 +0100
Subject: [PATCH 43/45] Update authmap-prod.cron

---
 docker/frontend/authmap-prod.cron | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/frontend/authmap-prod.cron b/docker/frontend/authmap-prod.cron
index 947aeefbe..b76717e13 100644
--- a/docker/frontend/authmap-prod.cron
+++ b/docker/frontend/authmap-prod.cron
@@ -1,2 +1,2 @@
-*/15 * * * * sleep $((RANDOM \% 601)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
+*/15 * * * * sleep $((RANDOM % 601)); /data/srv/current/config/frontend/mkauthmap  -c /data/srv/current/config/frontend/mkauth.conf -o /data/srv/state/frontend/etc/authmap.json --cert /etc/robots/robotcert.pem --key /etc/robots/robotkey.pem --ca-cert /etc/ssl/certs/CERN-bundle.pem ; [ $? -ne 0 ] && /bin/bash /data/alerts.sh
 

From bbc17ee1342bf28c373ad5d0ed4b1f30ff975cba Mon Sep 17 00:00:00 2001
From: Todor Ivanov <todor.trendafilov.ivanov@cern.ch>
Date: Thu, 14 Mar 2024 10:47:54 +0100
Subject: [PATCH 44/45] Swap MYSQL_ with MDB_ environment variables

---
 docker/pypi/wmagent/README.md            |  4 +--
 docker/pypi/wmagent/bin/manage           | 12 ++++-----
 docker/pypi/wmagent/bin/manage-common.sh | 34 ++++++++++++------------
 docker/pypi/wmagent/init.sh              |  4 +--
 4 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/docker/pypi/wmagent/README.md b/docker/pypi/wmagent/README.md
index 373a2c3b3..39d22d7c1 100644
--- a/docker/pypi/wmagent/README.md
+++ b/docker/pypi/wmagent/README.md
@@ -348,8 +348,8 @@ They are set at different moments along the process: i.e. at `buildtime` - when
 
 * **List  of variables set at `configtime` defined in $WMAgent.secrets file:**
 
-   * MYSQL_USER=*****
-   * MYSQL_PASS=*****
+   * MDB_USER=*****
+   * MDB_PASS=*****
 
     or
 
diff --git a/docker/pypi/wmagent/bin/manage b/docker/pypi/wmagent/bin/manage
index 3389545d5..7ee39c408 100755
--- a/docker/pypi/wmagent/bin/manage
+++ b/docker/pypi/wmagent/bin/manage
@@ -85,7 +85,7 @@ db_prompt(){
     case $AGENT_FLAVOR in
         'mysql')
             # _exec_mysql $@
-            mysql -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$wmaDBName --pager='less -SFX'
+            mysql -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --database=$wmaDBName --pager='less -SFX'
             ;;
         'oracle')
             _exec_oracle $@
@@ -110,7 +110,7 @@ init_wmagent(){
         'mysql')
             _exec_mysql "create database if not exists $wmaDBName" && \
                 echo "$FUNCNAME: MYSQL database: $wmaDBName has been created"
-            local database_options="--mysql_url=mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_HOST/$wmaDBName "
+            local database_options="--mysql_url=mysql://$MDB_USER:$MDB_PASS@$MDB_HOST/$wmaDBName "
             ;;
         'oracle')
             echo "$FUNCNAME: Using ORACLE user schema: $ORACLE_USER@$ORACLE_TNS "
@@ -230,17 +230,17 @@ clean_agent(){
 # Wipe out MySQL and Oracle databases
 # Will cause next start to recreate databases
 clean_mysql(){
-    [[ -z $MYSQL_USER ]] && { echo "$FUNCNAME: ERROR: Not using MySQL..."; exit 1 ; }
+    [[ -z $MDB_USER ]] && { echo "$FUNCNAME: ERROR: Not using MySQL..."; exit 1 ; }
     local errVal=0
     echo
-    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $WMA_DATABASE DATABASE."
+    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $wmaDBName DATABASE."
     echo -n "$FUNCNAME: Continue? [n]: "
     read x && [[ $x =~ (y|yes|yeS|yEs|Yes|yES|YEs|YeS|YES|Y) ]] || return 102
     echo "$FUNCNAME: ..."
     echo "$FUNCNAME: You still have 5 sec. to cancel before we proceed."
     echo
     sleep 5
-    echo "$FUNCNAME: Dropping MySQL DB... ";
+    echo "$FUNCNAME: Dropping MariaDB DB... ";
     _exec_mysql "drop database $wmaDBName"   || let errVal+=$?
     [[ $errVal -eq 0 ]] &&  rm -f $wmaInitSqlDB
     return $errVal
@@ -250,7 +250,7 @@ clean_oracle(){
     [[ -z $ORACLE_USER ]] && { echo "$FUNCNAME: ERROR: Not using ORACLE..."; exit 1 ; }
     local errVal=0
     echo
-    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $WMA_DATABASE DATABASE."
+    echo    "$FUNCNAME: THE CURRENT OPERATIONS WILL WIPE OUT THE $wmaDBName DATABASE."
     echo -n "$FUNCNAME: Continue? [n]: "
     read x && [[ $x =~ (y|yes|yeS|yEs|Yes|yES|YEs|YeS|YES|Y) ]] || return 102
     echo "$FUNCNAME: ..."
diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh
index 7646050c8..44a03c40a 100644
--- a/docker/pypi/wmagent/bin/manage-common.sh
+++ b/docker/pypi/wmagent/bin/manage-common.sh
@@ -1,6 +1,6 @@
 # Auxiliary script to hold common function definitions between init.sh and manage scripts
 
-# NOTE: At the current stage none of the global variables like $AGENT_FLAVOR or $MYSQL_PASS are
+# NOTE: At the current stage none of the global variables like $AGENT_FLAVOR or $MDB_PASS are
 #       defined. All of those come not from the environment as $WMA_* variables, but rather
 #       from loading the WMAgent.secrets file.  So loading of this script as a primary source
 #       with definitions should work, but calling any of those functions without
@@ -35,23 +35,23 @@ _exec_mysql() {
     local sqlStr=$1
     local dbName=$2
     if [[ -n $dbName ]]; then
-        mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$dbName --execute="$sqlStr"
+        mysql -sN -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --database=$dbName --execute="$sqlStr"
     else
-        mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --execute="$sqlStr"
+        mysql -sN -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --execute="$sqlStr"
     fi
 
     ## TODO: To add the same functionality for recognizing the type of call, similar to _exec_oracle
     #
     # if $isPipe || $noArgs
     # then
-    #     mysql -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$wmaDBName --pager='less -SFX'
+    #     mysql -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --database=$wmaDBName --pager='less -SFX'
     # else
     #     local sqlStr=$1
     #     local dbName=$2
     #     if [[ -n $dbName ]]; then
-    #         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --database=$dbName --execute="$sqlStr"
+    #         mysql -sN -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --database=$dbName --execute="$sqlStr"
     #     else
-    #         mysql -sN -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --execute="$sqlStr"
+    #         mysql -sN -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --execute="$sqlStr"
     #     fi
     # fi
 }
@@ -119,7 +119,7 @@ _sql_dumpSchema(){
     echo "$FUNCNAME: Dumping the current SQL schema of database: $wmaDBName to $wmaSchemaFile"
     case $AGENT_FLAVOR in
         'mysql')
-            mysqldump -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST --no-data --skip-dump-date --compact --skip-opt wmagent > $wmaSchemaFile
+            mysqldump -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST --no-data --skip-dump-date --compact --skip-opt wmagent > $wmaSchemaFile
             ;;
         'oracle')
             echo "$FUNCNAME: NOT implemented"
@@ -239,7 +239,7 @@ _status_of_couch(){
 
 _status_of_mysql(){
     echo "$FUNCNAME:"
-    mysqladmin -u $MYSQL_USER --password=$MYSQL_PASS -h $MYSQL_HOST  status
+    mysqladmin -u $MDB_USER --password=$MDB_PASS -h $MDB_HOST  status
     local errVal=$?
     [[ $errVal -ne 0 ]] && { echo "$FUNCNAME: ERROR: MySQL database unreachable!"; return $(false) ;}
     echo "$FUNCNAME: MySQL connection is OK!"
@@ -361,9 +361,9 @@ _load_wmasecrets(){
     local MATCH_ORACLE_PASS=`cat $WMA_SECRETS_FILE | grep ORACLE_PASS | sed s/ORACLE_PASS=//`
     local MATCH_ORACLE_TNS=`cat $WMA_SECRETS_FILE | grep ORACLE_TNS | sed s/ORACLE_TNS=//`
     local MATCH_GRAFANA_TOKEN=`cat $WMA_SECRETS_FILE | grep GRAFANA_TOKEN | sed s/GRAFANA_TOKEN=//`
-    local MATCH_MYSQL_USER=`cat $WMA_SECRETS_FILE | grep MYSQL_USER | sed s/MYSQL_USER=//`
-    local MATCH_MYSQL_PASS=`cat $WMA_SECRETS_FILE | grep MYSQL_PASS | sed s/MYSQL_PASS=//`
-    local MATCH_MYSQL_HOST=`cat $WMA_SECRETS_FILE | grep MYSQL_HOST | sed s/MYSQL_HOST=//`
+    local MATCH_MDB_USER=`cat $WMA_SECRETS_FILE | grep MDB_USER | sed s/MDB_USER=//`
+    local MATCH_MDB_PASS=`cat $WMA_SECRETS_FILE | grep MDB_PASS | sed s/MDB_PASS=//`
+    local MATCH_MDB_HOST=`cat $WMA_SECRETS_FILE | grep MDB_HOST | sed s/MDB_HOST=//`
     local MATCH_COUCH_USER=`cat $WMA_SECRETS_FILE | grep COUCH_USER | sed s/COUCH_USER=//`
     local MATCH_COUCH_PASS=`cat $WMA_SECRETS_FILE | grep COUCH_PASS | sed s/COUCH_PASS=//`
     local MATCH_COUCH_PORT=`cat $WMA_SECRETS_FILE | grep COUCH_PORT | sed s/COUCH_PORT=//`
@@ -393,9 +393,9 @@ _load_wmasecrets(){
     # database settings (mysql or oracle)
     if [ "x$MATCH_ORACLE_USER" == "x" ]; then
         AGENT_FLAVOR=mysql
-        MYSQL_USER=${MATCH_MYSQL_USER:-$USER};
-        MYSQL_PASS=${MATCH_MYSQL_PASS:-$MYSQL_PASS};
-        MYSQL_HOST=${MATCH_MYSQL_HOST:-127.0.0.1};
+        MDB_USER=${MATCH_MDB_USER:-$USER};
+        MDB_PASS=${MATCH_MDB_PASS:-$MDB_PASS};
+        MDB_HOST=${MATCH_MDB_HOST:-127.0.0.1};
     else
         AGENT_FLAVOR=oracle
         ORACLE_USER=$MATCH_ORACLE_USER;
@@ -465,9 +465,9 @@ _print_settings(){
     echo "ORACLE_PASS=               $ORACLE_PASS               "
     echo "ORACLE_TNS=                $ORACLE_TNS                "
     echo "GRAFANA_TOKEN=             $GRAFANA_TOKEN             "
-    echo "MYSQL_USER=                $MYSQL_USER                "
-    echo "MYSQL_PASS=                $MYSQL_PASS                "
-    echo "MYSQL_HOST=                $MYSQL_HOST                "
+    echo "MDB_USER=                  $MDB_USER                  "
+    echo "MDB_PASS=                  $MDB_PASS                  "
+    echo "MDB_HOST=                  $MDB_HOST                  "
     echo "COUCH_USER=                $COUCH_USER                "
     echo "COUCH_PASS=                $COUCH_PASS                "
     echo "COUCH_PORT=                $COUCH_PORT                "
diff --git a/docker/pypi/wmagent/init.sh b/docker/pypi/wmagent/init.sh
index 5ce424b2e..ad42f4827 100755
--- a/docker/pypi/wmagent/init.sh
+++ b/docker/pypi/wmagent/init.sh
@@ -139,7 +139,7 @@ deploy_to_host(){
             cp -f $WMA_DEPLOY_DIR/deploy/WMAgent.$agentType $WMA_SECRETS_FILE
             # Update WMagent.secrets file:
             echo "$FUNCNAME: Updating WMAgent.secrets file with the current host's details"
-            sed -i "s/MYSQL_USER=.*/MYSQL_USER=$WMA_USER/g" $WMA_SECRETS_FILE
+            sed -i "s/MDB_USER=.*/MDB_USER=$WMA_USER/g" $WMA_SECRETS_FILE
             sed -i "s/COUCH_USER=.*/COUCH_USER=$WMA_USER/g" $WMA_SECRETS_FILE
             sed -i "s/COUCH_HOST=127\.0\.0\.1/COUCH_HOST=$HOSTIP/g" $WMA_SECRETS_FILE
         fi
@@ -261,7 +261,7 @@ check_databases() {
     [[ -n $ORACLE_USER ]] && [[ -n $ORACLE_PASS ]] && [[ -n $ORACLE_TNS ]] && \
         oracleCred=true
 
-    [[ -n $MYSQL_USER ]] && [[ -n $MYSQL_PASS ]] && \
+    [[ -n $MDB_USER ]] && [[ -n $MDB_PASS ]] && \
         mysqlCred=true
 
     # Checking the relational databases:

From abeb082cdc57f373a42fff800ea4fb105eb0b212 Mon Sep 17 00:00:00 2001
From: "A. Pervaiz" <77356768+arooshap@users.noreply.github.com>
Date: Mon, 18 Mar 2024 15:48:44 +0100
Subject: [PATCH 45/45] Update copy_cron.sh

---
 docker/frontend/copy_cron.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/frontend/copy_cron.sh b/docker/frontend/copy_cron.sh
index 32c2f76ad..d630855cb 100644
--- a/docker/frontend/copy_cron.sh
+++ b/docker/frontend/copy_cron.sh
@@ -6,7 +6,7 @@ elif [ "$ENVIRONMENT" = "k8s-preprod" ]; then
   # Copy the development cron file
   echo "Copying authmap-preprod.cron"
   cp /tmp/authmap-preprod.cron /tmp/authmap.cron
-elif [ "$ENVIRONMENT" = "k8s-test" ]; then
+elif [[ "$ENVIRONMENT" == k8s-test* ]]; then
   # Copy the test cron file
   echo "Copying authmap-test.cron"
   cp /tmp/authmap-test.cron /tmp/authmap.cron