Skip to content

Commit

Permalink
Merge branch 'master' into aps-no-cas
Browse files Browse the repository at this point in the history
  • Loading branch information
vkuznet authored Jul 18, 2024
2 parents 59cf271 + 17f07fc commit a410f2b
Show file tree
Hide file tree
Showing 136 changed files with 5,045 additions and 732 deletions.
2 changes: 1 addition & 1 deletion docker/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ for pkg in $cmssw_pkgs; do

echo "### build $repo/$pkg"
if [ -n "$CMSK8STAG" ]; then
docker build --build-arg CMSK8S=$CMSK8S --build-arg CMSWEB_ENV=$CMSWEB_ENV -t $repo/$pkg -t $repo/$pkg:$CMSK8STAG $pkg
docker build --build-arg CMSK8S=$CMSK8S --build-arg CMSWEB_ENV=$CMSWEB_ENV -t $repo/$pkg -t $repo/$pkg:$CMSK8STAG $pkg --no-cache
docker tag $repo/$pkg:$CMSK8STAG $registry/$pkg:$CMSK8STAG
if [ "$pkg" == "reqmgr2ms" ] ; then
docker tag $repo/$pkg:$CMSK8STAG $registry/reqmgr2ms-output:$CMSK8STAG
Expand Down
75 changes: 75 additions & 0 deletions docker/cmsmon-hadoop-base/Dockerfile-spark3-el9
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Ref: https://gitlab.cern.ch/db/cerndb-infra-hadoop-conf/-/blob/qa/docker/Dockerfile-HDP3
FROM python:3.9 as builder

ENV WDIR=/data
WORKDIR $WDIR

# Create zip files of stomp.py=7.0.0 to submit Spark workers
RUN apt-get update -y && \
apt-get install -y zip subversion && \
# Install python stomp.py==7.0.0 module which is the latest working version with StompAMQ7
pip install --no-cache-dir -t stomp-v700 https://github.com/jasonrbriggs/stomp.py/archive/refs/tags/v7.0.0.zip && \
# Create zip file to send to Spark workers, because workers have old version of it (v3 or v4)
cd stomp-v700 && \
zip -r ../stomp-v700.zip . && \
cd .. && \
rm -rf stomp-v700

# -----------------------------------------------------------------------------------------
FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:20240703-1.x86_64

ENV WDIR=/data
WORKDIR $WDIR

ARG stage=stable

# Create YUM generic repository for Hadoop
RUN touch /etc/yum.repos.d/Cern-Hadoop-Components.repo && \
echo -e "[hdp9al-$(echo $stage)]\n\
name=CERN Hadoop repository [$(echo $stage)]\n\
baseurl=http://linuxsoft.cern.ch/internal/repos/hdp9al-$(echo $stage)/x86_64/os\n\
enabled=1\n\
gpgcheck=0\n\
priority=6" > /etc/yum.repos.d/Cern-Hadoop-Components.repo

# Other YUM repos needed
RUN touch /etc/yum.repos.d/config9al-$(echo $stage).repo && \
echo -e "[config9al-$(echo $stage)]\n\
name=Utilities for Configuration [$(echo $stage)]\n\
baseurl=http://linuxsoft.cern.ch/internal/repos/config9al-$(echo $stage)/x86_64/os\n\
enabled=1\n\
gpgcheck=True\n\
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kojiv2\n\
priority=30" > /etc/yum.repos.d/config9al-$(echo $stage).repo

# Define EPEL repository
RUN touch /etc/yum.repos.d/epel.repo && \
echo -e "[epel]\n\
name=Extra Packages for Enterprise Linux 9 [HEAD]\n\
baseurl=http://linuxsoft.cern.ch/epel/9/Everything/x86_64\n\
enabled=1\n\
gpgcheck=1\n\
gpgkey=http://linuxsoft.cern.ch/epel/RPM-GPG-KEY-EPEL-9\n\
exclude=collectd*,libcollectd*,mcollective,perl-Authen-Krb5,perl-Collectd,puppet,python*collectd_systemd*,koji*,python*koji*\n\
priority=20" > /etc/yum.repos.d/epel.repo


# Add repository files and GPG key
ADD http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo /etc/yum.repos.d/egi.repo
ADD http://linuxsoft.cern.ch/wlcg/wlcg-el9.repo /etc/yum.repos.d/wlcg-el9.repo
ADD RPM-GPG-KEY-wlcg /etc/pki/rpm-gpg/RPM-GPG-KEY-wlcg

# Update and install necessary packages
RUN dnf update -y && \
dnf upgrade -y && \
dnf install -y crontabs cronie cronie-anacron git-core zip unzip mesa-libGL pcre2-utf16 \
libXrender libSM libtirpc libicu ncurses which \
wlcg-voms-cms wlcg-iam-lsc-cms wlcg-iam-vomses-cms voms-clients \
python-pip python3-setuptools \
hadoop-bin-3.3 spark-bin-3.5 sqoop-bin-1.4 python3 cern-hadoop-config \
jq trustedbag-client cern-hadoop-xrootd-connector && \
dnf clean all

RUN hadoop-set-default-conf.sh analytix

COPY --from=builder /data $WDIR
2 changes: 1 addition & 1 deletion docker/cmsmon-mongo/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ARG MONGODBVER=5.0.9
ARG MONGODBVER=5.0.26
FROM mongo:$MONGODBVER
MAINTAINER Ceyhun Uzunoglu [email protected]

Expand Down
9 changes: 2 additions & 7 deletions docker/cmsmon-spark/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# cmsmon-spark
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-latest
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-el9-20240709
MAINTAINER Ceyhun Uzunoglu [email protected]

# tag to use
Expand All @@ -14,13 +14,8 @@ ENV HADOOP_CONF_DIR=/etc/hadoop/conf
ENV PATH="${PATH}:${WDIR}/CMSSpark/bin"
ENV PYTHONPATH "${PYTHONPATH}:${WDIR}:${WDIR}/CMSSpark/src/python:${WDIR}/CMSMonitoring/src/python"

# How to find: source LCG102 hadoop setup, which python, ln -ls python, voila!
ENV PYSPARK_PYTHON=/cvmfs/sft.cern.ch/lcg/releases/Python/3.9.12-9a1bc/x86_64-centos7-gcc8-opt/bin/python3
ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3
ENV LC_ALL=en_US.utf-8 LANG=en_US.utf-8

RUN CMSSPARK_TAG="$CMSSPARK_TAG" CMSMON_TAG="$CMSMON_TAG" ./install.sh && \
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.2 spark3
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.3 spark3

# Run crond
CMD ["crond", "-n", "-s", "&"]
2 changes: 1 addition & 1 deletion docker/cmsweb-base/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM cern/cc7-base:20230901-2
FROM cern/cc7-base:20240501-1
MAINTAINER Valentin Kuznetsov [email protected]
ADD http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo /etc/yum.repos.d/egi.repo
ADD http://linuxsoft.cern.ch/wlcg/wlcg-centos7.repo /etc/yum.repos.d/wlcg-centos7.repo
Expand Down
5 changes: 3 additions & 2 deletions docker/cmsweb/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ FROM registry.cern.ch/cmsweb/exporters:20231002-stable as exporters-builder
RUN mkdir -p /data


FROM cern/cc7-base:20230901-2
FROM cern/cc7-base:20240501-1

MAINTAINER Valentin Kuznetsov [email protected]

Expand All @@ -24,7 +24,8 @@ RUN yum install -y cern-get-certificate fetch-crl \
git-core zip unzip which file bzip2 e2fsprogs e2fsprogs-libs compat-libstdc++-33 \
CERN-CA-certs ca-certificates dummy-ca-certs ca-policy-lcg ca-policy-egi-core \
ca_EG-GRID ca_CERN-GridCA ca_CERN-LCG-IOTA-CA ca_CERN-Root-2 \
wlcg-voms-cms krb5-workstation krb5-libs pam_krb5 myproxy voms-clients-cpp voms-clients-java \
wlcg-voms-cms wlcg-iam-lsc-cms wlcg-iam-vomses-cms \
krb5-workstation krb5-libs pam_krb5 myproxy voms-clients-cpp voms-clients-java \
sudo openssl openssl-devel openssl-libs openssh openssh-clients python-backports-ssl_match_hostname \
cmake voms voms-devel globus-gsi-credential-devel globus-gsi-cert-utils-devel \
globus-common-devel globus-gsi-sysconfig globus-gsi-sysconfig-devel globus-gsi-callback-devel \
Expand Down
14 changes: 4 additions & 10 deletions docker/couchdb/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ RUN pip install CMSCouchapp

ENV WDIR=/data
ENV USER=_couchdb
ENV UGID=100022

RUN mkdir -p /etc/grid-security

# add new user
RUN useradd ${USER} && install -o ${USER} -d ${WDIR}
RUN groupadd -g ${UGID} ${USER}
RUN useradd -m ${USER} -u ${UGID} -g ${UGID}
RUN install -o ${USER} -d ${WDIR}
# add user to sudoers file
RUN echo "%$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
# switch to user
Expand All @@ -20,14 +23,6 @@ USER ${USER}
RUN mkdir -p $WDIR
WORKDIR ${WDIR}

# pass env variable to the build
ARG CMSK8S
ENV CMSK8S=$CMSK8S
ARG COUCH_USER
ENV COUCH_USER=$COUCH_USER
ARG COUCH_PASS
ENV COUCH_PASS=$COUCH_PASS

# get binary build of couchdb exporter
RUN curl -ksLO https://github.com/gesellix/couchdb-prometheus-exporter/releases/download/v27.0.0/couchdb-prometheus-exporter_27.0.0_Linux_x86_64.tar.gz
RUN tar xfz couchdb-prometheus-exporter_27.0.0_Linux_x86_64.tar.gz
Expand All @@ -49,7 +44,6 @@ RUN mkdir -p /data/srv/current/config/couchdb \
&& ln -s /etc/secrets/couch_creds /data/srv/current/auth/couchdb/couch_creds \
&& ln -s /etc/secrets/local.ini /data/srv/current/config/couchdb/local.ini
ADD manage /data/srv/current/config/couchdb/manage
ADD couchdb-logrotate.conf /data/srv/current/config/couchdb/couchdb-logrotate.conf

ENV PATH="/opt/couchdb/bin:/usr/local/bin/:${PATH}"

Expand Down
59 changes: 28 additions & 31 deletions docker/couchdb/README.md
Original file line number Diff line number Diff line change
@@ -1,31 +1,28 @@


### build image
docker build -t cmssw/couchdb .

### list images
docker images

### list of running containers
docker ps --no-trunc -aq

### remove all running containers
docker rm -f `docker ps --no-trunc -aq`

### run given image
docker run --rm -h `hostname -f` -v /tmp/vk:/etc/secrets -i -t cmssw/couchdb /bin/bash

### remove existing image
docker rmi cmssw/couchdb

### inspect running container
docker ps # find docker id
docker inspect <docker_id> | grep IPAddress

### push image to docker.com
docker push cmssw/couchdb

### references
https://stackoverflow.com/questions/18497688/run-a-docker-image-as-a-container#18498313
https://stackoverflow.com/questions/17236796/how-to-remove-old-docker-containers#17237701
http://goinbigdata.com/docker-run-vs-cmd-vs-entrypoint/
### CouchDB image construction
Image is built with the expected `_couchdb` user, also used to run the service.
* latest version has been tagged as `3.2.2-stable`.
* **configuration**: available under `/data/srv/auth/couchdb/`,
* **database**: both database and views are available under `/data/srv/state/couchdb/database/`.
* **couchapps**: couchapps libraries and code is available under `/data/srv/state/couchdb/stagingarea/`.
* **logs**: finally, service logs can be found at `/data/srv/logs/couchdb/`.

### Run CouchDB container
Connect to the cmsweb backend VM with the cmsweb account and:
```
cd /data/srv
curl https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/couchdb/docker-run.sh > docker-run.sh
chmod +x docker-run.sh
./docker-run.sh
```

### Stop CouchDB container
Connect to the cmsweb backend VM with the cmsweb account and:
```
docker stop couchdb
docker rm couchdb
```

### Checking status of CouchDB service
```
docker exec -it couchdb sh -c "/data/srv/current/config/couchdb/manage status"
```
30 changes: 30 additions & 0 deletions docker/couchdb/db_summary.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env python
"""
Script to list all databases in CouchDB and print a basic summary for each of them.
"""
import os
import sys
import socket
import requests
from itertools import chain

# Fetch a list of databases
resp = requests.get(f'http://localhost:5984/_all_dbs')
if resp.status_code >= 400:
print(f"Failed to list databases: {resp.status_code}")
else:
all_dbs = resp.json()
print(f"Node contains the following databases: {all_dbs}")


print(f"\n***** Summary of databases ****")
for db_name in all_dbs:
resp = requests.get(f'http://localhost:5984/{db_name}')
if resp.status_code >= 400:
print(f"Request failed for db {db_name} with status code: {resp.status_code}")
else:
data = {}
for kname, kdata in resp.json().items():
if kname in ["db_name", "doc_count", "doc_del_count", "sizes"]:
data[kname] = kdata
print(data)
34 changes: 34 additions & 0 deletions docker/couchdb/docker-run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash
COUCH_LOGS_DIR=/data/srv/logs/couchdb/
COUCH_DB_DIR=/data/srv/state/couchdb/database/
COUCH_STAGING_DIR=/data/srv/state/couchdb/stagingarea/
COUCH_USR=_couchdb

echo "Creating necessary directories on the host to persist logs and data"
mkdir -p $COUCH_LOGS_DIR
mkdir -p $COUCH_DB_DIR
mkdir -p $COUCH_STAGING_DIR
sudo chown $COUCH_USR:zh $COUCH_LOGS_DIR
sudo chown -R $COUCH_USR:zh $COUCH_DB_DIR/..

# Define directory to store credentials and standard configuration
COUCH_SECR_DIR=/data/srv/auth/couchdb/
echo "Creating directory to store credentials and local.ini under: $COUCH_SECR_DIR"
mkdir -p $COUCH_SECR_DIR
sudo chown -R $COUCH_USR:zh $COUCH_SECR_DIR

# Define command line arguments for docker run
dockerOpts=" \
--detach \
--network=host \
--hostname=$(hostname -f) \
--name=couchdb \
--mount type=bind,source=$COUCH_SECR_DIR,target=/etc/secrets \
--mount type=bind,source=$COUCH_DB_DIR,target=$COUCH_DB_DIR \
--mount type=bind,source=$COUCH_STAGING_DIR,target=$COUCH_STAGING_DIR \
--mount type=bind,source=$COUCH_LOGS_DIR,target=$COUCH_LOGS_DIR \
"

couch_tag=3.2.2-stable
echo "Executing docker run for CouchDB tag: $couch_tag"
docker run $dockerOpts registry.cern.ch/cmsweb/couchdb:$couch_tag && docker logs -f couchdb
29 changes: 3 additions & 26 deletions docker/couchdb/run.sh
Original file line number Diff line number Diff line change
@@ -1,33 +1,10 @@
#!/bin/bash
srv=`echo $USER | sed -e "s,_,,g"`

# overwrite host PEM files in /data/srv area
if [ -f /etc/secrets/robotkey.pem ]; then
sudo cp /etc/secrets/robotkey.pem /data/srv/current/auth/$srv/dmwm-service-key.pem
sudo cp /etc/secrets/robotcert.pem /data/srv/current/auth/$srv/dmwm-service-cert.pem
sudo chown $USER.$USER /data/srv/current/auth/$srv/dmwm-service-key.pem
sudo chown $USER.$USER /data/srv/current/auth/$srv/dmwm-service-cert.pem
fi
srv=$(echo $USER | sed -e "s,_,,g")

if [ -f /etc/secrets/couch_creds ]; then
export COUCH_CREDS=/etc/secrets/couch_creds
fi

# overwrite proxy if it is present in /etc/proxy
if [ -f /etc/proxy/proxy ]; then
export X509_USER_PROXY=/etc/proxy/proxy
mkdir -p /data/srv/state/$srv/proxy
if [ -f /data/srv/state/$srv/proxy/proxy.cert ]; then
rm /data/srv/state/$srv/proxy/proxy.cert
fi
ln -s /etc/proxy/proxy /data/srv/state/$srv/proxy/proxy.cert
mkdir -p /data/srv/current/auth/proxy
if [ -f /data/srv/current/auth/proxy/proxy ]; then
rm /data/srv/current/auth/proxy/proxy
fi
ln -s /etc/proxy/proxy /data/srv/current/auth/proxy/proxy
fi

# overwrite header-auth key file with one from secrets
if [ -f /etc/secrets/hmac ]; then
mkdir -p /data/srv/current/auth/wmcore-auth
Expand All @@ -53,7 +30,7 @@ fi

# use service configuration files from /etc/secrets if they are present
cdir=/data/srv/current/config/$srv
files=`ls $cdir`
files=$(ls $cdir)
for fname in $files; do
if [ -f /etc/secrets/$fname ]; then
if [ -f $cdir/$fname ]; then
Expand All @@ -63,7 +40,7 @@ for fname in $files; do
sudo chown $USER.$USER $cdir/$fname
fi
done
files=`ls /etc/secrets`
files=$(ls /etc/secrets)
for fname in $files; do
if [ ! -f $cdir/$fname ]; then
sudo cp /etc/secrets/$fname /data/srv/current/auth/$srv/$fname
Expand Down
11 changes: 3 additions & 8 deletions docker/cpueff-goweb/Dockerfile-cpueff-spark
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# cpueff-spark
FROM mongo:5.0.14 as builder
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-latest
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-el9-20240709
MAINTAINER Ceyhun Uzunoglu [email protected]

# tag to use
Expand All @@ -9,14 +9,9 @@ ENV WDIR=/data
WORKDIR $WDIR

ENV HADOOP_CONF_DIR=/etc/hadoop/conf
ENV PATH="${PATH}:${WDIR}/CMSMonitoring/cpueff-goweb/spark"
ENV PATH="${PATH}:${WDIR}:${WDIR}/CMSMonitoring/cpueff-goweb/spark"
ENV PYTHONPATH "${PYTHONPATH}:${WDIR}:${WDIR}/CMSSpark/src/python:${WDIR}/CMSMonitoring/src/python:${WDIR}/CMSMonitoring/cpueff-goweb/spark"

# How to find: source LCG102 hadoop setup, which python, ln -ls python, voila!
ENV PYSPARK_PYTHON=/cvmfs/sft.cern.ch/lcg/releases/Python/3.9.12-9a1bc/x86_64-centos7-gcc8-opt/bin/python3
ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3
ENV LC_ALL=en_US.utf-8 LANG=en_US.utf-8

COPY --from=builder /usr/bin/mongoimport $WDIR
COPY --from=builder /usr/bin/mongosh $WDIR

Expand All @@ -26,7 +21,7 @@ RUN mkdir -p $WDIR/logs && \
cd CMSMonitoring && git checkout tags/$CMSMON_TAG -b build && cd .. && \
zip -r CMSMonitoring.zip CMSMonitoring/src/python/CMSMonitoring/* && \
pip install --no-cache-dir stomp.py==7.0.0 click pyspark pandas numpy schema seaborn matplotlib plotly && \
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.2 spark3
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.3 spark3

WORKDIR $WDIR

Expand Down
2 changes: 1 addition & 1 deletion docker/dbs2go/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM registry.cern.ch/cmsweb/oracle:21_5 as oracle
FROM registry.cern.ch/cmsweb/oracle:21_5-stable as oracle
#FROM cmssw/oracle:21_1 as oracle
FROM cmssw/exporters:latest as exporters
FROM cmssw/filebeat:latest as filebeat
Expand Down
Loading

0 comments on commit a410f2b

Please sign in to comment.