-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into aps-no-cas
- Loading branch information
Showing
136 changed files
with
5,045 additions
and
732 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
# Ref: https://gitlab.cern.ch/db/cerndb-infra-hadoop-conf/-/blob/qa/docker/Dockerfile-HDP3 | ||
FROM python:3.9 as builder | ||
|
||
ENV WDIR=/data | ||
WORKDIR $WDIR | ||
|
||
# Create zip files of stomp.py=7.0.0 to submit Spark workers | ||
RUN apt-get update -y && \ | ||
apt-get install -y zip subversion && \ | ||
# Install python stomp.py==7.0.0 module which is the latest working version with StompAMQ7 | ||
pip install --no-cache-dir -t stomp-v700 https://github.com/jasonrbriggs/stomp.py/archive/refs/tags/v7.0.0.zip && \ | ||
# Create zip file to send to Spark workers, because workers have old version of it (v3 or v4) | ||
cd stomp-v700 && \ | ||
zip -r ../stomp-v700.zip . && \ | ||
cd .. && \ | ||
rm -rf stomp-v700 | ||
|
||
# ----------------------------------------------------------------------------------------- | ||
FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:20240703-1.x86_64 | ||
|
||
ENV WDIR=/data | ||
WORKDIR $WDIR | ||
|
||
ARG stage=stable | ||
|
||
# Create YUM generic repository for Hadoop | ||
RUN touch /etc/yum.repos.d/Cern-Hadoop-Components.repo && \ | ||
echo -e "[hdp9al-$(echo $stage)]\n\ | ||
name=CERN Hadoop repository [$(echo $stage)]\n\ | ||
baseurl=http://linuxsoft.cern.ch/internal/repos/hdp9al-$(echo $stage)/x86_64/os\n\ | ||
enabled=1\n\ | ||
gpgcheck=0\n\ | ||
priority=6" > /etc/yum.repos.d/Cern-Hadoop-Components.repo | ||
|
||
# Other YUM repos needed | ||
RUN touch /etc/yum.repos.d/config9al-$(echo $stage).repo && \ | ||
echo -e "[config9al-$(echo $stage)]\n\ | ||
name=Utilities for Configuration [$(echo $stage)]\n\ | ||
baseurl=http://linuxsoft.cern.ch/internal/repos/config9al-$(echo $stage)/x86_64/os\n\ | ||
enabled=1\n\ | ||
gpgcheck=True\n\ | ||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kojiv2\n\ | ||
priority=30" > /etc/yum.repos.d/config9al-$(echo $stage).repo | ||
|
||
# Define EPEL repository | ||
RUN touch /etc/yum.repos.d/epel.repo && \ | ||
echo -e "[epel]\n\ | ||
name=Extra Packages for Enterprise Linux 9 [HEAD]\n\ | ||
baseurl=http://linuxsoft.cern.ch/epel/9/Everything/x86_64\n\ | ||
enabled=1\n\ | ||
gpgcheck=1\n\ | ||
gpgkey=http://linuxsoft.cern.ch/epel/RPM-GPG-KEY-EPEL-9\n\ | ||
exclude=collectd*,libcollectd*,mcollective,perl-Authen-Krb5,perl-Collectd,puppet,python*collectd_systemd*,koji*,python*koji*\n\ | ||
priority=20" > /etc/yum.repos.d/epel.repo | ||
|
||
|
||
# Add repository files and GPG key | ||
ADD http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo /etc/yum.repos.d/egi.repo | ||
ADD http://linuxsoft.cern.ch/wlcg/wlcg-el9.repo /etc/yum.repos.d/wlcg-el9.repo | ||
ADD RPM-GPG-KEY-wlcg /etc/pki/rpm-gpg/RPM-GPG-KEY-wlcg | ||
|
||
# Update and install necessary packages | ||
RUN dnf update -y && \ | ||
dnf upgrade -y && \ | ||
dnf install -y crontabs cronie cronie-anacron git-core zip unzip mesa-libGL pcre2-utf16 \ | ||
libXrender libSM libtirpc libicu ncurses which \ | ||
wlcg-voms-cms wlcg-iam-lsc-cms wlcg-iam-vomses-cms voms-clients \ | ||
python-pip python3-setuptools \ | ||
hadoop-bin-3.3 spark-bin-3.5 sqoop-bin-1.4 python3 cern-hadoop-config \ | ||
jq trustedbag-client cern-hadoop-xrootd-connector && \ | ||
dnf clean all | ||
|
||
RUN hadoop-set-default-conf.sh analytix | ||
|
||
COPY --from=builder /data $WDIR |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
ARG MONGODBVER=5.0.9 | ||
ARG MONGODBVER=5.0.26 | ||
FROM mongo:$MONGODBVER | ||
MAINTAINER Ceyhun Uzunoglu [email protected] | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
# cmsmon-spark | ||
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-latest | ||
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-el9-20240709 | ||
MAINTAINER Ceyhun Uzunoglu [email protected] | ||
|
||
# tag to use | ||
|
@@ -14,13 +14,8 @@ ENV HADOOP_CONF_DIR=/etc/hadoop/conf | |
ENV PATH="${PATH}:${WDIR}/CMSSpark/bin" | ||
ENV PYTHONPATH "${PYTHONPATH}:${WDIR}:${WDIR}/CMSSpark/src/python:${WDIR}/CMSMonitoring/src/python" | ||
|
||
# How to find: source LCG102 hadoop setup, which python, ln -ls python, voila! | ||
ENV PYSPARK_PYTHON=/cvmfs/sft.cern.ch/lcg/releases/Python/3.9.12-9a1bc/x86_64-centos7-gcc8-opt/bin/python3 | ||
ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3 | ||
ENV LC_ALL=en_US.utf-8 LANG=en_US.utf-8 | ||
|
||
RUN CMSSPARK_TAG="$CMSSPARK_TAG" CMSMON_TAG="$CMSMON_TAG" ./install.sh && \ | ||
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.2 spark3 | ||
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.3 spark3 | ||
|
||
# Run crond | ||
CMD ["crond", "-n", "-s", "&"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
FROM cern/cc7-base:20230901-2 | ||
FROM cern/cc7-base:20240501-1 | ||
MAINTAINER Valentin Kuznetsov [email protected] | ||
ADD http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo /etc/yum.repos.d/egi.repo | ||
ADD http://linuxsoft.cern.ch/wlcg/wlcg-centos7.repo /etc/yum.repos.d/wlcg-centos7.repo | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,7 +2,7 @@ FROM registry.cern.ch/cmsweb/exporters:20231002-stable as exporters-builder | |
RUN mkdir -p /data | ||
|
||
|
||
FROM cern/cc7-base:20230901-2 | ||
FROM cern/cc7-base:20240501-1 | ||
|
||
MAINTAINER Valentin Kuznetsov [email protected] | ||
|
||
|
@@ -24,7 +24,8 @@ RUN yum install -y cern-get-certificate fetch-crl \ | |
git-core zip unzip which file bzip2 e2fsprogs e2fsprogs-libs compat-libstdc++-33 \ | ||
CERN-CA-certs ca-certificates dummy-ca-certs ca-policy-lcg ca-policy-egi-core \ | ||
ca_EG-GRID ca_CERN-GridCA ca_CERN-LCG-IOTA-CA ca_CERN-Root-2 \ | ||
wlcg-voms-cms krb5-workstation krb5-libs pam_krb5 myproxy voms-clients-cpp voms-clients-java \ | ||
wlcg-voms-cms wlcg-iam-lsc-cms wlcg-iam-vomses-cms \ | ||
krb5-workstation krb5-libs pam_krb5 myproxy voms-clients-cpp voms-clients-java \ | ||
sudo openssl openssl-devel openssl-libs openssh openssh-clients python-backports-ssl_match_hostname \ | ||
cmake voms voms-devel globus-gsi-credential-devel globus-gsi-cert-utils-devel \ | ||
globus-common-devel globus-gsi-sysconfig globus-gsi-sysconfig-devel globus-gsi-callback-devel \ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,31 +1,28 @@ | ||
|
||
|
||
### build image | ||
docker build -t cmssw/couchdb . | ||
|
||
### list images | ||
docker images | ||
|
||
### list of running containers | ||
docker ps --no-trunc -aq | ||
|
||
### remove all running containers | ||
docker rm -f `docker ps --no-trunc -aq` | ||
|
||
### run given image | ||
docker run --rm -h `hostname -f` -v /tmp/vk:/etc/secrets -i -t cmssw/couchdb /bin/bash | ||
|
||
### remove existing image | ||
docker rmi cmssw/couchdb | ||
|
||
### inspect running container | ||
docker ps # find docker id | ||
docker inspect <docker_id> | grep IPAddress | ||
|
||
### push image to docker.com | ||
docker push cmssw/couchdb | ||
|
||
### references | ||
https://stackoverflow.com/questions/18497688/run-a-docker-image-as-a-container#18498313 | ||
https://stackoverflow.com/questions/17236796/how-to-remove-old-docker-containers#17237701 | ||
http://goinbigdata.com/docker-run-vs-cmd-vs-entrypoint/ | ||
### CouchDB image construction | ||
Image is built with the expected `_couchdb` user, also used to run the service. | ||
* latest version has been tagged as `3.2.2-stable`. | ||
* **configuration**: available under `/data/srv/auth/couchdb/`, | ||
* **database**: both database and views are available under `/data/srv/state/couchdb/database/`. | ||
* **couchapps**: couchapps libraries and code is available under `/data/srv/state/couchdb/stagingarea/`. | ||
* **logs**: finally, service logs can be found at `/data/srv/logs/couchdb/`. | ||
|
||
### Run CouchDB container | ||
Connect to the cmsweb backend VM with the cmsweb account and: | ||
``` | ||
cd /data/srv | ||
curl https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/couchdb/docker-run.sh > docker-run.sh | ||
chmod +x docker-run.sh | ||
./docker-run.sh | ||
``` | ||
|
||
### Stop CouchDB container | ||
Connect to the cmsweb backend VM with the cmsweb account and: | ||
``` | ||
docker stop couchdb | ||
docker rm couchdb | ||
``` | ||
|
||
### Checking status of CouchDB service | ||
``` | ||
docker exec -it couchdb sh -c "/data/srv/current/config/couchdb/manage status" | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
#!/usr/bin/env python | ||
""" | ||
Script to list all databases in CouchDB and print a basic summary for each of them. | ||
""" | ||
import os | ||
import sys | ||
import socket | ||
import requests | ||
from itertools import chain | ||
|
||
# Fetch a list of databases | ||
resp = requests.get(f'http://localhost:5984/_all_dbs') | ||
if resp.status_code >= 400: | ||
print(f"Failed to list databases: {resp.status_code}") | ||
else: | ||
all_dbs = resp.json() | ||
print(f"Node contains the following databases: {all_dbs}") | ||
|
||
|
||
print(f"\n***** Summary of databases ****") | ||
for db_name in all_dbs: | ||
resp = requests.get(f'http://localhost:5984/{db_name}') | ||
if resp.status_code >= 400: | ||
print(f"Request failed for db {db_name} with status code: {resp.status_code}") | ||
else: | ||
data = {} | ||
for kname, kdata in resp.json().items(): | ||
if kname in ["db_name", "doc_count", "doc_del_count", "sizes"]: | ||
data[kname] = kdata | ||
print(data) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
#!/bin/bash | ||
COUCH_LOGS_DIR=/data/srv/logs/couchdb/ | ||
COUCH_DB_DIR=/data/srv/state/couchdb/database/ | ||
COUCH_STAGING_DIR=/data/srv/state/couchdb/stagingarea/ | ||
COUCH_USR=_couchdb | ||
|
||
echo "Creating necessary directories on the host to persist logs and data" | ||
mkdir -p $COUCH_LOGS_DIR | ||
mkdir -p $COUCH_DB_DIR | ||
mkdir -p $COUCH_STAGING_DIR | ||
sudo chown $COUCH_USR:zh $COUCH_LOGS_DIR | ||
sudo chown -R $COUCH_USR:zh $COUCH_DB_DIR/.. | ||
|
||
# Define directory to store credentials and standard configuration | ||
COUCH_SECR_DIR=/data/srv/auth/couchdb/ | ||
echo "Creating directory to store credentials and local.ini under: $COUCH_SECR_DIR" | ||
mkdir -p $COUCH_SECR_DIR | ||
sudo chown -R $COUCH_USR:zh $COUCH_SECR_DIR | ||
|
||
# Define command line arguments for docker run | ||
dockerOpts=" \ | ||
--detach \ | ||
--network=host \ | ||
--hostname=$(hostname -f) \ | ||
--name=couchdb \ | ||
--mount type=bind,source=$COUCH_SECR_DIR,target=/etc/secrets \ | ||
--mount type=bind,source=$COUCH_DB_DIR,target=$COUCH_DB_DIR \ | ||
--mount type=bind,source=$COUCH_STAGING_DIR,target=$COUCH_STAGING_DIR \ | ||
--mount type=bind,source=$COUCH_LOGS_DIR,target=$COUCH_LOGS_DIR \ | ||
" | ||
|
||
couch_tag=3.2.2-stable | ||
echo "Executing docker run for CouchDB tag: $couch_tag" | ||
docker run $dockerOpts registry.cern.ch/cmsweb/couchdb:$couch_tag && docker logs -f couchdb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,6 @@ | ||
# cpueff-spark | ||
FROM mongo:5.0.14 as builder | ||
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-latest | ||
FROM registry.cern.ch/cmsmonitoring/cmsmon-hadoop-base:spark3-el9-20240709 | ||
MAINTAINER Ceyhun Uzunoglu [email protected] | ||
|
||
# tag to use | ||
|
@@ -9,14 +9,9 @@ ENV WDIR=/data | |
WORKDIR $WDIR | ||
|
||
ENV HADOOP_CONF_DIR=/etc/hadoop/conf | ||
ENV PATH="${PATH}:${WDIR}/CMSMonitoring/cpueff-goweb/spark" | ||
ENV PATH="${PATH}:${WDIR}:${WDIR}/CMSMonitoring/cpueff-goweb/spark" | ||
ENV PYTHONPATH "${PYTHONPATH}:${WDIR}:${WDIR}/CMSSpark/src/python:${WDIR}/CMSMonitoring/src/python:${WDIR}/CMSMonitoring/cpueff-goweb/spark" | ||
|
||
# How to find: source LCG102 hadoop setup, which python, ln -ls python, voila! | ||
ENV PYSPARK_PYTHON=/cvmfs/sft.cern.ch/lcg/releases/Python/3.9.12-9a1bc/x86_64-centos7-gcc8-opt/bin/python3 | ||
ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3 | ||
ENV LC_ALL=en_US.utf-8 LANG=en_US.utf-8 | ||
|
||
COPY --from=builder /usr/bin/mongoimport $WDIR | ||
COPY --from=builder /usr/bin/mongosh $WDIR | ||
|
||
|
@@ -26,7 +21,7 @@ RUN mkdir -p $WDIR/logs && \ | |
cd CMSMonitoring && git checkout tags/$CMSMON_TAG -b build && cd .. && \ | ||
zip -r CMSMonitoring.zip CMSMonitoring/src/python/CMSMonitoring/* && \ | ||
pip install --no-cache-dir stomp.py==7.0.0 click pyspark pandas numpy schema seaborn matplotlib plotly && \ | ||
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.2 spark3 | ||
hadoop-set-default-conf.sh analytix && source hadoop-setconf.sh analytix 3.3 spark3 | ||
|
||
WORKDIR $WDIR | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.