diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..643ed3b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +# Never bake credentials into the image. +etc/.secrets +etc/.secrets/** + +# Local development artefacts. +.venv +venv +.git +.github +.claude +.pytest_cache +.ruff_cache +__pycache__ +**/__pycache__ +*.egg-info +sdf_cli.egg-info +.vscode +.DS_Store diff --git a/.github/workflows/ci-cd.yaml b/.github/workflows/ci-cd.yaml index acaff56..e6f2af9 100644 --- a/.github/workflows/ci-cd.yaml +++ b/.github/workflows/ci-cd.yaml @@ -4,7 +4,6 @@ on: push: branches: ["main"] pull_request: - branches: ["main"] env: UV_SYSTEM_PYTHON: 1 diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml new file mode 100644 index 0000000..8bacf7b --- /dev/null +++ b/.github/workflows/docker_publish.yaml @@ -0,0 +1,145 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + workflow_run: + workflows: [ "CI/CD" ] + types: + - completed + workflow_dispatch: + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + IMAGE_NAME: slaclab/sdf-cli + + +jobs: + build: + + runs-on: ubuntu-latest + if: >- + ${{ github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name == github.repository) }} + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio for signing and attestation. + id-token: write + # Allow to persist attestations + attestations: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} + + # Install the cosign tool + # https://github.com/sigstore/cosign-installer + - name: Install cosign + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve build branch and Docker tag + id: tag + env: + HEAD_BRANCH: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.ref_name }} + run: | + set -euo pipefail + + # Docker tags allow [a-zA-Z0-9._-] only, may not lead with '.' or '-', + # and are capped at 128 chars. Branch names like `feature/foo` are not + # usable verbatim. + SLUG=$(printf '%s' "${HEAD_BRANCH}" \ + | sed -e 's#[^a-zA-Z0-9._-]#-#g' -e 's#^[.-]#_#' \ + | cut -c1-100) + STAMP=$(date -u +"%Y%m%d-%H%M") + + if [ "${HEAD_BRANCH}" = "main" ]; then + TAG="${STAMP}" + else + TAG="${SLUG}-${STAMP}" + fi + + { + echo "branch=${HEAD_BRANCH}" + echo "slug=${SLUG}" + echo "tag=${TAG}" + echo "full_tag=${REGISTRY}/${IMAGE_NAME}:${TAG}" + } >> "$GITHUB_OUTPUT" + + echo "Building ${HEAD_BRANCH} as ${REGISTRY}/${IMAGE_NAME}:${TAG}" >> "$GITHUB_STEP_SUMMARY" + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: | + latest=false + tags: | + type=raw,value=${{ steps.tag.outputs.tag }} + type=raw,value=${{ steps.tag.outputs.slug }} + env: + GITHUB_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} + GITHUB_REF: ${{ github.event_name == 'workflow_run' && format('refs/heads/{0}', github.event.workflow_run.head_branch) || github.ref }} + + # Build and push Docker image with Buildx + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting published Docker image digest. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v2 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.build-and-push.outputs.digest }} + push-to-registry: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fdcd9a9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,149 @@ +# Container image for the Coact batch daemons + +# --------------------------------------------------------------------------- +# Slurm client build +# +# The Slurm client ships in the image rather than being mounted from the +# node's /opt/slurm, so the pod needs no hostPath. SLURM_VERSION must track +# the S3DF slurmctld/slurmdbd: a client newer than the servers is unsupported. +# --------------------------------------------------------------------------- +FROM rockylinux:9 AS slurm-build + +ARG http_proxy +ARG https_proxy +ARG no_proxy + +ARG SLURM_VERSION=25.11.8 +ARG SLURM_SHA256=34ace13f81011add6094569d13bfc4006ad8868201c2236e2905443c7e526393 + +# munge-devel and readline-devel live in CRB. +RUN dnf -y install epel-release dnf-plugins-core \ + && dnf config-manager --set-enabled crb \ + && dnf -y --setopt=install_weak_deps=False install \ + gcc make bzip2 perl python3 \ + munge-devel readline-devel \ + && dnf clean all + +WORKDIR /build +RUN curl -fsSLo slurm.tar.bz2 "https://download.schedmd.com/slurm/slurm-${SLURM_VERSION}.tar.bz2" \ + && echo "${SLURM_SHA256} slurm.tar.bz2" | sha256sum -c - \ + && tar xjf slurm.tar.bz2 --strip-components=1 \ + && rm slurm.tar.bz2 + +# Same prefix as the S3DF RPMs (/opt/slurm/slurm-, with slurm-curr +# symlinked to it), so PATH and SLURM_BIN_DIR are unchanged from bare metal. +RUN ./configure \ + --prefix=/opt/slurm/slurm-${SLURM_VERSION} \ + --libdir=/opt/slurm/slurm-${SLURM_VERSION}/lib64 \ + --sysconfdir=/etc/slurm \ + --disable-slurmrestd \ + && make -j"$(nproc)" \ + && make install \ + && rm -rf /opt/slurm/slurm-${SLURM_VERSION}/share /opt/slurm/slurm-${SLURM_VERSION}/include \ + && test -e /opt/slurm/slurm-${SLURM_VERSION}/lib64/slurm/auth_munge.so \ + && test -e /opt/slurm/slurm-${SLURM_VERSION}/lib64/slurm/accounting_storage_slurmdbd.so + +# --------------------------------------------------------------------------- +# Runtime image +# --------------------------------------------------------------------------- +FROM rockylinux:9 + +ARG SLURM_VERSION=25.11.8 + +LABEL org.opencontainers.image.source=https://github.com/slaclab/sdf-cli +LABEL org.opencontainers.image.description="Coact batch daemons (slurm job import, facility overage)" +LABEL edu.stanford.slac.slurm.version="${SLURM_VERSION}" + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +# Build behind the SDF proxy with: +# podman build --build-arg https_proxy=http://sdfproxy.sdf.slac.stanford.edu:3128 . +ARG http_proxy +ARG https_proxy +ARG no_proxy + +# --------------------------------------------------------------------------- +# OS packages +# --------------------------------------------------------------------------- +RUN dnf -y install epel-release \ + && dnf -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs install \ + munge \ + readline \ + sssd \ + sssd-client \ + nss-pam-ldapd \ + openldap-clients \ + krb5-workstation \ + python3.12 \ + python3.12-pip \ + tini \ + tzdata \ + procps-ng \ + which \ + glibc-langpack-en \ + shadow-utils \ + ca-certificates \ + && dnf clean all \ + && rm -rf /var/cache/dnf /var/cache/yum + +# Align the munge uid/gid with the SDF hosts +ARG MUNGE_UID=16952 +ARG MUNGE_GID=3761 +# usermod only re-owns the home directory, so the rest of munge's tree is +# re-owned explicitly -- munged refuses directories it does not own. +RUN groupmod -g $MUNGE_GID munge \ + && usermod -u $MUNGE_UID -g $MUNGE_GID munge \ + && chown -R munge:munge /etc/munge /var/lib/munge /var/log/munge /run/munge + +COPY --from=slurm-build /opt/slurm /opt/slurm +RUN ln -s slurm-${SLURM_VERSION} /opt/slurm/slurm-curr \ + && echo /opt/slurm/slurm-curr/lib64 > /etc/ld.so.conf.d/slurm.conf \ + && ldconfig + +# --------------------------------------------------------------------------- +# Runtime environment +# --------------------------------------------------------------------------- +ENV UV_PROJECT_ENVIRONMENT=/opt/venv \ + UV_PYTHON=/usr/bin/python3.12 \ + UV_LINK_MODE=copy \ + SLURM_BIN_DIR=/opt/slurm/slurm-curr/bin \ + SLURM_CONF=/run/slurm/conf/slurm.conf \ + PATH=/opt/venv/bin:/opt/slurm/slurm-curr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + TZ=America/Los_Angeles \ + LANG=en_US.UTF-8 \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /app + +# Dependency layer first so code changes do not invalidate the resolve. +COPY pyproject.toml uv.lock /app/ +RUN uv sync --frozen --no-install-project --no-dev + +COPY . /app +RUN uv sync --frozen --no-dev + +# ansible-runner 2.3.1 imports `pkg_resources` at module scope, and +# sdf_click.py imports modules/coactd.py (hence ansible_runner) unconditionally +# -- even for the `coact` batch subcommands. setuptools >= 81 dropped +# pkg_resources, so the CLI will not start without an older setuptools. +RUN uv pip install --python /opt/venv/bin/python "setuptools<81" + +# --------------------------------------------------------------------------- +# Identity / auth configuration +# --------------------------------------------------------------------------- +COPY etc/nsswitch.conf /etc/nsswitch.conf +COPY etc/krb5.conf /etc/krb5.conf +COPY etc/ldap.conf /etc/openldap/ldap.conf +COPY etc/sssd.conf /etc/sssd/sssd.conf +RUN chmod 0600 /etc/sssd/sssd.conf \ + && install -d -m 0755 /data + +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY munge-sidecar.sh /usr/local/bin/munge-sidecar.sh +RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh /usr/local/bin/munge-sidecar.sh /app/import-jobs.sh + +# tini reaps sssd and forwards signals; the entrypoint execs the CronJob +# command so the container exits with the job's own exit code. +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] +CMD ["python3", "/app/sdf_click.py", "--help"] diff --git a/Makefile b/Makefile index a84afe5..eff6e1e 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,32 @@ -VENV_DIR ?= venv -VENV_BIN ?= $(VENV_DIR)/bin/python3.9 -PIP_BIN ?= $(VENV_DIR)/bin/pip -PYTHON_BIN ?= python3.9 -VAULT_SECRET_PATH ?= secret/tid/coact - -secrets: - mkdir etc/.secrets/ -p - #set -e; for i in ldap_binddn ldap_bindpw; do vault kv get --field=$$i $(VAULT_SECRET_PATH) > etc/.secrets/$$i ; done +## sdf-cli / coact-daemon +## +## Container image build + Vault secret fetching for the batch daemons. + +# --- container image ------------------------------------------------------- +CONTAINER_RT ?= podman +REPO ?= ghcr.io/slaclab +IMAGE ?= sdf-cli +TAG ?= $(shell date +"%Y%m%d-%H%M") + +# --- vault ----------------------------------------------------------------- +VAULT_SECRET_PATH ?= secret/scs/coact +GROUPER_SECRET_PATH ?= secret/tid/scs/osmaint +GROUPER_SECRET_FIELD ?= password + +# --------------------------------------------------------------------------- +# Container image +# --------------------------------------------------------------------------- +build: ## Build the coact-daemon image + $(CONTAINER_RT) build . -f Dockerfile -t $(REPO)/$(IMAGE):$(TAG) + +# --------------------------------------------------------------------------- +# Secrets +# --------------------------------------------------------------------------- +secrets: ## Fetch daemon secrets from Vault into etc/.secrets/ + mkdir -p etc/.secrets/ set -e; for i in password; do vault kv get --field=$$i $(VAULT_SECRET_PATH)/service-account > etc/.secrets/$$i ; done + vault kv get --field=$(GROUPER_SECRET_FIELD) $(GROUPER_SECRET_PATH) > etc/.secrets/grouper_password chmod -R go-rwx etc/.secrets -clean-secrets: +clean-secrets: ## Remove etc/.secrets/ rm -rf etc/.secrets - -virtualenv: - $(PYTHON_BIN) -m venv $(VENV_DIR) - -venv: virtualenv - -pip: - $(VENV_BIN) -m pip install --upgrade pip - $(PIP_BIN) install -r requirements.txt - -# OS level dependencies -deps: - dnf groupinstall -y "Development Tools" - dnf install -y python36-devel openldap-devel - -# run this to configure the dev environment -environment: venv pip - -dev: environment - -update-sdf-ansible: - git submodule update --init --recursive - -apply: environment get-secrets update-sdf-ansible - -test: - $(VENV_BIN) sdf_click.py diff --git a/coact-jobs-import.sh b/coact-jobs-import.sh deleted file mode 100755 index 08e1d94..0000000 --- a/coact-jobs-import.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -export SDF_COACT_URI=coact.slac.stanford.edu:443/graphql-service - -while [ 1 ]; do ./import-jobs.sh; sleep 120; done diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..b78b3d3 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# Runtime bootstrap for the coact-daemon image. +# +# Starts sssd for NSS/LDAP resolution, waits for the munged sidecar's socket, +# then execs the command it was given. + +set -euo pipefail + +START_SSSD="${START_SSSD:-true}" +SSSD_WAIT_SECONDS="${SSSD_WAIT_SECONDS:-15}" +MUNGE_SOCKET="${MUNGE_SOCKET:-/run/munge/munge.socket.2}" +MUNGE_WAIT_SECONDS="${MUNGE_WAIT_SECONDS:-30}" + +log() { printf '[entrypoint] %s\n' "$*" >&2; } +warn() { printf '[entrypoint] WARNING: %s\n' "$*" >&2; } + +# -------------------------------------------------------------------------- +# sssd +# -------------------------------------------------------------------------- +start_sssd() { + if [ ! -f /etc/sssd/sssd.conf ]; then + warn "/etc/sssd/sssd.conf missing; skipping sssd" + return 0 + fi + + # Allow the LDAP endpoint to be retargeted without rebuilding the image. + if [ -n "${SSSD_LDAP_URI:-}" ]; then + log "overriding sssd ldap_uri with ${SSSD_LDAP_URI}" + sed -i "s|^ldap_uri = .*|ldap_uri = ${SSSD_LDAP_URI}|" /etc/sssd/sssd.conf + fi + if [ -n "${SSSD_LDAP_BASE_DN:-}" ]; then + log "overriding sssd ldap_search_base with ${SSSD_LDAP_BASE_DN}" + sed -i "s|^ldap_search_base = .*|ldap_search_base = ${SSSD_LDAP_BASE_DN}|" /etc/sssd/sssd.conf + fi + chmod 0600 /etc/sssd/sssd.conf + + # mkdir -p, not `install -d -m`: the sssd RPM already creates these + # owned by the sssd user, and chmod-ing a directory we do not own + # would need CAP_FOWNER purely to set the mode it already has. + mkdir -p /var/lib/sss/db /var/lib/sss/mc /var/lib/sss/pipes/private /var/log/sssd + + log "starting sssd" + if ! /usr/sbin/sssd -D; then + warn "sssd failed to start; NSS/LDAP name resolution unavailable (continuing)" + return 0 + fi + + local waited=0 + while ! pgrep -x sssd >/dev/null 2>&1; do + if [ "$waited" -ge "$SSSD_WAIT_SECONDS" ]; then + warn "sssd did not come up within ${SSSD_WAIT_SECONDS}s (continuing)" + return 0 + fi + sleep 1 + waited=$(( waited + 1 )) + done + log "sssd running" +} + +# -------------------------------------------------------------------------- +if [ "$START_SSSD" = "true" ]; then + start_sssd +else + log "START_SSSD=${START_SSSD}; not starting sssd" +fi + +# -------------------------------------------------------------------------- +# Slurm client +# +# munged runs in the pod's munge sidecar; wait for its socket rather than +# letting sacct fail to authenticate. +# -------------------------------------------------------------------------- +waited=0 +while ! [ -S "$MUNGE_SOCKET" ]; do + if [ "$waited" -ge "$MUNGE_WAIT_SECONDS" ]; then + warn "no munge socket at ${MUNGE_SOCKET} after ${MUNGE_WAIT_SECONDS}s; is the munged sidecar running?" + exit 1 + fi + sleep 1 + waited=$(( waited + 1 )) +done +log "munge socket present at ${MUNGE_SOCKET}" + +if [ ! -r "${SLURM_CONF:-/run/slurm/conf/slurm.conf}" ]; then + warn "no readable slurm.conf at ${SLURM_CONF:-/run/slurm/conf/slurm.conf}; is the slurm-conf ConfigMap mounted?" + exit 1 +fi +log "slurm.conf found at ${SLURM_CONF:-/run/slurm/conf/slurm.conf}" + +if ! command -v sacct >/dev/null 2>&1; then + warn "sacct not on PATH (${PATH})" + exit 1 +fi +# After the conf check: even `sacct -V` parses slurm.conf. +log "slurm client: $(sacct -V)" + +# `exec` with no arguments is a silent no-op that would exit 0 -- a green +# CronJob run that did nothing at all. +if [ "$#" -eq 0 ]; then + warn "no command given; nothing to run" + exit 1 +fi + +log "exec: $*" +exec "$@" diff --git a/etc/krb5.conf b/etc/krb5.conf new file mode 100644 index 0000000..9340935 --- /dev/null +++ b/etc/krb5.conf @@ -0,0 +1,36 @@ +# Kerberos configuration for the coact-daemon container. + +[libdefaults] + default_realm = SLAC.STANFORD.EDU + dns_lookup_realm = false + dns_lookup_kdc = false + rdns = false + forwardable = true + ticket_lifetime = 90000 + renew_lifetime = 7d + clockskew = 300 + +[realms] + SLAC.STANFORD.EDU = { + kdc = k5auth1.slac.stanford.edu:88 + kdc = k5auth2.slac.stanford.edu:88 + kdc = k5auth3.slac.stanford.edu:88 + master_kdc = k5auth1.slac.stanford.edu:88 + admin_server = k5admin.slac.stanford.edu + kpasswd_server = k5passwd.slac.stanford.edu + default_domain = slac.stanford.edu + } + SDF.SLAC.STANFORD.EDU = { + kdc = sdfkdc001.sdf.slac.stanford.edu:88 + kdc = sdfkdc002.sdf.slac.stanford.edu:88 + master_kdc = sdfkdc001.sdf.slac.stanford.edu:88 + admin_server = sdfkdc001.sdf.slac.stanford.edu:88 + kpasswd_server = sdfkdc001.sdf.slac.stanford.edu:88 + default_domain = sdf.slac.stanford.edu + } + +[domain_realm] + .slac.stanford.edu = SLAC.STANFORD.EDU + slac.stanford.edu = SLAC.STANFORD.EDU + .sdf.slac.stanford.edu = SDF.SLAC.STANFORD.EDU + sdf.slac.stanford.edu = SDF.SLAC.STANFORD.EDU diff --git a/etc/ldap.conf b/etc/ldap.conf new file mode 100644 index 0000000..f1d753a --- /dev/null +++ b/etc/ldap.conf @@ -0,0 +1,6 @@ +# OpenLDAP client configuration (/etc/openldap/ldap.conf). + +BASE dc=sdf,dc=slac,dc=stanford,dc=edu +URI ldap://sdfldapsrv.sdf.slac.stanford.edu +SASL_NOCANON on +SASL_REALM SDF.SLAC.STANFORD.EDU diff --git a/etc/nsswitch.conf b/etc/nsswitch.conf new file mode 100644 index 0000000..ba11096 --- /dev/null +++ b/etc/nsswitch.conf @@ -0,0 +1,20 @@ +# NSS configuration for the coact-daemon container. + +passwd: files sss +group: files sss +shadow: files sss +gshadow: files + +hosts: files dns myhostname +networks: files + +protocols: files +services: files +ethers: files +rpc: files + +netgroup: files sss +automount: files sss +publickey: files +bootparams: files +aliases: files diff --git a/etc/sssd.conf b/etc/sssd.conf new file mode 100644 index 0000000..fa5c6c1 --- /dev/null +++ b/etc/sssd.conf @@ -0,0 +1,32 @@ +# sssd configuration for the coact-daemon container. + +[sssd] +config_file_version = 2 +services = nss +domains = SLAC.STANFORD.EDU + +[nss] +filter_users = root,munge +filter_groups = root,munge +entry_cache_nowait_percentage = 75 +enum_cache_timeout = 300 + +[domain/SLAC.STANFORD.EDU] +id_provider = ldap +auth_provider = none +access_provider = permit +ldap_uri = ldap://sdfldap001.sdf.slac.stanford.edu +ldap_search_base = dc=sdf,dc=slac,dc=stanford,dc=edu +ldap_schema = rfc2307 +ldap_id_mapping = false +ldap_id_use_start_tls = false +ldap_tls_reqcert = never +ldap_tls_cacertdir = /etc/openldap/certs +entry_cache_timeout = 3600 +enumerate = false +cache_credentials = false + +# Short timeouts so an unreachable directory cannot stall a batch job. +ldap_network_timeout = 5 +ldap_opt_timeout = 10 +ldap_search_timeout = 10 diff --git a/import-jobs.sh b/import-jobs.sh index 69e086b..7a26292 100755 --- a/import-jobs.sh +++ b/import-jobs.sh @@ -1,42 +1,104 @@ #!/bin/bash +# +# Single-shot Slurm job accounting import. +# +# sacct -> slurmdump | slurmremap | slurmimport (then slurmrecalculate) +# +# Usage: +# ./import-jobs.sh # today (or yesterday just after midnight) +# ./import-jobs.sh 2026-08-31 # explicit date, for replay/backfill +set -euo pipefail -export PATH=$PATH:/opt/slurm/slurm-curr/bin -export SDF_COACT_URI=coact.slac.stanford.edu:443/graphql-service +: "${SDF_COACT_URI:?SDF_COACT_URI must be set, e.g. coact.slac.stanford.edu:443/graphql-service (no scheme)}" +export SDF_COACT_URI -export PYTHON_BIN=./old_venv/bin/python3 +SLURM_BIN_DIR="${SLURM_BIN_DIR:-/opt/slurm/slurm-curr/bin}" +case ":$PATH:" in + *":$SLURM_BIN_DIR:"*) ;; + *) PATH="$PATH:$SLURM_BIN_DIR" ;; +esac +export PATH -PASSWORD_FILE=./etc/.secrets/password +COACT_USERNAME="${COACT_USERNAME:-sdf-bot}" -if [ ! -z $1 ]; then - DATE=$@ +# Sourced from Vault via the coact-daemon secret; see +# deploy/kubernetes/overlays/dev/daemon/externalsecret.yaml. +: "${COACT_PASSWORD:?COACT_PASSWORD must be set (Vault secret/scs/coact-dev/service-account field 'password')}" +export COACT_PASSWORD + +JOB_HISTORY_DIR="${JOB_HISTORY_DIR:-/data/slurm-job-history}" +JOB_REMAPPED_DIR="${JOB_REMAPPED_DIR:-/data/slurm-job-remapped}" + +PYTHON="${PYTHON:-python3}" +SDF_CLICK="${SDF_CLICK:-$(dirname "$0")/sdf_click.py}" + +# -------------------------------------------------------------------------- +# Which day to import +# -------------------------------------------------------------------------- +if [ -n "${1:-}" ]; then + DATE="$*" else DATE=$(date +"%Y-%m-%d") -fi -# deal with first few minutes of new day; need to do full import of previous day before running it -MIDNIGHT=$(date -d 'today 00:00:00' "+%s") -NOW=$(date "+%s") -DIFF=$(( ($NOW - $MIDNIGHT) )) -if [[ $DIFF -lt 300 ]]; then - DATE=$(date -d 'yesterday' +"%Y-%m-%d") + # Deal with the first few minutes of a new day: the previous day needs a + # full import before today's partial import is meaningful. This is local + # time, which is why the container pins TZ (America/Los_Angeles). + MIDNIGHT=$(date -d 'today 00:00:00' "+%s") + NOW=$(date "+%s") + DIFF=$(( NOW - MIDNIGHT )) + if [ "$DIFF" -lt 300 ]; then + DATE=$(date -d 'yesterday' +"%Y-%m-%d") + fi fi -echo ">" $DATE" ("$(date)")" +mkdir -p "$JOB_HISTORY_DIR" "$JOB_REMAPPED_DIR" + +RAW_ARCHIVE="$JOB_HISTORY_DIR/$DATE" +REMAPPED_ARCHIVE="$JOB_REMAPPED_DIR/$DATE" +RAW_PARTIAL="$RAW_ARCHIVE.partial" +REMAPPED_PARTIAL="$REMAPPED_ARCHIVE.partial" -# full -$PYTHON_BIN ./sdf_click.py coact slurmdump --date $DATE \ - | tee ../slurm-job-history/$DATE \ - | $PYTHON_BIN ./sdf_click.py coact slurmremap \ - | tee ../slurm-job-remapped/$DATE \ - | $PYTHON_BIN ./sdf_click.py coact slurmimport --password-file $PASSWORD_FILE --output=upload >/dev/null +# Any exit before the promotion step below leaves the previous good archive +# for this date untouched. +trap 'rm -f "$RAW_PARTIAL" "$REMAPPED_PARTIAL"' EXIT -# just for 2023 imports -#cat ../slurm-job-remapped/$DATE | ./sdf.py coact slurmimport --password-file $PASSWORD_FILE --output=upload >/dev/null +echo "> $DATE ($(date))" + +# -------------------------------------------------------------------------- +# Full pipeline. +# +# The dumps are teed to `.partial` files and only moved into place once the +# whole pipeline has succeeded. +# +# `--output-error=warn-nopipe` keeps tee writing after a +# downstream stage closes the pipe, instead of dying on SIGPIPE with its +# buffered writes unflushed. +# -------------------------------------------------------------------------- +"$PYTHON" "$SDF_CLICK" coact slurmdump --date "$DATE" \ + | tee --output-error=warn-nopipe "$RAW_PARTIAL" \ + | "$PYTHON" "$SDF_CLICK" coact slurmremap \ + | tee --output-error=warn-nopipe "$REMAPPED_PARTIAL" \ + | "$PYTHON" "$SDF_CLICK" coact slurmimport \ + --username "$COACT_USERNAME" \ + --output=upload >/dev/null + +# A successful sacct always emits at least the header row, so an empty dump +# here means something went wrong that the exit codes did not surface. +if [ ! -s "$RAW_PARTIAL" ]; then + echo "error: raw sacct dump for $DATE is empty; keeping the existing archive" >&2 + exit 1 +fi +if [ ! -s "$REMAPPED_PARTIAL" ]; then + echo "error: remapped dump for $DATE is empty; keeping the existing archive" >&2 + exit 1 +fi -# don't pull data from slurm -#cat ../slurm-job-history/$DATE | ./sdf.py coact slurmremap | tee ../slurm-job-remapped/$DATE | ./sdf.py coact slurmimport --password-file $PASSWORD_FILE --output=upload >/dev/null +mv -f "$RAW_PARTIAL" "$RAW_ARCHIVE" +mv -f "$REMAPPED_PARTIAL" "$REMAPPED_ARCHIVE" -### -# recalculate summaries -### -$PYTHON_BIN ./sdf_click.py coact slurmrecalculate --password-file=$PASSWORD_FILE --date=$DATE +# -------------------------------------------------------------------------- +# Recalculate usage summaries +# -------------------------------------------------------------------------- +"$PYTHON" "$SDF_CLICK" coact slurmrecalculate \ + --username "$COACT_USERNAME" \ + --date "$DATE" diff --git a/modules/base.py b/modules/base.py index 8f0d9e9..212d89f 100644 --- a/modules/base.py +++ b/modules/base.py @@ -28,22 +28,23 @@ class MyHandler(GraphQlMixin): def run(self): self.back_channel = self.connect_graph_ql( username='user', - password_file='/path/to/password' + password='hunter2' ) # Use self.back_channel for GraphQL queries """ - + back_channel = None - - def connect_graph_ql(self, username: str, password_file: str, timeout: int = 60): + + def connect_graph_ql(self, username: str, password: str | None = None, password_file: str | None = None, timeout: int = 60): """ Connect to the GraphQL service. - + Args: username: The username for basic auth - password_file: Path to file containing the password + password: The password for basic auth + password_file: Path to a file containing the password; backwards compatabilty timeout: Connection timeout in seconds - + Returns: A connected GraphQL client """ @@ -51,6 +52,7 @@ def connect_graph_ql(self, username: str, password_file: str, timeout: int = 60) client = GraphQlClient() return client.connect_graph_ql( username=username, + password=password, password_file=password_file, timeout=timeout ) @@ -122,8 +124,8 @@ def configure_logging_from_verbose(verbose: int) -> None: def graphql_options(f): """ Decorator for GraphQL authentication options. - - Adds --username and --password-file options to commands. + + Adds --username and --password options to commands. """ f = click.option( '--username', @@ -131,9 +133,10 @@ def graphql_options(f): help='Basic auth username for graphql service' )(f) f = click.option( - '--password-file', + '--password', + envvar='COACT_PASSWORD', required=True, - type=click.Path(exists=True), - help='Basic auth password for graphql service' + show_envvar=True, + help='Basic auth password for graphql service. Prefer the COACT_PASSWORD environment variable' )(f) return f \ No newline at end of file diff --git a/modules/coact.py b/modules/coact.py index a151bc5..45459a1 100644 --- a/modules/coact.py +++ b/modules/coact.py @@ -158,6 +158,12 @@ def run_sacct( f"skipping ({len(fields)}, {int(fields[7])} < {int(fields[8])}) {line}" ) + process.wait() + if process.returncode != 0: + raise RuntimeError( + f"sacct exited {process.returncode}; refusing to report an empty job set" + ) + # ============================================================================ # SlurmRemap Command @@ -503,7 +509,7 @@ def remap_job_pre2024(self, d): help='Terminate if cannot parse data' ) @click.pass_context -def slurm_import(ctx, print_output, debug, username, password_file, batch, data, output, exit_on_error): +def slurm_import(ctx, print_output, debug, username, password, batch, data, output, exit_on_error): """Reads sacctmgr info from slurm and translates it to coact accounting stats.""" if debug: configure_logging_from_verbose(2) @@ -513,7 +519,7 @@ def slurm_import(ctx, print_output, debug, username, password_file, batch, data, importer = SlurmImporter( username=username, - password_file=password_file, + password=password, verbose=print_output, exit_on_error=exit_on_error ) @@ -535,9 +541,9 @@ class SlurmImporter(GraphQlMixin): "sdfmilan272": 1920, } - def __init__(self, username: str, password_file: str, verbose: bool = False, exit_on_error: bool = False): + def __init__(self, username: str, password: str, verbose: bool = False, exit_on_error: bool = False): self.username = username - self.password_file = password_file + self.password = password self.verbose = verbose self.exit_on_error = exit_on_error self._allocid = {} @@ -547,7 +553,7 @@ def run(self, data, output_format: str, batch_size: int) -> None: """Run the import process.""" self.back_channel = self.connect_graph_ql( username=self.username, - password_file=self.password_file, + password=self.password, timeout=300 ) self.get_metadata() @@ -850,7 +856,7 @@ def calc_resource_hours(startTs, endTs, tres: str, cluster: dict, alloc_nodes: O @common_options @graphql_options @click.pass_context -def slurm_recalculate(ctx, date, verbose, username, password_file): +def slurm_recalculate(ctx, date, verbose, username, password): """Recalculate the usage numbers from slurm jobs in Coact.""" configure_logging_from_verbose(verbose) ctx.obj['verbose'] = verbose @@ -858,7 +864,7 @@ def slurm_recalculate(ctx, date, verbose, username, password_file): client = GraphQlClient() back_channel = client.connect_graph_ql( username=username, - password_file=password_file, + password=password, timeout=300 ) @@ -879,7 +885,10 @@ def slurm_recalculate(ctx, date, verbose, username, password_file): @coact.command(name='overage') @click.option('--date', default=lambda: pdl.now().format('YYYY-MM-DD'), help='Recalculate jobs from this date (default: today)') @common_options -@graphql_options +# Deliberately not @graphql_options: the overage migration is deferred, so this +# command and its bare-metal wrapper stay on the password file. Only the +@click.option('--username', default='sdf-bot', help='Basic auth username for graphql service') +@click.option('--password-file', required=True, type=click.Path(exists=True), help='Basic auth password for graphql service') @click.option('--windows', type=int, multiple=True, default=[15, 60, 10080, 43800], help='Time windows to collate overage calculations') @click.option('--threshold', type=float, default=100.0, help='Percentage at which to be considered over allocation') @click.option('--dry-run', is_flag=True, default=False, help='Do not actually enforce job holding') @@ -1017,8 +1026,7 @@ def get(self, date: str) -> Iterator[OveragePoint]: ) logger.debug(f"Fetching usage data for date: {date}") data = self.get_data() - for point in self.overaged(data, threshold=self.threshold): - yield point + yield from self.overaged(data, threshold=self.threshold) def get_data(self) -> dict: """Fetch usage data from GraphQL.""" diff --git a/modules/utils/graphql.py b/modules/utils/graphql.py index 6f329be..e323e86 100644 --- a/modules/utils/graphql.py +++ b/modules/utils/graphql.py @@ -41,6 +41,12 @@ def get_password(self, password_file=None): password = f.read() return password + @staticmethod + def _normalize_password(password): + """Strip surrounding whitespace so a file with a trailing newline and an + environment variable holding the same secret produce the same header.""" + return password.strip() if password else password + def get_basic_auth_headers(self, username=None, password=None): headers = {} if username and password: @@ -50,10 +56,12 @@ def get_basic_auth_headers(self, username=None, password=None): def connect_graph_ql(self, graphql_uri='https://'+SDF_COACT_URI, get_schema=False, username=None, password_file=None, password=None, timeout=30): logger.trace(f"GraphQL connect: uri={graphql_uri}, username={username}, timeout={timeout}s") - logger.trace(f"GraphQL connect: password_file={password_file}, get_schema={get_schema}") + logger.trace(f"GraphQL connect: get_schema={get_schema}") if password_file: password = self.get_password(password_file=password_file) - logger.trace(f"GraphQL connect: loaded password from file (length={len(password.strip()) if password else 0})") + logger.trace(f"GraphQL connect: loaded password from {password_file}") + password = self._normalize_password(password) + logger.trace(f"GraphQL connect: password length={len(password) if password else 0}") logger.trace(f"GraphQL connect: creating AIOHTTPTransport to {graphql_uri}") self.transport = AIOHTTPTransport(url=graphql_uri, headers=self.get_basic_auth_headers(username=username, password=password)) logger.trace(f"GraphQL connect: creating Client with execute_timeout={timeout}") @@ -104,6 +112,7 @@ def connect_subscriber(self, graphql_uri='wss://'+SDF_COACT_URI, get_schema=Fals if password_file: password = self.get_password(password_file=password_file) logger.trace("GraphQL subscriber connect: loaded password from file") + password = self._normalize_password(password) logger.trace(f"GraphQL subscriber connect: creating WebsocketsTransport to {graphql_uri}") self.subscription_transport = WebsocketsTransport( url=graphql_uri, diff --git a/munge-sidecar.sh b/munge-sidecar.sh new file mode 100755 index 0000000..6f68fda --- /dev/null +++ b/munge-sidecar.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Pod-local munged for the coact-daemon image. +# +# The pod authenticates to slurmdbd with its own munged instead of the node's +# socket, so it needs no hostPath. Two modes, run as two containers: +# +# munge-sidecar.sh init root init container. Kubernetes volumes do not +# satisfy munged's ownership/permission checks (the +# Secret is a root-owned symlink, emptyDirs are 0777 +# without the sticky bit), so this copies the key +# into a munge-owned 0700 directory and tightens the +# shared socket directory. Needs only CAP_CHOWN. +# munge-sidecar.sh run munge user. Runs munged in the foreground as a +# native sidecar; the job container talks to it +# through the shared socket directory. + +set -euo pipefail + +MUNGE_KEY_SOURCE="${MUNGE_KEY_SOURCE:-/run/munge-key/munge.key}" +MUNGE_KEY_DIR="${MUNGE_KEY_DIR:-/etc/munge}" +MUNGE_SOCKET_DIR="${MUNGE_SOCKET_DIR:-/run/munge}" + +log() { printf '[munge-sidecar] %s\n' "$*" >&2; } + +init() { + if [ ! -s "$MUNGE_KEY_SOURCE" ]; then + log "no munge key at ${MUNGE_KEY_SOURCE}; is the coact-daemon-munge secret synced?" + exit 1 + fi + + # Take ownership, chmod, then hand to munge: changing the mode of a file + # root does not own would additionally need CAP_FOWNER. (A fresh emptyDir + # is already root's; a pre-populated volume may not be.) + chown root:root "$MUNGE_KEY_DIR" "$MUNGE_SOCKET_DIR" + chmod 0700 "$MUNGE_KEY_DIR" + chmod 0755 "$MUNGE_SOCKET_DIR" + install -m 0400 "$MUNGE_KEY_SOURCE" "$MUNGE_KEY_DIR/munge.key" + chown munge:munge "$MUNGE_KEY_DIR/munge.key" "$MUNGE_KEY_DIR" "$MUNGE_SOCKET_DIR" + + log "munge key installed in ${MUNGE_KEY_DIR}; socket directory ${MUNGE_SOCKET_DIR} prepared" +} + +run() { + log "starting munged" + exec /usr/sbin/munged --foreground \ + --key-file="$MUNGE_KEY_DIR/munge.key" \ + --socket="$MUNGE_SOCKET_DIR/munge.socket.2" +} + +case "${1:-}" in + init) init ;; + run) run ;; + *) log "usage: $0 init|run"; exit 2 ;; +esac diff --git a/tests/test_slurm_account_parsing.py b/tests/test_slurm_account_parsing.py index dedcd4f..93c51a3 100644 --- a/tests/test_slurm_account_parsing.py +++ b/tests/test_slurm_account_parsing.py @@ -54,7 +54,7 @@ def test_slurm_remapper(account_in, qos_in, account_out, qos_out): def test_slurm_importer_convert_dual_hierarchy(monkeypatch): from modules.coact import parse_datetime - importer = SlurmImporter(username="test", password_file="dummy") + importer = SlurmImporter(username="test", password="dummy") importer._clusters = {"milano": {"cpu": 64, "gpu": 0, "mem": 256 * 1073741824}} # Mock allocid lookup diff --git a/tests/test_slurm_node_memory.py b/tests/test_slurm_node_memory.py index 73a4239..dcbd571 100644 --- a/tests/test_slurm_node_memory.py +++ b/tests/test_slurm_node_memory.py @@ -15,7 +15,7 @@ def setup_method(self): """Create a SlurmImporter instance for testing.""" self.importer = SlurmImporter( username="test", - password_file="test", + password="test", verbose=False, exit_on_error=False )