Import Jobs Containerization

.dockerignore

@@ -0,0 +1,18 @@
+# Never bake credentials into the image.
+etc/.secrets
+etc/.secrets/**
+
+# Local development artefacts.
+.venv
+venv
+.git
+.github
+.claude
+.pytest_cache
+.ruff_cache
+__pycache__
+**/__pycache__
+*.egg-info
+sdf_cli.egg-info
+.vscode
+.DS_Store

.github/workflows/ci-cd.yaml

@@ -4,7 +4,6 @@ on:
   push:
     branches: ["main"]
   pull_request:
-    branches: ["main"]
 
 env:
   UV_SYSTEM_PYTHON: 1

.github/workflows/docker_publish.yaml

@@ -0,0 +1,145 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  workflow_run:
+    workflows: [ "CI/CD" ]
+    types:
+      - completed
+  workflow_dispatch:
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  IMAGE_NAME: slaclab/sdf-cli
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    if: >-
+      ${{ github.event_name != 'workflow_run' ||
+          (github.event.workflow_run.conclusion == 'success' &&
+           github.event.workflow_run.head_repository.full_name == github.repository) }}
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio for signing and attestation.
+      id-token: write
+      # Allow to persist attestations
+      attestations: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
+
+      # Install the cosign tool
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
+        with:
+          cosign-release: 'v2.2.4'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      # Login against a Docker registry
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Resolve build branch and Docker tag
+        id: tag
+        env:
+          HEAD_BRANCH: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.ref_name }}
+        run: |
+          set -euo pipefail
+
+          # Docker tags allow [a-zA-Z0-9._-] only, may not lead with '.' or '-',
+          # and are capped at 128 chars. Branch names like `feature/foo` are not
+          # usable verbatim.
+          SLUG=$(printf '%s' "${HEAD_BRANCH}" \
+            | sed -e 's#[^a-zA-Z0-9._-]#-#g' -e 's#^[.-]#_#' \
+            | cut -c1-100)
+          STAMP=$(date -u +"%Y%m%d-%H%M")
+
+          if [ "${HEAD_BRANCH}" = "main" ]; then
+            TAG="${STAMP}"
+          else
+            TAG="${SLUG}-${STAMP}"
+          fi
+
+          {
+            echo "branch=${HEAD_BRANCH}"
+            echo "slug=${SLUG}"
+            echo "tag=${TAG}"
+            echo "full_tag=${REGISTRY}/${IMAGE_NAME}:${TAG}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "Building ${HEAD_BRANCH} as ${REGISTRY}/${IMAGE_NAME}:${TAG}" >> "$GITHUB_STEP_SUMMARY"
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ steps.tag.outputs.tag }}
+            type=raw,value=${{ steps.tag.outputs.slug }}
+        env:
+          GITHUB_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
+          GITHUB_REF: ${{ github.event_name == 'workflow_run' && format('refs/heads/{0}', github.event.workflow_run.head_branch) || github.ref }}
+
+      # Build and push Docker image with Buildx
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting published Docker image digest.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
+
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)."
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.build-and-push.outputs.digest }}
+          push-to-registry: true

Dockerfile

@@ -0,0 +1,149 @@
+# Container image for the Coact batch daemons
+
+# ---------------------------------------------------------------------------
+# Slurm client build
+#
+# The Slurm client ships in the image rather than being mounted from the
+# node's /opt/slurm, so the pod needs no hostPath.  SLURM_VERSION must track
+# the S3DF slurmctld/slurmdbd: a client newer than the servers is unsupported.
+# ---------------------------------------------------------------------------
+FROM rockylinux:9 AS slurm-build
+
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+ARG SLURM_VERSION=25.11.8
+ARG SLURM_SHA256=34ace13f81011add6094569d13bfc4006ad8868201c2236e2905443c7e526393
+
+# munge-devel and readline-devel live in CRB.
+RUN dnf -y install epel-release dnf-plugins-core \
+ && dnf config-manager --set-enabled crb \
+ && dnf -y --setopt=install_weak_deps=False install \
+      gcc make bzip2 perl python3 \
+      munge-devel readline-devel \
+ && dnf clean all
+
+WORKDIR /build
+RUN curl -fsSLo slurm.tar.bz2 "https://download.schedmd.com/slurm/slurm-${SLURM_VERSION}.tar.bz2" \
+ && echo "${SLURM_SHA256}  slurm.tar.bz2" | sha256sum -c - \
+ && tar xjf slurm.tar.bz2 --strip-components=1 \
+ && rm slurm.tar.bz2
+
+# Same prefix as the S3DF RPMs (/opt/slurm/slurm-<ver>, with slurm-curr
+# symlinked to it), so PATH and SLURM_BIN_DIR are unchanged from bare metal.
+RUN ./configure \
+      --prefix=/opt/slurm/slurm-${SLURM_VERSION} \
+      --libdir=/opt/slurm/slurm-${SLURM_VERSION}/lib64 \
+      --sysconfdir=/etc/slurm \
+      --disable-slurmrestd \
+ && make -j"$(nproc)" \
+ && make install \
+ && rm -rf /opt/slurm/slurm-${SLURM_VERSION}/share /opt/slurm/slurm-${SLURM_VERSION}/include \
+ && test -e /opt/slurm/slurm-${SLURM_VERSION}/lib64/slurm/auth_munge.so \
+ && test -e /opt/slurm/slurm-${SLURM_VERSION}/lib64/slurm/accounting_storage_slurmdbd.so
+
+# ---------------------------------------------------------------------------
+# Runtime image
+# ---------------------------------------------------------------------------
+FROM rockylinux:9
+
+ARG SLURM_VERSION=25.11.8
+
+LABEL org.opencontainers.image.source=https://github.com/slaclab/sdf-cli
+LABEL org.opencontainers.image.description="Coact batch daemons (slurm job import, facility overage)"
+LABEL edu.stanford.slac.slurm.version="${SLURM_VERSION}"
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# Build behind the SDF proxy with:
+#   podman build --build-arg https_proxy=http://sdfproxy.sdf.slac.stanford.edu:3128 .
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+# ---------------------------------------------------------------------------
+# OS packages
+# ---------------------------------------------------------------------------
+RUN dnf -y install epel-release \
+ && dnf -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs install \
+      munge \
+      readline \
+      sssd \
+      sssd-client \
+      nss-pam-ldapd \
+      openldap-clients \
+      krb5-workstation \
+      python3.12 \
+      python3.12-pip \
+      tini \
+      tzdata \
+      procps-ng \
+      which \
+      glibc-langpack-en \
+      shadow-utils \
+      ca-certificates \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf /var/cache/yum
+
+# Align the munge uid/gid with the SDF hosts
+ARG MUNGE_UID=16952
+ARG MUNGE_GID=3761
+# usermod only re-owns the home directory, so the rest of munge's tree is
+# re-owned explicitly -- munged refuses directories it does not own.
+RUN groupmod -g $MUNGE_GID munge \
+ && usermod  -u $MUNGE_UID -g $MUNGE_GID munge \
+ && chown -R munge:munge /etc/munge /var/lib/munge /var/log/munge /run/munge
+
+COPY --from=slurm-build /opt/slurm /opt/slurm
+RUN ln -s slurm-${SLURM_VERSION} /opt/slurm/slurm-curr \
+ && echo /opt/slurm/slurm-curr/lib64 > /etc/ld.so.conf.d/slurm.conf \
+ && ldconfig
+
+# ---------------------------------------------------------------------------
+# Runtime environment
+# ---------------------------------------------------------------------------
+ENV UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_PYTHON=/usr/bin/python3.12 \
+    UV_LINK_MODE=copy \
+    SLURM_BIN_DIR=/opt/slurm/slurm-curr/bin \
+    SLURM_CONF=/run/slurm/conf/slurm.conf \
+    PATH=/opt/venv/bin:/opt/slurm/slurm-curr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    TZ=America/Los_Angeles \
+    LANG=en_US.UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR /app
+
+# Dependency layer first so code changes do not invalidate the resolve.
+COPY pyproject.toml uv.lock /app/
+RUN uv sync --frozen --no-install-project --no-dev
+
+COPY . /app
+RUN uv sync --frozen --no-dev
+
+# ansible-runner 2.3.1 imports `pkg_resources` at module scope, and
+# sdf_click.py imports modules/coactd.py (hence ansible_runner) unconditionally
+# -- even for the `coact` batch subcommands.  setuptools >= 81 dropped
+# pkg_resources, so the CLI will not start without an older setuptools.
+RUN uv pip install --python /opt/venv/bin/python "setuptools<81"
+
+# ---------------------------------------------------------------------------
+# Identity / auth configuration
+# ---------------------------------------------------------------------------
+COPY etc/nsswitch.conf /etc/nsswitch.conf
+COPY etc/krb5.conf     /etc/krb5.conf
+COPY etc/ldap.conf     /etc/openldap/ldap.conf
+COPY etc/sssd.conf     /etc/sssd/sssd.conf
+RUN chmod 0600 /etc/sssd/sssd.conf \
+ && install -d -m 0755 /data
+
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY munge-sidecar.sh     /usr/local/bin/munge-sidecar.sh
+RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh /usr/local/bin/munge-sidecar.sh /app/import-jobs.sh
+
+# tini reaps sssd and forwards signals; the entrypoint execs the CronJob
+# command so the container exits with the job's own exit code.
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
+CMD ["python3", "/app/sdf_click.py", "--help"]

Makefile

@@ -1,41 +1,32 @@
-VENV_DIR ?= venv
-VENV_BIN ?= $(VENV_DIR)/bin/python3.9
-PIP_BIN ?= $(VENV_DIR)/bin/pip
-PYTHON_BIN ?= python3.9
-VAULT_SECRET_PATH ?= secret/tid/coact
-
-secrets:
-	mkdir etc/.secrets/ -p
-	#set -e; for i in ldap_binddn ldap_bindpw; do vault kv get --field=$$i $(VAULT_SECRET_PATH) > etc/.secrets/$$i ; done
+## sdf-cli / coact-daemon
+##
+## Container image build + Vault secret fetching for the batch daemons.
+
+# --- container image -------------------------------------------------------
+CONTAINER_RT ?= podman
+REPO         ?= ghcr.io/slaclab
+IMAGE        ?= sdf-cli
+TAG          ?= $(shell date +"%Y%m%d-%H%M")
+
+# --- vault -----------------------------------------------------------------
+VAULT_SECRET_PATH     ?= secret/scs/coact
+GROUPER_SECRET_PATH   ?= secret/tid/scs/osmaint
+GROUPER_SECRET_FIELD  ?= password
+
+# ---------------------------------------------------------------------------
+# Container image
+# ---------------------------------------------------------------------------
+build: ## Build the coact-daemon image
+	$(CONTAINER_RT) build . -f Dockerfile -t $(REPO)/$(IMAGE):$(TAG)
+
+# ---------------------------------------------------------------------------
+# Secrets
+# ---------------------------------------------------------------------------
+secrets: ## Fetch daemon secrets from Vault into etc/.secrets/
+	mkdir -p etc/.secrets/
 	set -e; for i in password; do vault kv get --field=$$i $(VAULT_SECRET_PATH)/service-account > etc/.secrets/$$i ; done
+	vault kv get --field=$(GROUPER_SECRET_FIELD) $(GROUPER_SECRET_PATH) > etc/.secrets/grouper_password
 	chmod -R go-rwx etc/.secrets
 
-clean-secrets:
+clean-secrets: ## Remove etc/.secrets/
 	rm -rf etc/.secrets
-
-virtualenv:
-	$(PYTHON_BIN) -m venv $(VENV_DIR)
-
-venv: virtualenv
-
-pip:
-	$(VENV_BIN) -m pip install --upgrade pip
-	$(PIP_BIN) install -r requirements.txt
-
-# OS level dependencies
-deps:
-	dnf groupinstall -y "Development Tools"
-	dnf install -y python36-devel openldap-devel
-
-# run this to configure the dev environment
-environment: venv pip
-
-dev: environment
-
-update-sdf-ansible:
-	git submodule update --init --recursive
-
-apply: environment get-secrets update-sdf-ansible
-
-test:
-	$(VENV_BIN) sdf_click.py