diff --git a/cloud-info/Dockerfile b/cloud-info/Dockerfile index c2d9569c..74dc21fc 100644 --- a/cloud-info/Dockerfile +++ b/cloud-info/Dockerfile @@ -1,33 +1,57 @@ +FROM python:3 as build + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008 +RUN curl -s https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 \ + | apt-key add - \ + && echo "deb https://repository.egi.eu/sw/production/cas/1/current egi-igtf core" > /etc/apt/sources.list.d/igtf.list \ + && apt-get update \ + && apt-get install -y ca-policy-egi-core \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /cloud-info + +RUN python -m venv /cloud-info/venv +ENV PATH="/cloud-info/venv/bin:$PATH" + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt \ + && cat /etc/grid-security/certificates/*.pem >> "$(python -m requests.certs)" + +COPY . . + +RUN pip install --no-cache-dir . + +# The actual image FROM python:3 LABEL org.opencontainers.image.source=https://github.com/EGI-Federation/fedcloud-catchall-operations SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN mkdir /cloud-info -COPY requirements.txt /cloud-info/requirements.txt -RUN pip install --no-cache-dir -r /cloud-info/requirements.txt - -# CA certificates: install and add to python # hadolint ignore=DL3015, DL3008 -RUN curl -Ls \ - https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 \ - | apt-key add - \ - && echo 'deb http://repository.egi.eu/sw/production/cas/1/current egi-igtf core' \ - > /etc/apt/sources.list.d/cas.list \ - && apt-get update \ - && apt-get install -y jq \ - && apt-get install -y ca-policy-egi-core \ - && rm -rf /var/lib/apt/lists/* \ - && cat /etc/grid-security/certificates/*.pem >> "$(python -m requests.certs)" +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + jq rclone \ + && rm -rf /var/lib/apt/lists/* -COPY . /cloud-info/ -RUN pip install --no-cache-dir /cloud-info +RUN mkdir /cloud-info \ + && groupadd -g 1999 python \ + && useradd -r -u 1999 -g python python \ + && chown -R python:python /cloud-info +WORKDIR /cloud-info + +COPY --chown=python:python --from=build /cloud-info/venv ./venv COPY ams-wrapper.sh /usr/local/bin/ams-wrapper.sh COPY publisher.sh /usr/local/bin/publisher.sh COPY openstack.rc /etc/cloud-info-provider/openstack.rc COPY openstack.yaml /etc/cloud-info-provider/openstack.yaml +USER 1999 + +ENV PATH="/cloud-info/venv/bin:$PATH" CMD ["publisher.sh"] diff --git a/cloud-info/ams-wrapper.sh b/cloud-info/ams-wrapper.sh index 7f1c0b77..202c9fc9 100755 --- a/cloud-info/ams-wrapper.sh +++ b/cloud-info/ams-wrapper.sh @@ -41,7 +41,6 @@ AUTO_CONFIG_PATH="$(mktemp -d)" export CHECKIN_SECRETS_FILE="$CHECKIN_SECRETS_PATH/secrets.yaml" # TODO(enolfc): avoid creating new tokens for every provider export ACCESS_TOKEN_FILE="$AUTO_CONFIG_PATH/token.yaml" -USE_ACCESS_TOKEN=0 if token-generator; then # TODO(enolfc): even if this belows fails, we should use access token as it will provide # access to more projects @@ -49,10 +48,10 @@ if token-generator; then # this worked, let's update the env export CHECKIN_SECRETS_PATH="$AUTO_CONFIG_PATH/vos" export CLOUD_INFO_CONFIG="$AUTO_CONFIG_PATH/site.yaml" - USE_ACCESS_TOKEN=1 fi fi + # Any OS related parameter should be available as env variables if test "$CHECKIN_SECRETS_PATH" = ""; then # Case 1: manual config @@ -60,28 +59,26 @@ if test "$CHECKIN_SECRETS_PATH" = ""; then --middleware "$CLOUD_INFO_MIDDLEWARE" \ --ignore-share-errors \ --format glue21 >cloud-info.out -elif test "$USE_ACCESS_TOKEN" -eq 1; then - # Case 2: access token style - cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ - --middleware "$CLOUD_INFO_MIDDLEWARE" \ - --ignore-share-errors \ - --auth-refresher accesstoken \ - --format glue21 >cloud-info.out else - # Let's use the service account directly on the info provider - CHECKIN_DISCOVERY="https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration" - CLIENT_ID="$(yq -r '.fedcloudops.client_id' <"$CHECKIN_SECRETS_FILE")" - CLIENT_SECRET="$(yq -r '.fedcloudops.client_secret' <"$CHECKIN_SECRETS_FILE")" + # use service account for everyone + export OS_DISCOVERY_ENDPOINT="https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration" + export OS_CLIENT_ID="$(yq -r '.fedcloudops.client_id' <"$CHECKIN_SECRETS_FILE")" + export OS_CLIENT_SECRET="$(yq -r '.fedcloudops.client_secret' <"$CHECKIN_SECRETS_FILE")" + export OS_ACCESS_TOKEN_TYPE="access_token" + export OS_AUTH_TYPE="v3oidcclientcredentials" + export OS_OPENID_SCOPE="openid profile eduperson_entitlement email" cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ --middleware "$CLOUD_INFO_MIDDLEWARE" \ --ignore-share-errors \ - --os-auth-type v3oidcclientcredentials \ - --os-discovery-endpoint "$CHECKIN_DISCOVERY" \ - --os-client-id "$CLIENT_ID" \ - --os-client-secret "$CLIENT_SECRET" \ - --os-access-token-type access_token \ - --os-openid-scope "openid profile eduperson_entitlement email" \ --format glue21 >cloud-info.out + # Produce the json output also + RCLONE_CONFIG_S3="$(yq -r '.s3' <"$CHECKIN_SECRETS_FILE")" + if test "$RCLONE_CONFIG_S3" != "null"; then + cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \ + --middleware "$CLOUD_INFO_MIDDLEWARE" \ + --ignore-share-errors \ + --format glue21json >site.json + fi fi # Fail if there are no shares @@ -100,4 +97,17 @@ printf '"}]}' >>ams-payload curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload +if [ -f site.json ]; then + # Put this info into S3, configure rclone config with + # a provider named "s3" using env variables + export RCLONE_CONFIG_S3_TYPE=s3 + export RCLONE_CONFIG_S3_ACCESS_KEY_ID="$(yq -r '.s3.access_key_id' <"$CHECKIN_SECRETS_FILE")" + export RCLONE_CONFIG_S3_SECRET_ACCESS_KEY="$(yq -r '.s3.secret_access_key' <"$CHECKIN_SECRETS_FILE")" + export RCLONE_CONFIG_S3_ENDPOINT="$(yq -r '.s3.endpoint' <"$CHECKIN_SECRETS_FILE")" + export S3_BUCKET_NAME="$(yq -r '.s3.bucket' <"$CHECKIN_SECRETS_FILE")" + export RCLONE_CONFIG_S3_ACL=private + export RCLONE_CONFIG_S3_NO_CHECK_BUCKET=true + rclone copy site.json "s3:$S3_BUCKET_NAME/$SITE_NAME" +fi + rm -rf "$VO_CONFIG_PATH" diff --git a/cloud-info/cloud_info_catchall/config_generator.py b/cloud-info/cloud_info_catchall/config_generator.py index f9c3d071..a7aa79b0 100755 --- a/cloud-info/cloud_info_catchall/config_generator.py +++ b/cloud-info/cloud_info_catchall/config_generator.py @@ -49,6 +49,8 @@ def generate_shares(config, secrets): discoverer = RefresherShareDiscovery(config, secrets[s]) elif "access_token" in secrets[s]: discoverer = AccessTokenShareDiscovery(config, secrets[s]) + else: + continue token_shares = discoverer.get_token_shares() shares.update(token_shares) if not shares: diff --git a/cloud-info/cloud_info_catchall/share_discovery.py b/cloud-info/cloud_info_catchall/share_discovery.py index e9eb8b8f..57de67db 100644 --- a/cloud-info/cloud_info_catchall/share_discovery.py +++ b/cloud-info/cloud_info_catchall/share_discovery.py @@ -100,8 +100,3 @@ class AccessTokenShareDiscovery(ShareDiscovery): def get_token(self): return self.secret["access_token"] - - def build_share(self, project, access_token): - s = super().build_share(project, access_token) - s["auth"].update({"access_token": access_token}) - return s diff --git a/cloud-info/cloud_info_catchall/test_share_discovery.py b/cloud-info/cloud_info_catchall/test_share_discovery.py index 990fe881..ff5b5f70 100644 --- a/cloud-info/cloud_info_catchall/test_share_discovery.py +++ b/cloud-info/cloud_info_catchall/test_share_discovery.py @@ -162,7 +162,7 @@ def test_build_share(self): project = {"id": "foobar"} self.assertEqual( self.discoverer.build_share(project, "token"), - {"auth": {"project_id": "foobar", "access_token": "token"}}, + {"auth": {"project_id": "foobar"}}, ) diff --git a/cloud-info/cloud_info_catchall/token_generator.py b/cloud-info/cloud_info_catchall/token_generator.py index aa56a0ca..c79ec964 100755 --- a/cloud-info/cloud_info_catchall/token_generator.py +++ b/cloud-info/cloud_info_catchall/token_generator.py @@ -71,6 +71,9 @@ def generate_tokens(oidc_config, scopes, tokens, token_ttl, secrets): # not our thing if not isinstance(secrets[s], dict): continue + if "client_id" not in secrets[s] or "client_secret" not in secrets[s]: + # not suitable for us + continue if "refresh_token" in secrets[s]: # ignore those that have refresh token continue diff --git a/cloud-info/requirements.txt b/cloud-info/requirements.txt index b0c673c5..bfd6f90f 100644 --- a/cloud-info/requirements.txt +++ b/cloud-info/requirements.txt @@ -1,5 +1,6 @@ -# Cloud info version is 9d4c4c516b9311c77564444cb9ecbb059b7f2192 -git+https://github.com/EGI-Federation/cloud-info-provider.git@9d4c4c516b9311c77564444cb9ecbb059b7f2192 +# Cloud info version is 43cefc204b3e07211c6c37df2ee20eab845c3428 +# 43cefc204b3e07211c6c37df2ee20eab845c3428 includes json glue support +git+https://github.com/EGI-Federation/cloud-info-provider.git@43cefc204b3e07211c6c37df2ee20eab845c3428 git+https://github.com/ARGOeu/argo-ams-library@devel python-glanceclient python-novaclient diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 2e7ab19b..bd030f50 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -17,7 +17,6 @@ echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_ if ansible-playbook -i inventory.yaml \ --extra-vars @secrets.yaml \ --extra-vars @extra-vars.yaml \ - --extra-vars @vos.yaml \ playbook.yaml >ansible.log 2>&1; then status_summary="success" color="#6DBF59" @@ -32,6 +31,10 @@ fi # copy the secrets to the /etc/egi/vos dir which is readable from the containers cp secrets.yaml /etc/egi/vos/secrets.yaml +# make sure the container user (999) can access the files +chown -R 999:999 /etc/egi/ + + GITHUB_COMMIT_URL="https://api.github.com/repos/EGI-Federation/fedcloud-catchall-operations/commits/$COMMIT_SHA/pulls" # Find out PR we need to update diff --git a/deploy/vos.yaml b/deploy/vos.yaml deleted file mode 100644 index 33b60ee8..00000000 --- a/deploy/vos.yaml +++ /dev/null @@ -1,212 +0,0 @@ ---- -vos: - # Co-manage service account - EOServices-vo.indra.es: - auth: "{{ fedcloud_sa }}" - biinsight.eosc-hub.eu: - auth: "{{ fedcloud_sa }}" - cesga.es: - auth: "{{ fedcloud_sa }}" - cloud.egi.eu: - auth: "{{ fedcloud_sa }}" - cos4cloud-eosc.eu: - auth: "{{ fedcloud_sa }}" - covid19.eosc-synergy.eu: - auth: "{{ fedcloud_sa }}" - cryoem.instruct-eric.eu: - auth: "{{ fedcloud_sa }}" - crystal_channelling_simulation.vo.egi.eu: - auth: "{{ fedcloud_sa }}" - culturalheritage.vo.egi.eu: - auth: "{{ fedcloud_sa }}" - d4science.org: - auth: "{{ fedcloud_sa }}" - datafurn.eosc-hub.eu: - auth: "{{ fedcloud_sa }}" - eosc-synergy.eu: - auth: "{{ fedcloud_sa }}" - fermi-lat.infn.it: - auth: "{{ fedcloud_sa }}" - icecube: - auth: "{{ fedcloud_sa }}" - ispravision.vo.egi.eu: - auth: "{{ fedcloud_sa }}" - kampal.eosc-hub.eu: - auth: "{{ fedcloud_sa }}" - med.semmelweis-univ.hu: - auth: "{{ fedcloud_sa }}" - mswss.ui.savba.sk: - auth: "{{ fedcloud_sa }}" - mteam.data.kit.edu: - auth: "{{ fedcloud_sa }}" - o3as.data.kit.edu: - auth: "{{ fedcloud_sa }}" - openrisknet.org: - auth: "{{ fedcloud_sa }}" - peachnote.com: - auth: "{{ fedcloud_sa }}" - saps-vo.i3m.upv.es: - auth: "{{ fedcloud_sa }}" - training.egi.eu: - auth: "{{ fedcloud_sa }}" - umsa.cerit-sc.cz: - auth: "{{ fedcloud_sa }}" - vo.access.egi.eu: - auth: "{{ fedcloud_sa }}" - vo.ai4eosc.eu: - auth: "{{ fedcloud_sa }}" - vo.aneris.eu: - auth: "{{ fedcloud_sa }}" - vo.bd4nrg.eu: - auth: "{{ fedcloud_sa }}" - vo.binare-oy.eu: - auth: "{{ fedcloud_sa }}" - vo.beamide.com: - auth: "{{ fedcloud_sa }}" - vo.builtrix.tech: - auth: "{{ fedcloud_sa }}" - vo.carouseldancing.org: - auth: "{{ fedcloud_sa }}" - vo.cite.gr: - auth: "{{ fedcloud_sa }}" - vo.decido-project.eu: - auth: "{{ fedcloud_sa }}" - vo.deltares.nl: - auth: "{{ fedcloud_sa }}" - vo.digitbrain.eu: - auth: "{{ fedcloud_sa }}" - vo.ebrain-health.eu: - auth: "{{ fedcloud_sa }}" - vo.emphasisproject.eu: - auth: "{{ fedcloud_sa }}" - vo.emso-eric.eu: - auth: "{{ fedcloud_sa }}" - vo.enes.org: - auth: "{{ fedcloud_sa }}" - vo.environmental.egi.eu: - auth: "{{ fedcloud_sa }}" - vo.eu-openscreen.eu: - auth: "{{ fedcloud_sa }}" - vo.eurogeoss.eu: - auth: "{{ fedcloud_sa }}" - vo.eurosea.marine.ie: - auth: "{{ fedcloud_sa }}" - vo.e-rihs.eu: - auth: "{{ fedcloud_sa }}" - vo.geoss.eu: - auth: "{{ fedcloud_sa }}" - vo.i-nergy.eu: - auth: "{{ fedcloud_sa }}" - vo.imagine-ai.eu: - auth: "{{ fedcloud_sa }}" - vo.inteligg.com: - auth: "{{ fedcloud_sa }}" - vo.latitudo40.com.eu: - auth: "{{ fedcloud_sa }}" - vo.lethe-project.eu: - auth: "{{ fedcloud_sa }}" - vo.matrycs.eu: - auth: "{{ fedcloud_sa }}" - vo.max-centre.eu: - auth: "{{ fedcloud_sa }}" - vo.mightee.idia.za: - auth: "{{ fedcloud_sa }}" - vo.nextgeoss.eu: - auth: "{{ fedcloud_sa }}" - vo.notebooks.egi.eu: - auth: "{{ fedcloud_sa }}" - vo.obsea.es: - auth: "{{ fedcloud_sa }}" - vo.oipub.com: - auth: "{{ fedcloud_sa }}" - vo.openrdm.eu: - auth: "{{ fedcloud_sa }}" - vo.operas-eu.org: - auth: "{{ fedcloud_sa }}" - vo.pangeo.eu: - auth: "{{ fedcloud_sa }}" - vo.panosc.eu: - auth: "{{ fedcloud_sa }}" - vo.phiri.eu: - auth: "{{ fedcloud_sa }}" - vo.pithia.eu: - auth: "{{ fedcloud_sa }}" - vo.plocan.eu: - auth: "{{ fedcloud_sa }}" - vo.projectescape.eu: - auth: "{{ fedcloud_sa }}" - vo.seadatanet.org: - auth: "{{ fedcloud_sa }}" - vo.sphinxsys.org: - auth: "{{ fedcloud_sa }}" - vo.stars4all.eu: - auth: "{{ fedcloud_sa }}" - vo.thepund.it: - auth: "{{ fedcloud_sa }}" - vo.ubora-biomedical.org: - auth: "{{ fedcloud_sa }}" - vo.usegalaxy.eu: - auth: "{{ fedcloud_sa }}" - worsica.vo.incd.pt: - auth: "{{ fedcloud_sa }}" - wp9-pilot2.eosc-hub.eu: - auth: "{{ fedcloud_sa }}" - vo.radiotracers4psma.eu: - auth: "{{ fedcloud_sa }}" - # Perun based - aquamonitor.c-scale.eu: - auth: "{{ fedcloud_sa }}" - demo.fedcloud.egi.eu: - auth: "{{ fedcloud_sa }}" - eval.c-scale.eu: - auth: "{{ fedcloud_sa }}" - waterwatch.c-scale.eu: - auth: "{{ fedcloud_sa }}" - - # certificate based VOs - biomed: - auth: "{{ robot_sa }}" - cesga: - auth: "{{ robot_sa }}" - dteam: - auth: "{{ robot_sa }}" - eiscat.se: - auth: "{{ robot_sa }}" - enmr.eu: - auth: "{{ robot_sa }}" - fedcloud.egi.eu: - auth: "{{ robot_sa }}" - fusion: - auth: "{{ robot_sa }}" - ops: - auth: "{{ robot_sa }}" - vo.nbis.se: - auth: "{{ robot_sa }}" - - # Eduteams based - vo.eoscfuture-sp.panosc.eu: - auth: "{{ fedcloud_sa }}" - vo.europlanet-vespa.eu: - auth: "{{ fedcloud_sa }}" - lagoproject.net: - auth: "{{ fedcloud_sa }}" - - # Missing membership - chipster.csc.fi: - cms: - belle: - bioisi: - drihm.eu: - opencoast.eosc-hub.eu: - vo.clarin.eu: - vo.complex-systems.eu: - vo.envri-fair.eu: - vo.envrihub.eu: - vo.indigo-datacloud.eu: - vo.lifewatch.eu: - vo.inactive-sarscov2.eu: - university.eosc-synergy.eu: - # unclear - vo.elixir-europe.org: - # VOMS but down - blazarmonitoring.asi.it: diff --git a/sites/CESNET-MCC.yaml b/sites/CESNET-MCC.yaml index fb29e17b..629b960c 100644 --- a/sites/CESNET-MCC.yaml +++ b/sites/CESNET-MCC.yaml @@ -87,9 +87,6 @@ vos: - name: waterwatch.c-scale.eu auth: project_id: acf5d12568914e65a80150efb087cbb8 -- name: vo.carouseldancing.org - auth: - project_id: 7ace5ca998b849619bd34e922fbd2cb7 - name: vo.pangeo.eu auth: project_id: 05e0ff6e03774082aadacc75bfc1d783 diff --git a/sites/NCG-INGRID-PT.yaml b/sites/NCG-INGRID-PT.yaml index 3265be1d..501b1c5e 100644 --- a/sites/NCG-INGRID-PT.yaml +++ b/sites/NCG-INGRID-PT.yaml @@ -14,9 +14,6 @@ vos: - name: dev.intertwin.eu auth: project_id: 7e08ed7db02847a1858ce8ed0f93be3c -- name: ec-meloa.eu - auth: - project_id: 97095d3bc281493ebd1c108f2a031b0d - name: eosc-synergy.eu auth: project_id: ddf0c468c8af4e0bbb9808bfc0288381 diff --git a/vo-mappings.yaml b/vo-mappings.yaml index b0b0bcbb..19c311ee 100644 --- a/vo-mappings.yaml +++ b/vo-mappings.yaml @@ -47,7 +47,6 @@ vos: vo.bd4nrg.eu: "urn:mace:egi.eu:group:vo.bd4nrg.eu:role=vm_operator#aai.egi.eu" vo.beamide.com: "urn:mace:egi.eu:group:vo.beamide.com:role=vm_operator#aai.egi.eu" vo.binare-oy.eu: "urn:mace:egi.eu:group:vo.binare-oy.eu:role=vm_operator#aai.egi.eu" - vo.carouseldancing.org: "urn:mace:egi.eu:group:vo.carouseldancing.org:role=vm_operator#aai.egi.eu" vo.cite.gr: "urn:mace:egi.eu:group:vo.cite.gr:role=vm_operator#aai.egi.eu" vo.clarin.eu: "urn:mace:egi.eu:group:vo.clarin.eu:role=vm_operator#aai.egi.eu" vo.complex-systems.eu: "urn:mace:egi.eu:group:vo.complex-systems.eu:role=vm_operator#aai.egi.eu"