Skip to content

Commit

Permalink
Publish json to swift (#390)
Browse files Browse the repository at this point in the history
* Do not stop if AMS is not available

* Publish to swift

* Working script now

* Add swift variables to the role

* Formatting

* Remove nonexisting VO

* Remove not needed code

* Add swift config for cloud-info
  • Loading branch information
enolfc authored Jan 14, 2025
1 parent e50e754 commit 4d5ffc0
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 56 deletions.
105 changes: 50 additions & 55 deletions cloud-info/ams-wrapper.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,6 @@ GOCDB_ID=$(python -c "from __future__ import print_function; \
'$GOCDB_SERVICE_TYPE',
timeout=60)['gocdb_id'], end='')")

if test "$AMS_TOKEN_FILE" != ""; then
AMS_TOKEN=$(cat "$AMS_TOKEN_FILE")
elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \
ams = ArgoMessagingService(endpoint='$AMS_HOST', \
project='$AMS_PROJECT', \
cert='$HOSTCERT', \
key='$HOSTKEY'); \
print(ams.token)")
fi

if test "$SITE_NAME" = ""; then
SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")"
fi

SITE_TOPIC=$(echo "$SITE_NAME" | tr "." "-")
AMS_TOPIC="SITE_${SITE_TOPIC}_ENDPOINT_${GOCDB_ID}"

# exit if TOPIC is not available.
curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" >/dev/null 2>&1 ||
(
echo "Topic $AMS_TOPIC is not avaiable, aborting!"
false
)

# Attempt to generate the site configuration
AUTO_CONFIG_PATH="$(mktemp -d)"

Expand Down Expand Up @@ -73,13 +48,10 @@ else
--ignore-share-errors \
--format glue21 >cloud-info.out
# Produce the json output also
RCLONE_CONFIG_S3="$(yq -r '.s3' <"$CHECKIN_SECRETS_FILE")"
if test "$RCLONE_CONFIG_S3" != "null"; then
cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
--middleware "$CLOUD_INFO_MIDDLEWARE" \
--ignore-share-errors \
--format glue21json >site.json
fi
cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
--middleware "$CLOUD_INFO_MIDDLEWARE" \
--ignore-share-errors \
--format glue21json >cloud-info.json
fi

# Fail if there are no shares
Expand All @@ -89,32 +61,55 @@ grep -q GLUE2ShareID cloud-info.out ||
false
)

# Publishing on our own as message is too large for some providers
ARGO_URL="https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC:publish?key=$AMS_TOKEN"
# Publish to AMS
if test "$AMS_TOKEN_FILE" != ""; then
AMS_TOKEN=$(cat "$AMS_TOKEN_FILE")
elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \
ams = ArgoMessagingService(endpoint='$AMS_HOST', \
project='$AMS_PROJECT', \
cert='$HOSTCERT', \
key='$HOSTKEY'); \
print(ams.token)")
fi

if test "$SITE_NAME" = ""; then
SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")"
fi
SITE_TOPIC=$(echo "$SITE_NAME" | tr "." "-")
AMS_TOPIC="SITE_${SITE_TOPIC}_ENDPOINT_${GOCDB_ID}"
curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" >/dev/null 2>&1 &&
(
# Publishing to AMS on our own to ensure message fits
ARGO_URL="https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC:publish?key=$AMS_TOKEN"

printf '{"messages":[{"attributes":{},"data":"' >ams-payload
grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >>ams-payload
printf '"}]}' >>ams-payload
printf '{"messages":[{"attributes":{},"data":"' >ams-payload
grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >>ams-payload
printf '"}]}' >>ams-payload

curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload
curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload
)

if [ -f site.json ]; then
# Put this info into S3, configure rclone config with
# a provider named "s3" using env variables
export RCLONE_CONFIG_S3_TYPE=s3
RCLONE_CONFIG_S3_ACCESS_KEY_ID="$(yq -r '.s3.access_key_id' <"$CHECKIN_SECRETS_FILE")"
export RCLONE_CONFIG_S3_ACCESS_KEY_ID
RCLONE_CONFIG_S3_SECRET_ACCESS_KEY="$(yq -r '.s3.secret_access_key' <"$CHECKIN_SECRETS_FILE")"
export RCLONE_CONFIG_S3_SECRET_ACCESS_KEY
RCLONE_CONFIG_S3_ENDPOINT="$(yq -r '.s3.endpoint' <"$CHECKIN_SECRETS_FILE")"
export RCLONE_CONFIG_S3_ENDPOINT
S3_BUCKET_NAME="$(yq -r '.s3.bucket' <"$CHECKIN_SECRETS_FILE")"
export S3_BUCKET_NAME
RCLONE_CONFIG_S3_PROVIDER="$(yq -r '.s3.provider' <"$CHECKIN_SECRETS_FILE")"
export RCLONE_CONFIG_S3_PROVIDER
export RCLONE_CONFIG_S3_ACL=private
export RCLONE_CONFIG_S3_NO_CHECK_BUCKET=true
rclone copy site.json "s3:$S3_BUCKET_NAME/$SITE_NAME"
# Publish to object
if test -s cloud-info.json; then
if test "$SWIFT_SITE_NAME" != ""; then
OIDC_ACCESS_TOKEN=$(yq -r '.checkin.access_token' <"$ACCESS_TOKEN_FILE")
export OIDC_ACCESS_TOKEN
export EGI_VO="$SWIFT_VO_NAME"
SWIFT_URL=$(fedcloud openstack \
--site "$SWIFT_SITE_NAME" \
catalog show swift -f json |
jq -r '(.endpoints[] | select(.interface=="public")).url')
export RCLONE_CONFIG_REMOTE_TYPE="swift"
export RCLONE_CONFIG_REMOTE_ENV_AUTH="false"
export RCLONE_CONFIG_REMOTE_STORAGE_URL="$SWIFT_URL"
eval "$(fedcloud site env --site "$SWIFT_SITE_NAME")"
export RCLONE_CONFIG_REMOTE_AUTH_URL="$OS_AUTH_URL"
OS_AUTH_TOKEN=$(fedcloud openstack --site "$SWIFT_SITE_NAME" token issue -c id -f value)
export RCLONE_CONFIG_REMOTE_AUTH_TOKEN="$OS_AUTH_TOKEN"
rclone mkdir "remote:$SWIFT_CONTAINER_NAME"
rclone copy cloud-info.json "remote:$SWIFT_CONTAINER_NAME/$SITE_NAME"
fi
fi

rm -rf "$VO_CONFIG_PATH"
4 changes: 4 additions & 0 deletions deploy/playbook.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,7 @@
- role: catchall
vars:
site_config_dir: ../sites/
swift:
site: NCG-INGRID-PT
vo: vo.access.egi.eu
container: cloud-info
6 changes: 6 additions & 0 deletions deploy/roles/catchall/defaults/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ egi_group: "1999"
# No site information as default
sites: []

# Empty swift configuration as default
swift:
site:
vo:
container:

cloud_info_cron:
minute: "4,34"
hour: "*"
Expand Down
3 changes: 3 additions & 0 deletions deploy/roles/catchall/templates/cloud-info.env.j2
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,6 @@ OS_PROTOCOL={{ site.protocol | default('openid') }}
OS_REGION={{ site.region }}
{% endif %}
SITE_NAME={{ site.gocdb }}
SWIFT_SITE_NAME={{ swift.site }}
SWIFT_VO_NAME={{ swift.vo }}
SWIFT_CONTAINER_NAME={{ swift.container }}
1 change: 0 additions & 1 deletion vo-mappings.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ vos:
vo.enes.org: "urn:mace:egi.eu:group:vo.enes.org:role=vm_operator#aai.egi.eu"
vo.envrihub.eu: "urn:mace:egi.eu:group:vo.envrihub.eu:role=vm_operator#aai.egi.eu"
vo.envri-fair.eu: "urn:mace:egi.eu:group:vo.envri-fair.eu:role=vm_operator#aai.egi.eu"
vo.environmental.egi.eu: "urn:mace:egi.eu:group:vo.environmental.egi.eu:role=vm_operator#aai.egi.eu"
vo.europlanet-vespa.eu: "urn:mace:egi.eu:group:vo.europlanet-vespa.eu:role=vm_operator#aai.egi.eu"
vo.eurosea.marine.ie: "urn:mace:egi.eu:group:vo.eurosea.marine.ie:role=member#aai.egi.eu"
vo.geoss.eu: "urn:mace:egi.eu:group:vo.geoss.eu:role=vm_operator#aai.egi.eu"
Expand Down

0 comments on commit 4d5ffc0

Please sign in to comment.