Ansible and secrets update

.github/workflows/deploy.yml

@@ -25,17 +25,17 @@ jobs:
           curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 > jq
           chmod +x jq
           pip install yq git+https://github.com/tdviet/fedcloudclient.git
-          curl -L https://github.com/oidc-mytoken/client/releases/download/v0.3.0/mytoken_0.3.0_Linux_x86_64.tar.gz \
-            | tar -xzf -
-          mkdir ~/.mytoken
-          curl https://raw.githubusercontent.com/oidc-mytoken/client/master/config/example-config.yaml > ~/.mytoken/config.yaml
       - name: Configure providers access
         env:
-          MYTOKEN: ${{ secrets.MYTOKEN }}
           REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
+          ANSIBLE_SECRETS: ${{ secrets.ANSIBLE_SECRETS }}
         run: |
+          # using parametric scopes to only have access to cloud.egi.eu VO
+          SCOPE="openid%20email%20profile%20voperson_id"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu"
           OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \
-                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=openid%20email%20profile%20voperson_id%20eduperson_entitlement" \
+                            -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \
                           | jq -r ".access_token")
           echo "::add-mask::$OIDC_TOKEN"
           cd deploy
@@ -54,6 +54,12 @@ jobs:
           sed -i -e "s/deploy_secret/$DEPLOY_OS_TOKEN/" clouds.yaml
           mkdir -p ~/.config/openstack
           touch ~/.config/openstack/secure.yaml
+          FEDCLOUD_LOCKER_TOKEN="$(fedcloud secret locker create \
+                                   --oidc-access-token "$OIDC_TOKEN" \
+                                   --ttl 1h --num-uses 2)"
+          echo "::add-mask::$FEDCLOUD_LOCKER_TOKEN"
+          fedcloud secret put --locker-token "$FEDCLOUD_LOCKER_TOKEN" deploy "data=$ANSIBLE_SECRETS"
+          echo "FEDCLOUD_LOCKER_TOKEN=$FEDCLOUD_LOCKER_TOKEN" >> "$GITHUB_ENV"
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
         with:
@@ -71,16 +77,13 @@ jobs:
       - name: Adjust cloud-init file
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          ANSIBLE_SECRETS: ${{ secrets.ANSIBLE_SECRETS }}
         run: |
           cd deploy
           sed -i -e "s/%TOKEN%/${{ secrets.GITHUB_TOKEN }}/" cloud-init.yaml
           sed -i -e "s/%REF%/${{ github.sha }}/" cloud-init.yaml
           sed -i -e "s/%SHORT_REF%/$(git rev-parse --short HEAD)/" cloud-init.yaml
           sed -i -e "s#%SLACK_WEBHOOK_URL%#$SLACK_WEBHOOK_URL#" cloud-init.yaml
-          ANSIBLE_ENCODED_SECRETS="$(echo "$ANSIBLE_SECRETS" | base64 -w 0)"
-          echo "::add-mask::$ANSIBLE_ENCODED_SECRETS"
-          sed -i -e "s/%ANSIBLE_SECRETS%/$ANSIBLE_ENCODED_SECRETS/" cloud-init.yaml
+          sed -i -e "s/%FEDCLOUD_LOCKER_TOKEN%/$FEDCLOUD_LOCKER_TOKEN/" cloud-init.yaml
           sed -i -e "s/%CLOUDS_YAML%/$(base64 -w 0 < clouds.yaml)/" cloud-init.yaml
       - name: terraform plan
         id: plan
@@ -132,19 +135,24 @@ jobs:
           terraform output -raw instance-id
       - name: Re-configure providers access
         env:
-          MYTOKEN: ${{ secrets.MYTOKEN }}
           REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
         run: |
+          # using parametric scopes to only have access to cloud.egi.eu VO
+          SCOPE="openid%20email%20profile%20voperson_id"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu"
           OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \
-                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=openid%20email%20profile%20voperson_id%20eduperson_entitlement" \
+                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=$SCOPE" \
                           | jq -r ".access_token")
           echo "::add-mask::$OIDC_TOKEN"
           cd deploy
+          git checkout -- clouds.yaml
           BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
           BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
           BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \
                                                              --site "$BACKEND_SITE" --vo "$BACKEND_VO" -j | jq -r '.[0].Result.id')"
           echo "::add-mask::$BACKEND_OS_TOKEN"
+          echo "BACKEND_OS_TOKEN=$BACKEND_OS_TOKEN" >> "$GITHUB_ENV"
           sed -i -e "s/backend_secret/$BACKEND_OS_TOKEN/" clouds.yaml
           mkdir -p ~/.config/openstack
           touch ~/.config/openstack/secure.yaml
@@ -156,10 +164,9 @@ jobs:
           max_attempts: 20
           retry_wait_seconds: 40
           command: >
-            set -x &&
             pushd deploy &&
-            openstack --os-cloud backend object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" &&
-            openstack --os-cloud backend object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}"
+            openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" &&
+            openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}"
       - name: Look for errors
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |

.github/workflows/molecule.yml

@@ -0,0 +1,26 @@
+---
+name: Test role
+
+on: [push, pull_request]
+
+jobs:
+  molecule:
+    name: Runs molecule for the ansible role
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '>=3.9'
+      - name: Install dependencies
+        run: |
+          pip install molecule molecule-plugins[docker] pytest pytest-testinfra
+      - name: Test Ansible Bootstrap
+        run: |
+          cd deploy/roles/catchall
+          molecule test
+        env:
+          PY_COLORS: 1

deploy/cloud-init.yaml

@@ -23,6 +23,8 @@ packages:
   - ansible
   - jq
   - python3-openstackclient
+  - python3-pip
+  - python3.10-venv
   - retry
 
 write_files:
@@ -40,14 +42,14 @@ write_files:
       SLACK_WEBHOOK_URL="%SLACK_WEBHOOK_URL%"
       COMMIT_SHA="%REF%"
       SHORT_COMMIT_SHA="%SHORT_REF%"
+      FEDCLOUD_LOCKER_TOKEN="%FEDCLOUD_LOCKER_TOKEN%"
 
       # get the repo code and untar at cwd
       curl -L -H "Accept: application/vnd.github.v3+raw" \
               "https://api.github.com/repos/EGI-Federation/fedcloud-catchall-operations/tarball/$COMMIT_SHA" | \
               tar xz --strip=1
       cd deploy
-      echo "%ANSIBLE_SECRETS%" | base64 -d > ./secrets.yaml
-      ./deploy.sh "$OAUTH_TOKEN" "$COMMIT_SHA" \
+      ./deploy.sh "$OAUTH_TOKEN" "$COMMIT_SHA" "$FEDCLOUD_LOCKER_TOKEN" \
                   "$SHORT_COMMIT_SHA" "$SLACK_WEBHOOK_URL"
     path: /var/lib/cloud/scripts/per-boot/deploy.sh
     permissions: '0755'

deploy/deploy.sh

@@ -1,15 +1,25 @@
 #!/bin/sh
 # Configure current host with ansible
-# Expects as arguments the OAUTH_TOKEN, the COMMIT_SHA and the SLACK_WEBHOOK_URL
-
+# Expects as arguments:
+# - a GitHub OAUTH_TOKEN to update the PR
+# - the COMMIT_SHA
+# - a locker for fedcloud secret to obtain the secrets
+# - the SHORT_SHA used for pulling the docker image to use
+# - a SLACK_WEBHOOK_URL to report on the status
 set -e
 
 OAUTH_TOKEN="$1"
 COMMIT_SHA="$2"
-SHORT_SHA="$3"
-SLACK_WEBHOOK_URL="$4"
+FEDCLOUD_SECRET_LOCKER="$3"
+SHORT_SHA="$4"
+SLACK_WEBHOOK_URL="$5"
+
+# create a virtual env for fedcloudclient
+python3 -m venv "$PWD/.venv"
+"$PWD/.venv/bin/pip" install fedcloudclient
 
-ansible-galaxy install git+https://github.com/EGI-Federation/ansible-role-fedcloud-ops.git
+"$PWD/.venv/bin/fedcloud" secret get --locker-token "$FEDCLOUD_SECRET_LOCKER" \
+	deploy data >secrets.yaml
 
 echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA\"" >>extra-vars.yaml
 

deploy/playbook.yaml

@@ -2,7 +2,7 @@
 - hosts: all
   become: true
   roles:
-    - role: ansible-role-fedcloud-ops
+    - role: catchall
       tags: ["all", "docker"]
       vars:
         site_config_dir: ../sites/

deploy/roles/catchall/defaults/main.yaml

@@ -0,0 +1,22 @@
+# AMS details
+ams_project: egi_cloud_info
+ams_host: msg.argo.grnet.gr
+ams_token: secret
+
+# check-in endpoint
+checkin_token_endpoint: "https://aai.egi.eu/oidc/token"
+
+# docker image for the cloud info provider
+cloud_info_image: egifedcloud/ops-cloud-info:latest
+
+# site configuration location
+site_config_dir: sites
+
+# No site information as default
+sites: []
+
+cloud_info_cron:
+  minute: "4,34"
+  hour: "*"
+  weekday: "*"
+  timeout: "600"

deploy/roles/catchall/molecule/default/converge.yml

@@ -0,0 +1,28 @@
+---
+- name: Converge
+  hosts: all
+  tasks:
+    - name: "Include catchall role"
+      ansible.builtin.include_role:
+        name: "catchall"
+      vars:
+        sites:
+          - endpoint: https://example.com:5000/v3/
+            gocdb: foo.bar
+            vos:
+              - auth:
+                  project_id: a123456
+                name: sample_vo
+              - auth:
+                  project_id: b987659
+                name: vo.example.com
+          - endpoint: https://site.org:5000/v3/
+            gocdb: bar.foo
+            region: region1
+            vos:
+              - auth:
+                  project_id: a123456
+                name: sample_vo
+              - auth:
+                  project_id: b987659
+                name: vo.example.com

deploy/roles/catchall/molecule/default/molecule.yml

@@ -0,0 +1,13 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: instance
+    image: ubuntu:latest
+lint: ansible-lint --exclude .github/
+provisioner:
+  name: ansible
+verifier:
+  name: testinfra

deploy/roles/catchall/molecule/default/tests/test_default.py

@@ -0,0 +1,30 @@
+import hashlib
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ["MOLECULE_INVENTORY_FILE"]
+).get_hosts("all")
+
+
+def test_site_files(host):
+    endpoint_hash = hashlib.md5(b"https://example.com:5000/v3/").hexdigest()
+    filename = "foo-bar-%s" % endpoint_hash
+    assert host.file("/etc/egi/cloud-info/").is_directory
+    assert host.file("/etc/egi/cloud-info/%s.yaml" % filename).exists
+    assert not host.file("/etc/egi/cloud-info/%s.env" % filename).contains("OS_REGION")
+    assert host.file("/etc/egi/cloud-info/%s.env" % filename).exists
+    assert host.file("/etc/cron.d/cloud-info-%s" % filename).exists
+
+
+def test_site_files_region(host):
+    endpoint_hash = hashlib.md5(b"https://site.org:5000/v3/").hexdigest()
+    filename = "bar-foo-%s" % endpoint_hash
+    assert host.file("/etc/egi/cloud-info/").is_directory
+    assert host.file("/etc/egi/cloud-info/%s.yaml" % filename).exists
+    assert host.file("/etc/egi/cloud-info/%s.env" % filename).exists
+    assert host.file("/etc/egi/cloud-info/%s.env" % filename).contains(
+        "OS_REGION=region1"
+    )
+    assert host.file("/etc/cron.d/cloud-info-%s" % filename).exists

deploy/roles/catchall/requirements.txt

@@ -0,0 +1,4 @@
+molecule
+molecule-plugins[docker]
+pytest-testinfra
+ansible-lint

deploy/roles/catchall/tasks/cloud-info.yml

@@ -0,0 +1,26 @@
+---
+- name: Cloud-info config directory
+  ansible.builtin.template:
+    src: site-info.yaml.j2
+    dest: /etc/egi/cloud-info/{{ filename }}.yaml
+    mode: "600"
+
+- name: Cloud info env
+  ansible.builtin.template:
+    src: cloud-info.env.j2
+    dest: /etc/egi/cloud-info/{{ filename }}.env
+    mode: "600"
+
+- name: Cloud info cron
+  ansible.builtin.cron:
+    name: cloud-info-provider {{ site.gocdb }}
+    weekday: "{{ cloud_info_cron.weekday }}"
+    minute: "{{ cloud_info_cron.minute }}"
+    hour: "{{ cloud_info_cron.hour }}"
+    user: root
+    job: >
+      flock -n -w {{ cloud_info_cron.timeout }} /var/lock/cloud-info/{{ filename }}
+      docker run --rm -v /etc/egi:/etc/egi:ro
+      --env-file /etc/egi/cloud-info/{{ filename }}.env
+      {{ cloud_info_image }} >> /var/log/cloud-info/{{ filename }}.log 2>&1
+    cron_file: "cloud-info-{{ filename }}"

deploy/roles/catchall/tasks/docker.yml

@@ -0,0 +1,59 @@
+---
+- name: Install dependencies
+  ansible.builtin.apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - curl
+      - gnupg-agent
+      - software-properties-common
+    state: present
+    update_cache: true
+
+- name: Docker repo key
+  ansible.builtin.apt_key:
+    id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
+
+- name: Add docker repo
+  ansible.builtin.apt_repository:
+    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+    state: present
+
+- name: Install docker
+  ansible.builtin.apt:
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+    state: present
+    update_cache: true
+
+- name: Ensure docker config dir is present
+  ansible.builtin.file:
+    path: /etc/docker
+    state: directory
+    mode: "775"
+
+- name: Configure docker
+  ansible.builtin.copy:
+    # this is very CESNET-MCC specific, may be better to move as configurable
+    content: |
+      {
+        "mtu": 1442,
+        "exec-opts": ["native.cgroupdriver=systemd"],
+        "log-driver": "json-file",
+        "log-opts": {
+          "max-size": "100m"
+        },
+        "storage-driver": "overlay2"
+      }
+    dest: /etc/docker/daemon.json
+    mode: "660"
+
+- name: Restart docker
+  ansible.builtin.systemd:
+    name: docker
+    state: restarted
+    daemon_reload: true