int_test.yml

name: Integration tests
on:
  workflow_call:
    inputs:
      resource-group:
        description: Resource group for cluster creation
        type: string
        required: true
        default: ops-cli-int-test-rg
      runtime-init-args:
        description: Additional INIT arguments (beyond cluster name, resource group, schema registry).
        type: string
        required: false
        default: ''
      runtime-create-args:
        description: Additional CREATE arguments (beyond cluster name, resource group, instance name).
        type: string
        required: false
        default: ''
      custom-locations-oid:
        description: Custom Locations OID
        type: string
        required: false
        default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815'
      init-continue-on-error:
        description: Continue on error for init integration tests
        type: boolean
        required: false
        default: true
      use-container:
        description: Build container image for tests
        type: boolean
        required: false
        default: false
    secrets:
      # required for az login
      AZURE_CLIENT_ID:
        required: true
      AZURE_TENANT_ID:
        required: true
      AZURE_SUBSCRIPTION_ID:
        required: true
      # optional --sp-* init params
      AIO_SP_APP_ID:
      AIO_SP_OBJECT_ID:
      AIO_SP_SECRET:
  workflow_dispatch:
    inputs:
      resource-group:
        description: Resource group for cluster creation
        type: string
        required: true
        default: ops-cli-int-test-rg
      custom-locations-oid:
        description: Object ID of Custom Locations RP
        type: string
        required: false
        default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815'
      runtime-init-args:
        description: Additional INIT arguments (beyond cluster name, resource group, schema registry).
        type: string
        required: false
        default: ''
      runtime-create-args:
        description: Additional CREATE arguments (beyond cluster name, resource group, instance name).
        type: string
        required: false
        default: ''
      init-continue-on-error:
        description: Continue on error for init integration tests
        type: boolean
        required: false
        default: true
      use-container:
        description: Build container image for tests
        type: boolean
        required: false
        default: false

permissions:
  # required for OpenID federation
  contents: 'read'
  id-token: 'write'

env:
  RESOURCE_GROUP: "${{ inputs.resource-group }}"

jobs:
  test:
    env:
      CLUSTER_NAME: "opt${{ github.run_number }}${{ matrix.feature }}"
      INSTANCE_NAME: "inst${{ github.run_number }}${{ matrix.feature }}"
      CUSTOM_LOCATIONS_OID: ${{ inputs.custom-locations-oid }}
      EXTENSION_SOURCE_DIRECTORY: "./azure-iot-ops-cli-extension"
      K3S_VERSION: "v1.28.5+k3s1"
      CA_FILE: "test-ca.pem"
      CA_KEY_FILE: "test-ca-key.pem"
    strategy:
      fail-fast: false
      matrix:
        feature: [custom-input, default, insecure-listener, syncrules, redeploy, trustbundle]
        runtime-args:
          - ${{ inputs.runtime-init-args != '' || inputs.runtime-create-args != '' }}
        exclude:
          - feature: custom-input
            runtime-args: false
          - feature: default
            runtime-args: true
          - feature: insecure-listener
            runtime-args: true
          - feature: syncrules
            runtime-args: true
          - feature: redeploy
            runtime-args: true
          - feature: trustbundle
            runtime-args: true
    name: "Run cluster tests"
    runs-on: ubuntu-22.04
    steps:
      - name: "Determine Init Args"
        id: "init"
        run: |
          echo "NO_PREFLIGHT=false" >> $GITHUB_OUTPUT
          if [[ ${{ matrix.feature }} == "insecure-listener" ]]; then
            echo "CREATE_ARG=--add-insecure-listener --broker-listener-type NodePort" >> $GITHUB_OUTPUT
            echo "NO_PREFLIGHT=true" >> $GITHUB_OUTPUT
          elif [[ ${{ matrix.feature }} == "syncrules" ]]; then
            echo "CREATE_ARG=--enable-rsync" >> $GITHUB_OUTPUT
          elif [[ ${{ matrix.feature }} == "redeploy" ]]; then
            echo "REDEPLOY=True" >> $GITHUB_OUTPUT
          elif [[ ${{ matrix.feature }} == "custom-input" ]]; then
            echo "CREATE_ARG=${{ inputs.runtime-create-args }}" >> $GITHUB_OUTPUT
            echo "INIT_ARG=${{ inputs.runtime-init-args }}" >> $GITHUB_OUTPUT
          elif [[ ${{ matrix.feature }} == "trustbundle" ]]; then
            echo "CREATE_ARG=--trust-settings configMapName=example-bundle configMapKey=trust-bundle.pem issuerKind=Issuer issuerName=selfsigned-issuer " >> $GITHUB_OUTPUT
            echo "INIT_ARG=--user-trust true" >> $GITHUB_OUTPUT
          fi
      - name: "Output variables for future steps"
        id: "env_out"
        run: |
          echo "RESOURCE_GROUP=${{ env.RESOURCE_GROUP }}" >> $GITHUB_OUTPUT
          echo "CLUSTER_NAME=${{ env.CLUSTER_NAME }}" >> $GITHUB_OUTPUT
          echo "INSTANCE_NAME=${{ env.INSTANCE_NAME }}" >> $GITHUB_OUTPUT
      - name: "Setup python"
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: "Checkout extension source for build"
        uses: actions/checkout@v4
        with:
          # ensure source checkout uses our repo instead of calling workflow
          repository: azure/azure-iot-ops-cli-extension
          path: ${{ env.EXTENSION_SOURCE_DIRECTORY }}
      - name: "Build and install local IoT Ops extension from source"
        run: |
          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
          pip install -r dev_requirements.txt
          python -m setup bdist_wheel -d dist
          wheel=$(find ./dist/*.whl)
          az extension add --source $wheel -y
      - name: "Create k3s cluster"
        run: |
          # vars
          release_url="https://github.com/k3s-io/k3s/releases/download"
          install_dir="/usr/local/bin/k3s"

          # install k3s
          curl -Lo $install_dir "$release_url/${{ env.K3S_VERSION }}/k3s"
          chmod a+x $install_dir

          # config
          K3S_KUBECONFIG_MODE="644"

          # start cluster in background
          sudo k3s server --cluster-init &

          # allow time for cluster to initialize
          sleep 15s
      - name: "Write cluster config"
        run: |
          mkdir ~/.kube 2> /dev/null
          sudo k3s kubectl config view --raw > ~/.kube/config
          chmod 600 ~/.kube/config
          export KUBECONFIG=~/.kube/config
      - name: "Apply custom cluster issuer"
        if: ${{ matrix.feature == 'trustbundle' || contains(inputs.runtime-init-args, '--user-trust') }}
        run: |
          set -x
          helm repo add jetstack https://charts.jetstack.io --force-update

          helm install \
            cert-manager jetstack/cert-manager \
            --namespace cert-manager \
            --create-namespace \
            --version v1.13.6 \
            --set installCRDs=true \
            --set startupapicheck.timeout=5m

          kubectl create namespace azure-iot-operations

          helm upgrade trust-manager jetstack/trust-manager \
            --install \
            --namespace cert-manager \
            --set app.trust.namespace=cert-manager \
            --wait

          kubectl apply -f - <<EOF
          apiVersion: cert-manager.io/v1
          kind: Issuer
          metadata:
            name: selfsigned-issuer
            namespace: cert-manager
          spec:
            selfSigned: {}
          ---
          apiVersion: cert-manager.io/v1
          kind: Certificate
          metadata:
            name: trust-manager-example-ca
            namespace: cert-manager
          spec:
            isCA: true
            commonName: trust-manager-example-ca
            secretName: trust-manager-example-ca-secret
            privateKey:
              algorithm: ECDSA
              size: 256
            issuerRef:
              name: selfsigned-issuer
              kind: Issuer
              group: cert-manager.io
          EOF

          kubectl apply -f - <<EOF
          apiVersion: trust.cert-manager.io/v1alpha1
          kind: Bundle
          metadata:
            name: example-bundle
            namespace: cert-manager
          spec:
            sources:
            - useDefaultCAs: true
            - secret:
                name: "trust-manager-example-ca-secret"
                key: "tls.crt"
            target:
              configMap:
                key: "trust-bundle.pem"
              namespaceSelector:
                matchLabels:
                  kubernetes.io/metadata.name: azure-iot-operations
          EOF

          kubectl apply -f - <<EOF
          apiVersion: cert-manager.io/v1
          kind: Issuer
          metadata:
            name: selfsigned-issuer
            namespace: azure-iot-operations
          spec:
            selfSigned: {}
          EOF
      - name: "Az CLI login"
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      - name: "OIDC Token exchange service"
        run: |
          while true; do
            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
            az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -t ${{ secrets.AZURE_TENANT_ID }} --federated-token $token --output none
            # Sleep for 4 minutes
            sleep 240
          done &
      - name: "ARC connect cluster"
        uses: azure/azure-iot-ops-cli-extension/.github/actions/connect-arc@dev
        with:
          cluster-name: ${{ env.CLUSTER_NAME }}
          resource-group: ${{ env.RESOURCE_GROUP }}
          custom-locations-oid: ${{ env.CUSTOM_LOCATIONS_OID }}
      - name: "Tox test environment setup for init"
        run: |
          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
          python -m pip install tox
          tox r -vv -e python-init-int --notest
      - name: "Tox test environment setup for integration tests"
        if: ${{ matrix.feature == 'default' && !inputs.use-container }}
        run: |
          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
          tox r -vv -e python-int --notest
      - name: "Tox INIT Integration Tests"
        env:
          AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }}
          azext_edge_init_continue_on_error: ${{ inputs.init-continue-on-error || '' }}
          azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }}
          azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }}
          azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
          azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }}
          azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }}
          azext_edge_init_redeployment: ${{ steps.init.outputs.REDEPLOY }}
        run: |
          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
          tox r -e python-init-int --skip-pkg-install -- --durations=0
      - name: "Tox Integration Tests"
        if: ${{ matrix.feature == 'default' && !inputs.use-container }}
        env:
          azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }}
          azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }}
          azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
        run: |
          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
          tox r -e python-int --skip-pkg-install -- --durations=0
          coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4)
          echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY
      - name: "Containerized tests"
        if: ${{ matrix.feature == 'default' && inputs.use-container }}
        env:
          azext_edge_skip_init: true  # skip init tests in container
          azext_edge_init_redeployment: false  # ensure no redeployment in container
          AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }}
          azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }}
          azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }}
          azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
        run: |
          # volume mounts
          azure_dir=$(realpath ~/.azure)
          kubeconfig=$(realpath ~/.kube/config)
          kubeconfig_mount=/root/.kube/config
          tempLog=$(mktemp -d)

          # env vars
          envVars=()
          envVars+=("-e" "azext_edge_skip_init=$azext_edge_skip_init")
          envVars+=("-e" "azext_edge_init_redeployment=$azext_edge_init_redeployment")
          envVars+=("-e" "AIO_CLI_INIT_PREFLIGHT_DISABLED=$AIO_CLI_INIT_PREFLIGHT_DISABLED")
          envVars+=("-e" "azext_edge_rg=$azext_edge_rg")
          envVars+=("-e" "azext_edge_cluster=$azext_edge_cluster")
          envVars+=("-e" "azext_edge_instance=$azext_edge_instance")
          envVars+=("-e" "KUBECONFIG=$kubeconfig_mount")

          # Run tests
          set +e
          docker run \
            --rm \
            "${envVars[@]}" \
            -v "$kubeconfig:$kubeconfig_mount:ro" \
            -v "${azure_dir}:/root/.azure" \
            -v "${tempLog}:/usr/src/azure-iot-ops/junit" \
            --network host \
            $(docker build ${{ env.EXTENSION_SOURCE_DIRECTORY }} -q)
      - name: "Run smoke tests"
        run: |
          az iot ops support create-bundle
          az iot ops support create-bundle --svc broker --broker-traces
          az iot ops support create-bundle --ops-service acs secretstore
          az iot ops check
          az iot ops check --pre
          az iot ops check --post
          az iot ops check --as-object
          az iot ops check --svc broker --resources broker brokerlistener
          az iot ops asset query -g ${{ env.RESOURCE_GROUP }} --location westus -o table
      - name: "Delete Cluster for redeployment"
        if: ${{ matrix.feature == 'redeploy' }}
        run: |
          az iot ops delete -n ${{ env.INSTANCE_NAME }} -g ${{ env.RESOURCE_GROUP }} -y
      - name: "Redeploy cluster via tox"
        if: ${{ matrix.feature == 'redeploy' }}
        env:
          azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }}
          azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }}
          azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
          azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }}
          azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }}
        run: |
          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
          tox r -e python-init-int --skip-pkg-install -- --durations=0
      - name: "Delete AIO resources"
        if: ${{ always() }}
        run: |
          az iot ops delete --cluster ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} -y
      - name: "Delete connected cluster"
        if: ${{ always() }}
        run: |
          az resource delete  -v --name ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} --resource-type Microsoft.Kubernetes/connectedClusters --verbose