Integration tests #305
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration tests | |
| on: | |
| workflow_call: | |
| inputs: | |
| resource-group: | |
| description: Resource group for cluster creation | |
| type: string | |
| required: true | |
| default: ops-cli-int-test-rg | |
| runtime-init-args: | |
| description: Additional INIT arguments (beyond cluster name, resource group, schema registry). | |
| type: string | |
| required: false | |
| default: '' | |
| runtime-create-args: | |
| description: Additional CREATE arguments (beyond cluster name, resource group, instance name). | |
| type: string | |
| required: false | |
| default: '' | |
| custom-locations-oid: | |
| description: Custom Locations OID | |
| type: string | |
| required: false | |
| default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815' | |
| init-continue-on-error: | |
| description: Continue on error for init integration tests | |
| type: boolean | |
| required: false | |
| default: true | |
| use-container: | |
| description: Build container image for tests | |
| type: boolean | |
| required: false | |
| default: false | |
| secrets: | |
| # required for az login | |
| AZURE_CLIENT_ID: | |
| required: true | |
| AZURE_TENANT_ID: | |
| required: true | |
| AZURE_SUBSCRIPTION_ID: | |
| required: true | |
| workflow_dispatch: | |
| inputs: | |
| resource-group: | |
| description: Resource group for cluster creation | |
| type: string | |
| required: true | |
| default: ops-cli-int-test-rg | |
| custom-locations-oid: | |
| description: Object ID of Custom Locations RP | |
| type: string | |
| required: false | |
| default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815' | |
| runtime-init-args: | |
| description: Additional INIT arguments (beyond cluster name, resource group, schema registry). | |
| type: string | |
| required: false | |
| default: '' | |
| runtime-create-args: | |
| description: Additional CREATE arguments (beyond cluster name, resource group, instance name). | |
| type: string | |
| required: false | |
| default: '' | |
| init-continue-on-error: | |
| description: Continue on error for init integration tests | |
| type: boolean | |
| required: false | |
| default: true | |
| use-container: | |
| description: Build container image for tests | |
| type: boolean | |
| required: false | |
| default: false | |
| permissions: | |
| # required for OpenID federation | |
| contents: 'read' | |
| id-token: 'write' | |
| env: | |
| RESOURCE_GROUP: "${{ inputs.resource-group }}" | |
| K3S_VERSION: "v1.28.5+k3s1" | |
| CUSTOM_LOCATIONS_OID: "${{ inputs.custom-locations-oid }}" | |
| jobs: | |
| test: | |
| env: | |
| CLUSTER_NAME: "opt${{ github.run_number }}${{ matrix.feature }}" | |
| INSTANCE_NAME: "inst${{ github.run_number }}${{ matrix.feature }}" | |
| KEYVAULT_NAME: "kv${{ github.run_number }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| feature: [secretsync] # [custom-input, default, insecure-listener, syncrules, redeploy, trustbundle] | |
| # runtime-args: | |
| # - ${{ inputs.runtime-init-args != '' || inputs.runtime-create-args != '' }} | |
| # exclude: | |
| # - feature: custom-input | |
| # runtime-args: false | |
| # - feature: default | |
| # runtime-args: true | |
| # - feature: insecure-listener | |
| # runtime-args: true | |
| # - feature: syncrules | |
| # runtime-args: true | |
| # - feature: redeploy | |
| # runtime-args: true | |
| # - feature: trustbundle | |
| # runtime-args: true | |
| name: "Run cluster tests" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: "Determine Init Args" | |
| id: "init" | |
| run: | | |
| echo "NO_PREFLIGHT=false" >> $GITHUB_OUTPUT | |
| if [[ ${{ matrix.feature }} == "insecure-listener" ]]; then | |
| echo "CREATE_ARG=--add-insecure-listener --broker-listener-type NodePort" >> $GITHUB_OUTPUT | |
| echo "NO_PREFLIGHT=true" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "syncrules" ]]; then | |
| echo "CREATE_ARG=--enable-rsync" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "custom-input" ]]; then | |
| echo "CREATE_ARG=${{ inputs.runtime-create-args }}" >> $GITHUB_OUTPUT | |
| echo "INIT_ARG=${{ inputs.runtime-init-args }}" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "trustbundle" ]]; then | |
| echo "CREATE_ARG=--trust-settings configMapName=example-bundle configMapKey=trust-bundle.pem issuerKind=Issuer issuerName=selfsigned-issuer " >> $GITHUB_OUTPUT | |
| echo "INIT_ARG=--user-trust true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: "Output variables for future steps" | |
| id: "env_out" | |
| run: | | |
| echo "RESOURCE_GROUP=${{ env.RESOURCE_GROUP }}" >> $GITHUB_OUTPUT | |
| echo "CLUSTER_NAME=${{ env.CLUSTER_NAME }}" >> $GITHUB_OUTPUT | |
| echo "INSTANCE_NAME=${{ env.INSTANCE_NAME }}" >> $GITHUB_OUTPUT | |
| - name: "Setup python" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: "Checkout extension source for build" | |
| uses: actions/checkout@v4 | |
| - name: "Build and install local IoT Ops extension from source" | |
| run: | | |
| pip install -r dev_requirements.txt | |
| python -m setup bdist_wheel -d dist | |
| wheel=$(find ./dist/*.whl) | |
| az extension add --source $wheel -y | |
| - name: "Create k3s cluster" | |
| uses: ./.github/actions/create-k3s-cluster | |
| - name: "Apply custom cluster issuer" | |
| if: ${{ matrix.feature == 'trustbundle' || contains(inputs.runtime-init-args, '--user-trust') }} | |
| run: | | |
| set -x | |
| helm repo add jetstack https://charts.jetstack.io --force-update | |
| helm install \ | |
| cert-manager jetstack/cert-manager \ | |
| --namespace cert-manager \ | |
| --create-namespace \ | |
| --version v1.13.6 \ | |
| --set installCRDs=true \ | |
| --set startupapicheck.timeout=5m | |
| kubectl create namespace azure-iot-operations | |
| helm upgrade trust-manager jetstack/trust-manager \ | |
| --install \ | |
| --namespace cert-manager \ | |
| --set app.trust.namespace=cert-manager \ | |
| --wait | |
| kubectl apply -f - <<EOF | |
| apiVersion: cert-manager.io/v1 | |
| kind: Issuer | |
| metadata: | |
| name: selfsigned-issuer | |
| namespace: cert-manager | |
| spec: | |
| selfSigned: {} | |
| --- | |
| apiVersion: cert-manager.io/v1 | |
| kind: Certificate | |
| metadata: | |
| name: trust-manager-example-ca | |
| namespace: cert-manager | |
| spec: | |
| isCA: true | |
| commonName: trust-manager-example-ca | |
| secretName: trust-manager-example-ca-secret | |
| privateKey: | |
| algorithm: ECDSA | |
| size: 256 | |
| issuerRef: | |
| name: selfsigned-issuer | |
| kind: Issuer | |
| group: cert-manager.io | |
| EOF | |
| kubectl apply -f - <<EOF | |
| apiVersion: trust.cert-manager.io/v1alpha1 | |
| kind: Bundle | |
| metadata: | |
| name: example-bundle | |
| namespace: cert-manager | |
| spec: | |
| sources: | |
| - useDefaultCAs: true | |
| - secret: | |
| name: "trust-manager-example-ca-secret" | |
| key: "tls.crt" | |
| target: | |
| configMap: | |
| key: "trust-bundle.pem" | |
| namespaceSelector: | |
| matchLabels: | |
| kubernetes.io/metadata.name: azure-iot-operations | |
| EOF | |
| kubectl apply -f - <<EOF | |
| apiVersion: cert-manager.io/v1 | |
| kind: Issuer | |
| metadata: | |
| name: selfsigned-issuer | |
| namespace: azure-iot-operations | |
| spec: | |
| selfSigned: {} | |
| EOF | |
| - name: "Tox test environment setup for init" | |
| run: | | |
| python -m pip install tox | |
| tox r -vv -e python-init-int --notest | |
| - name: "Tox test environment setup for integration tests" | |
| if: ${{ matrix.feature == 'default' && !inputs.use-container }} | |
| run: | | |
| tox r -vv -e "python-all-int" --notest | |
| - name: "Tox test environment setup for integration tests" | |
| if: ${{ matrix.feature == 'secretsync' && !inputs.use-container }} | |
| run: | | |
| tox r -vv -e "python-secretsync-int" --notest | |
| - name: "Az CLI login" | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: "ARC connect cluster" | |
| uses: ./.github/actions/connect-arc | |
| with: | |
| cluster-name: ${{ env.CLUSTER_NAME }} | |
| resource-group: ${{ env.RESOURCE_GROUP }} | |
| custom-locations-oid: ${{ env.CUSTOM_LOCATIONS_OID }} | |
| enable-workload-identity: ${{ matrix.feature == 'secretsync' }} | |
| - name: "Az CLI login" | |
| if: ${{ matrix.feature == 'secretsync' && !inputs.use-container }} | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: "Tox INIT Integration Tests" | |
| env: | |
| AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }} | |
| azext_edge_init_continue_on_error: ${{ inputs.init-continue-on-error || '' }} | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }} | |
| azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }} | |
| # redeploy tested later in this workflow | |
| azext_edge_init_redeployment: false | |
| run: | | |
| tox r -e python-init-int --skip-pkg-install -- --durations=0 | |
| - name: "Tox Integration Tests" | |
| if: ${{ matrix.feature == 'default' && !inputs.use-container }} | |
| env: | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| run: | | |
| tox r -e "python-all-int" --skip-pkg-install -- --durations=0 --dist=loadfile -n auto | |
| coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4) | |
| echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY | |
| - name: "Tox SecretSync Integration Tests" | |
| if: ${{ matrix.feature == 'secretsync' && !inputs.use-container }} | |
| env: | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| azext_edge_sp_object_id: ${{ secrets.AZURE_CLIENT_ID }} | |
| run: | | |
| tox r -e "python-secretsync-int" --skip-pkg-install -- --durations=0 --dist=loadfile -n auto | |
| coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4) | |
| echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY | |
| - name: "Containerized tests" | |
| if: ${{ matrix.feature == 'default' && inputs.use-container }} | |
| env: | |
| azext_edge_skip_init: true # skip init tests in container | |
| azext_edge_init_redeployment: false # ensure no redeployment in container | |
| AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }} | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| run: | | |
| # volume mounts | |
| azure_dir=$(realpath ~/.azure) | |
| kubeconfig=$(realpath ~/.kube/config) | |
| kubeconfig_mount=/root/.kube/config | |
| tempLog=$(mktemp -d) | |
| # env vars | |
| envVars=() | |
| envVars+=("-e" "azext_edge_skip_init=$azext_edge_skip_init") | |
| envVars+=("-e" "azext_edge_init_redeployment=$azext_edge_init_redeployment") | |
| envVars+=("-e" "AIO_CLI_INIT_PREFLIGHT_DISABLED=$AIO_CLI_INIT_PREFLIGHT_DISABLED") | |
| envVars+=("-e" "azext_edge_rg=$azext_edge_rg") | |
| envVars+=("-e" "azext_edge_cluster=$azext_edge_cluster") | |
| envVars+=("-e" "azext_edge_instance=$azext_edge_instance") | |
| envVars+=("-e" "KUBECONFIG=$kubeconfig_mount") | |
| # Run tests | |
| set +e | |
| docker run \ | |
| --rm \ | |
| "${envVars[@]}" \ | |
| -v "$kubeconfig:$kubeconfig_mount:ro" \ | |
| -v "${azure_dir}:/root/.azure" \ | |
| -v "${tempLog}:/usr/src/azure-iot-ops/junit" \ | |
| --network host \ | |
| $(docker build . -q) | |
| - name: "Run smoke tests" | |
| run: | | |
| az iot ops support create-bundle | |
| az iot ops support create-bundle --svc broker --broker-traces | |
| az iot ops support create-bundle --ops-service acs secretstore | |
| az iot ops check | |
| az iot ops check --pre | |
| az iot ops check --post | |
| az iot ops check --as-object | |
| az iot ops check --svc broker --resources broker brokerlistener | |
| az iot ops asset query -g ${{ env.RESOURCE_GROUP }} --location westus -o table | |
| - name: "Delete Cluster for redeployment" | |
| if: ${{ matrix.feature == 'redeploy' }} | |
| run: | | |
| az iot ops delete -n ${{ env.INSTANCE_NAME }} -g ${{ env.RESOURCE_GROUP }} -y | |
| - name: "Redeploy cluster via tox" | |
| if: ${{ matrix.feature == 'redeploy' }} | |
| env: | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }} | |
| azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }} | |
| run: | | |
| tox r -e python-init-int --skip-pkg-install -- --durations=0 | |
| - name: "Delete AIO resources" | |
| if: ${{ always() }} | |
| run: | | |
| az iot ops delete --cluster ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} -y | |
| - name: "Delete connected cluster" | |
| if: ${{ always() }} | |
| run: | | |
| az resource delete -v --name ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} --resource-type Microsoft.Kubernetes/connectedClusters --verbose |