Create Azure Pipeline for Python 2.6 & 3.4 Unit Tests (#3284)

---------

Co-authored-by: narrieta@microsoft <narrieta>

.github/workflows/ci_pr.yml

@@ -8,49 +8,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  test-python-2_6-and-3_4-versions:
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - python-version: "2.6"
-          - python-version: "3.4"
-
-    name: "Python ${{ matrix.python-version }} Unit Tests"
-    runs-on: ubuntu-20.04
-    container:
-      image: ubuntu:16.04
-      volumes:
-        - /home/waagent:/home/waagent
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    env:
-      NOSEOPTS: "--verbose"
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install Python ${{ matrix.python-version }}
-      run: |
-        apt-get update
-        apt-get install -y curl bzip2 sudo python3
-        curl https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 -o python-${{ matrix.python-version }}.tar.bz2
-        sudo tar xjvf python-${{ matrix.python-version }}.tar.bz2 --directory /
-
-    - name: Test with nosetests
-      run: |
-        if [[ ${{ matrix.python-version }} == "2.6" ]]; then
-          source /home/waagent/virtualenv/python2.6.9/bin/activate
-        else
-          source /home/waagent/virtualenv/python3.4.8/bin/activate
-        fi
-        ./ci/nosetests.sh
-        exit $?
-
   test-python-2_7:
 
     strategy:

azurelinuxagent/ga/cgroupconfigurator.py

@@ -437,7 +437,7 @@ def _get_current_cpu_quota(unit_name):
 
                 # Calculate CPU percentage
                 cpu_percentage = (cpu_quota_us / 1000000) * 100
-                return "{:g}%".format(cpu_percentage)  # :g Removes trailing zeros after decimal point
+                return "{0:g}%".format(cpu_percentage)  # :g Removes trailing zeros after decimal point
             except Exception as e:
                 log_cgroup_warning("Error parsing current CPUQuotaPerSecUSec: {0}".format(ustr(e)))
                 return "unknown"

tests/data/ext/sample_ext-1.3.0/python.sh

@@ -4,7 +4,7 @@
 #
 python=$(command -v python 2> /dev/null)
 
-if [ -z "$PYTHON" ]; then
+if [ -z "$python" ]; then
   python=$(command -v python3)
 fi
 

tests/ga/test_cgroupapi.py

@@ -301,7 +301,7 @@ def mock_popen(command, *args, **kwargs):
                         shell=True,
                         timeout=300,
                         cwd=self.tmp_dir,
-                        env={},
+                        env={}.update(os.environ),
                         stdout=output_file,
                         stderr=output_file)
 
@@ -317,7 +317,7 @@ def test_start_extension_cgroups_v1_command_should_execute_the_command_in_a_cgro
                 shell=False,
                 timeout=300,
                 cwd=self.tmp_dir,
-                env={},
+                env={}.update(os.environ),
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)
 
@@ -344,7 +344,7 @@ def test_start_extension_cgroups_v1_command_should_use_systemd_to_execute_the_co
                     timeout=300,
                     shell=True,
                     cwd=self.tmp_dir,
-                    env={},
+                    env={}.update(os.environ),
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE)
 

tests/ga/test_cgroupconfigurator.py

@@ -354,7 +354,7 @@ def test_start_extension_command_should_not_use_systemd_when_cgroups_are_not_ena
                     timeout=300,
                     shell=False,
                     cwd=self.tmp_dir,
-                    env={},
+                    env={}.update(os.environ),
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE)
 
@@ -374,7 +374,7 @@ def test_start_extension_command_should_use_systemd_run_when_cgroups_v1_are_enab
                     timeout=300,
                     shell=False,
                     cwd=self.tmp_dir,
-                    env={},
+                    env={}.update(os.environ),
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE)
 
@@ -395,7 +395,7 @@ def test_start_extension_command_should_start_tracking_the_extension_cgroups(sel
                 timeout=300,
                 shell=False,
                 cwd=self.tmp_dir,
-                env={},
+                env={}.update(os.environ),
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)
 
@@ -426,7 +426,7 @@ def mock_popen(command_arg, *args, **kwargs):
                         timeout=300,
                         shell=False,
                         cwd=self.tmp_dir,
-                        env={},
+                        env={}.update(os.environ),
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
 
@@ -446,7 +446,7 @@ def test_start_extension_command_should_not_use_systemd_when_cgroup_v2_enabled(s
                         timeout=300,
                         shell=False,
                         cwd=self.tmp_dir,
-                        env={},
+                        env={}.update(os.environ),
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
 
@@ -480,7 +480,7 @@ def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_d
                             timeout=300,
                             shell=True,
                             cwd=self.tmp_dir,
-                            env={},
+                            env={}.update(os.environ),
                             stdout=output_file,
                             stderr=output_file)
 
@@ -526,7 +526,7 @@ def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_d
                             timeout=300,
                             shell=True,
                             cwd=self.tmp_dir,
-                            env={},
+                            env={}.update(os.environ),
                             stdout=stdout,
                             stderr=stderr)
 
@@ -571,7 +571,7 @@ def mock_popen(command, *args, **kwargs):
                         timeout=300,
                         shell=True,
                         cwd=self.tmp_dir,
-                        env={},
+                        env={}.update(os.environ),
                         stdout=stdout,
                         stderr=stderr)
 
@@ -856,7 +856,7 @@ def mock_popen(command, *args, **kwargs):
             # For the agent's processes, we use the current process and its parent (in the actual agent these would be the daemon and the extension
             # handler), and the commands started by the agent.
             #
-            # For other processes, we use process 1, a process that already completed, and an extension. Note that extensions are started using
+            # For other processes, we use a process that already completed, and an extension process. Note that extensions are started using
             # systemd-run and the process for that commands belongs to the agent's cgroup but the processes for the extension should be in a
             # different cgroup
             #
@@ -868,7 +868,7 @@ def get_completed_process():
                 return completed
 
             agent_processes = [os.getppid(), os.getpid()] + agent_command_processes + [start_extension.systemd_run_pid]
-            other_processes = [1, get_completed_process()] + extension_processes
+            other_processes = [get_completed_process()] + extension_processes
 
             with patch("azurelinuxagent.ga.cgroupapi.CgroupV1.get_processes", return_value=agent_processes + other_processes):
                 with self.assertRaises(CGroupsException) as context_manager:

tests/python_eol/Dockerfile

@@ -0,0 +1,60 @@
+#
+# Environment to execute the WALinuxAgent unit tests for some versions of Python that have reached EOL and are no longer available
+# in the official repositories.
+#
+# To build the image, set the PYTHON_VERSION argument to 2.6 or 3.4:
+#
+#    * docker build -t python2.6 --build-arg PYTHON_VERSION=2.6 .
+#    * docker build -t python3.4 --build-arg PYTHON_VERSION=3.4 .
+#
+# We add a couple of convenience functions to execute the unit tests to the profiles of waagent and root; these can be useful in interactive sessions. Note
+# that these functions assume the root of the source code has been mounted at /home/waagent/WALinuxAgent.
+#
+# Also, we precede "mesg n" with "tty -s" in root's profile to avoid the "standard input is not a tty" message when not running the container interactively.
+#
+# Sample commands:
+#
+#    * Start an interactive session: docker run --rm -it -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login
+#    * Run unit tests:               docker run --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-tests
+#    * Run tests that require root:  docker run --user root --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-sudo-tests
+#
+FROM ubuntu:16.04
+ARG PYTHON_VERSION
+LABEL description="Test environment for WALinuxAgent"
+
+SHELL ["/bin/bash", "-c"]
+
+RUN \
+    apt-get update && \
+    apt-get -y install curl bzip2 sudo && \
+    groupadd waagent && \
+    useradd --shell /bin/bash --create-home -g waagent waagent && \
+    curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2 && \
+    tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory / && \
+    rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2 && \
+    echo $'\
+        \n\
+        cd /home/waagent \n\
+        source /home/waagent/virtualenv/python'${PYTHON_VERSION}/bin/activate$' \n\
+        function run-tests { \n\
+            nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests \n\
+        } \n\
+        function run-sudo-tests { \n\
+            nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py \n\
+        } \n\
+        ' | tee -a /home/waagent/.profile >> ~/.profile && \
+    sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile && \
+    :
+
+#
+# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
+#
+RUN \
+    if [[ "${PYTHON_VERSION}" == "2.6" ]]; then \
+      apt-get -y install python3; \
+    fi
+
+USER waagent:waagent
+
+
+

tests/python_eol/execute_tests.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then
+  echo "Usage: execute_tests.sh 2.6|3.4"
+  exit 1
+fi
+
+EXIT_CODE=0
+PYTHON_VERSION=$1
+CONTAINER_IMAGE="waagenttests.azurecr.io/python$PYTHON_VERSION"
+CONTAINER_LOGS_DIRECTORY="/home/waagent/logs"
+CONTAINER_SOURCES_DIRECTORY="/home/waagent/WALinuxAgent"
+NOSETESTS_OPTIONS="--verbose --with-xunit"
+
+#
+# Give ownership of the logs directory to 'waagent' (UID 1000)
+#
+sudo chown 1000 "$LOGS_DIRECTORY"
+
+#
+# Give the current user access to the Docker daemon
+#
+sudo usermod -aG docker $USER
+newgrp docker < /dev/null
+
+#
+# Pull the container image and execute the tests
+#
+az acr login --name waagenttests --username "$CR_USER" --password "$CR_SECRET"
+
+docker pull "$CONTAINER_IMAGE"
+
+printf "\n***************************************** Running tests for Python $PYTHON_VERSION *****************************************\n\n"
+
+TEST_SUITE_OPTIONS="--xunit-testsuite-name='Python $PYTHON_VERSION' --xunit-file=$CONTAINER_LOGS_DIRECTORY/waagent-$PYTHON_VERSION.junit.xml"
+
+set -x
+docker run --rm \
+    --volume "$BUILD_SOURCESDIRECTORY":"$CONTAINER_SOURCES_DIRECTORY" \
+    --volume "$LOGS_DIRECTORY":"$CONTAINER_LOGS_DIRECTORY" \
+    "$CONTAINER_IMAGE" \
+    bash --login -c "nosetests $NOSETESTS_OPTIONS $TEST_SUITE_OPTIONS --ignore-files test_cgroupconfigurator_sudo.py $CONTAINER_SOURCES_DIRECTORY/tests" \
+|| EXIT_CODE=$(($EXIT_CODE || $?))
+set +x
+
+printf "\n************************************** Running tests for Python $PYTHON_VERSION [sudo] **************************************\n\n"
+
+TEST_SUITE_OPTIONS="--xunit-testsuite-name='Python $PYTHON_VERSION [sudo]' --xunit-file=$CONTAINER_LOGS_DIRECTORY/waagent-sudo-$PYTHON_VERSION.junit.xml"
+
+set -x
+docker run --rm \
+    --user root \
+    --volume "$BUILD_SOURCESDIRECTORY":"$CONTAINER_SOURCES_DIRECTORY" \
+    --volume "$LOGS_DIRECTORY":"$CONTAINER_LOGS_DIRECTORY" \
+    "$CONTAINER_IMAGE" \
+    bash --login -c "nosetests $NOSETESTS_OPTIONS $TEST_SUITE_OPTIONS $CONTAINER_SOURCES_DIRECTORY/tests/ga/test_cgroupconfigurator_sudo.py"\
+|| EXIT_CODE=$(($EXIT_CODE || $?))
+set +x
+
+exit "$EXIT_CODE"

tests/python_eol/pipeline.yml

@@ -0,0 +1,63 @@
+parameters:
+  - name: python_2_6
+    displayName: Python 2.6
+    type: boolean
+    default: true
+  - name: python_3_4
+    displayName: Python 3.4
+    type: boolean
+    default: true
+
+pool:
+  name: waagent-pool
+
+jobs:
+  - job: SelectPythonVersions
+    displayName: "Select Python versions"
+    steps:
+      - bash: |
+          # Create the test matrix, which is a JSON object with the selected Python versions, e.g. { "Python_2_6":{"VERSION":"2.6"}, "Python_3_4":{"VERSION":"3.4"} }
+          declare -a PYTHON_VERSIONS=()
+          if [ ${{ parameters.python_2_6 }} == "True" ]; then
+            PYTHON_VERSIONS+=('"Python_2_6":{"VERSION":"2.6"}')
+          fi
+          if [ ${{ parameters.python_3_4 }} == "True" ]; then
+            PYTHON_VERSIONS+=('"Python_3_4": {"VERSION":"3.4"}')
+          fi
+          PYTHON_VERSIONS=$(echo ${PYTHON_VERSIONS[@]} | sed 's/ /, /' | sed 's/.*/{ \0 }/')
+          echo "Python versions: $PYTHON_VERSIONS"
+          echo "##vso[task.setvariable variable=PYTHON_VERSIONS;isOutput=true]$PYTHON_VERSIONS"
+        name: "SetPythonVersions"
+
+  - job: "ExecuteTests"
+    displayName: "Execute tests"
+    dependsOn: SelectPythonVersions
+    timeoutInMinutes: 15
+    strategy:
+      matrix: $[ dependencies.SelectPythonVersions.outputs['SetPythonVersions.PYTHON_VERSIONS'] ]
+    steps:
+      - task: AzureKeyVault@2
+        displayName: "Fetch connection info"
+        inputs:
+          azureSubscription: $(connection_info)
+          KeyVaultName: 'waagenttests'
+          SecretsFilter: 'CR-USER, CR-SECRET'
+
+      - bash: |
+          mkdir $(Agent.TempDirectory)/logs
+          $(Build.SourcesDirectory)/tests/python_eol/execute_tests.sh $(VERSION)
+        displayName: "Execute tests"
+        continueOnError: true
+        env:
+          CR_USER: $(CR-USER)
+          CR_SECRET: $(CR-SECRET)
+          LOGS_DIRECTORY: $(Agent.TempDirectory)/logs
+
+      - task: PublishTestResults@2
+        displayName: 'Publish test results'
+        condition: always()
+        inputs:
+          testResultsFormat: 'JUnit'
+          testResultsFiles: 'waagent*.junit.xml'
+          searchFolder: $(Agent.TempDirectory)/logs
+          failTaskOnFailedTests: true