Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable unit tests for Python 2.6 & 3.4 on Github Actions #3296

Open
wants to merge 5 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions azurelinuxagent/common/utils/textutil.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
# Requires Python 2.6+ and Openssl 1.0+

import base64
import hashlib
import re
import struct
import sys
Expand Down Expand Up @@ -385,19 +384,6 @@ def is_str_empty(s):
return is_str_none_or_whitespace(s) or is_str_none_or_whitespace(s.rstrip(' \t\r\n\0'))


def hash_strings(string_list):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hash_strings was being used only by the Monitor thread. It has a dependency on hashlib/OpenSSL. The Monitor thread does not really need a cryptographic hash, so I replaced it with Python's own hash function.

"""
Compute a cryptographic hash of a list of strings

:param string_list: The strings to be hashed
:return: The cryptographic hash (digest) of the strings in the order provided
"""
sha1_hash = hashlib.sha1()
for item in string_list:
sha1_hash.update(item.encode())
return sha1_hash.digest()


def format_memory_value(unit, value):
units = {'bytes': 1, 'kilobytes': 1024, 'megabytes': 1024*1024, 'gigabytes': 1024*1024*1024}

Expand Down
7 changes: 3 additions & 4 deletions azurelinuxagent/ga/monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
from azurelinuxagent.common.protocol.imds import get_imds_client
from azurelinuxagent.common.protocol.util import get_protocol_util
from azurelinuxagent.common.utils.restutil import IOErrorCounter
from azurelinuxagent.common.utils.textutil import hash_strings
from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION
from azurelinuxagent.ga.periodic_operation import PeriodicOperation

Expand Down Expand Up @@ -145,9 +144,9 @@ def log_network_configuration(self):

def _operation(self):
raw_route_list = self.osutil.read_route_table()
digest = hash_strings(raw_route_list)
if digest != self.last_route_table_hash:
self.last_route_table_hash = digest
route_table_hash = ":".join([str(hash(r)) for r in raw_route_list])
if route_table_hash != self.last_route_table_hash:
self.last_route_table_hash = route_table_hash
route_list = self.osutil.get_list_of_routes(raw_route_list)
logger.info("Route table: [{0}]".format(",".join(map(networkutil.RouteEntry.to_json, route_list))))

Expand Down
2 changes: 0 additions & 2 deletions tests/common/osutil/test_default.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,12 @@ def test_valid_routes(self):
'eth0\t10813FA8\tC1BB910A\t000F\t0\t0\t0\tFFFFFFFF\t0\t0\t0 \n' \
'eth0\tFEA9FEA9\tC1BB910A\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0 \n' \
'docker0\t002BA8C0\t00000000\t0001\t0\t0\t10\t00FFFFFF\t0\t0\t0 \n'
known_sha1_hash = b'\x1e\xd1k\xae[\xf8\x9b\x1a\x13\xd0\xbbT\xa4\xe3Y\xa3\xdd\x0b\xbd\xa9'

mo = mock.mock_open(read_data=routing_table)
with patch(open_patch(), mo):
raw_route_list = osutil.DefaultOSUtil().read_route_table()

self.assertEqual(len(raw_route_list), 6)
self.assertEqual(textutil.hash_strings(raw_route_list), known_sha1_hash)

route_list = osutil.DefaultOSUtil().get_list_of_routes(raw_route_list)

Expand Down
16 changes: 0 additions & 16 deletions tests/common/utils/test_text_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
# Requires Python 2.6+ and Openssl 1.0+
#

import hashlib
import unittest

import azurelinuxagent.common.utils.textutil as textutil
Expand Down Expand Up @@ -116,21 +115,6 @@ def test_compress(self):
result = textutil.compress('[stdout]\nHello World\n\n[stderr]\n\n')
self.assertEqual('eJyLLi5JyS8tieXySM3JyVcIzy/KSeHiigaKphYVxXJxAQDAYQr2', result)

def test_hash_empty_list(self):
result = textutil.hash_strings([])
self.assertEqual(b'\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t', result)

def test_hash_list(self):
test_list = ["abc", "123"]
result_from_list = textutil.hash_strings(test_list)

test_string = "".join(test_list)
hash_from_string = hashlib.sha1()
hash_from_string.update(test_string.encode())

self.assertEqual(result_from_list, hash_from_string.digest())
self.assertEqual(hash_from_string.hexdigest(), '6367c48dd193d56ea7b0baad25b19455e529f5ee')

def test_empty_strings(self):
self.assertTrue(textutil.is_str_none_or_whitespace(None))
self.assertTrue(textutil.is_str_none_or_whitespace(' '))
Expand Down
8 changes: 6 additions & 2 deletions tests/lib/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,8 +411,12 @@ def emulate_assertListEqual(self, seq1, seq2, msg=None, seq_type=None):
diffMsg = '\n' + '\n'.join(
difflib.ndiff(pprint.pformat(seq1).splitlines(),
pprint.pformat(seq2).splitlines()))
standardMsg = self._truncateMessage(standardMsg, diffMsg)
msg = self._formatMessage(msg, standardMsg)
# _truncateMessage and _formatMessage are not defined on Python 2.6; output the entire diff in that case
Copy link
Member Author

@narrieta narrieta Jan 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This issue has always been here. The only reason I noticed it is because a test that uses assertListEqual failed for an unrelated issue and this code raised an exception.

if sys.version_info < (2, 7):
msg = standardMsg + "\n****************************************\n" + diffMsg
else:
standardMsg = self._truncateMessage(standardMsg, diffMsg)
msg = self._formatMessage(msg, standardMsg)
self.fail(msg)

def emulate_assertIsInstance(self, obj, object_type, msg=None):
Expand Down
121 changes: 92 additions & 29 deletions tests/python_eol/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,41 +18,104 @@
# * Run unit tests: docker run --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-tests
# * Run tests that require root: docker run --user root --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-sudo-tests
#
FROM ubuntu:16.04
FROM mcr.microsoft.com/mirror/docker/library/ubuntu:24.04
ARG PYTHON_VERSION
LABEL description="Test environment for WALinuxAgent"

SHELL ["/bin/bash", "-c"]

RUN \
apt-get update && \
apt-get -y install curl bzip2 sudo && \
groupadd waagent && \
useradd --shell /bin/bash --create-home -g waagent waagent && \
curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2 && \
tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory / && \
rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2 && \
echo $'\
\n\
cd /home/waagent \n\
source /home/waagent/virtualenv/python'${PYTHON_VERSION}/bin/activate$' \n\
function run-tests { \n\
nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests \n\
} \n\
function run-sudo-tests { \n\
nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py \n\
} \n\
' | tee -a /home/waagent/.profile >> ~/.profile && \
sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile && \
:

#
# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
#
RUN \
if [[ "${PYTHON_VERSION}" == "2.6" ]]; then \
apt-get -y install python3; \
RUN <<..
#
# Install the Python venv
#
apt-get update
apt-get -y install curl bzip2 sudo
groupadd waagent
useradd --shell /bin/bash --create-home -g waagent waagent
curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2
tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory /
chown -R waagent:waagent /home/waagent # The UID:GID in the tarball may not match those of the user, so we need to fix that.
Copy link
Member Author

@narrieta narrieta Jan 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ubuntu 24 has a new user, ubuntu, with PID 1000 so now the PID in the tarball does not match waagent's (which now gets 1001)

rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2

#
# Add the convenience functions to the profiles of waagent and root
#
(cat << ...
cd /home/waagent
source /home/waagent/virtualenv/python${PYTHON_VERSION}/bin/activate
function run-tests {
nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests
}
function run-sudo-tests {
nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py
}
...
) | tee -a /home/waagent/.profile >> ~/.profile
sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile

#
# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
#
if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
apt-get -y install python3
fi
#
# The python 2.6 and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16.
# Modules like hashlib and ssl will fail to import on more recent versions of Ubuntu. The Agent uses classes HTTPSConnection and HTTPS, which depend on the ssl module. Those classes
# are added conditionally on the import of ssl on httplib.py and http/client.py with code similar to:
#
# try:
# import ssl
# except ImportError:
# pass
# else:
# class HTTPSConnection(HTTPConnection):...
# class HTTPS(HTTP):...
# def FakeSocket (sock, sslobj):...
#
# Since the import fails, the classes will be undefined. To work around that, we define dummy items that raise NotImplementedError. The unit tests mock those classes anyway, so the
# actual implementation does not really matter.
#
if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
cat >> /opt/python/2.6.9/lib/python2.6/httplib.py << ...
# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries
class HTTPSConnection(HTTPConnection):
default_port = HTTPS_PORT

def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
raise NotImplementedError()

def connect(self):
raise NotImplementedError()

__all__.append("HTTPSConnection")

class HTTPS(HTTP):
_connection_class = HTTPSConnection

def __init__(self, host='', port=None, key_file=None, cert_file=None, strict=None):
raise NotImplementedError()

def FakeSocket (sock, sslobj):
raise NotImplementedError()
...

elif [[ "${PYTHON_VERSION}" == "3.4" ]]; then
cat >> /opt/python/3.4.8/lib/python3.4/http/client.py << ...
# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries
class HTTPSConnection(HTTPConnection):
default_port = HTTPS_PORT

def __init__(self, host, port=None, key_file=None, cert_file=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, context=None, check_hostname=None):
raise NotImplementedError()

def connect(self):
raise NotImplementedError()

__all__.append("HTTPSConnection")
...
fi
..

USER waagent:waagent

Expand Down
Loading