VM Tests: Run customized OS in VM. (#28)

Expand the basic test so that it takes the image produced by image
customizer and then runs it in a VM using QEMU/KVM via libvirt.

test/vmtests/README.md

@@ -1,13 +1,15 @@
 # VM Tests
 
-A test suite that runs the containerized version of the image customizer tool.
+A test suite that runs the containerized version of the image customizer tool and then
+boots the customized images.
 
 ## How to run
 
 Requirements:
 
 - Python3
-- Docker
+- QEMU/KVM
+- libvirt
 
 Steps:
 

test/vmtests/requirements.txt

@@ -1,2 +1,3 @@
 docker == 7.1.0
+libvirt-python == 10.9.0
 pytest == 8.3.3

test/vmtests/vmtests/conftest.py

@@ -7,12 +7,15 @@
 import string
 import tempfile
 from pathlib import Path
-from typing import Generator
+from typing import Generator, List
 
 import docker
+import libvirt  # type: ignore
 import pytest
 from docker import DockerClient
 
+from .utils.closeable import Closeable
+
 SCRIPT_PATH = Path(__file__).parent
 TEST_CONFIGS_DIR = SCRIPT_PATH.joinpath("../../../toolkit/tools/pkg/imagecustomizerlib/testdata")
 
@@ -91,3 +94,35 @@ def docker_client() -> Generator[DockerClient, None, None]:
     yield client
 
     client.close()  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def libvirt_conn() -> Generator[libvirt.virConnect, None, None]:
+    # Connect to libvirt.
+    libvirt_conn_str = f"qemu:///system"
+    libvirt_conn = libvirt.open(libvirt_conn_str)
+
+    yield libvirt_conn
+
+    libvirt_conn.close()
+
+
+# Fixture that will close resources after a test has run, so long as the '--keep-environment' flag is not specified.
+@pytest.fixture(scope="function")
+def close_list(keep_environment: bool) -> Generator[List[Closeable], None, None]:
+    vm_delete_list: List[Closeable] = []
+
+    yield vm_delete_list
+
+    if keep_environment:
+        return
+
+    exceptions = []
+    for vm in reversed(vm_delete_list):
+        try:
+            vm.close()
+        except Exception as ex:
+            exceptions.append(ex)
+
+    if len(exceptions) > 0:
+        raise ExceptionGroup("failed to close resources", exceptions)

test/vmtests/vmtests/test_no_change.py

@@ -1,22 +1,33 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+import os
 from pathlib import Path
+from typing import List
 
+import libvirt  # type: ignore
 from docker import DockerClient
 
 from .conftest import TEST_CONFIGS_DIR
+from .utils import local_client
+from .utils.closeable import Closeable
 from .utils.imagecustomizer import run_image_customizer
+from .utils.libvirt_utils import VmSpec, create_libvirt_domain_xml
+from .utils.libvirt_vm import LibvirtVm
 
 
 def test_no_change(
     docker_client: DockerClient,
     image_customizer_container_url: str,
     core_efi_azl2: Path,
     test_temp_dir: Path,
+    test_instance_name: str,
+    libvirt_conn: libvirt.virConnect,
+    close_list: List[Closeable],
 ) -> None:
     config_path = TEST_CONFIGS_DIR.joinpath("nochange-config.yaml")
     output_image_path = test_temp_dir.joinpath("image.qcow2")
+    diff_image_path = test_temp_dir.joinpath("image-diff.qcow2")
 
     run_image_customizer(
         docker_client,
@@ -26,3 +37,25 @@ def test_no_change(
         "qcow2",
         output_image_path,
     )
+
+    # Create a differencing disk for the VM.
+    # This will make it easier to manually debug what is in the image itself and what was set during first boot.
+    local_client.run(
+        ["qemu-img", "create", "-F", "qcow2", "-f", "qcow2", "-b", str(output_image_path), str(diff_image_path)],
+    ).check_exit_code()
+
+    # Ensure VM can write to the disk file.
+    os.chmod(diff_image_path, 0o666)
+
+    # Create VM.
+    vm_name = test_instance_name
+    domain_xml = create_libvirt_domain_xml(VmSpec(vm_name, 4096, 4, diff_image_path))
+
+    vm = LibvirtVm(vm_name, domain_xml, libvirt_conn)
+    close_list.append(vm)
+
+    # Start VM.
+    vm.start()
+
+    # Wait for VM to boot by waiting for it to request an IP address from the DHCP server.
+    vm.get_vm_ip_address(timeout=30)

test/vmtests/vmtests/utils/closeable.py

@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+from typing import Protocol
+
+
+# Interface for classes that have a 'close' method.
+class Closeable(Protocol):
+    def close(self) -> None:
+        pass

test/vmtests/vmtests/utils/libvirt_utils.py

@@ -0,0 +1,156 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import xml.etree.ElementTree as ET  # noqa: N817
+from pathlib import Path
+from typing import Dict
+
+
+class VmSpec:
+    def __init__(self, name: str, memory_mib: int, core_count: int, os_disk_path: Path):
+        self.name: str = name
+        self.memory_mib: int = memory_mib
+        self.core_count: int = core_count
+        self.os_disk_path: Path = os_disk_path
+
+
+# Create XML definition for a VM.
+def create_libvirt_domain_xml(vm_spec: VmSpec) -> str:
+    domain = ET.Element("domain")
+    domain.attrib["type"] = "kvm"
+
+    name = ET.SubElement(domain, "name")
+    name.text = vm_spec.name
+
+    memory = ET.SubElement(domain, "memory")
+    memory.attrib["unit"] = "MiB"
+    memory.text = str(vm_spec.memory_mib)
+
+    vcpu = ET.SubElement(domain, "vcpu")
+    vcpu.text = str(vm_spec.core_count)
+
+    os_tag = ET.SubElement(domain, "os")
+    os_tag.attrib["firmware"] = "efi"
+
+    os_type = ET.SubElement(os_tag, "type")
+    os_type.text = "hvm"
+
+    firmware = ET.SubElement(domain, "firmware")
+    firmware.attrib["secure-boot"] = "yes"
+    firmware.attrib["enrolled-keys"] = "yes"
+
+    features = ET.SubElement(domain, "features")
+
+    ET.SubElement(features, "acpi")
+
+    ET.SubElement(features, "apic")
+
+    cpu = ET.SubElement(domain, "cpu")
+    cpu.attrib["mode"] = "host-passthrough"
+
+    clock = ET.SubElement(domain, "clock")
+    clock.attrib["offset"] = "utc"
+
+    on_poweroff = ET.SubElement(domain, "on_poweroff")
+    on_poweroff.text = "destroy"
+
+    on_reboot = ET.SubElement(domain, "on_reboot")
+    on_reboot.text = "restart"
+
+    on_crash = ET.SubElement(domain, "on_crash")
+    on_crash.text = "destroy"
+
+    devices = ET.SubElement(domain, "devices")
+
+    serial = ET.SubElement(devices, "serial")
+    serial.attrib["type"] = "pty"
+
+    serial_target = ET.SubElement(serial, "target")
+    serial_target.attrib["type"] = "isa-serial"
+    serial_target.attrib["port"] = "0"
+
+    serial_target_model = ET.SubElement(serial_target, "model")
+    serial_target_model.attrib["name"] = "isa-serial"
+
+    console = ET.SubElement(devices, "console")
+    console.attrib["type"] = "pty"
+
+    console_target = ET.SubElement(console, "target")
+    console_target.attrib["type"] = "serial"
+    console_target.attrib["port"] = "0"
+
+    video = ET.SubElement(devices, "video")
+
+    video_model = ET.SubElement(video, "model")
+    video_model.attrib["type"] = "qxl"
+
+    graphics = ET.SubElement(devices, "graphics")
+    graphics.attrib["type"] = "spice"
+
+    network_interface = ET.SubElement(devices, "interface")
+    network_interface.attrib["type"] = "network"
+
+    network_interface_source = ET.SubElement(network_interface, "source")
+    network_interface_source.attrib["network"] = "default"
+
+    network_interface_model = ET.SubElement(network_interface, "model")
+    network_interface_model.attrib["type"] = "virtio"
+
+    next_disk_indexes: Dict[str, int] = {}
+    _add_disk_xml(
+        devices,
+        str(vm_spec.os_disk_path),
+        "disk",
+        "qcow2",
+        "virtio",
+        next_disk_indexes,
+    )
+
+    xml = ET.tostring(domain, "unicode")
+    return xml
+
+
+# Adds a disk to a libvirt domain XML document.
+def _add_disk_xml(
+    devices: ET.Element,
+    file_path: str,
+    device_type: str,
+    image_type: str,
+    bus_type: str,
+    next_disk_indexes: Dict[str, int],
+) -> None:
+    device_name = _gen_disk_device_name("vd", next_disk_indexes)
+
+    disk = ET.SubElement(devices, "disk")
+    disk.attrib["type"] = "file"
+    disk.attrib["device"] = device_type
+
+    disk_driver = ET.SubElement(disk, "driver")
+    disk_driver.attrib["name"] = "qemu"
+    disk_driver.attrib["type"] = image_type
+
+    disk_target = ET.SubElement(disk, "target")
+    disk_target.attrib["dev"] = device_name
+    disk_target.attrib["bus"] = bus_type
+
+    disk_source = ET.SubElement(disk, "source")
+    disk_source.attrib["file"] = file_path
+
+
+def _gen_disk_device_name(prefix: str, next_disk_indexes: Dict[str, int]) -> str:
+    disk_index = next_disk_indexes.get(prefix, 0)
+    next_disk_indexes[prefix] = disk_index + 1
+
+    match prefix:
+        case "vd" | "sd":
+            # The disk device name is required to follow the standard Linux device naming
+            # scheme. That is: [ sda, sdb, ..., sdz, sdaa, sdab, ... ]. However, it is
+            # unlikely that someone will ever need more than 26 disks. So, keep it simple
+            # for now.
+            if disk_index < 0 or disk_index > 25:
+                raise Exception(f"Unsupported disk index: {disk_index}.")
+            suffix = chr(ord("a") + disk_index)
+            return f"{prefix}{suffix}"
+
+        case _:
+            return f"{prefix}{disk_index}"