From 0bb79de9f6753027fff4fcb30e80de2b40d7714d Mon Sep 17 00:00:00 2001 From: Ohad Revah Date: Mon, 7 Sep 2026 14:10:11 +0300 Subject: [PATCH] Fix observability metrics test failures (#6219) Replace redundant virt-handler pod curl with Prometheus-based label validation. The direct curl was a single-shot check with no retry on conditionally-emitted metrics (e.g. storage_flush_requests_total), and the same metric pipeline is already validated via Prometheus query. Label validation (node, namespace) now runs on the Prometheus response. Fix fixture ordering in TestVmiSyncTotal by adding usefixtures for initial_vmi_sync_total_values on the first incremental test, ensuring the baseline is captured before migration fixtures run. assisted by: claude code claude-opus-4-6 https://redhat.atlassian.net/browse/CNV-96787 - **Tests** - Updated VM metrics tests to validate node and namespace labels directly from Prometheus results. - Added initial synchronization value setup to the VMI synchronization metrics test. - Consolidated metrics validation coverage around label-based checks, removing reliance on virt-handler pod metric retrieval. Signed-off-by: Ohad Co-authored-by: Claude Opus 4.6 (1M context) --- tests/observability/metrics/test_metrics.py | 26 ++--- tests/observability/metrics/utils.py | 111 ++++---------------- 2 files changed, 31 insertions(+), 106 deletions(-) diff --git a/tests/observability/metrics/test_metrics.py b/tests/observability/metrics/test_metrics.py index 592766f722..386062fe80 100644 --- a/tests/observability/metrics/test_metrics.py +++ b/tests/observability/metrics/test_metrics.py @@ -5,8 +5,7 @@ KUBEVIRT_VMI_INFO, ) from tests.observability.metrics.utils import ( - assert_vm_metric, - assert_vm_metric_virt_handler_pod, + assert_vm_metric_labels, compare_kubevirt_vmi_info_metric_with_vm_info, ) from utilities.constants import ( @@ -24,15 +23,14 @@ def test_cnv_vmi_monitoring_metrics_linux_vm( self, admin_client, prometheus, single_metric_vm, cnv_vmi_monitoring_metrics_matrix__function__ ): """ - Tests validating ability to perform various prometheus api queries on various metrics against a given vm. - This test also validates ability to pull metric information from a given vm's virt-handler pod and validates - appropriate information exists for that metrics. + Tests validating ability to perform various prometheus api queries on various metrics against a given vm + and validates appropriate label information (node, namespace) exists for those metrics. """ - assert_vm_metric( - prometheus=prometheus, query=cnv_vmi_monitoring_metrics_matrix__function__, vm_name=single_metric_vm.name - ) - assert_vm_metric_virt_handler_pod( - query=cnv_vmi_monitoring_metrics_matrix__function__, vm=single_metric_vm, admin_client=admin_client + assert_vm_metric_labels( + prometheus=prometheus, + query=cnv_vmi_monitoring_metrics_matrix__function__, + vm=single_metric_vm, + admin_client=admin_client, ) @@ -46,13 +44,11 @@ def test_cnv_vmi_monitoring_metrics_windows_vm( windows_vm_for_test, cnv_vmi_monitoring_metrics_matrix__function__, ): - assert_vm_metric( + assert_vm_metric_labels( prometheus=prometheus, query=cnv_vmi_monitoring_metrics_matrix__function__, - vm_name=windows_vm_for_test.name, - ) - assert_vm_metric_virt_handler_pod( - query=cnv_vmi_monitoring_metrics_matrix__function__, vm=windows_vm_for_test, admin_client=admin_client + vm=windows_vm_for_test, + admin_client=admin_client, ) diff --git a/tests/observability/metrics/utils.py b/tests/observability/metrics/utils.py index 3f17595fd9..6901124442 100644 --- a/tests/observability/metrics/utils.py +++ b/tests/observability/metrics/utils.py @@ -5,7 +5,7 @@ import urllib from contextlib import contextmanager from datetime import datetime, timezone -from typing import Any, Generator, Optional +from typing import Generator, Optional import bitmath from kubernetes.dynamic import DynamicClient @@ -50,7 +50,6 @@ TIMEOUT_30SEC, TIMEOUT_40MIN, USED, - VIRT_HANDLER, Images, ) from utilities.monitoring import get_metrics_value @@ -58,7 +57,6 @@ from utilities.virt import VirtualMachineForTests, running_vm LOGGER = logging.getLogger(__name__) -CURL_QUERY = "curl -k https://localhost:8443/metrics" SINGLE_VM = 1 COUNT_THREE = 3 @@ -99,103 +97,34 @@ def get_vm_metrics(prometheus: Prometheus, query: str, vm_name: str, timeout: in return None -def assert_vm_metric(prometheus: Prometheus, query: str, vm_name: str): - assert get_vm_metrics(prometheus=prometheus, query=query, vm_name=vm_name), ( - f"query: {query} has no result for vm: {vm_name}" - ) - - -def parse_vm_metric_results(raw_output: str) -> dict[str, Any]: - """ - Parse metrics received from virt-handler pod - - Args: - raw_output (str): raw metric output received from virt-handler pods - - Returns: - dict: Dictionary of parsed output - """ - regex_metrics = r"(?P\S+)\{(?P[^\}]+)\}[ ](?P\d+)" - metric_results: dict[str, Any] = {} - for line in raw_output.splitlines(): - if line.startswith("# HELP"): - metric, description = line[7:].split(" ", 1) - metric_results.setdefault(metric, {})["help"] = description - elif line.startswith("# TYPE"): - metric, metric_type = line[7:].split(" ", 1) - metric_results.setdefault(metric, {})["type"] = metric_type - elif re.match(regex_metrics, line): - match = re.match(regex_metrics, line) - if match: - metric_instance_dict = match.groupdict() - metric_instance_dict["labeldict"] = { - val[0]: val[-1] - for val in [label.partition("=") for label in metric_instance_dict["labels"].split(",")] - } - metric_results.setdefault(metric_instance_dict["metric"], {}).setdefault("results", []).append( - metric_instance_dict - ) - else: - metric, metric_type = line.split(" ", 1) - metric_results.setdefault(metric, {})["type"] = metric_type - return metric_results - - -def assert_vm_metric_virt_handler_pod(query: str, vm: VirtualMachineForTests, admin_client: DynamicClient): - """ - Get vm metric information from virt-handler pod - - Args: - query (str): Prometheus query string - vm (VirtualMachineForTests): A VirtualMachineForTests - admin_client (DynamicClient): Admin client for privileged operations - - """ - pod = vm.vmi.get_virt_handler_pod(privileged_client=admin_client) - output = parse_vm_metric_results(raw_output=pod.execute(command=["bash", "-c", f"{CURL_QUERY}"])) - assert output, f'No query output found from {VIRT_HANDLER} pod "{pod.name}" for query: "{CURL_QUERY}"' - metrics_list = [] - if query in output: - metrics_list = [ - result["labeldict"] - for result in output[query]["results"] - if "labeldict" in result and vm.name in result["labeldict"]["name"] - ] - assert metrics_list, ( - f'{VIRT_HANDLER} pod query:"{CURL_QUERY}" did not return any vm metric information for vm: {vm.name} ' - f"from {VIRT_HANDLER} pod: {pod.name}. " - ) - assert_validate_vm_metric(vm=vm, metrics_list=metrics_list, admin_client=admin_client) - - -def assert_validate_vm_metric( - vm: VirtualMachineForTests, metrics_list: list[dict[str, str]], admin_client: DynamicClient +def assert_vm_metric_labels( + prometheus: Prometheus, query: str, vm: VirtualMachineForTests, admin_client: DynamicClient ) -> None: - """ - Validate vm metric information fetched from virt-handler pod + """Validates that Prometheus metric results contain correct node and namespace labels for the VM. Args: - vm (VirtualMachineForTests): A VirtualMachineForTests - metrics_list (list): List of metrics entries collected from associated Virt-handler pod - admin_client (DynamicClient): Admin client for privileged operations - + prometheus: Prometheus client instance. + query: Prometheus query string. + vm: VM to validate metric labels for. + admin_client: Admin client for privileged operations. """ + results = get_vm_metrics(prometheus=prometheus, query=query, vm_name=vm.name) + assert results, f"query: {query} has no result for vm: {vm.name}" + vmi_node = vm.vmi.get_node(privileged_client=admin_client) - expected_values = { - "kubernetes_vmi_label_kubevirt_io_nodeName": vmi_node.name, + vm_results = [result["metric"] for result in results if result["metric"].get("name") == vm.name] + expected_labels = { "namespace": vm.namespace, "node": vmi_node.name, } - LOGGER.info(f"{VIRT_HANDLER} pod metrics associated with vm: {vm.name} are: {metrics_list}") - metric_data_mismatch = [ - entity - for key in expected_values - for entity in metrics_list - if not entity.get(key, None) or expected_values[key] not in entity[key] + label_mismatches = [ + metric + for metric in vm_results + for label, expected_value in expected_labels.items() + if metric.get(label) != expected_value ] - virt_handler_pod = vm.vmi.get_virt_handler_pod(privileged_client=admin_client) - assert not metric_data_mismatch, ( - f"Vm metric validation via {VIRT_HANDLER} pod {virt_handler_pod} failed: {metric_data_mismatch}" + assert not label_mismatches, ( + f"Metric label validation failed for vm {vm.name}. Expected: {expected_labels}, mismatched: {label_mismatches}" )