diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index c9b337a7..878eed69 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -2,7 +2,8 @@ import re import sys from enum import Enum -from typing import List, Tuple +from itertools import groupby +from typing import List, Set, Tuple import yaml @@ -62,6 +63,43 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp print("Build output check completed succesfully. No warnings") +def _known_arm64_diversion_warnings(output_lines: List[str]) -> Set[int]: + warning = "dpkg-shlibdeps: warning: diversions involved - output may be incorrect" + source = " diversion by libc6 from: /lib/ld-linux-aarch64.so.1" + target = " diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged" + known_warnings = set() + + # Keep unknown/malformed diversion records in the block so they veto it. + for is_diagnostic, block in groupby( + enumerate(output_lines), + lambda item: item[1] + .lstrip() + .startswith(("dpkg-shlibdeps:", "diversion", "local diversion")), + ): + if not is_diagnostic: + continue + records = list(block) + lines = [line for _, line in records] + if ( + set(lines) != {warning, source, target} + or lines.count(source) != lines.count(target) + or lines.count(warning) != 2 * lines.count(source) + ): + continue + + # Parallel dh_shlibdeps may interleave records, but every detail must + # follow a warning. Only complete libc6 ARM64 usr-merge blocks qualify. + pending_warnings = 0 + for line in lines: + pending_warnings += 1 if line == warning else -1 + if pending_warnings < 0: + break + else: + known_warnings.update(index for index, line in records if line == warning) + + return known_warnings + + def filter_warning_lines( output_lines: List[str], package_type: PackageType ) -> Tuple[List[str], List[str]]: @@ -76,7 +114,10 @@ def filter_warning_lines( package_specific_warning_lines = [] is_deb_warning_line = False is_rpm_warning_line = False - for output_line in output_lines: + known_diversion_warnings = set() + if package_type == PackageType.deb: + known_diversion_warnings = _known_arm64_diversion_warnings(output_lines) + for line_index, output_line in enumerate(output_lines): if package_type == PackageType.deb: if debian_lintian_starter in output_line: is_deb_warning_line = True @@ -87,7 +128,7 @@ def filter_warning_lines( package_specific_warning_lines.append(output_line) else: is_deb_warning_line = False - else: + elif line_index not in known_diversion_warnings: base_warning_lines.append(output_line) else: if rpm_lintian_starter in output_line: diff --git a/packaging_automation/tests/files/packaging_warning/sample_warning_build_output_deb_arm64_diversions.txt b/packaging_automation/tests/files/packaging_warning/sample_warning_build_output_deb_arm64_diversions.txt new file mode 100644 index 00000000..e21bf865 --- /dev/null +++ b/packaging_automation/tests/files/packaging_warning/sample_warning_build_output_deb_arm64_diversions.txt @@ -0,0 +1,26 @@ + dh_shlibdeps -a +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 from: /lib/ld-linux-aarch64.so.1 +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 from: /lib/ld-linux-aarch64.so.1 +dpkg-shlibdeps: warning: diversions involved - output may be incorrect +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged + diversion by libc6 from: /lib/ld-linux-aarch64.so.1 +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 from: /lib/ld-linux-aarch64.so.1 +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 from: /lib/ld-linux-aarch64.so.1 +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 from: /lib/ld-linux-aarch64.so.1 +dpkg-shlibdeps: warning: diversions involved - output may be incorrect + diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged + dh_installdeb -a diff --git a/packaging_automation/tests/test_packaging_warning_handler.py b/packaging_automation/tests/test_packaging_warning_handler.py index 4c4e9bd9..595d0d08 100644 --- a/packaging_automation/tests/test_packaging_warning_handler.py +++ b/packaging_automation/tests/test_packaging_warning_handler.py @@ -15,6 +15,13 @@ ) TEST_BASE_PATH = pathlib2.Path(__file__).parent +IGNORE_FILE = f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml" +DIVERSION_WARNING = ( + "dpkg-shlibdeps: warning: diversions involved - output may be incorrect" +) +DIVERSION_FROM = " diversion by libc6 from: /lib/ld-linux-aarch64.so.1" +DIVERSION_TO = " diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged" +KNOWN_DIVERSION = [DIVERSION_WARNING, DIVERSION_FROM, DIVERSION_WARNING, DIVERSION_TO] def test_parse_ignore_lists(): @@ -205,3 +212,210 @@ def test_validate_output_rpm_success(): f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml", PackageType.rpm, ) + + +@pytest.mark.parametrize( + "lines", + [ + pytest.param(KNOWN_DIVERSION, id="serial"), + pytest.param(KNOWN_DIVERSION * 3, id="repeated"), + pytest.param( + [DIVERSION_WARNING, DIVERSION_WARNING, DIVERSION_TO, DIVERSION_FROM], + id="interleaved", + ), + pytest.param( + KNOWN_DIVERSION + [" dh_gencontrol -a"] + KNOWN_DIVERSION, + id="separate-complete-blocks", + ), + ], +) +def test_known_arm64_diversion(lines): + original_lines = lines.copy() + assert filter_warning_lines(lines, PackageType.deb) == ([], []) + validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb) + assert lines == original_lines + + +def test_known_arm64_diversion_topn_noble_output(): + # Original topn/Noble output, including warning/warning/to/from interleaving. + output = ( + TEST_BASE_PATH + / "files/packaging_warning/sample_warning_build_output_deb_arm64_diversions.txt" + ).read_text( + encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, + errors=DEFAULT_UNICODE_ERROR_HANDLER, + ) + assert filter_warning_lines(output.splitlines(), PackageType.deb) == ([], []) + validate_output(output, IGNORE_FILE, PackageType.deb) + + +@pytest.mark.parametrize( + "source,target", + [ + pytest.param( + DIVERSION_FROM.replace("libc6", "other"), + DIVERSION_TO.replace("libc6", "other"), + id="unknown-owner", + ), + pytest.param( + DIVERSION_FROM.replace("libc6", "libc6:arm64"), + DIVERSION_TO, + id="different-owner-spelling", + ), + pytest.param( + DIVERSION_FROM.replace("diversion by libc6", "local diversion"), + DIVERSION_TO.replace("diversion by libc6", "local diversion"), + id="local-diversion", + ), + pytest.param( + DIVERSION_FROM.replace("aarch64", "x86-64"), + DIVERSION_TO.replace("aarch64", "x86-64"), + id="other-architecture", + ), + pytest.param( + DIVERSION_FROM.replace("/lib/", "/usr/lib/"), + DIVERSION_TO, + id="other-source-path", + ), + pytest.param( + DIVERSION_FROM, + DIVERSION_TO.replace(".usr-is-merged", ".distrib"), + id="other-target-path", + ), + pytest.param("", DIVERSION_TO, id="missing-source"), + pytest.param(DIVERSION_FROM, "", id="missing-target"), + pytest.param(DIVERSION_FROM, DIVERSION_TO + ".extra", id="target-suffix"), + pytest.param(DIVERSION_FROM + " extra", DIVERSION_TO, id="extra-text"), + pytest.param(DIVERSION_FROM.lstrip(), DIVERSION_TO, id="missing-indent"), + pytest.param("\t" + DIVERSION_FROM, DIVERSION_TO, id="extra-indent"), + pytest.param(DIVERSION_FROM, DIVERSION_TO + " ", id="trailing-space"), + pytest.param( + DIVERSION_FROM.replace("from:", "from"), + DIVERSION_TO, + id="malformed-record", + ), + pytest.param( + DIVERSION_FROM, + DIVERSION_TO.replace("to: /lib/ld-linux-aarch64.so.1.usr-is-merged", "to:"), + id="incomplete-record", + ), + ], +) +@pytest.mark.parametrize("interleaved", [False, True]) +def test_unknown_diversion_context(source, target, interleaved): + if interleaved: + lines = [DIVERSION_WARNING, DIVERSION_WARNING, target, source] + else: + lines = [DIVERSION_WARNING, source, DIVERSION_WARNING, target] + assert filter_warning_lines(lines, PackageType.deb) == ( + [DIVERSION_WARNING, DIVERSION_WARNING], + [], + ) + # Valid records elsewhere must not rescue missing or unknown context. + output = "\n".join(KNOWN_DIVERSION + lines + KNOWN_DIVERSION) + with pytest.raises(SystemExit) as error: + validate_output(output, IGNORE_FILE, PackageType.deb) + assert error.value.code == 1 + + +@pytest.mark.parametrize( + "lines", + [ + pytest.param([DIVERSION_WARNING], id="no-context"), + pytest.param([DIVERSION_WARNING, DIVERSION_FROM], id="source-only"), + pytest.param([DIVERSION_WARNING, DIVERSION_TO], id="target-only"), + pytest.param( + [DIVERSION_WARNING, DIVERSION_WARNING, DIVERSION_FROM], + id="truncated-interleaving", + ), + pytest.param([DIVERSION_WARNING, DIVERSION_FROM] * 2, id="unbalanced-source"), + pytest.param([DIVERSION_WARNING, DIVERSION_TO] * 2, id="unbalanced-target"), + pytest.param(KNOWN_DIVERSION + [DIVERSION_WARNING], id="extra-warning"), + pytest.param( + KNOWN_DIVERSION + [DIVERSION_FROM, DIVERSION_TO], + id="extra-details", + ), + pytest.param( + [DIVERSION_FROM, DIVERSION_WARNING, DIVERSION_WARNING, DIVERSION_TO], + id="detail-before-warning", + ), + pytest.param( + [DIVERSION_WARNING, DIVERSION_FROM, DIVERSION_TO, DIVERSION_WARNING], + id="detail-overtakes-warning", + ), + pytest.param( + KNOWN_DIVERSION + + ["dpkg-shlibdeps: warning: cannot find library libunknown.so"], + id="other-shlibdeps-warning", + ), + pytest.param( + KNOWN_DIVERSION + ["dpkg-shlibdeps: error: unknown diagnostic"], + id="extra-diagnostic-type", + ), + pytest.param( + KNOWN_DIVERSION + [" local diversion from: /unknown"], + id="extra-local-diversion", + ), + pytest.param( + KNOWN_DIVERSION + [DIVERSION_WARNING + " extra"], + id="nonliteral-warning", + ), + ], +) +def test_incomplete_or_ambiguous_diversion_block(lines): + assert filter_warning_lines(lines, PackageType.deb) == ( + [line for line in lines if "warning" in line.lower()], + [], + ) + with pytest.raises(SystemExit) as error: + validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb) + assert error.value.code == 1 + + +@pytest.mark.parametrize( + "boundary", + ["", "unrelated output", " dh_gencontrol -a", "compiler: warning: unrelated"], +) +def test_diversion_context_does_not_cross_boundaries(boundary): + lines = [ + DIVERSION_WARNING, + DIVERSION_FROM, + boundary, + DIVERSION_WARNING, + DIVERSION_TO, + ] + base_warnings, _ = filter_warning_lines(lines, PackageType.deb) + assert base_warnings.count(DIVERSION_WARNING) == 2 + with pytest.raises(SystemExit): + validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb) + + +@pytest.mark.parametrize("before", [False, True]) +def test_unrelated_diversion_warning_remains_fatal(before): + missing_context = [DIVERSION_WARNING] + blocks = ( + [missing_context, KNOWN_DIVERSION] + if before + else [KNOWN_DIVERSION, missing_context] + ) + lines = blocks[0] + [" dh_gencontrol -a"] + blocks[1] + assert filter_warning_lines(lines, PackageType.deb) == ([DIVERSION_WARNING], []) + with pytest.raises(SystemExit): + validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb) + + +def test_known_diversion_does_not_ignore_lintian(): + lintian_lines = ["package: W: unknown-warning", "package: E: unknown-error"] + lines = KNOWN_DIVERSION + ["Now running lintian"] + lintian_lines + assert filter_warning_lines(lines, PackageType.deb) == ([], lintian_lines) + with pytest.raises(SystemExit): + validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb) + + +def test_known_diversion_not_ignored_for_rpm(): + assert filter_warning_lines(KNOWN_DIVERSION, PackageType.rpm) == ( + [DIVERSION_WARNING, DIVERSION_WARNING], + [], + ) + with pytest.raises(SystemExit): + validate_output("\n".join(KNOWN_DIVERSION), IGNORE_FILE, PackageType.rpm)