From 660d8f64bf91160436a4f8ff047df2d826ab9c5b Mon Sep 17 00:00:00 2001 From: David Dallakyan Date: Sat, 15 Aug 2026 17:59:37 +0400 Subject: [PATCH 1/2] fix: coerce None filter_rate in Label/Int Filter charts PerformanceCustomDataset never set filter_rate, so Streamlit getRange/sort TypeError'd when the key was present with a None value. Mirror LabelFilterPerformanceCase and treat None as 0 on the chart axis. --- tests/test_custom_dataset_filter.py | 49 +++++++++++++++++++ tests/test_filter_charts.py | 41 ++++++++++++++++ vectordb_bench/backend/cases.py | 2 + .../frontend/components/int_filter/charts.py | 7 +-- .../components/label_filter/charts.py | 7 +-- 5 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 tests/test_custom_dataset_filter.py create mode 100644 tests/test_filter_charts.py diff --git a/tests/test_custom_dataset_filter.py b/tests/test_custom_dataset_filter.py new file mode 100644 index 000000000..a47349c41 --- /dev/null +++ b/tests/test_custom_dataset_filter.py @@ -0,0 +1,49 @@ +import pytest + +from vectordb_bench.backend.cases import CaseType, PerformanceCustomDataset +from vectordb_bench.backend.filter import LabelFilter +from vectordb_bench.models import CaseConfig + + +def _custom_case_kwargs(**overrides): + kwargs = { + "name": "custom-ds", + "description": "", + "load_timeout": 1.0, + "optimize_timeout": 1.0, + "dataset_config": { + "name": "custom-ds", + "dir": "/tmp/custom-ds", + "size": 1000, + "dim": 128, + "metric_type": "L2", + "file_count": 1, + }, + } + kwargs.update(overrides) + return kwargs + + +def test_custom_dataset_filter_sets_filter_rate_from_label_percentage(): + case = PerformanceCustomDataset(**_custom_case_kwargs(use_filter=True, label_percentage=0.01)) + assert case.filter_rate == pytest.approx(0.99) + assert isinstance(case.filters, LabelFilter) + assert case.filters.filter_rate == pytest.approx(0.99) + + +def test_custom_dataset_filter_matches_label_filter_formula(): + case = PerformanceCustomDataset(**_custom_case_kwargs(use_filter=True, label_percentage=0.2)) + assert case.filter_rate == pytest.approx(1.0 - 0.2) + + +def test_custom_dataset_without_filter_leaves_filter_rate_unset(): + case = PerformanceCustomDataset(**_custom_case_kwargs(use_filter=False)) + assert case.filter_rate is None + + +def test_case_config_builds_custom_dataset_with_filter_rate(): + case = CaseConfig( + case_id=CaseType.PerformanceCustomDataset, + custom_case=_custom_case_kwargs(use_filter=True, label_percentage=0.5), + ).case + assert case.filter_rate == pytest.approx(0.5) diff --git a/tests/test_filter_charts.py b/tests/test_filter_charts.py new file mode 100644 index 000000000..9fc79e0ee --- /dev/null +++ b/tests/test_filter_charts.py @@ -0,0 +1,41 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from vectordb_bench.frontend.components.int_filter import charts as int_charts +from vectordb_bench.frontend.components.label_filter import charts as label_charts + +NONE_FILTER_DATA = [ + { + "filter_rate": None, + "qps": 10.0, + "recall": 0.9, + "db_name": "A", + "dataset_name": "custom", + }, + { + "filter_rate": 0.99, + "qps": 20.0, + "recall": 0.8, + "db_name": "B", + "dataset_name": "custom", + }, +] + + +@pytest.mark.parametrize("charts", [label_charts, int_charts]) +def test_get_range_coerces_none_filter_rate(charts): + data = [{"filter_rate": None}, {"filter_rate": 0.5}] + xrange = charts.getRange("filter_rate", data, [0.05, 0.1]) + assert xrange[0] <= 0 + assert xrange[1] >= 0.5 + + +@pytest.mark.parametrize("charts", [label_charts, int_charts]) +def test_draw_chart_does_not_crash_when_filter_rate_is_none(charts): + st = MagicMock() + with patch.object(charts, "px") as px: + px.line.return_value = MagicMock() + charts.drawChart(st, list(NONE_FILTER_DATA), "qps") + px.line.assert_called_once() + st.plotly_chart.assert_called_once() diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 24eaf768e..90b6d6fc7 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -439,6 +439,7 @@ def __init__( gt_neighbors_field=dataset_config.gt_col_name, scalar_labels_file=f"{dataset_config.scalar_labels_name}.parquet", ) + filter_rate = (1.0 - label_percentage) if (use_filter and label_percentage is not None) else None super().__init__( name=name, description=description, @@ -448,6 +449,7 @@ def __init__( dataset=DatasetManager(data=dataset), use_filter=use_filter, label_percentage=label_percentage, + filter_rate=filter_rate, ) @property diff --git a/vectordb_bench/frontend/components/int_filter/charts.py b/vectordb_bench/frontend/components/int_filter/charts.py index 5c32a089e..a830f1c10 100644 --- a/vectordb_bench/frontend/components/int_filter/charts.py +++ b/vectordb_bench/frontend/components/int_filter/charts.py @@ -21,8 +21,9 @@ def drawChartByMetric(st, data, metrics=("qps", "recall"), **kwargs): def getRange(metric, data, padding_multipliers): - minV = min([d.get(metric, 0) for d in data]) - maxV = max([d.get(metric, 0) for d in data]) + values = [d.get(metric) or 0 for d in data] + minV = min(values) + maxV = max(values) padding = maxV - minV rangeV = [ minV - padding * padding_multipliers[0], @@ -39,7 +40,7 @@ def drawChart(st, data: list[object], metric): y = metric yrange = getRange(y, data, [0.2, 0.1]) - data.sort(key=lambda a: a[x]) + data.sort(key=lambda a: a.get(x) or 0) fig = px.line( data, diff --git a/vectordb_bench/frontend/components/label_filter/charts.py b/vectordb_bench/frontend/components/label_filter/charts.py index 5c32a089e..a830f1c10 100644 --- a/vectordb_bench/frontend/components/label_filter/charts.py +++ b/vectordb_bench/frontend/components/label_filter/charts.py @@ -21,8 +21,9 @@ def drawChartByMetric(st, data, metrics=("qps", "recall"), **kwargs): def getRange(metric, data, padding_multipliers): - minV = min([d.get(metric, 0) for d in data]) - maxV = max([d.get(metric, 0) for d in data]) + values = [d.get(metric) or 0 for d in data] + minV = min(values) + maxV = max(values) padding = maxV - minV rangeV = [ minV - padding * padding_multipliers[0], @@ -39,7 +40,7 @@ def drawChart(st, data: list[object], metric): y = metric yrange = getRange(y, data, [0.2, 0.1]) - data.sort(key=lambda a: a[x]) + data.sort(key=lambda a: a.get(x) or 0) fig = px.line( data, From 4b032d22dde3b0b11676929bc42da96fa3a32fc4 Mon Sep 17 00:00:00 2001 From: David Dallakyan Date: Thu, 27 Aug 2026 16:01:56 +0400 Subject: [PATCH 2/2] fix: set filter_rate on Cloud label-filter cases Derive it from LabelFilter so Cloud payload, cold-latency, and multi-tenant runs land on the Label Filter chart at the true rate. --- tests/test_cloud_cold_latency_case.py | 2 ++ tests/test_cloud_payload_case.py | 12 ++++++++++++ tests/test_custom_dataset_filter.py | 2 +- tests/test_multitenant_case.py | 2 ++ vectordb_bench/backend/cases.py | 12 +++++++++++- .../frontend/components/int_filter/charts.py | 1 + .../frontend/components/label_filter/charts.py | 1 + 7 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_cloud_cold_latency_case.py b/tests/test_cloud_cold_latency_case.py index 55f610b38..d38a05b70 100644 --- a/tests/test_cloud_cold_latency_case.py +++ b/tests/test_cloud_cold_latency_case.py @@ -55,7 +55,9 @@ def test_cloud_cold_latency_case_accepts_label_filter(): case = CloudColdLatencyCase(label_percentage=0.9) assert case.label_percentage == 0.9 + assert case.filter_rate == pytest.approx(0.1) assert case.filters.type == FilterOp.StrEqual + assert case.filters.filter_rate == pytest.approx(0.1) def test_cloud_cold_latency_case_rejects_two_filter_types(): diff --git a/tests/test_cloud_payload_case.py b/tests/test_cloud_payload_case.py index d3d97fb60..0c80c83ad 100644 --- a/tests/test_cloud_payload_case.py +++ b/tests/test_cloud_payload_case.py @@ -6,6 +6,7 @@ from vectordb_bench.backend.clients.api import EmptyDBCaseConfig from vectordb_bench.backend.data_source import DatasetSource from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.filter import FilterOp from vectordb_bench.backend.payload import PayloadProfile from vectordb_bench.backend.runner.mp_runner import MultiProcessingSearchRunner from vectordb_bench.backend.runner.serial_runner import SerialSearchRunner @@ -101,6 +102,17 @@ def test_case_runner_reuse_key_distinguishes_scalar_label_schema_requirement(): assert hash(ids_only_runner) != hash(scalar_label_runner) +def test_cloud_payload_label_filter_sets_filter_rate(): + case = CloudPayloadSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereMedium.value, + label_percentage=0.2, + ) + + assert case.filter_rate == pytest.approx(0.8) + assert case.filters.type == FilterOp.StrEqual + assert case.filters.filter_rate == pytest.approx(0.8) + + def test_cli_propagates_cloud_payload_dataset_selection(): custom_case = get_custom_case_config( { diff --git a/tests/test_custom_dataset_filter.py b/tests/test_custom_dataset_filter.py index a47349c41..04e491123 100644 --- a/tests/test_custom_dataset_filter.py +++ b/tests/test_custom_dataset_filter.py @@ -33,7 +33,7 @@ def test_custom_dataset_filter_sets_filter_rate_from_label_percentage(): def test_custom_dataset_filter_matches_label_filter_formula(): case = PerformanceCustomDataset(**_custom_case_kwargs(use_filter=True, label_percentage=0.2)) - assert case.filter_rate == pytest.approx(1.0 - 0.2) + assert case.filter_rate == pytest.approx(LabelFilter(label_percentage=0.2).filter_rate) def test_custom_dataset_without_filter_leaves_filter_rate_unset(): diff --git a/tests/test_multitenant_case.py b/tests/test_multitenant_case.py index 45939c32a..f7a031676 100644 --- a/tests/test_multitenant_case.py +++ b/tests/test_multitenant_case.py @@ -65,7 +65,9 @@ def test_case_config_constructs_multitenant_case(): assert isinstance(case, CloudMultiTenantSearchCase) assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.filter_rate == pytest.approx(0.99) assert case.filters.type == FilterOp.StrEqual + assert case.filters.filter_rate == pytest.approx(0.99) assert case.tenant_labels() == ["tenant_0000", "tenant_0001", "tenant_0002", "tenant_0003", "tenant_0004"] diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 90b6d6fc7..7a21f52ba 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -439,7 +439,11 @@ def __init__( gt_neighbors_field=dataset_config.gt_col_name, scalar_labels_file=f"{dataset_config.scalar_labels_name}.parquet", ) - filter_rate = (1.0 - label_percentage) if (use_filter and label_percentage is not None) else None + filter_rate = ( + LabelFilter(label_percentage=label_percentage).filter_rate + if (use_filter and label_percentage is not None) + else None + ) super().__init__( name=name, description=description, @@ -672,6 +676,8 @@ def __init__( "Cloud leaderboard search envelope case with explicit response payload profile. " f"Payload profile: {payload_profile.value}; dataset: {dataset_name}." ) + if label_percentage is not None: + filter_rate = LabelFilter(label_percentage=label_percentage).filter_rate super().__init__( name=name, description=description, @@ -741,6 +747,8 @@ def __init__( "Cloud leaderboard cold/warm serial latency case with explicit response payload profile. " f"Payload profile: {payload_profile.value}; dataset: {dataset_name}; query count: {query_count}." ) + if label_percentage is not None: + filter_rate = LabelFilter(label_percentage=label_percentage).filter_rate super().__init__( name=name, description=description, @@ -842,6 +850,8 @@ def __init__( raise ValueError(msg) dataset = dataset_with_size_type.get_manager() + if label_percentage is not None: + filter_rate = LabelFilter(label_percentage=label_percentage).filter_rate super().__init__( name=f"Cloud Multi-Tenant Search - {dataset_with_size_type.value}, {tenant_count} tenants", description=( diff --git a/vectordb_bench/frontend/components/int_filter/charts.py b/vectordb_bench/frontend/components/int_filter/charts.py index a830f1c10..4f292b9ef 100644 --- a/vectordb_bench/frontend/components/int_filter/charts.py +++ b/vectordb_bench/frontend/components/int_filter/charts.py @@ -21,6 +21,7 @@ def drawChartByMetric(st, data, metrics=("qps", "recall"), **kwargs): def getRange(metric, data, padding_multipliers): + # dict.get(metric, 0) still returns None when the key is present with a None value. values = [d.get(metric) or 0 for d in data] minV = min(values) maxV = max(values) diff --git a/vectordb_bench/frontend/components/label_filter/charts.py b/vectordb_bench/frontend/components/label_filter/charts.py index a830f1c10..4f292b9ef 100644 --- a/vectordb_bench/frontend/components/label_filter/charts.py +++ b/vectordb_bench/frontend/components/label_filter/charts.py @@ -21,6 +21,7 @@ def drawChartByMetric(st, data, metrics=("qps", "recall"), **kwargs): def getRange(metric, data, padding_multipliers): + # dict.get(metric, 0) still returns None when the key is present with a None value. values = [d.get(metric) or 0 for d in data] minV = min(values) maxV = max(values)