From b9c63c540e2e17df9271570b1c55d10fdee07aab Mon Sep 17 00:00:00 2001 From: Zijun Yang Date: Fri, 31 Jul 2026 10:41:21 +0800 Subject: [PATCH] feat(hologres): make HGraph build params configurable and tune defaults Expose HGraph index build params via hologreshgraph CLI options: quantization method, precise_io_type, full-compaction max file size, and the extra-column-id optimization, wiring them into HologresIndexConfig. Add a [hologres] pip install extra to pyproject.toml, log full_compact_max_file_size_mb during full compaction, and set defaults to rabitq quantization, full_compact_max_file_size_mb=16384, and use_extra_column_id enabled. --- README.md | 21 +++++++- pyproject.toml | 1 + .../backend/clients/hologres/cli.py | 50 ++++++++++++++++++- .../backend/clients/hologres/config.py | 39 +++++++++++---- .../backend/clients/hologres/hologres.py | 7 ++- 5 files changed, 105 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 847783545..2dbde748a 100644 --- a/README.md +++ b/README.md @@ -460,7 +460,7 @@ Options: It is recommended to use the following code for installation. ```shell -pip install 'vectordb-bench[hologres]' 'psycopg[binary]' pgvector +pip install 'vectordb-bench[hologres]' ``` Execute tests for the index types: HGraph. @@ -491,6 +491,25 @@ Options: --ef-search INTEGER hnsw ef-search [required] --index-type [HGraph] Type of index to use. Supported values: HGraph [required] + --use-reorder / --no-use-reorder + use reorder index [default: use-reorder] + --quantization-method [rabitq|sq8_uniform|fp32] + Base quantization type for the HGraph index. + Ignored when --no-use-reorder (fp32 is + forced). [default: rabitq] + --full-compact-max-file-size-mb INTEGER + Max file size (MB) for full compaction of + the HGraph index [default: 16384] + --precise-io-type [block_memory_io|reader_io] + Storage medium for the precise index (only + effective with --use-reorder). + block_memory_io: all in memory; reader_io: + precise index on disk. [default: + block_memory_io] + --use-extra-column-id / --no-use-extra-column-id + Embed the primary key 'id' in the index via + extra_columns to skip base-table lookups + [default: use-extra-column-id] --help Show this message and exit. ``` diff --git a/pyproject.toml b/pyproject.toml index 9039e8f70..878c0f388 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ seekdb = [ "mysql-connector-python" ] volc_mysql = [ "mysql-connector-python" ] pinot = [ "requests" ] adbpg = [ "psycopg", "psycopg-binary", "pgvector" ] +hologres = [ "psycopg", "psycopg-binary" ] [project.urls] Repository = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/hologres/cli.py b/vectordb_bench/backend/clients/hologres/cli.py index 7b90b006f..fe69c4297 100644 --- a/vectordb_bench/backend/clients/hologres/cli.py +++ b/vectordb_bench/backend/clients/hologres/cli.py @@ -21,7 +21,51 @@ class HologresTypedDict(CommonTypedDict): port: Annotated[int, click.option("--port", type=int, help="Hologres port", required=True)] -class HologresHGraphTypedDict(CommonTypedDict, HologresTypedDict, HNSWFlavor5): ... +class HologresHGraphTypedDict(CommonTypedDict, HologresTypedDict, HNSWFlavor5): + quantization_method: Annotated[ + str, + click.option( + "--quantization-method", + type=click.Choice(["rabitq", "sq8_uniform", "fp32"], case_sensitive=True), + default="rabitq", + show_default=True, + help="Base quantization type for the HGraph index. Ignored when --no-use-reorder (fp32 is forced).", + ), + ] + full_compact_max_file_size_mb: Annotated[ + int, + click.option( + "--full-compact-max-file-size-mb", + type=int, + default=16384, + show_default=True, + help="Max file size (MB) for full compaction of the HGraph index", + ), + ] + precise_io_type: Annotated[ + str, + click.option( + "--precise-io-type", + type=click.Choice(["block_memory_io", "reader_io"], case_sensitive=True), + default="block_memory_io", + show_default=True, + help=( + "Storage medium for the precise index (only effective with --use-reorder). " + "block_memory_io: all in memory; reader_io: precise index on disk." + ), + ), + ] + use_extra_column_id: Annotated[ + bool, + click.option( + "--use-extra-column-id/--no-use-extra-column-id", + is_flag=True, + type=bool, + default=True, + show_default=True, + help="Embed the primary key 'id' in the index via extra_columns to skip base-table lookups", + ), + ] @cli.command() @@ -45,6 +89,10 @@ def HologresHGraph(**parameters: Unpack[HologresHGraphTypedDict]): ef_construction=parameters["ef_construction"], ef_search=parameters["ef_search"], use_reorder=parameters["use_reorder"], + quantization_method=parameters["quantization_method"], + precise_io_type=parameters["precise_io_type"], + full_compact_max_file_size_mb=parameters["full_compact_max_file_size_mb"], + use_extra_column_id=parameters["use_extra_column_id"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/hologres/config.py b/vectordb_bench/backend/clients/hologres/config.py index ccd476556..d41c6522d 100644 --- a/vectordb_bench/backend/clients/hologres/config.py +++ b/vectordb_bench/backend/clients/hologres/config.py @@ -32,15 +32,27 @@ class HologresIndexConfig(BaseModel, DBCaseConfig): min_flush_proxima_row_count: int = 1000 min_compaction_proxima_row_count: int = 1000 max_total_size_to_merge_mb: int = 4096 - full_compact_max_file_size_mb: int = 4096 + full_compact_max_file_size_mb: int = 16384 - base_quantization_type: str = "sq8_uniform" + # Base quantization type for HGraph index. + # Available values: "rabitq", "sq8_uniform", "fp32" + # When use_reorder=False, this is ignored and "fp32" is used (no reorder requires full precision). + # "rabitq" requires use_reorder=True; rabitq_use_fht is automatically enabled when rabitq is selected. + quantization_method: str = "rabitq" precise_quantization_type: str = "fp32" + # Storage medium for the precise (high-precision) index. Only effective when use_reorder=True. + # "block_memory_io": both base and precise indexes in memory. + # "reader_io": base index in memory, precise index on disk. + precise_io_type: str = "block_memory_io" use_reorder: bool = True build_thread_count: int = 16 max_degree: int = 64 ef_construction: int = 400 + # When True, embeds the primary key ("id") in the HGraph index via extra_columns, + # avoiding a base-table lookup during search. + use_extra_column_id: bool = True + ef_search: int = 51 def index_param(self) -> dict: @@ -55,7 +67,7 @@ def search_param(self) -> dict: return { "distance_function": self.distance_function(), "order_direction": self.order_direction(), - "searcher_params": self.search_params(), + "searcher_params": self.searcher_params(), } def algorithm(self) -> str: @@ -98,22 +110,29 @@ def order_direction(self) -> str: return "ASC" def builder_params(self) -> dict: - if self.use_reorder: - self.base_quantization_type = "sq8_uniform" - else: - self.base_quantization_type = "fp32" + base_quantization_type = self.quantization_method if self.use_reorder else "fp32" - return { + params = { "max_total_size_to_merge_mb": self.max_total_size_to_merge_mb, "build_thread_count": self.build_thread_count, - "base_quantization_type": self.base_quantization_type, + "base_quantization_type": base_quantization_type, "max_degree": self.max_degree, "ef_construction": self.ef_construction, "precise_quantization_type": self.precise_quantization_type, "use_reorder": self.use_reorder, - "precise_io_type": "reader_io", } + if self.use_reorder: + params["precise_io_type"] = self.precise_io_type + + if base_quantization_type == "rabitq": + params["rabitq_use_fht"] = True + + if self.use_extra_column_id: + params["extra_columns"] = "id" + + return params + def searcher_params(self) -> dict: return { "ef_search": self.ef_search, diff --git a/vectordb_bench/backend/clients/hologres/hologres.py b/vectordb_bench/backend/clients/hologres/hologres.py index c396e23b0..33a517fdf 100644 --- a/vectordb_bench/backend/clients/hologres/hologres.py +++ b/vectordb_bench/backend/clients/hologres/hologres.py @@ -263,6 +263,7 @@ def _drop_table(self): def optimize(self, data_size: int | None = None): if self.case_config.create_index_after_load: + self._full_compact() self._create_index() self._full_compact() self._analyze() @@ -297,7 +298,10 @@ def _analyze(self): def _full_compact(self): cursor = self._get_cursor() - log.info(f"{self.name} client full compact table : {self.table_name}") + log.info( + f"{self.name} client full compact table : {self.table_name}, " + f"max_file_size_mb={self.case_config.full_compact_max_file_size_mb}" + ) cursor.execute( sql.SQL(""" SELECT hologres.hg_full_compact_table( @@ -331,6 +335,7 @@ def _create_index(self): ) log.info(f"{self.name} client create index on table : {self.table_name}, with sql: {sql_index.as_string()}") + log.info(f"{self.name} builder_params: {json.dumps(self.case_config.builder_params())}") try: cursor.execute(sql_index) conn.commit()