feat(hologres): make HGraph build params configurable and tune defaults

README.md

@@ -460,7 +460,7 @@ Options:
 
 It is recommended to use the following code for installation.
 ```shell
-pip install 'vectordb-bench[hologres]' 'psycopg[binary]' pgvector
+pip install 'vectordb-bench[hologres]'
 ```
 
 Execute tests for the index types: HGraph.
@@ -491,6 +491,25 @@ Options:
   --ef-search INTEGER             hnsw ef-search  [required]
   --index-type [HGraph]           Type of index to use. Supported values:
                                   HGraph [required]
+  --use-reorder / --no-use-reorder
+                                  use reorder index  [default: use-reorder]
+  --quantization-method [rabitq|sq8_uniform|fp32]
+                                  Base quantization type for the HGraph index.
+                                  Ignored when --no-use-reorder (fp32 is
+                                  forced).  [default: rabitq]
+  --full-compact-max-file-size-mb INTEGER
+                                  Max file size (MB) for full compaction of
+                                  the HGraph index  [default: 16384]
+  --precise-io-type [block_memory_io|reader_io]
+                                  Storage medium for the precise index (only
+                                  effective with --use-reorder).
+                                  block_memory_io: all in memory; reader_io:
+                                  precise index on disk.  [default:
+                                  block_memory_io]
+  --use-extra-column-id / --no-use-extra-column-id
+                                  Embed the primary key 'id' in the index via
+                                  extra_columns to skip base-table lookups
+                                  [default: use-extra-column-id]
   --help                          Show this message and exit.
   ```
 

pyproject.toml

@@ -87,6 +87,7 @@ seekdb          = [ "mysql-connector-python" ]
 volc_mysql      = [ "mysql-connector-python" ]
 pinot           = [ "requests" ]
 adbpg           = [ "psycopg", "psycopg-binary", "pgvector" ]
+hologres        = [ "psycopg", "psycopg-binary" ]
 
 [project.urls]
 Repository  = "https://github.com/zilliztech/VectorDBBench"

vectordb_bench/backend/clients/hologres/cli.py

@@ -21,7 +21,51 @@ class HologresTypedDict(CommonTypedDict):
     port: Annotated[int, click.option("--port", type=int, help="Hologres port", required=True)]
 
 
-class HologresHGraphTypedDict(CommonTypedDict, HologresTypedDict, HNSWFlavor5): ...
+class HologresHGraphTypedDict(CommonTypedDict, HologresTypedDict, HNSWFlavor5):
+    quantization_method: Annotated[
+        str,
+        click.option(
+            "--quantization-method",
+            type=click.Choice(["rabitq", "sq8_uniform", "fp32"], case_sensitive=True),
+            default="rabitq",
+            show_default=True,
+            help="Base quantization type for the HGraph index. Ignored when --no-use-reorder (fp32 is forced).",
+        ),
+    ]
+    full_compact_max_file_size_mb: Annotated[
+        int,
+        click.option(
+            "--full-compact-max-file-size-mb",
+            type=int,
+            default=16384,
+            show_default=True,
+            help="Max file size (MB) for full compaction of the HGraph index",
+        ),
+    ]
+    precise_io_type: Annotated[
+        str,
+        click.option(
+            "--precise-io-type",
+            type=click.Choice(["block_memory_io", "reader_io"], case_sensitive=True),
+            default="block_memory_io",
+            show_default=True,
+            help=(
+                "Storage medium for the precise index (only effective with --use-reorder). "
+                "block_memory_io: all in memory; reader_io: precise index on disk."
+            ),
+        ),
+    ]
+    use_extra_column_id: Annotated[
+        bool,
+        click.option(
+            "--use-extra-column-id/--no-use-extra-column-id",
+            is_flag=True,
+            type=bool,
+            default=True,
+            show_default=True,
+            help="Embed the primary key 'id' in the index via extra_columns to skip base-table lookups",
+        ),
+    ]
 
 
 @cli.command()
@@ -45,6 +89,10 @@ def HologresHGraph(**parameters: Unpack[HologresHGraphTypedDict]):
             ef_construction=parameters["ef_construction"],
             ef_search=parameters["ef_search"],
             use_reorder=parameters["use_reorder"],
+            quantization_method=parameters["quantization_method"],
+            precise_io_type=parameters["precise_io_type"],
+            full_compact_max_file_size_mb=parameters["full_compact_max_file_size_mb"],
+            use_extra_column_id=parameters["use_extra_column_id"],
         ),
         **parameters,
     )

vectordb_bench/backend/clients/hologres/config.py

@@ -32,15 +32,27 @@ class HologresIndexConfig(BaseModel, DBCaseConfig):
     min_flush_proxima_row_count: int = 1000
     min_compaction_proxima_row_count: int = 1000
     max_total_size_to_merge_mb: int = 4096
-    full_compact_max_file_size_mb: int = 4096
+    full_compact_max_file_size_mb: int = 16384
 
-    base_quantization_type: str = "sq8_uniform"
+    # Base quantization type for HGraph index.
+    # Available values: "rabitq", "sq8_uniform", "fp32"
+    # When use_reorder=False, this is ignored and "fp32" is used (no reorder requires full precision).
+    # "rabitq" requires use_reorder=True; rabitq_use_fht is automatically enabled when rabitq is selected.
+    quantization_method: str = "rabitq"
     precise_quantization_type: str = "fp32"
+    # Storage medium for the precise (high-precision) index. Only effective when use_reorder=True.
+    # "block_memory_io": both base and precise indexes in memory.
+    # "reader_io": base index in memory, precise index on disk.
+    precise_io_type: str = "block_memory_io"
     use_reorder: bool = True
     build_thread_count: int = 16
     max_degree: int = 64
     ef_construction: int = 400
 
+    # When True, embeds the primary key ("id") in the HGraph index via extra_columns,
+    # avoiding a base-table lookup during search.
+    use_extra_column_id: bool = True
+
     ef_search: int = 51
 
     def index_param(self) -> dict:
@@ -55,7 +67,7 @@ def search_param(self) -> dict:
         return {
             "distance_function": self.distance_function(),
             "order_direction": self.order_direction(),
-            "searcher_params": self.search_params(),
+            "searcher_params": self.searcher_params(),
         }
 
     def algorithm(self) -> str:
@@ -98,22 +110,29 @@ def order_direction(self) -> str:
         return "ASC"
 
     def builder_params(self) -> dict:
-        if self.use_reorder:
-            self.base_quantization_type = "sq8_uniform"
-        else:
-            self.base_quantization_type = "fp32"
+        base_quantization_type = self.quantization_method if self.use_reorder else "fp32"
 
-        return {
+        params = {
             "max_total_size_to_merge_mb": self.max_total_size_to_merge_mb,
             "build_thread_count": self.build_thread_count,
-            "base_quantization_type": self.base_quantization_type,
+            "base_quantization_type": base_quantization_type,
             "max_degree": self.max_degree,
             "ef_construction": self.ef_construction,
             "precise_quantization_type": self.precise_quantization_type,
             "use_reorder": self.use_reorder,
-            "precise_io_type": "reader_io",
         }
 
+        if self.use_reorder:
+            params["precise_io_type"] = self.precise_io_type
+
+        if base_quantization_type == "rabitq":
+            params["rabitq_use_fht"] = True
+
+        if self.use_extra_column_id:
+            params["extra_columns"] = "id"
+
+        return params
+
     def searcher_params(self) -> dict:
         return {
             "ef_search": self.ef_search,

vectordb_bench/backend/clients/hologres/hologres.py

@@ -263,6 +263,7 @@ def _drop_table(self):
 
     def optimize(self, data_size: int | None = None):
         if self.case_config.create_index_after_load:
+            self._full_compact()
             self._create_index()
         self._full_compact()
         self._analyze()
@@ -297,7 +298,10 @@ def _analyze(self):
 
     def _full_compact(self):
         cursor = self._get_cursor()
-        log.info(f"{self.name} client full compact table : {self.table_name}")
+        log.info(
+            f"{self.name} client full compact table : {self.table_name}, "
+            f"max_file_size_mb={self.case_config.full_compact_max_file_size_mb}"
+        )
         cursor.execute(
             sql.SQL("""
                 SELECT hologres.hg_full_compact_table(
@@ -331,6 +335,7 @@ def _create_index(self):
         )
 
         log.info(f"{self.name} client create index on table : {self.table_name}, with sql: {sql_index.as_string()}")
+        log.info(f"{self.name} builder_params: {json.dumps(self.case_config.builder_params())}")
         try:
             cursor.execute(sql_index)
             conn.commit()