Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bindings/c/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ set(SVS_C_API_SOURCES
src/filtered_search.hpp
src/index.hpp
src/index_builder.hpp
src/leanvec_training_data.hpp
src/storage.hpp
src/threadpool.hpp
src/types_support.hpp
Expand All @@ -45,6 +46,7 @@ set(SVS_C_API_SOURCES
src/svs_c.cpp
src/dispatcher_vamana.cpp
src/dispatcher_dynamic_vamana.cpp
src/leanvec_training_data.cpp
)

add_library(${TARGET_NAME} SHARED
Expand Down
19 changes: 17 additions & 2 deletions bindings/c/docs/C_API_Design.md
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ Defines how vectors are stored in memory, supporting various compression schemes
| **Simple** | FP32, FP16, INT8, UINT8, INT4, UINT4 | Uncompressed storage |
| **SQ** | INT8, UINT8 | Scalar quantization |
| **LVQ** | Primary: INT4/UINT4/INT8/UINT8<br>Residual: VOID/INT4/UINT4/INT8/UINT8 | Locally-adaptive vector quantization |
| **LeanVec** | Dimensions<br>Primary: data type<br>Secondary: data type | LeanVec dimensionality reduced storage |
| **LeanVec** | Dimensions<br>Primary: data type<br>Secondary: data type<br>Optional pre-trained matrices | LeanVec dimensionality reduced storage; matrices computed via PCA at build time, or trained up front (in-distribution or out-of-distribution) |

**Example:**
```c
Expand All @@ -301,11 +301,26 @@ svs_storage_h storage = svs_storage_create_lvq(
SVS_DATA_TYPE_UINT8, SVS_DATA_TYPE_UINT4, err
);

// LeanVec with 128 dimensions
// LeanVec with 128 dimensions; reduction matrices are computed from the
// dataset (PCA) at build time
svs_storage_h storage = svs_storage_create_leanvec(
128, SVS_DATA_TYPE_FLOAT16, SVS_DATA_TYPE_INT8, err
);

// LeanVec from matrices trained up front. Passing training queries learns
// out-of-distribution (OOD) matrices; passing 0/NULL learns in-distribution
// (PCA) matrices. The builder must already be configured with the correct
// dimension and thread pool. The storage's LeanVec dimensions come from the
// training data.
svs_leanvec_training_data_h td = svs_leanvec_training_data_build(
builder, 128, num_vectors, x, num_queries, x_q, err
);
svs_storage_h storage = svs_storage_create_leanvec_trained(
td, SVS_DATA_TYPE_INT4, SVS_DATA_TYPE_INT8, err
);
// The storage holds its own reference to the matrices.
svs_leanvec_training_data_free(td);

// Scalar quantization
svs_storage_h storage = svs_storage_create_sq(SVS_DATA_TYPE_INT8, err);
```
Expand Down
52 changes: 52 additions & 0 deletions bindings/c/include/svs/c/svs_c.h
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,7 @@ typedef struct svs_index_builder* svs_index_builder_h;
typedef struct svs_algorithm* svs_algorithm_h;
typedef struct svs_storage* svs_storage_h;
typedef struct svs_search_params* svs_search_params_h;
typedef struct svs_leanvec_training_data* svs_leanvec_training_data_h;

// Fully defined types; "_t" suffix indicates a fully defined struct
typedef enum svs_error_code svs_error_code_t;
Expand Down Expand Up @@ -643,6 +644,57 @@ SVS_API bool svs_storage_get_kind(
/// @param storage The storage handle to free
SVS_API void svs_storage_free(svs_storage_h storage);

/// @brief Train LeanVec dimensionality-reduction matrices from a data sample
/// @param builder The index builder handle
/// @param leanvec_dims The reduced number of LeanVec dimensions
/// @param num_vectors The number of data vectors in x
/// @param x Pointer to the data vectors [num_vectors x dim] (float array)
/// @param num_queries The number of training queries in x_q (0 for in-distribution)
/// @param x_q Pointer to the training queries [num_queries x dim], or NULL. When
/// provided, matrices are trained out-of-distribution (OOD) using these queries;
/// when num_queries is 0 or x_q is NULL, in-distribution (PCA) matrices are computed.
/// @param out_err An optional error handle to capture errors
/// @return A handle to the trained LeanVec matrices
/// @remarks
/// * The training data build process depends on the index builder's dimension and thread
/// pool configuration, so the builder must be configured with the correct dimension and
/// thread pool before calling this function.
/// * The input arrays are consumed synchronously and are not retained, so the caller may
/// free or modify @p x and @p x_q once this call returns.
/// * The training data built by this function is used to create a LeanVec storage
/// configuration via svs_storage_create_leanvec_trained(). The training data handle may be
/// freed once svs_storage_create_leanvec_trained() returns, as the storage retains a
/// reference to the trained matrices.
SVS_API svs_leanvec_training_data_h svs_leanvec_training_data_build(
svs_index_builder_h builder,
size_t leanvec_dims,
size_t num_vectors,
const float* x,
size_t num_queries,
const float* x_q, /*=NULL*/
svs_error_h out_err /*=NULL*/
);

/// @brief Free the LeanVec training data handle
/// @param training_data The training data handle to free
SVS_API void svs_leanvec_training_data_free(svs_leanvec_training_data_h training_data);

/// @brief Create a LeanVec storage configuration from pre-trained matrices
/// @param training_data The trained LeanVec matrices to use when reducing the data,
/// instead of computing PCA matrices at build time. The number of LeanVec dimensions
/// is taken from the training data. The storage retains a reference to the trained
/// matrices, so the training data handle may be freed once this call returns.
/// @param primary The data type of the primary quantization
/// @param secondary The data type of the secondary quantization
/// @param out_err An optional error handle to capture errors
/// @return A handle to the created LeanVec storage
SVS_API svs_storage_h svs_storage_create_leanvec_trained(
svs_leanvec_training_data_h training_data,
svs_data_type_t primary,
svs_data_type_t secondary,
svs_error_h out_err /*=NULL*/
);

/// @brief Create an index builder configuration
/// @param metric The distance metric to use
/// @param dimension The dimensionality of the vectors
Expand Down
22 changes: 19 additions & 3 deletions bindings/c/src/data_builder/leanvec.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include "svs/c/svs_c.h"

#include "leanvec_training_data.hpp"
#include "storage.hpp"
#include "types_support.hpp"

Expand All @@ -40,17 +41,26 @@
#endif // SVS_LEANVEC_HEADER

#include <filesystem>
#include <optional>
#include <stdexcept>
#include <utility>

namespace svs {

template <size_t I1, size_t I2, typename Allocator = svs::lib::Allocator<std::byte>>
class LeanVecDataBuilder {
size_t leanvec_dims_;
// Pre-trained (e.g. out-of-distribution) matrices; empty for PCA reduction.
std::optional<svs::leanvec::LeanVecMatrices<svs::Dynamic>> matrices_;

public:
LeanVecDataBuilder(size_t leanvec_dims)
: leanvec_dims_(leanvec_dims) {}
: leanvec_dims_(leanvec_dims)
, matrices_(std::nullopt) {}

LeanVecDataBuilder(svs::leanvec::LeanVecMatrices<svs::Dynamic> matrices)
: leanvec_dims_(matrices.num_cols())
, matrices_(std::move(matrices)) {}

using data_type = svs::leanvec::LeanDataset<
svs::leanvec::UsingLVQ<I1>,
Expand All @@ -67,7 +77,7 @@ class LeanVecDataBuilder {
const allocator_type& allocator = {}
) {
return data_type::reduce(
view, std::nullopt, pool, 0, svs::lib::MaybeStatic{leanvec_dims_}, allocator
view, matrices_, pool, 0, svs::lib::MaybeStatic{leanvec_dims_}, allocator
);
}

Expand Down Expand Up @@ -95,7 +105,13 @@ struct lib::

static To convert(From from) {
auto leanvec = static_cast<const c_runtime::StorageLeanVec*>(from);
return To{leanvec->lenavec_dims};
// `leanvec_dims` is taken from the training data at storage construction,
// so it is authoritative in both cases.
if (leanvec->training_data) {
return To{leanvec->training_data->matrices()};
}
assert(leanvec->leanvec_dims > 0);
return To{leanvec->leanvec_dims};
}
};

Expand Down
61 changes: 61 additions & 0 deletions bindings/c/src/leanvec_training_data.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright 2026 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifdef SVS_RUNTIME_ENABLE_LVQ_LEANVEC

#include "leanvec_training_data.hpp"

#include "error.hpp"

#include <svs/core/medioid.h>
#include <svs/lib/static.h>

namespace svs::c_runtime {

LeanVecTrainingData::matrices_type LeanVecTrainingData::compute_matrices(
svs::data::ConstSimpleDataView<float> data,
svs::data::ConstSimpleDataView<float> queries,
size_t leanvec_dims,
svs::threads::ThreadPoolHandle& pool
) {
if (!enabled()) {
throw svs::c_runtime::unsupported_hw(
"LeanVec training is not supported on this hardware"
);
}

// If queries are provided, compute out-of-distribution (OOD) matrices.
if (queries.size() > 0) {
return svs::leanvec::compute_leanvec_matrices_ood<svs::Dynamic>(
data, queries, pool, svs::lib::MaybeStatic{leanvec_dims}
);
}

// Else: PCA path (in-distribution)
auto means = svs::utils::compute_medioid(data, pool);
auto matrix = svs::leanvec::compute_leanvec_matrix<svs::Dynamic, svs::Dynamic>(
data, means, pool, svs::lib::MaybeStatic{leanvec_dims}
);
// A copy is used for the query matrix: in PCA mode data and query
// transforms are identical, and passing the same object twice trips
// use-after-move warnings and DenseArray double-free issues.
auto query_matrix = matrix;
return matrices_type{std::move(matrix), std::move(query_matrix)};
}

} // namespace svs::c_runtime

#endif // SVS_RUNTIME_ENABLE_LVQ_LEANVEC
85 changes: 85 additions & 0 deletions bindings/c/src/leanvec_training_data.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Copyright 2026 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once

#ifdef SVS_RUNTIME_ENABLE_LVQ_LEANVEC

#include "svs/c/svs_c.h"

#include <svs/core/data/simple.h>
#include <svs/cpuid.h>
#include <svs/lib/threads/threadpool.h>

#ifdef SVS_LEANVEC_HEADER
#include SVS_LEANVEC_HEADER
#else
#include <svs/leanvec/leanvec.h>
#endif

#include <cstddef>

namespace svs::c_runtime {

// Holds LeanVec dimensionality-reduction matrices trained from a data sample.
// Mirrors the runtime bindings' LeanVecTrainingData: matrices are computed once
// and later handed to LeanVecDataBuilder to reduce the dataset. When training
// queries are supplied the matrices are learned out-of-distribution (OOD),
// otherwise in-distribution (PCA) matrices are used for both data and queries.
class LeanVecTrainingData {
public:
using matrices_type = svs::leanvec::LeanVecMatrices<svs::Dynamic>;

LeanVecTrainingData(
svs::data::ConstSimpleDataView<float> data,
svs::data::ConstSimpleDataView<float> queries,
size_t leanvec_dims,
svs::threads::ThreadPoolHandle& pool
)
: matrices_{compute_matrices(data, queries, leanvec_dims, pool)} {}

size_t leanvec_dims() const { return matrices_.num_cols(); }
const matrices_type& matrices() const { return matrices_; }

static bool enabled() { return svs::detail::intel_enabled(); }

private:
matrices_type matrices_;

static matrices_type compute_matrices(
svs::data::ConstSimpleDataView<float> data,
svs::data::ConstSimpleDataView<float> queries,
size_t leanvec_dims,
svs::threads::ThreadPoolHandle& pool
);
};

} // namespace svs::c_runtime

#else // SVS_RUNTIME_ENABLE_LVQ_LEANVEC
namespace svs::c_runtime {
class LeanVecTrainingData {
public:
template <typename... Args> LeanVecTrainingData(Args&&...) {
throw svs::c_runtime::not_implemented(
"LeanVec training is not implemented in this build"
);
}

static bool enabled() { return false; }
};
} // namespace svs::c_runtime

#endif // SVS_RUNTIME_ENABLE_LVQ_LEANVEC
24 changes: 21 additions & 3 deletions bindings/c/src/storage.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#endif

#include <filesystem>
#include <memory>
#include <stdexcept>

namespace svs {
Expand All @@ -55,14 +56,20 @@ struct StorageSimple : public Storage {
}
};

class LeanVecTrainingData;
struct StorageLeanVec : public Storage {
size_t lenavec_dims;
size_t leanvec_dims;
size_t primary_bits;
size_t secondary_bits;
// Pre-trained reduction matrices; when set, they are used instead of PCA
// matrices computed at build time (enables out-of-distribution LeanVec).
// For trained storage, dimensions are read directly from these matrices;
// `leanvec_dims` is used only when `training_data` is null.
std::shared_ptr<const LeanVecTrainingData> training_data = nullptr;
Comment thread
Copilot marked this conversation as resolved.

StorageLeanVec(size_t lenavec_dims, svs_data_type_t primary, svs_data_type_t secondary)
StorageLeanVec(size_t leanvec_dims, svs_data_type_t primary, svs_data_type_t secondary)
: Storage{SVS_STORAGE_KIND_LEANVEC}
, lenavec_dims(lenavec_dims)
, leanvec_dims(leanvec_dims)
, primary_bits(to_bits_number(primary))
, secondary_bits(to_bits_number(secondary)) {
#ifdef SVS_RUNTIME_ENABLE_LVQ_LEANVEC
Expand All @@ -78,6 +85,17 @@ struct StorageLeanVec : public Storage {
#endif
}

// Construct from pre-trained matrices. `leanvec_dims` is the single value
// carried by the training data, so no reconciliation is needed.
StorageLeanVec(
std::shared_ptr<const LeanVecTrainingData> training_data,
svs_data_type_t primary,
svs_data_type_t secondary
)
: StorageLeanVec(0, primary, secondary) {
this->training_data = std::move(training_data);
}

static size_t to_bits_number(svs_data_type_t data_type) {
switch (data_type) {
case SVS_DATA_TYPE_INT4:
Expand Down
Loading
Loading