From 0509f61c7dd039fdc8fac80769c87948decb7938 Mon Sep 17 00:00:00 2001 From: Henrik Mohr Date: Tue, 3 Oct 2023 15:02:01 +0200 Subject: [PATCH] Batch db appends (size 128) --- CHANGELOG.md | 4 +++- README.md | 2 +- project.clj | 2 +- src/nd_db/core.clj | 15 ++++++++++----- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b332b0e..571403b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. This change - Timestamps for historical versions - Optimize (speed+size of) low level index format -## [0.9.0-beta6] - 2023-09-20 +## [0.9.0-beta6+7] - 2023-10-03 - Append documents to existing nd-db files (previously v1.0.0) g- Optional end-pointer parameter for versioning @@ -21,6 +21,8 @@ g- Optional end-pointer parameter for versioning - if `nddbmeta` doesn't exist, stop indexing after passed line number - this will create a new `.nddbmeta` file with a hash and metadata reflecting +Multiple documents are automatically written to db (and index) in batches of 128. + ## [0.9.0-beta5] - 2023-04-20 Minor refactoring diff --git a/README.md b/README.md index f7a5b82..9ec0d82 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # nd-db ```clojure -[com.luposlip/nd-db "0.9.0-beta6"] +[com.luposlip/nd-db "0.9.0-beta7"] ``` _Newline Delimited (read-only) Databases!_ diff --git a/project.clj b/project.clj index 9e28e83..e3262f8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.luposlip/nd-db "0.9.0-beta6" +(defproject com.luposlip/nd-db "0.9.0-beta7" :description "Clojure library to use newline delimited files as fast read-only databases." :url "https://github.com/luposlip/nd-db" :license {:name "Apache License, Version 2.0" diff --git a/src/nd_db/core.clj b/src/nd_db/core.clj index 94b2efc..8ca2968 100644 --- a/src/nd_db/core.clj +++ b/src/nd_db/core.clj @@ -180,8 +180,13 @@ meaning DON'T do parallel writes to database..!" (throw (ex-info "Can't write to historical database (when log-limit is set)!" {:log-limit log-limit}))) (let [docs (if (map? doc-or-docs) [doc-or-docs] - doc-or-docs) - docs-stringed (map doc-emitter docs)] - (-> db - (emit-docs (->> docs-stringed (str/join "\n"))) - (ndix/append docs docs-stringed)))) + doc-or-docs)] + (loop [all-docs-part (partition-all 128 docs) + aggr-db db] + (if (empty? all-docs-part) + aggr-db + (let [docs-part-stringed (map doc-emitter (first all-docs-part))] + (recur (rest all-docs-part) + (-> aggr-db + (emit-docs (->> docs-part-stringed (str/join "\n"))) + (ndix/append docs docs-part-stringed))))))))