From 917bdfd915a1508ef39b17bd7defb748950bd017 Mon Sep 17 00:00:00 2001 From: Jean-Francois Dockes Date: Thu, 4 Jan 2024 17:49:58 +0100 Subject: [PATCH] Add max_buffered_input_stream_size configuration variable to allow setting the maximum size of input streams to be fully read in memory --- doc/user.rst | 3 +++ src/config/Option.hxx | 1 + src/config/Templates.cxx | 1 + src/input/BufferedInputStream.cxx | 2 ++ src/input/BufferedInputStream.hxx | 8 ++++++-- src/input/Init.cxx | 13 +++++++++++++ 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/doc/user.rst b/doc/user.rst index 80da9dfda7..ac8c13c308 100644 --- a/doc/user.rst +++ b/doc/user.rst @@ -927,6 +927,9 @@ These settings are various limitations to prevent :program:`MPD` from using too - The maximum size a command list. Default is 2048 (2 MiB). * - **max_output_buffer_size KBYTES** - The maximum size of the output buffer to a client (maximum response size). Default is 8192 (8 MiB). + * - **max_buffered_input_stream_size KBYTES** + - Network streams of known size read from a server with seek ability and smaller than this size will be entirely read in memory. Larger streams are read by chunks, as needed. The default is 128 MiB, which may be too big for small machines. + Buffer Settings ^^^^^^^^^^^^^^^ diff --git a/src/config/Option.hxx b/src/config/Option.hxx index 70fb4f1507..88e5d0eb37 100644 --- a/src/config/Option.hxx +++ b/src/config/Option.hxx @@ -64,6 +64,7 @@ enum class ConfigOption { DESPOTIFY_HIGH_BITRATE, MIXRAMP_ANALYZER, + MAX_BUFFERED_INPUT_STREAM_SIZE, MAX }; diff --git a/src/config/Templates.cxx b/src/config/Templates.cxx index f712efb0f3..fdea9cee8e 100644 --- a/src/config/Templates.cxx +++ b/src/config/Templates.cxx @@ -61,6 +61,7 @@ const ConfigTemplate config_param_templates[] = { { "despotify_password", false, true }, { "despotify_high_bitrate", false, true }, { "mixramp_analyzer" }, + { "max_buffered_input_stream_size" }, }; static constexpr unsigned n_config_param_templates = diff --git a/src/input/BufferedInputStream.cxx b/src/input/BufferedInputStream.cxx index b47687b483..84b94ae0ec 100644 --- a/src/input/BufferedInputStream.cxx +++ b/src/input/BufferedInputStream.cxx @@ -5,6 +5,8 @@ #include +offset_type BufferedInputStream::MAX_SIZE = 128 * 1024 * 1024; + BufferedInputStream::BufferedInputStream(InputStreamPtr _input) :InputStream(_input->GetUriView(), _input->mutex), BufferingInputStream(std::move(_input)) diff --git a/src/input/BufferedInputStream.hxx b/src/input/BufferedInputStream.hxx index a84199a978..11403b4354 100644 --- a/src/input/BufferedInputStream.hxx +++ b/src/input/BufferedInputStream.hxx @@ -8,6 +8,7 @@ #include "BufferingInputStream.hxx" #include +#include /** * A "huge" buffer which remembers the (partial) contents of an @@ -15,8 +16,8 @@ * a "stream"; see IsEligible() for details. */ class BufferedInputStream final : public InputStream, BufferingInputStream { - // TODO: make configurable - static constexpr offset_type MAX_SIZE = 128 * 1024 * 1024; + // Default 128 MB, set in BufferedInputStream.cxx. Configurable, set from Init.cxx + static offset_type MAX_SIZE; public: BufferedInputStream(InputStreamPtr _input); @@ -46,6 +47,9 @@ public: size_t Read(std::unique_lock &lock, void *ptr, size_t size) override; + static void SetMaxSize(offset_type maxsize) noexcept { + MAX_SIZE = maxsize; + } private: /* virtual methods from class BufferingInputStream */ void OnBufferAvailable() noexcept override { diff --git a/src/input/Init.cxx b/src/input/Init.cxx index 8148d2bac6..54a5d88de1 100644 --- a/src/input/Init.cxx +++ b/src/input/Init.cxx @@ -4,9 +4,11 @@ #include "Init.hxx" #include "Registry.hxx" #include "InputPlugin.hxx" +#include "BufferedInputStream.hxx" #include "config/Data.hxx" #include "config/Option.hxx" #include "config/Block.hxx" +#include "config/Parser.hxx" #include "Log.hxx" #include "PluginUnavailable.hxx" #include "lib/fmt/RuntimeError.hxx" @@ -21,6 +23,7 @@ #endif static constexpr Domain input_domain("input"); +static constexpr size_t KILOBYTE = 1024; void input_stream_global_init(const ConfigData &config, EventLoop &event_loop) @@ -29,6 +32,16 @@ input_stream_global_init(const ConfigData &config, EventLoop &event_loop) InitUringInputPlugin(event_loop); #endif + if (auto *param = config.GetParam(ConfigOption::MAX_BUFFERED_INPUT_STREAM_SIZE)) { + offset_type buffer_size = param->With([](const char *s){ + size_t result = ParseSize(s, KILOBYTE); + if (result <= 0) + throw FmtRuntimeError("max buffered input size \"{}\" is not a positive integer", s); + return result; + }); + BufferedInputStream::SetMaxSize(buffer_size); + } + const ConfigBlock empty; for (unsigned i = 0; input_plugins[i] != nullptr; ++i) {