From 374db31e089347fe5b72c5eab226c519bdfe2717 Mon Sep 17 00:00:00 2001 From: James Turton <9107319+dzamo@users.noreply.github.com> Date: Tue, 26 Oct 2021 14:41:54 +0200 Subject: [PATCH] DRILL-8016: Default to lazy outbound connections for storage-jdbc and storage-splunk (#2342) * New lazy defaults for HikarCP setting min pool size = 0, max = 10. * Make storage-splunk plugin connect to Splunk lazily. * Remove commented code, enable code based on DRILL-8005. --- .../exec/store/jdbc/JdbcStoragePlugin.java | 28 +++++++++++++++++++ .../resources/bootstrap-storage-plugins.json | 4 +++ .../store/splunk/SplunkSchemaFactory.java | 21 +++++--------- .../exec/store/AbstractStoragePlugin.java | 2 +- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java index daacd0d2dd1..8ae93a38e24 100644 --- a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java +++ b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java @@ -120,6 +120,32 @@ public void close() { static HikariDataSource initDataSource(JdbcStorageConfig config) { try { Properties properties = new Properties(); + + /* + Set default HikariCP values which prefer to connect lazily to avoid overwhelming source + systems with connections which mostly remain idle. A data source that is present in N + storage configs replicated over P drillbits with a HikariCP minimumIdle value of Q will + have N×P×Q connections made to it eagerly. + The trade off of lazier connections is increased latency should there be a spike in user + queries involving a JDBC data source. When comparing the defaults that follow with e.g. the + HikariCP defaults, bear in mind that the context here is OLAP, not OLTP. It is normal + for queries to run for a long time and to be separated by long intermissions. Users who + prefer eager to lazy connections remain free to overwrite the following defaults in their + storage config. + */ + + // maximum amount of time that a connection is allowed to sit idle in the pool, 0 = forever + properties.setProperty("dataSource.idleTimeout", String.format("%d000", 1*60*60)); // 1 hour + // how frequently HikariCP will attempt to keep a connection alive, 0 = disabled + properties.setProperty("dataSource.keepaliveTime", String.format("%d000", 0)); + // maximum lifetime of a connection in the pool, 0 = forever + properties.setProperty("dataSource.maxLifetime", String.format("%d000", 6*60*60)); // 6 hours + // minimum number of idle connections that HikariCP tries to maintain in the pool, 0 = none + properties.setProperty("dataSource.minimumIdle", "0"); + // maximum size that the pool is allowed to reach, including both idle and in-use connections + properties.setProperty("dataSource.maximumPoolSize", "10"); + + // apply any HikariCP parameters the user may have set, overwriting defaults properties.putAll(config.getSourceParameters()); HikariConfig hikariConfig = new HikariConfig(properties); @@ -129,6 +155,8 @@ static HikariDataSource initDataSource(JdbcStorageConfig config) { UsernamePasswordCredentials credentials = config.getUsernamePasswordCredentials(); hikariConfig.setUsername(credentials.getUsername()); hikariConfig.setPassword(credentials.getPassword()); + // this serves as a hint to the driver, which *might* enable database optimizations + hikariConfig.setReadOnly(!config.isWritable()); return new HikariDataSource(hikariConfig); } catch (RuntimeException e) { diff --git a/contrib/storage-jdbc/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-jdbc/src/main/resources/bootstrap-storage-plugins.json index 8d044858a98..dd2fef1f877 100644 --- a/contrib/storage-jdbc/src/main/resources/bootstrap-storage-plugins.json +++ b/contrib/storage-jdbc/src/main/resources/bootstrap-storage-plugins.json @@ -8,6 +8,10 @@ "password": "xxx", "caseInsensitiveTableNames": false, "sourceParameters" : { + "idleTimeout": 3600000, + "keepaliveTime": 0, + "maxLifetime": 21600000, + "minimumIdle": 0, "maximumPoolSize": 10 }, "enabled": false diff --git a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkSchemaFactory.java b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkSchemaFactory.java index fa46ea01a25..bdd71f903e6 100644 --- a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkSchemaFactory.java +++ b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkSchemaFactory.java @@ -18,8 +18,6 @@ package org.apache.drill.exec.store.splunk; -import com.splunk.EntityCollection; -import com.splunk.Index; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.drill.exec.planner.logical.DynamicDrillTable; @@ -40,19 +38,10 @@ public class SplunkSchemaFactory extends AbstractSchemaFactory { private static final Logger logger = LoggerFactory.getLogger(SplunkSchemaFactory.class); private static final String SPL_TABLE_NAME = "spl"; private final SplunkStoragePlugin plugin; - private final EntityCollection indexes; public SplunkSchemaFactory(SplunkStoragePlugin plugin) { super(plugin.getName()); this.plugin = plugin; - SplunkPluginConfig config = plugin.getConfig(); - SplunkConnection connection = new SplunkConnection(config); - - - - // Get Splunk Indexes - connection.connect(); - indexes = connection.getIndexes(); } @Override @@ -110,11 +99,15 @@ private void registerIndexes() { registerTable(SPL_TABLE_NAME, new DynamicDrillTable(plugin, plugin.getName(), new SplunkScanSpec(plugin.getName(), SPL_TABLE_NAME, plugin.getConfig()))); - // Add all other indexes - for (String indexName : indexes.keySet()) { + // Retrieve and add all other Splunk indexes + SplunkPluginConfig config = plugin.getConfig(); + SplunkConnection connection = new SplunkConnection(config); + connection.connect(); + + for (String indexName : connection.getIndexes().keySet()) { logger.debug("Registering {}", indexName); registerTable(indexName, new DynamicDrillTable(plugin, plugin.getName(), - new SplunkScanSpec(plugin.getName(), indexName, plugin.getConfig()))); + new SplunkScanSpec(plugin.getName(), indexName, config))); } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java index 1dcd355ca9c..485e25b158b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java @@ -90,7 +90,7 @@ public Set getPhysicalOptimizerRules(OptimizerRulesContext /** * - * Note: Move this method to {@link StoragePlugin} interface in next major version release. + * TODO: Move this method to {@link StoragePlugin} interface in next major version release. */ public Set getOptimizerRules(OptimizerRulesContext optimizerContext, PlannerPhase phase) { switch (phase) {