Remove SQL for (most) Ledger State (#4575)

# Description

Resolves #4503 and #4502

This change removes SQL support for all ledger state, except for the
offers table. It also enforces background eviction. In order to not
break Horizon, core will not crash if any of the old flags or config
settings are used. Instead it will print a warning and ignore the flags.
The flags should be removed in the next protocol release.

# Checklist
- [x] Reviewed the
[contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#submitting-changes)
document
- [x] Rebased on top of master (no merge commits)
- [x] Ran `clang-format` v8.0.0 (via `make format` or the Visual Studio
extension)
- [x] Compiles
- [x] Ran all tests
- [ ] If change impacts performance, include supporting evidence per the
[performance
document](https://github.com/stellar/stellar-core/blob/master/performance-eval/performance-eval.md)

docs/stellar-core_example.cfg

@@ -229,14 +229,6 @@ FLOOD_DEMAND_BACKOFF_DELAY_MS = 500
 #   against each other.
 MAX_DEX_TX_OPERATIONS_IN_TX_SET = 0
 
-# DEPRECATED_SQL_LEDGER_STATE (bool) default false
-# When set to true, SQL is used to store all ledger state instead of
-# BucketListDB. This is not recommended and may cause performance degregradation.
-# This is deprecated and will be removed in the future. Note that offers table
-# is still maintained in SQL when this is set to false, but all other ledger
-# state tables are dropped.
-DEPRECATED_SQL_LEDGER_STATE = false
-
 # BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT (Integer) default 14
 # Determines page size used by BucketListDB for range indexes, where
 # pageSize == 2^BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT. If set to
@@ -258,11 +250,6 @@ BUCKETLIST_DB_INDEX_CUTOFF = 20
 # this value is ingnored and indexes are never persisted.
 BUCKETLIST_DB_PERSIST_INDEX = true
 
-# BACKGROUND_EVICTION_SCAN (bool) default true
-# Determines whether eviction scans occur in the background thread. Requires
-# that DEPRECATED_SQL_LEDGER_STATE is set to false.
-BACKGROUND_EVICTION_SCAN = true
-
 # EXPERIMENTAL_BACKGROUND_OVERLAY_PROCESSING (bool) default false
 # Determines whether some of overlay processing occurs in the background
 # thread.

docs/stellar-core_example_validators.cfg

@@ -4,7 +4,6 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Example configuration"
 
 DATABASE="sqlite3://example.db"
-DEPRECATED_SQL_LEDGER_STATE = false
 
 NODE_SEED="SA7FGJMMUIHNE3ZPI2UO5I632A7O5FBAZTXFAIEVFA4DSSGLHXACLAIT a3"
 NODE_HOME_DOMAIN="domainA"

docs/stellar-core_standalone.cfg

@@ -12,7 +12,6 @@ NODE_IS_VALIDATOR=true
 
 #DATABASE="postgresql://dbname=stellar user=postgres password=password host=localhost"
 DATABASE="sqlite3://stellar.db"
-DEPRECATED_SQL_LEDGER_STATE = false
 
 COMMANDS=["ll?level=debug"]
 

docs/stellar-core_testnet.cfg

@@ -4,7 +4,6 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 
 DATABASE="sqlite3://stellar.db"
-DEPRECATED_SQL_LEDGER_STATE = false
 
 # Stellar Testnet validators
 [[HOME_DOMAINS]]

docs/stellar-core_testnet_legacy.cfg

@@ -9,7 +9,6 @@ KNOWN_PEERS=[
 "core-testnet3.stellar.org"]
 
 DATABASE="sqlite3://stellar.db"
-DEPRECATED_SQL_LEDGER_STATE = false
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 

docs/stellar-core_testnet_validator.cfg

@@ -4,7 +4,6 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 
 DATABASE="sqlite3://stellar.db"
-DEPRECATED_SQL_LEDGER_STATE = false
 
 # Configuring the node as a validator
 # note that this is an unsafe configuration in this particular setup:

src/bucket/BucketApplicator.cpp

@@ -9,6 +9,7 @@
 #include "ledger/LedgerTxn.h"
 #include "ledger/LedgerTxnEntry.h"
 #include "main/Application.h"
+#include "util/GlobalChecks.h"
 #include "util/Logging.h"
 #include "util/types.h"
 #include <fmt/format.h>
@@ -21,14 +22,12 @@ BucketApplicator::BucketApplicator(Application& app,
                                    uint32_t minProtocolVersionSeen,
                                    uint32_t level,
                                    std::shared_ptr<LiveBucket const> bucket,
-                                   std::function<bool(LedgerEntryType)> filter,
                                    std::unordered_set<LedgerKey>& seenKeys)
     : mApp(app)
     , mMaxProtocolVersion(maxProtocolVersion)
     , mMinProtocolVersionSeen(minProtocolVersionSeen)
     , mLevel(level)
     , mBucketIter(bucket)
-    , mEntryTypeFilter(filter)
     , mSeenKeys(seenKeys)
 {
     auto protocolVersion = mBucketIter.getMetadata().ledgerVersion;
@@ -40,8 +39,8 @@ BucketApplicator::BucketApplicator(Application& app,
             protocolVersion, mMaxProtocolVersion));
     }
 
-    // Only apply offers if BucketListDB is enabled
-    if (mApp.getConfig().isUsingBucketListDB() && !bucket->isEmpty())
+    // Only apply offers
+    if (!bucket->isEmpty())
     {
         auto offsetOp = bucket->getOfferRange();
         if (offsetOp)
@@ -62,10 +61,8 @@ BucketApplicator::operator bool() const
 {
     // There is more work to do (i.e. (bool) *this == true) iff:
     // 1. The underlying bucket iterator is not EOF and
-    // 2. Either BucketListDB is not enabled (so we must apply all entry types)
-    //    or BucketListDB is enabled and we have offers still remaining.
-    return static_cast<bool>(mBucketIter) &&
-           (!mApp.getConfig().isUsingBucketListDB() || mOffersRemaining);
+    // 2. We have offers still remaining.
+    return static_cast<bool>(mBucketIter) && mOffersRemaining;
 }
 
 size_t
@@ -81,20 +78,19 @@ BucketApplicator::size() const
 }
 
 static bool
-shouldApplyEntry(std::function<bool(LedgerEntryType)> const& filter,
-                 BucketEntry const& e)
+shouldApplyEntry(BucketEntry const& e)
 {
     if (e.type() == LIVEENTRY || e.type() == INITENTRY)
     {
-        return filter(e.liveEntry().data.type());
+        return BucketIndex::typeNotSupported(e.liveEntry().data.type());
     }
 
     if (e.type() != DEADENTRY)
     {
         throw std::runtime_error(
             "Malformed bucket: unexpected non-INIT/LIVE/DEAD entry.");
     }
-    return filter(e.deadEntry().type());
+    return BucketIndex::typeNotSupported(e.deadEntry().type());
 }
 
 size_t
@@ -129,8 +125,7 @@ BucketApplicator::advance(BucketApplicator::Counters& counters)
         // returns the file offset at the end of the currently loaded entry.
         // This means we must read until pos is strictly greater than the upper
         // bound so that we don't skip the last offer in the range.
-        auto isUsingBucketListDB = mApp.getConfig().isUsingBucketListDB();
-        if (isUsingBucketListDB && mBucketIter.pos() > mUpperBoundOffset)
+        if (mBucketIter.pos() > mUpperBoundOffset)
         {
             mOffersRemaining = false;
             break;
@@ -139,89 +134,64 @@ BucketApplicator::advance(BucketApplicator::Counters& counters)
         BucketEntry const& e = *mBucketIter;
         LiveBucket::checkProtocolLegality(e, mMaxProtocolVersion);
 
-        if (shouldApplyEntry(mEntryTypeFilter, e))
+        if (shouldApplyEntry(e))
         {
-            if (isUsingBucketListDB)
+            if (e.type() == LIVEENTRY || e.type() == INITENTRY)
             {
-                if (e.type() == LIVEENTRY || e.type() == INITENTRY)
-                {
-                    auto [_, wasInserted] =
-                        mSeenKeys.emplace(LedgerEntryKey(e.liveEntry()));
+                auto [_, wasInserted] =
+                    mSeenKeys.emplace(LedgerEntryKey(e.liveEntry()));
 
-                    // Skip seen keys
-                    if (!wasInserted)
-                    {
-                        continue;
-                    }
-                }
-                else
+                // Skip seen keys
+                if (!wasInserted)
                 {
-                    // Only apply INIT and LIVE entries
-                    mSeenKeys.emplace(e.deadEntry());
                     continue;
                 }
             }
+            else
+            {
+                // Only apply INIT and LIVE entries
+                mSeenKeys.emplace(e.deadEntry());
+                continue;
+            }
 
             counters.mark(e);
 
-            if (e.type() == LIVEENTRY || e.type() == INITENTRY)
+            // DEAD and META entries skipped
+            releaseAssert(e.type() == LIVEENTRY || e.type() == INITENTRY);
+            // The last level can have live entries, but at that point we
+            // know that they are actually init entries because the earliest
+            // state of all entries is init, so we mark them as such here
+            if (mLevel == LiveBucketList::kNumLevels - 1 &&
+                e.type() == LIVEENTRY)
             {
-                // The last level can have live entries, but at that point we
-                // know that they are actually init entries because the earliest
-                // state of all entries is init, so we mark them as such here
-                if (mLevel == LiveBucketList::kNumLevels - 1 &&
-                    e.type() == LIVEENTRY)
-                {
-                    ltx->createWithoutLoading(e.liveEntry());
-                }
-                else if (
-                    protocolVersionIsBefore(
-                        mMinProtocolVersionSeen,
-                        LiveBucket::
-                            FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY))
+                ltx->createWithoutLoading(e.liveEntry());
+            }
+            else if (protocolVersionIsBefore(
+                         mMinProtocolVersionSeen,
+                         LiveBucket::
+                             FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY))
+            {
+                // Prior to protocol 11, INITENTRY didn't exist, so we need
+                // to check ltx to see if this is an update or a create
+                auto key = InternalLedgerEntry(e.liveEntry()).toKey();
+                if (ltx->getNewestVersion(key))
                 {
-                    // Prior to protocol 11, INITENTRY didn't exist, so we need
-                    // to check ltx to see if this is an update or a create
-                    auto key = InternalLedgerEntry(e.liveEntry()).toKey();
-                    if (ltx->getNewestVersion(key))
-                    {
-                        ltx->updateWithoutLoading(e.liveEntry());
-                    }
-                    else
-                    {
-                        ltx->createWithoutLoading(e.liveEntry());
-                    }
+                    ltx->updateWithoutLoading(e.liveEntry());
                 }
                 else
                 {
-                    if (e.type() == LIVEENTRY)
-                    {
-                        ltx->updateWithoutLoading(e.liveEntry());
-                    }
-                    else
-                    {
-                        ltx->createWithoutLoading(e.liveEntry());
-                    }
+                    ltx->createWithoutLoading(e.liveEntry());
                 }
             }
             else
             {
-                releaseAssertOrThrow(!isUsingBucketListDB);
-                if (protocolVersionIsBefore(
-                        mMinProtocolVersionSeen,
-                        LiveBucket::
-                            FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY))
+                if (e.type() == LIVEENTRY)
                 {
-                    // Prior to protocol 11, DEAD entries could exist
-                    // without LIVE entries in between
-                    if (ltx->getNewestVersion(e.deadEntry()))
-                    {
-                        ltx->eraseWithoutLoading(e.deadEntry());
-                    }
+                    ltx->updateWithoutLoading(e.liveEntry());
                 }
                 else
                 {
-                    ltx->eraseWithoutLoading(e.deadEntry());
+                    ltx->createWithoutLoading(e.liveEntry());
                 }
             }
 

src/bucket/BucketApplicator.h

@@ -26,7 +26,6 @@ class BucketApplicator
     uint32_t mLevel;
     LiveBucketInputIterator mBucketIter;
     size_t mCount{0};
-    std::function<bool(LedgerEntryType)> mEntryTypeFilter;
     std::unordered_set<LedgerKey>& mSeenKeys;
     std::streamoff mUpperBoundOffset{0};
     bool mOffersRemaining{true};
@@ -73,7 +72,6 @@ class BucketApplicator
     BucketApplicator(Application& app, uint32_t maxProtocolVersion,
                      uint32_t minProtocolVersionSeen, uint32_t level,
                      std::shared_ptr<LiveBucket const> bucket,
-                     std::function<bool(LedgerEntryType)> filter,
                      std::unordered_set<LedgerKey>& seenKeys);
     operator bool() const;
     size_t advance(Counters& counters);

src/bucket/BucketBase.cpp

@@ -397,8 +397,7 @@ BucketBase::merge(BucketManager& bucketManager, uint32_t maxProtocolVersion,
 
     MergeKey mk{keepTombstoneEntries, oldBucket->getHash(),
                 newBucket->getHash(), shadowHashes};
-    return out.getBucket(bucketManager,
-                         bucketManager.getConfig().isUsingBucketListDB(), &mk);
+    return out.getBucket(bucketManager, &mk);
 }
 
 template std::shared_ptr<LiveBucket> BucketBase::merge<LiveBucket>(

src/bucket/BucketIndexImpl.cpp

@@ -239,7 +239,7 @@ BucketIndexImpl<IndexT>::BucketIndexImpl(BucketManager& bm,
         ZoneValue(static_cast<int64_t>(count));
     }
 
-    if (bm.getConfig().isPersistingBucketListDBIndexes())
+    if (bm.getConfig().BUCKETLIST_DB_PERSIST_INDEX)
     {
         saveToDisk(bm, hash, ctx);
     }
@@ -260,7 +260,7 @@ BucketIndexImpl<BucketIndex::RangeIndex>::saveToDisk(
     BucketManager& bm, Hash const& hash, asio::io_context& ctx) const
 {
     ZoneScoped;
-    releaseAssert(bm.getConfig().isPersistingBucketListDBIndexes());
+    releaseAssert(bm.getConfig().BUCKETLIST_DB_PERSIST_INDEX);
     auto timer =
         LogSlowExecution("Saving index", LogSlowExecution::Mode::AUTOMATIC_RAII,
                          "took", std::chrono::milliseconds(100));
@@ -381,7 +381,6 @@ BucketIndex::createIndex(BucketManager& bm,
 
     ZoneScoped;
     auto const& cfg = bm.getConfig();
-    releaseAssertOrThrow(cfg.isUsingBucketListDB());
     releaseAssertOrThrow(!filename.empty());
     auto pageSize = effectivePageSize(cfg, fs::size(filename.string()));