diff --git a/src/database/Database.cpp b/src/database/Database.cpp index 44fa272d08..afbccf762d 100644 --- a/src/database/Database.cpp +++ b/src/database/Database.cpp @@ -216,6 +216,9 @@ Database::applySchemaUpgrade(unsigned long vers) mApp.getHistoryManager().dropSQLBasedPublish(); Upgrades::dropSupportUpgradeHistory(*this); break; + case 24: + mApp.getPersistentState().migrateToSlotStateTable(); + break; default: throw std::runtime_error("Unknown DB schema version"); } diff --git a/src/database/Database.h b/src/database/Database.h index 75d4c46805..009ee798f2 100644 --- a/src/database/Database.h +++ b/src/database/Database.h @@ -32,7 +32,7 @@ using PreparedStatementCache = // smallest schema version supported static constexpr unsigned long MIN_SCHEMA_VERSION = 21; -static constexpr unsigned long SCHEMA_VERSION = 23; +static constexpr unsigned long SCHEMA_VERSION = 24; /** * Helper class for borrowing a SOCI prepared statement handle into a local diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp index 6671c29d40..b0a44f5488 100644 --- a/src/herder/HerderImpl.cpp +++ b/src/herder/HerderImpl.cpp @@ -2008,7 +2008,7 @@ HerderImpl::restoreSCPState() // Load all known tx sets auto latestTxSets = mApp.getPersistentState().getTxSetsForAllSlots(); - for (auto const& txSet : latestTxSets) + for (auto const& [_, txSet] : latestTxSets) { try { @@ -2037,7 +2037,7 @@ HerderImpl::restoreSCPState() // load saved state from database auto latest64 = mApp.getPersistentState().getSCPStateAllSlots(); - for (auto const& state : latest64) + for (auto const& [_, state] : latest64) { try { @@ -2244,7 +2244,7 @@ HerderImpl::purgeOldPersistedTxSets() { auto hashesToDelete = mApp.getPersistentState().getTxSetHashesForAllSlots(); - for (auto const& state : + for (auto const& [_, state] : mApp.getPersistentState().getSCPStateAllSlots()) { try diff --git a/src/main/PersistentState.cpp b/src/main/PersistentState.cpp index adcf96d1a0..0b1e90815e 100644 --- a/src/main/PersistentState.cpp +++ b/src/main/PersistentState.cpp @@ -36,8 +36,8 @@ std::string PersistentState::kSQLCreateSCPStatement = "state TEXT" "); "; -std::string PersistentState::LCLTableName = "storestate"; -std::string PersistentState::SlotTableName = "slotstate"; +std::string PersistentState::kLCLTableName = "storestate"; +std::string PersistentState::kSlotTableName = "slotstate"; PersistentState::PersistentState(Application& app) : mApp(app) { @@ -45,7 +45,8 @@ PersistentState::PersistentState(Application& app) : mApp(app) } void -PersistentState::deleteTxSets(std::unordered_set hashesToDelete) +PersistentState::deleteTxSets(std::unordered_set hashesToDelete, + std::string table) { releaseAssert(threadIsMain()); soci::transaction tx(mApp.getDatabase().getRawSession()); @@ -53,7 +54,7 @@ PersistentState::deleteTxSets(std::unordered_set hashesToDelete) { auto name = getStoreStateNameForTxSet(hash); auto prep = mApp.getDatabase().getPreparedStatement( - "DELETE FROM slotstate WHERE statename = :n;", + fmt::format("DELETE FROM {} WHERE statename = :n;", table), mApp.getDatabase().getSession()); auto& st = prep.statement(); @@ -64,6 +65,66 @@ PersistentState::deleteTxSets(std::unordered_set hashesToDelete) tx.commit(); } +void +PersistentState::migrateToSlotStateTable() +{ + // No soci::transaction needed, because the migration in Database.cpp wraps + // everything in one transaction anyway. + releaseAssert(threadIsMain()); + auto& db = mApp.getDatabase(); + + // First, create the new table + db.getRawSession() << PersistentState::kSQLCreateSCPStatement; + + // Migrate all the tx sets + auto txSets = getTxSetsForAllSlots(kLCLTableName); + std::unordered_set keysToDelete; + for (auto const& txSet : txSets) + { + CLOG_INFO(Herder, "Migrating tx set {} to slotstate", + hexAbbrev(txSet.first)); + updateDb(getStoreStateNameForTxSet(txSet.first), txSet.second, + db.getSession(), kSlotTableName); + keysToDelete.insert(txSet.first); + } + + // Cleanup tx sets from the previous table + deleteTxSets(keysToDelete, kLCLTableName); + + // Migrate all SCP slot data + auto scpStates = getSCPStateAllSlots(kLCLTableName); + for (auto const& [i, scpState] : scpStates) + { + CLOG_INFO(Herder, "Migrating SCP state for slot {} to slotstate", i); + setSCPStateForSlot(i, scpState); + auto prep = mApp.getDatabase().getPreparedStatement( + "DELETE FROM storestate WHERE statename = :n;", + mApp.getDatabase().getSession()); + + auto& st = prep.statement(); + st.exchange(soci::use(getStoreStateName(kLastSCPDataXDR, i))); + st.define_and_bind(); + st.execute(true); + } + + // Migrate upgrade data + auto upgrades = getFromDb(getStoreStateName(kLedgerUpgrades), + db.getSession(), kLCLTableName); + if (!upgrades.empty()) + { + updateDb(getStoreStateName(kLedgerUpgrades), upgrades, db.getSession(), + kSlotTableName); + auto prep = mApp.getDatabase().getPreparedStatement( + "DELETE FROM storestate WHERE statename = :n;", + mApp.getDatabase().getSession()); + + auto& st = prep.statement(); + st.exchange(soci::use(getStoreStateName(kLedgerUpgrades))); + st.define_and_bind(); + st.execute(true); + } +} + void PersistentState::dropAll(Database& db) { @@ -72,7 +133,6 @@ PersistentState::dropAll(Database& db) soci::statement st = db.getRawSession().prepare << kSQLCreateStatement; st.execute(true); - // TODO: add a migration to move data to slotstate db.getRawSession() << "DROP TABLE IF EXISTS slotstate;"; soci::statement st2 = db.getRawSession().prepare << kSQLCreateSCPStatement; st2.execute(true); @@ -126,7 +186,7 @@ std::string PersistentState::getDBForEntry(PersistentState::Entry entry) { releaseAssert(entry != kLastEntry); - return entry <= kRebuildLedger ? LCLTableName : SlotTableName; + return entry <= kRebuildLedger ? kLCLTableName : kSlotTableName; } std::string @@ -144,21 +204,21 @@ PersistentState::setState(PersistentState::Entry entry, updateDb(getStoreStateName(entry), value, session, getDBForEntry(entry)); } -std::vector -PersistentState::getSCPStateAllSlots() +std::unordered_map +PersistentState::getSCPStateAllSlots(std::string table) { ZoneScoped; releaseAssert(threadIsMain()); // Collect all slots persisted - std::vector states; + std::unordered_map states; for (uint32 i = 0; i <= mApp.getConfig().MAX_SLOTS_TO_REMEMBER; i++) { auto val = getFromDb(getStoreStateName(kLastSCPDataXDR, i), - mApp.getDatabase().getSession(), SlotTableName); + mApp.getDatabase().getSession(), table); if (!val.empty()) { - states.push_back(val); + states.emplace(i, val); } } @@ -174,7 +234,7 @@ PersistentState::setSCPStateForSlot(uint64 slot, std::string const& value) auto slotIdx = static_cast( slot % (mApp.getConfig().MAX_SLOTS_TO_REMEMBER + 1)); updateDb(getStoreStateName(kLastSCPDataXDR, slotIdx), value, - mApp.getDatabase().getSession(), SlotTableName); + mApp.getDatabase().getSession(), kSlotTableName); } void @@ -190,7 +250,7 @@ PersistentState::setSCPStateV1ForSlot( for (auto const& txSet : txSets) { updateDb(getStoreStateNameForTxSet(txSet.first), txSet.second, - mApp.getDatabase().getSession(), SlotTableName); + mApp.getDatabase().getSession(), kSlotTableName); } tx.commit(); } @@ -202,7 +262,7 @@ PersistentState::shouldRebuildForType(LedgerEntryType let) releaseAssert(threadIsMain()); return !getFromDb(getStoreStateName(kRebuildLedger, let), - mApp.getDatabase().getSession(), LCLTableName) + mApp.getDatabase().getSession(), kLCLTableName) .empty(); } @@ -213,7 +273,7 @@ PersistentState::clearRebuildForType(LedgerEntryType let) releaseAssert(threadIsMain()); updateDb(getStoreStateName(kRebuildLedger, let), "", - mApp.getDatabase().getSession(), LCLTableName); + mApp.getDatabase().getSession(), kLCLTableName); } void @@ -230,7 +290,7 @@ PersistentState::setRebuildForType(LedgerEntryType let) } updateDb(getStoreStateName(kRebuildLedger, let), "1", - mApp.getDatabase().getSession(), LCLTableName); + mApp.getDatabase().getSession(), kLCLTableName); } void @@ -272,21 +332,23 @@ PersistentState::updateDb(std::string const& entry, std::string const& value, } } -std::vector -PersistentState::getTxSetsForAllSlots() +std::unordered_map +PersistentState::getTxSetsForAllSlots(std::string table) { ZoneScoped; releaseAssert(threadIsMain()); - std::vector result; + std::unordered_map result; + std::string key; std::string val; std::string pattern = mapping[kTxSet] + "%"; - std::string statementStr = - "SELECT state FROM slotstate WHERE statename LIKE :n;"; + std::string statementStr = fmt::format( + "SELECT statename, state FROM {} WHERE statename LIKE :n;", table); auto& db = mApp.getDatabase(); auto prep = db.getPreparedStatement(statementStr, db.getSession()); auto& st = prep.statement(); + st.exchange(soci::into(key)); st.exchange(soci::into(val)); st.exchange(soci::use(pattern)); st.define_and_bind(); @@ -295,9 +357,13 @@ PersistentState::getTxSetsForAllSlots() st.execute(true); } + Hash hash; + size_t len = binToHex(hash).size(); + while (st.got_data()) { - result.push_back(val); + result.emplace(hexToBin256(key.substr(mapping[kTxSet].size(), len)), + val); st.fetch(); } diff --git a/src/main/PersistentState.h b/src/main/PersistentState.h index d8f6c7bc1a..68494c5d21 100644 --- a/src/main/PersistentState.h +++ b/src/main/PersistentState.h @@ -44,8 +44,10 @@ class PersistentState SessionWrapper& session); // Special methods for SCP state (multiple slots) - std::vector getSCPStateAllSlots(); - std::vector getTxSetsForAllSlots(); + std::unordered_map + getSCPStateAllSlots(std::string table = kSlotTableName); + std::unordered_map + getTxSetsForAllSlots(std::string table = kSlotTableName); std::unordered_set getTxSetHashesForAllSlots(); void @@ -57,14 +59,16 @@ class PersistentState void setRebuildForType(LedgerEntryType let); bool hasTxSet(Hash const& txSetHash); - void deleteTxSets(std::unordered_set hashesToDelete); + void deleteTxSets(std::unordered_set hashesToDelete, + std::string table = kSlotTableName); + void migrateToSlotStateTable(); private: static std::string kSQLCreateStatement; static std::string kSQLCreateSCPStatement; static std::string mapping[kLastEntry]; - static std::string LCLTableName; - static std::string SlotTableName; + static std::string kLCLTableName; + static std::string kSlotTableName; Application& mApp;