Reusable module cache

src/bucket/BucketBase.cpp

@@ -51,6 +51,12 @@ BucketBase::getOfferRange() const
     return getIndex().getOfferRange();
 }
 
+std::optional<std::pair<std::streamoff, std::streamoff>>
+BucketBase::getContractCodeRange() const
+{
+    return getIndex().getContractCodeRange();
+}
+
 void
 BucketBase::setIndex(std::unique_ptr<BucketIndex const>&& index)
 {

src/bucket/BucketBase.h

@@ -93,6 +93,11 @@ class BucketBase : public NonMovableOrCopyable
     std::optional<std::pair<std::streamoff, std::streamoff>>
     getOfferRange() const;
 
+    // Returns [lowerBound, upperBound) of file offsets for all contract code
+    // entries in the bucket, or std::nullopt if no contract code exists
+    std::optional<std::pair<std::streamoff, std::streamoff>>
+    getContractCodeRange() const;
+
     // Sets index, throws if index is already set
     void setIndex(std::unique_ptr<BucketIndex const>&& index);
 

src/bucket/BucketIndex.h

@@ -130,6 +130,11 @@ class BucketIndex : public NonMovableOrCopyable
     virtual std::optional<std::pair<std::streamoff, std::streamoff>>
     getOfferRange() const = 0;
 
+    // Returns lower bound and upper bound for contract code entry positions in
+    // the given bucket, or std::nullopt if no contract code entries exist
+    virtual std::optional<std::pair<std::streamoff, std::streamoff>>
+    getContractCodeRange() const = 0;
+
     // Returns page size for index. InidividualIndex returns 0 for page size
     virtual std::streamoff getPageSize() const = 0;
 

src/bucket/BucketIndexImpl.cpp

@@ -598,6 +598,20 @@ BucketIndexImpl<IndexT>::getOfferRange() const
     return getOffsetBounds(lowerBound, upperBound);
 }
 
+template <class IndexT>
+std::optional<std::pair<std::streamoff, std::streamoff>>
+BucketIndexImpl<IndexT>::getContractCodeRange() const
+{
+    // Get the smallest and largest possible contract code keys
+    LedgerKey lowerBound(CONTRACT_CODE);
+    lowerBound.contractCode().hash.fill(std::numeric_limits<uint8_t>::min());
+
+    LedgerKey upperBound(CONTRACT_CODE);
+    upperBound.contractCode().hash.fill(std::numeric_limits<uint8_t>::max());
+
+    return getOffsetBounds(lowerBound, upperBound);
+}
+
 #ifdef BUILD_TESTS
 template <class IndexT>
 bool

src/bucket/BucketIndexImpl.h

@@ -100,6 +100,9 @@ template <class IndexT> class BucketIndexImpl : public BucketIndex
     virtual std::optional<std::pair<std::streamoff, std::streamoff>>
     getOfferRange() const override;
 
+    virtual std::optional<std::pair<std::streamoff, std::streamoff>>
+    getContractCodeRange() const override;
+
     virtual std::streamoff
     getPageSize() const override
     {

src/bucket/BucketSnapshot.cpp

@@ -282,6 +282,41 @@ LiveBucketSnapshot::scanForEviction(
     return Loop::INCOMPLETE;
 }
 
+// Scans contract code entries in the bucket.
+Loop
+LiveBucketSnapshot::scanForContractCode(
+    std::function<Loop(LedgerEntry const&)> callback) const
+{
+    ZoneScoped;
+    if (isEmpty())
+    {
+        return Loop::INCOMPLETE;
+    }
+
+    auto range = mBucket->getContractCodeRange();
+    if (!range)
+    {
+        return Loop::INCOMPLETE;
+    }
+
+    auto& stream = getStream();
+    stream.seek(range->first);
+
+    BucketEntry be;
+    while (stream.pos() < range->second && stream.readOne(be))
+    {
+        if (be.type() != DEADENTRY &&
+            be.liveEntry().data.type() == CONTRACT_CODE)
+        {
+            if (callback(be.liveEntry()) == Loop::COMPLETE)
+            {
+                return Loop::COMPLETE;
+            }
+        }
+    }
+    return Loop::INCOMPLETE;
+}
+
 template <class BucketT>
 XDRInputFileStream&
 BucketSnapshotBase<BucketT>::getStream() const

src/bucket/BucketSnapshot.h

@@ -87,6 +87,10 @@ class LiveBucketSnapshot : public BucketSnapshotBase<LiveBucket>
                          std::list<EvictionResultEntry>& evictableKeys,
                          SearchableLiveBucketListSnapshot const& bl,
                          uint32_t ledgerVers) const;
+
+    // Scans contract code entries in the bucket.
+    Loop
+    scanForContractCode(std::function<Loop(LedgerEntry const&)> callback) const;
 };
 
 class HotArchiveBucketSnapshot : public BucketSnapshotBase<HotArchiveBucket>

src/bucket/SearchableBucketList.cpp

@@ -62,6 +62,18 @@ SearchableLiveBucketListSnapshot::scanForEviction(
     return result;
 }
 
+void
+SearchableLiveBucketListSnapshot::scanForContractCode(
+    std::function<Loop(LedgerEntry const&)> callback) const
+{
+    ZoneScoped;
+    releaseAssert(mSnapshot);
+    auto f = [&callback](auto const& b) {
+        return b.scanForContractCode(callback);
+    };
+    loopAllBuckets(f, *mSnapshot);
+}
+
 template <class BucketT>
 std::optional<std::vector<typename BucketT::LoadT>>
 SearchableBucketListSnapshotBase<BucketT>::loadKeysInternal(

src/bucket/SearchableBucketList.h

@@ -35,6 +35,9 @@ class SearchableLiveBucketListSnapshot
         std::shared_ptr<EvictionStatistics> stats,
         StateArchivalSettings const& sas, uint32_t ledgerVers) const;
 
+    void
+    scanForContractCode(std::function<Loop(LedgerEntry const&)> callback) const;
+
     friend SearchableSnapshotConstPtr
     BucketSnapshotManager::copySearchableLiveBucketListSnapshot() const;
 };

src/ledger/LedgerManager.h

@@ -7,6 +7,7 @@
 #include "catchup/LedgerApplyManager.h"
 #include "history/HistoryManager.h"
 #include "ledger/NetworkConfig.h"
+#include "rust/RustBridge.h"
 #include <memory>
 
 namespace stellar
@@ -200,6 +201,12 @@ class LedgerManager
     virtual void manuallyAdvanceLedgerHeader(LedgerHeader const& header) = 0;
 
     virtual SorobanMetrics& getSorobanMetrics() = 0;
+    virtual rust_bridge::SorobanModuleCache& getModuleCache() = 0;
+
+    // Compiles all contracts in the current ledger, for ledger protocols
+    // starting at minLedgerVersion and running through to
+    // Config::CURRENT_LEDGER_PROTOCOL_VERSION (to enable upgrades).
+    virtual void compileAllContractsInLedger(uint32_t minLedgerVersion) = 0;
 
     virtual ~LedgerManager()
     {

src/ledger/LedgerManagerImpl.cpp

@@ -22,6 +22,7 @@
 #include "ledger/LedgerTxn.h"
 #include "ledger/LedgerTxnEntry.h"
 #include "ledger/LedgerTxnHeader.h"
+#include "ledger/SharedModuleCacheCompiler.h"
 #include "main/Application.h"
 #include "main/Config.h"
 #include "main/ErrorMessages.h"
@@ -42,6 +43,7 @@
 #include "work/WorkScheduler.h"
 #include "xdrpp/printer.h"
 
+#include <cstdint>
 #include <fmt/format.h>
 
 #include "xdr/Stellar-ledger.h"
@@ -155,6 +157,7 @@ LedgerManagerImpl::LedgerManagerImpl(Application& app)
     , mCatchupDuration(
           app.getMetrics().NewTimer({"ledger", "catchup", "duration"}))
     , mState(LM_BOOTING_STATE)
+    , mModuleCache(::rust_bridge::new_module_cache())
 
 {
     setupLedgerCloseMetaStream();
@@ -403,6 +406,9 @@ LedgerManagerImpl::loadLastKnownLedger(bool restoreBucketlist)
         updateNetworkConfig(ltx);
         mSorobanNetworkConfigReadOnly = mSorobanNetworkConfigForApply;
     }
+
+    // Prime module cache with ledger content.
+    compileAllContractsInLedger(latestLedgerHeader->ledgerVersion);
 }
 
 Database&
@@ -566,6 +572,27 @@ LedgerManagerImpl::getSorobanMetrics()
     return mSorobanMetrics;
 }
 
+rust_bridge::SorobanModuleCache&
+LedgerManagerImpl::getModuleCache()
+{
+    return *mModuleCache;
+}
+
+void
+LedgerManagerImpl::compileAllContractsInLedger(uint32_t minLedgerVersion)
+{
+    auto& moduleCache = getModuleCache();
+    std::vector<uint32_t> ledgerVersions;
+    for (uint32_t i = minLedgerVersion;
+         i <= Config::CURRENT_LEDGER_PROTOCOL_VERSION; i++)
+    {
+        ledgerVersions.push_back(i);
+    }
+    auto compiler = std::make_shared<SharedModuleCacheCompiler>(
+        mApp, moduleCache, ledgerVersions);
+    compiler->run();
+}
+
 void
 LedgerManagerImpl::publishSorobanMetrics()
 {

src/ledger/LedgerManagerImpl.h

@@ -11,6 +11,7 @@
 #include "ledger/NetworkConfig.h"
 #include "ledger/SorobanMetrics.h"
 #include "main/PersistentState.h"
+#include "rust/RustBridge.h"
 #include "transactions/TransactionFrame.h"
 #include "util/XDRStream.h"
 #include "xdr/Stellar-ledger.h"
@@ -152,6 +153,9 @@ class LedgerManagerImpl : public LedgerManager
     // Update cached ledger state values managed by this class.
     void advanceLedgerPointers(CloseLedgerOutput const& output);
 
+    // The reusable / inter-ledger soroban module cache.
+    ::rust::Box<rust_bridge::SorobanModuleCache> mModuleCache;
+
   protected:
     // initialLedgerVers must be the ledger version at the start of the ledger
     // and currLedgerVers is the ledger version in the current ltx header. These
@@ -251,5 +255,7 @@ class LedgerManagerImpl : public LedgerManager
     {
         return mCurrentlyApplyingLedger;
     }
+    rust_bridge::SorobanModuleCache& getModuleCache() override;
+    void compileAllContractsInLedger(uint32_t minLedgerVersion) override;
 };
 }