Dipdup 8.0 fixes

.env.dist

@@ -0,0 +1,23 @@
+POSTGRES_HOST=db
+POSTGRES_PORT=5432
+POSTGRES_DB=dipdup
+POSTGRES_USER=dipdup
+POSTGRES_PASSWORD=changeme
+
+TZKT_URL=https://api.parisnet.tzkt.io
+
+METADATA_URL=https://metadata.dipdup.net
+TEZOS_NODE_URL=https://rpc.tzkt.io/parisnet
+ETHERLINK_NODE_WS=wss://etherlink.dipdup.net/
+ETHERLINK_NODE_URL=https://etherlink.dipdup.net
+ETHERLINK_SUBSQUID_URL=https://etherlink-archive.dipdup.net
+ROLLUP_NODE_URL=https://etherlink-rollup-paris.dipdup.net
+
+SMART_ROLLUP_ADDRESS=sr1JBmCsMoXmCeeYQWB3YYYqP9d68wUXQzkC
+NATIVE_TICKETER=KT1Bdyc1UcmjgPr3prJziMyfjSCPK6SEjfqs
+
+HASURA_URL=https://hasura.etherlink.dipdup.net
+ADMIN_SECRET=changeme
+
+DIPDUP_NO_SYMLINK=1
+DIPDUP_DEBUG=0

Dockerfile

@@ -1,12 +1,11 @@
-ARG PYTHON_VERSION=3.11
+ARG PYTHON_VERSION=3.12
 ARG SOURCE_DIR=bridge_indexer
 ARG POETRY_PATH=/opt/poetry
 ARG VENV_PATH=/opt/venv
 ARG APP_PATH=/opt/app
 ARG APP_USER=dipdup
 
-
-FROM python:${PYTHON_VERSION}-slim-buster as builder-base
+FROM python:${PYTHON_VERSION}-slim-bookworm as builder-base
 
 ARG VENV_PATH
 ARG POETRY_PATH
@@ -50,7 +49,7 @@ RUN poetry install --only main --sync --no-interaction --no-ansi -vvv \
  && rm -rf `find $VIRTUAL_ENV/lib -name __pycache__`
 
 
-FROM python:${PYTHON_VERSION}-slim-buster as runtime-base
+FROM python:${PYTHON_VERSION}-slim-bookworm as runtime-base
 
 ARG VENV_PATH
 ENV PATH="$VENV_PATH/bin:$PATH"

README.md

@@ -0,0 +1,21 @@
+# Etherlink Bridge Indexer
+DipDup indexer for tracking Bridge operations.
+## Setup with docker
+#### Build docker image
+* `docker build . -t etherlink-bridge-indexer`
+#### Setup environment
+* copy example file: `cp .env.dist .env` and replace sample values with your settings
+* edit `bridge_indexer/dipdup.yaml` if required
+#### Run with compose
+* `docker compose up -d` 
+## Local setup
+#### Prepare virtual environment and install dependencies
+```shell
+poetry shell
+poetry install --sync --no-root
+```
+#### Setup environment
+* copy example file: `cp .env.dist .env` and replace sample values with your settings
+* edit `bridge_indexer/dipdup.yaml` if required
+#### Run as python module
+`python -m dipdup -c . run`

bridge_indexer/dipdup.yaml

@@ -1,4 +1,4 @@
-spec_version: 2.0
+spec_version: 3.0
 package: bridge_indexer
 
 
@@ -26,7 +26,7 @@ datasources:
 
   metadata:
     kind: tzip_metadata
-    network: oxfordnet
+    network: parisnet
     url: ${METADATA_URL}
 
   tezos_node:
@@ -40,7 +40,6 @@ datasources:
   etherlink_subsquid:
     kind: evm.subsquid
     url: ${ETHERLINK_SUBSQUID_URL:-~}
-    node: etherlink_node
 
   rollup_node:
     kind: http
@@ -53,29 +52,34 @@ contracts:
     address: ${SMART_ROLLUP_ADDRESS}
     typename: rollup
 
+  tezos_native_ticketer:
+    kind: tezos
+    address: ${NATIVE_TICKETER}
+
   etherlink_rollup_kernel:
     kind: evm
-    address: 0x0000000000000000000000000000000000000000
+    address: "0x0000000000000000000000000000000000000000"
     typename: kernel
 
   l2_tzbtc_token:
     kind: evm
-    address: 0x54b20569c0aa92a5618589142b2d5cc5fe6fe426
+    address: 0x87dcbf128677ba36e79d47daf4eb4e51610e0150
     typename: l2_token
   l2_sirs_token:
     kind: evm
-    address: 0x1b674b7d28d3da4bb0cbda959de2bcebe2ac83e8
+    address: 0xcb5d40c6b1bdf5cd51b3801351b0a68d101a561b
     typename: l2_token
   l2_usdt_token:
     kind: evm
-    address: 0xe4bf1873cfdaed33bab4e0f5788cba6d03a267f0
+    address: 0x8554cd57c0c3e5ab9d1782c9063279fa9bfa4680
     typename: l2_token
 
 
 indexes:
   tezos_rollup_cement:
-    kind: tezos.tzkt.operations
-    datasource: tzkt
+    kind: tezos.operations
+    datasources:
+      - tzkt
     types:
       - sr_cement
     contracts:
@@ -87,8 +91,9 @@ indexes:
             destination: tezos_smart_rollup
 
   tezos_rollup_operations:
-    kind: tezos.tzkt.operations
-    datasource: tzkt
+    kind: tezos.operations
+    datasources:
+      - tzkt
     types:
       - sr_execute
       - transaction
@@ -106,8 +111,10 @@ indexes:
             destination: tezos_smart_rollup
 
   etherlink_kernel_events:
-    kind: evm.subsquid.events
-    datasource: etherlink_subsquid
+    kind: evm.events
+    datasources:
+      - etherlink_subsquid
+      - etherlink_node
     handlers:
       - callback: etherlink.on_deposit
         contract: etherlink_rollup_kernel
@@ -118,8 +125,10 @@ indexes:
         name: Withdrawal
 
   etherlink_token_events:
-    kind: evm.subsquid.events
-    datasource: etherlink_subsquid
+    kind: evm.events
+    datasources:
+      - etherlink_subsquid
+      - etherlink_node
     handlers:
       - callback: etherlink.on_transfer
         contract: l2_tzbtc_token
@@ -133,8 +142,10 @@ indexes:
 
 
   etherlink_kernel_transactions:
-    kind: evm.subsquid.transactions
-    datasource: etherlink_subsquid
+    kind: evm.transactions
+    datasources:
+      - etherlink_subsquid
+      - etherlink_node
     handlers:
       - callback: etherlink.on_xtz_deposit
         from_: etherlink_rollup_kernel

bridge_indexer/handlers/bridge_matcher.py

@@ -12,8 +12,7 @@
 from bridge_indexer.models import TezosDepositOperation
 from bridge_indexer.models import TezosWithdrawOperation
 
-LAYERS_TIMESTAMP_GAP_MIN = timedelta(seconds=8-1)
-LAYERS_TIMESTAMP_GAP_MAX = timedelta(seconds=5*8+1)
+LAYERS_TIMESTAMP_GAP_MAX = timedelta(seconds=20*7)
 
 
 class BridgeMatcher:
@@ -101,8 +100,7 @@ async def check_pending_etherlink_xtz_deposits():
                     l2_transaction=None,
                     l1_transaction__inbox_message_id__gt=0,
                     l1_transaction__ticket=l2_deposit.l2_token.ticket,
-                    l1_transaction__timestamp__gt=l2_deposit.timestamp + LAYERS_TIMESTAMP_GAP_MIN,
-                    l1_transaction__timestamp__lt=l2_deposit.timestamp + LAYERS_TIMESTAMP_GAP_MAX,
+                    l1_transaction__timestamp__gte=l2_deposit.timestamp - LAYERS_TIMESTAMP_GAP_MAX,
                     l1_transaction__l2_account=l2_deposit.l2_account,
                     l1_transaction__amount=l2_deposit.amount[:-12],
                 )

bridge_indexer/handlers/etherlink/on_deposit.py

@@ -4,14 +4,14 @@
 from dipdup.context import HandlerContext
 from dipdup.models import Index
 from dipdup.models import IndexStatus
-from dipdup.models.evm_subsquid import SubsquidEvent
+from dipdup.models.evm import EvmEvent
 
 from bridge_indexer.handlers import setup_handler_logger
 from bridge_indexer.handlers.bridge_matcher import BridgeMatcher
 from bridge_indexer.models import EtherlinkDepositOperation
 from bridge_indexer.models import EtherlinkToken
 from bridge_indexer.models import TezosTicket
-from bridge_indexer.types.kernel.evm_events.deposit import Deposit
+from bridge_indexer.types.kernel.evm_events.deposit import DepositPayload
 
 
 async def _validate_ticket(ticket_hash):
@@ -39,7 +39,7 @@ async def register_etherlink_token(token_contract: str, ticket_hash: int) -> Eth
 
 async def on_deposit(
     ctx: HandlerContext,
-    event: SubsquidEvent[Deposit],
+    event: EvmEvent[DepositPayload],
 ) -> None:
     setup_handler_logger(ctx)
     ctx.logger.info(f'Etherlink Deposit Event found: 0x{event.data.transaction_hash}')

bridge_indexer/handlers/etherlink/on_transfer.py

@@ -1,10 +1,11 @@
 from dipdup.context import HandlerContext
-from dipdup.models.evm_subsquid import SubsquidEvent
+from dipdup.models.evm import EvmEvent
+
 from tortoise.exceptions import DoesNotExist
 
 from bridge_indexer.handlers import setup_handler_logger
 from bridge_indexer.models import EtherlinkTokenHolder
-from bridge_indexer.types.l2_token.evm_events.transfer import Transfer
+from bridge_indexer.types.l2_token.evm_events.transfer import TransferPayload
 
 
 async def on_balance_update(
@@ -35,7 +36,7 @@ async def on_balance_update(
 
 async def on_transfer(
     ctx: HandlerContext,
-    event: SubsquidEvent[Transfer],
+    event: EvmEvent[TransferPayload],
 ) -> None:
     setup_handler_logger(ctx)
     ctx.logger.info(f'Etherlink Token Transfer Event found: 0x{event.data.transaction_hash}')

bridge_indexer/handlers/etherlink/on_withdraw.py

@@ -1,19 +1,19 @@
 from dipdup.context import HandlerContext
 from dipdup.models import Index
 from dipdup.models import IndexStatus
-from dipdup.models.evm_subsquid import SubsquidEvent
+from dipdup.models.evm import EvmEvent
 from tortoise.exceptions import DoesNotExist
 
 from bridge_indexer.handlers import setup_handler_logger
 from bridge_indexer.handlers.bridge_matcher import BridgeMatcher
 from bridge_indexer.models import EtherlinkToken
 from bridge_indexer.models import EtherlinkWithdrawOperation
-from bridge_indexer.types.kernel.evm_events.withdrawal import Withdrawal
+from bridge_indexer.types.kernel.evm_events.withdrawal import WithdrawalPayload
 
 
 async def on_withdraw(
     ctx: HandlerContext,
-    event: SubsquidEvent[Withdrawal],
+    event: EvmEvent[WithdrawalPayload],
 ) -> None:
     setup_handler_logger(ctx)
     ctx.logger.info(f'Etherlink Withdraw Event found: 0x{event.data.transaction_hash}')

bridge_indexer/handlers/etherlink/on_xtz_deposit.py

@@ -4,8 +4,7 @@
 from dipdup.context import HandlerContext
 from dipdup.models import Index
 from dipdup.models import IndexStatus
-from dipdup.models.evm_node import EvmNodeTransactionData
-from dipdup.models.evm_subsquid import SubsquidTransactionData
+from dipdup.models.evm import EvmTransactionData
 
 from bridge_indexer.handlers import setup_handler_logger
 from bridge_indexer.handlers.bridge_matcher import BridgeMatcher
@@ -14,7 +13,7 @@
 from bridge_indexer.models import TezosTicket
 
 
-async def _validate_xtz_transaction(transaction: SubsquidTransactionData | EvmNodeTransactionData):
+async def _validate_xtz_transaction(transaction: EvmTransactionData):
     validators = [
         transaction.value > 0,
         transaction.from_ == '0x0000000000000000000000000000000000000000',
@@ -27,7 +26,7 @@ async def _validate_xtz_transaction(transaction: SubsquidTransactionData | EvmNo
 
 async def on_xtz_deposit(
     ctx: HandlerContext,
-    transaction: SubsquidTransactionData | EvmNodeTransactionData,
+    transaction: EvmTransactionData,
 ) -> None:
     setup_handler_logger(ctx)
     ctx.logger.info(f'Etherlink XTZ Deposit Transaction found: 0x{transaction.hash}')

bridge_indexer/handlers/rollup_message.py

@@ -1,9 +1,11 @@
 import asyncio
+from typing import AsyncGenerator
 from typing import TYPE_CHECKING
 
 from dipdup.datasources.http import HttpDatasource
-from dipdup.datasources.tezos_tzkt import TzktDatasource
-from dipdup.models.tezos_tzkt import TzktOperationData
+from dipdup.datasources.tezos_tzkt import Datasource
+from dipdup.datasources.tezos_tzkt import TezosTzktDatasource
+from dipdup.models.tezos import TezosOperationData
 
 from bridge_indexer.models import BridgeOperation
 from bridge_indexer.models import BridgeWithdrawOperation
@@ -18,7 +20,7 @@
 
 
 class InboxMessageService:
-    def __init__(self, tzkt: TzktDatasource, bridge: 'BridgeConstantStorage'):
+    def __init__(self, tzkt: Datasource, bridge: 'BridgeConstantStorage'):
         self._tzkt = tzkt
         self._bridge = bridge
 
@@ -57,12 +59,12 @@ async def _prepare_inbox(self, inbox_level):
                 inbox.append(inbox_message)
             await RollupInboxMessage.bulk_create(inbox)
 
-    async def _read_inbox(self, inbox_level: int):
+    async def _read_inbox(self, inbox_level: int) -> AsyncGenerator[RollupInboxMessage, None]:
         await self._prepare_inbox(inbox_level)
         async for inbox_message in RollupInboxMessage.filter(level=inbox_level, l1_deposits__isnull=True).order_by('id'):
             yield inbox_message
 
-    async def match_transaction_with_inbox(self, data: TzktOperationData) -> RollupInboxMessage:
+    async def match_transaction_with_inbox(self, data: TezosOperationData) -> RollupInboxMessage:
         async for inbox_message in self._read_inbox(data.level):
             if data.parameter_json == inbox_message.parameter:
                 return inbox_message
@@ -76,7 +78,7 @@ async def find_by_index(self, inbox_level: int, index: int):
 
 
 class OutboxMessageService:
-    def __init__(self, tzkt: TzktDatasource, rollup_node: HttpDatasource, protocol: 'ProtocolConstantStorage'):
+    def __init__(self, tzkt: TezosTzktDatasource, rollup_node: HttpDatasource, protocol: 'ProtocolConstantStorage'):
         self._tzkt = tzkt
         self._rollup_node = rollup_node
         self._protocol = protocol

bridge_indexer/handlers/service_container.py

@@ -1,5 +1,5 @@
 from dipdup.context import DipDupContext
-from dipdup.datasources.tezos_tzkt import TzktDatasource
+from dipdup.datasources.tezos_tzkt import TezosTzktDatasource
 from dipdup.datasources.tzip_metadata import TzipMetadataDatasource
 from pydantic import BaseModel
 
@@ -28,7 +28,7 @@ class Config:
     ticket_service: TicketService
     inbox_message_service: InboxMessageService
     outbox_message_service: OutboxMessageService
-    tzkt: TzktDatasource
+    tzkt: TezosTzktDatasource
     metadata: TzipMetadataDatasource
 
 
@@ -41,13 +41,13 @@ def __init__(self, ctx: DipDupContext):
 
     @staticmethod
     def register(ctx):
-        tzkt = ctx.get_tzkt_datasource('tzkt')
+        tzkt = ctx.get_tezos_tzkt_datasource('tzkt')
         rollup_node = ctx.get_http_datasource('rollup_node')
         metadata = ctx.get_metadata_datasource('metadata')
 
         bridge = BridgeConstantStorage(
             smart_rollup_address=ctx.config.get_tezos_contract('tezos_smart_rollup').address,
-            native_ticketer='KT1Q6aNZ9aGro4DvBKwhKvVdia2UmVGsS9zE',
+            native_ticketer=ctx.config.get_tezos_contract('tezos_native_ticketer').address,
         )
         protocol = ProtocolConstantStorage(
             smart_rollup_commitment_period=20,

bridge_indexer/handlers/tezos/on_cement_commitment.py

@@ -1,5 +1,5 @@
 from dipdup.context import HandlerContext
-from dipdup.models.tezos_tzkt import TzktSmartRollupCement
+from dipdup.models.tezos import TezosSmartRollupCement
 
 from bridge_indexer.handlers import setup_handler_logger
 from bridge_indexer.handlers.bridge_matcher import BridgeMatcher
@@ -14,7 +14,7 @@
 
 async def on_cement_commitment(
     ctx: HandlerContext,
-    cement: TzktSmartRollupCement,
+    cement: TezosSmartRollupCement,
 ) -> None:
     setup_handler_logger(ctx)
     new_record, _ = await RollupCementedCommitment.update_or_create(