Regenerate watcher to backfill event data (#45)

* Regenerate watcher with latest codegen

* Make new Event columns nullable

* Add a script to backfill event data

* Revert "Make new Event columns nullable"

This reverts commit fdcab98.

* Use backfill events data command from @cerc-io/cli

* Update package json script

Author: prathamesh0

environments/local.toml

@@ -23,7 +23,7 @@
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # Server GQL config
+  # GQL server config
   [server.gql]
     path = "/graphql"
 
@@ -45,6 +45,14 @@
       maxAge = 15
       timeTravelMaxAge = 86400 # 1 day
 
+  # ETH RPC server config
+  [server.ethRPC]
+    enabled = true
+    path = "/rpc"
+
+    # Max number of logs that can be returned in a single getLogs request (default: 10000)
+    getLogsResultLimit = 10000
+
 [metrics]
   host = "127.0.0.1"
   port = 9000

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/sushiswap-v3-watcher-ts",
-  "version": "0.1.16",
+  "version": "0.1.17",
   "description": "sushiswap-v3-watcher-ts",
   "private": true,
   "main": "dist/index.js",
@@ -25,7 +25,8 @@
     "import-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/import-state.js",
     "import-state:dev": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
     "inspect-cid": "DEBUG=vulcanize:* ts-node src/cli/inspect-cid.ts",
-    "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts"
+    "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts",
+    "backfill-events-data": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true node --enable-source-maps dist/cli/backfill-events-data.js"
   },
   "repository": {
     "type": "git",
@@ -39,11 +40,11 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@cerc-io/cli": "^0.2.103",
-    "@cerc-io/ipld-eth-client": "^0.2.103",
-    "@cerc-io/solidity-mapper": "^0.2.103",
-    "@cerc-io/util": "^0.2.103",
-    "@cerc-io/graph-node": "^0.2.103",
+    "@cerc-io/cli": "^0.2.108",
+    "@cerc-io/ipld-eth-client": "^0.2.108",
+    "@cerc-io/solidity-mapper": "^0.2.108",
+    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/graph-node": "^0.2.108",
     "@ethersproject/providers": "^5.4.4",
     "debug": "^4.3.1",
     "decimal.js": "^10.3.1",

src/cli/backfill-events-data.ts

@@ -0,0 +1,26 @@
+//
+// Copyright 2024 Vulcanize, Inc.
+//
+
+import 'reflect-metadata';
+import debug from 'debug';
+
+import { BackfillEventsDataCmd } from '@cerc-io/cli';
+
+import { Database } from '../database';
+import { Event } from '../entity/Event';
+
+const log = debug('vulcanize:backfill-events-data');
+
+const main = async (): Promise<void> => {
+  const backFillCmd = new BackfillEventsDataCmd();
+  await backFillCmd.init(Database);
+
+  await backFillCmd.exec(Event);
+};
+
+main().catch(err => {
+  log(err);
+}).finally(() => {
+  process.exit(0);
+});

src/database.ts

@@ -169,6 +169,12 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.getEventsInRange(repo, fromBlockNumber, toBlockNumber);
   }
 
+  async getEvents (options: FindManyOptions<Event>): Promise<Array<Event>> {
+    const repo = this._conn.getRepository(Event);
+
+    return this._baseDatabase.getEvents(repo, options);
+  }
+
   async saveEventEntity (queryRunner: QueryRunner, entity: Event): Promise<Event> {
     const repo = queryRunner.manager.getRepository(Event);
     return this._baseDatabase.saveEventEntity(repo, entity);

src/entity/Event.ts

@@ -27,6 +27,21 @@ export class Event {
   @Column('varchar', { length: 256 })
     eventName!: string;
 
+  @Column('varchar', { length: 66 })
+    topic0!: string;
+
+  @Column('varchar', { length: 66, nullable: true })
+    topic1!: string | null;
+
+  @Column('varchar', { length: 66, nullable: true })
+    topic2!: string | null;
+
+  @Column('varchar', { length: 66, nullable: true })
+    topic3!: string | null;
+
+  @Column('varchar')
+    data!: string;
+
   @Column('text')
     eventInfo!: string;
 

src/indexer.ts

@@ -192,6 +192,10 @@ export class Indexer implements IndexerInterface {
     return this._storageLayoutMap;
   }
 
+  get contractMap (): Map<string, ethers.utils.Interface> {
+    return this._contractMap;
+  }
+
   get graphWatcher (): GraphWatcher {
     return this._graphWatcher;
   }
@@ -503,6 +507,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
+  async removeContract (address: string, kind: string): Promise<void> {
+    return this._baseIndexer.removeContract(address, kind);
+  }
+
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
     this._baseIndexer.updateStateStatusMap(address, stateStatus);
   }
@@ -543,6 +551,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getEventsInRange(fromBlockNumber, toBlockNumber, this._serverConfig.gql.maxEventsBlockRange);
   }
 
+  async getEvents (options: FindManyOptions<Event>): Promise<Array<Event>> {
+    return this._db.getEvents(options);
+  }
+
   async getSyncStatus (): Promise<SyncStatus | undefined> {
     return this._baseIndexer.getSyncStatus();
   }

src/schema.gql

@@ -4775,8 +4775,8 @@ type Token {
   totalValueLockedUSD: BigDecimal!
   totalValueLockedUSDUntracked: BigDecimal!
   derivedETH: BigDecimal!
-  whitelistPools: [Pool!]!
-  tokenDayData: [TokenDayData!]!
+  whitelistPools(where: Pool_filter, orderBy: Pool_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Pool!]!
+  tokenDayData(where: TokenDayData_filter, orderBy: TokenDayData_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [TokenDayData!]!
 }
 
 type Pool {
@@ -4809,13 +4809,13 @@ type Pool {
   totalValueLockedUSD: BigDecimal!
   totalValueLockedUSDUntracked: BigDecimal!
   liquidityProviderCount: BigInt!
-  poolHourData: [PoolHourData!]!
-  poolDayData: [PoolDayData!]!
-  mints: [Mint!]!
-  burns: [Burn!]!
-  swaps: [Swap!]!
-  collects: [Collect!]!
-  ticks: [Tick!]!
+  poolHourData(where: PoolHourData_filter, orderBy: PoolHourData_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [PoolHourData!]!
+  poolDayData(where: PoolDayData_filter, orderBy: PoolDayData_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [PoolDayData!]!
+  mints(where: Mint_filter, orderBy: Mint_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Mint!]!
+  burns(where: Burn_filter, orderBy: Burn_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Burn!]!
+  swaps(where: Swap_filter, orderBy: Swap_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Swap!]!
+  collects(where: Collect_filter, orderBy: Collect_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Collect!]!
+  ticks(where: Tick_filter, orderBy: Tick_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Tick!]!
 }
 
 type PoolHourData {
@@ -4889,11 +4889,11 @@ type Transaction {
   timestamp: BigInt!
   gasUsed: BigInt!
   gasPrice: BigInt!
-  mints: [Mint!]!
-  burns: [Burn!]!
-  swaps: [Swap!]!
-  flashed: [Flash!]!
-  collects: [Collect!]!
+  mints(where: Mint_filter, orderBy: Mint_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Mint!]!
+  burns(where: Burn_filter, orderBy: Burn_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Burn!]!
+  swaps(where: Swap_filter, orderBy: Swap_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Swap!]!
+  flashed(where: Flash_filter, orderBy: Flash_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Flash!]!
+  collects(where: Collect_filter, orderBy: Collect_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Collect!]!
 }
 
 type Burn {
@@ -5025,8 +5025,8 @@ type Position {
   transaction: Transaction!
   feeGrowthInside0LastX128: BigInt!
   feeGrowthInside1LastX128: BigInt!
-  increaseEvents: [IncreaseEvent!]!
-  decreaseEvents: [IncreaseEvent!]!
+  increaseEvents(where: IncreaseEvent_filter, orderBy: IncreaseEvent_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [IncreaseEvent!]!
+  decreaseEvents(where: IncreaseEvent_filter, orderBy: IncreaseEvent_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [IncreaseEvent!]!
 }
 
 type IncreaseEvent {