indexer: optimize idx.BlockList performance

turns out, a unified query with `COUNT(*) OVER() AS total_count` is 10x slower
than two separate queries `SELECT *` and `SELECT COUNT(*)`

also, optimize even further (~1000x) for the most common query:
when listing all blocks without filters, don't even count, just return last height

the benchmark code used to test is included

Author: altergui

vochain/indexer/bench_test.go

@@ -2,6 +2,7 @@ package indexer
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"math/big"
 	"sync"
@@ -14,6 +15,7 @@ import (
 	"go.vocdoni.io/dvote/test/testcommon/testutil"
 	"go.vocdoni.io/dvote/util"
 	"go.vocdoni.io/dvote/vochain"
+	indexerdb "go.vocdoni.io/dvote/vochain/indexer/db"
 	"go.vocdoni.io/dvote/vochain/state"
 	"go.vocdoni.io/dvote/vochain/transaction/vochaintx"
 	"go.vocdoni.io/proto/build/go/models"
@@ -197,3 +199,84 @@ func BenchmarkNewProcess(b *testing.B) {
 	log.Infof("indexed %d new processes, took %s",
 		numProcesses, time.Since(startTime))
 }
+
+func BenchmarkBlockList(b *testing.B) {
+	app := vochain.TestBaseApplication(b)
+
+	idx, err := New(app, Options{DataDir: b.TempDir()})
+	qt.Assert(b, err, qt.IsNil)
+
+	count := 100000
+
+	createDummyBlocks(b, idx, count)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	benchmarkBlockList := func(b *testing.B,
+		limit int, offset int, chainID string, hash string, proposerAddress string,
+	) {
+		b.ReportAllocs()
+		b.ResetTimer()
+
+		for i := 0; i < b.N; i++ {
+			blocks, total, err := idx.BlockList(limit, offset, chainID, hash, proposerAddress)
+			qt.Assert(b, err, qt.IsNil)
+			qt.Assert(b, blocks, qt.HasLen, limit, qt.Commentf("%+v", blocks))
+			qt.Assert(b, blocks[0].TxCount, qt.Equals, int64(0))
+			qt.Assert(b, total, qt.Equals, uint64(count))
+		}
+	}
+
+	// Run sub-benchmarks with different limits and filters
+	b.Run("BlockListLimit1", func(b *testing.B) {
+		benchmarkBlockList(b, 1, 0, "", "", "")
+	})
+
+	b.Run("BlockListLimit10", func(b *testing.B) {
+		benchmarkBlockList(b, 10, 0, "", "", "")
+	})
+
+	b.Run("BlockListLimit100", func(b *testing.B) {
+		benchmarkBlockList(b, 100, 0, "", "", "")
+	})
+
+	b.Run("BlockListOffset", func(b *testing.B) {
+		benchmarkBlockList(b, 10, count/2, "", "", "")
+	})
+
+	b.Run("BlockListWithChainID", func(b *testing.B) {
+		benchmarkBlockList(b, 10, 0, "test", "", "")
+	})
+
+	b.Run("BlockListWithHashSubstr", func(b *testing.B) {
+		benchmarkBlockList(b, 10, 0, "", "cafe", "")
+	})
+	b.Run("BlockListWithHashExact", func(b *testing.B) {
+		benchmarkBlockList(b, 10, 0, "", "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe", "")
+	})
+}
+
+func createDummyBlocks(b *testing.B, idx *Indexer, n int) {
+	idx.blockMu.Lock()
+	defer idx.blockMu.Unlock()
+
+	queries := idx.blockTxQueries()
+	for h := 1; h <= n; h++ {
+		_, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{
+			ChainID: "test",
+			Height:  int64(h),
+			Time:    time.Now(),
+			Hash: nonNullBytes([]byte{
+				0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe,
+				0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe,
+			}),
+			ProposerAddress: nonNullBytes([]byte{0xfe, 0xde}),
+			LastBlockHash:   nonNullBytes([]byte{0xca, 0xfe}),
+		},
+		)
+		qt.Assert(b, err, qt.IsNil)
+	}
+	err := idx.blockTx.Commit()
+	qt.Assert(b, err, qt.IsNil)
+}

vochain/indexer/block.go

@@ -71,22 +71,30 @@ func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress
 	for _, row := range results {
 		list = append(list, indexertypes.BlockFromDBRow(&row))
 	}
-	if len(results) == 0 {
-		return list, 0, nil
+	count, err := idx.CountBlocks(chainID, hash, proposerAddress)
+	if err != nil {
+		return nil, 0, err
 	}
-	return list, uint64(results[0].TotalCount), nil
+	return list, count, nil
 }
 
 // CountBlocks returns how many blocks are indexed.
-func (idx *Indexer) CountBlocks() (uint64, error) {
-	results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{
-		Limit: 1,
+// If all args passed are empty ("") it will return the last block height, as an optimization.
+func (idx *Indexer) CountBlocks(chainID, hash, proposerAddress string) (uint64, error) {
+	if chainID == "" && hash == "" && proposerAddress == "" {
+		count, err := idx.readOnlyQuery.LastBlockHeight(context.TODO())
+		if err != nil {
+			return 0, err
+		}
+		return uint64(count), nil
+	}
+	count, err := idx.readOnlyQuery.CountBlocks(context.TODO(), indexerdb.CountBlocksParams{
+		ChainID:         chainID,
+		HashSubstr:      hash,
+		ProposerAddress: proposerAddress,
 	})
 	if err != nil {
 		return 0, err
 	}
-	if len(results) == 0 {
-		return 0, nil
-	}
-	return uint64(results[0].TotalCount), nil
+	return uint64(count), nil
 }

vochain/indexer/db/blocks.sql.go

@@ -444,7 +444,7 @@ func (idx *Indexer) ReindexBlocks(inTest bool) {
 		return
 	}
 
-	idxBlockCount, err := idx.CountBlocks()
+	idxBlockCount, err := idx.CountBlocks("", "", "")
 	if err != nil {
 		log.Warnf("indexer CountBlocks returned error: %s", err)
 	}

vochain/indexer/db/db.go

@@ -21,11 +21,15 @@ SELECT * FROM blocks
 WHERE hash = ?
 LIMIT 1;
 
+-- name: LastBlockHeight :one
+SELECT height FROM blocks
+ORDER BY height DESC
+LIMIT 1;
+
 -- name: SearchBlocks :many
 SELECT
     b.*,
-    COUNT(t.block_index) AS tx_count,
-    COUNT(*) OVER() AS total_count
+    COUNT(t.block_index) AS tx_count
 FROM blocks AS b
 LEFT JOIN transactions AS t
     ON b.height = t.block_height
@@ -44,3 +48,18 @@ GROUP BY b.height
 ORDER BY b.height DESC
 LIMIT sqlc.arg(limit)
 OFFSET sqlc.arg(offset);
+
+-- name: CountBlocks :one
+SELECT COUNT(*)
+FROM blocks AS b
+WHERE (
+    (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id))
+    AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query
+    AND (
+        sqlc.arg(hash_substr) = ''
+        OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr)))
+        OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0)
+        -- TODO: consider keeping an hash_hex column for faster searches
+    )
+    AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address)))
+);