Add account tx and snark job views

Add search
Add charts

Author: Peter Pratscher

Makefile

@@ -5,14 +5,17 @@ BUILDDATE=`date -u +"%Y-%m-%dT%H:%M:%S%:z"`
 PACKAGE=eth2-exporter
 LDFLAGS="-X ${PACKAGE}/version.Version=${VERSION} -X ${PACKAGE}/version.BuildDate=${BUILDDATE} -X ${PACKAGE}/version.GitCommit=${GITCOMMIT} -X ${PACKAGE}/version.GitDate=${GITDATE}"
 
-all: explorer frontend
+all: explorer frontend statistics
 
 lint:
 	golint ./...
 
 explorer:
 	go build --ldflags=${LDFLAGS} -o bin/indexer cmd/indexer/main.go
 
+statistics:
+	go build --ldflags=${LDFLAGS} -o bin/statistics cmd/statistics/main.go
+
 frontend:
 	rm -rf bin/templates
 	rm -rf bin/static

cmd/frontend/main.go

@@ -80,9 +80,13 @@ func main() {
 	router.HandleFunc("/blocks/data", handlers.BlocksData).Methods("GET")
 	router.HandleFunc("/account/{pk}", handlers.Account).Methods("GET")
 	router.HandleFunc("/account/{pk}/data_blocks", handlers.AccountBlocksData).Methods("GET")
+	router.HandleFunc("/account/{pk}/data_txs", handlers.AccountTxData).Methods("GET")
+	router.HandleFunc("/account/{pk}/data_snarkjobs", handlers.AccountSnarkJobsData).Methods("GET")
 	router.HandleFunc("/accounts", handlers.Accounts).Methods("GET")
 	router.HandleFunc("/accounts/data", handlers.AccountsData).Methods("GET")
+	router.HandleFunc("/charts", handlers.Charts).Methods("GET")
 	router.HandleFunc("/status", handlers.Status).Methods("GET")
+	router.HandleFunc("/search", handlers.Search).Methods("POST")
 
 	router.PathPrefix("/").Handler(http.FileServer(http.Dir("static")))
 

cmd/statistics/main.go

@@ -0,0 +1,80 @@
+/*
+ *    Copyright 2020 bitfly gmbh
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package main
+
+import (
+	"coda-explorer/db"
+	"flag"
+	"fmt"
+	"github.com/jmoiron/sqlx"
+	"github.com/sirupsen/logrus"
+	"log"
+	"time"
+)
+
+var logger = logrus.New().WithField("module", "main")
+
+// Helper application to re-generate all statistics
+func main() {
+	dbHost := flag.String("dbHost", "", "Database host")
+	dbPort := flag.String("dbPort", "", "Database port")
+	dbUser := flag.String("dbUser", "", "Database user")
+	dbPassword := flag.String("dbPassword", "", "Database password")
+	dbName := flag.String("dbName", "", "Database name")
+
+	flag.Parse()
+
+	dbConn, err := sqlx.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", *dbUser, *dbPassword, *dbHost, *dbPort, *dbName))
+	if err != nil {
+		logger.Fatal(err)
+	}
+	// The golang postgres sql driver does not properly implement PingContext
+	// therefore we use a timer to catch db connection timeouts
+	dbConnectionTimeout := time.NewTimer(15 * time.Second)
+	go func() {
+		<-dbConnectionTimeout.C
+		log.Fatal("Timeout while connecting to the database")
+	}()
+	err = dbConn.Ping()
+	if err != nil {
+		logger.Fatal(err)
+	}
+	dbConnectionTimeout.Stop()
+
+	logger.Info("database connection established")
+
+	db.DB = dbConn
+	defer db.DB.Close()
+
+	var startTime time.Time
+	err = db.DB.Get(&startTime, "SELECT MIN(ts) FROM blocks")
+	if err != nil {
+		logger.Fatalf("error retrieving start time from blocks table: %v", err)
+	}
+
+	currTime := startTime.Truncate(time.Hour * 24)
+	endTime := time.Now().Truncate(time.Hour * 24)
+	for currTime.Before(endTime) {
+		logger.Infof("exporting statistics for day %v", currTime)
+		err := db.GenerateAndSaveStatistics(currTime)
+		if err != nil {
+			logger.Fatalf("error generating statistics for day %v: %v", currTime, err)
+		}
+		currTime = currTime.Add(time.Hour * 24)
+	}
+	logger.Infof("regenerated all statistics")
+}

handlers/account.go

@@ -68,7 +68,7 @@ func Account(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// BlocksData will return information about blocks
+// AccountBlocksData will return information about blocks mined by an account
 func AccountBlocksData(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
@@ -101,15 +101,12 @@ func AccountBlocksData(w http.ResponseWriter, r *http.Request) {
 
 	var blocksCount int64
 
-	err = db.DB.Get(&blocksCount, "SELECT blocksproposed FROM accounts WHERE publickey = $1", pk)
+	err = db.DB.Get(&blocksCount, "SELECT least(blocksproposed, 10000) FROM accounts WHERE publickey = $1", pk)
 	if err != nil {
 		logger.Errorf("error retrieving blockproposed for account %v: %v", pk, err)
 		http.Error(w, "Internal server error", 503)
 		return
 	}
-	if blocksCount > 10000 {
-		blocksCount = 10000
-	}
 
 	var blocks []*types.Block
 
@@ -154,3 +151,169 @@ func AccountBlocksData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+// AccountBlocksData will return information about tx sent and received by an account
+func AccountTxData(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	q := r.URL.Query()
+
+	draw, err := strconv.ParseInt(q.Get("draw"), 10, 64)
+	if err != nil {
+		logger.Errorf("error converting datatables data parameter from string to int: %v", err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+	start, err := strconv.ParseInt(q.Get("start"), 10, 64)
+	if err != nil {
+		logger.Errorf("error converting datatables start parameter from string to int: %v", err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+	length, err := strconv.ParseInt(q.Get("length"), 10, 64)
+	if err != nil {
+		logger.Errorf("error converting datatables length parameter from string to int: %v", err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+	if length > 100 {
+		length = 100
+	}
+
+	vars := mux.Vars(r)
+	pk := vars["pk"]
+
+	var txCount int64
+
+	err = db.DB.Get(&txCount, "SELECT least(count(*), 10000) FROM accounttransactions WHERE publickey = $1 AND canonical", pk)
+	if err != nil {
+		logger.Errorf("error retrieving tx count for account %v: %v", pk, err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+
+	var txs []*types.TxPageData
+
+	err = db.DB.Select(&txs, `SELECT userjobs.*, blocks.height, blocks.slot, blocks.epoch, blocks.ts
+										FROM accounttransactions 
+										LEFT JOIN userjobs ON accounttransactions.blockstatehash = userjobs.blockstatehash AND accounttransactions.id = userjobs.id
+										LEFT JOIN blocks ON accounttransactions.blockstatehash = blocks.statehash
+										WHERE accounttransactions.publickey = $1 AND accounttransactions.canonical
+										ORDER BY ts DESC LIMIT $2 OFFSET $3`, pk, length, start)
+
+	if err != nil {
+		logger.Errorf("error retrieving tx data for account %v: %v", pk, err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+
+	tableData := make([][]interface{}, len(txs))
+	for i, tx := range txs {
+		tableData[i] = []interface{}{
+			tx.ID,
+			tx.Ts.Unix(),
+			tx.Height,
+			tx.Sender,
+			tx.Recipient,
+			tx.Amount,
+			tx.Fee,
+			tx.Delegation,
+			tx.BlockStateHash,
+		}
+	}
+
+	data := &types.DataTableResponse{
+		Draw:            draw,
+		RecordsTotal:    txCount,
+		RecordsFiltered: txCount,
+		Data:            tableData,
+	}
+
+	err = json.NewEncoder(w).Encode(data)
+	if err != nil {
+		logger.Errorf("error enconding json response for %v route: %v", r.URL.String(), err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+}
+
+// AccountBlocksData will return information about blocks mined by an account
+func AccountSnarkJobsData(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	q := r.URL.Query()
+
+	draw, err := strconv.ParseInt(q.Get("draw"), 10, 64)
+	if err != nil {
+		logger.Errorf("error converting datatables data parameter from string to int: %v", err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+	start, err := strconv.ParseInt(q.Get("start"), 10, 64)
+	if err != nil {
+		logger.Errorf("error converting datatables start parameter from string to int: %v", err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+	length, err := strconv.ParseInt(q.Get("length"), 10, 64)
+	if err != nil {
+		logger.Errorf("error converting datatables length parameter from string to int: %v", err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+	if length > 100 {
+		length = 100
+	}
+
+	vars := mux.Vars(r)
+	pk := vars["pk"]
+
+	var blocksCount int64
+
+	err = db.DB.Get(&blocksCount, "SELECT least(snarkjobs, 10000) FROM accounts WHERE publickey = $1", pk)
+	if err != nil {
+		logger.Errorf("error retrieving snarkjobs for account %v: %v", pk, err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+
+	var snarkJobs []*types.SnarkJobPageData
+
+	err = db.DB.Select(&snarkJobs, `SELECT snarkjobs.*, blocks.height, blocks.slot, blocks.epoch, blocks.ts
+										FROM snarkjobs 
+										LEFT JOIN blocks On snarkjobs.blockstatehash = blocks.statehash
+										WHERE prover = $1 AND snarkjobs.canonical
+										ORDER BY blocks.height DESC LIMIT $2 OFFSET $3`, pk, length, start)
+
+	if err != nil {
+		logger.Errorf("error retrieving snark job data for account %v: %v", pk, err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+
+	tableData := make([][]interface{}, len(snarkJobs))
+	for i, sj := range snarkJobs {
+		tableData[i] = []interface{}{
+			sj.Jobids,
+			sj.Prover,
+			sj.Fee,
+			sj.Ts.Unix(),
+			sj.Height,
+			sj.BlockStateHash,
+		}
+	}
+
+	data := &types.DataTableResponse{
+		Draw:            draw,
+		RecordsTotal:    blocksCount,
+		RecordsFiltered: blocksCount,
+		Data:            tableData,
+	}
+
+	err = json.NewEncoder(w).Encode(data)
+	if err != nil {
+		logger.Errorf("error enconding json response for %v route: %v", r.URL.String(), err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+}

handlers/charts.go

@@ -0,0 +1,62 @@
+/*
+ *    Copyright 2020 bitfly gmbh
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package handlers
+
+import (
+	"coda-explorer/db"
+	"coda-explorer/templates"
+	"coda-explorer/types"
+	"coda-explorer/version"
+	"html/template"
+	"net/http"
+)
+
+// ChartBlocks will return information about the daily produced blocks using a go template
+var chartsTemplate = template.Must(template.New("blocks").Funcs(templates.GetTemplateFuncs()).ParseFiles("templates/layout.html", "templates/charts.html"))
+
+func Charts(w http.ResponseWriter, r *http.Request) {
+
+	w.Header().Set("Content-Type", "text/html")
+
+	var stats []*types.Statistic
+	err := db.DB.Select(&stats, "SELECT * FROM statistics WHERE value > 0 ORDER BY ts, indicator")
+	if err != nil {
+		logger.Errorf("error retrieving statistcs data for route %v: %v", r.URL.String(), err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+
+	data := &types.PageData{
+		Meta: &types.Meta{
+			Title:       "coda explorer",
+			Description: "",
+			Path:        "",
+		},
+		ShowSyncingMessage: false,
+		Active:             "charts",
+		Data:               stats,
+		Version:            version.Version,
+	}
+
+	err = chartsTemplate.ExecuteTemplate(w, "layout", data)
+
+	if err != nil {
+		logger.Errorf("error executing template for %v route: %v", r.URL.String(), err)
+		http.Error(w, "Internal server error", 503)
+		return
+	}
+}