refactored and use printBlock for prety display

Signed-off-by: Krasi Georgiev <[email protected]>

Author: Krasi Georgiev

cmd/tsdb/main.go

@@ -49,6 +49,7 @@ func main() {
 		listCmdHumanReadable = listCmd.Flag("human-readable", "print human readable values").Short('h').Bool()
 		listPath             = listCmd.Arg("db path", "database path (default is benchout/storage)").Default("benchout/storage").String()
 		scanCmd              = cli.Command("scan", "scans the db and lists corrupted blocks")
+		scanCmdHumanReadable = scanCmd.Flag("human-readable", "print human readable values").Short('h').Bool()
 		scanPath             = scanCmd.Arg("dir", "database path (default is current dir ./)").Default("./").ExistingDir()
 	)
 
@@ -60,15 +61,16 @@ func main() {
 			samplesFile: *benchSamplesFile,
 		}
 		wb.run()
+
 	case listCmd.FullCommand():
 		db, err := tsdb.Open(*listPath, nil, nil, nil)
 		if err != nil {
 			exitWithError(err)
 		}
 		printBlocks(db.Blocks(), listCmdHumanReadable)
-	case scanCmd.FullCommand():
 
-		// Scan for temporary files.
+	case scanCmd.FullCommand():
+		fmt.Println("Scanning for temporary files...")
 		var tmpFiles []string
 		filepath.Walk(*scanPath, func(path string, f os.FileInfo, _ error) error {
 			if !f.IsDir() {
@@ -80,48 +82,40 @@ func main() {
 		})
 		if len(tmpFiles) > 0 {
 			fmt.Println(`
-Scanning found some temporary files!
 These are usually caused by a crash or incomplete compaction and 
-are safe to delete as long as you are sure that no other application is currently using this database folder.`)
+are safe to delete as long as you are sure that no other application is currently using this database.`)
 
-			promptDelete(tmpFiles)
+			promptDelete(tmpFiles, scanCmdHumanReadable)
 
 		}
 
-		useless, err := tsdb.Scan(*scanPath)
+		scanner, err := tsdb.NewDBScanner(*scanPath)
 		if err != nil {
 			exitWithError(err)
 		}
 
-		if len(useless) == 0 {
-			fmt.Println("Hooray! The db has no corrupted blocks (or the scan tool is broken).\U0001f638")
-			return
-		}
-
-		for err, paths := range useless {
-			fmt.Println(err)
-			for _, path := range paths {
-				fmt.Println(path)
-			}
-			switch err.(type) {
-			case tsdb.ErrorOpen:
-				fmt.Println(`
-Scanning found some blocks that cannot be opened!
-Deleting these will allow a clean startup, but you will loose all data in the listed time ranges.`)
-			case tsdb.ErrorSequence:
-				fmt.Println(`
-Scanning found some overlapping blocks!
-Deleting these will allow a clean startup, but you will loose all data in the listed time ranges.`)
-			}
-
-			promptDelete(paths)
+		for err, useless := range scanner.Scan() {
+			fmt.Printf("Deleting these will loose all data in the listed time ranges.%v", err)
+			promptDelete(useless, scanCmdHumanReadable)
 		}
 	}
 	flag.CommandLine.Set("log.level", "debug")
 }
 
-func promptDelete(paths []string) {
-	fmt.Println(paths)
+func promptDelete(i interface{}, humanReadable *bool) {
+	var paths []string
+	switch i.(type) {
+	case []*tsdb.Block:
+		blocks := i.([]*tsdb.Block)
+		for _, b := range blocks {
+			paths = append(paths, b.Dir())
+		}
+		printBlocks(blocks, humanReadable)
+	case []string:
+		paths = i.([]string)
+		printTmps(paths, humanReadable)
+	}
+
 	fmt.Printf("Do you want to delete these (y/N)? ")
 	var s string
 	_, err := fmt.Scanln(&s)
@@ -144,7 +138,7 @@ func promptDelete(paths []string) {
 		return
 	}
 	fmt.Printf("%v is not a valid answer \n", s)
-	promptDelete(paths)
+	promptDelete(i, humanReadable)
 }
 
 type writeBenchmark struct {
@@ -425,6 +419,11 @@ func exitWithError(err error) {
 	os.Exit(1)
 }
 
+func printTmps(tmps []string, humanReadable *bool) {
+
+	fmt.Println(tmps)
+}
+
 func printBlocks(blocks []*tsdb.Block, humanReadable *bool) {
 	tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
 	defer tw.Flush()

scan.go

@@ -2,59 +2,93 @@ package tsdb
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/tsdb/chunks"
+	"github.com/prometheus/tsdb/index"
 )
 
-type ErrorOpen string
-
-func (e ErrorOpen) Error() string {
-	return fmt.Sprintf("block open error: %v", e)
+type DBScanner struct {
+	db   *DB
+	dirs []string
 }
 
-type ErrorSequence string
-
-func (e ErrorSequence) Error() string {
-	return fmt.Sprintf("block sequence error %v", e)
-}
+func NewDBScanner(dir string) (*DBScanner, error) {
 
-// Scan checks the integrity of the tsdb for a given directory and returns a list of files and folders that can be deleted.
-// It returns a map of error to explain the reason and slice of dirs for further processing.
-func Scan(dir string) (map[error][]string, error) {
-	useless := make(map[error][]string)
-
-	db := &DB{
-		dir:    dir,
-		logger: nil,
-		opts:   &Options{},
+	dbScanner := &DBScanner{
+		db: &DB{
+			dir:    dir,
+			logger: nil,
+			opts:   &Options{},
+		},
 	}
 
-	dirs, err := blockDirs(db.dir)
+	dirs, err := blockDirs(dir)
 	if err != nil {
-		return nil, errors.Wrap(err, "find blocks")
+		return nil, err
 	}
+	dbScanner.dirs = dirs
+	return dbScanner, nil
+}
 
-	// Scan for blocks that can't be opened.
-	var errOpen ErrorOpen
-	for _, dir := range dirs {
-		b, err := OpenBlock(dir, nil)
-		if err != nil {
-			errOpen = ErrorOpen(err.Error())
-			useless[errOpen] = append(useless[errOpen], dir)
+// Scan checks the integrity of the tsdb and returns a slice of blocks that return an error while processing.
+// The blocks are grouped in a map by the error.
+func (db *DBScanner) Scan() map[error][]*Block {
+
+	uselessBlocks := make(map[error][]*Block)
+	var goodBlocks []*Block
+
+	for _, dir := range db.dirs {
+		var err error
+		block := &Block{
+			dir: dir,
+		}
+		for {
+			meta, err := readMetaFile(dir)
+			if err != nil {
+				err = errors.Wrap(err, "reading block meta")
+				break
+			}
+			block.meta = *meta
+
+			if block.chunkr, err = chunks.NewDirReader(chunkDir(dir), nil); err != nil {
+				err = errors.Wrap(err, "reading block chunk")
+				break
+			}
+
+			if block.indexr, err = index.NewFileReader(filepath.Join(dir, "index")); err != nil {
+				err = errors.Wrap(err, "reading block index")
+				break
+			}
+
+			if block.tombstones, err = readTombstones(dir); err != nil {
+				err = errors.Wrap(err, "reading block tumbstone")
+				break
+			}
 			break
 		}
 
-		db.blocks = append(db.blocks, b)
+		if err != nil {
+			uselessBlocks[err] = append(uselessBlocks[err], block)
+		} else {
+			goodBlocks = append(goodBlocks, block)
+		}
+
 	}
 
 	// Scan for overlaping blocks.
-	if overlaps := validateBlockSequence(db.blocks); overlaps != nil {
-		var dirs []string
-		for _, b := range db.blocks {
-			dirs = append(dirs, b.Dir())
+	for time, overlappingMetas := range validateBlockSequence(goodBlocks) {
+		err := fmt.Errorf("overlapping blocks range: %v , %v", time.Min, time.Min)
+		for _, m := range overlappingMetas {
+			for _, b := range goodBlocks {
+				if m.ULID == b.Meta().ULID {
+					uselessBlocks[err] = append(uselessBlocks[err], b)
+					break
+				}
+			}
 		}
-		useless[ErrorSequence(overlaps.String())] = dirs
-	}
 
-	return useless, nil
+	}
+	return uselessBlocks
 }