don't bail for missing meta.js files

Author: krasi-georgiev

compact.go

@@ -157,7 +157,8 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) {
 	for _, dir := range dirs {
 		meta, err := readMetaFile(dir)
 		if err != nil {
-			return nil, err
+			// No need to return an error as this will be deleted when reloading the db.
+			continue
 		}
 		dms = append(dms, dirMeta{dir, meta})
 	}
@@ -313,13 +314,15 @@ func (c *LeveledCompactor) Compact(dest string, dirs ...string) (uid ulid.ULID,
 	for _, d := range dirs {
 		b, err := OpenBlock(d, c.chunkPool)
 		if err != nil {
-			return uid, err
+			level.Error(c.logger).Log("msg", "couldn't open a block", "dir", d, "err", err.Error())
+			continue
 		}
 		defer b.Close()
 
 		meta, err := readMetaFile(d)
 		if err != nil {
-			return uid, err
+			level.Error(c.logger).Log("msg", "reading meta file", "dir", d, "err", err.Error())
+			continue
 		}
 
 		metas = append(metas, meta)

db.go

@@ -319,7 +319,7 @@ func (db *DB) retentionCutoff() (b bool, err error) {
 	last := blocks[len(db.blocks)-1]
 
 	mint := last.Meta().MaxTime - int64(db.opts.RetentionDuration)
-	dirs, err := retentionCutoffDirs(db.dir, mint)
+	dirs, err := retentionCutoffDirs(db.logger, db.dir, mint)
 	if err != nil {
 		return false, err
 	}
@@ -433,7 +433,7 @@ func (db *DB) compact() (changes bool, err error) {
 
 // retentionCutoffDirs returns all directories of blocks in dir that are strictly
 // before mint.
-func retentionCutoffDirs(dir string, mint int64) ([]string, error) {
+func retentionCutoffDirs(l log.Logger, dir string, mint int64) ([]string, error) {
 	df, err := fileutil.OpenDir(dir)
 	if err != nil {
 		return nil, errors.Wrapf(err, "open directory")
@@ -450,7 +450,8 @@ func retentionCutoffDirs(dir string, mint int64) ([]string, error) {
 	for _, dir := range dirs {
 		meta, err := readMetaFile(dir)
 		if err != nil {
-			return nil, errors.Wrapf(err, "read block meta %s", dir)
+			delDirs = append(delDirs, dir)
+			continue
 		}
 		// The first block we encounter marks that we crossed the boundary
 		// of deletable blocks.
@@ -504,10 +505,8 @@ func (db *DB) reload(deleteable ...string) (err error) {
 	for _, dir := range dirs {
 		meta, err := readMetaFile(dir)
 		if err != nil {
-			return errors.Wrapf(err, "read meta information %s", dir)
-		}
-		// If the block is pending for deletion, don't add it to the new block set.
-		if stringsContain(deleteable, dir) {
+			deleteable = append(deleteable, dir)
+			level.Error(db.logger).Log("msg", "dir set for deletion due to error in the meta file", "dir", dir, "err", err.Error())
 			continue
 		}
 
@@ -541,8 +540,14 @@ func (db *DB) reload(deleteable ...string) (err error) {
 		if err := b.Close(); err != nil {
 			level.Warn(db.logger).Log("msg", "closing block failed", "err", err)
 		}
-		if err := os.RemoveAll(b.Dir()); err != nil {
-			level.Warn(db.logger).Log("msg", "deleting block failed", "err", err)
+		deleteable = append(deleteable, b.Dir())
+	}
+
+	for _, d := range deleteable {
+		if _, err := os.Stat(d); err == nil {
+			if err := os.RemoveAll(d); err != nil {
+				level.Warn(db.logger).Log("msg", "deleting block failed", "err", err)
+			}
 		}
 	}
 

repair.go

@@ -30,6 +30,10 @@ func repairBadIndexVersion(logger log.Logger, dir string) error {
 			continue
 		}
 		d = path.Join(dir, d)
+		// Skip dirs with missing meta. These will be deleted when reloading the db.
+		if _, err := os.Stat(filepath.Join(dir, metaFilename)); os.IsNotExist(err) {
+			continue
+		}
 
 		meta, err := readBogusMetaFile(d)
 		if err != nil {