don't bail for missing meta.js files

Author: krasi-georgiev

compact.go

@@ -157,7 +157,8 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) {
 	for _, dir := range dirs {
 		meta, err := readMetaFile(dir)
 		if err != nil {
-			return nil, err
+			level.Error(c.logger).Log("msg", "reading meta file", "dir", dir, "err", err.Error())
+			continue
 		}
 		dms = append(dms, dirMeta{dir, meta})
 	}
@@ -313,13 +314,15 @@ func (c *LeveledCompactor) Compact(dest string, dirs ...string) (uid ulid.ULID,
 	for _, d := range dirs {
 		b, err := OpenBlock(d, c.chunkPool)
 		if err != nil {
-			return uid, err
+			level.Error(c.logger).Log("msg", "couldn't open a block", "dir", d, "err", err.Error())
+			continue
 		}
 		defer b.Close()
 
 		meta, err := readMetaFile(d)
 		if err != nil {
-			return uid, err
+			level.Error(c.logger).Log("msg", "reading meta file", "dir", d, "err", err.Error())
+			continue
 		}
 
 		metas = append(metas, meta)

db.go

@@ -319,7 +319,7 @@ func (db *DB) retentionCutoff() (b bool, err error) {
 	last := blocks[len(db.blocks)-1]
 
 	mint := last.Meta().MaxTime - int64(db.opts.RetentionDuration)
-	dirs, err := retentionCutoffDirs(db.dir, mint)
+	dirs, err := retentionCutoffDirs(db.logger, db.dir, mint)
 	if err != nil {
 		return false, err
 	}
@@ -433,7 +433,7 @@ func (db *DB) compact() (changes bool, err error) {
 
 // retentionCutoffDirs returns all directories of blocks in dir that are strictly
 // before mint.
-func retentionCutoffDirs(dir string, mint int64) ([]string, error) {
+func retentionCutoffDirs(l log.Logger, dir string, mint int64) ([]string, error) {
 	df, err := fileutil.OpenDir(dir)
 	if err != nil {
 		return nil, errors.Wrapf(err, "open directory")
@@ -450,7 +450,8 @@ func retentionCutoffDirs(dir string, mint int64) ([]string, error) {
 	for _, dir := range dirs {
 		meta, err := readMetaFile(dir)
 		if err != nil {
-			return nil, errors.Wrapf(err, "read block meta %s", dir)
+			level.Error(l).Log("msg", "reading meta file", "dir", dir, "err", err.Error())
+			continue
 		}
 		// The first block we encounter marks that we crossed the boundary
 		// of deletable blocks.
@@ -504,7 +505,8 @@ func (db *DB) reload(deleteable ...string) (err error) {
 	for _, dir := range dirs {
 		meta, err := readMetaFile(dir)
 		if err != nil {
-			return errors.Wrapf(err, "read meta information %s", dir)
+			level.Error(db.logger).Log("msg", "reading meta file", "dir", dir, "err", err.Error())
+			continue
 		}
 		// If the block is pending for deletion, don't add it to the new block set.
 		if stringsContain(deleteable, dir) {
@@ -515,7 +517,8 @@ func (db *DB) reload(deleteable ...string) (err error) {
 		if !ok {
 			b, err = OpenBlock(dir, db.chunkPool)
 			if err != nil {
-				return errors.Wrapf(err, "open block %s", dir)
+				level.Error(db.logger).Log("msg", "couldn't open a block", "dir", dir, "err", err.Error())
+				continue
 			}
 		}