db: log extreme values for deletion estimates

Check `PointDeletionsBytesEstimate` and `RangeDeletionsBytesEstimate`
for extreme values and log them (at most once every 30s). This is to
help identify the cause of very high level scores observed in a
cluster.

Author: RaduBerinde

compaction.go

@@ -3259,7 +3259,7 @@ func (c *compaction) makeVersionEdit(result compact.Result) (*versionEdit, error
 		// If the file didn't contain any range deletions, we can fill its
 		// table stats now, avoiding unnecessarily loading the table later.
 		maybeSetStatsFromProperties(
-			fileMeta.PhysicalMeta(), &t.WriterMeta.Properties.CommonProperties,
+			fileMeta.PhysicalMeta(), &t.WriterMeta.Properties.CommonProperties, c.logger,
 		)
 
 		if t.WriterMeta.HasPointKeys {

ingest.go

@@ -357,7 +357,7 @@ func ingestLoad1(
 	// disallowing removal of an open file. Under MemFS, if we don't populate
 	// meta.Stats here, the file will be loaded into the file cache for
 	// calculating stats before we can remove the original link.
-	maybeSetStatsFromProperties(meta.PhysicalMeta(), &r.Properties.CommonProperties)
+	maybeSetStatsFromProperties(meta.PhysicalMeta(), &r.Properties.CommonProperties, opts.Logger)
 
 	{
 		iter, err := r.NewIter(sstable.NoTransforms, nil /* lower */, nil /* upper */, sstable.AssertNoBlobHandles)

table_stats.go

@@ -8,7 +8,9 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"time"
 
+	"github.com/cockroachdb/crlib/crtime"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
@@ -17,6 +19,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/sstable/block"
+	"github.com/cockroachdb/redact"
 )
 
 // In-memory statistics about tables help inform compaction picking, but may
@@ -131,6 +134,7 @@ func (d *DB) collectTableStats() bool {
 	for _, c := range collected {
 		c.tableMetadata.Stats = c.TableStats
 		maybeCompact = maybeCompact || fileCompensation(c.tableMetadata) > 0
+		sanityCheckStats(c.tableMetadata, d.opts.Logger, "collected stats")
 		c.tableMetadata.StatsMarkValid()
 	}
 
@@ -653,7 +657,29 @@ func (d *DB) estimateReclaimedSizeBeneath(
 	return estimate, hintSeqNum, nil
 }
 
-func maybeSetStatsFromProperties(meta *tableMetadata, props *sstable.CommonProperties) bool {
+var lastSanityCheckStatsLog crtime.AtomicMono
+
+func sanityCheckStats(meta *tableMetadata, logger Logger, info string) {
+	// Values for PointDeletionsBytesEstimate and RangeDeletionsBytesEstimate that
+	// exceed this value are most likely indicative of a bug.
+	const maxDeletionBytesEstimate = 16 << 30 // 16 GiB
+
+	if meta.Stats.PointDeletionsBytesEstimate > maxDeletionBytesEstimate ||
+		meta.Stats.RangeDeletionsBytesEstimate > maxDeletionBytesEstimate {
+		if v := lastSanityCheckStatsLog.Load(); v == 0 || v.Elapsed() > 30*time.Second {
+			logger.Errorf("%s: table %s has extreme deletion bytes estimates: point=%d range=%d",
+				info, meta.FileNum,
+				redact.Safe(meta.Stats.PointDeletionsBytesEstimate),
+				redact.Safe(meta.Stats.RangeDeletionsBytesEstimate),
+			)
+			lastSanityCheckStatsLog.Store(crtime.NowMono())
+		}
+	}
+}
+
+func maybeSetStatsFromProperties(
+	meta *tableMetadata, props *sstable.CommonProperties, logger Logger,
+) bool {
 	// If a table contains range deletions or range key deletions, we defer the
 	// stats collection. There are two main reasons for this:
 	//
@@ -697,6 +723,7 @@ func maybeSetStatsFromProperties(meta *tableMetadata, props *sstable.CommonPrope
 	meta.Stats.ValueBlocksSize = props.ValueBlocksSize
 	meta.Stats.CompressionType = block.CompressionFromString(props.CompressionName)
 	meta.StatsMarkValid()
+	sanityCheckStats(meta, logger, "stats from properties")
 	return true
 }