CNDB-15508: Query planner metrics

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -638,6 +638,13 @@ public enum CassandraRelevantProperties
      */
     SAI_QUERY_KIND_PER_QUERY_METRICS_ENABLED("cassandra.sai.metrics.query_kind.per_query.enabled", "false"),
 
+    /**
+     * Whether to enable SAI query plan metrics such as the estimated cost, estimated number of rows,
+     * number of indexes used in the original and optimized query plan, etc.
+     * These metrics are counters and histograms.
+     */
+    SAI_QUERY_PLAN_METRICS_ENABLED("cassandra.sai.metrics.query_plan.enabled", "true"),
+
     /**
      * Whether to enable SAI index metrics such as memtable flush metrics, compaction metrics, and disk usage metrics.
      * These metrics include timers, histograms, counters, and gauges for index operations.

src/java/org/apache/cassandra/index/sai/QueryContext.java

@@ -19,11 +19,15 @@
 package org.apache.cassandra.index.sai;
 
 import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.plan.Plan;
 import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.apache.cassandra.utils.MonotonicClock;
 
@@ -88,9 +92,7 @@ public class QueryContext
 
     private float annRerankFloor = 0.0f; // only called from single-threaded setup code
 
-    // Determines the order of using indexes for filtering and sorting.
-    // Null means the query execution order hasn't been decided yet.
-    private FilterSortOrder filterSortOrder = null;
+    private PlanInfo queryPlanInfo;
 
     @VisibleForTesting
     public QueryContext()
@@ -219,12 +221,6 @@ public void addAnnGraphSearchLatency(long val)
         annGraphSearchLatency += val;
     }
 
-    public void setFilterSortOrder(FilterSortOrder filterSortOrder)
-    {
-        checkThreadOwnership();
-        this.filterSortOrder = filterSortOrder;
-    }
-
     public void checkpoint()
     {
         checkThreadOwnership();
@@ -250,17 +246,10 @@ public void updateAnnRerankFloor(float observedFloor)
             annRerankFloor = max(annRerankFloor, observedFloor);
     }
 
-    /**
-     * Determines the order of filtering and sorting operations.
-     * Currently used only by vector search.
-     */
-    public enum FilterSortOrder
+    public void recordQueryPlan(Plan.RowsIteration originalPlan, Plan.RowsIteration optimizedPlan)
     {
-        /** First get the matching keys from the non-vector indexes, then use vector index to return the top K by similarity order */
-        SEARCH_THEN_ORDER,
-
-        /** First get the candidates in ANN order from the vector index, then fetch the rows and filter them until we find K matching the predicates */
-        SCAN_THEN_FILTER
+        if (CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean())
+            this.queryPlanInfo = new PlanInfo(originalPlan, optimizedPlan);
     }
 
     public Snapshot snapshot()
@@ -311,7 +300,9 @@ public static class Snapshot
         public final long triePostingsDecodes;
         public final long queryTimeouts;
         public final long annGraphSearchLatency;
-        public final FilterSortOrder filterSortOrder;
+
+        @Nullable
+        public final PlanInfo queryPlanInfo;
 
         /**
          * Creates a snapshot of all the metrics in the given {@link QueryContext}.
@@ -339,7 +330,38 @@ private Snapshot(QueryContext context)
             triePostingsDecodes = context.triePostingsDecodes;
             queryTimeouts = context.queryTimeouts;
             annGraphSearchLatency = context.annGraphSearchLatency;
-            filterSortOrder = context.filterSortOrder;
+            queryPlanInfo = context.queryPlanInfo;
+        }
+    }
+
+    /**
+     * Captures relevant information about a query plan, both original and optimized.
+     */
+    public static class PlanInfo
+    {
+        public final boolean searchExecutedBeforeOrder;
+        public final boolean filterExecutedAfterOrderedScan;
+
+        public final long costEstimated;
+        public final long rowsToReturnEstimated;
+        public final long rowsToFetchEstimated;
+        public final long keysToIterateEstimated;
+        public final int logSelectivityEstimated;
+
+        public final int indexReferencesInQuery;
+        public final int indexReferencesInPlan;
+
+        public PlanInfo(@Nonnull Plan.RowsIteration originalPlan, @Nonnull Plan.RowsIteration optimizedPlan)
+        {
+            this.costEstimated = Math.round(optimizedPlan.fullCost());
+            this.rowsToReturnEstimated = Math.round(optimizedPlan.expectedRows());
+            this.rowsToFetchEstimated = Math.round(optimizedPlan.estimatedRowsToFetch());
+            this.keysToIterateEstimated = Math.round(optimizedPlan.estimatedKeysToIterate());
+            this.logSelectivityEstimated = Math.min(20, (int) Math.floor(-Math.log10(optimizedPlan.selectivity())));
+            this.indexReferencesInQuery = originalPlan.referencedIndexCount();
+            this.indexReferencesInPlan = optimizedPlan.referencedIndexCount();
+            this.searchExecutedBeforeOrder = optimizedPlan.isSearchThenOrderHybrid();
+            this.filterExecutedAfterOrderedScan = optimizedPlan.isOrderedScanThenFilterHybrid();
         }
     }
 }

src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java

@@ -34,6 +34,9 @@
 import com.google.common.collect.Streams;
 import com.google.common.util.concurrent.Runnables;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.DecoratedKey;
@@ -55,6 +58,7 @@
 import org.apache.cassandra.index.sai.memory.MemoryIndex.PkWithFrequency;
 import org.apache.cassandra.index.sai.plan.Expression;
 import org.apache.cassandra.index.sai.plan.Orderer;
+import org.apache.cassandra.index.sai.plan.QueryController;
 import org.apache.cassandra.index.sai.utils.BM25Utils;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
 import org.apache.cassandra.index.sai.utils.PrimaryKeyWithByteComparable;
@@ -74,6 +78,8 @@
 
 public class TrieMemtableIndex extends AbstractMemtableIndex
 {
+    private static final Logger logger = LoggerFactory.getLogger(TrieMemtableIndex.class);
+
     private final ShardBoundaries boundaries;
     private final MemoryIndex[] rangeIndexes;
     private final IndexContext indexContext;
@@ -104,6 +110,8 @@ public TrieMemtableIndex(IndexContext indexContext, Memtable memtable, int shard
         }
         this.sensorContext = Context.from(indexContext);
         this.requestTracker = RequestTracker.instance;
+
+        logger.debug("Creating TrieMemtableIndex for index {} with shard count: {}", indexContext.getIndexName(), boundaries.shardCount());
     }
 
     @Override

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -22,6 +22,8 @@
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nullable;
+
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
@@ -115,7 +117,7 @@ public void record(QueryContext context, ReadCommand command)
         {
             final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(snapshot.totalQueryTimeNs);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+            if (snapshot.queryPlanInfo != null && snapshot.queryPlanInfo.searchExecutedBeforeOrder)
             {
                 Tracing.trace("Index query accessed memtable indexes, {}, and {}, selected {} before ranking, " +
                               "post-filtered {} in {}, and took {} microseconds.",
@@ -199,8 +201,8 @@ public static class PerTable extends AbstractQueryMetrics
         public final Counter totalRowTombstonesFetched;
         public final Counter totalQueriesCompleted;
 
-        public final Counter sortThenFilterQueriesCompleted;
-        public final Counter filterThenSortQueriesCompleted;
+        @Nullable
+        public final QueryPlanMetrics queryPlanMetrics;
 
         /**
          * @param table the table to measure metrics for
@@ -220,9 +222,9 @@ public PerTable(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
             totalRowTombstonesFetched = Metrics.counter(createMetricName("TotalRowTombstonesFetched"));
             totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
             totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
-
-            sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
-            filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
+            queryPlanMetrics = (CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean())
+                                 ? new QueryPlanMetrics()
+                                 : null;
         }
 
         @Override
@@ -243,10 +245,42 @@ public void record(QueryContext.Snapshot snapshot)
             totalRowsReturned.inc(snapshot.rowsReturned);
             totalRowTombstonesFetched.inc(snapshot.rowTombstonesFetched);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SCAN_THEN_FILTER)
-                sortThenFilterQueriesCompleted.inc();
-            else if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
-                filterThenSortQueriesCompleted.inc();
+            QueryContext.PlanInfo queryPlanInfo = snapshot.queryPlanInfo;
+            if (queryPlanInfo != null && queryPlanMetrics != null)
+            {
+                queryPlanMetrics.totalCostEstimated.inc(queryPlanInfo.costEstimated);
+                queryPlanMetrics.totalRowsToReturnEstimated.inc(queryPlanInfo.rowsToReturnEstimated);
+                queryPlanMetrics.totalRowsToFetchEstimated.inc(queryPlanInfo.rowsToFetchEstimated);
+                queryPlanMetrics.totalKeysToIterateEstimated.inc(queryPlanInfo.keysToIterateEstimated);
+
+                if (queryPlanInfo.filterExecutedAfterOrderedScan)
+                    queryPlanMetrics.sortThenFilterQueriesCompleted.inc();
+                if (queryPlanInfo.searchExecutedBeforeOrder)
+                    queryPlanMetrics.filterThenSortQueriesCompleted.inc();
+            }
+        }
+
+        public class QueryPlanMetrics
+        {
+            public final Counter totalRowsToReturnEstimated;
+            public final Counter totalRowsToFetchEstimated;
+            public final Counter totalKeysToIterateEstimated;
+            public final Counter totalCostEstimated;
+
+            public final Counter sortThenFilterQueriesCompleted;
+            public final Counter filterThenSortQueriesCompleted;
+
+
+            public QueryPlanMetrics()
+            {
+                totalRowsToReturnEstimated = Metrics.counter(createMetricName("TotalRowsToReturnEstimated"));
+                totalRowsToFetchEstimated = Metrics.counter(createMetricName("TotalRowsToFetchEstimated"));
+                totalKeysToIterateEstimated = Metrics.counter(createMetricName("TotalKeysToIterateEstimated"));
+                totalCostEstimated = Metrics.counter(createMetricName("TotalCostEstimated"));
+
+                sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
+                filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
+            }
         }
     }
 
@@ -293,6 +327,9 @@ public static class PerQuery extends AbstractQueryMetrics
          */
         public final Timer annGraphSearchLatency;
 
+        @Nullable
+        public final QueryPlanMetrics queryPlanMetrics;
+
         /**
          * @param table the table to measure metrics for
          * @param queryKind an identifier for the kind of query which metrics are being recorded for
@@ -323,6 +360,10 @@ public PerQuery(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
 
             // Key vector metrics that translate to performance
             annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
+
+            queryPlanMetrics = CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean()
+                                 ? new QueryPlanMetrics()
+                                 : null;
         }
 
         @Override
@@ -362,6 +403,75 @@ public void record(QueryContext.Snapshot snapshot)
             {
                 annGraphSearchLatency.update(snapshot.annGraphSearchLatency, TimeUnit.NANOSECONDS);
             }
+
+            QueryContext.PlanInfo queryPlanInfo = snapshot.queryPlanInfo;
+            if (queryPlanInfo != null && queryPlanMetrics != null)
+            {
+                queryPlanMetrics.costEstimated.update(queryPlanInfo.costEstimated);
+                queryPlanMetrics.rowsToReturnEstimated.update(queryPlanInfo.rowsToReturnEstimated);
+                queryPlanMetrics.rowsToFetchEstimated.update(queryPlanInfo.rowsToFetchEstimated);
+                queryPlanMetrics.keysToIterateEstimated.update(queryPlanInfo.keysToIterateEstimated);
+                queryPlanMetrics.logSelectivityEstimated.update(queryPlanInfo.logSelectivityEstimated);
+                queryPlanMetrics.indexReferencesInQuery.update(queryPlanInfo.indexReferencesInQuery);
+                queryPlanMetrics.indexReferencesInPlan.update(queryPlanInfo.indexReferencesInPlan);
+            }
         }
+
+        /// Metrics related to query planning.
+        /// Moved to separate class so they can be enabled/disabled as a group.
+        public class QueryPlanMetrics
+        {
+            /**
+             * Query execution cost as estimated by the planner
+             */
+            public final Histogram costEstimated;
+
+            /**
+             * Number of rows to be returned from the query as estimated by the planner
+             */
+            public final Histogram rowsToReturnEstimated;
+
+            /**
+             * Number of rows to be fetched by the query as estimated by the planner
+             */
+            public final Histogram rowsToFetchEstimated;
+
+            /**
+             * Number of keys to be iterated by the query as estimated by the planner
+             */
+            public final Histogram keysToIterateEstimated;
+
+            /**
+             * Negative decimal logarithm of selectivity of the query, before applying the LIMIT clause.
+             * We use logarithm because selectivity values can be very small (e.g. 10^-9).
+             */
+            public final Histogram logSelectivityEstimated;
+
+            /**
+             * Number of indexes referenced by the optimized query plan.
+             * The same index referenced from unrelated query clauses,
+             * leading to separate index searches, are counted separately.
+             */
+            public final Histogram indexReferencesInPlan;
+
+            /**
+             * Number of indexes referenced by the original query plan before optimization (as stated in the query text)
+             */
+            public final Histogram indexReferencesInQuery;
+
+            QueryPlanMetrics()
+            {
+                costEstimated = Metrics.histogram(createMetricName("CostEstimated"), false);
+                rowsToReturnEstimated = Metrics.histogram(createMetricName("RowsToReturnEstimated"), true);
+                rowsToFetchEstimated = Metrics.histogram(createMetricName("RowsToFetchEstimated"), true);
+                keysToIterateEstimated = Metrics.histogram(createMetricName("KeysToIterateEstimated"), true);
+                logSelectivityEstimated = Metrics.histogram(createMetricName("LogSelectivityEstimated"), true);
+                indexReferencesInPlan = Metrics.histogram(createMetricName("IndexReferencesInPlan"), true);
+                indexReferencesInQuery = Metrics.histogram(createMetricName("IndexReferencesInQuery"), false);
+            }
+        }
+
     }
+
+
 }