CNDB-15508: Query planner metrics

This commit adds new metrics related to the operation of SAI query
planner. The metrics should help checking if the query planner makes
proper decisions by correlating them with the other metrics,
e.g. the metrics of the actual query execution.

Per-query metrics (histograms):
- `RowsToReturnEstimated`: the estimated number of rows to be returned
  by the query
- `RowsToFetchEstimated`: the estimated number of rows the query
  is going to fetch from storage
- `KeysToIterateEstimated`: the estimated number of primary keys to
  read from the indexes when executing the query to completion
- `CostEstimated`: the abstract cost of query execution
- `LogSelectivityEstimated`: minus decimal logarithm of query
  selectivity,  before applying the query LIMIT (0 means the query
  selects all rows, 5 means it selects 10^(-5) = 0.00001 subset
  of rows)
- `IndexReferencesInQuery`: the number of index references in the
  unoptimized query execution plan (the same index may
  be referenced multiple times and counts separately)
- `IndexReferencesInPlan`: the number of index references in the
  optimized query execution plan (the same index may
  be referenced multiple times and counts separately)

Per-table:
- `TotalRowsToReturnEstimated`: the sum of all estimates of returned
  rows from all completed queries
- `TotalRowsToFetchEstimated`: the sum of all estimates of fetched
  rows from all completed queries
- `TotalKeysToIterateEstimated`: the sum of all estimates of iterated
  primary keys from all completed queries
- `TotalCostEstimated`: counts the sum of all cost estimates from
  all completed queries

# Conflicts:
#	src/java/org/apache/cassandra/index/sai/QueryContext.java

Author: Piotr Kołaczkowski

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -638,6 +638,13 @@ public enum CassandraRelevantProperties
      */
     SAI_QUERY_KIND_PER_QUERY_METRICS_ENABLED("cassandra.sai.metrics.query_kind.per_query.enabled", "false"),
 
+    /**
+     * Whether to enable SAI query plan metrics such as the estimated cost, estimated number of rows,
+     * number of indexes used in the original and optimized query plan, etc.
+     * These metrics are counters and histograms.
+     */
+    SAI_QUERY_PLAN_METRICS_ENABLED("cassandra.sai.metrics.query_plan.enabled", "true"),
+
     /**
      * Whether to enable SAI index metrics such as memtable flush metrics, compaction metrics, and disk usage metrics.
      * These metrics include timers, histograms, counters, and gauges for index operations.

src/java/org/apache/cassandra/index/sai/QueryContext.java

@@ -19,11 +19,15 @@
 package org.apache.cassandra.index.sai;
 
 import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.plan.Plan;
 import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.apache.cassandra.utils.MonotonicClock;
 
@@ -88,9 +92,7 @@ public class QueryContext
 
     private float annRerankFloor = 0.0f; // only called from single-threaded setup code
 
-    // Determines the order of using indexes for filtering and sorting.
-    // Null means the query execution order hasn't been decided yet.
-    private FilterSortOrder filterSortOrder = null;
+    private PlanInfo queryPlanInfo;
 
     @VisibleForTesting
     public QueryContext()
@@ -219,12 +221,6 @@ public void addAnnGraphSearchLatency(long val)
         annGraphSearchLatency += val;
     }
 
-    public void setFilterSortOrder(FilterSortOrder filterSortOrder)
-    {
-        checkThreadOwnership();
-        this.filterSortOrder = filterSortOrder;
-    }
-
     public void checkpoint()
     {
         checkThreadOwnership();
@@ -250,17 +246,10 @@ public void updateAnnRerankFloor(float observedFloor)
             annRerankFloor = max(annRerankFloor, observedFloor);
     }
 
-    /**
-     * Determines the order of filtering and sorting operations.
-     * Currently used only by vector search.
-     */
-    public enum FilterSortOrder
+    public void recordQueryPlan(Plan.RowsIteration originalPlan, Plan.RowsIteration optimizedPlan)
     {
-        /** First get the matching keys from the non-vector indexes, then use vector index to return the top K by similarity order */
-        SEARCH_THEN_ORDER,
-
-        /** First get the candidates in ANN order from the vector index, then fetch the rows and filter them until we find K matching the predicates */
-        SCAN_THEN_FILTER
+        if (CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean())
+            this.queryPlanInfo = new PlanInfo(originalPlan, optimizedPlan);
     }
 
     public Snapshot snapshot()
@@ -311,7 +300,9 @@ public static class Snapshot
         public final long triePostingsDecodes;
         public final long queryTimeouts;
         public final long annGraphSearchLatency;
-        public final FilterSortOrder filterSortOrder;
+
+        @Nullable
+        public final PlanInfo queryPlanInfo;
 
         /**
          * Creates a snapshot of all the metrics in the given {@link QueryContext}.
@@ -339,7 +330,38 @@ private Snapshot(QueryContext context)
             triePostingsDecodes = context.triePostingsDecodes;
             queryTimeouts = context.queryTimeouts;
             annGraphSearchLatency = context.annGraphSearchLatency;
-            filterSortOrder = context.filterSortOrder;
+            queryPlanInfo = context.queryPlanInfo;
+        }
+    }
+
+    /**
+     * Captures relevant information about a query plan, both original and optimized.
+     */
+    public static class PlanInfo
+    {
+        public final boolean searchExecutedBeforeOrder;
+        public final boolean filterExecutedAfterOrderedScan;
+
+        public final long costEstimated;
+        public final long rowsToReturnEstimated;
+        public final long rowsToFetchEstimated;
+        public final long keysToIterateEstimated;
+        public final int logSelectivityEstimated;
+
+        public final int indexReferencesInQuery;
+        public final int indexReferencesInPlan;
+
+        public PlanInfo(@Nonnull Plan.RowsIteration originalPlan, @Nonnull Plan.RowsIteration optimizedPlan)
+        {
+            this.costEstimated = Math.round(optimizedPlan.fullCost());
+            this.rowsToReturnEstimated = Math.round(optimizedPlan.expectedRows());
+            this.rowsToFetchEstimated = Math.round(optimizedPlan.estimatedRowsToFetch());
+            this.keysToIterateEstimated = Math.round(optimizedPlan.estimatedKeysToIterate());
+            this.logSelectivityEstimated = Math.min(20, (int) Math.floor(-Math.log10(optimizedPlan.selectivity())));
+            this.indexReferencesInQuery = originalPlan.referencedIndexCount();
+            this.indexReferencesInPlan = optimizedPlan.referencedIndexCount();
+            this.searchExecutedBeforeOrder = optimizedPlan.isSearchThenOrderHybrid();
+            this.filterExecutedAfterOrderedScan = optimizedPlan.isOrderedScanThenFilterHybrid();
         }
     }
 }

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -22,6 +22,8 @@
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nullable;
+
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
@@ -115,7 +117,7 @@ public void record(QueryContext context, ReadCommand command)
         {
             final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(snapshot.totalQueryTimeNs);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+            if (snapshot.queryPlanInfo != null && snapshot.queryPlanInfo.searchExecutedBeforeOrder)
             {
                 Tracing.trace("Index query accessed memtable indexes, {}, and {}, selected {} before ranking, " +
                               "post-filtered {} in {}, and took {} microseconds.",
@@ -199,8 +201,8 @@ public static class PerTable extends AbstractQueryMetrics
         public final Counter totalRowTombstonesFetched;
         public final Counter totalQueriesCompleted;
 
-        public final Counter sortThenFilterQueriesCompleted;
-        public final Counter filterThenSortQueriesCompleted;
+        @Nullable
+        public final QueryPlanMetrics queryPlanMetrics;
 
         /**
          * @param table the table to measure metrics for
@@ -220,9 +222,9 @@ public PerTable(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
             totalRowTombstonesFetched = Metrics.counter(createMetricName("TotalRowTombstonesFetched"));
             totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
             totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
-
-            sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
-            filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
+            queryPlanMetrics = (CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean())
+                                 ? new QueryPlanMetrics()
+                                 : null;
         }
 
         @Override
@@ -243,10 +245,42 @@ public void record(QueryContext.Snapshot snapshot)
             totalRowsReturned.inc(snapshot.rowsReturned);
             totalRowTombstonesFetched.inc(snapshot.rowTombstonesFetched);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SCAN_THEN_FILTER)
-                sortThenFilterQueriesCompleted.inc();
-            else if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
-                filterThenSortQueriesCompleted.inc();
+            QueryContext.PlanInfo queryPlanInfo = snapshot.queryPlanInfo;
+            if (queryPlanInfo != null && queryPlanMetrics != null)
+            {
+                queryPlanMetrics.totalCostEstimated.inc(queryPlanInfo.costEstimated);
+                queryPlanMetrics.totalRowsToReturnEstimated.inc(queryPlanInfo.rowsToReturnEstimated);
+                queryPlanMetrics.totalRowsToFetchEstimated.inc(queryPlanInfo.rowsToFetchEstimated);
+                queryPlanMetrics.totalKeysToIterateEstimated.inc(queryPlanInfo.keysToIterateEstimated);
+
+                if (queryPlanInfo.filterExecutedAfterOrderedScan)
+                    queryPlanMetrics.sortThenFilterQueriesCompleted.inc();
+                if (queryPlanInfo.searchExecutedBeforeOrder)
+                    queryPlanMetrics.filterThenSortQueriesCompleted.inc();
+            }
+        }
+
+        public class QueryPlanMetrics
+        {
+            public final Counter totalRowsToReturnEstimated;
+            public final Counter totalRowsToFetchEstimated;
+            public final Counter totalKeysToIterateEstimated;
+            public final Counter totalCostEstimated;
+
+            public final Counter sortThenFilterQueriesCompleted;
+            public final Counter filterThenSortQueriesCompleted;
+
+
+            public QueryPlanMetrics()
+            {
+                totalRowsToReturnEstimated = Metrics.counter(createMetricName("TotalRowsToReturnEstimated"));
+                totalRowsToFetchEstimated = Metrics.counter(createMetricName("TotalRowsToFetchEstimated"));
+                totalKeysToIterateEstimated = Metrics.counter(createMetricName("TotalKeysToIterateEstimated"));
+                totalCostEstimated = Metrics.counter(createMetricName("TotalCostEstimated"));
+
+                sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
+                filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
+            }
         }
     }
 
@@ -293,6 +327,9 @@ public static class PerQuery extends AbstractQueryMetrics
          */
         public final Timer annGraphSearchLatency;
 
+        @Nullable
+        public final QueryPlanMetrics queryPlanMetrics;
+
         /**
          * @param table the table to measure metrics for
          * @param queryKind an identifier for the kind of query which metrics are being recorded for
@@ -323,6 +360,10 @@ public PerQuery(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
 
             // Key vector metrics that translate to performance
             annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
+
+            queryPlanMetrics = CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean()
+                                 ? new QueryPlanMetrics()
+                                 : null;
         }
 
         @Override
@@ -362,6 +403,75 @@ public void record(QueryContext.Snapshot snapshot)
             {
                 annGraphSearchLatency.update(snapshot.annGraphSearchLatency, TimeUnit.NANOSECONDS);
             }
+
+            QueryContext.PlanInfo queryPlanInfo = snapshot.queryPlanInfo;
+            if (queryPlanInfo != null && queryPlanMetrics != null)
+            {
+                queryPlanMetrics.costEstimated.update(queryPlanInfo.costEstimated);
+                queryPlanMetrics.rowsToReturnEstimated.update(queryPlanInfo.rowsToReturnEstimated);
+                queryPlanMetrics.rowsToFetchEstimated.update(queryPlanInfo.rowsToFetchEstimated);
+                queryPlanMetrics.keysToIterateEstimated.update(queryPlanInfo.keysToIterateEstimated);
+                queryPlanMetrics.logSelectivityEstimated.update(queryPlanInfo.logSelectivityEstimated);
+                queryPlanMetrics.indexReferencesInQuery.update(queryPlanInfo.indexReferencesInQuery);
+                queryPlanMetrics.indexReferencesInPlan.update(queryPlanInfo.indexReferencesInPlan);
+            }
         }
+
+        /// Metrics related to query planning.
+        /// Moved to separate class so they can be enabled/disabled as a group.
+        public class QueryPlanMetrics
+        {
+            /**
+             * Query execution cost as estimated by the planner
+             */
+            public final Histogram costEstimated;
+
+            /**
+             * Number of rows to be returned from the query as estimated by the planner
+             */
+            public final Histogram rowsToReturnEstimated;
+
+            /**
+             * Number of rows to be fetched by the query as estimated by the planner
+             */
+            public final Histogram rowsToFetchEstimated;
+
+            /**
+             * Number of keys to be iterated by the query as estimated by the planner
+             */
+            public final Histogram keysToIterateEstimated;
+
+            /**
+             * Negative decimal logarithm of selectivity of the query, before applying the LIMIT clause.
+             * We use logarithm because selectivity values can be very small (e.g. 10^-9).
+             */
+            public final Histogram logSelectivityEstimated;
+
+            /**
+             * Number of indexes referenced by the optimized query plan.
+             * The same index referenced from unrelated query clauses,
+             * leading to separate index searches, are counted separately.
+             */
+            public final Histogram indexReferencesInPlan;
+
+            /**
+             * Number of indexes referenced by the original query plan before optimization (as stated in the query text)
+             */
+            public final Histogram indexReferencesInQuery;
+
+            QueryPlanMetrics()
+            {
+                costEstimated = Metrics.histogram(createMetricName("CostEstimated"), false);
+                rowsToReturnEstimated = Metrics.histogram(createMetricName("RowsToReturnEstimated"), true);
+                rowsToFetchEstimated = Metrics.histogram(createMetricName("RowsToFetchEstimated"), true);
+                keysToIterateEstimated = Metrics.histogram(createMetricName("KeysToIterateEstimated"), true);
+                logSelectivityEstimated = Metrics.histogram(createMetricName("LogSelectivityEstimated"), true);
+                indexReferencesInPlan = Metrics.histogram(createMetricName("IndexReferencesInPlan"), true);
+                indexReferencesInQuery = Metrics.histogram(createMetricName("IndexReferencesInQuery"), false);
+            }
+        }
+
     }
+
+
 }