Update src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

Co-authored-by: Andrés de la Peña <[email protected]>

Author: 2 people

src/java/org/apache/cassandra/index/sai/QueryContext.java

@@ -246,7 +246,7 @@ public void updateAnnRerankFloor(float observedFloor)
             annRerankFloor = max(annRerankFloor, observedFloor);
     }
 
-    public void recordQueryPlan(Plan.RowsIteration originalPlan, Plan.RowsIteration.RowsIteration optimizedPlan)
+    public void recordQueryPlan(Plan.RowsIteration originalPlan, Plan.RowsIteration optimizedPlan)
     {
         if (CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean())
             this.queryPlanInfo = new PlanInfo(originalPlan, optimizedPlan);
@@ -342,22 +342,22 @@ public static class PlanInfo
         public final boolean searchExecutedBeforeOrder;
         public final boolean filterExecutedAfterOrderedScan;
 
-        public final double rowsToReturnEstimated;
-        public final double rowsToFetchEstimated;
-        public final double keysToIterateEstimated;
-        public final double selectivityEstimated;
-        public final double costEstimated;
+        public final long costEstimated;
+        public final long rowsToReturnEstimated;
+        public final long rowsToFetchEstimated;
+        public final long keysToIterateEstimated;
+        public final int logSelectivityEstimated;
 
         public final int indexReferencesInQuery;
         public final int indexReferencesInPlan;
 
         public PlanInfo(@Nonnull Plan.RowsIteration originalPlan, @Nonnull Plan.RowsIteration optimizedPlan)
         {
-            this.costEstimated = optimizedPlan.fullCost();
-            this.rowsToReturnEstimated = optimizedPlan.expectedRows();
-            this.rowsToFetchEstimated = optimizedPlan.estimatedRowsToFetch();
-            this.keysToIterateEstimated = optimizedPlan.estimatedKeysToIterate();
-            this.selectivityEstimated = optimizedPlan.selectivity();
+            this.costEstimated = Math.round(optimizedPlan.fullCost());
+            this.rowsToReturnEstimated = Math.round(optimizedPlan.expectedRows());
+            this.rowsToFetchEstimated = Math.round(optimizedPlan.estimatedRowsToFetch());
+            this.keysToIterateEstimated = Math.round(optimizedPlan.estimatedKeysToIterate());
+            this.logSelectivityEstimated = Math.min(20, (int) Math.floor(-Math.log10(optimizedPlan.selectivity())));
             this.indexReferencesInQuery = originalPlan.referencedIndexCount();
             this.indexReferencesInPlan = optimizedPlan.referencedIndexCount();
             this.searchExecutedBeforeOrder = optimizedPlan.isSearchThenOrderHybrid();

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -22,6 +22,8 @@
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nullable;
+
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
@@ -199,13 +201,8 @@ public static class PerTable extends AbstractQueryMetrics
         public final Counter totalRowTombstonesFetched;
         public final Counter totalQueriesCompleted;
 
-        public final Counter totalRowsToReturnEstimated;
-        public final Counter totalRowsToFetchEstimated;
-        public final Counter totalKeysToIterateEstimated;
-        public final Counter totalCostEstimated;
-
-        public final Counter sortThenFilterQueriesCompleted;
-        public final Counter filterThenSortQueriesCompleted;
+        @Nullable
+        public final QueryPlanMetrics queryPlanMetrics;
 
         /**
          * @param table the table to measure metrics for
@@ -225,13 +222,9 @@ public PerTable(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
             totalRowTombstonesFetched = Metrics.counter(createMetricName("TotalRowTombstonesFetched"));
             totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
             totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
-            totalRowsToReturnEstimated = Metrics.counter(createMetricName("TotalRowsToReturnEstimated"));
-            totalRowsToFetchEstimated = Metrics.counter(createMetricName("TotalRowsToFetchEstimated"));
-            totalKeysToIterateEstimated = Metrics.counter(createMetricName("TotalKeysToIterateEstimated"));
-            totalCostEstimated = Metrics.counter(createMetricName("TotalCostEstimated"));
-
-            sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
-            filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
+            queryPlanMetrics = (CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean())
+                                 ? new QueryPlanMetrics()
+                                 : null;
         }
 
         @Override
@@ -253,17 +246,40 @@ public void record(QueryContext.Snapshot snapshot)
             totalRowTombstonesFetched.inc(snapshot.rowTombstonesFetched);
 
             QueryContext.PlanInfo queryPlanInfo = snapshot.queryPlanInfo;
-            if (queryPlanInfo != null)
+            if (queryPlanInfo != null && queryPlanMetrics != null)
             {
-                totalCostEstimated.inc(Math.round(queryPlanInfo.costEstimated));
-                totalRowsToReturnEstimated.inc(Math.round(queryPlanInfo.rowsToReturnEstimated));
-                totalRowsToFetchEstimated.inc(Math.round(queryPlanInfo.rowsToFetchEstimated));
-                totalKeysToIterateEstimated.inc(Math.round(queryPlanInfo.keysToIterateEstimated));
+                queryPlanMetrics.totalCostEstimated.inc(queryPlanInfo.costEstimated);
+                queryPlanMetrics.totalRowsToReturnEstimated.inc(queryPlanInfo.rowsToReturnEstimated);
+                queryPlanMetrics.totalRowsToFetchEstimated.inc(queryPlanInfo.rowsToFetchEstimated);
+                queryPlanMetrics.totalKeysToIterateEstimated.inc(queryPlanInfo.keysToIterateEstimated);
 
                 if (queryPlanInfo.filterExecutedAfterOrderedScan)
-                    sortThenFilterQueriesCompleted.inc();
+                    queryPlanMetrics.sortThenFilterQueriesCompleted.inc();
                 if (queryPlanInfo.searchExecutedBeforeOrder)
-                    filterThenSortQueriesCompleted.inc();
+                    queryPlanMetrics.filterThenSortQueriesCompleted.inc();
+            }
+        }
+
+        public class QueryPlanMetrics
+        {
+            public final Counter totalRowsToReturnEstimated;
+            public final Counter totalRowsToFetchEstimated;
+            public final Counter totalKeysToIterateEstimated;
+            public final Counter totalCostEstimated;
+
+            public final Counter sortThenFilterQueriesCompleted;
+            public final Counter filterThenSortQueriesCompleted;
+
+
+            public QueryPlanMetrics()
+            {
+                totalRowsToReturnEstimated = Metrics.counter(createMetricName("TotalRowsToReturnEstimated"));
+                totalRowsToFetchEstimated = Metrics.counter(createMetricName("TotalRowsToFetchEstimated"));
+                totalKeysToIterateEstimated = Metrics.counter(createMetricName("TotalKeysToIterateEstimated"));
+                totalCostEstimated = Metrics.counter(createMetricName("TotalCostEstimated"));
+
+                sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
+                filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
             }
         }
     }
@@ -311,33 +327,8 @@ public static class PerQuery extends AbstractQueryMetrics
          */
         public final Timer annGraphSearchLatency;
 
-        /** Query execution cost as estimated by the planner */
-        public final Histogram costEstimated;
-
-        /** Number of rows to be returned from the query as estimated by the planner */
-        public final Histogram rowsToReturnEstimated;
-
-        /** Number of rows to be fetched by the query as estimated by the planner */
-        public final Histogram rowsToFetchEstimated;
-
-        /** Number of rows to be fetched by the query as estimated by the planner */
-        public final Histogram keysToIterateEstimated;
-
-        /**
-         * Negative deceimal logarithm of selectivity of the query, before applying the LIMIT clause.
-         * We use logarithm because selectivity values can be very small (e.g. 10^-9).
-         */
-        public final Histogram logSelectivityEstimated;
-
-        /**
-         * Number of indexes referenced by the optimized query plan.
-         * The same index referenced from unrelated query clauses,
-         * leading to separate index searches, are counted separately.
-         */
-        public final Histogram indexReferencesInPlan;
-
-        /** Number of indexes referenced by the original query plan before optimization (as stated in the query text) */
-        public final Histogram indexReferencesInQuery;
+        @Nullable
+        public final QueryPlanMetrics queryPlanMetrics;
 
         /**
          * @param table the table to measure metrics for
@@ -370,13 +361,9 @@ public PerQuery(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
             // Key vector metrics that translate to performance
             annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
 
-            costEstimated = Metrics.histogram(createMetricName("CostEstimated"), false);
-            rowsToReturnEstimated = Metrics.histogram(createMetricName("RowsToReturnEstimated"), true);
-            rowsToFetchEstimated = Metrics.histogram(createMetricName("RowsToFetchEstimated"), true);
-            keysToIterateEstimated = Metrics.histogram(createMetricName("KeysToIterateEstimated"), true);
-            logSelectivityEstimated = Metrics.histogram(createMetricName("LogSelectivityEstimated"), true);
-            indexReferencesInPlan = Metrics.histogram(createMetricName("IndexReferencesInPlan"), true);
-            indexReferencesInQuery = Metrics.histogram(createMetricName("IndexReferencesInQuery"), false);
+            queryPlanMetrics = CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean()
+                                 ? new QueryPlanMetrics()
+                                 : null;
         }
 
         @Override
@@ -418,17 +405,73 @@ public void record(QueryContext.Snapshot snapshot)
             }
 
             QueryContext.PlanInfo queryPlanInfo = snapshot.queryPlanInfo;
-            if (queryPlanInfo != null)
+            if (queryPlanInfo != null && queryPlanMetrics != null)
             {
-                costEstimated.update(Math.round(queryPlanInfo.costEstimated));
-                rowsToReturnEstimated.update(Math.round(queryPlanInfo.rowsToReturnEstimated));
-                rowsToFetchEstimated.update(Math.round(queryPlanInfo.rowsToFetchEstimated));
-                keysToIterateEstimated.update(Math.round(queryPlanInfo.keysToIterateEstimated));
-                double logSelectivity = -Math.log10(queryPlanInfo.selectivityEstimated);
-                logSelectivityEstimated.update((int) (Math.min(20, Math.floor(logSelectivity))));
-                indexReferencesInQuery.update(queryPlanInfo.indexReferencesInQuery);
-                indexReferencesInPlan.update(queryPlanInfo.indexReferencesInPlan);
+                queryPlanMetrics.costEstimated.update(queryPlanInfo.costEstimated);
+                queryPlanMetrics.rowsToReturnEstimated.update(queryPlanInfo.rowsToReturnEstimated);
+                queryPlanMetrics.rowsToFetchEstimated.update(queryPlanInfo.rowsToFetchEstimated);
+                queryPlanMetrics.keysToIterateEstimated.update(queryPlanInfo.keysToIterateEstimated);
+                queryPlanMetrics.logSelectivityEstimated.update(queryPlanInfo.logSelectivityEstimated);
+                queryPlanMetrics.indexReferencesInQuery.update(queryPlanInfo.indexReferencesInQuery);
+                queryPlanMetrics.indexReferencesInPlan.update(queryPlanInfo.indexReferencesInPlan);
             }
         }
+
+        /// Metrics related to query planning.
+        /// Moved to separate class so they can be enabled/disabled as a group.
+        public class QueryPlanMetrics
+        {
+            /**
+             * Query execution cost as estimated by the planner
+             */
+            public final Histogram costEstimated;
+
+            /**
+             * Number of rows to be returned from the query as estimated by the planner
+             */
+            public final Histogram rowsToReturnEstimated;
+
+            /**
+             * Number of rows to be fetched by the query as estimated by the planner
+             */
+            public final Histogram rowsToFetchEstimated;
+
+            /**
+             * Number of keys to be iterated by the query as estimated by the planner
+             */
+            public final Histogram keysToIterateEstimated;
+
+            /**
+             * Negative decimal logarithm of selectivity of the query, before applying the LIMIT clause.
+             * We use logarithm because selectivity values can be very small (e.g. 10^-9).
+             */
+            public final Histogram logSelectivityEstimated;
+
+            /**
+             * Number of indexes referenced by the optimized query plan.
+             * The same index referenced from unrelated query clauses,
+             * leading to separate index searches, are counted separately.
+             */
+            public final Histogram indexReferencesInPlan;
+
+            /**
+             * Number of indexes referenced by the original query plan before optimization (as stated in the query text)
+             */
+            public final Histogram indexReferencesInQuery;
+
+            QueryPlanMetrics()
+            {
+                costEstimated = Metrics.histogram(createMetricName("CostEstimated"), false);
+                rowsToReturnEstimated = Metrics.histogram(createMetricName("RowsToReturnEstimated"), true);
+                rowsToFetchEstimated = Metrics.histogram(createMetricName("RowsToFetchEstimated"), true);
+                keysToIterateEstimated = Metrics.histogram(createMetricName("KeysToIterateEstimated"), true);
+                logSelectivityEstimated = Metrics.histogram(createMetricName("LogSelectivityEstimated"), true);
+                indexReferencesInPlan = Metrics.histogram(createMetricName("IndexReferencesInPlan"), true);
+                indexReferencesInQuery = Metrics.histogram(createMetricName("IndexReferencesInQuery"), false);
+            }
+        }
+
     }
+
+
 }

src/java/org/apache/cassandra/index/sai/plan/Plan.java

@@ -531,6 +531,8 @@ public final double selectivity()
 
     /**
      * Returns the number of indexes referenced by this plan.
+     * The same index referenced from unrelated query clauses,
+     * leading to separate index searches, are counted separately.
      */
     public final int referencedIndexCount()
     {
@@ -1757,7 +1759,7 @@ protected Fetch withAccess(Access access)
      * In order to return one row in the result set it may need to retrieve many rows from the source node.
      * Hence, it will typically have higher cost-per-row than the source node, and will return fewer rows.
      */
-    public static class Filter extends RowsIteration
+    static class Filter extends RowsIteration
     {
         private final RowFilter filter;
         private final LazyTransform<RowsIteration> source;

test/unit/org/apache/cassandra/index/sai/metrics/AbstractMetricsTest.java

@@ -20,9 +20,11 @@
 import java.util.concurrent.TimeUnit;
 import javax.management.ObjectName;
 
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.index.sai.SAITester;
 import org.apache.cassandra.utils.Throwables;
 
@@ -32,6 +34,11 @@
 @Ignore
 public abstract class AbstractMetricsTest extends SAITester
 {
+    boolean indexMetricsEnabled;
+    boolean perQueryKindPerTableMetricsEnabled;
+    boolean perQueryKindPerQueryMetricsEnabled;
+    boolean queryPlanMetricsEnabled;
+
     @Before
     public void initializeTest() throws Throwable
     {
@@ -40,6 +47,20 @@ public void initializeTest() throws Throwable
         startJMXServer();
 
         createMBeanServerConnection();
+
+        indexMetricsEnabled = CassandraRelevantProperties.SAI_INDEX_METRICS_ENABLED.getBoolean();
+        perQueryKindPerQueryMetricsEnabled = CassandraRelevantProperties.SAI_QUERY_KIND_PER_QUERY_METRICS_ENABLED.getBoolean();
+        perQueryKindPerTableMetricsEnabled = CassandraRelevantProperties.SAI_QUERY_KIND_PER_TABLE_METRICS_ENABLED.getBoolean();
+        queryPlanMetricsEnabled = CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.getBoolean();
+    }
+
+    @After
+    public void cleanupTest() throws Throwable
+    {
+        CassandraRelevantProperties.SAI_INDEX_METRICS_ENABLED.setBoolean(indexMetricsEnabled);
+        CassandraRelevantProperties.SAI_QUERY_KIND_PER_TABLE_METRICS_ENABLED.setBoolean(perQueryKindPerTableMetricsEnabled);
+        CassandraRelevantProperties.SAI_QUERY_KIND_PER_QUERY_METRICS_ENABLED.setBoolean(perQueryKindPerQueryMetricsEnabled);
+        CassandraRelevantProperties.SAI_QUERY_PLAN_METRICS_ENABLED.setBoolean(indexMetricsEnabled);
     }
 
     protected void waitForIndexCompaction(String keyspace, String table, String index)
@@ -85,7 +106,6 @@ protected void waitForHistogramMeanBetween(ObjectName name, double min, double m
         }, 10, TimeUnit.SECONDS);
     }
 
-
     protected void waitForGreaterThanZero(ObjectName name)
     {
         waitForAssert(() -> {