CNDB-15919: Optimize SAI NOT queries, push logic into posting lists

michaeljmarshall · michaeljmarshall · commit 25d7f7eac783 · 2025-11-06T16:13:48.000-06:00
diff --git a/src/java/org/apache/cassandra/index/sai/SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/SSTableIndex.java
@@ -95,7 +95,7 @@ private static SearchableIndex createSearchableIndex(SSTableContext sstableConte
         if (CassandraRelevantProperties.SAI_INDEX_READS_DISABLED.getBoolean())
         {
             logger.info("Creating dummy (empty) index searcher for sstable {} as SAI index reads are disabled", sstableContext.sstable.descriptor);
-            return new EmptyIndex();
+            return new EmptyIndex(sstableContext);
         }
 
         return perIndexComponents.onDiskFormat().newSearchableIndex(sstableContext, perIndexComponents);
@@ -252,17 +252,10 @@ private KeyRangeIterator getNonEqIterator(Expression expression,
                                               QueryContext context,
                                               boolean defer) throws IOException
     {
-        KeyRangeIterator allKeys = allSSTableKeys(keyRange);
         if (TypeUtil.supportsRounding(expression.validator))
-        {
-            return allKeys;
-        }
+            return allSSTableKeys(keyRange);
         else
-        {
-            Expression negExpression = expression.negated();
-            KeyRangeIterator matchedKeys = searchableIndex.search(negExpression, keyRange, context, defer);
-            return KeyRangeAntiJoinIterator.create(allKeys, matchedKeys);
-        }
+            return searchableIndex.search(expression, keyRange, context, defer);
     }
 
     public KeyRangeIterator search(Expression expression,
diff --git a/src/java/org/apache/cassandra/index/sai/disk/EmptyIndex.java b/src/java/org/apache/cassandra/index/sai/disk/EmptyIndex.java
@@ -27,6 +27,7 @@
 import org.apache.cassandra.db.virtual.SimpleDataSet;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
 import org.apache.cassandra.index.sai.disk.v1.Segment;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
 import org.apache.cassandra.index.sai.plan.Expression;
@@ -38,6 +39,13 @@
 
 public class EmptyIndex implements SearchableIndex
 {
+    private final SSTableContext sstableContext;
+
+    public EmptyIndex(SSTableContext sstableContext)
+    {
+        this.sstableContext = sstableContext;
+    }
+
     @Override
     public long indexFileCacheSize()
     {
@@ -98,7 +106,9 @@ public KeyRangeIterator search(Expression expression,
                                    QueryContext context,
                                    boolean defer) throws IOException
     {
-        return KeyRangeIterator.empty();
+        return expression.getOp().isNonEquality()
+               ? PrimaryKeyMapIterator.create(sstableContext, keyRange)
+               : KeyRangeIterator.empty();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/IndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexSearcher.java
@@ -37,6 +37,7 @@
 import org.apache.cassandra.index.sai.disk.PostingList;
 import org.apache.cassandra.index.sai.disk.PostingListKeyRangeIterator;
 import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.v1.postings.ComplementPostingList;
 import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
 import org.apache.cassandra.index.sai.iterators.RowIdToPrimaryKeyWithSortKeyIterator;
 import org.apache.cassandra.index.sai.plan.Expression;
@@ -96,7 +97,28 @@ protected IndexSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
      * @param defer        create the iterator in a deferred state
      * @return {@link KeyRangeIterator} that matches given expression
      */
-    public abstract KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, boolean defer) throws IOException;
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, boolean defer) throws IOException
+    {
+        if (expression.getOp().isNonEquality())
+        {
+            var negated = expression.negated();
+            var postingList = searchInternal(negated, keyRange, queryContext, defer);
+            // TODO handle the complexity of which row ids to use here.
+            int minSegmentRowId = metadata.toSegmentRowId(0);
+            int maxSegmentRowId = metadata.toSegmentRowId(primaryKeyMapFactory.count() - 1);
+
+            var complement = new ComplementPostingList(minSegmentRowId, maxSegmentRowId, postingList);
+            // TODO this needs to use min/max keys for the whole table
+            return toPrimaryKeyIterator(complement, queryContext);
+        }
+        else
+        {
+            var postingList = searchInternal(expression, keyRange, queryContext, defer);
+            return toPrimaryKeyIterator(postingList, queryContext);
+        }
+    }
+
+    protected abstract PostingList searchInternal(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext, boolean defer) throws IOException;
 
     /**
      * Order the rows by the given Orderer.  Used for ORDER BY clause when
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcher.java
@@ -134,13 +134,7 @@ public long indexFileCacheSize()
     }
 
     @SuppressWarnings("resource")
-    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
-    {
-        PostingList postingList = searchPosting(exp, context);
-        return toPrimaryKeyIterator(postingList, context);
-    }
-
-    private PostingList searchPosting(Expression exp, QueryContext context)
+    protected PostingList searchInternal(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
     {
         if (logger.isTraceEnabled())
             logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/KDTreeIndexSearcher.java
@@ -84,13 +84,7 @@ public long indexFileCacheSize()
     }
 
     @Override
-    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
-    {
-        PostingList postingList = searchPosting(exp, context);
-        return toPrimaryKeyIterator(postingList, context);
-    }
-
-    private PostingList searchPosting(Expression exp, QueryContext context)
+    protected PostingList searchInternal(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
     {
         if (logger.isTraceEnabled())
             logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
@@ -163,7 +163,7 @@ public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexComponents.ForRead per
     public SearchableIndex newSearchableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents)
     {
         return perIndexComponents.isEmpty()
-               ? new EmptyIndex()
+               ? new EmptyIndex(sstableContext)
                : new V1SearchableIndex(sstableContext, perIndexComponents);
     }
 
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ComplementPostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/ComplementPostingList.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.PostingList;
+
+public class ComplementPostingList implements PostingList
+{
+    private final PostingList source;
+    private final int lastSourceRowId;
+    private int nextRowId;
+    private int nextSourceRowId = -1;
+
+    /**
+     * A posting list that complements the provided posting list within the specified range.
+     *
+     * @param minSegmentRowId inclusive minimum row id
+     * @param maxSegmentRowId exclusive maximum row id
+     * @param source posting list to complement
+     */
+    public ComplementPostingList(int minSegmentRowId, int maxSegmentRowId, PostingList source)
+    {
+        this.nextRowId = minSegmentRowId;
+        this.lastSourceRowId = maxSegmentRowId;
+        this.source = source == null ? PostingList.EMPTY : source;
+    }
+
+    @Override
+    public int nextPosting() throws IOException
+    {
+        if (nextSourceRowId == -1)
+            nextSourceRowId = source.nextPosting();
+
+        // Move both pointers forward
+        while (nextSourceRowId == nextRowId)
+        {
+            nextRowId++;
+            nextSourceRowId = source.nextPosting();
+        }
+
+        if (nextRowId > lastSourceRowId)
+            return END_OF_STREAM;
+
+        return nextRowId++;
+    }
+
+    @Override
+    public int size()
+    {
+        return (lastSourceRowId - nextRowId) - source.size() + 1;
+    }
+
+    @Override
+    public int advance(int targetRowID) throws IOException
+    {
+        nextRowId = targetRowID;
+        nextSourceRowId = source.advance(targetRowID);
+        return nextPosting();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v2/V2VectorIndexSearcher.java
@@ -135,13 +135,7 @@ public ProductQuantization getPQ()
     }
 
     @Override
-    public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
-    {
-        PostingList results = searchPosting(context, exp, keyRange);
-        return toPrimaryKeyIterator(results, context);
-    }
-
-    private PostingList searchPosting(QueryContext context, Expression exp, AbstractBounds<PartitionPosition> keyRange) throws IOException
+    protected PostingList searchInternal(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException
     {
         if (logger.isTraceEnabled())
             logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);

Original file line number	Diff line number	Diff line change
`@@ -134,13 +134,7 @@ public long indexFileCacheSize()`
`134`	`134`	`}`
`135`	`135`
`136`	`136`	`@SuppressWarnings("resource")`
`137`		`- public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException`
`138`		`- {`
`139`		`- PostingList postingList = searchPosting(exp, context);`
`140`		`- return toPrimaryKeyIterator(postingList, context);`
`141`		`- }`
`142`		`-`
`143`		`- private PostingList searchPosting(Expression exp, QueryContext context)`
	`137`	`+ protected PostingList searchInternal(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException`
`144`	`138`	`{`
`145`	`139`	`if (logger.isTraceEnabled())`
`146`	`140`	`logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);`
Original file line number	Diff line number	Diff line change
`@@ -84,13 +84,7 @@ public long indexFileCacheSize()`
`84`	`84`	`}`
`85`	`85`
`86`	`86`	`@Override`
`87`		`- public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException`
`88`		`- {`
`89`		`- PostingList postingList = searchPosting(exp, context);`
`90`		`- return toPrimaryKeyIterator(postingList, context);`
`91`		`- }`
`92`		`-`
`93`		`- private PostingList searchPosting(Expression exp, QueryContext context)`
	`87`	`+ protected PostingList searchInternal(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException`
`94`	`88`	`{`
`95`	`89`	`if (logger.isTraceEnabled())`
`96`	`90`	`logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);`
Original file line number	Diff line number	Diff line change
`@@ -163,7 +163,7 @@ public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexComponents.ForRead per`
`163`	`163`	`public SearchableIndex newSearchableIndex(SSTableContext sstableContext, IndexComponents.ForRead perIndexComponents)`
`164`	`164`	`{`
`165`	`165`	`return perIndexComponents.isEmpty()`
`166`		`- ? new EmptyIndex()`
	`166`	`+ ? new EmptyIndex(sstableContext)`
`167`	`167`	`: new V1SearchableIndex(sstableContext, perIndexComponents);`
`168`	`168`	`}`
`169`	`169`
Original file line number	Diff line number	Diff line change
`@@ -135,13 +135,7 @@ public ProductQuantization getPQ()`
`135`	`135`	`}`
`136`	`136`
`137`	`137`	`@Override`
`138`		`- public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException`
`139`		`- {`
`140`		`- PostingList results = searchPosting(context, exp, keyRange);`
`141`		`- return toPrimaryKeyIterator(results, context);`
`142`		`- }`
`143`		`-`
`144`		`- private PostingList searchPosting(QueryContext context, Expression exp, AbstractBounds<PartitionPosition> keyRange) throws IOException`
	`138`	`+ protected PostingList searchInternal(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context, boolean defer) throws IOException`
`145`	`139`	`{`
`146`	`140`	`if (logger.isTraceEnabled())`
`147`	`141`	`logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);`