push step limit down storage (#5883)

push step limit down storage "

Author: nevermore3

src/graph/executor/query/ExpandAllExecutor.cpp

@@ -142,6 +142,9 @@ folly::Future<Status> ExpandAllExecutor::getNeighbors() {
   std::vector<Value> vids(nextStepVids_.size());
   std::move(nextStepVids_.begin(), nextStepVids_.end(), vids.begin());
   QueryExpressionContext qec(qctx()->ectx());
+  auto stepLimit =
+      stepLimits_.empty() ? std::numeric_limits<int64_t>::max() : stepLimits_[currentStep_ - 2];
+  auto limit = std::min(stepLimit, expand_->limit(qec));
   return storageClient
       ->getNeighbors(param,
                      {nebula::kVid},
@@ -153,9 +156,9 @@ folly::Future<Status> ExpandAllExecutor::getNeighbors() {
                      expand_->edgeProps(),
                      nullptr,
                      false,
-                     false,
+                     sample_,
                      std::vector<storage::cpp2::OrderBy>(),
-                     expand_->limit(qec),
+                     limit,
                      expand_->filter(),
                      nullptr)
       .via(runner())
@@ -402,6 +405,12 @@ folly::Future<Status> ExpandAllExecutor::handleResponse(RpcResponse&& resps) {
       buildResult(curVertexProps, edgeProps);
     }
 
+    if (!stepLimits_.empty()) {
+      // if stepLimits_ is not empty, do not use cache
+      nextStepVids_.emplace(dst);
+      continue;
+    }
+
     if (adjList_.find(dst) == adjList_.end()) {
       nextStepVids_.emplace(dst);
     } else {
@@ -413,7 +422,9 @@ folly::Future<Status> ExpandAllExecutor::handleResponse(RpcResponse&& resps) {
     adjList_.emplace(curVid, std::move(adjEdgeProps));
   }
 
-  resetNextStepVids(visitedVids);
+  if (stepLimits_.empty()) {
+    resetNextStepVids(visitedVids);
+  }
 
   if (!preVisitedVids_.empty()) {
     getNeighborsFromCache(dst2VidsMap, visitedVids, samples);

src/graph/executor/query/ExpandExecutor.cpp

@@ -141,6 +141,8 @@ folly::Future<Status> ExpandExecutor::getNeighbors() {
                                           qctx_->plan()->isProfileEnabled());
   std::vector<Value> vids(nextStepVids_.size());
   std::move(nextStepVids_.begin(), nextStepVids_.end(), vids.begin());
+  auto stepLimit =
+      stepLimits_.empty() ? std::numeric_limits<int64_t>::max() : stepLimits_[currentStep_ - 1];
   return storageClient
       ->getNeighbors(param,
                      {nebula::kVid},
@@ -152,9 +154,9 @@ folly::Future<Status> ExpandExecutor::getNeighbors() {
                      expand_->edgeProps(),
                      nullptr,
                      false,
-                     false,
+                     sample_,
                      std::vector<storage::cpp2::OrderBy>(),
-                     -1,
+                     stepLimit,
                      nullptr,
                      nullptr)
       .via(runner())
@@ -323,6 +325,11 @@ folly::Future<Status> ExpandExecutor::handleResponse(RpcResponse&& resps) {
         if (currentStep_ >= maxSteps_) {
           continue;
         }
+        if (!stepLimits_.empty()) {
+          // do not use cache when stepLimits_ is not empty
+          nextStepVids_.emplace(dst);
+          continue;
+        }
         if (adjDsts_.find(dst) == adjDsts_.end()) {
           nextStepVids_.emplace(dst);
         } else {

src/graph/planner/plan/Query.h

@@ -422,7 +422,7 @@ class ExpandAll : public Expand {
     } else if (edgeColumns_) {
       colNames = edgeColumns_->names();
     }
-    setLimit(-1);
+    setLimit(std::numeric_limits<int64_t>::max());
     setColNames(colNames);
   }
 

tests/tck/features/go/SampleLimit.feature

@@ -72,4 +72,3 @@ Feature: Sample and limit
       | like._dst |
       | /[\s\w+]/ |
       | /[\s\w+]/ |
-      | /[\s\w+]/ |

tests/tck/features/go/SampleLimit.intVid.feature

@@ -62,4 +62,3 @@ Feature: Sample and limit
       | like._dst |
       | /[\d\-+]/ |
       | /[\d\-+]/ |
-      | /[\d\-+]/ |

tests/tck/features/optimizer/PushSampleDownRule.feature

@@ -28,7 +28,6 @@ Feature: Push Limit down rule
     Then the result should be, in any order:
       | like._dst |
       | /[\w\s]+/ |
-      | /[\w\s]+/ |
     And the execution plan should be:
       | id | name      | dependencies | operator info              |
       | 4  | Project   | 3            |                            |
@@ -42,7 +41,6 @@ Feature: Push Limit down rule
     Then the result should be, in any order:
       | src       | likeness | dst       |
       | /[\w\s]+/ | /\d\d/   | /[\w\s]+/ |
-      | /[\w\s]+/ | /\d\d/   | /[\w\s]+/ |
     And the execution plan should be:
       | id | name         | dependencies | profiling data | operator info                                 |
       | 8  | Project      | 7            |                |                                               |