perf: SQL optimizations

Author: lvca

engine/src/main/java/com/arcadedb/query/sql/executor/AbstractTraverseStep.java

@@ -31,22 +31,29 @@
  */
 public abstract class AbstractTraverseStep extends AbstractExecutionStep {
   protected final WhereClause                  whileClause;
+  // Optional emit filter pushed down from an outer SELECT's WHERE. Non-matching vertices still drive expansion, they just don't appear in the results. Null
+  // means "emit everything traversed", preserving the historical behavior.
+  protected final WhereClause                  postFilter;
   protected final List<TraverseProjectionItem> projections;
   protected final PInteger                     maxDepth;
 
-  protected       List<Result> entryPoints = null;
-  protected final List<Result> results     = new ArrayList<>();
+  // ArrayDeque, not ArrayList: entryPoints is used as a stack (addFirst/removeFirst in the hot loop). ArrayList would make both O(n), turning the traversal into
+  // O(n^2) on the queue depth and was the main cause of slow TRAVERSE on deep/wide subgraphs.
+  protected       Deque<Result> entryPoints = null;
+  protected final List<Result>  results     = new ArrayList<>();
 
   // Visited set for traversal dedup. Graph traversal is inherently sparse (RIDs scatter across buckets with random offsets), so RidHashSet's primitive-packed
   // open-addressing hash wins over RidSet's bitmap on both memory and build time. See performance.RidDedupSetBenchmark.
   final RidHashSet traversed;
 
   public AbstractTraverseStep(final List<TraverseProjectionItem> projections, final WhereClause whileClause,
+      final WhereClause postFilter,
       final PInteger maxDepth,
       final CommandContext context) {
     super(context);
     this.traversed = new RidHashSet();
     this.whileClause = whileClause;
+    this.postFilter = postFilter;
     this.maxDepth = maxDepth;
     this.projections = projections.stream().map(TraverseProjectionItem::copy).collect(Collectors.toList());
   }
@@ -93,7 +100,7 @@ public Result next() {
 
   private void fetchNextBlock(final CommandContext context, final int nRecords) {
     if (this.entryPoints == null)
-      this.entryPoints = new ArrayList<>();
+      this.entryPoints = new ArrayDeque<>();
 
     while (this.results.isEmpty()) {
       if (this.entryPoints.isEmpty())

engine/src/main/java/com/arcadedb/query/sql/executor/BreadthFirstTraverseStep.java

@@ -35,7 +35,12 @@ public class BreadthFirstTraverseStep extends AbstractTraverseStep {
 
   public BreadthFirstTraverseStep(final List<TraverseProjectionItem> projections, final WhereClause whileClause, final PInteger maxDepth,
       final CommandContext context) {
-    super(projections, whileClause, maxDepth, context);
+    this(projections, whileClause, null, maxDepth, context);
+  }
+
+  public BreadthFirstTraverseStep(final List<TraverseProjectionItem> projections, final WhereClause whileClause, final WhereClause postFilter,
+      final PInteger maxDepth, final CommandContext context) {
+    super(projections, whileClause, postFilter, maxDepth, context);
   }
 
   @Override
@@ -100,7 +105,8 @@ private Result toTraverseResult(final Result item) {
   protected void fetchNextResults(final CommandContext context, final int nRecords) {
     if (!this.entryPoints.isEmpty()) {
       final TraverseResult item = (TraverseResult) this.entryPoints.removeFirst();
-      this.results.add(item);
+      if (postFilter == null || postFilter.matchesFilters(item, context))
+        this.results.add(item);
       for (final TraverseProjectionItem proj : projections) {
         final Object nextStep = proj.execute(item, context);
         final Integer depth = item.depth != null ? item.depth : (Integer) item.getMetadata("$depth");

engine/src/main/java/com/arcadedb/query/sql/executor/DepthFirstTraverseStep.java

@@ -35,7 +35,12 @@ public class DepthFirstTraverseStep extends AbstractTraverseStep {
 
   public DepthFirstTraverseStep(final List<TraverseProjectionItem> projections, final WhereClause whileClause, final PInteger maxDepth,
       final CommandContext context) {
-    super(projections, whileClause, maxDepth, context);
+    this(projections, whileClause, null, maxDepth, context);
+  }
+
+  public DepthFirstTraverseStep(final List<TraverseProjectionItem> projections, final WhereClause whileClause, final WhereClause postFilter,
+      final PInteger maxDepth, final CommandContext context) {
+    super(projections, whileClause, postFilter, maxDepth, context);
   }
 
   @Override
@@ -100,7 +105,8 @@ else if (item.isElement() && item.getElement().get().getIdentity() != null) {
   protected void fetchNextResults(final CommandContext context, final int nRecords) {
     if (!this.entryPoints.isEmpty()) {
       final TraverseResult item = (TraverseResult) this.entryPoints.removeFirst();
-      this.results.add(item);
+      if (postFilter == null || postFilter.matchesFilters(item, context))
+        this.results.add(item);
       for (final TraverseProjectionItem proj : projections) {
         final Object nextStep = proj.execute(item, context);
         final Integer depth = item.depth != null ? item.depth : (Integer) item.getMetadata("$depth");

engine/src/main/java/com/arcadedb/query/sql/executor/SelectExecutionPlanner.java

@@ -68,6 +68,7 @@
 import com.arcadedb.query.sql.parser.SelectStatement;
 import com.arcadedb.query.sql.parser.SuffixIdentifier;
 import com.arcadedb.query.sql.parser.Statement;
+import com.arcadedb.query.sql.parser.TraverseStatement;
 import com.arcadedb.query.sql.parser.SubQueryCollector;
 import com.arcadedb.query.sql.parser.Timeout;
 import com.arcadedb.query.sql.parser.WhereClause;
@@ -1250,7 +1251,7 @@ else if (variableValue instanceof String typeName) {
       }
       handleBucketsAsTarget(info.fetchExecutionPlan, info, buckets, context);
     } else if (target.getStatement() != null) {
-      handleSubqueryAsTarget(info.fetchExecutionPlan, target.getStatement(), context);
+      handleSubqueryAsTarget(info.fetchExecutionPlan, maybePushWhereIntoTraverse(target.getStatement(), info), context);
     } else if (target.getFunctionCall() != null) {
       //        handleFunctionCallAsTarget(result, target.getFunctionCall(), context);//TODO
       throw new CommandExecutionException("function call as target is not supported yet");
@@ -3458,6 +3459,33 @@ private void handleSubqueryAsTarget(final SelectExecutionPlan plan, final Statem
     plan.chain(new SubQueryStep(subExecutionPlan, context, subCtx));
   }
 
+  /**
+   * When the outer SELECT has shape `SELECT ... FROM (TRAVERSE ...) WHERE <row-local cond>`, push the WHERE into the TRAVERSE step as a post-emit filter.
+   * Non-matching vertices still drive expansion (the sub-graph is fully walked), they are just not emitted. This avoids materializing every intermediate vertex
+   * as a Result and running each through a separate FilterStep, which is the dominant cost on the common pattern `... WHERE @type = 'X'` that filters a minority
+   * class out of a multi-type traversal.
+   * <p>
+   * The original TraverseStatement node is not mutated: a copy is returned and the outer WHERE is cleared when consumed.
+   */
+  private Statement maybePushWhereIntoTraverse(final Statement subQuery, final QueryPlanningInfo info) {
+    if (!(subQuery instanceof TraverseStatement traverse))
+      return subQuery;
+    if (info.whereClause == null || info.whereClause.getBaseExpression() == null)
+      return subQuery;
+    // Conservative: skip if WHERE references parent scope, LET variables, or $current/$parent (anything starting with '$').
+    if (info.whereClause.refersToParent())
+      return subQuery;
+    if (info.whereClause.toString().contains("$"))
+      return subQuery;
+
+    final TraverseStatement copy = (TraverseStatement) traverse.copy();
+    copy.setPostFilter(info.whereClause.copy());
+    // WHERE has been consumed by the inner TRAVERSE, so the outer plan must not add another FilterStep for it.
+    info.whereClause = null;
+    info.flattenedWhereClause = null;
+    return copy;
+  }
+
   private boolean isOrderByRidDesc(final QueryPlanningInfo info) {
     if (!hasTargetWithSortedRids(info))
       return false;

engine/src/main/java/com/arcadedb/query/sql/executor/TraverseExecutionPlanner.java

@@ -37,6 +37,7 @@ public class TraverseExecutionPlanner {
   private final List<TraverseProjectionItem> projections;
   private final FromClause                   target;
   private final WhereClause                  whileClause;
+  private final WhereClause                  postFilter;
   private final TraverseStatement.Strategy   strategy;
   private final PInteger                     maxDepth;
   private final Skip                         skip;
@@ -50,6 +51,7 @@ public TraverseExecutionPlanner(final TraverseStatement statement) {
 
     this.target = statement.getTarget();
     this.whileClause = statement.getWhileClause() == null ? null : statement.getWhileClause().copy();
+    this.postFilter = statement.getPostFilter() == null ? null : statement.getPostFilter().copy();
 
     this.strategy = statement.getStrategy() == null ? TraverseStatement.Strategy.DEPTH_FIRST : statement.getStrategy();
     this.maxDepth = statement.getMaxDepth() == null ? null : statement.getMaxDepth().copy();
@@ -78,10 +80,10 @@ public InternalExecutionPlan createExecutionPlan(final CommandContext context) {
   private void handleTraversal(final SelectExecutionPlan result, final CommandContext context) {
     switch (strategy) {
       case BREADTH_FIRST:
-        result.chain(new BreadthFirstTraverseStep(this.projections, this.whileClause, maxDepth, context));
+        result.chain(new BreadthFirstTraverseStep(this.projections, this.whileClause, this.postFilter, maxDepth, context));
         break;
       case DEPTH_FIRST:
-        result.chain(new DepthFirstTraverseStep(this.projections, this.whileClause, maxDepth, context));
+        result.chain(new DepthFirstTraverseStep(this.projections, this.whileClause, this.postFilter, maxDepth, context));
         break;
     }
     //TODO

engine/src/main/java/com/arcadedb/query/sql/parser/TraverseStatement.java

@@ -42,6 +42,9 @@ public enum Strategy {
   protected Skip                         skip;
   protected Strategy                     strategy;
   protected PInteger                     maxDepth;
+  // Optional emit filter set by the outer SELECT planner when `SELECT ... FROM (TRAVERSE ...) WHERE <row-local cond>` is detected. Evaluated per visited vertex
+  // before emission only: non-matching vertices still drive expansion, they are just not returned as results. Not exposed in grammar.
+  protected WhereClause                  postFilter;
 
   public TraverseStatement(final int id) {
     super(id);
@@ -159,6 +162,7 @@ public Statement copy() {
     result.limit = limit == null ? null : limit.copy();
     result.strategy = strategy;
     result.maxDepth = maxDepth == null ? null : maxDepth.copy();
+    result.postFilter = postFilter == null ? null : postFilter.copy();
     return result;
   }
 
@@ -181,7 +185,9 @@ public boolean equals(final Object o) {
       return false;
     if (strategy != that.strategy)
       return false;
-    return Objects.equals(maxDepth, that.maxDepth);
+    if (!Objects.equals(maxDepth, that.maxDepth))
+      return false;
+    return Objects.equals(postFilter, that.postFilter);
   }
 
   @Override
@@ -192,6 +198,7 @@ public int hashCode() {
     result = 31 * result + (limit != null ? limit.hashCode() : 0);
     result = 31 * result + (strategy != null ? strategy.hashCode() : 0);
     result = 31 * result + (maxDepth != null ? maxDepth.hashCode() : 0);
+    result = 31 * result + (postFilter != null ? postFilter.hashCode() : 0);
     return result;
   }
 
@@ -247,5 +254,13 @@ public Skip getSkip() {
   public void setSkip(final Skip skip) {
     this.skip = skip;
   }
+
+  public WhereClause getPostFilter() {
+    return postFilter;
+  }
+
+  public void setPostFilter(final WhereClause postFilter) {
+    this.postFilter = postFilter;
+  }
 }
 /* JavaCC - OriginalChecksum=47399a3a3d5a423768bbdc70ee957464 (do not edit this line) */

engine/src/test/java/com/arcadedb/query/sql/executor/TraverseStatementExecutionTest.java

@@ -201,6 +201,109 @@ void traverseInBatchTx() {
     });
   }
 
+  @Test
+  void branchingTreeTraverse() {
+    // Regression test for the ArrayList->ArrayDeque swap on entryPoints. A shallow but wide tree stresses the deque depth more than any linear chain covered by
+    // the other tests; with the old ArrayList the addFirst/removeFirst hot path was O(n) and the full traversal was O(n^2), which made non-trivial subgraphs
+    // visibly slow even though the visited count was small. Correctness (total visited, per-depth count) must be unchanged.
+    database.transaction(() -> {
+      final String v = "BranchingTreeV";
+      final String e = "BranchingTreeE";
+      database.getSchema().createVertexType(v);
+      database.getSchema().createEdgeType(e);
+
+      final int fanout = 20;
+      final RID root = database.command("sql", "create vertex " + v + " set name = 'root'").next().getIdentity().get();
+      final Map<String, Object> params = new HashMap<>();
+      params.put("from", root);
+
+      int expectedTotal = 1;
+      for (int i = 0; i < fanout; i++) {
+        final RID child = database.command("sql", "create vertex " + v + " set name = 'c" + i + "'").next().getIdentity().get();
+        params.put("to", child);
+        database.command("sql", "create edge " + e + " from :from to :to", params).close();
+        expectedTotal++;
+        for (int j = 0; j < fanout; j++) {
+          final RID grand = database.command("sql", "create vertex " + v + " set name = 'g" + i + "_" + j + "'").next().getIdentity().get();
+          params.put("from", child);
+          params.put("to", grand);
+          database.command("sql", "create edge " + e + " from :from to :to", params).close();
+          expectedTotal++;
+        }
+        params.put("from", root);
+      }
+
+      params.clear();
+      params.put("root", root);
+      try (final ResultSet rs = database.query("sql", "select count(*) as c from (traverse out('" + e + "') from :root)", params)) {
+        assertThat(rs.hasNext()).isTrue();
+        assertThat(((Number) rs.next().getProperty("c")).intValue()).isEqualTo(expectedTotal);
+      }
+    });
+  }
+
+  @Test
+  void postFilterPushdownByType() {
+    // Outer `SELECT ... FROM (TRAVERSE ...) WHERE @type = 'X'` should evaluate the @type filter inside the traverse step, so non-matching vertices are visited
+    // (to keep expansion correct) but not emitted. Correctness is what this test guards; the perf win is that the outer SubQueryStep no longer shuttles every
+    // intermediate vertex through a FilterStep.
+    database.transaction(() -> {
+      final String folder = "PushdownFolderV";
+      final String leaf = "PushdownLeafV";
+      final String edge = "PushdownE";
+      database.getSchema().createVertexType(folder);
+      database.getSchema().createVertexType(leaf);
+      database.getSchema().createEdgeType(edge);
+
+      final RID root = database.command("sql", "create vertex " + folder + " set name = 'root'").next().getIdentity().get();
+      final Map<String, Object> params = new HashMap<>();
+      int expectedLeaves = 0;
+      for (int i = 0; i < 5; i++) {
+        final RID sub = database.command("sql", "create vertex " + folder + " set name = 'sub" + i + "'").next().getIdentity().get();
+        params.clear();
+        params.put("from", root);
+        params.put("to", sub);
+        database.command("sql", "create edge " + edge + " from :from to :to", params).close();
+        for (int j = 0; j < 4; j++) {
+          final RID l = database.command("sql", "create vertex " + leaf + " set name = 'leaf" + i + "_" + j + "'").next().getIdentity().get();
+          params.clear();
+          params.put("from", sub);
+          params.put("to", l);
+          database.command("sql", "create edge " + edge + " from :from to :to", params).close();
+          expectedLeaves++;
+        }
+      }
+
+      params.clear();
+      params.put("root", root);
+      try (final ResultSet rs = database.query("sql",
+          "select @rid as rid, @type as t from (traverse out('" + edge + "') from :root) where @type = '" + leaf + "'", params)) {
+        int count = 0;
+        while (rs.hasNext()) {
+          final Result r = rs.next();
+          assertThat((String) r.getProperty("t")).isEqualTo(leaf);
+          count++;
+        }
+        assertThat(count).isEqualTo(expectedLeaves);
+      }
+
+      // Count form - the pushdown should not change count-aggregation semantics.
+      try (final ResultSet rs = database.query("sql",
+          "select count(*) as c from (traverse out('" + edge + "') from :root) where @type = '" + leaf + "'", params)) {
+        assertThat(rs.hasNext()).isTrue();
+        assertThat(((Number) rs.next().getProperty("c")).intValue()).isEqualTo(expectedLeaves);
+      }
+
+      // Safety: when outer WHERE references a LET variable, pushdown must NOT fire (because the LET lives in the outer scope) and behavior must still be
+      // correct.
+      try (final ResultSet rs = database.query("sql",
+          "select count(*) as c from (traverse out('" + edge + "') from :root) let $t = '" + leaf + "' where @type = $t", params)) {
+        assertThat(rs.hasNext()).isTrue();
+        assertThat(((Number) rs.next().getProperty("c")).intValue()).isEqualTo(expectedLeaves);
+      }
+    });
+  }
+
   @Test
   void traverseFromRID() {
     database.command("sql", "CREATE VERTEX TYPE TVtx IF NOT EXISTS");