From 762bbf0650fb697f3382dcff4af536accefd7279 Mon Sep 17 00:00:00 2001 From: Andy Seaborne Date: Sun, 28 Jan 2024 14:42:15 +0000 Subject: [PATCH] GH-2218: Implement LATERAL as per SEP-0006/SEP-0007 --- .../apache/jena/sparql/core/Substitute.java | 35 +- .../apache/jena/sparql/core/VarExprList.java | 8 +- .../jena/sparql/engine/binding/Binding.java | 4 + .../sparql/engine/binding/BindingBase.java | 12 +- .../engine/iterator/QueryIterLateral.java | 303 +++++++++++++++++- .../sparql/engine/main/JoinClassifier.java | 7 +- .../engine/main/LeftJoinClassifier.java | 3 + .../sparql/engine/ref/EvaluatorDispatch.java | 154 ++++----- .../jena/sparql/engine/ref/RefEval.java | 2 +- .../jena/sparql/lang/SyntaxVarScope.java | 53 +-- jena-arq/testing/ARQ/Lateral/data2.ttl | 9 + .../ARQ/Lateral/lateral-in-optional.arq | 15 + .../ARQ/Lateral/lateral-in-optional.srj | 22 ++ jena-arq/testing/ARQ/Lateral/manifest.ttl | 20 ++ .../ARQ/Lateral/optional-in-lateral.arq | 16 + .../ARQ/Lateral/optional-in-lateral.srj | 27 ++ 16 files changed, 565 insertions(+), 125 deletions(-) create mode 100644 jena-arq/testing/ARQ/Lateral/data2.ttl create mode 100644 jena-arq/testing/ARQ/Lateral/lateral-in-optional.arq create mode 100644 jena-arq/testing/ARQ/Lateral/lateral-in-optional.srj create mode 100644 jena-arq/testing/ARQ/Lateral/optional-in-lateral.arq create mode 100644 jena-arq/testing/ARQ/Lateral/optional-in-lateral.srj diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/core/Substitute.java b/jena-arq/src/main/java/org/apache/jena/sparql/core/Substitute.java index 1f7970560fe..19cbadc2963 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/core/Substitute.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/core/Substitute.java @@ -20,16 +20,19 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; import org.apache.jena.graph.Triple; import org.apache.jena.sparql.algebra.Op; +import org.apache.jena.sparql.algebra.Transform; import org.apache.jena.sparql.algebra.TransformCopy; import org.apache.jena.sparql.algebra.Transformer; import org.apache.jena.sparql.algebra.op.*; import org.apache.jena.sparql.engine.binding.Binding; import org.apache.jena.sparql.engine.binding.BindingFactory; +import org.apache.jena.sparql.engine.iterator.QueryIterLateral; import org.apache.jena.sparql.expr.Expr; import org.apache.jena.sparql.expr.ExprList; import org.apache.jena.sparql.pfunction.PropFuncArg; @@ -40,12 +43,42 @@ * Substitution in SPARQL algebra. *

* See also {@link QueryTransformOps} and {@link UpdateTransformOps} which operate on SPARQL syntax. + *

+ * {@link #inject} provides the substitution, while leaving a variable present, used by LATERAL. */ public class Substitute { + /** + * Inject takes an {@link Op} to transform using a {Binding binding}. The + * transformation assumes the Ope structure is legal for the operation. The + * transformation is to wrap each place a variable is used (BGP, GRAPH, Path and + * some equivalent operations) with a {@code BIND} to restrict the vartibale to a specific value + * while still retaining the variable (e.g for FILETERs). + *

+ * 

+     *   (bgp
+     *     (?s :p 123)
+     *     (?s :q ?a)
+     *   )
+     *
+ * with binding {@code ?s = :x } becomes + * 
+     * (assign (?s :x)
+     *    (bgp
+     *      (:x :p 123)
+     *      (:x :q ?a)
+     *    ))
+     *
+ */ + public static Op inject(Op opInput, Binding binding) { + Set injectVars = binding.varsMentioned(); + Transform transform = new QueryIterLateral.TransformInject(injectVars, binding::get); + Op opOutput = Transformer.transform(transform, opInput); + return opOutput; + } + public static Op substitute(Op op, Binding binding) { // Want to avoid cost if the binding is empty // but the empty test is not zero-cost on non-empty things. - if ( isNotNeeded(binding) ) return op; return Transformer.transform(new OpSubstituteWorker(binding), op); diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/core/VarExprList.java b/jena-arq/src/main/java/org/apache/jena/sparql/core/VarExprList.java index 26756d90279..f2b4336d833 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/core/VarExprList.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/core/VarExprList.java @@ -35,10 +35,10 @@ public class VarExprList { private List vars; private LinkedHashMap exprs; // Preserve order. - public VarExprList(List vars) { - this.vars = new ArrayList<>(vars); - this.exprs = new LinkedHashMap<>(); - } +// public VarExprList(List vars) { +// this.vars = new ArrayList<>(vars); +// this.exprs = new LinkedHashMap<>(); +// } public VarExprList(VarExprList other) { this.vars = new ArrayList<>(other.vars); diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/Binding.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/Binding.java index c3fc2b044d2..9bb7fcf4b72 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/Binding.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/Binding.java @@ -19,6 +19,7 @@ package org.apache.jena.sparql.engine.binding; import java.util.Iterator; +import java.util.Set; import java.util.function.BiConsumer; import org.apache.jena.graph.Node; @@ -56,6 +57,9 @@ public static BindingBuilder builder(Binding parent) { /** Iterate over all variables of this binding. */ public Iterator vars(); + /** Iterate over all variables of this binding. */ + public Set varsMentioned(); + /** Operate on each entry. */ public void forEach(BiConsumer action); diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/BindingBase.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/BindingBase.java index 4716290bae8..0e7fe54610c 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/BindingBase.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/binding/BindingBase.java @@ -19,6 +19,8 @@ package org.apache.jena.sparql.engine.binding; import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.function.BiConsumer; import org.apache.jena.atlas.iterator.IteratorConcat; @@ -45,14 +47,20 @@ protected BindingBase(Binding _parent) { // public Binding getParent() { return parent; } @Override - final public Iterator vars() - { + final public Iterator vars() { Iterator iter = vars1(); if ( parent != null ) iter = IteratorConcat.concat(parent.vars(), iter); return iter; } + @Override + final public Set varsMentioned() { + Set result = new LinkedHashSet<>(); + vars().forEachRemaining(result::add); + return result; + } + /** Operate on each entry. */ @Override public void forEach(BiConsumer action) { diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/iterator/QueryIterLateral.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/iterator/QueryIterLateral.java index 734c9278a53..1162c6cb802 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/iterator/QueryIterLateral.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/iterator/QueryIterLateral.java @@ -18,23 +18,36 @@ package org.apache.jena.sparql.engine.iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +import org.apache.jena.atlas.lib.SetUtils; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.Triple; import org.apache.jena.sparql.algebra.Op; -import org.apache.jena.sparql.algebra.op.OpTable; -import org.apache.jena.sparql.core.Substitute; +import org.apache.jena.sparql.algebra.TransformCopy; +import org.apache.jena.sparql.algebra.op.*; +import org.apache.jena.sparql.core.*; import org.apache.jena.sparql.engine.ExecutionContext; import org.apache.jena.sparql.engine.QueryIterator; import org.apache.jena.sparql.engine.binding.Binding; +import org.apache.jena.sparql.engine.binding.BindingBuilder; +import org.apache.jena.sparql.engine.binding.BindingFactory; import org.apache.jena.sparql.engine.main.QC; +import org.apache.jena.sparql.expr.NodeValue; +import org.apache.jena.sparql.util.VarUtils; public class QueryIterLateral extends QueryIterRepeatApply { - private final Op lateralOp; + private final Op subOp; private final boolean isUnit; - public QueryIterLateral(QueryIterator input, Op lateralOp, ExecutionContext execCxt) { + public QueryIterLateral(QueryIterator input, Op subOp, ExecutionContext execCxt) { super(input, execCxt); - this.lateralOp = lateralOp; - this.isUnit = isJoinIdentity(lateralOp); + this.subOp = subOp; + this.isUnit = isJoinIdentity(subOp); } private boolean isJoinIdentity(Op op) { @@ -47,7 +60,283 @@ private boolean isJoinIdentity(Op op) { protected QueryIterator nextStage(Binding binding) { if ( isUnit ) return QueryIterSingleton.create(binding, super.getExecContext()); - Op op = Substitute.substitute(lateralOp, binding); + Op op = Substitute.inject(subOp, binding); return QC.execute(op, binding, super.getExecContext()); } + + /* + * This transform applies variable substitution by injecting a binding for the variable and + * also substituting the variable in the op. + * The second step - substitution - is strictly unnecessary it happens anyway. + */ + public static class TransformInject extends TransformCopy { + + private final Set injectVars; + private final Set varsAsNodes; + private final Function replacement; + private static final boolean substitute = true; + + // Replacement becomes binding.?? + // Or "op call injection"!! + public TransformInject(Set injectVars, Function replacement) { + this.injectVars = injectVars; + this.varsAsNodes = Set.copyOf(injectVars); + this.replacement = replacement; + } + + @Override + public Op transform(OpBGP opBGP) { + if ( injectVars.isEmpty()) + return opBGP; + BasicPattern bp = opBGP.getPattern(); + List triples = bp.getList(); + // Alt - build up triple by triple. + //generateAssignmentTriple(triple, assigns, builder); + Set bgpVars = new LinkedHashSet<>(); + VarUtils.addVarsTriples(bgpVars, triples); + Set x = SetUtils.intersection(bgpVars, injectVars); + if ( x.isEmpty()) + return opBGP; + VarExprList assigns = new VarExprList(); + BindingBuilder builder = BindingFactory.builder(); + for ( Var var : x ) + generateAssignmentVar(var, assigns, builder); + + if ( assigns.isEmpty() ) + return super.transform(opBGP); + + // If no substitutions, do less work. + Binding substitutions = builder.build(); + Op opExec = substitute + ? Substitute.substitute(opBGP, substitutions) + : opBGP; + opExec = OpAssign.create(opExec, assigns); + return opExec; + } + +// @Override +// public Op transform(OpQuadBlock opQuadBlock) { +// opQuadBlock.getPattern(); +// } + + @Override + public Op transform(OpQuadPattern opQuadPattern) { + // "One node : Commonality with OpGraph, OpService (OpDatasetNames) + // Function for maker. + BasicPattern bgp = opQuadPattern.getBasicPattern(); + VarExprList assigns = new VarExprList(); + BindingBuilder builder = BindingFactory.builder(); + Node gn = opQuadPattern.getGraphNode(); + + generateAssignmentNode(gn, assigns, builder); + + for ( Triple t : bgp ) { + generateAssignmentTriple(t, assigns, builder); + } + if ( assigns.isEmpty() ) + return super.transform(opQuadPattern); + // If no substitutions, do less work. + Op opExec = opQuadPattern; + if ( substitute ) { + Binding substitutions = builder.build(); + opExec = Substitute.substitute(opQuadPattern, substitutions); + } + opExec = OpAssign.create(opExec, assigns); + return opExec; + } + + // XXX Extract - do one node + + @Override + public Op transform(OpService opService, Op subOp) { + Node g = opService.getService(); + if ( ! workToDo(g) ) + return super.transform(opService, subOp); + // Fast small lists? + VarExprList assigns = new VarExprList(); + BindingBuilder builder = BindingFactory.builder(); + Var var = Var.alloc(g); + generateAssignmentVar(var, assigns, builder); + if ( assigns.isEmpty() ) + return super.transform(opService, subOp); + // subOp has already been processed. + + Node g2 = g; + Op op2; + if ( substitute ) { + Binding substitutions = builder.build(); + g2 = substitutions.get(var); + op2 = new OpService(g2, subOp, opService.getSilent()); + } else + op2 = new OpService(g, subOp, opService.getSilent()); + Op opExec = OpAssign.create(op2, assigns); + return opExec; + } + + @Override + public Op transform(OpGraph opGraph, Op subOp) { + Node g = opGraph.getNode(); + if ( ! workToDo(g) ) + return super.transform(opGraph, subOp); + // Fast small lists? + VarExprList assigns = new VarExprList(); + BindingBuilder builder = BindingFactory.builder(); + Var var = Var.alloc(g); + generateAssignmentVar(var, assigns, builder); + if ( assigns.isEmpty() ) + return super.transform(opGraph, subOp); + // subOp has already been processed. + + Node g2 = g; + Op op2; + if ( substitute ) { + Binding substitutions = builder.build(); + g2 = substitutions.get(var); + op2 = new OpGraph(g2, subOp); + } else + op2 = new OpGraph(g, subOp); + Op opExec = OpAssign.create(op2, assigns); + return opExec; + } + + @Override + public Op transform(OpDatasetNames opDatasetNames) { + Node g = opDatasetNames.getGraphNode(); + if ( ! workToDo(g) ) + return super.transform(opDatasetNames); + Var var = Var.alloc(g); + Node g2 = replacement.apply(var); + Op op2 = new OpGraph(g2, OpTable.unit()); + return op2; + } + +// Binding for variables occurs in several places in SPARQL: + // +// Basic Graph Pattern Matching +// Property Path Patterns +// evaluation of algebra form Graph(var,P) involving a variable (from the syntax GRAPH ?variable {…}) + + @Override + public Op transform(OpPath opPath) { + // Exactly one of predicate and path is defined. + TriplePath path = opPath.getTriplePath(); + VarExprList assigns = new VarExprList(); + BindingBuilder builder = BindingFactory.builder(); + + if ( path.isTriple() ) { + Triple t1 = path.asTriple(); + generateAssignmentTriple(t1, assigns, builder); + // XXX Triple t2 = Substitute.substitute(triple, binding); + Triple t2 = applyReplacement(t1, replacement); + if ( t1.equals(t2) ) + return opPath; + TriplePath path2 = new TriplePath(t2); + return new OpPath(path2); + } + + // Path, not predicate. + Node s = path.getSubject(); + Node o = path.getObject(); + if ( ! workToDo(s) && ! workToDo(o) ) + return super.transform(opPath); + + generateAssignmentNode(s, assigns, builder); + generateAssignmentNode(o, assigns, builder); + + if ( assigns.isEmpty() ) + return super.transform(opPath); + + Node s2 = applyReplacement(s, replacement); + Node o2 = applyReplacement(o, replacement); + TriplePath path2 = new TriplePath(s2, path.getPath(), o2); + Op op2 = new OpPath(path2); + Op opExec = OpAssign.create(op2, assigns); + return opExec; + } + + @Override + public Op transform(OpTriple opTriple) { + Triple triple = opTriple.getTriple(); + VarExprList assigns = new VarExprList(); + BindingBuilder builder = BindingFactory.builder(); + generateAssignmentTriple(triple, assigns, builder); + if ( assigns.isEmpty() ) + return super.transform(opTriple); + Triple t2 = triple; + if ( substitute ) + t2 = applyReplacement(triple, replacement); + Op op2 = new OpTriple(t2); + Op opExec = OpAssign.create(op2, assigns); + return opExec; + } + + private Triple applyReplacement(Triple triple, Function replacement) { + Node s2 = applyReplacement(triple.getSubject(), replacement); + Node p2 = applyReplacement(triple.getPredicate(), replacement); + Node o2 = applyReplacement(triple.getObject(), replacement); + Triple t2 = Triple.create(s2, p2, o2); + return t2; + } + + private void generateAssignmentTriple(Triple triple, VarExprList assigns, BindingBuilder builder) { + Node s = triple.getSubject(); + Node p = triple.getPredicate(); + Node o = triple.getObject(); + if ( ! workToDo(s) && ! workToDo(p) && ! workToDo(o) ) + return; + generateAssignmentNode(s, assigns, builder); + generateAssignmentNode(p, assigns, builder); + generateAssignmentNode(o, assigns, builder); + } + + private static Node applyReplacement(Node n, Function replacement) { + if ( n instanceof Var x ) { + Node x2 = replacement.apply(x); + return ( x2 == null ) ? n : x2; + } + return n; + } + +// // Allow for nulls. +// private Triple transform(Triple triple) { +// +// if ( assigns.isEmpty() ) +// return triple; +// +// // XXX assignments() +// +// Triple triple2 = Triple.create(s, p, o); +// return triple2; +// } + + // XXX Other, non-std, ops. + + private void generateAssignmentNode(Node n, VarExprList assigns, BindingBuilder builder) { + if ( n == null ) + return; + if ( ! Var.isVar(n) ) + return; + generateAssignmentVar(Var.alloc(n), assigns, builder); + } + + private void generateAssignmentVar(Var var, VarExprList assigns, BindingBuilder builder) { + Node value = replacement.apply(var); + if ( value != null ) { + if ( ! builder.contains(var) ) { + builder.add(var, value); + assigns.add(var, NodeValue.makeNode(value)); + } + } + } + + // XXX Needed? To avoid object allocation. + private boolean workToDo(Node n) { + if ( n == null ) + return false; + if ( ! Var.isVar(n) ) + return false; + Var v = Var.alloc(n); + return null != replacement.apply(v); + } + } } diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/JoinClassifier.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/JoinClassifier.java index 194654085a9..2855857b439 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/JoinClassifier.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/JoinClassifier.java @@ -60,6 +60,9 @@ static public boolean isLinear(Op _left, Op _right) { if ( right instanceof OpTopN ) return false ; if ( right instanceof OpOrder ) return false ; + // Lateral is different. + if ( right instanceof OpLateral ) return false ; + // Assume something will not commute these later on. return check(left, right) ; } @@ -67,7 +70,9 @@ static public boolean isLinear(Op _left, Op _right) { // -- pre check for ops we can't handle in a linear fashion. // These are the negation patterns (minus and diff) // FILTER NOT EXISTS is safe - it's defined by iteration like the linear execution algorithm. - private static class UnsafeLineraOpException extends RuntimeException {} + private static class UnsafeLineraOpException extends RuntimeException { + @Override public Throwable fillInStackTrace() { return this; } + } private static OpVisitor checkForUnsafeVisitor = new OpVisitorBase() { @Override public void visit(OpMinus opMinus) { throw new UnsafeLineraOpException(); } @Override public void visit(OpDiff opDiff) { throw new UnsafeLineraOpException(); } diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/LeftJoinClassifier.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/LeftJoinClassifier.java index b94a601315b..85c59069e2e 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/LeftJoinClassifier.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/main/LeftJoinClassifier.java @@ -25,6 +25,7 @@ import org.apache.jena.sparql.algebra.Op ; import org.apache.jena.sparql.algebra.OpVars ; import org.apache.jena.sparql.algebra.op.OpExt ; +import org.apache.jena.sparql.algebra.op.OpLateral; import org.apache.jena.sparql.algebra.op.OpLeftJoin ; import org.apache.jena.sparql.algebra.op.OpModifier ; import org.apache.jena.sparql.core.Var ; @@ -59,6 +60,8 @@ static public boolean isLinear(Op left, Op right) { // With SELECT *, it's as if the subquery were just the pattern. if ( right instanceof OpModifier ) return false ; + if ( right instanceof OpLateral ) + return false ; Set leftVars = OpVars.visibleVars(left) ; if ( print ) { diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/EvaluatorDispatch.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/EvaluatorDispatch.java index 1df12f683bb..f1b1de27902 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/EvaluatorDispatch.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/EvaluatorDispatch.java @@ -35,88 +35,76 @@ /** Class to provide type-safe eval() dispatch using the visitor support of Op */ -public class EvaluatorDispatch implements OpVisitor -{ +public class EvaluatorDispatch implements OpVisitor { private Deque stack = new ArrayDeque<>(); protected Evaluator evaluator; - public EvaluatorDispatch(Evaluator evaluator) - { + public EvaluatorDispatch(Evaluator evaluator) { this.evaluator = evaluator; } - protected Table eval(Op op) - { + protected Table eval(Op op) { op.visit(this); return pop(); } - Table getResult() - { + Table getResult() { if ( stack.size() != 1 ) - Log.warn(this, "Warning: getResult: stack size = "+stack.size()); + Log.warn(this, "Warning: getResult: stack size = " + stack.size()); Table table = pop(); return table; } @Override - public void visit(OpBGP opBGP) - { + public void visit(OpBGP opBGP) { Table table = evaluator.basicPattern(opBGP.getPattern()); push(table); } @Override - public void visit(OpQuadPattern quadPattern) - { + public void visit(OpQuadPattern quadPattern) { push(RefEval.evalQuadPattern(quadPattern, evaluator)); } @Override - public void visit(OpQuadBlock quadBlock) - { + public void visit(OpQuadBlock quadBlock) { push(eval(quadBlock.convertOp())); - //push(Eval.evalQuadPattern(quadBlock, evaluator)); + // push(Eval.evalQuadPattern(quadBlock, evaluator)); } @Override - public void visit(OpTriple opTriple) - { + public void visit(OpTriple opTriple) { visit(opTriple.asBGP()); } @Override - public void visit(OpQuad opQuad) - { + public void visit(OpQuad opQuad) { visit(opQuad.asQuadPattern()); } + @Override - public void visit(OpPath opPath) - { + public void visit(OpPath opPath) { Table table = evaluator.pathPattern(opPath.getTriplePath()); push(table); } @Override - public void visit(OpProcedure opProc) - { + public void visit(OpProcedure opProc) { Table table = eval(opProc.getSubOp()); table = evaluator.procedure(table, opProc.getProcId(), opProc.getArgs()); push(table); } @Override - public void visit(OpPropFunc opPropFunc) - { + public void visit(OpPropFunc opPropFunc) { Table table = eval(opPropFunc.getSubOp()); table = evaluator.propertyFunction(table, opPropFunc.getProperty(), opPropFunc.getSubjectArgs(), opPropFunc.getObjectArgs()); push(table); } @Override - public void visit(OpJoin opJoin) - { + public void visit(OpJoin opJoin) { Table left = eval(opJoin.getLeft()); Table right = eval(opJoin.getRight()); Table table = evaluator.join(left, right); @@ -124,13 +112,11 @@ public void visit(OpJoin opJoin) } @Override - public void visit(OpSequence opSequence) - { + public void visit(OpSequence opSequence) { // Evaluation is as a sequence of joins. Table table = TableFactory.createUnit(); - for ( Iterator iter = opSequence.iterator(); iter.hasNext(); ) - { + for ( Iterator iter = opSequence.iterator() ; iter.hasNext() ; ) { Op op = iter.next(); Table eltTable = eval(op); table = evaluator.join(table, eltTable); @@ -139,13 +125,11 @@ public void visit(OpSequence opSequence) } @Override - public void visit(OpDisjunction opDisjunction) - { + public void visit(OpDisjunction opDisjunction) { // Evaluation is as a concatentation of alternatives Table table = TableFactory.createEmpty(); - for ( Iterator iter = opDisjunction.iterator(); iter.hasNext(); ) - { + for ( Iterator iter = opDisjunction.iterator() ; iter.hasNext() ; ) { Op op = iter.next(); Table eltTable = eval(op); table = evaluator.union(table, eltTable); @@ -154,17 +138,26 @@ public void visit(OpDisjunction opDisjunction) } @Override - public void visit(OpLeftJoin opLeftJoin) - { + public void visit(OpLeftJoin opLeftJoin) { Table left = eval(opLeftJoin.getLeft()); Table right = eval(opLeftJoin.getRight()); + + try { + left.rows(); + } catch (Throwable th) { + Op x = opLeftJoin.getLeft(); + eval(x); + + System.out.println("NULL"); + } + + Table table = evaluator.leftJoin(left, right, opLeftJoin.getExprs()); push(table); } @Override - public void visit(OpDiff opDiff) - { + public void visit(OpDiff opDiff) { Table left = eval(opDiff.getLeft()); Table right = eval(opDiff.getRight()); Table table = evaluator.diff(left, right); @@ -172,8 +165,7 @@ public void visit(OpDiff opDiff) } @Override - public void visit(OpMinus opMinus) - { + public void visit(OpMinus opMinus) { Table left = eval(opMinus.getLeft()); Table right = eval(opMinus.getRight()); Table table = evaluator.minus(left, right); @@ -181,8 +173,7 @@ public void visit(OpMinus opMinus) } @Override - public void visit(OpUnion opUnion) - { + public void visit(OpUnion opUnion) { Table left = eval(opUnion.getLeft()); Table right = eval(opUnion.getRight()); Table table = evaluator.union(left, right); @@ -190,8 +181,7 @@ public void visit(OpUnion opUnion) } @Override - public void visit(OpConditional opCond) - { + public void visit(OpConditional opCond) { Table left = eval(opCond.getLeft()); // Ref engine - don't care about efficiency Table right = eval(opCond.getRight()); @@ -200,60 +190,53 @@ public void visit(OpConditional opCond) } @Override - public void visit(OpLateral opLateral) - { + public void visit(OpLateral opLateral) { Table left = eval(opLateral.getLeft()); Table table = evaluator.lateral(left, opLateral.getRight()); push(table); } @Override - public void visit(OpFilter opFilter) - { + public void visit(OpFilter opFilter) { Table table = eval(opFilter.getSubOp()); table = evaluator.filter(opFilter.getExprs(), table); push(table); } @Override - public void visit(OpGraph opGraph) - { + public void visit(OpGraph opGraph) { push(RefEval.evalGraph(opGraph, evaluator)); } @Override - public void visit(OpService opService) - { + public void visit(OpService opService) { QueryIterator qIter = Service.exec(opService, ARQ.getContext()); Table table = TableFactory.create(qIter); push(table); } @Override - public void visit(OpDatasetNames dsNames) - { + public void visit(OpDatasetNames dsNames) { push(RefEval.evalDS(dsNames, evaluator)); } @Override - public void visit(OpTable opTable) - { + public void visit(OpTable opTable) { push(opTable.getTable()); } @Override - public void visit(OpExt opExt) - { throw new QueryExecException("Encountered OpExt during execution of reference engine"); } + public void visit(OpExt opExt) { + throw new QueryExecException("Encountered OpExt during execution of reference engine"); + } @Override - public void visit(OpNull opNull) - { + public void visit(OpNull opNull) { push(TableFactory.createEmpty()); } @Override - public void visit(OpLabel opLabel) - { + public void visit(OpLabel opLabel) { if ( opLabel.hasSubOp() ) push(eval(opLabel.getSubOp())); else @@ -262,93 +245,86 @@ public void visit(OpLabel opLabel) } @Override - public void visit(OpList opList) - { + public void visit(OpList opList) { Table table = eval(opList.getSubOp()); table = evaluator.list(table); push(table); } @Override - public void visit(OpOrder opOrder) - { + public void visit(OpOrder opOrder) { Table table = eval(opOrder.getSubOp()); table = evaluator.order(table, opOrder.getConditions()); push(table); } @Override - public void visit(OpTopN opTop) - { + public void visit(OpTopN opTop) { Table table = eval(opTop.getSubOp()); - //table = evaluator.topN(table, opTop.getLimti(), opTop.getConditions()); + // table = evaluator.topN(table, opTop.getLimti(), opTop.getConditions()); table = evaluator.order(table, opTop.getConditions()); table = evaluator.slice(table, 0, opTop.getLimit()); push(table); } @Override - public void visit(OpProject opProject) - { + public void visit(OpProject opProject) { Table table = eval(opProject.getSubOp()); table = evaluator.project(table, opProject.getVars()); push(table); } @Override - public void visit(OpDistinct opDistinct) - { + public void visit(OpDistinct opDistinct) { Table table = eval(opDistinct.getSubOp()); table = evaluator.distinct(table); push(table); } @Override - public void visit(OpReduced opReduced) - { + public void visit(OpReduced opReduced) { Table table = eval(opReduced.getSubOp()); table = evaluator.reduced(table); push(table); } @Override - public void visit(OpSlice opSlice) - { + public void visit(OpSlice opSlice) { Table table = eval(opSlice.getSubOp()); table = evaluator.slice(table, opSlice.getStart(), opSlice.getLength()); push(table); } @Override - public void visit(OpAssign opAssign) - { + public void visit(OpAssign opAssign) { Table table = eval(opAssign.getSubOp()); table = evaluator.assign(table, opAssign.getVarExprList()); push(table); } @Override - public void visit(OpExtend opExtend) - { + public void visit(OpExtend opExtend) { Table table = eval(opExtend.getSubOp()); table = evaluator.extend(table, opExtend.getVarExprList()); push(table); } @Override - public void visit(OpGroup opGroup) - { + public void visit(OpGroup opGroup) { Table table = eval(opGroup.getSubOp()); table = evaluator.groupBy(table, opGroup.getGroupVars(), opGroup.getAggregators()); push(table); } - protected void push(Table table) { stack.push(table); } - protected Table pop() - { + protected void push(Table table) { + stack.push(table); + } + + protected Table pop() { if ( stack.size() == 0 ) Log.warn(this, "Warning: pop: empty stack"); - return stack.pop(); + Table table = stack.pop(); + return table; } } diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/RefEval.java b/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/RefEval.java index ba212ff3f8c..83ddfd42563 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/RefEval.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/engine/ref/RefEval.java @@ -49,7 +49,6 @@ import org.apache.jena.sparql.engine.iterator.QueryIterRoot; import org.apache.jena.sparql.engine.main.QC; -// Spit out a few of the longer ops. public class RefEval { public static Table eval(Evaluator evaluator, Op op) { EvaluatorDispatch ev = new EvaluatorDispatch(evaluator); @@ -58,6 +57,7 @@ public static Table eval(Evaluator evaluator, Op op) { return table; } + // Spit out a few of the longer ops. static Table evalDS(OpDatasetNames opDSN, Evaluator evaluator) { Node graphNode = opDSN.getGraphNode(); if ( graphNode.isURI() ) { diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/lang/SyntaxVarScope.java b/jena-arq/src/main/java/org/apache/jena/sparql/lang/SyntaxVarScope.java index cb79fd01ce8..5e0690ad151 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/lang/SyntaxVarScope.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/lang/SyntaxVarScope.java @@ -31,21 +31,29 @@ /** Calculate in-scope variables from the AST */ public class SyntaxVarScope { + // @formatter:off /* SPARQL 1.1 "in scope" rules These define the variables from a pattern that are * in-scope These are not the usage rules. * - * Syntax Form In-scope variables + * 18.2.1 Variable Scope * - * Basic Graph Pattern (BGP) v occurs in the BGP Path v occurs in the path Group - * { P1 P2 ... } v is in-scope if in-scope in one or more of P1, P2, ... GRAPH - * term { P } v is term or v is in-scope in P { P1 } UNION { P2 } v is in-scope - * in P1 or in-scope in P2 OPTIONAL {P} v is in-scope in P SERVICE term {P} v is - * term or v is in-scope in P (expr AS v) for BIND, SELECT and GROUP BY v is - * in-scope SELECT ..v .. { P } v is in-scope if v is mentioned as a project - * variable SELECT * { P } v is in-scope in P VALUES var (values) v is in-scope - * if v is in varlist VALUES varlist (values) v is in-scope if v is in varlist */ - - // Weakness : EXISTS inside FILTERs? + * Syntax Form In-scope variables + * 
+     * Basic Graph Pattern (BGP) v occurs in the BGP
+     * Path                      v occurs in the path
+     * Group { P1 P2 ... }       v is in-scope if in-scope in one or more of P1, P2, ...
+     * GRAPH term { P }          v is term or v is in-scope in P
+     * { P1 } UNION { P2 }       v is in-scope in P1 or in-scope in P2
+     * OPTIONAL {P}              v is in-scope in P
+     * SERVICE term {P}          v is term or v is in-scope in P
+     * (expr AS v) for BIND, SELECT and GROUP BY   v is in-scope
+     * SELECT ..v .. { P }       v is in-scope if v is mentioned as a project variable
+     * SELECT * { P }            v is in-scope in P
+     * VALUES var (values)       v is in-scope
+     * VALUES varlist (values)   v is in-scope if v is in varlist
+     *
+ */ + //@formatter:on public static void check(Query query) { if ( query.getQueryPattern() == null ) @@ -220,17 +228,17 @@ public void visit(ElementGroup el) { for ( int i = 0 ; i < el.size() ; i++ ) { Element e = el.get(i); // Tests. - if ( e instanceof ElementBind ) { + if ( e instanceof ElementBind eltBind) { Collection accScope = calcScopeAll(el.getElements(), i); - checkBIND(accScope, (ElementBind)e); + checkBIND(accScope, eltBind); } - if ( e instanceof ElementService ) { + if ( e instanceof ElementService eltSvc ) { Collection accScope = calcScopeAll(el.getElements(), i); - checkSERVICE(accScope, (ElementService)e); + checkSERVICE(accScope, eltSvc); } - if ( e instanceof ElementLateral ) { + if ( e instanceof ElementLateral eltLat) { Collection accScope = calcScopeAll(el.getElements(), i); - checkLATERAL(accScope, e); + checkLATERAL(accScope, eltLat); } } } @@ -285,6 +293,7 @@ public void visit(ElementData eltData) { @Override public void visit(ElementSubQuery eltSubQuery) { + // Only called when there is an expression. eltSubQuery.getQuery().getProject().forEachExpr((var, expr) -> { if ( accScope.contains(var) ) @@ -293,19 +302,23 @@ public void visit(ElementSubQuery eltSubQuery) { // Check inside query pattern Query subQuery = eltSubQuery.getQuery(); Collection accScope2 = accScope; - //eltSubQuery.getQuery().setResultVars(); if ( ! subQuery.isQueryResultStar() ) { List projectVars = eltSubQuery.getQuery().getProject().getVars(); - // Calculate variables passed down : scope which are in the project vars. + // Copy accScope2 = new ArrayList<>(accScope); + // Calculate variables passed down : remove any that are not in the project vars + // Any reused name will be renamed apart later. accScope2.removeIf(v->!projectVars.contains(v)); } + if ( accScope2.isEmpty() ) + // No work to do. + return; Element el2 = eltSubQuery.getQuery().getQueryPattern(); checkLATERAL(accScope2, el2); } }; - // Does not walk into subqueries but we need to change the scoep for sub-queries. + // Does not walk into subqueries but we need to change the scope for sub-queries. ElementWalker.walk(el, checker); } } diff --git a/jena-arq/testing/ARQ/Lateral/data2.ttl b/jena-arq/testing/ARQ/Lateral/data2.ttl new file mode 100644 index 00000000000..35745fbbfa2 --- /dev/null +++ b/jena-arq/testing/ARQ/Lateral/data2.ttl @@ -0,0 +1,9 @@ +@prefix ex: . + +ex:s1 a ex:T ; + ex:p "11" , "12" , "13" . + +ex:s2 a ex:T ; + ex:p "21" , "22" , "23" . + +ex:s3 a ex:T . diff --git a/jena-arq/testing/ARQ/Lateral/lateral-in-optional.arq b/jena-arq/testing/ARQ/Lateral/lateral-in-optional.arq new file mode 100644 index 00000000000..a746707a5d5 --- /dev/null +++ b/jena-arq/testing/ARQ/Lateral/lateral-in-optional.arq @@ -0,0 +1,15 @@ + PREFIX ex: + + SELECT ?s ?o + WHERE + { ?s a ex:T + OPTIONAL + { LATERAL + { SELECT ?s ?o + WHERE + { ?s ex:p ?o } + ORDER BY ?o + LIMIT 2 + } + } + } \ No newline at end of file diff --git a/jena-arq/testing/ARQ/Lateral/lateral-in-optional.srj b/jena-arq/testing/ARQ/Lateral/lateral-in-optional.srj new file mode 100644 index 00000000000..1a521b0769e --- /dev/null +++ b/jena-arq/testing/ARQ/Lateral/lateral-in-optional.srj @@ -0,0 +1,22 @@ +{ "head": { + "vars": [ "s" , "o" ] + } , + "results": { + "bindings": [ + { + "s": { "type": "uri" , "value": "http://example.org/s3" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s1" } , + "o": { "type": "literal" , "value": "11" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s1" } , + "o": { "type": "literal" , "value": "12" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s2" } + } + ] + } +} diff --git a/jena-arq/testing/ARQ/Lateral/manifest.ttl b/jena-arq/testing/ARQ/Lateral/manifest.ttl index 08276f22a1c..ea2b7ee3a00 100644 --- a/jena-arq/testing/ARQ/Lateral/manifest.ttl +++ b/jena-arq/testing/ARQ/Lateral/manifest.ttl @@ -37,6 +37,8 @@ PREFIX skos: <#lateral-3> <#lateral-4> <#lateral-5> + <#lateral-6> + <#lateral-7> ). <#lateral-1> rdf:type mf:QueryEvaluationTest ; @@ -84,6 +86,24 @@ PREFIX skos: mf:result . +<#lateral-6> rdf:type mf:QueryEvaluationTest ; + mf:name "LATERAL - LATERAL inside OPTIONAL" ; + mf:action [ + qt:query ; + qt:data + ] ; + mf:result + . + +<#lateral-7> rdf:type mf:QueryEvaluationTest ; + mf:name "LATERAL - OPTIONAL inside LATERAL" ; + mf:action [ + qt:query ; + qt:data + ] ; + mf:result + . + ## ?x :q ?z . LATERAL { ?s :q ?o FILTER (?z = ?o) } diff --git a/jena-arq/testing/ARQ/Lateral/optional-in-lateral.arq b/jena-arq/testing/ARQ/Lateral/optional-in-lateral.arq new file mode 100644 index 00000000000..19834d9ee72 --- /dev/null +++ b/jena-arq/testing/ARQ/Lateral/optional-in-lateral.arq @@ -0,0 +1,16 @@ + PREFIX ex: + + SELECT ?s ?o + WHERE + { ?s a ex:T + LATERAL + { OPTIONAL + { SELECT ?s ?o + WHERE + { ?s ex:p ?o } + ORDER BY ?o + LIMIT 2 + } + } + } + \ No newline at end of file diff --git a/jena-arq/testing/ARQ/Lateral/optional-in-lateral.srj b/jena-arq/testing/ARQ/Lateral/optional-in-lateral.srj new file mode 100644 index 00000000000..97d1bd12c8a --- /dev/null +++ b/jena-arq/testing/ARQ/Lateral/optional-in-lateral.srj @@ -0,0 +1,27 @@ +{ "head": { + "vars": [ "s" , "o" ] + } , + "results": { + "bindings": [ + { + "s": { "type": "uri" , "value": "http://example.org/s3" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s1" } , + "o": { "type": "literal" , "value": "11" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s1" } , + "o": { "type": "literal" , "value": "12" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s2" } , + "o": { "type": "literal" , "value": "21" } + } , + { + "s": { "type": "uri" , "value": "http://example.org/s2" } , + "o": { "type": "literal" , "value": "22" } + } + ] + } +}