apacheGH-1615: LATERAL: Parse and evaluate

jena-arq/Grammar/arq.jj

@@ -35,6 +35,8 @@ import org.apache.jena.sparql.expr.aggregate.lib.* ;
 import org.apache.jena.update.* ;
 import org.apache.jena.sparql.modify.request.* ;
 import org.apache.jena.sparql.core.Quad ;
+import java.util.List;
+import java.util.ArrayList;
 public class ARQParser extends ARQParserBase
 {}
 PARSER_END(ARQParser)
@@ -612,6 +614,8 @@ Element GraphPatternNotTriples() : { Element el = null ; }
    el = GroupOrUnionGraphPattern()
  |
    el = OptionalGraphPattern()
+ |
+   el = LateralGraphPattern()
  |
    el = MinusGraphPattern()
  |
@@ -637,6 +641,10 @@ Element OptionalGraphPattern() : { Element el ; }
 { <OPTIONAL> el = GroupGraphPattern()
     { return new ElementOptional(el) ; }
 }
+Element LateralGraphPattern() : { Element el ; }
+{ <LATERAL> el = GroupGraphPattern()
+    { return new ElementLateral(el) ; }
+}
 Element GraphGraphPattern() : { Element el ; Node n ;}
 {
   <GRAPH> n = VarOrIri() el = GroupGraphPattern()
@@ -1783,6 +1791,7 @@ TOKEN [IGNORE_CASE] :
 | < SUBJECT: "SUBJECT" >
 | < PREDICATE: "PREDICATE" >
 | < OBJECT: "OBJECT" >
+| < LATERAL: "LATERAL" >
 | < EXISTS: "exists" >
 | < NOT: "not" >
 | < AS: "as" >

jena-arq/Grammar/main.jj

@@ -99,6 +99,8 @@ import org.apache.jena.sparql.modify.request.* ;
 #endif
 #ifdef ARQ
 import org.apache.jena.sparql.core.Quad ;
+import java.util.List;
+import java.util.ArrayList;
 #endif
 
 public class CLASS extends PARSERBASE
@@ -879,6 +881,10 @@ Element GraphPatternNotTriples() : { Element el = null ; }
    el = GroupOrUnionGraphPattern()
  |
    el = OptionalGraphPattern()
+#ifdef ARQ
+ |
+   el = LateralGraphPattern()
+#endif
  |
    el = MinusGraphPattern()
  |
@@ -910,13 +916,19 @@ Element OptionalGraphPattern() : { Element el ; }
     { return new ElementOptional(el) ; }
 }
 
+#ifdef ARQ
+Element LateralGraphPattern() : { Element el ; }
+{ <LATERAL> el = GroupGraphPattern()
+    { return new ElementLateral(el) ; }
+}
+#endif
+
 Element GraphGraphPattern() : { Element el ; Node n ;}
 {
   <GRAPH> n = VarOrIri() el = GroupGraphPattern()
     { return new ElementNamedGraph(n, el) ; }
 }
 
-
 Element ServiceGraphPattern() : { Element el ; Node n ; boolean silent = false ; }
 {
   <SERVICE> 
@@ -2522,6 +2534,7 @@ TOKEN [IGNORE_CASE] :
 |  < SUBJECT:     "SUBJECT" >
 |  < PREDICATE:   "PREDICATE" >
 |  < OBJECT:      "OBJECT" >
+|  < LATERAL:     "LATERAL" >
 #endif
 |  < EXISTS:      "exists" >
 |  < NOT:         "not" >

jena-arq/Grammar/tokens.txt

@@ -104,6 +104,7 @@
 [<GRAPH>]      ::= 'GRAPH'
 [<UNION>]      ::= 'UNION'
 [<OPTIONAL>]   ::= 'OPTIONAL'
+[<LATERAL>]    ::= 'LATERAL'
 [<BOUND>]      ::= 'BOUND'
 [<COALESCE>]   ::= 'COALESCE'
 [<IF>]         ::= 'IF'

jena-arq/src/main/java/org/apache/jena/sparql/algebra/AlgebraGenerator.java

@@ -301,6 +301,11 @@ protected Op compileOneInGroup(Element elt, Op current, Deque<Op> acc) {
             return compileElementOptional(eltOpt, current);
         }
 
+        if ( elt instanceof ElementLateral ) {
+            ElementLateral eltLateral = (ElementLateral)elt;
+            return compileElementLateral(eltLateral, current);
+        }
+
         if ( elt instanceof ElementMinus ) {
             ElementMinus elt2 = (ElementMinus)elt;
             Op op = compileElementMinus(current, elt2);
@@ -397,6 +402,12 @@ protected Op compileElementOptional(ElementOptional eltOpt, Op current) {
         return current;
     }
 
+    protected Op compileElementLateral(ElementLateral eltLateral, Op current) {
+        Element subElt = eltLateral.getLateralElement();
+        Op op = compileElement(subElt);
+        return OpLateral.create(current, op);
+    }
+
     protected Op compileBasicPattern(BasicPattern pattern) {
         return new OpBGP(pattern);
     }

jena-arq/src/main/java/org/apache/jena/sparql/algebra/OpAsQuery.java

@@ -37,11 +37,15 @@
 import org.apache.jena.sparql.core.Var ;
 import org.apache.jena.sparql.core.VarExprList ;
 import org.apache.jena.sparql.engine.QueryIterator ;
+import org.apache.jena.sparql.engine.Rename;
 import org.apache.jena.sparql.expr.* ;
 import org.apache.jena.sparql.expr.aggregate.Aggregator ;
 import org.apache.jena.sparql.pfunction.PropFuncArg ;
 import org.apache.jena.sparql.syntax.* ;
-import org.apache.jena.sparql.syntax.syntaxtransform.*;
+import org.apache.jena.sparql.syntax.syntaxtransform.ElementTransform;
+import org.apache.jena.sparql.syntax.syntaxtransform.ElementTransformCleanGroupsOfOne;
+import org.apache.jena.sparql.syntax.syntaxtransform.ElementTransformer;
+import org.apache.jena.sparql.syntax.syntaxtransform.ExprTransformApplyElementTransform;
 import org.apache.jena.sparql.util.graph.GraphList ;
 import org.apache.jena.vocabulary.RDF ;
 
@@ -698,9 +702,25 @@ public void visit(OpUnion opUnion) {
             currentGroup().addElement(elUnion) ;
         }
 
+        @Override
+        public void visit(OpLateral opLateral) {
+            Element eLeft = asElement(opLateral.getLeft()) ;
+            ElementGroup eRight = asElementGroup(opLateral.getRight()) ;
+            ElementGroup g = currentGroup() ;
+            if ( !emptyGroup(eLeft) ) {
+                if ( eLeft instanceof ElementGroup )
+                    g.getElements().addAll(((ElementGroup)eLeft).getElements()) ;
+                else
+                    g.addElement(eLeft) ;
+            }
+            ElementLateral eltLateral = new ElementLateral(eRight) ;
+            g.addElement(eltLateral) ;
+        }
+
         @Override
         public void visit(OpConditional opCondition) {
-            // Possibly imperfect because there might be filters outside the OpConditional.
+            // Possibly completely reversing a query exactly because
+            // there might be filters outside the OpConditional.
             convertLeftJoin(opCondition.getLeft(), opCondition.getRight(), null);
         }
 
@@ -822,7 +842,10 @@ private void newLevel(Op op) {
         }
 
         private void convertAsSubQuery(Op op) {
-            Converter subConverter = new Converter(op) ;
+            // Reverse scoped renaming.
+            // ?/x is illegal so the original query string must have had ?x at this point for any number of "/"
+            Op op1 = Rename.reverseVarRename(op, true);
+            Converter subConverter = new Converter(op1) ;
             ElementSubQuery subQuery = new ElementSubQuery(subConverter.convert()) ;
             ElementGroup g = currentGroup() ;
             g.addElement(subQuery) ;

jena-arq/src/main/java/org/apache/jena/sparql/algebra/OpVisitor.java

@@ -49,6 +49,7 @@ public interface OpVisitor
     public void visit(OpUnion opUnion) ;
     public void visit(OpDiff opDiff) ;
     public void visit(OpMinus opMinus) ;
+    public void visit(OpLateral opLateral) ;
     public void visit(OpConditional opCondition) ;
 
     // OpN

jena-arq/src/main/java/org/apache/jena/sparql/algebra/OpVisitorBase.java

@@ -49,6 +49,8 @@ public OpVisitorBase() {}
 
     @Override public void visit(OpLeftJoin opLeftJoin)          {}
 
+    @Override public void visit(OpLateral opLateral)            {}
+
     @Override public void visit(OpConditional opCond)           {}
 
     @Override public void visit(OpMinus opMinus)                {}

jena-arq/src/main/java/org/apache/jena/sparql/algebra/OpVisitorByType.java

@@ -103,6 +103,9 @@ public void visit(OpMinus opMinus)
     public void visit(OpUnion opUnion)
     { visit2(opUnion) ; }
 
+    @Override public void visit(OpLateral opLateral)
+    { visit2(opLateral) ; }
+
     @Override
     public void visit(OpConditional opCond)
     { visit2(opCond) ; }

jena-arq/src/main/java/org/apache/jena/sparql/algebra/Transform.java

@@ -51,6 +51,7 @@ public interface Transform
     public Op transform(OpDiff opDiff, Op left, Op right) ;
     public Op transform(OpMinus opMinus, Op left, Op right) ;
     public Op transform(OpUnion opUnion, Op left, Op right) ;
+    public Op transform(OpLateral opLater, Op left, Op right) ;
     public Op transform(OpConditional opCondition, Op left, Op right) ;
 
     // OpN

jena-arq/src/main/java/org/apache/jena/sparql/algebra/TransformCopy.java

@@ -80,6 +80,8 @@ public class TransformCopy implements Transform
     @Override
     public Op transform(OpUnion opUnion, Op left, Op right)         { return xform(opUnion, left, right) ; }
     @Override
+    public Op transform(OpLateral opLateral, Op left, Op right)     { return xform(opLateral, left, right) ; }
+    @Override
     public Op transform(OpConditional opCond, Op left, Op right)    { return xform(opCond, left, right) ; }
 
     @Override

jena-arq/src/main/java/org/apache/jena/sparql/algebra/TransformSingle.java

@@ -24,7 +24,7 @@
 
 /**
  * Special purpose base class for a single transformation.
- *  
+ *
  * When writing {@link Transform}s to be applied to a tree,
  * extend {@link TransformCopy}, not this class.
  */
@@ -76,6 +76,8 @@ public class TransformSingle implements Transform
     @Override
     public Op transform(OpUnion opUnion, Op left, Op right)         { return opUnion ; }
     @Override
+    public Op transform(OpLateral opCopLateral, Op left, Op right)  { return opCopLateral ; }
+    @Override
     public Op transform(OpConditional opCond, Op left, Op right)    { return opCond ; }
 
     @Override

jena-arq/src/main/java/org/apache/jena/sparql/algebra/TransformWrapper.java

@@ -78,6 +78,8 @@ public TransformWrapper(Transform transform)
     @Override
     public Op transform(OpUnion opUnion, Op left, Op right)         { return transform.transform(opUnion, left, right) ; }
     @Override
+    public Op transform(OpLateral opLateral, Op left, Op right)     { return transform.transform(opLateral, left, right) ; }
+    @Override
     public Op transform(OpConditional opCond, Op left, Op right)    { return transform.transform(opCond, left, right) ; }
 
     @Override

jena-arq/src/main/java/org/apache/jena/sparql/algebra/op/OpLateral.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.sparql.algebra.op;
+
+import org.apache.jena.sparql.algebra.Op;
+import org.apache.jena.sparql.algebra.OpVisitor;
+import org.apache.jena.sparql.algebra.Transform;
+import org.apache.jena.sparql.sse.Tags;
+import org.apache.jena.sparql.util.NodeIsomorphismMap;
+
+public class OpLateral extends Op2
+{
+    public static Op create(Op left, Op right) {
+        return new OpLateral(left, right);
+    }
+
+    private OpLateral(Op left, Op right) {
+        super(left, right);
+    }
+
+    @Override
+    public String getName() { return Tags.tagLateral; }
+
+    @Override
+    public Op apply(Transform transform, Op left, Op right)
+    { return transform.transform(this, left, right); }
+
+    @Override
+    public void visit(OpVisitor opVisitor) { opVisitor.visit(this); }
+
+    @Override
+    public Op2 copy(Op newLeft, Op newRight)
+    { return new OpLateral(newLeft, newRight); }
+
+    @Override
+    public boolean equalTo(Op op2, NodeIsomorphismMap labelMap) {
+        if ( ! ( op2 instanceof OpLateral) ) return false;
+        return super.sameArgumentsAs((Op2)op2, labelMap);
+    }
+}

jena-arq/src/main/java/org/apache/jena/sparql/algebra/optimize/TransformFilterImplicitJoin.java

@@ -120,10 +120,14 @@ private static Op apply(ExprList exprs, Op subOp) {
         // ---- Check if the subOp is the right shape to transform.
         Op op = subOp;
 
-        // Special case : deduce that the filter will always "eval unbound"
-        // hence eliminate all rows. Return the empty table.
+        // LATERAL : This is not longer true.
+//        // Special case : deduce that the filter will always "eval unbound"
+//        // hence eliminate all rows. Return the empty table.
+//        if (testSpecialCaseUnused(subOp, joins, remaining))
+//            return OpTable.empty();
+        // But simply skipping this causes (filter) to become (assign) which fails as (assign) does not handle errors.
         if (testSpecialCaseUnused(subOp, joins, remaining))
-            return OpTable.empty();
+            return null;
 
         // Special case: the deep left op of a OpConditional/OpLeftJoin is unit
         // table.

jena-arq/src/main/java/org/apache/jena/sparql/algebra/walker/OpVisitorByTypeAndExpr.java

@@ -133,6 +133,11 @@ public default void visit(OpUnion opUnion) {
         visit2(opUnion);
     }
 
+    @Override
+    public default void visit(OpLateral opLateral) {
+        visit2(opLateral);
+    }
+
     @Override
     public default void visit(OpConditional opCond) {
         visit2(opCond);