diff --git a/docs/reference.md b/docs/reference.md
index ab3e31be..b703f406 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -38,7 +38,7 @@ Contributions are welcome!
In Morel but not Standard ML:
* `from` expression with `in`, `suchthat`, `join`, `where`, `group`,
- `compute`, `order`, `yield` clauses
+ `compute`, `order`, `skip`, `take`, `yield` clauses
* `union`, `except`, `intersect`, `elem`, `notelem` operators
* "*lab* `=`" is optional in `exprow`
* identifiers may be quoted
@@ -159,7 +159,9 @@ In Standard ML but not in Morel:
compute clause (a > 1)
| order orderItem1 , ... , orderItemo
order clause (o ≥ 1)
- | yield exp
+ | skip exp skip clause
+ | take exp take clause
+ | yield exp yield clause
groupKey → [ id = ] exp
agg → [ id = ] exp [ of exp ]
orderItem → exp [ desc ]
diff --git a/src/main/java/net/hydromatic/morel/ast/Ast.java b/src/main/java/net/hydromatic/morel/ast/Ast.java
index 48473742..d74ed1f2 100644
--- a/src/main/java/net/hydromatic/morel/ast/Ast.java
+++ b/src/main/java/net/hydromatic/morel/ast/Ast.java
@@ -1707,6 +1707,58 @@ public Where copy(Exp exp) {
}
}
+ /** A {@code skip} clause in a {@code from} expression. */
+ public static class Skip extends FromStep {
+ public final Exp exp;
+
+ Skip(Pos pos, Exp exp) {
+ super(pos, Op.SKIP);
+ this.exp = exp;
+ }
+
+ @Override AstWriter unparse(AstWriter w, int left, int right) {
+ return w.append(" skip ").append(exp, 0, 0);
+ }
+
+ @Override public AstNode accept(Shuttle shuttle) {
+ return shuttle.visit(this);
+ }
+
+ @Override public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ public Skip copy(Exp exp) {
+ return this.exp.equals(exp) ? this : new Skip(pos, exp);
+ }
+ }
+
+ /** A {@code take} clause in a {@code from} expression. */
+ public static class Take extends FromStep {
+ public final Exp exp;
+
+ Take(Pos pos, Exp exp) {
+ super(pos, Op.TAKE);
+ this.exp = exp;
+ }
+
+ @Override AstWriter unparse(AstWriter w, int left, int right) {
+ return w.append(" take ").append(exp, 0, 0);
+ }
+
+ @Override public AstNode accept(Shuttle shuttle) {
+ return shuttle.visit(this);
+ }
+
+ @Override public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ public Take copy(Exp exp) {
+ return this.exp.equals(exp) ? this : new Take(pos, exp);
+ }
+ }
+
/** A {@code yield} clause in a {@code from} expression. */
public static class Yield extends FromStep {
public final Exp exp;
diff --git a/src/main/java/net/hydromatic/morel/ast/AstBuilder.java b/src/main/java/net/hydromatic/morel/ast/AstBuilder.java
index ac9493c1..b46ab652 100644
--- a/src/main/java/net/hydromatic/morel/ast/AstBuilder.java
+++ b/src/main/java/net/hydromatic/morel/ast/AstBuilder.java
@@ -483,6 +483,14 @@ public Ast.FromStep where(Pos pos, Ast.Exp exp) {
return new Ast.Where(pos, exp);
}
+ public Ast.FromStep skip(Pos pos, Ast.Exp exp) {
+ return new Ast.Skip(pos, exp);
+ }
+
+ public Ast.FromStep take(Pos pos, Ast.Exp exp) {
+ return new Ast.Take(pos, exp);
+ }
+
public Ast.FromStep yield(Pos pos, Ast.Exp exp) {
return new Ast.Yield(pos, exp);
}
diff --git a/src/main/java/net/hydromatic/morel/ast/Core.java b/src/main/java/net/hydromatic/morel/ast/Core.java
index 74b8799c..b1138bdf 100644
--- a/src/main/java/net/hydromatic/morel/ast/Core.java
+++ b/src/main/java/net/hydromatic/morel/ast/Core.java
@@ -1238,6 +1238,66 @@ public Where copy(Exp exp, List bindings) {
}
}
+ /** A {@code skip} clause in a {@code from} expression. */
+ public static class Skip extends FromStep {
+ public final Exp exp;
+
+ Skip(ImmutableList bindings, Exp exp) {
+ super(Op.SKIP, bindings);
+ this.exp = requireNonNull(exp, "exp");
+ }
+
+ @Override public Skip accept(Shuttle shuttle) {
+ return shuttle.visit(this);
+ }
+
+ @Override public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override protected AstWriter unparse(AstWriter w, From from, int ordinal,
+ int left, int right) {
+ return w.append(" skip ").append(exp, 0, 0);
+ }
+
+ public Skip copy(Exp exp, List bindings) {
+ return exp == this.exp
+ && bindings.equals(this.bindings)
+ ? this
+ : core.skip(bindings, exp);
+ }
+ }
+
+ /** A {@code take} clause in a {@code from} expression. */
+ public static class Take extends FromStep {
+ public final Exp exp;
+
+ Take(ImmutableList bindings, Exp exp) {
+ super(Op.TAKE, bindings);
+ this.exp = requireNonNull(exp, "exp");
+ }
+
+ @Override public Take accept(Shuttle shuttle) {
+ return shuttle.visit(this);
+ }
+
+ @Override public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override protected AstWriter unparse(AstWriter w, From from, int ordinal,
+ int left, int right) {
+ return w.append(" take ").append(exp, 0, 0);
+ }
+
+ public Take copy(Exp exp, List bindings) {
+ return exp == this.exp
+ && bindings.equals(this.bindings)
+ ? this
+ : core.take(bindings, exp);
+ }
+ }
+
/** An {@code order} clause in a {@code from} expression. */
public static class Order extends FromStep {
public final ImmutableList orderItems;
diff --git a/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java b/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java
index fb29a98f..d99faffb 100644
--- a/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java
+++ b/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java
@@ -521,6 +521,14 @@ public Core.Where where(List bindings, Core.Exp exp) {
return new Core.Where(ImmutableList.copyOf(bindings), exp);
}
+ public Core.Skip skip(List bindings, Core.Exp exp) {
+ return new Core.Skip(ImmutableList.copyOf(bindings), exp);
+ }
+
+ public Core.Take take(List bindings, Core.Exp exp) {
+ return new Core.Take(ImmutableList.copyOf(bindings), exp);
+ }
+
public Core.Yield yield_(List bindings, Core.Exp exp) {
return new Core.Yield(ImmutableList.copyOf(bindings), exp);
}
diff --git a/src/main/java/net/hydromatic/morel/ast/FromBuilder.java b/src/main/java/net/hydromatic/morel/ast/FromBuilder.java
index 69430d35..2d1149c4 100644
--- a/src/main/java/net/hydromatic/morel/ast/FromBuilder.java
+++ b/src/main/java/net/hydromatic/morel/ast/FromBuilder.java
@@ -29,6 +29,7 @@
import org.apache.calcite.util.Util;
import org.checkerframework.checker.nullness.qual.Nullable;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -192,6 +193,19 @@ public FromBuilder where(Core.Exp condition) {
return addStep(core.where(bindings, condition));
}
+ public FromBuilder skip(Core.Exp count) {
+ if (count.op == Op.INT_LITERAL
+ && ((Core.Literal) count).value.equals(BigDecimal.ZERO)) {
+ // skip "skip 0"
+ return this;
+ }
+ return addStep(core.skip(bindings, count));
+ }
+
+ public FromBuilder take(Core.Exp count) {
+ return addStep(core.take(bindings, count));
+ }
+
public FromBuilder group(SortedMap groupExps,
SortedMap aggregates) {
return addStep(core.group(groupExps, aggregates));
@@ -378,6 +392,14 @@ private class StepHandler extends Visitor {
where(where.exp);
}
+ @Override protected void visit(Core.Skip skip) {
+ skip(skip.exp);
+ }
+
+ @Override protected void visit(Core.Take take) {
+ take(take.exp);
+ }
+
@Override protected void visit(Core.Yield yield) {
yield_(false, yield.bindings, yield.exp);
}
diff --git a/src/main/java/net/hydromatic/morel/ast/Op.java b/src/main/java/net/hydromatic/morel/ast/Op.java
index 033e29e5..03bddabb 100644
--- a/src/main/java/net/hydromatic/morel/ast/Op.java
+++ b/src/main/java/net/hydromatic/morel/ast/Op.java
@@ -137,6 +137,8 @@ public enum Op {
COMPUTE,
ORDER,
ORDER_ITEM,
+ SKIP,
+ TAKE,
YIELD,
AGGREGATE,
IF;
diff --git a/src/main/java/net/hydromatic/morel/ast/Shuttle.java b/src/main/java/net/hydromatic/morel/ast/Shuttle.java
index 470a2436..4182d128 100644
--- a/src/main/java/net/hydromatic/morel/ast/Shuttle.java
+++ b/src/main/java/net/hydromatic/morel/ast/Shuttle.java
@@ -240,6 +240,14 @@ protected AstNode visit(Ast.Where where) {
return ast.where(where.pos, where.exp.accept(this));
}
+ protected AstNode visit(Ast.Skip skip) {
+ return ast.skip(skip.pos, skip.exp.accept(this));
+ }
+
+ protected AstNode visit(Ast.Take take) {
+ return ast.take(take.pos, take.exp.accept(this));
+ }
+
protected AstNode visit(Ast.Yield yield) {
return ast.yield(yield.pos, yield.exp.accept(this));
}
@@ -395,6 +403,14 @@ protected Core.Where visit(Core.Where where) {
return where.copy(where.exp.accept(this), where.bindings);
}
+ protected Core.Skip visit(Core.Skip skip) {
+ return skip.copy(skip.exp.accept(this), skip.bindings);
+ }
+
+ protected Core.Take visit(Core.Take take) {
+ return take.copy(take.exp.accept(this), take.bindings);
+ }
+
protected Core.Group visit(Core.Group group) {
return group.copy(visitSortedMap(group.groupExps),
visitSortedMap(group.aggregates));
diff --git a/src/main/java/net/hydromatic/morel/ast/Visitor.java b/src/main/java/net/hydromatic/morel/ast/Visitor.java
index 5a2fbd39..f12e9136 100644
--- a/src/main/java/net/hydromatic/morel/ast/Visitor.java
+++ b/src/main/java/net/hydromatic/morel/ast/Visitor.java
@@ -206,6 +206,14 @@ protected void visit(Ast.Where where) {
where.exp.accept(this);
}
+ protected void visit(Ast.Skip skip) {
+ skip.exp.accept(this);
+ }
+
+ protected void visit(Ast.Take take) {
+ take.exp.accept(this);
+ }
+
protected void visit(Ast.Yield yield) {
yield.exp.accept(this);
}
@@ -326,6 +334,14 @@ protected void visit(Core.Where where) {
where.exp.accept(this);
}
+ protected void visit(Core.Skip skip) {
+ skip.exp.accept(this);
+ }
+
+ protected void visit(Core.Take take) {
+ take.exp.accept(this);
+ }
+
protected void visit(Core.NonRecValDecl valDecl) {
valDecl.pat.accept(this);
valDecl.exp.accept(this);
diff --git a/src/main/java/net/hydromatic/morel/compile/CalciteCompiler.java b/src/main/java/net/hydromatic/morel/compile/CalciteCompiler.java
index 6a30e9b3..838a6486 100644
--- a/src/main/java/net/hydromatic/morel/compile/CalciteCompiler.java
+++ b/src/main/java/net/hydromatic/morel/compile/CalciteCompiler.java
@@ -260,7 +260,7 @@ Code toRel4(Environment env, Code code, Type type) {
} else {
for (Core.Exp arg : args) {
cx.relBuilder.values(new String[]{"T"}, true);
- yield_(cx, arg);
+ yield_(cx, ImmutableList.of(), arg);
}
cx.relBuilder.union(true, args.size());
}
@@ -303,9 +303,9 @@ Code toRel4(Environment env, Code code, Type type) {
return false;
}
final JsonBuilder jsonBuilder = new JsonBuilder();
+ final RelJson relJson = RelJson.create().withJsonBuilder(jsonBuilder);
final String jsonRowType =
- jsonBuilder.toJsonString(
- new RelJson(jsonBuilder).toJson(rowType));
+ jsonBuilder.toJsonString(relJson.toJson(rowType));
final String morelCode = apply.toString();
ThreadLocals.let(CalciteFunctions.THREAD_ENV,
new CalciteFunctions.Context(new Session(ImmutableMap.of()), cx.env,
@@ -369,7 +369,7 @@ private static void harmonizeRowTypes(RelBuilder relBuilder, int inputCount) {
}
}
- @Override protected RelCode compileFrom(Context cx, Core.From from) {
+ @Override protected Code compileFrom(Context cx, Core.From from) {
final Code code = super.compileFrom(cx, from);
return new RelCode() {
@Override public Describer describe(Describer describer) {
@@ -400,7 +400,7 @@ private static void harmonizeRowTypes(RelBuilder relBuilder, int inputCount) {
|| getLast(from.steps).op != Op.YIELD) {
final Core.Exp implicitYieldExp =
core.implicitYieldExp(typeSystem, from.steps);
- cx = yield_(cx, implicitYieldExp);
+ cx = yield_(cx, ImmutableList.of(), implicitYieldExp);
}
return true;
}
@@ -411,6 +411,10 @@ private RelContext step(RelContext cx, int i, Core.FromStep fromStep) {
return join(cx, i, (Core.Scan) fromStep);
case WHERE:
return where(cx, (Core.Where) fromStep);
+ case SKIP:
+ return skip(cx, (Core.Skip) fromStep);
+ case TAKE:
+ return take(cx, (Core.Take) fromStep);
case ORDER:
return order(cx, (Core.Order) fromStep);
case GROUP:
@@ -425,25 +429,28 @@ private RelContext step(RelContext cx, int i, Core.FromStep fromStep) {
}
private RelContext yield_(RelContext cx, Core.Yield yield) {
- return yield_(cx, yield.exp);
+ return yield_(cx, yield.bindings, yield.exp);
}
- private RelContext yield_(RelContext cx, Core.Exp exp) {
+ private RelContext yield_(RelContext cx, List bindings,
+ Core.Exp exp) {
final Core.Tuple tuple;
switch (exp.op) {
case ID:
final Core.Id id = (Core.Id) exp;
tuple = toRecord(cx, id);
if (tuple != null) {
- return yield_(cx, tuple);
+ return yield_(cx, bindings, tuple);
}
break;
case TUPLE:
tuple = (Core.Tuple) exp;
+ final List names =
+ ImmutableList.copyOf(tuple.type().argNameTypes().keySet());
cx.relBuilder.project(transform(tuple.args, e -> translate(cx, e)),
- ImmutableList.copyOf(tuple.type().argNameTypes().keySet()));
- return cx;
+ names);
+ return getRelContext(cx, cx.env.bindAll(bindings), names);
}
RexNode rex = translate(cx, exp);
cx.relBuilder.project(rex);
@@ -489,7 +496,7 @@ record = toRecord(cx, id);
if (cx.map.containsKey(id.idPat.name)) {
// Not a record, so must be a scalar. It is represented in Calcite
// as a record with one field.
- final VarData fn = cx.map.get(id.idPat.name);
+ final VarData fn = requireNonNull(cx.map.get(id.idPat.name));
return fn.apply(cx.relBuilder);
}
break;
@@ -611,9 +618,9 @@ private RexNode morelScalar(RelContext cx, Core.Exp exp) {
final RelDataType calciteType =
Converters.toCalciteType(exp.type, typeFactory);
final JsonBuilder jsonBuilder = new JsonBuilder();
+ final RelJson relJson = RelJson.create().withJsonBuilder(jsonBuilder);
final String jsonType =
- jsonBuilder.toJsonString(
- new RelJson(jsonBuilder).toJson(calciteType));
+ jsonBuilder.toJsonString(relJson.toJson(calciteType));
final String morelCode = exp.toString();
return cx.relBuilder.getRexBuilder().makeCall(calciteType,
CalciteFunctions.SCALAR_OPERATOR,
@@ -695,6 +702,26 @@ private RelContext where(RelContext cx, Core.Where where) {
return cx;
}
+ private RelContext skip(RelContext cx, Core.Skip skip) {
+ if (skip.exp.op != Op.INT_LITERAL) {
+ throw new AssertionError("skip requires literal: " + skip.exp);
+ }
+ int offset = ((Core.Literal) skip.exp).unwrap(Integer.class);
+ int fetch = -1; // per Calcite: "negative means no limit"
+ cx.relBuilder.limit(offset, fetch);
+ return cx;
+ }
+
+ private RelContext take(RelContext cx, Core.Take take) {
+ if (take.exp.op != Op.INT_LITERAL) {
+ throw new AssertionError("take requires literal: " + take.exp);
+ }
+ int offset = 0;
+ int fetch = ((Core.Literal) take.exp).unwrap(Integer.class);
+ cx.relBuilder.limit(offset, fetch);
+ return cx;
+ }
+
private RelContext order(RelContext cx, Core.Order order) {
final List exps = new ArrayList<>();
order.orderItems.forEach(i -> {
@@ -709,7 +736,6 @@ private RelContext order(RelContext cx, Core.Order order) {
}
private RelContext group(RelContext cx, Core.Group group) {
- final SortedMap map = new TreeMap<>();
final List bindings = new ArrayList<>();
final List nodes = new ArrayList<>();
final List names = new ArrayList<>();
@@ -734,20 +760,24 @@ private RelContext group(RelContext cx, Core.Group group) {
// Create an Aggregate operator.
cx.relBuilder.aggregate(groupKey, aggregateCalls);
+ return getRelContext(cx, cx.env.bindAll(bindings), names);
+ }
+ private static RelContext getRelContext(RelContext cx, Environment env,
+ List names) {
// Permute the fields so that they are sorted by name, per Morel records.
final List sortedNames =
Ordering.natural().immutableSortedCopy(names);
cx.relBuilder.rename(names)
.project(cx.relBuilder.fields(sortedNames));
final RelDataType rowType = cx.relBuilder.peek().getRowType();
- sortedNames.forEach(name -> {
- final int i = map.size();
- map.put(name, new VarData(PrimitiveType.UNIT, i, rowType));
- });
+ final SortedMap map = new TreeMap<>();
+ sortedNames.forEach(name ->
+ map.put(name,
+ new VarData(PrimitiveType.UNIT, map.size(), rowType)));
// Return a context containing a variable for each output field.
- return new RelContext(cx.env.bindAll(bindings), cx, cx.relBuilder,
+ return new RelContext(env, cx, cx.relBuilder,
ImmutableSortedMap.copyOfSorted(map), 1);
}
diff --git a/src/main/java/net/hydromatic/morel/compile/Compiler.java b/src/main/java/net/hydromatic/morel/compile/Compiler.java
index 4dc633f5..d7f4d91c 100644
--- a/src/main/java/net/hydromatic/morel/compile/Compiler.java
+++ b/src/main/java/net/hydromatic/morel/compile/Compiler.java
@@ -151,7 +151,7 @@ void apply(Consumer outLines, Consumer outBindings,
}
/** Compilation context. */
- static class Context {
+ public static class Context {
final Environment env;
Context(Environment env) {
@@ -388,6 +388,16 @@ && getOnlyElement(bindings).id.type.equals(elementType)) {
final Code filterCode = compile(cx, where.exp);
return () -> Codes.whereRowSink(filterCode, nextFactory.get());
+ case SKIP:
+ final Core.Skip skip = (Core.Skip) firstStep;
+ final Code skipCode = compile(cx, skip.exp);
+ return () -> Codes.skipRowSink(skipCode, nextFactory.get());
+
+ case TAKE:
+ final Core.Take take = (Core.Take) firstStep;
+ final Code takeCode = compile(cx, take.exp);
+ return () -> Codes.takeRowSink(takeCode, nextFactory.get());
+
case YIELD:
final Core.Yield yield = (Core.Yield) firstStep;
if (steps.size() == 1) {
diff --git a/src/main/java/net/hydromatic/morel/compile/Resolver.java b/src/main/java/net/hydromatic/morel/compile/Resolver.java
index 2318d776..5dff9118 100644
--- a/src/main/java/net/hydromatic/morel/compile/Resolver.java
+++ b/src/main/java/net/hydromatic/morel/compile/Resolver.java
@@ -919,6 +919,16 @@ Core.Exp run(Ast.From from) {
fromBuilder.where(r.toCore(where.exp));
}
+ @Override protected void visit(Ast.Skip skip) {
+ final Resolver r = withEnv(env); // do not use 'from' bindings
+ fromBuilder.skip(r.toCore(skip.exp));
+ }
+
+ @Override protected void visit(Ast.Take take) {
+ final Resolver r = withEnv(env); // do not use 'from' bindings
+ fromBuilder.take(r.toCore(take.exp));
+ }
+
@Override protected void visit(Ast.Yield yield) {
final Resolver r = withEnv(fromBuilder.bindings());
fromBuilder.yield_(r.toCore(yield.exp));
diff --git a/src/main/java/net/hydromatic/morel/compile/TypeResolver.java b/src/main/java/net/hydromatic/morel/compile/TypeResolver.java
index 3180d370..cd70681a 100644
--- a/src/main/java/net/hydromatic/morel/compile/TypeResolver.java
+++ b/src/main/java/net/hydromatic/morel/compile/TypeResolver.java
@@ -546,6 +546,22 @@ private Pair deduceStepType(TypeEnv env,
fromSteps.add(where.copy(filter2));
return Pair.of(env2, v);
+ case SKIP:
+ final Ast.Skip skip = (Ast.Skip) step;
+ final Unifier.Variable v11 = unifier.variable();
+ final Ast.Exp skipCount = deduceType(env2, skip.exp, v11);
+ equiv(v11, toTerm(PrimitiveType.INT));
+ fromSteps.add(skip.copy(skipCount));
+ return Pair.of(env2, v);
+
+ case TAKE:
+ final Ast.Take take = (Ast.Take) step;
+ final Unifier.Variable v12 = unifier.variable();
+ final Ast.Exp takeCount = deduceType(env2, take.exp, v12);
+ equiv(v12, toTerm(PrimitiveType.INT));
+ fromSteps.add(take.copy(takeCount));
+ return Pair.of(env2, v);
+
case YIELD:
final Ast.Yield yield = (Ast.Yield) step;
final Unifier.Variable v6 = unifier.variable();
diff --git a/src/main/java/net/hydromatic/morel/eval/Codes.java b/src/main/java/net/hydromatic/morel/eval/Codes.java
index 7533d3d8..1a65ebef 100644
--- a/src/main/java/net/hydromatic/morel/eval/Codes.java
+++ b/src/main/java/net/hydromatic/morel/eval/Codes.java
@@ -507,6 +507,7 @@ public static Code from(Supplier rowSinkFactory) {
@Override public Object eval(EvalEnv env) {
final RowSink rowSink = rowSinkFactory.get();
+ rowSink.start(env);
rowSink.accept(env);
return rowSink.result(env);
}
@@ -524,6 +525,16 @@ public static RowSink whereRowSink(Code filterCode, RowSink rowSink) {
return new WhereRowSink(filterCode, rowSink);
}
+ /** Creates a {@link RowSink} for a {@code skip} clause. */
+ public static RowSink skipRowSink(Code filterCode, RowSink rowSink) {
+ return new SkipRowSink(filterCode, rowSink);
+ }
+
+ /** Creates a {@link RowSink} for a {@code take} clause. */
+ public static RowSink takeRowSink(Code filterCode, RowSink rowSink) {
+ return new TakeRowSink(filterCode, rowSink);
+ }
+
/** Creates a {@link RowSink} for a {@code order} clause. */
public static RowSink orderRowSink(
Iterable extends Map.Entry> codes,
@@ -3042,26 +3053,47 @@ public Object eval(EvalEnv env) {
/** Accepts rows produced by a supplier as part of a {@code from} clause. */
public interface RowSink extends Describable {
+ void start(EvalEnv env);
void accept(EvalEnv env);
List