diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index 54600a5021f..464afc534d5 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -80,7 +80,7 @@ which involves a handy, automatic, encoding of Rascal values into json values. data Response = response(Status status, str mimeType, map[str,str] header, str content) | fileResponse(loc file, str mimeType, map[str,str] header) - | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", bool explicitConstructorNames=false, bool explicitDataTypes=false) ; diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 6cecd64debc..fcebd8fabbb 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -100,7 +100,7 @@ public IValue fromJSON(IValue type, IString src) { } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins) { + public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -108,6 +108,8 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null) .setCalendarFormat(dateTimeFormat.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .read(in, start); } catch (IOException e) { @@ -119,7 +121,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, } } - public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins) { + public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -127,6 +129,8 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null) .setCalendarFormat(dateTimeFormat.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .read(in, start); } catch (IOException e) { @@ -137,7 +141,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool } } - public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) { + public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) { if (indent.intValue() > 0) { out.setIndent(" ".substring(0, indent.intValue() % 9)); @@ -148,13 +152,15 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .write(out, value); } catch (IOException e) { throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null); } } - public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) { + public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { StringWriter string = new StringWriter(); try (JsonWriter out = new JsonWriter(string)) { @@ -166,6 +172,8 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .write(out, value); return values.string(string.toString()); @@ -173,7 +181,4 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null); } } - - - } diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index f751745311d..bcf08efca99 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -15,25 +15,23 @@ module lang::json::IO @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{ -Use writeJSON -} +@synopsis{Maps any Rascal value to a JSON string} +@deprecated{use ((writeJSON))} public java str toJSON(value v); @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{ -Use asJSON -} +@synopsis{Maps any Rascal value to a JSON string, optionally in compact form.} +@deprecated{use ((asJSON))} public java str toJSON(value v, bool compact); @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{ -Use readJSON -} +@deprecated{use ((readJSON))} +@synopsis{Parses a JSON string and maps it to the requested type of Rascal value.} public java &T fromJSON(type[&T] typ, str src); @javaClass{org.rascalmpl.library.lang.json.IO} -@synopsis{reads JSON values from a stream +@synopsis{reads JSON values from a stream} +@description{ In general the translation behaves as follows: * Objects translate to map[str,value] by default, unless a node is expected (properties are then translated to keyword fields) * Arrays translate to lists by default, or to a set if that is expected or a tuple if that is expected. Arrays may also be interpreted as constructors or nodes (see below) @@ -44,20 +42,40 @@ In general the translation behaves as follows: * If num, int, real or rat are expected both strings and number values are mapped * If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]}} -java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); +java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{parses JSON values from a string In general the translation behaves as the same as for ((readJSON)).} -java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); +java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false); @javaClass{org.rascalmpl.library.lang.json.IO} -@synopsis{writes `val` to the location `target`} +@synopsis{Serializes a value as a JSON string and stream it} @description{ - If `dateTimeAsInt` is set to `true`, the dateTime values are converted to an int that represents the number of milliseconds from 1970-01-01T00:00Z. - If `indent` is set to a number greater than 0, the JSON file will be formatted with `indent` number of spaces as indentation. +This function tries to map Rascal values to JSON values in a natural way. +In particular it tries to create a value that has the same number of recursive levels, +such that one constructor maps to one object. The serialization is typically _lossy_ since +JSON values by default do not explicitly encode the class or constructor while Rascal data types do. + +If you need the names of constructors or data-types in your result, then use the parameters: +* `explicitConstructorNames=true` will store the name of every constructor in a field `_constructor` +* `explicitDataTypes=true` will store the name of the ADT in a field called `_type` + +The `dateTimeFormat` parameter dictates how `datetime` values will be printed. + +The `unpackedLocations` parameter will produce an object with many fields for every property of a `loc` value, but +if set to false a `loc` will be printed as a string. +} +@pitfalls{ +* It is understood that Rascal's number types have arbitrary precision, but this is not supported by the JSON writer. +As such when an `int` is printed that does not fit into a JVM `long`, there will be truncation to the lower 64 bits. +For `real` numbers that are larger than JVM's double you get "negative infinity" or "positive infinity" as a result. } -java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true); +java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); @javaClass{org.rascalmpl.library.lang.json.IO} -java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true); +@synopsis{Serializes a value as a JSON string and stores it as a string} +@description{ +This function uses `writeJSON` and stores the result in a string. +} +java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 23b64f81606..e43e658afa9 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.uri.URIUtil; import io.usethesource.vallang.IInteger; @@ -59,6 +58,8 @@ public class JsonValueReader { private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + private boolean explicitConstructorNames; + private boolean explicitDataTypes; /** * @param vf factory which will be used to construct values @@ -105,6 +106,20 @@ protected SimpleDateFormat initialValue() { return this; } + public JsonValueReader setExplicitConstructorNames(boolean value) { + this.explicitConstructorNames = value; + return this; + } + + public JsonValueReader setExplicitDataTypes(boolean value) { + this.explicitDataTypes = value; + if (value) { + this.explicitConstructorNames = true; + } + return this; + } + + /** * Read and validate a Json stream as an IValue * @param in json stream @@ -477,7 +492,6 @@ private int getCol() { } } - @Override public IValue visitAbstractData(Type type) throws IOException { if (in.peek() == JsonToken.STRING) { @@ -493,18 +507,75 @@ public IValue visitAbstractData(Type type) throws IOException { throw new IOException("no nullary constructor found for " + type); } - assert in.peek() == JsonToken.BEGIN_OBJECT; + Set alternatives = null; + + in.beginObject(); + int startPos = getPos(); + int startLine = getLine(); + int startCol = getCol(); + + // use explicit information in the JSON to select and filter constructors from the TypeStore + // we expect always to have the field _constructor before _type. + if (explicitConstructorNames || explicitDataTypes) { + String consName = null; + String typeName = null; // this one is optional, and the order with cons is not defined. + + String consLabel = in.nextName(); + + // first we read either a cons name or a type name + if (explicitConstructorNames && "_constructor".equals(consLabel)) { + consName = in.nextString(); + } + else if (explicitDataTypes && "_type".equals(consLabel)) { + typeName = in.nextString(); + } + + // optionally read the second field + if (explicitDataTypes && typeName == null) { + // we've read a constructor name, but we still need a type name + consLabel = in.nextName(); + if (explicitDataTypes && "_type".equals(consLabel)) { + typeName = in.nextString(); + } + } + else if (explicitDataTypes && consName == null) { + // we've read type name, but we still need a constructor name + consLabel = in.nextName(); + if (explicitDataTypes && "_constructor".equals(consLabel)) { + consName = in.nextString(); + } + } + + if (explicitDataTypes && typeName == null) { + throw new IOException("Missing a _type field: " + in.getPath()); + } + else if (explicitConstructorNames && consName == null) { + throw new IOException("Missing a _constructor field: " + in.getPath()); + } + + if (typeName != null && consName != null) { + // first focus on the given type name + var dataType = TF.abstractDataType(store, typeName); + alternatives = store.lookupConstructor(dataType, consName); + } + else { + // we only have a constructor name + // lookup over all data types by constructor name + alternatives = store.lookupConstructors(consName); + } + } + else { + alternatives = store.lookupAlternatives(type); + } - Set alternatives = store.lookupAlternatives(type); if (alternatives.size() > 1) { monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); } + else if (alternatives.size() == 0) { + throw new IOException("No fitting constructor found for " + in.getPath()); + } Type cons = alternatives.iterator().next(); - in.beginObject(); - int startPos = getPos(); - int startLine = getLine(); - int startCol = getCol(); IValue[] args = new IValue[cons.getArity()]; Map kwParams = new HashMap<>(); @@ -532,7 +603,20 @@ else if (cons.hasKeywordField(label, store)) { } } else { // its a normal arg, pass its label to the child - throw new IOException("Unknown field " + label + ":" + in.getPath()); + if (!explicitConstructorNames && "_constructor".equals(label)) { + // ignore additional _constructor fields. + in.nextString(); // skip the constructor value + continue; + } + else if (!explicitDataTypes && "_type".equals(label)) { + // ignore additional _type fields. + in.nextString(); // skip the type value + continue; + } + else { + // field label does not match data type definition + throw new IOException("Unknown field " + label + ":" + in.getPath()); + } } } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index c20838b972f..5e9cc93500b 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -44,6 +44,8 @@ public class JsonValueWriter { private boolean datesAsInts = true; private boolean unpackedLocations = false; private boolean dropOrigins = true; + private boolean explicitConstructorNames = false; + private boolean explicitDataTypes; public JsonValueWriter() { setCalendarFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); @@ -78,6 +80,16 @@ public JsonValueWriter setDropOrigins(boolean setting) { return this; } + public JsonValueWriter setExplicitConstructorNames(boolean setting) { + this.explicitConstructorNames = setting; + return this; + } + + public JsonValueWriter setExplicitDataTypes(boolean setting) { + this.explicitDataTypes = setting; + return this; + } + public void write(JsonWriter out, IValue value) throws IOException { value.accept(new IValueVisitor() { @@ -222,13 +234,24 @@ public Void visitNode(INode o) throws IOException { @Override public Void visitConstructor(IConstructor o) throws IOException { - if (o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { + if (!explicitConstructorNames && !explicitDataTypes && o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { // enums! out.value(o.getName()); return null; } out.beginObject(); + + if (explicitConstructorNames || explicitDataTypes) { + out.name("_constructor"); + out.value(o.getName()); + } + + if (explicitDataTypes) { + out.name("_type"); + out.value(o.getType().getName()); + } + int i = 0; for (IValue arg : o) { out.name(o.getConstructorType().getFieldName(i)); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index ef8e837ee0f..0fd17e4119f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -69,4 +69,37 @@ test bool originTracking() { } return true; +} + +test bool explicitConstructorNames() { + example = data4(e=z()); + json = asJSON(example, explicitConstructorNames=true); + + assert json == "{\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\"}}"; + + assert parseJSON(#DATA4, json, explicitConstructorNames=true) == example; + + // here we can't be sure to get z() back, but we will get some Enum + assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitConstructorNames=false); + + return true; +} + +test bool explicitDataTypes() { + example = data4(e=z()); + json = asJSON(example, explicitDataTypes=true); + + assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; + + // _constructor and _type must be the first fields + assert parseJSON(#DATA4, json, explicitDataTypes=true) == example; + + // _type and _constructor may appear in a different order + flippedJson = "{\"_type\":\"DATA4\",\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; + assert parseJSON(#DATA4, flippedJson, explicitDataTypes=true) == example; + + // here we can't be sure to get z() back, but we will get some Enum + assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); + + return true; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index f149bb36340..8ed61b9d41e 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -238,10 +238,15 @@ private void handleJSONResponse(Map output, IConstructor re IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); - + IValue ecn = kws.getParameter("explicitConstructorNames"); + IValue edt = kws.getParameter("explicitDataTypes"); + JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/org/rascalmpl/library/util/Webserver.java b/src/org/rascalmpl/library/util/Webserver.java index 292241ed33a..d6c80fe1a7d 100644 --- a/src/org/rascalmpl/library/util/Webserver.java +++ b/src/org/rascalmpl/library/util/Webserver.java @@ -238,10 +238,15 @@ private Response translateJsonResponse(Method method, IConstructor cons) { IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue ecn = kws.getParameter("explicitConstructorNames"); + IValue edt = kws.getParameter("explicitDataTypes"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; try { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/org/rascalmpl/repl/REPLContentServer.java b/src/org/rascalmpl/repl/REPLContentServer.java index de811bcb6ef..becc0a80871 100644 --- a/src/org/rascalmpl/repl/REPLContentServer.java +++ b/src/org/rascalmpl/repl/REPLContentServer.java @@ -135,10 +135,15 @@ private static Response translateJsonResponse(Method method, IConstructor cons) IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue ecn = kws.getParameter("explicitConstructorNames"); + IValue edt = kws.getParameter("explicitDataTypes"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; try { final ByteArrayOutputStream baos = new ByteArrayOutputStream();