Merge branch 'main' into add-categories-to-rascal-grammar

pom.xml

@@ -29,6 +29,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <lucence-version>7.5.0</lucence-version>
+        <maven-version>3.9.8</maven-version>
         <exec.mainClass>org.rascalmpl.shell.RascalShell</exec.mainClass>
         <rascal.test.memory>2</rascal.test.memory>
         <maven.compiler.release>11</maven.compiler.release>
@@ -316,10 +317,15 @@
                                 </transformer>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                     <resource>org/rascalmpl/uri/resolvers.config</resource>
-                                    <resource>io/usethesource/vallang/type/types.config </resource>
                                 </transformer>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                    <resource>io/usethesource/vallang/type/types.config</resource>
+                                </transformer>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                    <resource>META-INF/sisu/javax.inject.Named</resource> <!-- Needed for dependency injection in MavenCli -->
                                 </transformer>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ComponentsXmlResourceTransformer"/> <!-- Needed for dependency injection in MavenCli -->
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
                                     <addHeader>false</addHeader>
                                 </transformer>
@@ -514,5 +520,30 @@
             <artifactId>icu4j</artifactId>
             <version>74.2</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.maven</groupId>
+            <artifactId>maven-embedder</artifactId>
+            <version>${maven-version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <version>3.7.1</version>
+        </dependency>
+        <dependency> <!-- needed for maven-embedder-->
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-nop</artifactId>
+            <version>1.7.36</version>
+        </dependency>
+        <dependency> <!-- needed for maven-embedder-->
+            <groupId>org.apache.maven</groupId>
+            <artifactId>maven-compat</artifactId>
+            <version>${maven-version}</version>
+        </dependency>
+        <dependency> <!-- needed for maven-embedder-->
+            <groupId>org.apache.maven</groupId>
+            <artifactId>maven-core</artifactId>
+            <version>${maven-version}</version>
+        </dependency>
     </dependencies>
 </project>

src/org/rascalmpl/exceptions/RascalStackOverflowError.java

@@ -0,0 +1,49 @@
+/*******************************************************************************
+ * Copyright (c) 2024 CWI
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+
+ *   * Jurgen J. Vinju - [email protected] - CWI
+*******************************************************************************/
+package org.rascalmpl.exceptions;
+
+import org.rascalmpl.ast.AbstractAST;
+import org.rascalmpl.interpreter.env.Environment;
+
+/**
+ * This class captures the runtime state of the interpreter at the moment
+ * we detect a java StackOverflowError. Since we can not use any stack depth
+ * at that moment to create a real Throw exception, we only copy the deepest
+ * environment and wrap it in here. Later on the level of the REPL, when the
+ * stack is completely unrolled, we can convert the stack trace to a Rascal trace.
+ */
+public class RascalStackOverflowError extends RuntimeException {
+    private static final long serialVersionUID = -3947588548271683963L;
+    private final Environment deepestEnvironment;
+    private final AbstractAST currentAST;
+
+    public RascalStackOverflowError(AbstractAST current, Environment deepest) {
+        this.deepestEnvironment = deepest;
+        this.currentAST = current;
+    }
+
+    public Throw makeThrow() {
+        StackTrace trace = new StackTrace();
+        Environment env = deepestEnvironment;
+
+        while (env != null) {
+            trace.add(env.getLocation(), env.getName());
+            env = env.getCallerScope();
+        }
+
+        return RuntimeExceptionFactory.stackOverflow(currentAST, trace);
+    }
+
+    public Environment getEnvironment() {
+        return deepestEnvironment;
+    }
+}

src/org/rascalmpl/library/Content.rsc

@@ -80,7 +80,7 @@ which involves a handy, automatic, encoding of Rascal values into json values.
 data Response 
   = response(Status status, str mimeType, map[str,str] header, str content)
   | fileResponse(loc file, str mimeType, map[str,str] header)
-  | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
+  | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", bool explicitConstructorNames=false, bool explicitDataTypes=false)
   ;
 
 

src/org/rascalmpl/library/String.rsc

@@ -20,6 +20,7 @@ module String
 
 extend Exception;
 import List;
+import ParseTree;
 
 @synopsis{All functions in this module that have a charset parameter use this as default.}
 private str DEFAULT_CHARSET = "UTF-8";
@@ -519,11 +520,46 @@ for the allowed syntax in `charSet`.
 ```rascal-shell
 import String;
 squeeze("hello", "el");
+// the other squeeze function uses character class types instead:
+squeeze("hello", "el") == squeeze("hello", #[el]);
 ```
 }
 @javaClass{org.rascalmpl.library.Prelude}
+@deprecated{Use the other squeence function that accepts Rascal character class syntax.}
 public java str squeeze(str src, str charSet);
 
+@synopsis{Squeeze repeated occurrences of characters.}
+@description{
+Squeeze repeated occurrences in `src` of characters, if they are a member of `&CharClass`, removed.
+
+* `src` is any string
+* `&CharClass` is a reified character class type such as `[a-z]` (a type that is a subtype of the class of all characters `![]`)
+* To pass in a char-class type used the type reifier operator: `#[a-z]` or `#![]`
+}
+@benefits{
+* to squeeze all characters use the universal character class: `#![]` (the negation of the empty class).
+* this function is type-safe; you can only pass in correct reified character classes like `#[A-Za-z]`.
+}
+@pitfalls{
+* `![]` excludes the 0'th unicode character, so we can not squeeze the unicode codepoint `0` using this function. 
+If you really need to squeeze 0 then it's best to write your own:
+```rascal
+visit (x) { 
+  case /<dot:.>+/ => "\a00" when dot == "\a00" 
+}
+````
+* Do not confuse the character `0` (codepoint 48) with the zero codepoint: `#[0] != #[\a00]`
+}
+@examples{
+```rascal-shell
+import String;
+squeeze("hello", #[el]);
+```
+}
+public str squeeze(str src, type[&CharClass] _:type[![]] _) = visit(src) {
+    case /<c:.><c>+/ => c
+      when &CharClass _ := Tree::char(charAt(c, 0))
+};
 
 
 @synopsis{Split a string into a list of strings based on a literal separator.}

src/org/rascalmpl/library/lang/json/IO.java

@@ -100,14 +100,16 @@ public IValue fromJSON(IValue type, IString src) {
     }
 
 
-	public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins) {
+	public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
       TypeStore store = new TypeStore();
       Type start = new TypeReifier(values).valueToType((IConstructor) type, store);
 
       try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) {
 		in.setLenient(lenient.getValue());
         return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null)
             .setCalendarFormat(dateTimeFormat.getValue())
+			.setExplicitConstructorNames(explicitConstructorNames.getValue())
+			.setExplicitDataTypes(explicitDataTypes.getValue())
             .read(in, start);
       }
       catch (IOException e) {
@@ -119,14 +121,16 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat,
       }
     }
 
-	public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins) {
+	public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
 	      TypeStore store = new TypeStore();
 	      Type start = new TypeReifier(values).valueToType((IConstructor) type, store);
 
 	      try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) {
 			in.setLenient(lenient.getValue());
 	        return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null)
 	            .setCalendarFormat(dateTimeFormat.getValue())
+				.setExplicitConstructorNames(explicitConstructorNames.getValue())
+				.setExplicitDataTypes(explicitDataTypes.getValue())
 	            .read(in, start);
 	      }
 	      catch (IOException e) {
@@ -137,7 +141,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool
 	      }
 	    }
 
-	public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) {
+	public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
 	    try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) {
 	        if (indent.intValue() > 0) {
 	            out.setIndent("        ".substring(0, indent.intValue() % 9));
@@ -148,13 +152,15 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations
 	        .setDatesAsInt(dateTimeAsInt.getValue())
 	        .setUnpackedLocations(unpackedLocations.getValue())
 			.setDropOrigins(dropOrigins.getValue())
+			.setExplicitConstructorNames(explicitConstructorNames.getValue())
+			.setExplicitDataTypes(explicitDataTypes.getValue())
 	        .write(out, value);
 	    } catch (IOException e) {
 	        throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null);
 	    } 
 	}
 
-	public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) {
+	public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
 	    StringWriter string = new StringWriter();
 
 	    try (JsonWriter out = new JsonWriter(string)) {
@@ -166,14 +172,13 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor
 	        	.setDatesAsInt(dateTimeAsInt.getValue())
 	        	.setUnpackedLocations(unpackedLocations.getValue())
 				.setDropOrigins(dropOrigins.getValue())
+				.setExplicitConstructorNames(explicitConstructorNames.getValue())
+				.setExplicitDataTypes(explicitDataTypes.getValue())
 	        	.write(out, value);
 
 	        return values.string(string.toString());
 	    } catch (IOException e) {
 	        throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null);
 	    } 
 	}
-
-
-
 }

src/org/rascalmpl/library/lang/json/IO.rsc

@@ -15,25 +15,23 @@
 module lang::json::IO
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
-@deprecated{
-Use writeJSON
-}
+@synopsis{Maps any Rascal value to a JSON string}
+@deprecated{use ((writeJSON))}
 public java str toJSON(value v);
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
-@deprecated{
-Use asJSON
-}
+@synopsis{Maps any Rascal value to a JSON string, optionally in compact form.}
+@deprecated{use ((asJSON))}
 public java str toJSON(value v, bool compact);
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
-@deprecated{
-Use readJSON
-}
+@deprecated{use ((readJSON))}
+@synopsis{Parses a JSON string and maps it to the requested type of Rascal value.}
 public java &T fromJSON(type[&T] typ, str src);
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
-@synopsis{reads JSON values from a stream
+@synopsis{reads JSON values from a stream}
+@description{
 In general the translation behaves as follows:
  * Objects translate to map[str,value] by default, unless a node is expected (properties are then translated to keyword fields)
  * Arrays translate to lists by default, or to a set if that is expected or a tuple if that is expected. Arrays may also be interpreted as constructors or nodes (see below)
@@ -44,20 +42,40 @@ In general the translation behaves as follows:
  * If num, int, real or rat are expected both strings and number values are mapped
  * If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of
    a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]}}
-java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false);
+java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false);
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
 @synopsis{parses JSON values from a string
 In general the translation behaves as the same as for ((readJSON)).}
-java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false);
+java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false);
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
-@synopsis{writes `val` to the location `target`}
+@synopsis{Serializes a value as a JSON string and stream it}
 @description{
-  If `dateTimeAsInt` is set to `true`, the dateTime values are converted to an int that represents the number of milliseconds from 1970-01-01T00:00Z.
-  If `indent` is set to a number greater than 0, the JSON file will be formatted with `indent` number of spaces as indentation.
+This function tries to map Rascal values to JSON values in a natural way.
+In particular it tries to create a value that has the same number of recursive levels,
+such that one constructor maps to one object. The serialization is typically _lossy_ since
+JSON values by default do not explicitly encode the class or constructor while Rascal data types do.
+
+If you need the names of constructors or data-types in your result, then use the parameters:
+* `explicitConstructorNames=true` will store the name of every constructor in a field `_constructor`
+* `explicitDataTypes=true` will store the name of the ADT in a field called `_type`
+
+The `dateTimeFormat` parameter dictates how `datetime` values will be printed.
+
+The `unpackedLocations` parameter will produce an object with many fields for every property of a `loc` value, but
+if set to false a `loc` will be printed as a string.
+}
+@pitfalls{
+* It is understood that Rascal's number types have arbitrary precision, but this is not supported by the JSON writer.
+As such when an `int` is printed that does not fit into a JVM `long`, there will be truncation to the lower 64 bits.
+For `real` numbers that are larger than JVM's double you get "negative infinity" or "positive infinity" as a result.
 }
-java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true);
+java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false);
 
 @javaClass{org.rascalmpl.library.lang.json.IO}
-java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true);
+@synopsis{Serializes a value as a JSON string and stores it as a string}
+@description{
+This function uses `writeJSON` and stores the result in a string.
+}
+java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false);