From 0c8bc494d50753fd627961fa0d438ae530e309ba Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 24 Sep 2024 11:51:37 +0200 Subject: [PATCH 01/20] Started working on decent test support --- .../recovery/PicoMassRecoveryTests.rsc | 28 +++++ .../recovery/RascalMassRecoveryTests.rsc | 15 +++ .../concrete/recovery/RecoveryTestSupport.rsc | 100 ++++++++++++++++++ .../parser/uptr/UPTRNodeFactory.java | 5 +- 4 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc new file mode 100644 index 00000000000..339d69df267 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc @@ -0,0 +1,28 @@ +module lang::rascal::tests::concrete::recovery::PicoMassRecoveryTests + +import lang::pico::\syntax::Main; +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +str picoFac = "begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 1; + % check this out % + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end"; + +bool testFacDeletions() { + TestStats stats = testSingleCharDeletions(#Program, picoFac); + printStats(stats); + return true; +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc new file mode 100644 index 00000000000..c5d220efb31 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc @@ -0,0 +1,15 @@ +module lang::rascal::tests::concrete::recovery::RascalMassRecoveryTests + +import lang::rascal::\syntax::Rascal; +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import IO; +import ParseTree; + +start[Module] (value input, loc origin) rascalParser = parser(#start[Module], allowAmbiguity=true, allowRecovery=true); +str source = readFile(|std:///lang/rascal/vis/ImportGraph.rsc|); + +bool testRascalDeletions() { + TestStats stats = testSingleCharDeletions(rascalParser, source); + printStats(stats); + return true; +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc new file mode 100644 index 00000000000..57377b3f312 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -0,0 +1,100 @@ +module lang::rascal::tests::concrete::recovery::RecoveryTestSupport + +import ParseTree; +import String; +import IO; +import util::Benchmark; + +public data TestStats = testStats(int totalAttempts=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, list[tuple[loc,int]] slowParses = []); + +private TestStats testRecovery(&T (value input, loc origin) parser, str input, loc source) { + TestStats stats = testStats(totalAttempts=1); + + int begin = realTime(); + try { + Tree t = parser(input, source); + if (hasErrors(t)) { + stats.successfulRecoveries = 1; + print("+"); + + //errors = findAllErrors(t); + //count = size(errors); + //best = findBestError(t); + //println("successful recovery, errors found, best: , loc: "); + //for (err <- errors) { + // println(""); + //} + } else { + stats.successfulParses = 1; + print("."); + } + } catch ParseError(_): { + stats.parseErrors = 1; + print("?"); + } + int duration = realTime() - begin; + + slowParse = []; + if (duration > 200) { + stats.slowParses = []; + print("!"); + } + + return stats; +} + +TestStats testSingleCharDeletions(type[&T] grammar, str input) = testSingleCharDeletions(parser(grammar, allowAmbiguity=true, allowRecovery=true), input); + +TestStats testSingleCharDeletions(&T (value input, loc origin) parser, str input) { + TestStats totalStats = testStats(); + int len = size(input); + int i = 0; + + while (i < len) { + str modifiedInput = substring(input, 0, i) + substring(input, i+1); + TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deleted=<"">|); + totalStats = mergeStats(totalStats, singleRunStats); + i = i+1; + } + + return totalStats; +} + +TestStats mergeStats(TestStats stats1, TestStats stats2) { + TestStats result = stats1; + result.totalAttempts += stats2.totalAttempts; + result.successfulParses += stats2.successfulParses; + result.successfulRecoveries += stats2.successfulRecoveries; + result.failedRecoveries += stats2.failedRecoveries; + result.parseErrors += stats2.parseErrors; + result.slowParses += stats2.slowParses; + return result; +} + +private int percentage(int number, int total) { + return (100*number)/total; +} + +void printStats(TestStats stats) { + println(); + println("Total parses: "); + println("Succesful parses: ( % of total)"); + int totalFailed = stats.totalAttempts - stats.successfulParses; + println("Succesful recoveries: ( % of failed)"); + println("Failed recoveries: ( % of failed)"); + println("Parse errors: ( % of failed)"); + + if (stats.slowParses == []) { + println("No slow parses."); + } else { + println(" slow parses:"); + for ( <- stats.slowParses) { + println(": ms."); + } + } +} + +void testRecovery(type[&T] grammar, loc input) { + TestStats stats = testSingleCharDeletions(grammar, readFile(input)); + printStats(stats); +} \ No newline at end of file diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 3b5e8a84c68..34ff966fccd 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -145,8 +145,9 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createSkippedNode(int[] characters) { - IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); - return VF.appl(SKIPPED, chars); + //IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); + //return VF.appl(SKIPPED, VF.list(charValues)); + return createLiteralNode(characters, SKIPPED); } public ITree createErrorNode(ArrayList children, Object production) { From 89d07fe19d6e2019c13dfdcbf95e6aa139685523 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 24 Sep 2024 15:39:16 +0200 Subject: [PATCH 02/20] Added Java and C recovery tests --- .../library/lang/c90/examples/hello-world.c | 19 +++++++++++ .../library/lang/pico/examples/fac.pico | 18 ++++++++++ .../recovery/LanguageRecoveryTests.rsc | 10 ++++++ .../concrete/recovery/RecoveryTestSupport.rsc | 33 +++++++++++++++++-- 4 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/c90/examples/hello-world.c create mode 100644 src/org/rascalmpl/library/lang/pico/examples/fac.pico create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc diff --git a/src/org/rascalmpl/library/lang/c90/examples/hello-world.c b/src/org/rascalmpl/library/lang/c90/examples/hello-world.c new file mode 100644 index 00000000000..3e04f5ac330 --- /dev/null +++ b/src/org/rascalmpl/library/lang/c90/examples/hello-world.c @@ -0,0 +1,19 @@ + +int print(const char *text); + +void printHello(char *name) { + print("Hello "); + print(name); + print("!"); +} + +int main(int argc, char *argv[]) { + char *name; + if (argc > 1) { + name = argv[1]; + } else { + name = "World"; + } + + printHello(name); +} diff --git a/src/org/rascalmpl/library/lang/pico/examples/fac.pico b/src/org/rascalmpl/library/lang/pico/examples/fac.pico new file mode 100644 index 00000000000..95a05f3e2e5 --- /dev/null +++ b/src/org/rascalmpl/library/lang/pico/examples/fac.pico @@ -0,0 +1,18 @@ +begin declare input : natural, + output : natural, + repnr : natural, + rep : natural, + s1 : string, + s2 : string; + input := 14; + output := 1; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc new file mode 100644 index 00000000000..042844ab59d --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc @@ -0,0 +1,10 @@ +module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +void runLanguageTests() { + testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); + testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); + testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); + testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 57377b3f312..6468d46e717 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -1,9 +1,16 @@ + +@bootstrapParser module lang::rascal::tests::concrete::recovery::RecoveryTestSupport +import lang::rascal::\syntax::Rascal; import ParseTree; import String; import IO; import util::Benchmark; +import Grammar; + +import lang::rascal::grammar::definition::Modules; + public data TestStats = testStats(int totalAttempts=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, list[tuple[loc,int]] slowParses = []); @@ -94,7 +101,27 @@ void printStats(TestStats stats) { } } -void testRecovery(type[&T] grammar, loc input) { - TestStats stats = testSingleCharDeletions(grammar, readFile(input)); - printStats(stats); +private str syntaxLocToModuleName(loc syntaxFile) { + str path = replaceLast(substring(syntaxFile.path, 1), ".rsc", ""); + return replaceAll(path, "/", "::"); +} + +loc zippedFile(str zip, str path) { + loc res = getResource("m3/snakes-and-ladders-project-source.zip"); + loc zipFile = res[scheme="jar+"][path=res.path + "!/"]; + return zipFile + path; +} + +void testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { + Module \module = parse(#start[Module], syntaxFile).top; + str modName = syntaxLocToModuleName(syntaxFile); + gram = modules2grammar(modName, {\module}); + + if (sym:\start(\sort(topSort)) <- gram.starts) { + println("Error recovery of () on :"); + TestStats stats = testSingleCharDeletions(type(sym, gram.rules), readFile(testInput)); + printStats(stats); + } else { + println("Cannot find top sort in "); + } } \ No newline at end of file From 09de7e05e7eb307b96529dbad39b3ff9e208516c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 25 Sep 2024 07:56:52 +0200 Subject: [PATCH 03/20] Removed obsolete tests, replaced by tests in LanguageRecoveryTests.rsc --- .../recovery/PicoMassRecoveryTests.rsc | 28 ------------------- .../recovery/RascalMassRecoveryTests.rsc | 15 ---------- 2 files changed, 43 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc deleted file mode 100644 index 339d69df267..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc +++ /dev/null @@ -1,28 +0,0 @@ -module lang::rascal::tests::concrete::recovery::PicoMassRecoveryTests - -import lang::pico::\syntax::Main; -import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; - -str picoFac = "begin declare input : natural, - output : natural, - repnr : natural, - rep : natural; - input := 14; - output := 1; - % check this out % - while input - 1 do - rep := output; - repnr := input; - while repnr - 1 do - output := output + rep; - repnr := repnr - 1 - od; - input := input - 1 - od -end"; - -bool testFacDeletions() { - TestStats stats = testSingleCharDeletions(#Program, picoFac); - printStats(stats); - return true; -} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc deleted file mode 100644 index c5d220efb31..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc +++ /dev/null @@ -1,15 +0,0 @@ -module lang::rascal::tests::concrete::recovery::RascalMassRecoveryTests - -import lang::rascal::\syntax::Rascal; -import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; -import IO; -import ParseTree; - -start[Module] (value input, loc origin) rascalParser = parser(#start[Module], allowAmbiguity=true, allowRecovery=true); -str source = readFile(|std:///lang/rascal/vis/ImportGraph.rsc|); - -bool testRascalDeletions() { - TestStats stats = testSingleCharDeletions(rascalParser, source); - printStats(stats); - return true; -} \ No newline at end of file From 0401f728f6b45eb1c19b95b8fc441e246780b40d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 25 Sep 2024 09:27:18 +0200 Subject: [PATCH 04/20] Added "delete until end of line" test --- .../library/lang/pico/examples/fac.pico | 2 +- .../recovery/LanguageRecoveryTests.rsc | 13 ++++-- .../concrete/recovery/RecoveryTestSupport.rsc | 41 +++++++++++++++++-- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/org/rascalmpl/library/lang/pico/examples/fac.pico b/src/org/rascalmpl/library/lang/pico/examples/fac.pico index 95a05f3e2e5..92e8ff7bb34 100644 --- a/src/org/rascalmpl/library/lang/pico/examples/fac.pico +++ b/src/org/rascalmpl/library/lang/pico/examples/fac.pico @@ -15,4 +15,4 @@ begin declare input : natural, od; input := input - 1 od -end \ No newline at end of file +end diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc index 042844ab59d..dc07c4fb46a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc @@ -2,9 +2,14 @@ module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +void runCTest() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +void runPicoTest() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +void runJavaTest() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +void runRascalTest() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); + void runLanguageTests() { - testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); - testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); - testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); - testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); + runPicoTest(); + runCTest(); + runJavaTest(); + runRascalTest(); } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 6468d46e717..9116f32c7f1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -67,6 +67,24 @@ TestStats testSingleCharDeletions(&T (value input, loc origin) parser, str input return totalStats; } +TestStats testDeleteUntilEol(&T (value input, loc origin) parser, str input) { + TestStats totalStats = testStats(); + int lineStart = 0; + list[int] lineEndings = findAll(input, "\n"); + + for (int lineEnd <- lineEndings) { + lineLength = lineEnd - lineStart; + for (int pos <- [lineStart..lineEnd]) { + modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); + TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deletedUntilEol=<",">|); + totalStats = mergeStats(totalStats, singleRunStats); + } + lineStart = lineEnd+1; + } + + return totalStats; +} + TestStats mergeStats(TestStats stats1, TestStats stats2) { TestStats result = stats1; result.totalAttempts += stats2.totalAttempts; @@ -107,7 +125,7 @@ private str syntaxLocToModuleName(loc syntaxFile) { } loc zippedFile(str zip, str path) { - loc res = getResource("m3/snakes-and-ladders-project-source.zip"); + loc res = getResource(zip); loc zipFile = res[scheme="jar+"][path=res.path + "!/"]; return zipFile + path; } @@ -115,12 +133,27 @@ loc zippedFile(str zip, str path) { void testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); - gram = modules2grammar(modName, {\module}); + Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { + println("--------------------------------------------------------------------------------"); println("Error recovery of () on :"); - TestStats stats = testSingleCharDeletions(type(sym, gram.rules), readFile(testInput)); - printStats(stats); + type[value] begin = type(sym, gram.rules); + testParser = parser(begin, allowAmbiguity=true, allowRecovery=true); + str input = readFile(testInput); + + println("Single char deletions:"); + TestStats singleCharDeletionStats = testSingleCharDeletions(testParser, input); + printStats(singleCharDeletionStats); + TestStats totalStats = singleCharDeletionStats; + + println("Deletes until end-of-line:"); + TestStats deleteUntilEolStats = testDeleteUntilEol(testParser, input); + printStats(deleteUntilEolStats); + totalStats = mergeStats(totalStats, deleteUntilEolStats); + + println("Total stats:"); + printStats(totalStats); } else { println("Cannot find top sort in "); } From 0464d897dd99063aa5da064828b92dcc52d160e8 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 11:44:54 +0200 Subject: [PATCH 05/20] Improved performance of `findBestError`, improved statistics reporting --- src/org/rascalmpl/library/ParseTree.rsc | 48 +---- .../concrete/recovery/BasicRecoveryTests.rsc | 6 +- .../recovery/LanguageRecoveryTests.rsc | 13 +- .../concrete/recovery/RecoveryTestSupport.rsc | 183 +++++++++++------- .../parser/uptr/UPTRNodeFactory.java | 2 - 5 files changed, 138 insertions(+), 114 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2877ecaaee3..cbacbb70c4e 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -144,7 +144,7 @@ extend Message; extend List; import String; -import Node; +import Set; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -807,20 +807,19 @@ str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringCh This filter removes error trees until no ambiguities caused by error recovery are left. Note that regular ambiguous trees remain in the parse forest. } -Tree defaultErrorDisambiguationFilter(t: appl(Production prod, args)) { - Tree result = appl(prod, [defaultErrorDisambiguationFilter(arg) | arg <- args]); - return setKeywordParameters(result, getKeywordParameters(t)); +Tree defaultErrorDisambiguationFilter(Tree t) { + return visit(t) { + case a:amb(_) => ambDisambiguation(a) + }; } -Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { +private Tree ambDisambiguation(amb(set[Tree] alternatives)) { // Go depth-first - set[Tree] disambiguatedAlts = { defaultErrorDisambiguationFilter(alt) | Tree alt <- alternatives }; - - set[Tree] errorTrees = { alt | Tree alt <- disambiguatedAlts, /appl(error(_,_,_), _) := alt }; - set[Tree] nonErrorTrees = { alt | Tree alt <- disambiguatedAlts, /appl(error(_,_,_), _) !:= alt }; + rel[int score, Tree alt] scoredErrorTrees = { | Tree alt <- alternatives }; + set[Tree] nonErrorTrees = scoredErrorTrees[0]; if (nonErrorTrees == {}) { - return getBestErrorTree(errorTrees); + return (getFirstFrom(scoredErrorTrees) | it.score > c.score ? c : it | c <- scoredErrorTrees).alt; } if ({Tree single} := nonErrorTrees) { @@ -832,34 +831,7 @@ Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { return amb(nonErrorTrees); } -private Tree getBestErrorTree(set[Tree] trees) { - Tree best = char(0); - int bestErrorCount = -1; - int bestErrorLength = 0; - - for (tree <- trees) { - list[Tree] errors = findAllErrors(tree); - int errorCount = size(errors); - int errorLength = 0; - - for (err <- errors) { - errorLength += getSkipped(err).src.length; - } - - if (bestErrorCount == -1 || errorCount < bestErrorCount || (errorCount == bestErrorCount && errorLength < bestErrorLength)) { - best = tree; - bestErrorCount = errorCount; - bestErrorLength = errorLength; - } - } - - if (bestErrorCount != -1) { - return best; - } - - // trees must have been empty - fail; -} +private int scoreErrors(Tree t) = (0 | it + getSkipped(e).src.length | /e:appl(error(_,_,_),_) := t); // Handle char and cycle nodes default Tree defaultErrorDisambiguationFilter(Tree t) = t; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index c9edce905b3..ab6d77258cd 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -24,7 +24,7 @@ syntax T = ABC End; syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -private Tree parseS(str input, bool visualize=false) +Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { @@ -38,10 +38,10 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $"); - return getErrorText(findFirstError(t)) == "x c"; + return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x c"; } test bool ax() { Tree t = parseS("a x $"); - return getErrorText(findFirstError(t)) == "x "; + return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc index dc07c4fb46a..aa72c006915 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc @@ -2,10 +2,15 @@ module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; -void runCTest() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); -void runPicoTest() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); -void runJavaTest() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); -void runRascalTest() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); +void runCTest() { testRecoveryC(); } +void runPicoTest() { testRecoveryPico(); } +void runJavaTest() { testRecoveryJava(); } +void runRascalTest() { testRecoveryRascal(); } + +TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); void runLanguageTests() { runPicoTest(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 9116f32c7f1..8320972d82f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -1,5 +1,3 @@ - -@bootstrapParser module lang::rascal::tests::concrete::recovery::RecoveryTestSupport import lang::rascal::\syntax::Rascal; @@ -8,67 +6,97 @@ import String; import IO; import util::Benchmark; import Grammar; +import analysis::statistics::Descriptive; import lang::rascal::grammar::definition::Modules; +public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); +public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); -public data TestStats = testStats(int totalAttempts=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, list[tuple[loc,int]] slowParses = []); +private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { + int startTime = 0; + int duration = 0; + TestMeasurement measurement = successfulParse(); + try { + startTime = realTime(); + Tree t = standardParser(input, source); + duration = realTime() - startTime; + measurement = successfulParse(source=source, duration=duration); + } catch ParseError(_): { + startTime = realTime(); + try { + Tree t = recoveryParser(input, source); + duration = realTime() - startTime; + Tree best = findBestError(t); + errorSize = size(getErrorText(best)); + measurement = recovered(source=source, duration=duration, errorSize=errorSize); + } catch ParseError(_): { + duration = realTime() - startTime; + measurement = parseError(source=source, duration=duration); + } + } -private TestStats testRecovery(&T (value input, loc origin) parser, str input, loc source) { - TestStats stats = testStats(totalAttempts=1); + return measurement; +} - int begin = realTime(); - try { - Tree t = parser(input, source); - if (hasErrors(t)) { - stats.successfulRecoveries = 1; - print("+"); - - //errors = findAllErrors(t); - //count = size(errors); - //best = findBestError(t); - //println("successful recovery, errors found, best: , loc: "); - //for (err <- errors) { - // println(""); - //} - } else { - stats.successfulParses = 1; - print("."); +TestStats updateStats(TestStats stats, TestMeasurement measurement) { + switch (measurement) { + case successfulParse(): { + print("."); + stats.successfulParses += 1; + } + case recovered(errorSize=errorSize): + if (errorSize <= stats.recoverySuccessLimit) { + print("+"); + stats.successfulRecoveries += 1; + } else { + print("-"); + stats.failedRecoveries += 1; + } + case parseError(): { + print("?"); + stats.parseErrors += 1; } - } catch ParseError(_): { - stats.parseErrors = 1; - print("?"); } - int duration = realTime() - begin; - - slowParse = []; - if (duration > 200) { - stats.slowParses = []; + + if (measurement.duration > stats.slowParseLimit) { print("!"); + stats.slowParses += 1; } + stats.measurements = stats.measurements + measurement; + return stats; } -TestStats testSingleCharDeletions(type[&T] grammar, str input) = testSingleCharDeletions(parser(grammar, allowAmbiguity=true, allowRecovery=true), input); +TestStats mergeStats(TestStats stats1, TestStats stats2) { + return testStats( + stats1.slowParseLimit, stats1.recoverySuccessLimit, + successfulParses = stats1.successfulParses + stats2.successfulParses, + successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, + failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, + parseErrors = stats1.parseErrors + stats2.parseErrors, + slowParses = stats1.slowParses + stats2.slowParses, + measurements = stats1.measurements + stats2.measurements); +} -TestStats testSingleCharDeletions(&T (value input, loc origin) parser, str input) { - TestStats totalStats = testStats(); +TestStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { + TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); int len = size(input); int i = 0; while (i < len) { str modifiedInput = substring(input, 0, i) + substring(input, i+1); - TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deleted=<"">|); - totalStats = mergeStats(totalStats, singleRunStats); + TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); + stats = updateStats(stats, measurement); i = i+1; } - return totalStats; + return stats; } -TestStats testDeleteUntilEol(&T (value input, loc origin) parser, str input) { - TestStats totalStats = testStats(); +TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { + TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); int lineStart = 0; list[int] lineEndings = findAll(input, "\n"); @@ -76,24 +104,13 @@ TestStats testDeleteUntilEol(&T (value input, loc origin) parser, str input) { lineLength = lineEnd - lineStart; for (int pos <- [lineStart..lineEnd]) { modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); - TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deletedUntilEol=<",">|); - totalStats = mergeStats(totalStats, singleRunStats); + TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); + stats = updateStats(stats, measurement); } lineStart = lineEnd+1; } - return totalStats; -} - -TestStats mergeStats(TestStats stats1, TestStats stats2) { - TestStats result = stats1; - result.totalAttempts += stats2.totalAttempts; - result.successfulParses += stats2.successfulParses; - result.successfulRecoveries += stats2.successfulRecoveries; - result.failedRecoveries += stats2.failedRecoveries; - result.parseErrors += stats2.parseErrors; - result.slowParses += stats2.slowParses; - return result; + return stats; } private int percentage(int number, int total) { @@ -102,21 +119,39 @@ private int percentage(int number, int total) { void printStats(TestStats stats) { println(); - println("Total parses: "); - println("Succesful parses: ( % of total)"); - int totalFailed = stats.totalAttempts - stats.successfulParses; + int measurementCount = size(stats.measurements); + println("Total parses: "); + println("Succesful parses: ( % of total)"); + int totalFailed = measurementCount - stats.successfulParses; println("Succesful recoveries: ( % of failed)"); println("Failed recoveries: ( % of failed)"); println("Parse errors: ( % of failed)"); - if (stats.slowParses == []) { + if (stats.slowParses == 0) { println("No slow parses."); } else { - println(" slow parses:"); - for ( <- stats.slowParses) { - println(": ms."); - } + slowest = (getFirstFrom(stats.measurements) | it.duration < e.duration ? it : e | e <- stats.measurements); + println(" slow parses, slowest parse: ( ms)"); } + + println(); + println("95th percentiles:"); + + list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; + list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; + list[int] failedRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize > stats.recoverySuccessLimit ]; + list[int] parseErrorTimes = [ duration | parseError(duration=duration) <- stats.measurements ]; + + println("Succesful parse time: ms"); + println("Succesful recovery time: ms"); + println("Failed recovery time: ms"); + println("Parse error time: ms"); + + list[int] errorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements ]; + println("Recovery error size characters"); + + list[int] successfulErrorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; + println("Successful recovery size: characters"); } private str syntaxLocToModuleName(loc syntaxFile) { @@ -130,31 +165,45 @@ loc zippedFile(str zip, str path) { return zipFile + path; } -void testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { +TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { - println("--------------------------------------------------------------------------------"); + println("=========================================================================="); println("Error recovery of () on :"); type[value] begin = type(sym, gram.rules); - testParser = parser(begin, allowAmbiguity=true, allowRecovery=true); + standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); + recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); str input = readFile(testInput); + int startTime = realTime(); + standardParser(input, testInput); + int referenceDuration = realTime() - startTime; + int slowParseLimit = referenceDuration*100; + int recoverySuccessLimit = size(input)/4; + + println(); println("Single char deletions:"); - TestStats singleCharDeletionStats = testSingleCharDeletions(testParser, input); + TestStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); printStats(singleCharDeletionStats); TestStats totalStats = singleCharDeletionStats; + println(); println("Deletes until end-of-line:"); - TestStats deleteUntilEolStats = testDeleteUntilEol(testParser, input); + TestStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); printStats(deleteUntilEolStats); totalStats = mergeStats(totalStats, deleteUntilEolStats); - println("Total stats:"); + println(); + println("Overall stats"); + print("-------------"); printStats(totalStats); + println(); + + return totalStats; } else { - println("Cannot find top sort in "); + throw "Cannot find top sort in "; } } \ No newline at end of file diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 34ff966fccd..006187c7f9a 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -145,8 +145,6 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createSkippedNode(int[] characters) { - //IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); - //return VF.appl(SKIPPED, VF.list(charValues)); return createLiteralNode(characters, SKIPPED); } From 6c1efe3cbfc31797de6915af9ccc4b3e122c0c38 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 14:28:18 +0200 Subject: [PATCH 06/20] Added some more languages to test and the output of a baseline run --- .../library/lang/diff/unified/UnifiedDiff.rsc | 2 + .../lang/diff/unified/examples/example.diff | 39 ++ .../library/lang/dot/examples/parse-state.dot | 46 ++ .../lang/dot/examples/parser-state.dot | 65 +++ .../rascalmpl/library/lang/dot/syntax/Dot.rsc | 2 +- .../recovery/ErrorRecoveryBenchmark.rsc | 29 ++ .../recovery/LanguageRecoveryTests.rsc | 20 - .../rascal/tests/concrete/recovery/NOTES.md | 413 ++++++++++++++++++ .../concrete/recovery/RecoveryTestSupport.rsc | 24 +- 9 files changed, 611 insertions(+), 29 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/diff/unified/examples/example.diff create mode 100644 src/org/rascalmpl/library/lang/dot/examples/parse-state.dot create mode 100644 src/org/rascalmpl/library/lang/dot/examples/parser-state.dot create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md diff --git a/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc b/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc index d4e84750c73..4e2d114f003 100644 --- a/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc +++ b/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc @@ -3,6 +3,8 @@ @contributor{Tijs van der Storm - storm@cwi.nl (CWI)} module lang::diff::unified::UnifiedDiff +start syntax DiffFile = Diff; + syntax Diff = Header old Header new Chunk* chunks ; diff --git a/src/org/rascalmpl/library/lang/diff/unified/examples/example.diff b/src/org/rascalmpl/library/lang/diff/unified/examples/example.diff new file mode 100644 index 00000000000..ea75dfad7a5 --- /dev/null +++ b/src/org/rascalmpl/library/lang/diff/unified/examples/example.diff @@ -0,0 +1,39 @@ +--- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java ++++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +@@ -1,6 +1,7 @@ + package org.rascalmpl.parser.uptr; + + import java.net.URI; ++import java.util.Arrays; + import java.util.IdentityHashMap; + import java.util.Map; + +@@ -21,7 +22,9 @@ import org.rascalmpl.values.parsetrees.ProductionAdapter; + import org.rascalmpl.values.parsetrees.TreeAdapter; + + public class UPTRNodeFactory implements INodeConstructorFactory{ +- private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); ++ private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); ++ private static final IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_IterStar, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT)))))); ++ + private boolean allowAmb; + + public UPTRNodeFactory(boolean allowAmbiguity){ +@@ -141,7 +144,14 @@ public class UPTRNodeFactory implements INodeConstructorFactory children, Object production) { ++ IConstructor prod = (IConstructor) production; ++ IConstructor errorProd = VF.constructor(RascalValueFactory.Production_Error, prod.get(0), prod, VF.integer(children.size()-1)); ++ return buildAppl(children, errorProd); ++ } ++ + } diff --git a/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot new file mode 100644 index 00000000000..bda91c2557e --- /dev/null +++ b/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot @@ -0,0 +1,46 @@ +digraph Parser { +"Parser"["label"="Parser\nInput: \"void f(){if(1){}}\"\nLocation: 0 ('v')\nStep 2: Reducing terminals"]; +"todo-1"["label"="<0> 0", "shape"="record"]; +"-2"["label"="Epsilon: \n.0@0 ,matchable,end\n?\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; +"124"["label"="SeparatedList: 124\n.0@0 ,expandable,end\n124\nin: 'default -> tags'"]; +"12858"["label"="NonTerminal: Tags\n.0@0 \nTags\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; +"-1"["label"="NonTerminal: FunctionDeclaration\n.0@-1 \nFunctionDeclaration"]; +"12858" -> "-1"; +"124" -> "12858"; +"-2" -> "124"; +"todo-1":"0":sw -> "-2"["label"="Stack"]; +"46484886"["shape"="octagon", "label"="Epsilon"]; +"todo-1":"0":se -> "46484886"["label"="Node"]; +"todoLists":"1" -> "todo-1"; +"todoLists"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8 | <9> 9 | <10> 10 | <11> 11 | <12> 12 | <13> 13 | <14> 14 | <15> 15", "shape"="record"]; +"Parser" -> "todoLists"["label"="todo lists"]; +"stacksToExpand"["label"="", "shape"="record"]; +"Parser" -> "stacksToExpand"["label"="stacks to expand"]; +"terminalsToReduce"["label"="<0> 0", "shape"="record", "color"="red"]; +"terminalsToReduce":"0":sw -> "-2"["label"="Stack"]; +"terminalsToReduce":"0":se -> "46484886"["label"="Node"]; +"Parser" -> "terminalsToReduce"["label"="terminals to reduce"]; +"nonTerminalsToReduce"["label"="", "shape"="record"]; +"Parser" -> "nonTerminalsToReduce"["label"="non-terminals to reduce"]; +"122"["label"="NonTerminal: Tag\n.0@0 ,end\nTag\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; +"122" -> "124"; +"unexpandableNodes":"0" -> "122"; +"unexpandableNodes"["label"="<0> 0", "shape"="record"]; +"12824"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"FunctionDeclaration\")' ':' 12828 0"]; +"unmatchableLeafNodes":"0" -> "12824"; +"128"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Tags\")' ':' 132 0"]; +"unmatchableLeafNodes":"1" -> "128"; +"2043"["label"="Literal: \n.0@-1 ,matchable\n'@'\nin: '@' Name '=' Expression"]; +"unmatchableLeafNodes":"2" -> "2043"; +"2065"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"Tag\"))' ':' 2069 0"]; +"unmatchableLeafNodes":"3" -> "2065"; +"unmatchableLeafNodes"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3", "shape"="record"]; +"unmatchableMidProductionNodes"["shape"="record", "label"=""]; +"filteredNodes"["label"="", "shape"="record"]; +"error"["label"="Errors"]; +"Parser" -> "error"["label"="error tracking"]; +"error" -> "unexpandableNodes"["label"="unexpandable"]; +"error" -> "unmatchableLeafNodes"["label"="unmatchable leafs"]; +"error" -> "unmatchableMidProductionNodes"["label"="unmatchable mid-prod"]; +"error" -> "filteredNodes"["label"="filtered"]; +} diff --git a/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot new file mode 100644 index 00000000000..e0405c03a8a --- /dev/null +++ b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot @@ -0,0 +1,65 @@ +digraph Parser { +"Parser"["label"="Parser\nInput: \"void f(){if(1){}}\"\nLocation: 0 ('v')\nStep 5: Reducing terminals"]; +"todo-1"["label"="<0> 0", "shape"="record"]; +"-2"["label"="Epsilon: \n.0@0 ,matchable,end\n?\nin: 'lex(\"LAYOUT\") -> regular(\iter-star(lex(\"LAYOUT\")))'"]; +"7226"["label"="List: 7226\n.0@0 ,expandable,end\n7226\nin: 'LAYOUTLIST -> \iter-star(lex(\"LAYOUT\"))'"]; +"12860"["label"="NonTerminal: LAYOUTLIST\n.1@0 \nlayouts_LAYOUTLIST\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; +"-1"["label"="NonTerminal: FunctionDeclaration\n.0@-1 \nFunctionDeclaration"]; +"12860" -> "-1"; +"7226" -> "12860"; +"-2" -> "7226"; +"todo-1":"0" -> "-2"["label"="Stack"]; +"46484886"["shape"="octagon", "label"="Epsilon"]; +"todo-1":"0":se -> "46484886"["label"="Node"]; +"todoLists":"1" -> "todo-1"; +"todoLists"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8 | <9> 9 | <10> 10 | <11> 11 | <12> 12 | <13> 13 | <14> 14 | <15> 15", "shape"="record"]; +"Parser" -> "todoLists"["label"="todo lists"]; +"stacksToExpand"["label"="", "shape"="record"]; +"Parser" -> "stacksToExpand"["label"="stacks to expand"]; +"terminalsToReduce"["label"="<0> 0", "shape"="record", "color"="red"]; +"terminalsToReduce":"0":sw -> "-2"["label"="Stack"]; +"terminalsToReduce":"0":se -> "46484886"["label"="Node"]; +"Parser" -> "terminalsToReduce"["label"="terminals to reduce"]; +"nonTerminalsToReduce"["label"="", "shape"="record"]; +"Parser" -> "nonTerminalsToReduce"["label"="non-terminals to reduce"]; +"122"["label"="NonTerminal: Tag\n.0@0 ,end\nTag\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; +"124"["label"="SeparatedList: 124\n.0@0 ,expandable,end\n124\nin: 'default -> tags'"]; +"12858"["label"="NonTerminal: Tags\n.0@0 \nTags\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; +"12858" -> "-1"; +"124" -> "12858"; +"122" -> "124"; +"unexpandableNodes":"0" -> "122"; +"13120"["label"="NonTerminal: Comment\n.0@0 ,end\nComment\nin: 'LAYOUT -> Comment'"]; +"7221"["label"="NonTerminal: LAYOUT\n.0@0 ,end\nLAYOUT\nin: 'lex(\"LAYOUT\") -> regular(\iter-star(lex(\"LAYOUT\")))'"]; +"7221" -> "7226"; +"13120" -> "7221"; +"unexpandableNodes":"1" -> "13120"; +"unexpandableNodes"["label"="<0> 0 | <1> 1", "shape"="record"]; +"12824"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"FunctionDeclaration\")' ':' 12828 0"]; +"unmatchableLeafNodes":"0" -> "12824"; +"128"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Tags\")' ':' 132 0"]; +"unmatchableLeafNodes":"1" -> "128"; +"2043"["label"="Literal: \n.0@-1 ,matchable\n'@'\nin: '@' Name '=' Expression"]; +"unmatchableLeafNodes":"2" -> "2043"; +"2065"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"Tag\"))' ':' 2069 0"]; +"unmatchableLeafNodes":"3" -> "2065"; +"13122"["label"="Char: \n.0@-1 ,matchable,end\n9-13,32,133,160,5760,6158,8192-8202,8232-8233,8239,8287,12288\nin: 'LAYOUT -> [range(9,13),range(32,32),range(133,133),range(160,160),range(5760,5760),range(6158,6158),range(8192,8202),range(8232,8233),range(8239,8239),range(8287,8287),range(12288,12288)]'"]; +"unmatchableLeafNodes":"4" -> "13122"; +"13125"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"LAYOUT\"))' ':' 13129 0"]; +"unmatchableLeafNodes":"5" -> "13125"; +"7373"["label"="Literal: \n.0@-1 ,matchable\n'/*'\nin: '/*' 7379 '*/'"]; +"unmatchableLeafNodes":"6" -> "7373"; +"7382"["label"="Literal: \n.0@-1 ,matchable\n'//'\nin: '//' 7386"]; +"unmatchableLeafNodes":"7" -> "7382"; +"7389"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Comment\")' ':' 7393 0"]; +"unmatchableLeafNodes":"8" -> "7389"; +"unmatchableLeafNodes"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8", "shape"="record"]; +"unmatchableMidProductionNodes"["shape"="record", "label"=""]; +"filteredNodes"["label"="", "shape"="record"]; +"error"["label"="Errors"]; +"Parser" -> "error"["label"="error tracking"]; +"error" -> "unexpandableNodes"["label"="unexpandable"]; +"error" -> "unmatchableLeafNodes"["label"="unmatchable leafs"]; +"error" -> "unmatchableMidProductionNodes"["label"="unmatchable mid-prod"]; +"error" -> "filteredNodes"["label"="filtered"]; +} diff --git a/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc b/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc index 7fcf9b98ca7..d50f8f66a52 100644 --- a/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc +++ b/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc @@ -68,7 +68,7 @@ syntax NodeId | Id Port ; -syntax Port = ":" Id Id? +syntax Port = ":" Id (":" Id)? // | ":" Id // | ":" CompassPt ; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc new file mode 100644 index 00000000000..a0459cc5237 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -0,0 +1,29 @@ +module lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +void runTestC() { testRecoveryC(); } +void runTestDiff() { testRecoveryDiff(); } +void runTestDot() { testRecoveryDot(); } +void runTestJava() { testRecoveryJava(); } +void runTestJson() { testRecoveryJson(); } +void runTestPico() { testRecoveryPico(); } +void runTestRascal() { testRecoveryRascal(); } + +TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +TestStats testRecoveryDiff() = testErrorRecovery(|std:///lang/diff/unified/UnifiedDiff.rsc|, "DiffFile", |std:///lang/diff/unified/examples/example.diff|); +TestStats testRecoveryDot() = testErrorRecovery(|std:///lang/dot/syntax/Dot.rsc|, "DOT", |std:///lang/dot/examples/parser-state.dot|); +TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +TestStats testRecoveryJson() = testErrorRecovery(|std:///lang/json/syntax/JSON.rsc|, "JSONText", |std:///lang/json/examples/ex01.json|); +TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); + +void runLanguageTests() { + testRecoveryC(); + testRecoveryDiff(); + testRecoveryDot(); + testRecoveryJava(); + testRecoveryJson(); + testRecoveryPico(); + testRecoveryRascal(); +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc deleted file mode 100644 index aa72c006915..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc +++ /dev/null @@ -1,20 +0,0 @@ -module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests - -import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; - -void runCTest() { testRecoveryC(); } -void runPicoTest() { testRecoveryPico(); } -void runJavaTest() { testRecoveryJava(); } -void runRascalTest() { testRecoveryRascal(); } - -TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); -TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); -TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); -TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); - -void runLanguageTests() { - runPicoTest(); - runCTest(); - runJavaTest(); - runRascalTest(); -} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md new file mode 100644 index 00000000000..5eb4723935e --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md @@ -0,0 +1,413 @@ + +# Error recovery benchmark + +Next to some basic integration tests, this directory contains a simple error recovery benchmark (see `ErrorRecoveryBenchmark.rsc`). +This benchmark is meant to provide feedback on the quality and speed of error recovery and to be able to track progress in these areas. + +The benchmark tests error recovery for a number of languages by performing modifications on a valid input file and attempting to parse the result. +Currently only two types of modifications are implemented: + +- Deletion of single characters +- Deletion until the next end-of-line character + +We anticipate more types of modifications later, possibly based on the parse tree instead of the raw source file. + +## Baseline test + +Below the output of a baseline test is included. This provides a basic idea of what information the benchmark provides and also serves as a comparison for assessing the result of future improvements. + +Note that a benchmark test is documented by a string of characters including: + +- `.`: After modification the input still parses without errors +- '+': After modification error recovery is able to recover by skipping a reasonable amount of input +- '-': After modification error recovery is able to recover by skipping an excessive amount of input (>25% of the contents of the file) +- '?': After modification parsing results in an unrecoverable parse error + +The rest of the output should (hopefully) be self-descriptive: + +``` +rascal>import lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark; +rascal>runLanguageTests(); +========================================================================== +Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|: +☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator Globa 🕛 0:00:03.468 + +Single char deletions: +...........+................++....................+..........+.+.........++......+++.........+....++.........++.+++..-............?....................+++.+......................................................................?....++.-.....+.++.?.................++++....+.++++.?...........+.+.....?+....?................+....+?..?.. +Total parses: 333 +Succesful parses: 281 (84 % of total) +Succesful recoveries: 42 (80 % of failed) +Failed recoveries: 2 (3 % of failed) +Parse errors: 8 (15 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 3/2/8 ms +Failed recovery time : 1/2/2 ms +Parse error time : 1/1/2 ms +Recovery error size : 17/5/83 characters +Successful recovery size : 13/4/50 characters + +Deletes until end-of-line: +...........++++++++++++++++++...++++++++++++++++++++++?----++.........++++++++++....+++++++++++....++++++++++.-..?++??++++???????????????????????++............................................................+++++?????.????+++++++++++??......++++++++++++++....??+++??......+++++++??????+.???.....????++++++??????.?. +Total parses: 314 +Succesful parses: 119 (37 % of total) +Succesful recoveries: 128 (65 % of failed) +Failed recoveries: 5 (2 % of failed) +Parse errors: 62 (31 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 2/2/9 ms +Failed recovery time : 2/2/3 ms +Parse error time : 1/1/2 ms +Recovery error size : 38/36/82 characters +Successful recovery size : 36/35/82 characters + +Overall stats +------------- +Total parses: 647 +Succesful parses: 400 (61 % of total) +Succesful recoveries: 170 (68 % of failed) +Failed recoveries: 7 (2 % of failed) +Parse errors: 70 (28 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/9 ms +Failed recovery time : 1/2/3 ms +Parse error time : 1/1/2 ms +Recovery error size : 33/27/83 characters +Successful recovery size : 30/26/82 characters + +========================================================================== +Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|: + +Single char deletions: +++++....................................................+++++....................................................++++++.++++.++++.+...................................+.+.....................+.........................+..................................+......................+.----...---...----...........................................................-....................................................-.-.........................................................................................-..........................................................................................................-..........................................................................................................-...............................................................................................................................................................................................................................................................................................................................-.-...........................-..-.................................................????....???......???..................................................................................?...?............................................................................................................................................................................................?.................................................?...?.?..............................................................................?.................................................?..................................................................................................................................?.........................................?...?.?.? +Total parses: 1776 +Succesful parses: 1702 (95 % of total) +Succesful recoveries: 30 (40 % of failed) +Failed recoveries: 21 (28 % of failed) +Parse errors: 23 (31 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 1/1/2 ms +Failed recovery time : 5/6/9 ms +Parse error time : 0/0/1 ms +Recovery error size : 386/161/837 characters +Successful recovery size : 96/96/161 characters + +Deletes until end-of-line: +++++....................................................++++....................................................++++++++++++++++..................................++....................+........................+.................................+.....................+-----!-!-!-----!-!----..........................................................-...................................................--........................................................................................-.........................................................................................................-.........................................................................................................+..............................................................................................................................................................................................................................................................................................................................--..........................-.-................................................????????????????????.................................................................................?..??.............?.......................................................?..................................................?.....?......................................................?................................................?..??.............................................................................?................................................?.................................................................................................................................?........................................?..??. +Total parses: 1737 +Succesful parses: 1643 (94 % of total) +Succesful recoveries: 31 (32 % of failed) +Failed recoveries: 26 (27 % of failed) +Parse errors: 37 (39 % of failed) +5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 1/2/2 ms +Failed recovery time : 6/6/17 ms +Parse error time : 1/1/2 ms +Recovery error size : 398/160/777 characters +Successful recovery size : 102/147/161 characters + +Overall stats +------------- +Total parses: 3513 +Succesful parses: 3345 (95 % of total) +Succesful recoveries: 61 (36 % of failed) +Failed recoveries: 47 (27 % of failed) +Parse errors: 60 (35 % of failed) +5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 1/1/2 ms +Failed recovery time : 6/6/13 ms +Parse error time : 0/1/2 ms +Recovery error size : 392/161/837 characters +Successful recovery size : 99/109/161 characters + +========================================================================== +Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|: +☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.307 + +Single char deletions: +++??????.......?..+......+-+.....+++...............+..................+................................................++...+......--+.....+++.....+..+.....+++......++...+..+-+.....+++............................................+.......+.............................+.......+.....++...-....--+.....+++..........................................................................+.......+....++...+.....+-+.....+++.................................................................................................................++...+..+++.....+++.............................................................++...+.....+.++.+..+...+....+.++.+.....+...+..+.++.+....+...+......+.+.+.++.+..+++.....+++.....++...+........+++.....+++.......+..+.....+++.......++...+......+.+.+....++.+........+++.....+++....++...+.........+.+.+.++.+......+...+.........+++.....+++.........................................................................................................................................+..+.....+++......++...+......+.++.+.........+++.....+++..........++...+..............+++.....++++..+.....+++......++...+......+.++.+..............+++.....+++................++...+.................+++.....+++.....+..+.....+++......+..+.....+++...++...+.................+.+.+....++.+..+++.....+++.....++...+.................+.+.+....++.+........+-+.....+++....++...+......+.++.+.................+-+.....+++...................++...+....................+-+.....++++..+.....+++......++...+......+.-+.+....................+-+.....+++.......................++...-...--+.....+++............................................+....+...................................+....+............+...........+......++...-...--+.....+++....................................................................++...+.....+++.....+++.............................................................................................++...+.....+.++.+..-...-...-.++.-.....-...-...-.++.-...-...-.................-.-.-.-+.-...-...-.....--+.....+++.................................................................++...-....-++.....+++.................................................+.......+.............................+.......+.....++...+....+.++.+....+...+.....+.++.+....+...+.................+.+.+.++.+.....+...+.................+-+.....+++.............+..+.....+++......++...+.....+++.....+++.........................................+....................+...............++...+....................+.+.+.++.+.....+...+...+++.....+++.........................................+.....+.............++...+....................+.+.+.++.+...+...+....+++.....+++.............................................................++...+....................+.+.+.++.+....+...+....+++.....+++....................................................+....+...............++...+....................+.+.+.-+.+....+...+.....+++.....+++..............................................................................................................................................................................................................................................................................................++...+....................+.+.+.++.+.....+...+.....+++.....+++....................................................+.......+................++...+....................+.+.+.++.+.....+...+....+++.....+++.....................................................++...+....................+.+.+.++.+....+...+....+++.....+++................................................++...+....................+.+.+.++.+....+...+....+++.....+++.........................................+........+..............++...+....................+.+.+.+?.+....+...+....................+++.....+++.....................................................................+..+.....+++......++...?.............................?++.....+++......+..+.....+++++...?.............?++.....++++..+.....+++......++...?.....?++.....+++......++...?......?.+?.?.....?++.....+++..............++...?.....?.+?.?.................?++.....+++............++...?.....?.+?.?....................?++.....+++.................++...?.....?.+?.?.............................?++.....+++....................++...?.....?.+?.?.............?++.....++?........??...?.. +Total parses: 4239 +Succesful parses: 3555 (83 % of total) +Succesful recoveries: 599 (87 % of failed) +Failed recoveries: 42 (6 % of failed) +Parse errors: 43 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 5/5/8 ms +Succesful recovery time : 9/9/17 ms +Failed recovery time : 10/10/17 ms +Parse error time : 3/3/6 ms +Recovery error size : 233/44/2220 characters +Successful recovery size : 65/38/241 characters + +Deletes until end-of-line: +????????????????..-------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++.++++++++++++++++++++++++++++++++++...---.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++..+..+++...+++++..+..++++++...+++..+..+++++...+++++++..++..+..+++.++++++++++++++++...+++++++++.+++++++++++++++++++++++++++++++++++++...+++++++..++.+...+..+++++++++.+++++++++++++++...++++++++++..++..+..+++++++...++++++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++..+..++++++++++.+++++++++++++++++++++...+++++++++++++++.+++++++++++++++++++++++++++++...+++++++..+..+++++++++++++++.+++++++++++++++++++++++++++...++++++++++++++++++.+++++++++++++++++++++++++++++++++++++++++++++++++...++++++++++++++++++.+++.+...+..+++.++++++++++++++++...++++++++++++++++++..++.....+..+++++++++.+++++++++++++++...+++++++..+..++++++++++++++++++.++++++++++++++++++++++++++++++...+++++++++++++++++++++.+++++++++++++++++++++++++++++...-------..+..---------------------.++++++++++++++++++++++++++++++++++...----.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...------..+..---...----..+..------...----..+..----...------------------..--..+..----...------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++..+..+++++...++++++..+..+++++...++++++++++++++++++..++..+..++++++...++++++++++++++++++.++++++++++++++++++++++++++++++++++++++++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..?..+++++...?????????????????????.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...??????????????????????????????.+++++++++++++++++++++++++++++...??????????????.+++++++++++++++++++++++++++++...??????.+++++++++++++++++...???????..?..??????.+++++++++++++++++++++++++...??????..?..??????????????????.+++++++++++++++++++++++...??????..?..?????????????????????.++++++++++++++++++++++++++++...??????..?..??????????????????????????????.+++++++++++++++++++++++++++++++...??????..?????????????????.???????????????????..?. +Total parses: 4174 +Succesful parses: 402 (9 % of total) +Succesful recoveries: 3438 (91 % of failed) +Failed recoveries: 99 (2 % of failed) +Parse errors: 235 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 6/6/13 ms +Succesful recovery time : 9/8/18 ms +Failed recovery time : 11/10/25 ms +Parse error time : 4/4/7 ms +Recovery error size : 150/63/391 characters +Successful recovery size : 81/61/297 characters + +Overall stats +------------- +Total parses: 8413 +Succesful parses: 3957 (47 % of total) +Succesful recoveries: 4037 (90 % of failed) +Failed recoveries: 141 (3 % of failed) +Parse errors: 278 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 5/5/9 ms +Succesful recovery time : 9/8/18 ms +Failed recovery time : 11/10/19 ms +Parse error time : 3/4/7 ms +Recovery error size : 163/61/408 characters +Successful recovery size : 79/57/289 characters + +========================================================================== +Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///C:/Users/pieter/Documents/projects/wk/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|: +☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumer 🕛 0:00:08.222 + +Single char deletions: +++++++++......-..++++++.++++++..........+++++++++.......-...++++++...........-....+....+....+........+.-........+....+.........++..+...+..........++++++.....................++.+...++++++.....+..?.?. +Total parses: 198 +Succesful parses: 131 (66 % of total) +Succesful recoveries: 61 (91 % of failed) +Failed recoveries: 4 (5 % of failed) +Parse errors: 2 (2 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/6 ms +Failed recovery time : 2/2/4 ms +Parse error time : 0/1/1 ms +Recovery error size : 18/14/69 characters +Successful recovery size : 14/13/31 characters + +Deletes until end-of-line: +.++++++------------------------------------------------++++++++++++++++++++-------------------------...+++++++++++++++++++++++..+.......++++++++++++++++++++++++++++++++...+++++++++++??? +Total parses: 185 +Succesful parses: 16 (8 % of total) +Succesful recoveries: 93 (55 % of failed) +Failed recoveries: 73 (43 % of failed) +Parse errors: 3 (1 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/5 ms +Failed recovery time : 1/2/3 ms +Parse error time : 0/1/1 ms +Recovery error size : 48/42/112 characters +Successful recovery size : 18/11/45 characters + +Overall stats +------------- +Total parses: 383 +Succesful parses: 147 (38 % of total) +Succesful recoveries: 154 (65 % of failed) +Failed recoveries: 77 (32 % of failed) +Parse errors: 5 (2 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/6 ms +Failed recovery time : 1/2/4 ms +Parse error time : 0/1/1 ms +Recovery error size : 39/21/112 characters +Successful recovery size : 16/12/42 characters + +========================================================================== +Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|: +☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.619 + +Single char deletions: +?........?.....??.?............+.....++..+..?............+......++.+..?............+.....++..+....................??............?.........??.?................+...++....+......................................++................+......-+....+................-.....+-..+...-............?+............?...??.?...+....+....+......?........?..? +Total parses: 337 +Succesful parses: 285 (84 % of total) +Succesful recoveries: 27 (51 % of failed) +Failed recoveries: 4 (7 % of failed) +Parse errors: 21 (40 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/0/1 ms +Succesful recovery time : 0/1/1 ms +Failed recovery time : 0/0/1 ms +Parse error time : 0/1/1 ms +Recovery error size : 34/21/124 characters +Successful recovery size : 21/16/57 characters + +Deletes until end-of-line: +?.????????????????............+++++++??????............++++++++?????............????????????????????????????????.????????????????????????................++++++++++++++++++++++++++++++++++++++++++++++++++................--------+++++.-----------------------------.-----------+.??????????????????+????????????????????.???????. +Total parses: 324 +Succesful parses: 75 (23 % of total) +Succesful recoveries: 72 (28 % of failed) +Failed recoveries: 48 (19 % of failed) +Parse errors: 129 (51 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/0/1 ms +Succesful recovery time : 0/1/2 ms +Failed recovery time : 0/1/1 ms +Parse error time : 0/0/1 ms +Recovery error size : 86/57/294 characters +Successful recovery size : 42/57/57 characters + +Overall stats +------------- +Total parses: 661 +Succesful parses: 360 (54 % of total) +Succesful recoveries: 99 (32 % of failed) +Failed recoveries: 52 (17 % of failed) +Parse errors: 150 (49 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/0/1 ms +Succesful recovery time : 0/1/2 ms +Failed recovery time : 0/1/1 ms +Parse error time : 0/0/1 ms +Recovery error size : 76/57/293 characters +Successful recovery size : 37/39/57 characters + +========================================================================== +Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|: +☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.719 + +Single char deletions: +?????.-------.......+.+++++++-.........................+.++++++++............................+.++++++++....................+.++++++++................+..+.+++++++................+..+.------+..............++...+...............++.+-........-----.......+.-.--.................++.......+..................++......-............+++++.......+.+.++......................++........+....+.....................++.......+.+............+++..................--.......+.+........--..???.. +Total parses: 472 +Succesful parses: 355 (75 % of total) +Succesful recoveries: 81 (69 % of failed) +Failed recoveries: 28 (23 % of failed) +Parse errors: 8 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/1 ms +Succesful recovery time : 2/3/5 ms +Failed recovery time : 1/2/4 ms +Parse error time : 0/0/1 ms +Recovery error size : 87/10/463 characters +Successful recovery size : 16/6/70 characters + +Deletes until end-of-line: +?????--------..++++++++++++++-..................++++++++++++++++......................+++++++++++++++................+++++++++++++................+++++++++++.-------------------------+........+++++++++++........-----------.+++++++-----------------.............+++++++++++++............+++++++-------.++++++++++++++++++++++++++++...............++++++++++++++++++++++.+++++++++++++++++++++++......++.------------+.------------------++......++.+++++++-.???. +Total parses: 454 +Succesful parses: 151 (33 % of total) +Succesful recoveries: 195 (64 % of failed) +Failed recoveries: 100 (33 % of failed) +Parse errors: 8 (2 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/1 ms +Succesful recovery time : 2/2/4 ms +Failed recovery time : 1/1/4 ms +Parse error time : 0/0/1 ms +Recovery error size : 101/41/268 characters +Successful recovery size : 28/26/70 characters + +Overall stats +------------- +Total parses: 926 +Succesful parses: 506 (54 % of total) +Succesful recoveries: 276 (65 % of failed) +Failed recoveries: 128 (30 % of failed) +Parse errors: 16 (3 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/1 ms +Succesful recovery time : 2/2/5 ms +Failed recovery time : 1/1/4 ms +Parse error time : 0/0/1 ms +Recovery error size : 97/33/272 characters +Successful recovery size : 25/15/70 characters + +========================================================================== +Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|: + +Single char deletions: +?........+...............................................................................................?.?...........+......................................................................................................................................................................................................................................................................................................................................................................................................................?.?........+....................................................................................................................................................................................................................................................................................................?.?........+..................................................................................................?.?.!......!....!..!.!.!????????..!.!.+!+......++...++.............??????+....++..........+.??????+...++......+.??????+....++......++.......++..........++.......+.??????+....++......+++......++......+.??????+.........+.??????+....++..........+.??????+....++...........+.+++++++..+.+++++++........++......++.....-..+........+..............................................................................................?.....?...........?.....................+..................+.?................+!+.......+++............++.............................++.+..+........?......................................................................................?.....?...........?.....................+..................+.?.....++...............................................................................................+.....................+................+.........++++................+!.................................++.+..+........?.................................................................................................?.....?...........?.....................+..................-!.?.....+.+................+.........+?..........++...........................................................................+....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++!++++++.........+.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+.....++............................................................................................++...........................................................................................................................................+......+.........+.........+........+.+.........++.+....+.++.+......+....++.....................+.........++++..............++++++.+.......+...++.....+......+++++++....+.........+......++.+.+.......+...++............................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++.+..+........+........................................................................................?.+++++.............+.............+........+...+............+.+.+++........+...+...............+.+++........+...+....+.......+.+.+++........+...+....+.......+.+.+++........+...+....+.......+.++.+?..?........?..........................................................................................?.............?...............?....+...+.....+.?..............+.+.....+...+.+...++..............++.................+.+...............+.+...+............++.................?.............+...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++.................+.+.+...........+..........+.....++..........................+..................+++..+..................+.+..+..................++.......................+.?..?........?.............................................................?.............?...............?........?.?.....+++.?..................?............+....++..........+....+.....+.....+.+.................+.++..........++++++.............+...................+.+.+....++...................+.+.+....+.+.....+...................+.+.+....+.+.....+.....................+.++....+.....++.........+?.....?.....++++++..........+.+.+.............................++?.? +Total parses: 4314 +Succesful parses: 3815 (88 % of total) +Succesful recoveries: 393 (78 % of failed) +Failed recoveries: 2 (0 % of failed) +Parse errors: 104 (20 % of failed) +14 slow parses, slowest parse: |unknown:///?deleted=1137| (1 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 14/12/20 ms +Succesful recovery time : 29/21/69 ms +Failed recovery time : 183/184/351 ms +Parse error time : 10/5/40 ms +Recovery error size : 65/19/274 characters +Successful recovery size : 59/18/233 characters + +Deletes until end-of-line: +.+........?????????????????????????????????????????????????????????????????????????????????????????????????++++++++++++.................................................................................................................................................................................................................................................................................................................................................................................................................??+++++++++...............................................................................................................................................................................................................................................................................................??+++++++++................................................................................................?.?..............????????....+?......+?...+?...........?????++++++++++++++++++.?????+++++++++++++.?????++++++++++++++++++++++++++++++++++++++++++++.?????+++++++++++++++++++++++++++++++.?????+++++++++++.?????++++++++++++++++++.?????+++++++++++++++++++.+++++++++.+++++-------------------------.+........???????????????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++++++++++++++++++++++++.+........???????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+..........................................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++++++++++++++.+........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!??-!-!-!-!-!?????????????-!----!??.........???????????????++++++++++?.........+...........................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+............................................................................................+...........................................................................................................................................+++++++?????----++-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........+++.+++++++++++++?????+++????????+?+++++++.....+++++++++++++++++++++++++++++++..+++++++++++++.....+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????+++.+........?????????????????????????????????????????????????????????????????????????????????????????+++++++++++++++++++++++++++++++++........++++.........+..+..++++.....+++++++++++++++++++++++++++.....++++++++++++++++++++++++++.....++++++++++++++++++++++++++++++++++++++++++++++++++++++??.?........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????+++++++++++??????.......+........?++++++++++++++++++++++++++++++++...........+......????????????????????++++++++++++++++++++++++++++++++++++++++++++++...........+..+..++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....++++++++++++++++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+++++++++++++++++?.?........???????????????????????????????????????????????????????????????????????????????????????????????????????????????.........++++++++++++++++++++++++++++.........+++++++++++++++++++..+++++++++++++++++++++++++++++++++++++++++++++++...............+..+..++?+++++.............++++++++++?++++++++++++.............++++++++++?++++++++++++++++++++++++++++++++++++++++++++++++???????????????+++++++++++++++++++++++++.??????????????++????????????+? +Total parses: 4206 +Succesful parses: 1542 (36 % of total) +Succesful recoveries: 1611 (60 % of failed) +Failed recoveries: 55 (2 % of failed) +Parse errors: 998 (37 % of failed) +64 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 13/13/21 ms +Succesful recovery time : 28/18/66 ms +Failed recovery time : 135/18/319 ms +Parse error time : 7/6/14 ms +Recovery error size : 158/40/998 characters +Successful recovery size : 116/38/561 characters + +Overall stats +------------- +Total parses: 8520 +Succesful parses: 5357 (62 % of total) +Succesful recoveries: 2004 (63 % of failed) +Failed recoveries: 57 (1 % of failed) +Parse errors: 1102 (34 % of failed) +78 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 14/13/20 ms +Succesful recovery time : 28/18/67 ms +Failed recovery time : 137/18/331 ms +Parse error time : 7/6/14 ms +Recovery error size : 140/36/981 characters +Successful recovery size : 105/32/407 characters +``` diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 8320972d82f..52cba322632 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -7,6 +7,7 @@ import IO; import util::Benchmark; import Grammar; import analysis::statistics::Descriptive; +import util::Math; import lang::rascal::grammar::definition::Modules; @@ -135,23 +136,30 @@ void printStats(TestStats stats) { } println(); - println("95th percentiles:"); + println("Statistics (average/median/95th percentile):"); + + void printStats(str label, list[int] values, str unit) { + int mean = sum(values)/size(values); + print(left(label, 40)); + print(": "); + println("// "); + } list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; list[int] failedRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize > stats.recoverySuccessLimit ]; list[int] parseErrorTimes = [ duration | parseError(duration=duration) <- stats.measurements ]; - println("Succesful parse time: ms"); - println("Succesful recovery time: ms"); - println("Failed recovery time: ms"); - println("Parse error time: ms"); + printStats("Succesful parse time", successfulParseTimes, "ms"); + printStats("Succesful recovery time", successfulRecoveryTimes, "ms"); + printStats("Failed recovery time", failedRecoveryTimes, "ms"); + printStats("Parse error time", parseErrorTimes, "ms"); list[int] errorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements ]; - println("Recovery error size characters"); + printStats("Recovery error size", errorSizes, "characters"); list[int] successfulErrorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; - println("Successful recovery size: characters"); + printStats("Successful recovery size", successfulErrorSizes, "characters"); } private str syntaxLocToModuleName(loc syntaxFile) { @@ -181,7 +189,7 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { int startTime = realTime(); standardParser(input, testInput); int referenceDuration = realTime() - startTime; - int slowParseLimit = referenceDuration*100; + int slowParseLimit = referenceDuration*10; int recoverySuccessLimit = size(input)/4; println(); From c93de3e4194f502cf30c4bbf10adf652e94575cb Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 15:51:09 +0200 Subject: [PATCH 07/20] Removed space before % --- .../tests/concrete/recovery/RecoveryTestSupport.rsc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 52cba322632..5638a550924 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -122,11 +122,11 @@ void printStats(TestStats stats) { println(); int measurementCount = size(stats.measurements); println("Total parses: "); - println("Succesful parses: ( % of total)"); + println("Succesful parses: (% of total)"); int totalFailed = measurementCount - stats.successfulParses; - println("Succesful recoveries: ( % of failed)"); - println("Failed recoveries: ( % of failed)"); - println("Parse errors: ( % of failed)"); + println("Succesful recoveries: (% of failed)"); + println("Failed recoveries: (% of failed)"); + println("Parse errors: (% of failed)"); if (stats.slowParses == 0) { println("No slow parses."); From 5fbc8cd6160ddb41c175278a5f4a5eee130f1062 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 15:52:05 +0200 Subject: [PATCH 08/20] Reintroduced 'private' --- .../lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index ab6d77258cd..26655e8edb7 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -24,7 +24,7 @@ syntax T = ABC End; syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -Tree parseS(str input, bool visualize=false) +private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { From 11c705e95913b5284dd2a0cd2d94ce2677b99238 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 27 Sep 2024 12:26:04 +0200 Subject: [PATCH 09/20] Added removed compass direction --- src/org/rascalmpl/library/lang/dot/examples/parser-state.dot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot index e0405c03a8a..fc3047e35a5 100644 --- a/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot +++ b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot @@ -8,7 +8,7 @@ digraph Parser { "12860" -> "-1"; "7226" -> "12860"; "-2" -> "7226"; -"todo-1":"0" -> "-2"["label"="Stack"]; +"todo-1":"0":sw -> "-2"["label"="Stack"]; "46484886"["shape"="octagon", "label"="Epsilon"]; "todo-1":"0":se -> "46484886"["label"="Node"]; "todoLists":"1" -> "todo-1"; From 480eb12ac2e799ade1282e013c476cce980fa7a5 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 27 Sep 2024 12:27:01 +0200 Subject: [PATCH 10/20] Added Rascal batch testing support --- .../recovery/ErrorRecoveryBenchmark.rsc | 4 ++ .../concrete/recovery/RecoveryTestSupport.rsc | 58 ++++++++++++++++--- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index a0459cc5237..89a0f47793a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -26,4 +26,8 @@ void runLanguageTests() { testRecoveryJson(); testRecoveryPico(); testRecoveryRascal(); +} + +void runRascalBatchTest() { + batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 3, 4096); } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 5638a550924..9598a87a579 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -173,7 +173,10 @@ loc zippedFile(str zip, str path) { return zipFile + path; } -TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { +TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = +testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); + +TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input, int slowParseLimit=0, int recoverySuccessLimit=0) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); @@ -184,13 +187,17 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { type[value] begin = type(sym, gram.rules); standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); - str input = readFile(testInput); - int startTime = realTime(); - standardParser(input, testInput); - int referenceDuration = realTime() - startTime; - int slowParseLimit = referenceDuration*10; - int recoverySuccessLimit = size(input)/4; + if (slowParseLimit == 0) { + int startTime = realTime(); + standardParser(input, testInput); + int referenceDuration = realTime() - startTime; + slowParseLimit = referenceDuration*10; + } + + if (recoverySuccessLimit == 0) { + recoverySuccessLimit = size(input)/4; + } println(); println("Single char deletions:"); @@ -214,4 +221,39 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { } else { throw "Cannot find top sort in "; } -} \ No newline at end of file +} + +TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize) { + int count = 0; + + int slowParseLimit = 200; + int recoverySuccessLimit = 100; + + TestStats totalStats = testStats(slowParseLimit, recoverySuccessLimit); + + println("Batch testing in directory "); + for (entry <- listEntries(dir)) { + loc file = dir + entry; + if (isFile(file)) { + if (endsWith(file.path, ext)) { + str content = readFile(file); + if (size(content) <= maxFileSize) { + TestStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, slowParseLimit = slowParseLimit, recoverySuccessLimit = recoverySuccessLimit); + mergeStats(totalStats, fileStats); + count += 1; + } + } + } else if (isDirectory(file)) { + TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); + totalStats = mergeStats(totalStats, dirStats); + count += size(dirStats.measurements); + } + + if (count > maxFiles) { + return totalStats; + } + } + + return totalStats; +} + From 9bbc1e04bb9658149665854ed3d23cdef26fe55c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Sat, 28 Sep 2024 19:22:07 +0200 Subject: [PATCH 11/20] Fixed issue with infinite recursion during parsing --- .../recovery/ErrorRecoveryBenchmark.rsc | 10 +++++++++- .../concrete/recovery/RecoveryTestSupport.rsc | 20 ++++++++++++------- .../uptr/recovery/ToTokenRecoverer.java | 14 ++++++++++--- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index 89a0f47793a..ebf66e0a26b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -2,6 +2,9 @@ module lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import IO; +import util::Benchmark; + void runTestC() { testRecoveryC(); } void runTestDiff() { testRecoveryDiff(); } void runTestDot() { testRecoveryDot(); } @@ -29,5 +32,10 @@ void runLanguageTests() { } void runRascalBatchTest() { - batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 3, 4096); + int startTime = realTime(); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 100, 4096); + int duration = realTime() - startTime; + println("================================================================"); + println("Rascal batch test done in seconds, total result:"); + printStats(stats); } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 9598a87a579..0bab795ef8e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -12,7 +12,7 @@ import util::Math; import lang::rascal::grammar::definition::Modules; public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); -public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); +public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int filesTested=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { int startTime = 0; @@ -73,6 +73,7 @@ TestStats updateStats(TestStats stats, TestMeasurement measurement) { TestStats mergeStats(TestStats stats1, TestStats stats2) { return testStats( stats1.slowParseLimit, stats1.recoverySuccessLimit, + filesTested = stats1.filesTested + stats2.filesTested, successfulParses = stats1.successfulParses + stats2.successfulParses, successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, @@ -139,10 +140,14 @@ void printStats(TestStats stats) { println("Statistics (average/median/95th percentile):"); void printStats(str label, list[int] values, str unit) { - int mean = sum(values)/size(values); print(left(label, 40)); print(": "); - println("// "); + if (values == []) { + print("-/-/- "); + } else { + int mean = sum(values)/size(values); + println("// "); + } } list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; @@ -210,10 +215,10 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu TestStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); printStats(deleteUntilEolStats); totalStats = mergeStats(totalStats, deleteUntilEolStats); + totalStats.filesTested = 1; println(); - println("Overall stats"); - print("-------------"); + println("Cumulative stats for "); printStats(totalStats); println(); @@ -236,17 +241,18 @@ TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int m loc file = dir + entry; if (isFile(file)) { if (endsWith(file.path, ext)) { + println("Testing file ( left)"); str content = readFile(file); if (size(content) <= maxFileSize) { TestStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, slowParseLimit = slowParseLimit, recoverySuccessLimit = recoverySuccessLimit); - mergeStats(totalStats, fileStats); + totalStats = mergeStats(totalStats, fileStats); count += 1; } } } else if (isDirectory(file)) { TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); totalStats = mergeStats(totalStats, dirStats); - count += size(dirStats.measurements); + count += dirStats.filesTested; } if (count > maxFiles) { diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index cfcf8637ad4..4ec262c54da 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -52,6 +52,8 @@ public class ToTokenRecoverer implements IRecoverer { private IdDispenser stackNodeIdDispenser; private ExpectsProvider expectsProvider; + private Set processedNodes = new HashSet<>(); + public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, IdDispenser stackNodeIdDispenser) { this.uri = uri; this.expectsProvider = expectsProvider; @@ -67,7 +69,6 @@ public DoubleArrayList, AbstractNode> reviveStac // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to // improve error recovery. - ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); @@ -346,6 +347,14 @@ private DoubleArrayList, AbstractNode> reviveFai new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { + AbstractStackNode failedNode = failedNodes.get(i); + + // Protect against endless loop + long id = (long) failedNode.getId() << 32 | failedNode.getStartLocation(); + if (!processedNodes.add(id)) { + continue; + } + findRecoveryNodes(failedNodes.get(i), recoveryNodes); } @@ -368,7 +377,7 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes) { for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { @@ -377,7 +386,6 @@ private static void collectUnmatchableMidProductionNodes(int location, AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference // updates of the static version - // Merge the information on the predecessors into the failed node. for (int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); From 080c4f6cb67e5cc038299648f37fa95873a9c358 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Sun, 29 Sep 2024 07:37:46 +0200 Subject: [PATCH 12/20] Readded missing line --- .../lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index 311ee832f5f..23b3af5d2f5 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -42,5 +42,6 @@ test bool axc() { } test bool ax() { + Tree t = parseS("a x $"); return getErrorText(findBestError(t)) == "x "; } From 781a26058efa38dd9f26ece9d583321d97444722 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 30 Sep 2024 08:51:02 +0200 Subject: [PATCH 13/20] Improved benchmarking, both in speed and stat quality --- .../recovery/ErrorRecoveryBenchmark.rsc | 17 +- .../concrete/recovery/RecoveryTestSupport.rsc | 287 ++++++++++++------ 2 files changed, 203 insertions(+), 101 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index ebf66e0a26b..cf7f25f1525 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -13,13 +13,13 @@ void runTestJson() { testRecoveryJson(); } void runTestPico() { testRecoveryPico(); } void runTestRascal() { testRecoveryRascal(); } -TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); -TestStats testRecoveryDiff() = testErrorRecovery(|std:///lang/diff/unified/UnifiedDiff.rsc|, "DiffFile", |std:///lang/diff/unified/examples/example.diff|); -TestStats testRecoveryDot() = testErrorRecovery(|std:///lang/dot/syntax/Dot.rsc|, "DOT", |std:///lang/dot/examples/parser-state.dot|); -TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); -TestStats testRecoveryJson() = testErrorRecovery(|std:///lang/json/syntax/JSON.rsc|, "JSONText", |std:///lang/json/examples/ex01.json|); -TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); -TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); +FileStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +FileStats testRecoveryDiff() = testErrorRecovery(|std:///lang/diff/unified/UnifiedDiff.rsc|, "DiffFile", |std:///lang/diff/unified/examples/example.diff|); +FileStats testRecoveryDot() = testErrorRecovery(|std:///lang/dot/syntax/Dot.rsc|, "DOT", |std:///lang/dot/examples/parser-state.dot|); +FileStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +FileStats testRecoveryJson() = testErrorRecovery(|std:///lang/json/syntax/JSON.rsc|, "JSONText", |std:///lang/json/examples/ex01.json|); +FileStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +FileStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); void runLanguageTests() { testRecoveryC(); @@ -33,8 +33,9 @@ void runLanguageTests() { void runRascalBatchTest() { int startTime = realTime(); - TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 100, 4096); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 1000, 10000); int duration = realTime() - startTime; + println(); println("================================================================"); println("Rascal batch test done in seconds, total result:"); printStats(stats); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 0bab795ef8e..68dfda9118c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -8,11 +8,17 @@ import util::Benchmark; import Grammar; import analysis::statistics::Descriptive; import util::Math; +import Set; +import List; import lang::rascal::grammar::definition::Modules; +alias FrequencyTable = map[int val, int count]; + public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); -public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int filesTested=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); +public data FileStats = fileStats(int totalParses = 0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, FrequencyTable parseTimeRatios=()); + +public data TestStats = testStats(int filesTested=0, int testCount=0, FrequencyTable successfulParsePct=(), FrequencyTable successfulRecoveryPct=(), FrequencyTable failedRecoveryPct=(), FrequencyTable parseErrorPct=(), FrequencyTable slowParsePct=(), FrequencyTable parseTimeRatios=()); private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { int startTime = 0; @@ -40,65 +46,122 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser return measurement; } -TestStats updateStats(TestStats stats, TestMeasurement measurement) { +FileStats updateStats(FileStats stats, TestMeasurement measurement, int referenceParseTime, int recoverySuccessLimit) { + stats.totalParses += 1; + + int ratio = measurement.duration/referenceParseTime; + int parseTimeRatio = ratio == 0 ? 0 : round(log2(measurement.duration/referenceParseTime)); + switch (measurement) { case successfulParse(): { print("."); stats.successfulParses += 1; } - case recovered(errorSize=errorSize): - if (errorSize <= stats.recoverySuccessLimit) { + case recovered(errorSize=errorSize): { + stats.parseTimeRatios = increment(stats.parseTimeRatios, parseTimeRatio); + if (errorSize <= recoverySuccessLimit) { print("+"); stats.successfulRecoveries += 1; } else { print("-"); stats.failedRecoveries += 1; } + } case parseError(): { + stats.parseTimeRatios = increment(stats.parseTimeRatios, parseTimeRatio); print("?"); stats.parseErrors += 1; } } - if (measurement.duration > stats.slowParseLimit) { + if (measurement.duration > referenceParseTime*10) { print("!"); stats.slowParses += 1; } - stats.measurements = stats.measurements + measurement; - return stats; } -TestStats mergeStats(TestStats stats1, TestStats stats2) { - return testStats( - stats1.slowParseLimit, stats1.recoverySuccessLimit, - filesTested = stats1.filesTested + stats2.filesTested, +FileStats mergeFileStats(FileStats stats1, FileStats stats2) { + return fileStats( + totalParses = stats1.totalParses + stats2.totalParses, successfulParses = stats1.successfulParses + stats2.successfulParses, successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, parseErrors = stats1.parseErrors + stats2.parseErrors, slowParses = stats1.slowParses + stats2.slowParses, - measurements = stats1.measurements + stats2.measurements); + parseTimeRatios = mergeFrequencyTables(stats1.parseTimeRatios, stats2.parseTimeRatios) + ); +} + +FrequencyTable increment(FrequencyTable frequencyTable, int val) { + if (val in frequencyTable) { + frequencyTable[val] += 1; + } else { + frequencyTable[val] = 1; + } + + return frequencyTable; } -TestStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { - TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); +TestStats consolidateStats(TestStats cumulativeStats, FileStats fileStats) { + int totalFailed = fileStats.totalParses - fileStats.successfulParses; + + cumulativeStats.successfulParsePct = increment(cumulativeStats.successfulParsePct, percentage(fileStats.successfulParses, fileStats.totalParses)); + cumulativeStats.successfulRecoveryPct = increment(cumulativeStats.successfulRecoveryPct, percentage(fileStats.successfulRecoveries, totalFailed)); + cumulativeStats.failedRecoveryPct = increment(cumulativeStats.failedRecoveryPct, percentage(fileStats.failedRecoveries, totalFailed)); + cumulativeStats.parseErrorPct = increment(cumulativeStats.parseErrorPct, percentage(fileStats.parseErrors, totalFailed)); + cumulativeStats.slowParsePct = increment(cumulativeStats.slowParsePct, percentage(fileStats.slowParses, totalFailed)); + cumulativeStats.parseTimeRatios = mergeFrequencyTables(cumulativeStats.parseTimeRatios, fileStats.parseTimeRatios); + + cumulativeStats.filesTested += 1; + cumulativeStats.testCount += fileStats.totalParses; + + return cumulativeStats; +} + +map[int,int] mergeFrequencyTables(map[int,int] hist1, map[int,int] hist2) { + for (int pct <- hist2) { + if (pct in hist1) { + hist1[pct] += hist2[pct]; + } else { + hist1[pct] = hist2[pct]; + } + } + + return hist1; +} + +TestStats mergeStats(TestStats stats, TestStats stats2) { + stats.filesTested += stats2.filesTested; + stats.testCount += stats2.testCount; + stats.successfulParsePct = mergeFrequencyTables(stats.successfulParsePct, stats2.successfulParsePct); + stats.successfulRecoveryPct = mergeFrequencyTables(stats.successfulRecoveryPct, stats2.successfulRecoveryPct); + stats.failedRecoveryPct = mergeFrequencyTables(stats.failedRecoveryPct, stats2.failedRecoveryPct); + stats.parseErrorPct = mergeFrequencyTables(stats.parseErrorPct, stats2.parseErrorPct); + stats.slowParsePct = mergeFrequencyTables(stats.slowParsePct, stats2.slowParsePct); + stats.parseTimeRatios = mergeFrequencyTables(stats.parseTimeRatios, stats2.parseTimeRatios); + + return stats; +} + +FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { + FileStats stats = fileStats(); int len = size(input); int i = 0; while (i < len) { str modifiedInput = substring(input, 0, i) + substring(input, i+1); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); - stats = updateStats(stats, measurement); + stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); i = i+1; } return stats; } -TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { - TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); +FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { + FileStats stats = fileStats(); int lineStart = 0; list[int] lineEndings = findAll(input, "\n"); @@ -107,7 +170,7 @@ TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va for (int pos <- [lineStart..lineEnd]) { modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); - stats = updateStats(stats, measurement); + stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); } lineStart = lineEnd+1; } @@ -116,55 +179,101 @@ TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va } private int percentage(int number, int total) { - return (100*number)/total; + return total == 0 ? 0 : (100*number)/total; } -void printStats(TestStats stats) { - println(); - int measurementCount = size(stats.measurements); - println("Total parses: "); - println("Succesful parses: (% of total)"); - int totalFailed = measurementCount - stats.successfulParses; - println("Succesful recoveries: (% of failed)"); - println("Failed recoveries: (% of failed)"); - println("Parse errors: (% of failed)"); - - if (stats.slowParses == 0) { - println("No slow parses."); - } else { - slowest = (getFirstFrom(stats.measurements) | it.duration < e.duration ? it : e | e <- stats.measurements); - println(" slow parses, slowest parse: ( ms)"); +int statLabelWidth = 40; +int statFieldWidth = 7; + + +void printFileStats(FileStats fileStats) { + void printStat(str label, int stat, int total, bool printPct=true) { + int pct = total == 0 ? 0 : stat*100/total; + print(left(label + ":", statLabelWidth)); + str pctStr = printPct ? " ( %)" : ""; + println(left("", statFieldWidth)); } println(); - println("Statistics (average/median/95th percentile):"); + printStat("Total parses", fileStats.totalParses, fileStats.totalParses); + printStat("Successful parses", fileStats.successfulParses, fileStats.totalParses); + int failedParses = fileStats.totalParses - fileStats.successfulParses; + printStat("Successful recoveries", fileStats.successfulRecoveries, failedParses); + printStat("Failed recoveries", fileStats.failedRecoveries, failedParses); + printStat("Parse errors", fileStats.parseErrors, failedParses); + printStat("Slow parses", fileStats.slowParses, failedParses); + printFrequencyTableHeader(); + printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2/%"); +} - void printStats(str label, list[int] values, str unit) { - print(left(label, 40)); - print(": "); - if (values == []) { - print("-/-/- "); - } else { - int mean = sum(values)/size(values); - println("// "); +void printFrequencyTableHeader() { + print(left("", statLabelWidth+1)); + print(right("mean", statFieldWidth)); + print(right("median", statFieldWidth)); + print(right("95 %", statFieldWidth)); + print(right("min", statFieldWidth)); + println(right("max", statFieldWidth)); +} + +void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit = "%") { + print(left(label + " ():", statLabelWidth)); + + int totalCount = (0 | it+frequencyTable[val] | val <- frequencyTable); + + int total = 0; + int median = 0; + int medianCount = 0; + int cumulativeCount = 0; + int ninetyFivePercentileLimit = round(totalCount * 0.95); + int ninetyFivePercentile = -1; + int minVal = 1000000000; + int maxVal = -1000000000; + + for (val <- sort(toList(frequencyTable.val))) { + minVal = min(minVal, val); + maxVal = max(maxVal, val); + + int count = frequencyTable[val]; + cumulativeCount += count; + + if (ninetyFivePercentile == -1 && cumulativeCount >= ninetyFivePercentileLimit) { + ninetyFivePercentile = val; } - } - list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; - list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; - list[int] failedRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize > stats.recoverySuccessLimit ]; - list[int] parseErrorTimes = [ duration | parseError(duration=duration) <- stats.measurements ]; + total += val*count; + + if (count > medianCount) { + medianCount = count; + median = val; + } + } - printStats("Succesful parse time", successfulParseTimes, "ms"); - printStats("Succesful recovery time", successfulRecoveryTimes, "ms"); - printStats("Failed recovery time", failedRecoveryTimes, "ms"); - printStats("Parse error time", parseErrorTimes, "ms"); + if (totalCount == 0) { + print("-"); + } else { + int mean = total/totalCount; + print(right("", statFieldWidth)); + print(right("", statFieldWidth)); + print(right("", statFieldWidth)); + print(right("", statFieldWidth)); + println(right("", statFieldWidth)); + } +} - list[int] errorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements ]; - printStats("Recovery error size", errorSizes, "characters"); +void printStats(TestStats stats) { + if (stats.filesTested != 1) { + println("Files tested: "); + } + println("Total parses: "); + printFrequencyTableHeader(); + printFrequencyTableStats("Succesful parses", stats.successfulParsePct); + printFrequencyTableStats("Succesful recoveries", stats.successfulRecoveryPct); + printFrequencyTableStats("Failed recoveries", stats.failedRecoveryPct); + printFrequencyTableStats("Parse errors", stats.parseErrorPct); + printFrequencyTableStats("Slow parses", stats.slowParsePct); + printFrequencyTableStats("Parse time ratios", stats.parseTimeRatios, unit = "log2/%"); - list[int] successfulErrorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; - printStats("Successful recovery size", successfulErrorSizes, "characters"); + println(); } private str syntaxLocToModuleName(loc syntaxFile) { @@ -178,51 +287,41 @@ loc zippedFile(str zip, str path) { return zipFile + path; } -TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = -testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); +FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); -TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input, int slowParseLimit=0, int recoverySuccessLimit=0) { +FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { - println("=========================================================================="); println("Error recovery of () on :"); type[value] begin = type(sym, gram.rules); standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); - if (slowParseLimit == 0) { - int startTime = realTime(); - standardParser(input, testInput); - int referenceDuration = realTime() - startTime; - slowParseLimit = referenceDuration*10; - } + int startTime = realTime(); + standardParser(input, testInput); + int referenceParseTime = realTime() - startTime; - if (recoverySuccessLimit == 0) { - recoverySuccessLimit = size(input)/4; - } + recoverySuccessLimit = size(input)/4; println(); println("Single char deletions:"); - TestStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); - printStats(singleCharDeletionStats); - TestStats totalStats = singleCharDeletionStats; + FileStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); + printFileStats(singleCharDeletionStats); println(); println("Deletes until end-of-line:"); - TestStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); - printStats(deleteUntilEolStats); - totalStats = mergeStats(totalStats, deleteUntilEolStats); - totalStats.filesTested = 1; + FileStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); + printFileStats(deleteUntilEolStats); + FileStats stats = mergeFileStats(singleCharDeletionStats, deleteUntilEolStats); println(); - println("Cumulative stats for "); - printStats(totalStats); - println(); - - return totalStats; + println("-----------------------------------------------------------"); + println("Total test stats for :"); + printFileStats(stats); + return stats; } else { throw "Cannot find top sort in "; } @@ -231,35 +330,37 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize) { int count = 0; - int slowParseLimit = 200; - int recoverySuccessLimit = 100; - - TestStats totalStats = testStats(slowParseLimit, recoverySuccessLimit); + TestStats cumulativeStats = testStats(); - println("Batch testing in directory "); + println("Batch testing in directory (maxFiles=, maxFileSize=)"); for (entry <- listEntries(dir)) { loc file = dir + entry; if (isFile(file)) { if (endsWith(file.path, ext)) { - println("Testing file ( left)"); str content = readFile(file); if (size(content) <= maxFileSize) { - TestStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, slowParseLimit = slowParseLimit, recoverySuccessLimit = recoverySuccessLimit); - totalStats = mergeStats(totalStats, fileStats); + println("========================================================================"); + println("Testing file ( left)"); + FileStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content); + cumulativeStats = consolidateStats(cumulativeStats, fileStats); + println(); + println("------------------------------------------------------------------------"); + println("Cumulative stats after testing :"); + printStats(cumulativeStats); count += 1; } } } else if (isDirectory(file)) { TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); - totalStats = mergeStats(totalStats, dirStats); + cumulativeStats = mergeStats(cumulativeStats, dirStats); count += dirStats.filesTested; } - if (count > maxFiles) { - return totalStats; + if (count >= maxFiles) { + break; } } - return totalStats; + return cumulativeStats; } From f95fba1e5445f380377f53eba353edb2f24b7781 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 30 Sep 2024 13:09:58 +0200 Subject: [PATCH 14/20] Fixed cumulative stats display --- .../recovery/ErrorRecoveryBenchmark.rsc | 22 ++++++++++++++++--- .../concrete/recovery/RecoveryTestSupport.rsc | 21 ++++++------------ 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index cf7f25f1525..6642b08d605 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -4,6 +4,8 @@ import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; import IO; import util::Benchmark; +import String; +import List; void runTestC() { testRecoveryC(); } void runTestDiff() { testRecoveryDiff(); } @@ -31,12 +33,26 @@ void runLanguageTests() { testRecoveryRascal(); } -void runRascalBatchTest() { +void runRascalBatchTest(int maxFiles=1000, int maxFileSize=4000) { int startTime = realTime(); - TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 1000, 10000); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, maxFileSize); int duration = realTime() - startTime; println(); - println("================================================================"); + println("========================im========================================"); println("Rascal batch test done in seconds, total result:"); printStats(stats); +} + +int main(list[str] args) { + int maxFiles = 1000; + int maxFileSize = 4000; + if (size(args) == 2) { + maxFiles = toInt(args[0]); + maxFileSize = toInt(args[1]); + } else if (size(args) != 0) { + println("Usage: ErrorRecoveryBenchmark "); + } + + runRascalBatchTest(maxFiles=maxFiles, maxFileSize=maxFileSize); + return 0; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 68dfda9118c..456124096d1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -190,7 +190,7 @@ void printFileStats(FileStats fileStats) { void printStat(str label, int stat, int total, bool printPct=true) { int pct = total == 0 ? 0 : stat*100/total; print(left(label + ":", statLabelWidth)); - str pctStr = printPct ? " ( %)" : ""; + str pctStr = printPct ? " (%)" : ""; println(left("", statFieldWidth)); } @@ -203,14 +203,14 @@ void printFileStats(FileStats fileStats) { printStat("Parse errors", fileStats.parseErrors, failedParses); printStat("Slow parses", fileStats.slowParses, failedParses); printFrequencyTableHeader(); - printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2/%"); + printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2(ratio)"); } void printFrequencyTableHeader() { print(left("", statLabelWidth+1)); print(right("mean", statFieldWidth)); print(right("median", statFieldWidth)); - print(right("95 %", statFieldWidth)); + print(right("95%", statFieldWidth)); print(right("min", statFieldWidth)); println(right("max", statFieldWidth)); } @@ -327,11 +327,7 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu } } -TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize) { - int count = 0; - - TestStats cumulativeStats = testStats(); - +TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize, TestStats cumulativeStats=testStats()) { println("Batch testing in directory (maxFiles=, maxFileSize=)"); for (entry <- listEntries(dir)) { loc file = dir + entry; @@ -340,23 +336,20 @@ TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int m str content = readFile(file); if (size(content) <= maxFileSize) { println("========================================================================"); - println("Testing file ( left)"); + println("Testing file # ( of left)"); FileStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content); cumulativeStats = consolidateStats(cumulativeStats, fileStats); println(); println("------------------------------------------------------------------------"); println("Cumulative stats after testing :"); printStats(cumulativeStats); - count += 1; } } } else if (isDirectory(file)) { - TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); - cumulativeStats = mergeStats(cumulativeStats, dirStats); - count += dirStats.filesTested; + cumulativeStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles, maxFileSize, cumulativeStats=cumulativeStats); } - if (count >= maxFiles) { + if (cumulativeStats.filesTested >= maxFiles) { break; } } From b9d991b92d9425548d16743f960c97005fb03186 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 30 Sep 2024 13:14:48 +0200 Subject: [PATCH 15/20] Fixed division-by-zero when reference parse time happens to be 0 --- .../rascal/tests/concrete/recovery/RecoveryTestSupport.rsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 456124096d1..998bb842013 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -49,8 +49,8 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser FileStats updateStats(FileStats stats, TestMeasurement measurement, int referenceParseTime, int recoverySuccessLimit) { stats.totalParses += 1; - int ratio = measurement.duration/referenceParseTime; - int parseTimeRatio = ratio == 0 ? 0 : round(log2(measurement.duration/referenceParseTime)); + int ratio = referenceParseTime == 0 ? measurement.duration : measurement.duration/referenceParseTime; + int parseTimeRatio = ratio == 0 ? 0 : round(log2(ratio)); switch (measurement) { case successfulParse(): { From 88a3647925fc284581da211aaa9030c6310d9937 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 09:47:20 +0200 Subject: [PATCH 16/20] Added bug tests and some support for selective testing --- .gitattributes | 1 + .../concrete/recovery/RecoveryTestSupport.rsc | 17 ++++++-- .../recovery/bugs/InfiniteLoop2Bug.rsc | 14 +++++++ .../recovery/bugs/InfiniteLoopBug.rsc | 20 ++++++++++ .../recovery/bugs/InfiniteLoopInput.txt | 3 ++ .../concrete/recovery/bugs/OvertakeBug.rsc | 11 ++++++ .../recovery/bugs/OvertakeBugInput.txt | 39 +++++++++++++++++++ 7 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 .gitattributes create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..2ce24899f27 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.dot diff=-astextplain diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 998bb842013..261b47aef75 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -33,6 +33,7 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser startTime = realTime(); try { Tree t = recoveryParser(input, source); + rprintln(t); duration = realTime() - startTime; Tree best = findBestError(t); errorSize = size(getErrorText(best)); @@ -160,14 +161,23 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & return stats; } -FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { +FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1) { FileStats stats = fileStats(); - int lineStart = 0; + int lineStart = begin; list[int] lineEndings = findAll(input, "\n"); for (int lineEnd <- lineEndings) { lineLength = lineEnd - lineStart; for (int pos <- [lineStart..lineEnd]) { + + // Check boundaries (only used for quick bug testing) + if (end != -1 && end < pos) { + return stats; + } + if (pos < begin) { + continue; + } + modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); @@ -295,7 +305,6 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { - println("Error recovery of () on :"); type[value] begin = type(sym, gram.rules); standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); @@ -306,6 +315,8 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu recoverySuccessLimit = size(input)/4; + println("Error recovery of () on , reference parse time: ms."); + println(); println("Single char deletions:"); FileStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc new file mode 100644 index 00000000000..4beae613ec8 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc @@ -0,0 +1,14 @@ +module lang::rascal::tests::concrete::recovery::bugs::InfiniteLoop2Bug + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testBug() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///lang/rascal/grammar/tests/TestGrammars.rsc|); + testDeleteUntilEol(standardParser, recoveryParser, input, 200, 100, begin=278, end=278); +} + diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc new file mode 100644 index 00000000000..8710b829d53 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc @@ -0,0 +1,20 @@ +module lang::rascal::tests::concrete::recovery::bugs::InfiniteLoopBug + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testInfiniteLoop1() { + str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt|); + p = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + println("starting parse"); + p(input, |unknown:///?visualize=false|); +} + +void testInfiniteLoop2() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///analysis/m3/FlowGraph.rsc|); + testDeleteUntilEol(standardParser, recoveryParser, input, 200, 100); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt new file mode 100644 index 00000000000..26359b3e558 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt @@ -0,0 +1,3 @@ +module M +alias G = rel[loc, set[E +data P = t() | f(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc new file mode 100644 index 00000000000..2880cef3a39 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc @@ -0,0 +1,11 @@ +module lang::rascal::tests::concrete::recovery::bugs::OvertakeBug + +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +bool testOvertakeBug() { + str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt|); + parser(#Module, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=false|); + return true; +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt new file mode 100644 index 00000000000..a6821039bc7 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt @@ -0,0 +1,39 @@ +module analysis::diff::edits::ExecuteTextEdits + +extend analysis::diff::edits::TextEdits; +import IO; +import String; +import List; + +void executeDocumentEdits(list[DocumentEdit] edits) { + for (e <- edits) { + executeDocumentEdit(e); + } +} + +void executeDocumentEdit(removed(loc f)) { + remove(f.top; +} + +void executeDocumentEdit(created(loc f)) { + writeFile(f, ""); +} + +void executeDocumentEdit(renamed(loc from, loc to)) { + move(from.top, to.top, overwrite=true); +} + +void executeDocumentEdit(changed(loc file, list[TextEdit] edits)) { + assert isSorted(edits, less=bool (TextEdit e1, TextEdit e2) { + return e1.range.offset < e2.range.offset; + }); + + str content = readFile(file); + + for (replace(loc range, str repl) <- reverse(edits)) { + assert range.top == file.top; + content = ""; + } + + writeFile(file.top, content); +} \ No newline at end of file From b17dd426d2002e883ff9c034fc36f333153bd256 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 09:56:44 +0200 Subject: [PATCH 17/20] Removed unnecessary changes --- .../library/lang/dot/examples/parse-state.dot | 46 ------------------- .../uptr/recovery/ToTokenRecoverer.java | 2 +- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/dot/examples/parse-state.dot diff --git a/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot deleted file mode 100644 index bda91c2557e..00000000000 --- a/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot +++ /dev/null @@ -1,46 +0,0 @@ -digraph Parser { -"Parser"["label"="Parser\nInput: \"void f(){if(1){}}\"\nLocation: 0 ('v')\nStep 2: Reducing terminals"]; -"todo-1"["label"="<0> 0", "shape"="record"]; -"-2"["label"="Epsilon: \n.0@0 ,matchable,end\n?\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; -"124"["label"="SeparatedList: 124\n.0@0 ,expandable,end\n124\nin: 'default -> tags'"]; -"12858"["label"="NonTerminal: Tags\n.0@0 \nTags\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; -"-1"["label"="NonTerminal: FunctionDeclaration\n.0@-1 \nFunctionDeclaration"]; -"12858" -> "-1"; -"124" -> "12858"; -"-2" -> "124"; -"todo-1":"0":sw -> "-2"["label"="Stack"]; -"46484886"["shape"="octagon", "label"="Epsilon"]; -"todo-1":"0":se -> "46484886"["label"="Node"]; -"todoLists":"1" -> "todo-1"; -"todoLists"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8 | <9> 9 | <10> 10 | <11> 11 | <12> 12 | <13> 13 | <14> 14 | <15> 15", "shape"="record"]; -"Parser" -> "todoLists"["label"="todo lists"]; -"stacksToExpand"["label"="", "shape"="record"]; -"Parser" -> "stacksToExpand"["label"="stacks to expand"]; -"terminalsToReduce"["label"="<0> 0", "shape"="record", "color"="red"]; -"terminalsToReduce":"0":sw -> "-2"["label"="Stack"]; -"terminalsToReduce":"0":se -> "46484886"["label"="Node"]; -"Parser" -> "terminalsToReduce"["label"="terminals to reduce"]; -"nonTerminalsToReduce"["label"="", "shape"="record"]; -"Parser" -> "nonTerminalsToReduce"["label"="non-terminals to reduce"]; -"122"["label"="NonTerminal: Tag\n.0@0 ,end\nTag\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; -"122" -> "124"; -"unexpandableNodes":"0" -> "122"; -"unexpandableNodes"["label"="<0> 0", "shape"="record"]; -"12824"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"FunctionDeclaration\")' ':' 12828 0"]; -"unmatchableLeafNodes":"0" -> "12824"; -"128"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Tags\")' ':' 132 0"]; -"unmatchableLeafNodes":"1" -> "128"; -"2043"["label"="Literal: \n.0@-1 ,matchable\n'@'\nin: '@' Name '=' Expression"]; -"unmatchableLeafNodes":"2" -> "2043"; -"2065"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"Tag\"))' ':' 2069 0"]; -"unmatchableLeafNodes":"3" -> "2065"; -"unmatchableLeafNodes"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3", "shape"="record"]; -"unmatchableMidProductionNodes"["shape"="record", "label"=""]; -"filteredNodes"["label"="", "shape"="record"]; -"error"["label"="Errors"]; -"Parser" -> "error"["label"="error tracking"]; -"error" -> "unexpandableNodes"["label"="unexpandable"]; -"error" -> "unmatchableLeafNodes"["label"="unmatchable leafs"]; -"error" -> "unmatchableMidProductionNodes"["label"="unmatchable mid-prod"]; -"error" -> "filteredNodes"["label"="filtered"]; -} diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 4ec262c54da..eb213f6dc5a 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -377,7 +377,7 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes) { for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { From 6103116aeb7823c9fd706524fb94693964c257ae Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 09:58:28 +0200 Subject: [PATCH 18/20] Removed unnecessary changes --- src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index eb213f6dc5a..98b5e14490f 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -69,6 +69,7 @@ public DoubleArrayList, AbstractNode> reviveStac // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to // improve error recovery. + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); @@ -386,6 +387,7 @@ private static void collectUnmatchableMidProductionNodes(int location, AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference // updates of the static version + // Merge the information on the predecessors into the failed node. for (int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); From 2b00b16e580dc783ecb34d0e6180e081d5205e01 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 10:35:00 +0200 Subject: [PATCH 19/20] Printing a newline after each line of input that is processed --- .../rascal/tests/concrete/recovery/NOTES.md | 1132 +++++++++++------ .../concrete/recovery/RecoveryTestSupport.rsc | 5 +- 2 files changed, 782 insertions(+), 355 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md index 5eb4723935e..a4c00877185 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md @@ -22,392 +22,816 @@ Note that a benchmark test is documented by a string of characters including: - '+': After modification error recovery is able to recover by skipping a reasonable amount of input - '-': After modification error recovery is able to recover by skipping an excessive amount of input (>25% of the contents of the file) - '?': After modification parsing results in an unrecoverable parse error +- A newline is printed after each line of the input file The rest of the output should (hopefully) be self-descriptive: ``` +$ /usr/bin/env C:\\Program\ Files\\Eclipse\ Adoptium\\jdk-11.0.14.101-hotspot\\bin\\java.exe -agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=localhost:58567 @C:\\Users\\pieter\\AppData\\Local\\Temp\\cp_89hnnok9nfsf1vtykxy3jiv5n.argfile org.rascalmpl.shell.RascalShell +Windows console mode could not be enabled, errno=1 +Version: Rascal version not specified in META-INF/MANIFEST.MF??? +INFO: detected |lib://rascal| at |file:///D:/rascal/rascal/target/classes/| rascal>import lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark; +☑ Loading modules 🕗 0:00:03.047 +ok rascal>runLanguageTests(); -========================================================================== -Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|: -☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator Globa 🕛 0:00:03.468 +☑ Loading modules 🕐 0:00:02.331 +☑ Generating parser; RealLiteral Tags ModuleActuals Renamings KeywordFormal MidPathChars OptionalExpression Expression Class PathPart Signature ShellCommand Catch DataTarget KeywordFormals Renaming Tag Type LocationLiteral Declaration FunctionBody Field Range OptionalComma PrePathChars StringConstant Variable Type 🕛 0:00:12.763 +☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator GlobalDeclaration Comment Parameter Parameter 🕛 0:00:03.927 +Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|, reference parse time: 47 ms. Single char deletions: -...........+................++....................+..........+.+.........++......+++.........+....++.........++.+++..-............?....................+++.+......................................................................?....++.-.....+.++.?.................++++....+.++++.?...........+.+.....?+....?................+....+?..?.. -Total parses: 333 -Succesful parses: 281 (84 % of total) -Succesful recoveries: 42 (80 % of failed) -Failed recoveries: 2 (3 % of failed) -Parse errors: 8 (15 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 3/2/8 ms -Failed recovery time : 1/2/2 ms -Parse error time : 1/1/2 ms -Recovery error size : 17/5/83 characters -Successful recovery size : 13/4/50 characters +..........+................+-. +. +................+..........+.+ +........++......+++ +........+....++ +........++.+++ +.- +. +.........?....................+++.+........................................................ +.............? +...++.-.....+.++.? +................++++ +...+.++++.? +..........+.+.....?+ +...? +. +.............+....+? +.? +. +Total parses: 314 (100%) +Successful parses: 262 (83%) +Successful recoveries: 41 (78%) +Failed recoveries: 3 (5%) +Parse errors: 8 (15%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -...........++++++++++++++++++...++++++++++++++++++++++?----++.........++++++++++....+++++++++++....++++++++++.-..?++??++++???????????????????????++............................................................+++++?????.????+++++++++++??......++++++++++++++....??+++??......+++++++??????+.???.....????++++++??????.?. -Total parses: 314 -Succesful parses: 119 (37 % of total) -Succesful recoveries: 128 (65 % of failed) -Failed recoveries: 5 (2 % of failed) -Parse errors: 62 (31 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 2/2/9 ms -Failed recovery time : 2/2/3 ms -Parse error time : 1/1/2 ms -Recovery error size : 38/36/82 characters -Successful recovery size : 36/35/82 characters - -Overall stats -------------- -Total parses: 647 -Succesful parses: 400 (61 % of total) -Succesful recoveries: 170 (68 % of failed) -Failed recoveries: 7 (2 % of failed) -Parse errors: 70 (28 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/9 ms -Failed recovery time : 1/2/3 ms -Parse error time : 1/1/2 ms -Recovery error size : 33/27/83 characters -Successful recovery size : 30/26/82 characters - -========================================================================== -Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|: + +..........+++++++++++++++++-. + +++++++++++++++++++++++?----++ +........++++++++++ +...+++++++++++ +...++++++++++ +- + +?++??++++???????????????????????++........................................................ +...+++++????? +????+++++++++++?? +.....++++++++++++++ +...??+++?? +.....+++++++??????+ +??? + +...?????+++++?????? +? + +Total parses: 295 (100%) +Successful parses: 100 (33%) +Successful recoveries: 126 (64%) +Failed recoveries: 6 (3%) +Parse errors: 63 (32%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/c90/examples/hello-world.c|: + +Total parses: 609 (100%) +Successful parses: 362 (59%) +Successful recoveries: 167 (67%) +Failed recoveries: 9 (3%) +Parse errors: 71 (28%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; Range Content Month Date DiffFile Diff Hours Decimal TimeZone DateTime Sign ChunkStart Year Day Line Minutes Seconds Chunk Time Header Indicator FileName 🕛 0:00:01.032 +Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|, reference parse time: 20 ms. Single char deletions: -++++....................................................+++++....................................................++++++.++++.++++.+...................................+.+.....................+.........................+..................................+......................+.----...---...----...........................................................-....................................................-.-.........................................................................................-..........................................................................................................-..........................................................................................................-...............................................................................................................................................................................................................................................................................................................................-.-...........................-..-.................................................????....???......???..................................................................................?...?............................................................................................................................................................................................?.................................................?...?.?..............................................................................?.................................................?..................................................................................................................................?.........................................?...?.?.? -Total parses: 1776 -Succesful parses: 1702 (95 % of total) -Succesful recoveries: 30 (40 % of failed) -Failed recoveries: 21 (28 % of failed) -Parse errors: 23 (31 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 1/1/2 ms -Failed recovery time : 5/6/9 ms -Parse error time : 0/0/1 ms -Recovery error size : 386/161/837 characters -Successful recovery size : 96/96/161 characters +++++.................................................... ++++++.................................................... +++++++.++++.++++ +.+.................................. +.+ +.+.................... +.+........................ +.+................................. +.+..................... +.+ +.----...---...----.......................................................... +.-................................................... +.- +.-........................................................................................ +.-......................................................................................................... +.-......................................................................................................... +.-.............................................................................................................................................................................................................................................................................................................................. +.- +.-.......................... +.-. +.-................................................ +.????....???......???................................................................................. +.?.. +.? +............... +......................................................... +.................................................... +....... +........................................................ +.?................................................ +.?.. +.? +.?............................................................................. +.?................................................ +.?................................................................................................................................. +.?........................................ +.?.. +.? +.?. +? +Total parses: 1776 (100%) +Successful parses: 1702 (95%) +Successful recoveries: 30 (40%) +Failed recoveries: 21 (28%) +Parse errors: 23 (31%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -++++....................................................++++....................................................++++++++++++++++..................................++....................+........................+.................................+.....................+-----!-!-!-----!-!----..........................................................-...................................................--........................................................................................-.........................................................................................................-.........................................................................................................+..............................................................................................................................................................................................................................................................................................................................--..........................-.-................................................????????????????????.................................................................................?..??.............?.......................................................?..................................................?.....?......................................................?................................................?..??.............................................................................?................................................?.................................................................................................................................?........................................?..??. -Total parses: 1737 -Succesful parses: 1643 (94 % of total) -Succesful recoveries: 31 (32 % of failed) -Failed recoveries: 26 (27 % of failed) -Parse errors: 37 (39 % of failed) -5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 1/2/2 ms -Failed recovery time : 6/6/17 ms -Parse error time : 1/1/2 ms -Recovery error size : 398/160/777 characters -Successful recovery size : 102/147/161 characters - -Overall stats -------------- -Total parses: 3513 -Succesful parses: 3345 (95 % of total) -Succesful recoveries: 61 (36 % of failed) -Failed recoveries: 47 (27 % of failed) -Parse errors: 60 (35 % of failed) -5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 1/1/2 ms -Failed recovery time : 6/6/13 ms -Parse error time : 0/1/2 ms -Recovery error size : 392/161/837 characters -Successful recovery size : 99/109/161 characters - -========================================================================== -Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|: -☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.307 +++++.................................................... +++++.................................................... ++++++++++++++++ ++.................................. ++ ++.................... ++........................ ++................................. ++..................... ++ +-----------------.......................................................... +-................................................... +- +-........................................................................................ +-......................................................................................................... +-......................................................................................................... ++.............................................................................................................................................................................................................................................................................................................................. +- +-.......................... +-. +-................................................ +????????????????????................................................................................. +?.. +? +?............. +?....................................................... +?.................................................. +?..... +?...................................................... +?................................................ +?.. +? +?............................................................................. +?................................................ +?................................................................................................................................. +?........................................ +?.. +? +?. + +Total parses: 1737 (100%) +Successful parses: 1643 (94%) +Successful recoveries: 31 (32%) +Failed recoveries: 26 (27%) +Parse errors: 37 (39%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/diff/unified/examples/example.diff|: + +Total parses: 3513 (100%) +Successful parses: 3345 (95%) +Successful recoveries: 61 (36%) +Failed recoveries: 47 (27%) +Parse errors: 60 (35%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.529 +Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|, reference parse time: 25 ms. Single char deletions: -++??????.......?..+......+-+.....+++...............+..................+................................................++...+......--+.....+++.....+..+.....+++......++...+..+-+.....+++............................................+.......+.............................+.......+.....++...-....--+.....+++..........................................................................+.......+....++...+.....+-+.....+++.................................................................................................................++...+..+++.....+++.............................................................++...+.....+.++.+..+...+....+.++.+.....+...+..+.++.+....+...+......+.+.+.++.+..+++.....+++.....++...+........+++.....+++.......+..+.....+++.......++...+......+.+.+....++.+........+++.....+++....++...+.........+.+.+.++.+......+...+.........+++.....+++.........................................................................................................................................+..+.....+++......++...+......+.++.+.........+++.....+++..........++...+..............+++.....++++..+.....+++......++...+......+.++.+..............+++.....+++................++...+.................+++.....+++.....+..+.....+++......+..+.....+++...++...+.................+.+.+....++.+..+++.....+++.....++...+.................+.+.+....++.+........+-+.....+++....++...+......+.++.+.................+-+.....+++...................++...+....................+-+.....++++..+.....+++......++...+......+.-+.+....................+-+.....+++.......................++...-...--+.....+++............................................+....+...................................+....+............+...........+......++...-...--+.....+++....................................................................++...+.....+++.....+++.............................................................................................++...+.....+.++.+..-...-...-.++.-.....-...-...-.++.-...-...-.................-.-.-.-+.-...-...-.....--+.....+++.................................................................++...-....-++.....+++.................................................+.......+.............................+.......+.....++...+....+.++.+....+...+.....+.++.+....+...+.................+.+.+.++.+.....+...+.................+-+.....+++.............+..+.....+++......++...+.....+++.....+++.........................................+....................+...............++...+....................+.+.+.++.+.....+...+...+++.....+++.........................................+.....+.............++...+....................+.+.+.++.+...+...+....+++.....+++.............................................................++...+....................+.+.+.++.+....+...+....+++.....+++....................................................+....+...............++...+....................+.+.+.-+.+....+...+.....+++.....+++..............................................................................................................................................................................................................................................................................................++...+....................+.+.+.++.+.....+...+.....+++.....+++....................................................+.......+................++...+....................+.+.+.++.+.....+...+....+++.....+++.....................................................++...+....................+.+.+.++.+....+...+....+++.....+++................................................++...+....................+.+.+.++.+....+...+....+++.....+++.........................................+........+..............++...+....................+.+.+.+?.+....+...+....................+++.....+++.....................................................................+..+.....+++......++...?.............................?++.....+++......+..+.....+++++...?.............?++.....++++..+.....+++......++...?.....?++.....+++......++...?......?.+?.?.....?++.....+++..............++...?.....?.+?.?.................?++.....+++............++...?.....?.+?.?....................?++.....+++.................++...?.....?.+?.?.............................?++.....+++....................++...?.....?.+?.?.............?++.....++?........??...?.. -Total parses: 4239 -Succesful parses: 3555 (83 % of total) -Succesful recoveries: 599 (87 % of failed) -Failed recoveries: 42 (6 % of failed) -Parse errors: 43 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 5/5/8 ms -Succesful recovery time : 9/9/17 ms -Failed recovery time : 10/10/17 ms -Parse error time : 3/3/6 ms -Recovery error size : 233/44/2220 characters -Successful recovery size : 65/38/241 characters +++??????.......? +.+......+-+.....+++...............+..................+................................................++. +.+......--+.....+++.....+..+.....+++......++. +.+..+-+.....+++............................................+.......+.............................+.......+.....++. +.-....--+.....+++..........................................................................+.......+....++. +.+.....+-+.....+++.................................................................................................................++. +.+..+++.....+++.............................................................++. +.+.....+.++.+..+. +.+....+.++.+.....+. +.+..+.++.+....+. +.+......+.+.+....++.+..+++.....+++.....++. +.+........+++.....+++.......+..+.....+++.......++. +.+......+.+.+....++.+........+++.....+++....++. +.+.........+.+.+.++.+......+. +.+.........+++.....+++.........................................................................................................................................+..+.....+++......++. +.+......+.++.+.........+++.....+++..........++. +.+..............+++.....++++..+.....+++......++. +.+......+.++.+..............+++.....+++................++. +.+.................+++.....+++.....+..+.....+++......+..+.....+++...++. +.+.................+.+.+....++.+..+++.....+++.....++. +.+.................+.+.+....++.+........+-+.....+++....++. +.+......+.++.+.................+-+.....+++...................++. +.+....................+-+.....++++..+.....+++......++. +.+......+.-+.+....................+-+.....+++.......................++. +.-...--+.....+++............................................+....+...................................+....+............+...........+......++. +.-...--+.....+++....................................................................++. +.+.....+++.....+++.............................................................................................++. +.+.....+.++.+..-. +.-...-.++.-.....-. +.-...-.++.-...-. +.-.................-.-.-.-+.-...-. +.-.....--+.....+++.................................................................++. +.-....-++.....+++.................................................+.......+.............................+.......+.....++. +.+....+.++.+....+. +.+.....+.++.+....+. +.+.................+.+.+.++.+.....+. +.+.................+-+.....+++.............+..+.....+++......++. +.+.....+++.....+++.........................................+....................+...............++. +.+....................+.+.+.++.+.....+. +.+...+++.....+++.........................................+.....+.............++. +.+....................+.+.+.++.+...+. +.+....+++.....+++.............................................................++. +.+....................+.+.+.++.+....+. +.+....+++.....+++....................................................+....+...............++. +.+....................+.+.+.-+.+....+. +.+.....+++.....+++..............................................................................................................................................................................................................................................................................................++. +.+....................+.+.+.++.+.....+. +.+.....+++.....+++....................................................+.......+................++. +.+....................+.+.+.++.+.....+. +.+....+++.....+++.....................................................++. +.+....................+.+.+.++.+....+. +.+....+++.....+++................................................++. +.+....................+.+.+.++.+....+. +.+....+++.....+++.........................................+........+..............++. +.+....................+.+.+.+?.+....+. +.+....................+++.....+++.....................................................................+..+.....+++......++. +.?.............................?++.....+++......+..+.....+++++. +.?.............?++.....++++..+.....+++......++. +.?.....?++.....+++......++. +.?......?.+?.?.....?++.....+++..............++. +.?.....?.+?.?.................?++.....+++............++. +.?.....?.+?.?....................?++.....+++.................++. +.?.....?.+?.?.............................?++.....+++....................++. +.?.....?.+?.?.............?++.....++?........??. +.? +. +Total parses: 4177 (100%) +Successful parses: 3493 (83%) +Successful recoveries: 599 (87%) +Failed recoveries: 42 (6%) +Parse errors: 43 (6%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -????????????????..-------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++.++++++++++++++++++++++++++++++++++...---.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++..+..+++...+++++..+..++++++...+++..+..+++++...+++++++..++..+..+++.++++++++++++++++...+++++++++.+++++++++++++++++++++++++++++++++++++...+++++++..++.+...+..+++++++++.+++++++++++++++...++++++++++..++..+..+++++++...++++++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++..+..++++++++++.+++++++++++++++++++++...+++++++++++++++.+++++++++++++++++++++++++++++...+++++++..+..+++++++++++++++.+++++++++++++++++++++++++++...++++++++++++++++++.+++++++++++++++++++++++++++++++++++++++++++++++++...++++++++++++++++++.+++.+...+..+++.++++++++++++++++...++++++++++++++++++..++.....+..+++++++++.+++++++++++++++...+++++++..+..++++++++++++++++++.++++++++++++++++++++++++++++++...+++++++++++++++++++++.+++++++++++++++++++++++++++++...-------..+..---------------------.++++++++++++++++++++++++++++++++++...----.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...------..+..---...----..+..------...----..+..----...------------------..--..+..----...------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++..+..+++++...++++++..+..+++++...++++++++++++++++++..++..+..++++++...++++++++++++++++++.++++++++++++++++++++++++++++++++++++++++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..?..+++++...?????????????????????.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...??????????????????????????????.+++++++++++++++++++++++++++++...??????????????.+++++++++++++++++++++++++++++...??????.+++++++++++++++++...???????..?..??????.+++++++++++++++++++++++++...??????..?..??????????????????.+++++++++++++++++++++++...??????..?..?????????????????????.++++++++++++++++++++++++++++...??????..?..??????????????????????????????.+++++++++++++++++++++++++++++++...??????..?????????????????.???????????????????..?. -Total parses: 4174 -Succesful parses: 402 (9 % of total) -Succesful recoveries: 3438 (91 % of failed) -Failed recoveries: 99 (2 % of failed) -Parse errors: 235 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 6/6/13 ms -Succesful recovery time : 9/8/18 ms -Failed recovery time : 11/10/25 ms -Parse error time : 4/4/7 ms -Recovery error size : 150/63/391 characters -Successful recovery size : 81/61/297 characters - -Overall stats -------------- -Total parses: 8413 -Succesful parses: 3957 (47 % of total) -Succesful recoveries: 4037 (90 % of failed) -Failed recoveries: 141 (3 % of failed) -Parse errors: 278 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 5/5/9 ms -Succesful recovery time : 9/8/18 ms -Failed recovery time : 11/10/19 ms -Parse error time : 3/4/7 ms -Recovery error size : 163/61/408 characters -Successful recovery size : 79/57/289 characters - -========================================================================== -Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///C:/Users/pieter/Documents/projects/wk/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|: -☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumer 🕛 0:00:08.222 +???????????????? +.-------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++.++++++++++++++++++++++++++++++++++. +.---.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++..+..+++. +.+++++..+..++++++. +.+++..+..+++++. +.+++++++..++.....+..+++.++++++++++++++++. +.+++++++++.+++++++++++++++++++++++++++++++++++++. +.+++++++..++.+...+..+++++++++.+++++++++++++++. +.++++++++++..++..+..+++++++. +.++++++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++..+..++++++++++.+++++++++++++++++++++. +.+++++++++++++++.+++++++++++++++++++++++++++++. +.+++++++..+..+++++++++++++++.+++++++++++++++++++++++++++. +.++++++++++++++++++.+++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++++++++++++++.+++.+...+..+++.++++++++++++++++. +.++++++++++++++++++..++.....+..+++++++++.+++++++++++++++. +.+++++++..+..++++++++++++++++++.++++++++++++++++++++++++++++++. +.+++++++++++++++++++++.+++++++++++++++++++++++++++++. +.-------..+..---------------------.++++++++++++++++++++++++++++++++++. +.----.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.------..+..---. +.----..+..------. +.----..+..----. +.------------------..--..+..----. +.------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++..+..+++++. +.++++++..+..+++++. +.++++++++++++++++++..++..+..++++++. +.++++++++++++++++++.++++++++++++++++++++++++++++++++++++++++++. +.++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++++. +.++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++++. +.++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.+++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..?..+++++. +.?????????????????????.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.??????????????????????????????.+++++++++++++++++++++++++++++. +.??????????????.+++++++++++++++++++++++++++++. +.??????.+++++++++++++++++. +.???????..?..??????.+++++++++++++++++++++++++. +.??????..?..??????????????????.+++++++++++++++++++++++. +.??????..?..?????????????????????.++++++++++++++++++++++++++++. +.??????..?..??????????????????????????????.+++++++++++++++++++++++++++++++. +.??????..?????????????????.???????????????????. +? + +Total parses: 4112 (100%) +Successful parses: 340 (8%) +Successful recoveries: 3438 (91%) +Failed recoveries: 99 (2%) +Parse errors: 235 (6%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 1 + +----------------------------------------------------------- +Total test stats for |std:///lang/dot/examples/parser-state.dot|: + +Total parses: 8289 (100%) +Successful parses: 3833 (46%) +Successful recoveries: 4037 (90%) +Failed recoveries: 141 (3%) +Parse errors: 278 (6%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 1 +☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumeral AbstractMethodMod StringLiteral Field 🕛 0:00:07.668 +Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|, reference parse time: 91 ms. Single char deletions: -++++++++......-..++++++.++++++..........+++++++++.......-...++++++...........-....+....+....+........+.-........+....+.........++..+...+..........++++++.....................++.+...++++++.....+..?.?. -Total parses: 198 -Succesful parses: 131 (66 % of total) -Succesful recoveries: 61 (91 % of failed) -Failed recoveries: 4 (5 % of failed) -Parse errors: 2 (2 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/6 ms -Failed recovery time : 2/2/4 ms -Parse error time : 0/1/1 ms -Recovery error size : 18/14/69 characters -Successful recovery size : 14/13/31 characters +++++++++......- +. +.++++++.++++++..........+++++++++.......- +. +..++++++...........-....+....+....+........+.- +........+....+.........++ +..+ +. +..+........ +..++++++.....................++.+ +...++++++.....+ +..? +.? +. +Total parses: 198 (100%) +Successful parses: 131 (66%) +Successful recoveries: 61 (91%) +Failed recoveries: 4 (5%) +Parse errors: 2 (2%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -.++++++------------------------------------------------++++++++++++++++++++-------------------------...+++++++++++++++++++++++..+.......++++++++++++++++++++++++++++++++...+++++++++++??? -Total parses: 185 -Succesful parses: 16 (8 % of total) -Succesful recoveries: 93 (55 % of failed) -Failed recoveries: 73 (43 % of failed) -Parse errors: 3 (1 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/5 ms -Failed recovery time : 1/2/3 ms -Parse error time : 0/1/1 ms -Recovery error size : 48/42/112 characters -Successful recovery size : 18/11/45 characters - -Overall stats -------------- -Total parses: 383 -Succesful parses: 147 (38 % of total) -Succesful recoveries: 154 (65 % of failed) -Failed recoveries: 77 (32 % of failed) -Parse errors: 5 (2 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/6 ms -Failed recovery time : 1/2/4 ms -Parse error time : 0/1/1 ms -Recovery error size : 39/21/112 characters -Successful recovery size : 16/12/42 characters - -========================================================================== -Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|: -☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.619 +.++++++-------- + +---------------------------------------- + +++++++++++++++++++++------------------------- +...+++++++++++++++++++++ +++ + +..+....... +++++++++++++++++++++++++++++++++ +...+++++++++++ +?? +? + +Total parses: 185 (100%) +Successful parses: 16 (8%) +Successful recoveries: 93 (55%) +Failed recoveries: 73 (43%) +Parse errors: 3 (1%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|: + +Total parses: 383 (100%) +Successful parses: 147 (38%) +Successful recoveries: 154 (65%) +Failed recoveries: 77 (32%) +Parse errors: 5 (2%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.691 +Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|, reference parse time: 12 ms. Single char deletions: -?........?.....??.?............+.....++..+..?............+......++.+..?............+.....++..+....................??............?.........??.?................+...++....+......................................++................+......-+....+................-.....+-..+...-............?+............?...??.?...+....+....+......?........?..? -Total parses: 337 -Succesful parses: 285 (84 % of total) -Succesful recoveries: 27 (51 % of failed) -Failed recoveries: 4 (7 % of failed) -Parse errors: 21 (40 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/0/1 ms -Succesful recovery time : 0/1/1 ms -Failed recovery time : 0/0/1 ms -Parse error time : 0/1/1 ms -Recovery error size : 34/21/124 characters -Successful recovery size : 21/16/57 characters +?. +.......?.....??.?. +...........+.....++..+..?. +...........+......++.+..?. +...........+.....++..+....................??. +...........?.........??.?. +...............+...++....+......................................++. +...............+......-+....+. +...............-.....+-..+...-. +...........?+. +...........?...??.?...+....+....+......?. +.......?. +.? +Total parses: 337 (100%) +Successful parses: 285 (84%) +Successful recoveries: 27 (51%) +Failed recoveries: 4 (7%) +Parse errors: 21 (40%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -?.????????????????............+++++++??????............++++++++?????............????????????????????????????????.????????????????????????................++++++++++++++++++++++++++++++++++++++++++++++++++................--------+++++.-----------------------------.-----------+.??????????????????+????????????????????.???????. -Total parses: 324 -Succesful parses: 75 (23 % of total) -Succesful recoveries: 72 (28 % of failed) -Failed recoveries: 48 (19 % of failed) -Parse errors: 129 (51 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/0/1 ms -Succesful recovery time : 0/1/2 ms -Failed recovery time : 0/1/1 ms -Parse error time : 0/0/1 ms -Recovery error size : 86/57/294 characters -Successful recovery size : 42/57/57 characters - -Overall stats -------------- -Total parses: 661 -Succesful parses: 360 (54 % of total) -Succesful recoveries: 99 (32 % of failed) -Failed recoveries: 52 (17 % of failed) -Parse errors: 150 (49 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/0/1 ms -Succesful recovery time : 0/1/2 ms -Failed recovery time : 0/1/1 ms -Parse error time : 0/0/1 ms -Recovery error size : 76/57/293 characters -Successful recovery size : 37/39/57 characters - -========================================================================== -Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|: -☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.719 +?. +????????????????. +...........+++++++??????. +...........++++++++?????. +...........????????????????????????????????. +????????????????????????. +...............++++++++++++++++++++++++++++++++++++++++++++++++++. +...............--------+++++. +-----------------------------. +-----------+. +??????????????????+????????????????????. +???????. + +Total parses: 324 (100%) +Successful parses: 75 (23%) +Successful recoveries: 72 (28%) +Failed recoveries: 48 (19%) +Parse errors: 129 (51%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/json/examples/ex01.json|: + +Total parses: 661 (100%) +Successful parses: 360 (54%) +Successful recoveries: 99 (32%) +Failed recoveries: 52 (17%) +Parse errors: 150 (49%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.660 +Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|, reference parse time: 9 ms. Single char deletions: -?????.-------.......+.+++++++-.........................+.++++++++............................+.++++++++....................+.++++++++................+..+.+++++++................+..+.------+..............++...+...............++.+-........-----.......+.-.--.................++.......+..................++......-............+++++.......+.+.++......................++........+....+.....................++.......+.+............+++..................--.......+.+........--..???.. -Total parses: 472 -Succesful parses: 355 (75 % of total) -Succesful recoveries: 81 (69 % of failed) -Failed recoveries: 28 (23 % of failed) -Parse errors: 8 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/1 ms -Succesful recovery time : 2/3/5 ms -Failed recovery time : 1/2/4 ms -Parse error time : 0/0/1 ms -Recovery error size : 87/10/463 characters -Successful recovery size : 16/6/70 characters +?????.-------.......+.+++++++-.. +......................+.++++++++...... +.....................+.++++++++ +...................+.++++++++ +...............+..+.+++++++ +...............+..+.------+ +.............++...+ +..............++.+- +.......-----.......+.-.--. +...............++.......+ +.................++......- +...........+++++.......+.+.++ +.....................++........+....+ +....................++.......+.+ +...........+++ +.................??.......+.+ +.......-- +.??? +. +Total parses: 454 (100%) +Successful parses: 337 (74%) +Successful recoveries: 81 (69%) +Failed recoveries: 26 (22%) +Parse errors: 10 (8%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -?????--------..++++++++++++++-..................++++++++++++++++......................+++++++++++++++................+++++++++++++................+++++++++++.-------------------------+........+++++++++++........-----------.+++++++-----------------.............+++++++++++++............+++++++-------.++++++++++++++++++++++++++++...............++++++++++++++++++++++.+++++++++++++++++++++++......++.------------+.------------------++......++.+++++++-.???. -Total parses: 454 -Succesful parses: 151 (33 % of total) -Succesful recoveries: 195 (64 % of failed) -Failed recoveries: 100 (33 % of failed) -Parse errors: 8 (2 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/1 ms -Succesful recovery time : 2/2/4 ms -Failed recovery time : 1/1/4 ms -Parse error time : 0/0/1 ms -Recovery error size : 101/41/268 characters -Successful recovery size : 28/26/70 characters - -Overall stats -------------- -Total parses: 926 -Succesful parses: 506 (54 % of total) -Succesful recoveries: 276 (65 % of failed) -Failed recoveries: 128 (30 % of failed) -Parse errors: 16 (3 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/1 ms -Succesful recovery time : 2/2/5 ms -Failed recovery time : 1/1/4 ms -Parse error time : 0/0/1 ms -Recovery error size : 97/33/272 characters -Successful recovery size : 25/15/70 characters - -========================================================================== -Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|: +?????--------..++++++++++++++-.. +...............++++++++++++++++...... +...............+++++++++++++++ +...............+++++++++++++ +...............+++++++++++ +-------------------------+ +.......+++++++++++ +.......----------- ++++++++-----------------. +...........+++++++++++++ +...........+++++++------- +++++++++++++++++++++++++++++ +..............++++++++++++++++++++++ ++++++++++++++++++++++++......++ +------------+ +??????????????????++......++ ++++++++- +??? + +Total parses: 436 (100%) +Successful parses: 133 (30%) +Successful recoveries: 195 (64%) +Failed recoveries: 82 (27%) +Parse errors: 26 (8%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/pico/examples/fac.pico|: + +Total parses: 890 (100%) +Successful parses: 470 (52%) +Successful recoveries: 276 (65%) +Failed recoveries: 108 (25%) +Parse errors: 36 (8%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|, reference parse time: 20 ms. Single char deletions: -?........+...............................................................................................?.?...........+......................................................................................................................................................................................................................................................................................................................................................................................................................?.?........+....................................................................................................................................................................................................................................................................................................?.?........+..................................................................................................?.?.!......!....!..!.!.!????????..!.!.+!+......++...++.............??????+....++..........+.??????+...++......+.??????+....++......++.......++..........++.......+.??????+....++......+++......++......+.??????+.........+.??????+....++..........+.??????+....++...........+.+++++++..+.+++++++........++......++.....-..+........+..............................................................................................?.....?...........?.....................+..................+.?................+!+.......+++............++.............................++.+..+........?......................................................................................?.....?...........?.....................+..................+.?.....++...............................................................................................+.....................+................+.........++++................+!.................................++.+..+........?.................................................................................................?.....?...........?.....................+..................-!.?.....+.+................+.........+?..........++...........................................................................+....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++!++++++.........+.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+.....++............................................................................................++...........................................................................................................................................+......+.........+.........+........+.+.........++.+....+.++.+......+....++.....................+.........++++..............++++++.+.......+...++.....+......+++++++....+.........+......++.+.+.......+...++............................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++.+..+........+........................................................................................?.+++++.............+.............+........+...+............+.+.+++........+...+...............+.+++........+...+....+.......+.+.+++........+...+....+.......+.+.+++........+...+....+.......+.++.+?..?........?..........................................................................................?.............?...............?....+...+.....+.?..............+.+.....+...+.+...++..............++.................+.+...............+.+...+............++.................?.............+...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++.................+.+.+...........+..........+.....++..........................+..................+++..+..................+.+..+..................++.......................+.?..?........?.............................................................?.............?...............?........?.?.....+++.?..................?............+....++..........+....+.....+.....+.+.................+.++..........++++++.............+...................+.+.+....++...................+.+.+....+.+.....+...................+.+.+....+.+.....+.....................+.++....+.....++.........+?.....?.....++++++..........+.+.+.............................++?.? -Total parses: 4314 -Succesful parses: 3815 (88 % of total) -Succesful recoveries: 393 (78 % of failed) -Failed recoveries: 2 (0 % of failed) -Parse errors: 104 (20 % of failed) -14 slow parses, slowest parse: |unknown:///?deleted=1137| (1 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 14/12/20 ms -Succesful recovery time : 29/21/69 ms -Failed recovery time : 183/184/351 ms -Parse error time : 10/5/40 ms -Recovery error size : 65/19/274 characters -Successful recovery size : 59/18/233 characters +?........+...............................................................................................? +.?...........+ +...................................................................................................... +.................................................................................................... +................................................................................................................ +........................................................................................... +.? +.?........+ +.................................................................................................... +............................................................................ +............................................................................................... +.................... +.? +.?........+ +................................................................................................. +.? +.?............... +????????....++......++...++........... +. +.??????+....++..........+ +.??????+...++......+ +.??????+....++......++.......++..........++.......+ +.??????+....++......+++......++......+ +.??????+.........+ +.??????+....++..........+ +.??????+....++...........+ +.+++++++..+ +.+++++++........++......++.....- +. +.+........+..............................................................................................? +.....?...........?.....................+..................+.? +................+!+.......+++............++.............................++ +.+ +. +.+........?......................................................................................? +.....?...........?.....................+..................+.? +.....++..................................................................................... +..........+.....................+................+.........++++ +................+!.................................++ +.+ +. +.+........?.................................................................................................? +.....?...........?.....................+..................-!.? +.....+.+................+.........+? +..... +.....++........................................................................... ++....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ +.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ +.....++....................................................................................... +.....++........................................................................................................................................... ++......+ +. +........+.........+........+.+ +.........++.+....+.++.+......+....?+ +.....................+ +.........++++. +.............++++++.+.......+...++ +.....+ +. +.....+++++++....+.........+......++.+.+.......+...++ +. +...........................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++ +.+ +. +.+........+........................................................................................? +.+++++.............+.............+ +........+...+............+.+.+++ +........+...+...............+.+++ +........+...+....+.......+.+.+++ +........+...+....+.......+.+.+++ +........+...+....+.......+.++ +.+? +. +.?........?..........................................................................................? +.............?...............?....+...+.....+.? +..............+.+.....+...+.+...++..............++ +..... +............+.+...............+.+...+............++ +. +................?.............+ +...............+.+.+.+.......+...+..........++ +...............+.+.+.+.......+...+..........++ +...............+.+.+.+.......+...+..........++ +.................+.+.+...........+..........+ +.....++ +. +.........................+..................+++..+..................+.+..+..................++ +. +......................+ +.? +. +.?........?.............................................................? +.............?...............?........?.? +.....+++.? +..................?............+....++ +. +.........+....+.....+.....+.+.................+.++ +. +.........++++++.............+ +...................+.+.+....++ +...................+.+.+....+.+.....+ +...................+.+.+....+.+.....+ +.....................+.++....+.....++ +.........+? +.....? +.....++++++..........+.+.+. +............................++? +.? +Total parses: 4314 (100%) +Successful parses: 3815 (88%) +Successful recoveries: 392 (78%) +Failed recoveries: 2 (0%) +Parse errors: 105 (21%) +Slow parses: 4 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 1 0 4 Deletes until end-of-line: -.+........?????????????????????????????????????????????????????????????????????????????????????????????????++++++++++++.................................................................................................................................................................................................................................................................................................................................................................................................................??+++++++++...............................................................................................................................................................................................................................................................................................??+++++++++................................................................................................?.?..............????????....+?......+?...+?...........?????++++++++++++++++++.?????+++++++++++++.?????++++++++++++++++++++++++++++++++++++++++++++.?????+++++++++++++++++++++++++++++++.?????+++++++++++.?????++++++++++++++++++.?????+++++++++++++++++++.+++++++++.+++++-------------------------.+........???????????????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++++++++++++++++++++++++.+........???????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+..........................................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++++++++++++++.+........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!??-!-!-!-!-!?????????????-!----!??.........???????????????++++++++++?.........+...........................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+............................................................................................+...........................................................................................................................................+++++++?????----++-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........+++.+++++++++++++?????+++????????+?+++++++.....+++++++++++++++++++++++++++++++..+++++++++++++.....+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????+++.+........?????????????????????????????????????????????????????????????????????????????????????????+++++++++++++++++++++++++++++++++........++++.........+..+..++++.....+++++++++++++++++++++++++++.....++++++++++++++++++++++++++.....++++++++++++++++++++++++++++++++++++++++++++++++++++++??.?........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????+++++++++++??????.......+........?++++++++++++++++++++++++++++++++...........+......????????????????????++++++++++++++++++++++++++++++++++++++++++++++...........+..+..++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....++++++++++++++++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+++++++++++++++++?.?........???????????????????????????????????????????????????????????????????????????????????????????????????????????????.........++++++++++++++++++++++++++++.........+++++++++++++++++++..+++++++++++++++++++++++++++++++++++++++++++++++...............+..+..++?+++++.............++++++++++?++++++++++++.............++++++++++?++++++++++++++++++++++++++++++++++++++++++++++++???????????????+++++++++++++++++++++++++.??????????????++????????????+? -Total parses: 4206 -Succesful parses: 1542 (36 % of total) -Succesful recoveries: 1611 (60 % of failed) -Failed recoveries: 55 (2 % of failed) -Parse errors: 998 (37 % of failed) -64 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 13/13/21 ms -Succesful recovery time : 28/18/66 ms -Failed recovery time : 135/18/319 ms -Parse error time : 7/6/14 ms -Recovery error size : 158/40/998 characters -Successful recovery size : 116/38/561 characters - -Overall stats -------------- -Total parses: 8520 -Succesful parses: 5357 (62 % of total) -Succesful recoveries: 2004 (63 % of failed) -Failed recoveries: 57 (1 % of failed) -Parse errors: 1102 (34 % of failed) -78 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 14/13/20 ms -Succesful recovery time : 28/18/67 ms -Failed recovery time : 137/18/331 ms -Parse error time : 7/6/14 ms -Recovery error size : 140/36/981 characters -Successful recovery size : 105/32/407 characters +.+........???????????????????????????????????????????????????????????????????????????????????????????????? +?++++++++++++ +..................................................................................................... +................................................................................................... +............................................................................................................... +.......................................................................................... +? +?+++++++++ +................................................................................................... +........................................................................... +.............................................................................................. +................... +? +?+++++++++ +................................................................................................ +? +.?.............. +????????....+?......+?...+?.......... + +.?????++++++++++++++++++ +.?????+++++++++++++ +.?????++++++++++++++++++++++++++++++++++++++++++++ +.?????+++++++++++++++++++++++++++++++ +.?????+++++++++++ +.?????++++++++++++++++++ +.?????+++++++++++++++++++ +.+++++++++ +.+++++------------------------- + +.+........??????????????????????????????????????????????????????????????????????????????????????????????? +++++++???????????+++++++++++++++++++++++++++++++++++++++++?? +.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++?+++++++++++++++++++++++++++++++ ++ + +.+........??????????????????????????????????????????????????????????????????????????????????????? +++++++???????????+++++++++++++++++++++++++++++++++++++++++?? +.....+..................................................................................... +.....+++++..++++++++++++++++++++++++++++++++++++++++++++++++++ +.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++ ++ + +.+........?????????????????????????????????????????????????????????????????????????????????????????????????? +??????????????????-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!??-!-!-!-!-!?????????????-!----!?? +.........???????????????++++++++++? +.... +.....+........................................................................... ++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.........++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.....+....................................................................................... +.....+........................................................................................................................................... ++++++++ + +?????----++-+++++++++++++++++ ++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++ +.........+++. ++++++++++++++?????+++????????+?++ ++++++ + +.....+++++++++++++++++++++++++++++++..+++++++++++++ + +.....+!+!+!+!+!+!++!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????++ ++ + +.+........????????????????????????????????????????????????????????????????????????????????????????? ++++++++++++++++++++++++++++++++++ +........++++.........+..+..++++ +.....+++++++++++++++++++++++++++ +.....++++++++++++++++++++++++++ +.....++++++++++++++++++++++++++ +++++++++++++++++++++++++++++ +?? + +.?........??????????????????????????????????????????????????????????????????????????????????????????? +?????????????????????????????+++++++++++?????? +.......+........?++++++++++++++++++++++++++++++++ +.... +.......+......????????????????????++++++++++++++++ + +++++++++++++++++++++++++++++++ +...........+..+..++?+++++++++++++++++++++++++ +.........++++++++++?+++++++++++++++++++++++++ +.........++++++++++?+++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++ +++++++ + +.....++++++++++++++++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.....+++++++++++++++++ +? + +.?........?????????????????????????????????????????????????????????????? +???????????????????????????????????????? +????????? +.........++++++++++++++++++++++++++++ + +.........+++++++++++++++++++..+++++++++++++++++++ + +++++++++++++++++++++++++++++ +...............+..+..++?+++++ +.............++++++++++?++++++++++++ +.............++++++++++?++++++++++++ +++++++++++++++++++++++++++++++++++++ +?????????? +????? ++++++++++++++++++++++++++. +??????????????++????????????+? + +Total parses: 4206 (100%) +Successful parses: 1542 (36%) +Successful recoveries: 1610 (60%) +Failed recoveries: 55 (2%) +Parse errors: 999 (37%) +Slow parses: 63 (2%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 1 0 4 + +----------------------------------------------------------- +Total test stats for |std:///lang/rascal/vis/ImportGraph.rsc|: + +Total parses: 8520 (100%) +Successful parses: 5357 (62%) +Successful recoveries: 2002 (63%) +Failed recoveries: 57 (1%) +Parse errors: 1104 (34%) +Slow parses: 67 (2%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 1 0 4 +ok ``` diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 261b47aef75..87cc9f5765b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -33,7 +33,6 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser startTime = realTime(); try { Tree t = recoveryParser(input, source); - rprintln(t); duration = realTime() - startTime; Tree best = findBestError(t); errorSize = size(getErrorText(best)); @@ -156,6 +155,9 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); i = i+1; + if (i < len && substring(input, i, i+1) == "\n") { + println(); + } } return stats; @@ -183,6 +185,7 @@ FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); } lineStart = lineEnd+1; + println(); } return stats; From e09396476c7613733cf14141993a6595c132c29f Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 11:16:04 +0200 Subject: [PATCH 20/20] Fixed off-by-one error in newline printing --- .../rascal/tests/concrete/recovery/NOTES.md | 541 +++++++++--------- .../concrete/recovery/RecoveryTestSupport.rsc | 2 +- 2 files changed, 271 insertions(+), 272 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md index a4c00877185..f88771a2e4c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md @@ -35,31 +35,30 @@ rascal>import lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark; ☑ Loading modules 🕗 0:00:03.047 ok rascal>runLanguageTests(); -☑ Loading modules 🕐 0:00:02.331 -☑ Generating parser; RealLiteral Tags ModuleActuals Renamings KeywordFormal MidPathChars OptionalExpression Expression Class PathPart Signature ShellCommand Catch DataTarget KeywordFormals Renaming Tag Type LocationLiteral Declaration FunctionBody Field Range OptionalComma PrePathChars StringConstant Variable Type 🕛 0:00:12.763 -☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator GlobalDeclaration Comment Parameter Parameter 🕛 0:00:03.927 -Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|, reference parse time: 47 ms. +☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator GlobalDeclaration Comment Parameter Parameter 🕛 0:00:04.556 +Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|, reference parse time: 42 ms. Single char deletions: -..........+................+-. . -................+..........+.+ -........++......+++ -........+....++ -........++.+++ -.- +.........+................+-.. . -.........?....................+++.+........................................................ -.............? -...++.-.....+.++.? -................++++ -...+.++++.? -..........+.+.....?+ -...? +...............+..........+.+. +.......++......+++. +.......+....++. +.......++.+++. +-. . -.............+....+? -.? +........?....................+++.+......................................................... +............?. +..++.-.....+.++.?. +...............++++. +..+.++++.?. +.........+.+.....?+. +..?. . +............+....+?. +?. + Total parses: 314 (100%) Successful parses: 262 (83%) Successful recoveries: 41 (78%) @@ -110,56 +109,56 @@ Parse errors: 71 (28%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; Range Content Month Date DiffFile Diff Hours Decimal TimeZone DateTime Sign ChunkStart Year Day Line Minutes Seconds Chunk Time Header Indicator FileName 🕛 0:00:01.032 -Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|, reference parse time: 20 ms. +☑ Generating parser; Range Content Month Date DiffFile Diff Hours Decimal TimeZone DateTime Sign ChunkStart Year Day Line Minutes Seconds Chunk Time Header Indicator FileName 🕛 0:00:00.677 +Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|, reference parse time: 10 ms. Single char deletions: -++++.................................................... -+++++.................................................... -++++++.++++.++++ -.+.................................. -.+ -.+.................... -.+........................ -.+................................. -.+..................... -.+ -.----...---...----.......................................................... -.-................................................... -.- -.-........................................................................................ -.-......................................................................................................... -.-......................................................................................................... -.-.............................................................................................................................................................................................................................................................................................................................. -.- -.-.......................... -.-. -.-................................................ -.????....???......???................................................................................. -.?.. -.? +++++....................................................+ +++++....................................................+ ++++++.++++.++++. ++................................... ++. ++..................... ++......................... ++.................................. ++...................... ++. +----...---...----........................................................... +-.................................................... +-. +-......................................................................................... +-.......................................................................................................... +-.......................................................................................................... +-............................................................................................................................................................................................................................................................................................................................... +-. +-........................... +-.. +-................................................. +????....???......???.................................................................................. +?... +?. ............... ......................................................... .................................................... ....... ........................................................ -.?................................................ -.?.. -.? -.?............................................................................. -.?................................................ -.?................................................................................................................................. -.?........................................ -.?.. -.? -.?. -? +?................................................. +?... +?. +?.............................................................................. +?................................................. +?.................................................................................................................................. +?......................................... +?... +?. +?.? + Total parses: 1776 (100%) Successful parses: 1702 (95%) Successful recoveries: 30 (40%) Failed recoveries: 21 (28%) Parse errors: 23 (31%) -Slow parses: 0 (0%) +Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 @@ -211,7 +210,7 @@ Failed recoveries: 26 (27%) Parse errors: 37 (39%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 0 +Parse time ratios (log2(ratio)): 0 0 0 0 1 ----------------------------------------------------------- Total test stats for |std:///lang/diff/unified/examples/example.diff|: @@ -223,77 +222,77 @@ Failed recoveries: 47 (27%) Parse errors: 60 (35%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.529 -Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|, reference parse time: 25 ms. +Parse time ratios (log2(ratio)): 0 0 0 0 1 +☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.346 +Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|, reference parse time: 38 ms. Single char deletions: -++??????.......? -.+......+-+.....+++...............+..................+................................................++. -.+......--+.....+++.....+..+.....+++......++. -.+..+-+.....+++............................................+.......+.............................+.......+.....++. -.-....--+.....+++..........................................................................+.......+....++. -.+.....+-+.....+++.................................................................................................................++. -.+..+++.....+++.............................................................++. -.+.....+.++.+..+. -.+....+.++.+.....+. -.+..+.++.+....+. -.+......+.+.+....++.+..+++.....+++.....++. -.+........+++.....+++.......+..+.....+++.......++. -.+......+.+.+....++.+........+++.....+++....++. -.+.........+.+.+.++.+......+. -.+.........+++.....+++.........................................................................................................................................+..+.....+++......++. -.+......+.++.+.........+++.....+++..........++. -.+..............+++.....++++..+.....+++......++. -.+......+.++.+..............+++.....+++................++. -.+.................+++.....+++.....+..+.....+++......+..+.....+++...++. -.+.................+.+.+....++.+..+++.....+++.....++. -.+.................+.+.+....++.+........+-+.....+++....++. -.+......+.++.+.................+-+.....+++...................++. -.+....................+-+.....++++..+.....+++......++. -.+......+.-+.+....................+-+.....+++.......................++. -.-...--+.....+++............................................+....+...................................+....+............+...........+......++. -.-...--+.....+++....................................................................++. -.+.....+++.....+++.............................................................................................++. -.+.....+.++.+..-. -.-...-.++.-.....-. -.-...-.++.-...-. -.-.................-.-.-.-+.-...-. -.-.....--+.....+++.................................................................++. -.-....-++.....+++.................................................+.......+.............................+.......+.....++. -.+....+.++.+....+. -.+.....+.++.+....+. -.+.................+.+.+.++.+.....+. -.+.................+-+.....+++.............+..+.....+++......++. -.+.....+++.....+++.........................................+....................+...............++. -.+....................+.+.+.++.+.....+. -.+...+++.....+++.........................................+.....+.............++. -.+....................+.+.+.++.+...+. -.+....+++.....+++.............................................................++. -.+....................+.+.+.++.+....+. -.+....+++.....+++....................................................+....+...............++. -.+....................+.+.+.-+.+....+. -.+.....+++.....+++..............................................................................................................................................................................................................................................................................................++. -.+....................+.+.+.++.+.....+. -.+.....+++.....+++....................................................+.......+................++. -.+....................+.+.+.++.+.....+. -.+....+++.....+++.....................................................++. -.+....................+.+.+.++.+....+. -.+....+++.....+++................................................++. -.+....................+.+.+.++.+....+. -.+....+++.....+++.........................................+........+..............++. -.+....................+.+.+.+?.+....+. -.+....................+++.....+++.....................................................................+..+.....+++......++. -.?.............................?++.....+++......+..+.....+++++. -.?.............?++.....++++..+.....+++......++. -.?.....?++.....+++......++. -.?......?.+?.?.....?++.....+++..............++. -.?.....?.+?.?.................?++.....+++............++. -.?.....?.+?.?....................?++.....+++.................++. -.?.....?.+?.?.............................?++.....+++....................++. -.?.....?.+?.?.............?++.....++?........??. -.? -. +++??????.......?. ++......+-+.....+++...............+..................+................................................++.. ++......--+.....+++.....+..+.....+++......++.. ++..+-+.....+++............................................+.......+.............................+.......+.....++.. +-....--+.....+++..........................................................................+.......+....++.. ++.....+-+.....+++.................................................................................................................++.. ++..+++.....+++.............................................................++.. ++.....+.++.+..+.. ++....+.++.+.....+.. ++..+.++.+....+.. ++......+.+.+....++.+..+++.....+++.....++.. ++........+++.....+++.......+..+.....+++.......++.. ++......+.+.+....++.+........+++.....+++....++.. ++.........+.+.+.++.+......+.. ++.........+++.....+++.........................................................................................................................................+..+.....+++......++.. ++......+.++.+.........+++.....+++..........++.. ++..............+++.....++++..+.....+++......++.. ++......+.++.+..............+++.....+++................++.. ++.................+++.....+++.....+..+.....+++......+..+.....+++...++.. ++.................+.+.+....++.+..+++.....+++.....++.. ++.................+.+.+....++.+........+-+.....+++....++.. ++......+.++.+.................+-+.....+++...................++.. ++....................+-+.....++++..+.....+++......++.. ++......+.-+.+....................+-+.....+++.......................++.. +-...--+.....+++............................................+....+...................................+....+............+...........+......++.. +-...--+.....+++....................................................................++.. ++.....+++.....+++.............................................................................................++.. ++.....+.++.+..-.. +-...-.++.-.....-.. +-...-.++.-...-.. +-.................-.-.-.-+.-...-.. +-.....--+.....+++.................................................................++.. +-....-++.....+++.................................................+.......+.............................+.......+.....++.. ++....+.++.+....+.. ++.....+.++.+....+.. ++.................+.+.+.++.+.....+.. ++.................+-+.....+++.............+..+.....+++......++.. ++.....+++.....+++.........................................+....................+...............++.. ++....................+.+.+.++.+.....+.. ++...+++.....+++.........................................+.....+.............++.. ++....................+.+.+.++.+...+.. ++....+++.....+++.............................................................++.. ++....................+.+.+.++.+....+.. ++....+++.....+++....................................................+....+...............++.. ++....................+.+.+.-+.+....+.. ++.....+++.....+++..............................................................................................................................................................................................................................................................................................++.. ++....................+.+.+.++.+.....+.. ++.....+++.....+++....................................................+.......+................++.. ++....................+.+.+.++.+.....+.. ++....+++.....+++.....................................................++.. ++....................+.+.+.++.+....+.. ++....+++.....+++................................................++.. ++....................+.+.+.++.+....+.. ++....+++.....+++.........................................+........+..............++.. ++....................+.+.+.+?.+....+.. ++....................+++.....+++.....................................................................+..+.....+++......++.. +?.............................?++.....+++......+..+.....+++++.. +?.............?++.....++++..+.....+++......++.. +?.....?++.....+++......++.. +?......?.+?.?.....?++.....+++..............++.. +?.....?.+?.?.................?++.....+++............++.. +?.....?.+?.?....................?++.....+++.................++.. +?.....?.+?.?.............................?++.....+++....................++.. +?.....?.+?.?.............?++.....++?........??.. +?. + Total parses: 4177 (100%) Successful parses: 3493 (83%) Successful recoveries: 599 (87%) @@ -377,7 +376,7 @@ Failed recoveries: 99 (2%) Parse errors: 235 (6%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 1 +Parse time ratios (log2(ratio)): 0 0 0 0 0 ----------------------------------------------------------- Total test stats for |std:///lang/dot/examples/parser-state.dot|: @@ -389,25 +388,25 @@ Failed recoveries: 141 (3%) Parse errors: 278 (6%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 1 -☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumeral AbstractMethodMod StringLiteral Field 🕛 0:00:07.668 -Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|, reference parse time: 91 ms. +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumeral AbstractMethodMod StringLiteral Field 🕛 0:00:13.312 +Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|, reference parse time: 118 ms. Single char deletions: -++++++++......- -. -.++++++.++++++..........+++++++++.......- +++++++++......-. . -..++++++...........-....+....+....+........+.- -........+....+.........++ -..+ +++++++.++++++..........+++++++++.......-. . -..+........ -..++++++.....................++.+ -...++++++.....+ -..? -.? +.++++++...........-....+....+....+........+.-. +.......+....+.........++. +.+. . +.+......... +.++++++.....................++.+. +..++++++.....+. +.?. +?. + Total parses: 198 (100%) Successful parses: 131 (66%) Successful recoveries: 61 (91%) @@ -452,23 +451,23 @@ Parse errors: 5 (2%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.691 -Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|, reference parse time: 12 ms. +☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:01.228 +Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|, reference parse time: 17 ms. Single char deletions: -?. -.......?.....??.?. -...........+.....++..+..?. -...........+......++.+..?. -...........+.....++..+....................??. -...........?.........??.?. -...............+...++....+......................................++. -...............+......-+....+. -...............-.....+-..+...-. -...........?+. -...........?...??.?...+....+....+......?. -.......?. -.? +?.. +......?.....??.?.. +..........+.....++..+..?.. +..........+......++.+..?.. +..........+.....++..+....................??.. +..........?.........??.?.. +..............+...++....+......................................++.. +..............+......-+....+.. +..............-.....+-..+...-.. +..........?+.. +..........?...??.?...+....+....+......?.. +......?.. +? Total parses: 337 (100%) Successful parses: 285 (84%) Successful recoveries: 27 (51%) @@ -512,29 +511,29 @@ Parse errors: 150 (49%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.660 -Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|, reference parse time: 9 ms. +☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:01.139 +Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|, reference parse time: 16 ms. Single char deletions: -?????.-------.......+.+++++++-.. -......................+.++++++++...... -.....................+.++++++++ -...................+.++++++++ -...............+..+.+++++++ -...............+..+.------+ -.............++...+ -..............++.+- -.......-----.......+.-.--. -...............++.......+ -.................++......- -...........+++++.......+.+.++ -.....................++........+....+ -....................++.......+.+ -...........+++ -.................??.......+.+ -.......-- -.??? -. +?????.-------.......+.+++++++-... +.....................+.++++++++....... +....................+.++++++++. +..................+.++++++++. +..............+..+.+++++++. +..............+..+.------+. +............++...+. +.............++.+-. +......-----.......+.-.--.. +..............++.......+. +................++......-. +..........+++++.......+.+.++. +....................++........+....+. +...................++.......+.+. +..........+++. +................??.......+.+. +......--. +???. + Total parses: 454 (100%) Successful parses: 337 (74%) Successful recoveries: 81 (69%) @@ -569,7 +568,7 @@ Successful parses: 133 (30%) Successful recoveries: 195 (64%) Failed recoveries: 82 (27%) Parse errors: 26 (8%) -Slow parses: 0 (0%) +Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 @@ -584,125 +583,125 @@ Parse errors: 36 (8%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|, reference parse time: 20 ms. +Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|, reference parse time: 17 ms. Single char deletions: -?........+...............................................................................................? -.?...........+ +?........+...............................................................................................?. +?...........+. ...................................................................................................... .................................................................................................... ................................................................................................................ ........................................................................................... -.? -.?........+ +?. +?........+. .................................................................................................... ............................................................................ ............................................................................................... .................... -.? -.?........+ +?. +?........+. ................................................................................................. -.? -.?............... -????????....++......++...++........... +?. +?...............? +???????....++......++...++............ . -.??????+....++..........+ -.??????+...++......+ -.??????+....++......++.......++..........++.......+ -.??????+....++......+++......++......+ -.??????+.........+ -.??????+....++..........+ -.??????+....++...........+ -.+++++++..+ -.+++++++........++......++.....- +??????+....++..........+. +??????+...++......+. +??????+....++......++.......++..........++.......+. +??????+....++......+++......++......+. +??????+.........+. +??????+....++..........+. +??????+....++...........+. ++++++++..+. ++++++++........++......++.....-. . -.+........+..............................................................................................? -.....?...........?.....................+..................+.? -................+!+.......+++............++.............................++ -.+ ++........+..............................................................................................?. +....?...........?.....................+..................+.?. +...............+!+.......+++............++.............................++!. ++. . -.+........?......................................................................................? -.....?...........?.....................+..................+.? -.....++..................................................................................... -..........+.....................+................+.........++++ -................+!.................................++ -.+ ++........?......................................................................................?. +....?...........?.....................+..................+.?. +....++...................................................................................... +.........+.....................+!................+.........++++. +...............+!.................................++. ++. . -.+........?.................................................................................................? -.....?...........?.....................+..................-!.? -.....+.+................+.........+? ++........?.................................................................................................?. +....?...........?.....................+..................-!.?. +....+!.+................+.........+?. ..... -.....++........................................................................... -+....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ -.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ -.....++....................................................................................... -.....++........................................................................................................................................... -+......+ +....++...........................................................................+ +....+!.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+. +......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+. +....++........................................................................................ +....++...........................................................................................................................................+ +......+. . -........+.........+........+.+ -.........++.+....+.++.+......+....?+ -.....................+ -.........++++. -.............++++++.+.......+...++ -.....+ +.......+.........+........+.+. +........++.+....+.++.+......+....?+. +....................+. +........++++.. +............++++++.+.......+...++. +....+. . -.....+++++++....+.........+......++.+.+.......+...++ +....+++++++....+.........+......++.+.+.......+...++. . -...........................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++ -.+ +..........................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++. ++. . -.+........+........................................................................................? -.+++++.............+.............+ -........+...+............+.+.+++ -........+...+...............+.+++ -........+...+....+.......+.+.+++ -........+...+....+.......+.+.+++ -........+...+....+.......+.++ -.+? ++........+........................................................................................?. ++++++.............+.............+. +.......+...+............+.+.+++. +.......+...+...............+.+++. +.......+...+....+.......+.+.+++. +.......+...+....+.......+.+.+++. +.......+...+....+.......+.++. ++?. . -.?........?..........................................................................................? -.............?...............?....+...+.....+.? -..............+.+.....+...+.+...++..............++ +?........?..........................................................................................?. +............?...............?....+...+.....+.?. +.............+!.+.....+...+.+...++..............++. ..... -............+.+...............+.+...+............++ +...........+.+...............+.+...+............++. . -................?.............+ -...............+.+.+.+.......+...+..........++ -...............+.+.+.+.......+...+..........++ -...............+.+.+.+.......+...+..........++ -.................+.+.+...........+..........+ -.....++ +...............?.............+. +..............+.+.+.+.......+...+..........++. +..............+.+.+.+.......+...+..........++. +..............+.+.+.+.......+...+..........++. +................+.+.+...........+..........+. +....++. . -.........................+..................+++..+..................+.+..+..................++ +........................+..................+++..+..................+.+..+..................++. . -......................+ -.? +.....................+. +?. . -.?........?.............................................................? -.............?...............?........?.? -.....+++.? -..................?............+....++ +?........?.............................................................?. +............?...............?........?.?. +....+++.?. +.................?............+....++. . -.........+....+.....+.....+.+.................+.++ +........+....+.....+.....+.+.................+.++. . -.........++++++.............+ -...................+.+.+....++ -...................+.+.+....+.+.....+ -...................+.+.+....+.+.....+ -.....................+.++....+.....++ -.........+? -.....? -.....++++++..........+.+.+. -............................++? -.? +........++++++.............+. +..................+.+.+....++. +..................+.+.+....+.+.....+. +..................+.+.+....+.+.....+. +....................+.++....+.....++. +........+?. +....?. +....++++++..........+.+.+.. +...........................++?. +? Total parses: 4314 (100%) Successful parses: 3815 (88%) Successful recoveries: 392 (78%) Failed recoveries: 2 (0%) Parse errors: 105 (21%) -Slow parses: 4 (0%) +Slow parses: 9 (1%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 1 0 4 +Parse time ratios (log2(ratio)): 0 0 2 0 5 Deletes until end-of-line: .+........???????????????????????????????????????????????????????????????????????????????????????????????? @@ -736,14 +735,14 @@ Deletes until end-of-line: .+........??????????????????????????????????????????????????????????????????????????????????????????????? ++++++???????????+++++++++++++++++++++++++++++++++++++++++?? -.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++?+++++++++++++++++++++++++++++++ +.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++?+++++++++++++++++++++++++++++++! + .+........??????????????????????????????????????????????????????????????????????????????????????? ++++++???????????+++++++++++++++++++++++++++++++++++++++++?? .....+..................................................................................... .....+++++..++++++++++++++++++++++++++++++++++++++++++++++++++ -.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++ +.....+!+!+!+!+!++!+!+!+!+!+++++++++++++!++++++++++++++++++++++ + .+........?????????????????????????????????????????????????????????????????????????????????????????????????? @@ -766,7 +765,7 @@ Deletes until end-of-line: .....+++++++++++++++++++++++++++++++..+++++++++++++ -.....+!+!+!+!+!+!++!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????++ +.....+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????++ + .+........????????????????????????????????????????????????????????????????????????????????????????? @@ -818,9 +817,9 @@ Successful parses: 1542 (36%) Successful recoveries: 1610 (60%) Failed recoveries: 55 (2%) Parse errors: 999 (37%) -Slow parses: 63 (2%) +Slow parses: 66 (2%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 1 0 4 +Parse time ratios (log2(ratio)): 0 0 2 0 5 ----------------------------------------------------------- Total test stats for |std:///lang/rascal/vis/ImportGraph.rsc|: @@ -830,8 +829,8 @@ Successful parses: 5357 (62%) Successful recoveries: 2002 (63%) Failed recoveries: 57 (1%) Parse errors: 1104 (34%) -Slow parses: 67 (2%) +Slow parses: 75 (2%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 1 0 4 +Parse time ratios (log2(ratio)): 0 0 2 0 5 ok ``` diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 87cc9f5765b..ac73a8f53f9 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -154,10 +154,10 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & str modifiedInput = substring(input, 0, i) + substring(input, i+1); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); - i = i+1; if (i < len && substring(input, i, i+1) == "\n") { println(); } + i = i+1; } return stats;