[GR-47807] Better STOML parsing and code-owners tests.

PullRequest: mx/1854

ci/common.jsonnet

@@ -203,6 +203,12 @@ local common_json = import "../common.json";
       },
     },
 
+    gradle:: {
+      downloads+: {
+        GRADLE_JAVA_HOME: jdks_data["oraclejdk21"],
+      }
+    },
+
     local code_tools = {
       downloads+: if 'jdk_version' in self && self.jdk_version > 21 then {
         TOOLS_JAVA_HOME: jdks_data['oraclejdk21'],
@@ -251,7 +257,7 @@ local common_json = import "../common.json";
       } else {},
     },
 
-    graalpy:: {
+    graalpy:: self.gradle + {
       packages+: if (self.os == "linux") then {
         libffi: '>=3.2.1',
         bzip2: '>=1.0.6',

common.json

@@ -4,11 +4,11 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.36.0",
+  "mx_version": "7.36.1",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {
-    "galahad-jdk":        {"name": "jpg-jdk",   "version": "24",      "build_id": "jdk-24+26-3309", "platformspecific": true, "extrabundles": ["static-libs"]},
+    "galahad-jdk":        {"name": "jpg-jdk",   "version": "24",      "build_id": "jdk-24+27-3513", "platformspecific": true, "extrabundles": ["static-libs"]},
 
     "oraclejdk17":        {"name": "jpg-jdk",   "version": "17.0.7",  "build_id": "jdk-17.0.7+8", "platformspecific": true, "extrabundles": ["static-libs"]},
     "labsjdk-ce-17":      {"name": "labsjdk",   "version": "ce-17.0.7+4-jvmci-23.1-b02", "platformspecific": true },
@@ -45,13 +45,13 @@
 
     "oraclejdk23":       {"name": "jpg-jdk",   "version": "23", "build_id": "jdk-23+37", "platformspecific": true, "extrabundles": ["static-libs"]},
 
-    "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "24",      "build_id": "jdk-24+26", "platformspecific": true, "extrabundles": ["static-libs"]},
-    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-24+26-jvmci-b01", "platformspecific": true },
-    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-24+26-jvmci-b01-debug", "platformspecific": true },
-    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-24+26-jvmci-b01-sulong", "platformspecific": true },
-    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-24+26-jvmci-b01", "platformspecific": true },
-    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-24+26-jvmci-b01-debug", "platformspecific": true },
-    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-24+26-jvmci-b01-sulong", "platformspecific": true }
+    "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "24",      "build_id": "jdk-24+27", "platformspecific": true, "extrabundles": ["static-libs"]},
+    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-24+27-jvmci-b01", "platformspecific": true },
+    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-24+27-jvmci-b01-debug", "platformspecific": true },
+    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-24+27-jvmci-b01-sulong", "platformspecific": true },
+    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-24+27-jvmci-b01", "platformspecific": true },
+    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-24+27-jvmci-b01-debug", "platformspecific": true },
+    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-24+27-jvmci-b01-sulong", "platformspecific": true }
   },
 
   "eclipse": {

src/mx/_impl/mx.py

@@ -18365,7 +18365,7 @@ def alarm_handler(signum, frame):
 _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache'))
 
 # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue
-version = VersionSpec("7.36.2")  # [GR-60385] Catch exceptions communicating with compiler daemon.
+version = VersionSpec("7.36.3")  # [GR-47807] Basic tests for codeowners.
 
 _mx_start_datetime = datetime.utcnow()
 

src/mx/_impl/mx_codeowners.py

@@ -27,6 +27,7 @@
 
 __all__ = [
     "FileOwners",
+    "stoml_parse_rules",
 ]
 
 import argparse
@@ -37,6 +38,9 @@
 from . import mx
 from . import mx_stoml
 
+def stoml_parse_rules(toml_string):
+    return mx_stoml.parse_string(toml_string)
+
 class _TomlParsingException(Exception):
     pass
 
@@ -67,9 +71,7 @@ def _load_toml_from_fd(fd):
     # No other libraries to try, falling back to our simplified parser
     try:
         tree = mx_stoml.parse_fd(fd)
-        return {
-            'rule': tree,
-        }
+        return tree
     except RuntimeError as e:
         mx.log_error(f"Failed at parsing {fd.name} using the in-house parser. You can try again after installing the 'tomllib' or 'toml' package.")
         raise _TomlParsingException(e)

src/mx/_impl/mx_gate.py

@@ -591,6 +591,10 @@ def _run_mx_suite_tests():
     bench_rules_tests.tests()
     java_argument_file_test.tests()
 
+    from tests import code_owners_tests, stoml_tests
+    code_owners_tests.tests()
+    stoml_tests.tests()
+
     from tests import test_maven_projects
     test_maven_projects.tests()
 

src/mx/_impl/mx_stoml.py

@@ -26,6 +26,7 @@
 #
 
 from argparse import ArgumentParser
+import re
 
 def parse_fd(fd, path="<fd>"):
     content = fd.read().decode('utf-8')
@@ -40,6 +41,8 @@ def parse_string(content, path="<toml-string>"):
     return parser.parse(path, content)
 
 class _Streamer:
+    EOF = ""
+
     def __init__(self, path, content):
         self.path = path
         self.content = content
@@ -61,7 +64,16 @@ def terminate(self, message):
     def peek(self, ahead=0):
         if self.pos + ahead < len(self.content):
             return self.content[self.pos + ahead]
-        return ""
+        return _Streamer.EOF
+
+    def peek_to_whitespace(self):
+        token = ""
+        for i in range(len(self.content) - self.pos + 1):
+            next_char = self.peek(i)
+            if (next_char == _Streamer.EOF) or next_char.isspace():
+                break
+            token = token + next_char
+        return token
 
     def pull(self, expected=None):
         if expected is None:
@@ -73,13 +85,30 @@ def pull(self, expected=None):
         self.slurp(len(expected))
 
     def pullSpaces(self):
-        while self.peek().isspace():
-            self.pull()
+        inside_comment = False
+        while True:
+            next_char = self.peek()
+            if next_char == _Streamer.EOF:
+                break
+            if inside_comment:
+                if next_char in ["\r", "\n"]:
+                    inside_comment = False
+                self.pull()
+                continue
+            if next_char.isspace():
+                self.pull()
+                continue
+            if next_char == "#":
+                self.pull()
+                inside_comment = True
+                continue
+            # Some other character and we are not inside a comment
+            break
 
     def slurp(self, count):
         for _ in range(0, count):
             character = self.peek()
-            if character in ("\n", ""):
+            if character in ("\n", _Streamer.EOF):
                 self.lines.append(self.line)
                 self.line = ""
                 self.row = self.row + 1
@@ -90,33 +119,47 @@ def slurp(self, count):
             self.pos = self.pos + 1
 
 class _StomlParser:
+    TABLE_MARKER = re.compile(r"^\[(?P<name>.*)]$")
+    ARRAY_OF_TABLES_MARKER = re.compile(r"^\[\[(?P<name>.*)]]$")
+
     def parse(self, path, content):
-        rules = []
+        tree = {}
         streamer = _Streamer(path, content)
-        self.root(streamer, rules)
-        return rules
+        self.root(streamer, tree)
+        return tree
 
-    def root(self, streamer, rules):
+    def root(self, streamer, tree):
         while True:
-            while streamer.peek().isspace():
-                streamer.pull()
-            if streamer.peek() == "":
+            streamer.pullSpaces()
+            if streamer.peek() == _Streamer.EOF:
                 return
-            streamer.pull("[[rule]]")
-            rule = self.rule(streamer)
-            rules.append(rule)
+            next_token = streamer.peek_to_whitespace()
+
+            is_table = _StomlParser.TABLE_MARKER.match(next_token)
+            is_array_of_tables = _StomlParser.ARRAY_OF_TABLES_MARKER.match(next_token)
+
+            if is_array_of_tables:
+                streamer.pull(next_token)
+                table_name = is_array_of_tables.group("name")
+                if not table_name in tree:
+                    tree[table_name] = []
+                tree[table_name].append(self.parse_table(streamer))
+            elif is_table:
+                streamer.pull(next_token)
+                tree[is_table.group("name")] = self.parse_table(streamer)
+            else:
+                streamer.terminate("Expected table or array of tables.")
 
-    def rule(self, streamer):
-        rule = {}
+    def parse_table(self, streamer):
+        result = {}
         while True:
-            while streamer.peek().isspace():
-                streamer.pull()
+            streamer.pullSpaces()
             if self.valid_identifier_character(streamer.peek()):
-                self.keyvalue(streamer, rule)
+                self.keyvalue(streamer, result)
             else:
-                return rule
+                return result
 
-    def keyvalue(self, streamer, rule):
+    def keyvalue(self, streamer, result):
         key = self.identifier(streamer)
         streamer.pullSpaces()
         streamer.pull("=")
@@ -127,10 +170,12 @@ def keyvalue(self, streamer, rule):
         elif streamer.peek() == "[":
             # list of strings
             value = self.list(streamer)
+        elif streamer.peek_to_whitespace() in ["true", "false"]:
+            value = self.boolean(streamer)
         else:
             value = None
-            streamer.terminate("Expected either a string or a list of strings.")
-        rule[key] = value
+            streamer.terminate("Expected either a string or a boolean or a list of strings.")
+        result[key] = value
 
     def valid_identifier_character(self, c):
         return c.isalpha() or c == "_"
@@ -142,6 +187,15 @@ def identifier(self, streamer):
             streamer.pull()
         return ident
 
+    def boolean(self, streamer):
+        val = self.identifier(streamer)
+        if val == "true":
+            return True
+        elif val == "false":
+            return False
+        else:
+            streamer.terminate("Expected either true or false.")
+
     def string(self, streamer):
         streamer.pull("\"")
         content = ""