DOCSP-3615: Implement kind-of-working language server

.gitignore

@@ -1,4 +1,6 @@
 __pycache__/
 .mypy_cache/
 *.pyc
-.DS_Store
+.DS_Store
+parser/snooty.py
+*.dist/

parser/Makefile

@@ -1,4 +1,4 @@
-.PHONY: lint run test
+.PHONY: lint run test nuitka
 
 run:
 	pipenv run mypy --strict snooty
@@ -10,3 +10,8 @@ lint:
 
 test: lint
 	pipenv run pytest
+
+nuitka:
+	echo 'from snooty import main; main.main()' > snooty.py
+	nuitka3 --lto --standalone --python-flag=no_site --remove-output snooty.py
+	rm snooty.py

parser/Pipfile

@@ -4,15 +4,16 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pymongo = "*"
+pymongo = {extras = ["snappy", "srv", "tls"], version = "*"}
 docutils = "*"
 dnspython = "*"
 watchdog = "*"
 toml = "*"
 pyyaml = "*"
 typing-extensions = "*"
-pytest = "*"
+python-jsonrpc-server = "*"
 
 [dev-packages]
-mypy = "*"
 "flake8" = "*"
+mypy = "*"
+pytest = "*"

parser/snooty/gizaparser/__init__.py

@@ -1,54 +1 @@
-import yaml
-from yaml.composer import Composer
-from yaml.constructor import Constructor
-from typing import Dict, List, Tuple, Type, TypeVar
-from .flutter import check_type, LoadError
-from ..types import SerializableType
 from . import steps  # NoQA
-
-_T = TypeVar('_T')
-
-
-class ParseError(Exception):
-    pass
-
-
-def load_yaml(text: str) -> List[SerializableType]:
-    loader = yaml.Loader(text)
-
-    def compose_node(parent: yaml.nodes.Node, index: int) -> yaml.nodes.Node:
-        # the line number where the previous token has ended (plus empty lines)
-        line = loader.line
-        node = Composer.compose_node(loader, parent, index)
-        node.__line__ = line + 1
-        return node
-
-    def construct_mapping(node: yaml.nodes.Node, deep: bool = False) -> Dict:
-        mapping = Constructor.construct_mapping(loader, node, deep=deep)
-        mapping['__line__'] = node.__line__
-        return mapping
-
-    setattr(loader, 'compose_node', compose_node)
-    setattr(loader, 'construct_mapping', construct_mapping)
-    result: List[SerializableType] = []
-    while True:
-        data = loader.get_data()
-        if data:
-            result.append(data)
-        else:
-            break
-
-    return result
-
-
-def parse(ty: Type[_T], path: str) -> Tuple[List[_T], str]:
-    with open(path, 'r') as f:
-        text = f.read()
-
-    parsed_yaml = load_yaml(text)
-
-    try:
-        parsed = [check_type(ty, data) for data in parsed_yaml]
-        return parsed, text
-    except LoadError as err:
-        raise ParseError(str(err))

parser/snooty/gizaparser/apiargs.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from typing import Dict, Optional
-from .flutter import checked
+from ..flutter import checked
 from .nodes import Node, Inherit
 
 

parser/snooty/gizaparser/extracts.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from typing import Dict, Optional
-from .flutter import checked
+from ..flutter import checked
 from .nodes import Node, Inherit
 
 

parser/snooty/gizaparser/nodes.py

@@ -4,9 +4,10 @@
 import os.path
 import re
 from dataclasses import dataclass
-from typing import Callable, Dict, Set, Generic, Optional, TypeVar, Tuple, Iterator, Sequence
-from .flutter import checked
-from ..types import Page, EmbeddedRstParser
+from typing import Dict, Set, Generic, Optional, TypeVar, Tuple, Iterator, Sequence, List
+from typing_extensions import Protocol
+from ..flutter import checked
+from ..types import Diagnostic, Page, EmbeddedRstParser
 
 PAT_SUBSTITUTION = re.compile(r'\{\{([\w-]+)\}\}')
 logger = logging.getLogger(__name__)
@@ -77,6 +78,9 @@ def set_dependencies(self, obj: str, dependencies: Set[str]) -> None:
         for dependency in dependencies:
             self.dependents[dependency].add(obj)
 
+    def __delitem__(self, obj: str) -> None:
+        pass
+
 
 @dataclass
 class GizaFile(Generic[T]):
@@ -136,12 +140,19 @@ def __iter__(self) -> Iterator[Tuple[str, GizaFile[T]]]:
     def __len__(self) -> int:
         return len(self.nodes)
 
+    def __delitem__(self, file_id: str) -> None:
+        del self.dg[file_id]
+        del self.nodes[file_id]
 
-@dataclass
-class GizaCategory(Generic[T]):
+
+class GizaCategory(Generic[T], Protocol):
     """A GizaCategory stores metadata about a "category" of Giza YAML files. For
        example, "steps", or "apiargs". Each GizaCategory contains all types necessary
        to transform a given path into a Page."""
     registry: GizaRegistry[T]
-    parse: Callable[[str], Tuple[Sequence[T], str]]
-    to_page: Callable[[Page, Sequence[T], EmbeddedRstParser], None]
+
+    def parse(self,
+              path: str,
+              text: Optional[str] = None) -> Tuple[Sequence[T], str, List[Diagnostic]]: ...
+
+    def to_page(self, page: Page, data: Sequence[T], rst_parser: EmbeddedRstParser) -> None: ...

parser/snooty/gizaparser/options.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from typing import Dict, Optional
-from .flutter import checked
+from ..flutter import checked
 from .nodes import Node, Inherit