Tree-sitter

.gitattributes

@@ -0,0 +1 @@
+modules/treesitter/src/main/scala/playground/generated/** linguist-generated

.github/workflows/ci.yml

@@ -55,5 +55,22 @@ jobs:
         run: nix develop --command bash -c 'cd vscode-extension && yarn && SERVER_VERSION=$(cat ../.version) xvfb-run --auto-servernum yarn test'
 
       - name: Show extension test logs
-        if: job.status == 'failure'
+        if: always() && job.status == 'failure'
         run: cat vscode-extension/fixture/smithyql-log.txt | tail --lines 1000
+
+  build-parser:
+    name: "Build parser"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, macos-13]
+    runs-on: ${{matrix.os}}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/[email protected]
+      - uses: coursier/setup-action@v1
+        with:
+          apps: sbt
+          jvm: adoptium:1.21
+      - name: Parser tests
+        # intentionally not setting up nix
+        run: sbt treesitter/test

build.sbt

@@ -87,6 +87,7 @@ val commonSettings = Seq(
   Test / scalacOptions += "-Wconf:cat=deprecation:silent,msg=Specify both message and version:silent",
   scalacOptions += "-release:11",
   mimaFailOnNoPrevious := false,
+  resolvers += "Sonatype S01 snapshots" at "https://s01.oss.sonatype.org/content/repositories/snapshots",
 )
 
 def module(
@@ -133,8 +134,28 @@ lazy val parser = module("parser")
   .dependsOn(
     ast % "test->test;compile->compile",
     source % "test->test;compile->compile",
+    treesitter % "test->compile",
   )
 
+lazy val treesitter = module("treesitter")
+  .settings(
+    libraryDependencies ++= Seq(
+      "org.polyvariant.treesitter4s" %% "core" % "0.4.0"
+    )
+  )
+
+lazy val parsergen = module("parser-gen")
+  .settings(
+    libraryDependencies ++= Seq(
+      "dev.optics" %% "monocle-core" % "3.3.0",
+      "com.disneystreaming.smithy4s" %% "smithy4s-json" % smithy4sVersion.value,
+      ("org.scalameta" %% "scalameta" % "4.11.0").cross(CrossVersion.for3Use2_13),
+      "org.polyvariant.treesitter4s" %% "core" % "0.4.0",
+    ),
+    scalacOptions -= "-release:11",
+  )
+  .enablePlugins(Smithy4sCodegenPlugin)
+
 // Formatter for the SmithyQL language constructs
 lazy val formatter = module("formatter")
   .settings(
@@ -189,6 +210,7 @@ lazy val core = module("core")
     examples % "test->compile",
     pluginCore,
     ast,
+    treesitter,
     source % "test->test;compile->compile",
     parser % "test->compile;test->test",
     formatter,
@@ -228,17 +250,27 @@ lazy val e2e = module("e2e")
     buildInfoKeys ++=
       Seq[BuildInfoKey.Entry[_]]( // do you know how to simplify this? let me know please!
         Def
-          .task((lsp / Compile / fullClasspath).value.map(_.data).map(_.toString))
-          .taskValue
-          .named("lspClassPath"),
-        Def
-          .task(
-            (lsp / Compile / mainClass).value.getOrElse(sys.error("didn't find main class in lsp"))
+          .task {
+            s"""${(lsp / organization).value}::${(lsp / moduleName).value}:${(lsp / version).value}"""
+          }
+          // todo: replace with a full publishLocal before e2e in particular gets run (but not before tests run normally)
+          .dependsOn(
+            lsp / publishLocal,
+            languageSupport / publishLocal,
+            core / publishLocal,
+            parser / publishLocal,
+            pluginCore / publishLocal,
+            source / publishLocal,
+            treesitter / publishLocal,
+            ast / publishLocal,
+            formatter / publishLocal,
+            protocol4s / publishLocal,
           )
           .taskValue
-          .named("lspMainClass"),
+          .named("lspArtifact")
       ),
     publish / skip := true,
+    Test / fork := true,
   )
   .dependsOn(lsp)
 
@@ -260,11 +292,13 @@ lazy val root = project
     core,
     examples,
     parser,
+    parsergen,
     formatter,
     languageSupport,
     lsp,
     protocol4s,
     pluginCore,
     pluginSample,
     e2e,
+    treesitter,
   )

flake.nix

@@ -1,10 +1,10 @@
 {
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
     flake-utils.url = "github:numtide/flake-utils";
   };
 
-  outputs = { nixpkgs, flake-utils, ... }:
+  outputs = { self, nixpkgs, flake-utils, ... }:
     flake-utils.lib.eachDefaultSystem (
       system:
       let
@@ -18,10 +18,40 @@
             pkgs.sbt
             pkgs.jless
             pkgs.gnupg
+            (pkgs.tree-sitter.override { webUISupport = true; })
             # temporary, while we don't download coursier ourselves
             pkgs.coursier
           ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.xvfb-run ];
         };
+        packages.tree-sitter-smithyql =
+          pkgs.stdenv.mkDerivation {
+            name = "tree-sitter-smithyql";
+            src = ./tree-sitter-smithyql;
+            buildInputs = [pkgs.tree-sitter pkgs.nodejs];
+            buildPhase = ''
+              tree-sitter generate
+              make
+            '';
+            installPhase = if system == "x86_64-darwin" || system == "aarch64-darwin" then ''
+              cp libtree-sitter-smithyql.dylib $out
+            '' else ''
+              cp libtree-sitter-smithyql.so $out
+            '';
+          };
+        packages.tree-sitter-smithyql-all =
+          pkgs.stdenv.mkDerivation {
+            name = "tree-sitter-smithyql-all";
+            src = ./tree-sitter-smithyql;
+            dontBuild=true;
+            installPhase = ''
+              mkdir $out
+              cd $out
+              mkdir darwin-aarch64 && cp ${self.packages.aarch64-darwin.tree-sitter-smithyql} darwin-aarch64/libtree-sitter-smithyql.dylib
+              mkdir darwin-x86-64 && cp ${self.packages.x86_64-darwin.tree-sitter-smithyql} darwin-x86-64/libtree-sitter-smithyql.dylib
+              mkdir linux-aarch64 && cp ${self.packages.aarch64-linux.tree-sitter-smithyql} linux-aarch64/libtree-sitter-smithyql.so
+              mkdir linux-x86-64 && cp ${self.packages.x86_64-linux.tree-sitter-smithyql} linux-x86-64/libtree-sitter-smithyql.so
+            '';
+          };
       }
     );
 }

modules/ast/src/main/scala/playground/smithyql/AST.scala

@@ -4,6 +4,7 @@ import cats.Applicative
 import cats.Functor
 import cats.Id
 import cats.Show
+import cats.arrow.FunctionK
 import cats.data.NonEmptyList
 import cats.kernel.Eq
 import cats.kernel.Order
@@ -87,7 +88,7 @@ final case class SourceFile[F[_]](
 
   def mapK[G[_]: Functor](
     fk: F ~> G
-  ): AST[G] = SourceFile(
+  ): SourceFile[G] = SourceFile(
     prelude = prelude.mapK(fk),
     statements = fk(statements).map(_.map(_.mapK(fk))),
   )
@@ -132,9 +133,9 @@ final case class OperationName[F[_]](
   text: String
 ) extends AST[F] {
 
-  def mapK[G[_]: Functor](
-    fk: F ~> G
-  ): OperationName[G] = copy()
+  def mapK[G[_]: Functor](fk: FunctionK[F, G]): OperationName[G] = retag[G]
+
+  def retag[G[_]]: OperationName[G] = copy()
 
 }
 
@@ -189,7 +190,7 @@ final case class QueryOperationName[F[_]](
     fk: F ~> G
   ): QueryOperationName[G] = QueryOperationName(
     identifier.map(fk(_)),
-    fk(operationName).map(_.mapK(fk)),
+    fk(operationName).map(_.retag[G]),
   )
 
 }

modules/ast/src/test/scala/playground/Assertions.scala

@@ -23,7 +23,7 @@ object Assertions extends Expectations.Helpers {
         val stringWithResets = d.show()(conf).linesWithSeparators.map(Console.RESET + _).mkString
 
         failure(
-          s"Diff failed:\n${Console.RESET}(${conf.left("expected")}, ${conf.right("actual")})\n\n" + stringWithResets
+          s"Diff failed:\n${Console.RESET}(${conf.left("actual")}, ${conf.right("expected")})\n\n" + stringWithResets
         )
     }
 

modules/core/src/main/scala/playground/ASTAdapter.scala

@@ -0,0 +1,12 @@
+package playground
+
+import cats.syntax.all.*
+import playground.smithyql.QualifiedIdentifier
+
+object ASTAdapter {
+
+  def decodeQI(qi: playground.generated.nodes.QualifiedIdentifier): Option[QualifiedIdentifier] =
+    (qi.namespace.map(_.source).toNel, qi.selection.map(_.source))
+      .mapN(QualifiedIdentifier.apply)
+
+}

modules/core/src/main/scala/playground/MultiServiceResolver.scala

@@ -9,6 +9,7 @@ import playground.smithyql.UseClause
 import playground.smithyql.WithSource
 
 object MultiServiceResolver {
+  import playground.smithyql.tsutils.*
 
   /** Determines which service should be used for a query. The rules are:
     *   - If the operation name has a service identifier, there MUST be a service with that name
@@ -37,6 +38,40 @@ object MultiServiceResolver {
       case None => resolveImplicit(queryOperationName.operationName, serviceIndex, useClauses)
     }
 
+  /** Determines which service should be used for a query. The rules are:
+    *   - If the operation name has a service identifier, there MUST be a service with that name
+    *     that contains the given operation.
+    *   - If there's no service identifier, find all matching services that are included in the use
+    *     clauses. MUST find exactly one entry.
+    *
+    * In other cases, such as when we can't find a unique entry, or the explicitly referenced
+    * service doesn't have an operation with a matching name, we fail. The latter might eventually
+    * be refactored to a separate piece of code.
+    *
+    * **Important**!
+    *
+    * This method assumes that all of the use clauses match the available service set. It does NOT
+    * perform a check on that. For the actual check, see PreludeCompiler.
+    */
+  def resolveServiceTs(
+    queryOperationName: playground.generated.nodes.QueryOperationName,
+    serviceIndex: ServiceIndex,
+    useClauses: List[playground.generated.nodes.UseClause],
+  ): EitherNel[CompilationError, QualifiedIdentifier] =
+    queryOperationName.name match {
+      case Some(opName) =>
+        // todo: this should be an Option in codegen. might be a bad grammar
+        queryOperationName.identifier.headOption match {
+          case Some(explicitRef) => resolveExplicitTs(serviceIndex, explicitRef, opName)
+
+          case None => resolveImplicitTs(opName, serviceIndex, useClauses)
+        }
+      case None =>
+        // TODO: operation name is invalid or something like that
+        ???
+
+    }
+
   private def resolveExplicit(
     index: ServiceIndex,
     explicitRef: WithSource[QualifiedIdentifier],
@@ -66,6 +101,41 @@ object MultiServiceResolver {
       case Some(_) => explicitRef.value.asRight
     }
 
+  private def resolveExplicitTs(
+    index: ServiceIndex,
+    explicitRef: playground.generated.nodes.QualifiedIdentifier,
+    operationName: playground.generated.nodes.OperationName,
+  ): EitherNel[CompilationError, QualifiedIdentifier] =
+    ASTAdapter.decodeQI(explicitRef) match {
+      case None => ??? /* todo - I don't really know xD */
+      // explicit reference exists, but doesn't parse
+      case Some(ref) =>
+        index.getService(ref) match {
+          // explicit reference exists, but the service doesn't
+          case None =>
+            CompilationError
+              .error(
+                CompilationErrorDetails.UnknownService(index.serviceIds.toList),
+                explicitRef.range,
+              )
+              .leftNel
+
+          // the service exists, but doesn't have the requested operation
+          case Some(service)
+              if !service.operationNames.contains_(OperationName(operationName.source)) =>
+            CompilationError
+              .error(
+                CompilationErrorDetails.OperationMissing(service.operationNames.toList),
+                operationName.range,
+              )
+              .leftNel
+
+          // all good
+          case Some(_) => ref.asRight
+        }
+
+    }
+
   private def resolveImplicit(
     operationName: WithSource[OperationName[WithSource]],
     index: ServiceIndex,
@@ -90,4 +160,28 @@ object MultiServiceResolver {
     }
   }
 
+  private def resolveImplicitTs(
+    operationName: playground.generated.nodes.OperationName,
+    index: ServiceIndex,
+    useClauses: List[playground.generated.nodes.UseClause],
+  ): EitherNel[CompilationError, QualifiedIdentifier] = {
+    val matchingServices = index
+      .getServices(useClauses.flatMap(_.identifier).flatMap(ASTAdapter.decodeQI).toSet)
+      .filter(_.hasOperation(OperationName(operationName.source)))
+
+    matchingServices match {
+      case one :: Nil => one.id.asRight
+      case _ =>
+        CompilationError
+          .error(
+            CompilationErrorDetails
+              .AmbiguousService(
+                workspaceServices = index.serviceIds.toList
+              ),
+            operationName.range,
+          )
+          .leftNel
+    }
+  }
+
 }