Codegen update 4 (#263)

* Use codegen directly from tests
- use codegen module as a test dep
- upgrade Scala in generated code 3.3.0 -> 3.3.1
* Move test provider publishing to tests from Justfile
* Make sure we don't have polymorphism issues with SchemaProvider.addSchema

Justfile

@@ -261,9 +261,6 @@ clean-test-integration: clean-test-integration-codegen
 
 # Runs integration tests for core
 test-integration-core: publish-local-codegen publish-local-core install-language-plugin publish-local-compiler-plugin
-	just generate-provider-sdk random 4.13.2
-	just publish-local-provider-sdk random 4.13.2
-	PULUMI_SCALA_PLUGIN_LOCAL_PATH={{language-plugin-output-dir}} \
 	scala-cli --power test integration-tests --test-only 'besom.integration.core*'
 
 # Runs integration tests for compiler plugin
@@ -272,7 +269,6 @@ test-integration-compiler-plugin: publish-local-codegen publish-local-core insta
 
 # Runs integration tests for language plugin
 test-integration-language-plugin: publish-local-codegen publish-local-core install-language-plugin publish-local-compiler-plugin
-	PULUMI_SCALA_PLUGIN_LOCAL_PATH={{language-plugin-output-dir}} \
 	scala-cli --power test integration-tests --test-only 'besom.integration.languageplugin*'
 
 # Runs integration tests for codegen

codegen/src/Main.scala

@@ -1,52 +1,87 @@
 package besom.codegen
 
+import besom.codegen.SchemaProvider.{ProviderName, SchemaVersion}
+
 object Main {
   def main(args: Array[String]): Unit = {
-    if (args(0) == "test")
-      args.drop(1).toList match {
-        case schemaPath :: outputPath :: providerName :: schemaVersion :: besomVersion :: Nil =>
-          generateTestPackageSources(
-            schemaPath = os.Path(schemaPath),
-            outputPath = os.Path(outputPath),
-            providerName = providerName,
-            schemaVersion = schemaVersion,
-            besomVersion = besomVersion
-          )
-          sys.exit(0)
-        case _ =>
-          System.err.println("Codegen's expected test arguments: <schemaPath> <outputPath> <providerName> <schemaVersion> <besomVersion>")
-          sys.exit(1)
-      }
-
     args.toList match {
       case schemasDirPath :: outputDirBasePath :: providerName :: schemaVersion :: besomVersion :: Nil =>
         generatePackageSources(
-          schemasDirPath = os.Path(schemasDirPath),
-          outputDirBasePath = os.Path(outputDirBasePath),
+          schemasDir = os.Path(schemasDirPath),
+          codegenDir = os.Path(outputDirBasePath),
           providerName = providerName,
           schemaVersion = schemaVersion,
           besomVersion = besomVersion
         )
       case _ =>
-        System.err.println("Codegen's expected arguments: <schemasDirPath> <outputDirBasePath> <providerName> <schemaVersion> <besomVersion>")
+        System.err.println(
+          "Codegen's expected arguments: <schemasDirPath> <outputDirBasePath> <providerName> <schemaVersion> <besomVersion>"
+        )
         sys.exit(1)
     }
   }
 
-  def generatePackageSources(schemasDirPath: os.Path, outputDirBasePath: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = {
-    val schemaProvider = new DownloadingSchemaProvider(schemaCacheDirPath = schemasDirPath)
-    val destinationDir = outputDirBasePath / providerName / schemaVersion
+  // noinspection ScalaWeakerAccess
+  def generatePackageSources(
+    schemasDir: os.Path,
+    codegenDir: os.Path,
+    providerName: String,
+    schemaVersion: String,
+    besomVersion: String,
+    preLoadSchemas: Map[(ProviderName, SchemaVersion), os.Path] = Map()
+  ): os.Path = {
+    implicit val providerConfig: Config.ProviderConfig = Config.providersConfigs(providerName)
+    implicit val logger: Logger                        = new Logger
+
+    val schemaProvider = new DownloadingSchemaProvider(schemaCacheDirPath = schemasDir)
+    val outputDir      = codegenDir / providerName / schemaVersion
+
+    // Print diagnostic information
+    val preLoadInfo = if (preLoadSchemas.nonEmpty) {
+      val preloadList = preLoadSchemas
+        .map { case ((name, version), path) =>
+          s"   - $name:$version -> ${path.relativeTo(os.pwd)}"
+        }
+        .mkString("\n")
+      s"""| - Pre-load schemas:
+          |$preloadList
+          |""".stripMargin
+    } else {
+      ""
+    }
+    println(
+      s"""|Generating package '$providerName:$schemaVersion' into '${outputDir.relativeTo(os.pwd)}'
+          | - Besom version   : $besomVersion
+          | - Scala version   : ${CodeGen.scalaVersion}
+          | - Java version    : ${CodeGen.javaVersion}
+          |""".stripMargin + preLoadInfo
+    )
+
+    // Pre-load schemas from files if needed
+    preLoadSchemas.foreach { case ((name, version), path) =>
+      schemaProvider.addSchemaFile(name, version, path)
+    }
+
+    generatePackageSources(schemaProvider, outputDir, providerName, schemaVersion, besomVersion)
+    println(s"Finished generating provider '$providerName' codebase")
 
-    generatePackageSources(schemaProvider, destinationDir, providerName, schemaVersion, besomVersion)
+    outputDir
   }
 
-  def generatePackageSources(schemaProvider: SchemaProvider, destinationDir: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = {
-    println(s"Generating provider SDK for $providerName")
-
+  private def generatePackageSources(
+    schemaProvider: SchemaProvider,
+    outputDir: os.Path,
+    providerName: String,
+    schemaVersion: String,
+    besomVersion: String
+  )(implicit logger: Logger, providerConfig: Config.ProviderConfig): Unit = {
     val pulumiPackage = schemaProvider.pulumiPackage(providerName = providerName, schemaVersion = schemaVersion)
-
-    implicit val providerConfig: Config.ProviderConfig = Config.providersConfigs(providerName)
-    implicit val logger: Logger = new Logger
+    println(
+      s"""|Loaded package: ${pulumiPackage.name} ${pulumiPackage.version.getOrElse("")}
+          | - Resources: ${pulumiPackage.resources.size}
+          | - Types    : ${pulumiPackage.types.size}
+          |""".stripMargin
+    )
 
     implicit val typeMapper: TypeMapper = new TypeMapper(
       defaultProviderName = providerName,
@@ -56,32 +91,29 @@ object Main {
     )
 
     // make sure we don't have a dirty state
-    os.remove.all(destinationDir)
-    os.makeDir.all(destinationDir)
+    os.remove.all(outputDir)
+    os.makeDir.all(outputDir)
 
     val codeGen = new CodeGen
     try {
-      codeGen.sourcesFromPulumiPackage(
-        pulumiPackage,
-        schemaVersion = schemaVersion,
-        besomVersion = besomVersion
-      ).foreach { sourceFile =>
-        val filePath = destinationDir / sourceFile.filePath.osSubPath
-        os.makeDir.all(filePath / os.up)
-        os.write(filePath, sourceFile.sourceCode)
-      }
+      codeGen
+        .sourcesFromPulumiPackage(
+          pulumiPackage,
+          schemaVersion = schemaVersion,
+          besomVersion = besomVersion
+        )
+        .foreach { sourceFile =>
+          val filePath = outputDir / sourceFile.filePath.osSubPath
+          os.makeDir.all(filePath / os.up)
+          os.write(filePath, sourceFile.sourceCode, createFolders = true)
+        }
       println("Finished generating SDK codebase")
     } finally {
       if (logger.nonEmpty) {
-        val logFile = destinationDir / ".codegen-log.txt"
+        val logFile = outputDir / ".codegen-log.txt"
         println(s"Some problems were encountered during the code generation. See ${logFile}")
         logger.writeToFile(logFile)
       }
     }
   }
-
-  def generateTestPackageSources(schemaPath: os.Path, outputPath: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = {
-    val schemaProvider = new TestSchemaProvider(schemaPath)
-    generatePackageSources(schemaProvider, outputPath, providerName, schemaVersion, besomVersion)
-  }
 }

codegen/src/SchemaProvider.scala

@@ -1,7 +1,5 @@
 package besom.codegen
 
-import besom.codegen.SchemaProvider.{ProviderName, SchemaVersion}
-
 import scala.collection.mutable.ListBuffer
 import besom.codegen.metaschema._
 import besom.codegen.Utils.PulumiPackageOps
@@ -25,10 +23,12 @@ trait SchemaProvider {
     val enumTypeTokensBuffer   = ListBuffer.empty[String]
     val objectTypeTokensBuffer = ListBuffer.empty[String]
 
+    // post-process the package to improve its quality
     pulumiPackage.types.foreach {
       case (typeToken, _: EnumTypeDefinition) =>
         enumTypeTokensBuffer += typeToken.toLowerCase // Unifying to lower case to circumvent inconsistencies in low quality schemas (e.g. aws)
-      case (typeToken, _: ObjectTypeDefinition) => objectTypeTokensBuffer += typeToken.toLowerCase
+      case (typeToken, _: ObjectTypeDefinition) =>
+        objectTypeTokensBuffer += typeToken.toLowerCase
     }
 
     PulumiPackageInfo(
@@ -41,14 +41,8 @@ trait SchemaProvider {
   }
 }
 
-class TestSchemaProvider(schemaPath: os.Path) extends SchemaProvider {
-  override def pulumiPackage(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackage =
-    pulumiPackage(schemaPath)
-  override def packageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo =
-    loadPackageInfo(pulumiPackage(schemaPath))
-}
-
 class DownloadingSchemaProvider(schemaCacheDirPath: os.Path) extends SchemaProvider {
+
   import SchemaProvider._
 
   private val packageInfos: collection.mutable.Map[(ProviderName, SchemaVersion), PulumiPackageInfo] =
@@ -58,25 +52,36 @@ class DownloadingSchemaProvider(schemaCacheDirPath: os.Path) extends SchemaProvi
     val schemaFilePath = schemaCacheDirPath / providerName / schemaVersion / "schema.json"
 
     if (!os.exists(schemaFilePath)) {
-      val schemaSource = s"${providerName}@${schemaVersion}"
+      val schemaSource = s"$providerName@$schemaVersion"
       os.makeDir.all(schemaFilePath / os.up)
-      os.proc("pulumi", "plugin", "install", "resource", providerName, schemaVersion).call()
       os.proc("pulumi", "package", "get-schema", schemaSource).call(stdout = schemaFilePath)
     }
 
     schemaFilePath
   }
 
+  def addSchemaFile(providerName: ProviderName, schemaVersion: SchemaVersion, content: os.Path): os.Path = {
+    val schemaFilePath = schemaCacheDirPath / providerName / schemaVersion / "schema.json"
+    os.copy.over(content, schemaFilePath, replaceExisting = true, createFolders = true)
+    schemaFilePath
+  }
+
+  def addSchemaString(providerName: ProviderName, schemaVersion: SchemaVersion, content: String): os.Path = {
+    val schemaFilePath = schemaCacheDirPath / providerName / schemaVersion / "schema.json"
+    os.write.over(schemaFilePath, content, createFolders = true)
+    schemaFilePath
+  }
+
   def pulumiPackage(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackage =
     pulumiPackage(downloadedSchemaFilePath(providerName, schemaVersion))
 
-  private def loadPackageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo =
-    loadPackageInfo(pulumiPackage(providerName, schemaVersion))
-
   def packageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo = {
     packageInfos.getOrElseUpdate(
       (providerName, schemaVersion),
       loadPackageInfo(providerName, schemaVersion)
     )
   }
+
+  private def loadPackageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo =
+    loadPackageInfo(pulumiPackage(providerName, schemaVersion))
 }

integration-tests/CodegenTests.test.scala

@@ -21,28 +21,6 @@ class CodegenTests extends munit.FunSuite {
 
   val testdata = os.pwd / "integration-tests" / "resources" / "testdata"
 
-  def codegen(
-    schemaPath: os.Path,
-    codegenOutputDir: os.Path,
-    schemaName: String,
-    schemaVersion: String,
-    besomVersion: String
-  ): os.proc =
-    pproc(
-      "scala-cli",
-      "run",
-      "codegen",
-      "--suppress-experimental-feature-warning",
-      "--suppress-directives-in-multiple-files-warning",
-      "--",
-      "test",
-      schemaPath,
-      codegenOutputDir,
-      schemaName,
-      schemaVersion,
-      besomVersion
-    )
-
   val slowFileList = List(
     "docker"
   )
@@ -119,13 +97,8 @@ class CodegenTests extends munit.FunSuite {
       case _                                            => name
     }
     test(options) {
-      val outputDir = codegenDir / data.name / "0.0.0"
-      val result    = codegen(data.schema, outputDir, data.name, "0.0.0", coreVersion).call(check = false)
-      val output    = result.out.text()
-      assert(output.contains("Finished generating SDK codebase"), s"Output:\n$output\n")
-      assert(result.exitCode == 0)
-
-      val compiled = scalaCli.compile(outputDir).call(check = false)
+      val outputDir = codegen.generatePackageFromSchema(data.schema, data.name, "0.0.0")
+      val compiled  = scalaCli.compile(outputDir).call(check = false)
       assert {
         clue(data)
         compiled.exitCode == 0

integration-tests/CoreTests.test.scala

@@ -21,11 +21,14 @@ class CoreTests extends munit.FunSuite {
 
   FunFixture[pulumi.FixtureContext](
     setup = {
+      val schemaName = "random"
+      val providerSource = codegen.generatePackage(schemaName, providerRandomSchemaVersion)
+      scalaCli.publishLocal(providerSource)
       pulumi.fixture.setup(
         wd / "resources" / "random-example",
         projectFiles = Map(
           "project.scala" ->
-            (defaultProjectFile + s"""//> using dep org.virtuslab::besom-random:$providerRandomVersion""")
+            (defaultProjectFile + s"""//> using dep org.virtuslab::besom-$schemaName:$providerRandomVersion""")
         )
       )
     },

integration-tests/LanguagePluginTest.test.scala

@@ -15,7 +15,7 @@ class LanguagePluginTest extends munit.FunSuite {
   val wd                  = os.pwd / "integration-tests"
   val resourcesDir        = wd / "resources"
   val executorsDir        = resourcesDir / "executors"
-  val bootstrapLibJarPath = scalaPluginLocalPath.get / "bootstrap.jar"
+  val bootstrapLibJarPath = languagePluginDir / "bootstrap.jar"
 
   val projectFile =
     s"""|//> using scala $scalaVersion