Add overlay support to codegen (#402)

- add support for overlays in codegen
- refactor codegen to allow for the required changes to be made

codegen/src/CodeGen.test.scala

@@ -1,7 +1,8 @@
 package besom.codegen.metaschema
 
 import besom.codegen.*
-import besom.codegen.Config.{CodegenConfig, ProviderConfig}
+import besom.codegen.Config
+import besom.codegen.Utils.PulumiPackageOps
 
 import scala.meta.*
 import scala.meta.dialects.Scala33
@@ -809,23 +810,23 @@ class CodeGenTest extends munit.FunSuite {
     )
   ).foreach(data =>
     test(data.name.withTags(data.tags)) {
-      implicit val config: Config.CodegenConfig = CodegenConfig()
-      implicit val logger: Logger               = new Logger(config.logLevel)
+      given config: Config                 = Config()
+      given logger: Logger                 = Logger()
+      given schemaProvider: SchemaProvider = DownloadingSchemaProvider()
 
-      implicit val schemaProvider: SchemaProvider =
-        new DownloadingSchemaProvider(schemaCacheDirPath = Config.DefaultSchemasDir)
-      val (pulumiPackage, packageInfo) = schemaProvider.packageInfo(
-        PackageMetadata(defaultTestSchemaName, "0.0.0"),
-        PulumiPackage.fromString(data.json)
+      val pulumiPackage = PulumiPackage.fromString(data.json)
+      val packageInfo = schemaProvider.packageInfo(
+        pulumiPackage.toPackageMetadata(),
+        pulumiPackage
       )
-      implicit val providerConfig: ProviderConfig = Config.providersConfigs(packageInfo.name)
-      implicit val tm: TypeMapper                 = new TypeMapper(packageInfo, schemaProvider)
+
+      given TypeMapper = TypeMapper(packageInfo)
 
       val codegen = new CodeGen
       if (data.expectedError.isDefined)
-        interceptMessage[Exception](data.expectedError.get)(codegen.scalaFiles(pulumiPackage, packageInfo))
+        interceptMessage[Exception](data.expectedError.get)(codegen.scalaFiles(packageInfo))
       else
-        codegen.scalaFiles(pulumiPackage, packageInfo).foreach {
+        codegen.scalaFiles(packageInfo).foreach {
           case SourceFile(FilePath(f: String), code: String) if data.expected.contains(f) =>
             assertNoDiff(code, data.expected(f))
             code.parse[Source].get

codegen/src/Config.scala

@@ -2,6 +2,25 @@ package besom.codegen
 
 import besom.model.SemanticVersion
 
+case class Config(
+  logLevel: Logger.Level = Logger.Level.Info,
+  besomVersion: String = Config.DefaultBesomVersion,
+  javaVersion: String = Config.DefaultJavaVersion,
+  scalaVersion: String = Config.DefaultScalaVersion,
+  schemasDir: os.Path = Config.DefaultSchemasDir,
+  codegenDir: os.Path = Config.DefaultCodegenDir,
+  overlaysDir: os.Path = Config.DefaultOverlaysDir,
+  outputDir: Option[os.RelPath] = None,
+  providers: String => Config.Provider = Config.DefaultProvidersConfigs
+):
+  val coreShortVersion: String = SemanticVersion
+    .parseTolerant(besomVersion)
+    .fold(
+      e => throw GeneralCodegenException(s"Invalid besom version: ${besomVersion}", e),
+      _.copy(patch = 0).toShortString
+    )
+end Config
+
 // noinspection ScalaWeakerAccess
 object Config {
 
@@ -14,33 +33,19 @@ object Config {
     } catch {
       case ex: java.nio.file.NoSuchFileException =>
         throw GeneralCodegenException(
-          "Expected './version.txt' file or explicit 'besom.codegen.Config.CodegenConfig(besomVersion = \"1.2.3\")",
+          "Expected './version.txt' file or explicit 'besom.codegen.Config(besomVersion = \"1.2.3\")",
           ex
         )
     }
   }
-  val DefaultSchemasDir: os.Path = os.pwd / ".out" / "schemas"
-  val DefaultCodegenDir: os.Path = os.pwd / ".out" / "codegen"
-
-  case class CodegenConfig(
-    besomVersion: String = DefaultBesomVersion,
-    schemasDir: os.Path = DefaultSchemasDir,
-    codegenDir: os.Path = DefaultCodegenDir,
-    outputDir: Option[os.RelPath] = None,
-    scalaVersion: String = DefaultScalaVersion,
-    javaVersion: String = DefaultJavaVersion,
-    logLevel: Logger.Level = Logger.Level.Info
-  ):
-    val coreShortVersion: String = SemanticVersion
-      .parseTolerant(besomVersion)
-      .fold(
-        e => throw GeneralCodegenException(s"Invalid besom version: ${besomVersion}", e),
-        _.copy(patch = 0).toShortString
-      )
+  val DefaultSchemasDir: os.Path  = os.pwd / ".out" / "schemas"
+  val DefaultCodegenDir: os.Path  = os.pwd / ".out" / "codegen"
+  val DefaultOverlaysDir: os.Path = os.pwd / "codegen" / "resources" / "overlays"
 
-  case class ProviderConfig(
-    noncompiledModules: Seq[String] = Seq.empty
+  case class Provider(
+    nonCompiledModules: Seq[String] = Seq.empty,
+    moduleToPackages: Map[String, String] = Map.empty
   )
 
-  val providersConfigs: Map[String, ProviderConfig] = Map().withDefaultValue(ProviderConfig())
+  val DefaultProvidersConfigs: Map[String, Provider] = Map().withDefault(_ => Config.Provider())
 }

codegen/src/DefinitionCoordinates.test.scala

@@ -1,14 +1,13 @@
 package besom.codegen
 
 import besom.codegen.metaschema.PulumiPackage
+import besom.codegen.Utils.PulumiPackageOps
 
 import scala.meta.*
 import scala.meta.dialects.Scala33
 
 //noinspection ScalaFileName,TypeAnnotation
 class DefinitionCoordinatesTest extends munit.FunSuite {
-  implicit val logger: Logger                        = new Logger
-  implicit val providerConfig: Config.ProviderConfig = Config.ProviderConfig()
 
   case class Data(
     token: PulumiToken,
@@ -60,8 +59,16 @@ class DefinitionCoordinatesTest extends munit.FunSuite {
 
   tests.foreach(data => {
     test(s"Type: ${data.token.asString}".withTags(data.tags.toSet)) {
+      given config: Config                 = Config()
+      given logger: Logger                 = Logger()
+      given schemaProvider: SchemaProvider = DownloadingSchemaProvider()
+
       val pulumiPackage = PulumiPackage("test")
-      val coords        = data.token.toCoordinates(pulumiPackage)
+      val packageInfo   = schemaProvider.packageInfo(pulumiPackage.toPackageMetadata(), pulumiPackage)
+
+      val coords = data.token.toCoordinates(packageInfo)
+
+      given Config.Provider = packageInfo.providerConfig
 
       data.expected.foreach {
         case ResourceClassExpectations(fullPackageName, fullyQualifiedTypeRef, filePath, asArgsType) =>

codegen/src/Logger.scala

@@ -2,7 +2,7 @@ package besom.codegen
 
 import scala.collection.mutable.ListBuffer
 
-class Logger(val printLevel: Logger.Level = Logger.Level.Info) {
+class Logger(val printLevel: Logger.Level) {
   import Logger.Level.*
 
   private val buffer     = ListBuffer.empty[String]
@@ -40,6 +40,8 @@ class Logger(val printLevel: Logger.Level = Logger.Level.Info) {
 }
 
 object Logger {
+  def apply()(using config: Config) = new Logger(config.logLevel)
+
   sealed abstract class Level(val level: Int) extends Ordered[Level] {
     override def compare(that: Level): Int = level.compare(that.level)
   }

codegen/src/Main.scala

@@ -1,27 +1,25 @@
 package besom.codegen
 
-import besom.codegen.Config.{CodegenConfig, ProviderConfig}
 import besom.codegen.UpickleApi.*
-import besom.codegen.metaschema.PulumiPackage
 import besom.codegen.{PackageVersion, SchemaFile}
 
 object Main {
   def main(args: Array[String]): Unit = {
     val result = args.toList match {
       case "named" :: name :: version :: Nil =>
-        implicit val codegenConfig: CodegenConfig = CodegenConfig()
+        given Config = Config()
         generator.generatePackageSources(metadata = PackageMetadata(name, version))
       case "named" :: name :: version :: outputDir :: Nil =>
-        implicit val codegenConfig: CodegenConfig = CodegenConfig(outputDir = Some(os.rel / outputDir))
+        given Config = Config(outputDir = Some(os.rel / outputDir))
         generator.generatePackageSources(metadata = PackageMetadata(name, version))
       case "metadata" :: metadataPath :: Nil =>
-        implicit val codegenConfig: CodegenConfig = CodegenConfig()
+        given Config = Config()
         generator.generatePackageSources(metadata = PackageMetadata.fromJsonFile(os.Path(metadataPath)))
       case "metadata" :: metadataPath :: outputDir :: Nil =>
-        implicit val codegenConfig: CodegenConfig = CodegenConfig(outputDir = Some(os.rel / outputDir))
+        given Config = Config(outputDir = Some(os.rel / outputDir))
         generator.generatePackageSources(metadata = PackageMetadata.fromJsonFile(os.Path(metadataPath)))
       case "schema" :: name :: version :: schemaPath :: Nil =>
-        implicit val codegenConfig: CodegenConfig = CodegenConfig()
+        given Config = Config()
         generator.generatePackageSources(
           metadata = PackageMetadata(name, version),
           schema = Some(os.Path(schemaPath))
@@ -70,26 +68,24 @@ object generator {
   def generatePackageSources(
     metadata: PackageMetadata,
     schema: Option[SchemaFile] = None
-  )(implicit config: CodegenConfig): Result = {
-    implicit val logger: Logger = new Logger(config.logLevel)
-    implicit val schemaProvider: DownloadingSchemaProvider = new DownloadingSchemaProvider(
-      schemaCacheDirPath = config.schemasDir
-    )
+  )(using config: Config): Result = {
+    given logger: Logger                            = Logger()
+    given schemaProvider: DownloadingSchemaProvider = DownloadingSchemaProvider()
 
     // detect possible problems with GH API throttling
     // noinspection ScalaUnusedSymbol
     if !sys.env.contains("GITHUB_TOKEN") then logger.warn("Setting GITHUB_TOKEN environment variable might solve some problems")
 
-    val (pulumiPackage, packageInfo) = schemaProvider.packageInfo(metadata, schema)
-    val packageName                  = packageInfo.name
-    val packageVersion               = packageInfo.version
+    val packageInfo    = schemaProvider.packageInfo(metadata, schema)
+    val packageName    = packageInfo.name
+    val packageVersion = packageInfo.version
 
-    implicit val providerConfig: ProviderConfig = Config.providersConfigs(packageName)
+    given typeMapper: TypeMapper = TypeMapper(packageInfo)
 
     val outputDir: os.Path =
       config.outputDir.getOrElse(os.rel / packageName / packageVersion.asString).resolveFrom(config.codegenDir)
 
-    val total = generatePackageSources(pulumiPackage, packageInfo, outputDir)
+    val total = generatePackageSources(packageInfo, outputDir)
     logger.info(s"Finished generating package '$packageName:$packageVersion' codebase (${total} files)")
 
     val dependencies = schemaProvider.dependencies(packageName, packageVersion).map { case (name, version) =>
@@ -107,39 +103,36 @@ object generator {
   }
 
   private def generatePackageSources(
-    pulumiPackage: PulumiPackage,
     packageInfo: PulumiPackageInfo,
     outputDir: os.Path
-  )(implicit
+  )(using
     logger: Logger,
-    codegenConfig: CodegenConfig,
-    providerConfig: ProviderConfig,
-    schemaProvider: SchemaProvider
+    config: Config,
+    schemaProvider: SchemaProvider,
+    typeMapper: TypeMapper
   ): Int = {
     // Print diagnostic information
     logger.info {
       val relOutputDir = outputDir.relativeTo(os.pwd)
       s"""|Generating package '${packageInfo.name}:${packageInfo.version}' into '$relOutputDir'
-          | - Besom version   : ${codegenConfig.besomVersion}
-          | - Scala version   : ${codegenConfig.scalaVersion}
-          | - Java version    : ${codegenConfig.javaVersion}
+          | - Besom version   : ${config.besomVersion}
+          | - Scala version   : ${config.scalaVersion}
+          | - Java version    : ${config.javaVersion}
           |
-          | - Resources: ${pulumiPackage.resources.size}
-          | - Types    : ${pulumiPackage.types.size}
-          | - Functions: ${pulumiPackage.functions.size}
-          | - Config   : ${pulumiPackage.config.variables.size}
+          | - Resources: ${packageInfo.pulumiPackage.resources.size}
+          | - Types    : ${packageInfo.pulumiPackage.types.size}
+          | - Functions: ${packageInfo.pulumiPackage.functions.size}
+          | - Config   : ${packageInfo.pulumiPackage.config.variables.size}
           |""".stripMargin
     }
 
-    implicit val typeMapper: TypeMapper = new TypeMapper(packageInfo, schemaProvider)
-
     // make sure we don't have a dirty state
     os.remove.all(outputDir)
     os.makeDir.all(outputDir)
 
     val codeGen = new CodeGen
     try {
-      val sources = codeGen.sourcesFromPulumiPackage(pulumiPackage, packageInfo)
+      val sources = codeGen.sourcesFromPulumiPackage(packageInfo)
       sources
         .foreach { sourceFile =>
           val filePath = outputDir / sourceFile.filePath.osSubPath

codegen/src/Overlay.scala

@@ -0,0 +1,20 @@
+package besom.codegen
+
+object Overlay:
+  def readFiles(
+    packageInfo: PulumiPackageInfo,
+    token: PulumiToken,
+    scalaDefinitions: Vector[ScalaDefinitionCoordinates]
+  )(using config: Config, logger: Logger): Vector[SourceFile] =
+    given Config.Provider = packageInfo.providerConfig
+
+    scalaDefinitions.flatMap { scalaDefinition =>
+      val relativeFilePath = scalaDefinition.filePath.copy(scalaDefinition.filePath.pathParts.tail) // drop "src" from the path
+      val path             = config.overlaysDir / packageInfo.name / relativeFilePath.osSubPath
+      if os.exists(path) then
+        val content = os.read(path)
+        Vector(SourceFile(scalaDefinition.filePath, content))
+      else
+        logger.info(s"Token '${token.asString}' was not generated because it was marked as overlay and not found at '${path}'")
+        Vector.empty
+    }

codegen/src/PackageMetadata.test.scala

@@ -45,8 +45,7 @@ class PackageMetadataTest extends munit.FunSuite {
 }
 
 class PackageVersionTest extends munit.FunSuite {
-
-  implicit val logger: Logger = new Logger
+  given Logger = new Logger(Logger.Level.Info)
 
   test("parse") {
     assertEquals(PackageVersion("v6.7.0").map(_.asString), Some("6.7.0"))

codegen/src/PropertyInfo.test.scala

@@ -1,6 +1,6 @@
 package besom.codegen.metaschema
 
-import besom.codegen.Config.CodegenConfig
+import besom.codegen.Config
 import besom.codegen._
 
 import scala.meta._
@@ -110,16 +110,15 @@ class PropertyInfoTest extends munit.FunSuite {
     )
   ).foreach(data =>
     test(data.name.withTags(data.tags)) {
-      implicit val config: Config.CodegenConfig = CodegenConfig()
-      implicit val logger: Logger               = new Logger(config.logLevel)
+      given Config                         = Config()
+      given Logger                         = Logger()
+      given schemaProvider: SchemaProvider = DownloadingSchemaProvider()
 
-      implicit val schemaProvider: SchemaProvider =
-        new DownloadingSchemaProvider(schemaCacheDirPath = Config.DefaultSchemasDir)
-      val (_, packageInfo) = data.metadata match {
+      val packageInfo = data.metadata match {
         case m @ TestPackageMetadata => schemaProvider.packageInfo(m, PulumiPackage(name = m.name))
         case _                       => schemaProvider.packageInfo(data.metadata)
       }
-      implicit val tm: TypeMapper = new TypeMapper(packageInfo, schemaProvider)
+      given TypeMapper = TypeMapper(packageInfo)
 
       val (name, definition) = UpickleApi.read[Map[String, PropertyDefinition]](data.json).head
       val property           = PropertyInfo.from(name, definition, isPropertyRequired = false)

codegen/src/PulumiDefinitionCoordinates.scala

@@ -47,7 +47,7 @@ case class PulumiDefinitionCoordinates private (
       case _ => throw PulumiDefinitionCoordinatesError(s"Invalid definition name '$definitionName'")
   }
 
-  def resourceMethod(implicit logger: Logger): ScalaDefinitionCoordinates = {
+  def asFunctionClass(using Logger): ScalaDefinitionCoordinates = {
     val (methodPrefix, methodName) = splitMethodName(definitionName)
     ScalaDefinitionCoordinates(
       providerPackageParts = providerPackageParts,
@@ -56,15 +56,15 @@ case class PulumiDefinitionCoordinates private (
       selectionName = Some(mangleMethodName(methodName))
     )
   }
-  def methodArgsClass(implicit logger: Logger): ScalaDefinitionCoordinates = {
+  def asFunctionArgsClass(using Logger): ScalaDefinitionCoordinates = {
     val (methodPrefix, methodName) = splitMethodName(definitionName)
     ScalaDefinitionCoordinates(
       providerPackageParts = providerPackageParts,
       modulePackageParts = modulePackageParts,
       definitionName = Some((methodPrefix.toSeq :+ mangleTypeName(methodName)).mkString("") + "Args")
     )
   }
-  def methodResultClass(implicit logger: Logger): ScalaDefinitionCoordinates = {
+  def asFunctionResultClass(using Logger): ScalaDefinitionCoordinates = {
     val (methodPrefix, methodName) = splitMethodName(definitionName)
     ScalaDefinitionCoordinates(
       providerPackageParts = providerPackageParts,
@@ -73,7 +73,7 @@ case class PulumiDefinitionCoordinates private (
     )
   }
 
-  def asConfigClass(implicit logger: Logger): ScalaDefinitionCoordinates = {
+  def asConfigClass(using Logger): ScalaDefinitionCoordinates = {
     ScalaDefinitionCoordinates(
       providerPackageParts = providerPackageParts,
       modulePackageParts = modulePackageParts,