diff --git a/codegen/src/CodeGen.scala b/codegen/src/CodeGen.scala index 39073e2c..fb720f34 100644 --- a/codegen/src/CodeGen.scala +++ b/codegen/src/CodeGen.scala @@ -1,6 +1,5 @@ package besom.codegen -import besom.codegen.Config.{CodegenConfig, ProviderConfig} import besom.codegen.PackageVersion import besom.codegen.Utils.* import besom.codegen.metaschema.* @@ -10,9 +9,8 @@ import scala.meta.* import scala.meta.dialects.Scala33 //noinspection ScalaWeakerAccess,TypeAnnotation -class CodeGen(implicit - codegenConfig: CodegenConfig, - providerConfig: ProviderConfig, +class CodeGen(using + config: Config, typeMapper: TypeMapper, schemaProvider: SchemaProvider, logger: Logger @@ -20,10 +18,9 @@ class CodeGen(implicit import CodeGen.* def sourcesFromPulumiPackage( - pulumiPackage: PulumiPackage, packageInfo: PulumiPackageInfo ): Seq[SourceFile] = - scalaFiles(pulumiPackage, packageInfo) ++ Seq( + scalaFiles(packageInfo) ++ Seq( projectConfigFile( schemaName = packageInfo.name, packageVersion = packageInfo.version @@ -31,27 +28,26 @@ class CodeGen(implicit resourcePluginMetadataFile( pluginName = packageInfo.name, pluginVersion = packageInfo.version, - pluginDownloadUrl = pulumiPackage.pluginDownloadURL + pluginDownloadUrl = packageInfo.pulumiPackage.pluginDownloadURL ) ) def scalaFiles( - pulumiPackage: PulumiPackage, packageInfo: PulumiPackageInfo ): Seq[SourceFile] = { - val (configFiles, configDependencies) = sourceFilesForConfig(pulumiPackage, packageInfo) + val (configFiles, configDependencies) = sourceFilesForConfig(packageInfo) configFiles ++ - sourceFilesForProviderResource(pulumiPackage) ++ - sourceFilesForNonResourceTypes(pulumiPackage, configDependencies) ++ - sourceFilesForResources(pulumiPackage) ++ - sourceFilesForFunctions(pulumiPackage) + sourceFilesForProviderResource(packageInfo) ++ + sourceFilesForNonResourceTypes(packageInfo, configDependencies) ++ + sourceFilesForResources(packageInfo) ++ + sourceFilesForFunctions(packageInfo) } def projectConfigFile(schemaName: String, packageVersion: PackageVersion): SourceFile = { - val besomVersion = codegenConfig.besomVersion - val scalaVersion = codegenConfig.scalaVersion - val javaVersion = codegenConfig.javaVersion - val coreShortVersion = codegenConfig.coreShortVersion + val besomVersion = config.besomVersion + val scalaVersion = config.scalaVersion + val javaVersion = config.javaVersion + val coreShortVersion = config.coreShortVersion val dependencies = packageDependencies(schemaProvider.dependencies(schemaName, packageVersion)) @@ -105,46 +101,64 @@ class CodeGen(implicit SourceFile(filePath = filePath, sourceCode = fileContent) } - def sourceFilesForProviderResource(pulumiPackage: PulumiPackage): Seq[SourceFile] = { - val typeToken = pulumiPackage.providerTypeToken - val moduleToPackageParts = pulumiPackage.moduleToPackageParts - val providerToPackageParts = pulumiPackage.providerToPackageParts + def sourceFilesForProviderResource(packageInfo: PulumiPackageInfo): Seq[SourceFile] = { + val typeToken = packageInfo.providerTypeToken + val moduleToPackageParts = packageInfo.moduleToPackageParts + val providerToPackageParts = packageInfo.providerToPackageParts val typeCoordinates = PulumiDefinitionCoordinates.fromRawToken(typeToken, moduleToPackageParts, providerToPackageParts) + given Config.Provider = packageInfo.providerConfig sourceFilesForResource( typeCoordinates = typeCoordinates, - resourceDefinition = pulumiPackage.provider, - methods = pulumiPackage.parsedMethods(pulumiPackage.provider), + resourceDefinition = packageInfo.pulumiPackage.provider, + methods = packageInfo.parseMethods(packageInfo.pulumiPackage.provider), isProvider = true ) } def sourceFilesForNonResourceTypes( - pulumiPackage: PulumiPackage, + packageInfo: PulumiPackageInfo, configDependencies: Seq[ConfigDependency] ): Seq[SourceFile] = { - pulumiPackage.parsedTypes.flatMap { case (coordinates, typeDefinition) => - typeDefinition match { - case enumDef: EnumTypeDefinition => - sourceFilesForEnum( - typeCoordinates = coordinates, - enumDefinition = enumDef + given Config.Provider = packageInfo.providerConfig + + packageInfo.parsedTypes.flatMap { + case (coordinates, (enumDef: EnumTypeDefinition, false)) => + sourceFilesForEnum( + typeCoordinates = coordinates, + enumDefinition = enumDef + ) + case (coordinates, (_: EnumTypeDefinition, true)) => + Overlay.readFiles( + packageInfo, + coordinates.token, + Vector( + coordinates.asEnumClass ) - case objectDef: ObjectTypeDefinition => - sourceFilesForObjectType( - typeCoordinates = coordinates, - objectTypeDefinition = objectDef, - configDependencies = configDependencies + ) + case (coordinates, (objectDef: ObjectTypeDefinition, false)) => + sourceFilesForObjectType( + typeCoordinates = coordinates, + objectTypeDefinition = objectDef, + configDependencies = configDependencies + ) + case (coordinates, (_: ObjectTypeDefinition, true)) => + Overlay.readFiles( + packageInfo, + coordinates.token, + Vector( + coordinates.asObjectClass(asArgsType = false), + coordinates.asObjectClass(asArgsType = true) ) - } + ) }.toSeq } def sourceFilesForEnum( typeCoordinates: PulumiDefinitionCoordinates, enumDefinition: EnumTypeDefinition - ): Seq[SourceFile] = { + )(using Config.Provider): Seq[SourceFile] = { val classCoordinates = typeCoordinates.asEnumClass val enumClassName = classCoordinates.definitionTypeName.getOrElse( @@ -210,7 +224,7 @@ class CodeGen(implicit typeCoordinates: PulumiDefinitionCoordinates, objectTypeDefinition: ObjectTypeDefinition, configDependencies: Seq[ConfigDependency] - ): Seq[SourceFile] = { + )(using Config.Provider): Seq[SourceFile] = { val typeToken = typeCoordinates.token val baseClassCoordinates = typeCoordinates.asObjectClass(asArgsType = false) val argsClassCoordinates = typeCoordinates.asObjectClass(asArgsType = true) @@ -256,15 +270,27 @@ class CodeGen(implicit Seq(baseClassSourceFile, argsClassSourceFile) } - def sourceFilesForResources(pulumiPackage: PulumiPackage): Seq[SourceFile] = { - pulumiPackage.parsedResources - .map { case (coordinates, resourceDefinition) => - sourceFilesForResource( - typeCoordinates = coordinates, - resourceDefinition = resourceDefinition, - methods = pulumiPackage.parsedMethods(resourceDefinition), - isProvider = false - ) + def sourceFilesForResources(packageInfo: PulumiPackageInfo): Seq[SourceFile] = { + given Config.Provider = packageInfo.providerConfig + + packageInfo.parsedResources + .map { + case (coordinates, (resourceDefinition, false)) => + sourceFilesForResource( + typeCoordinates = coordinates, + resourceDefinition = resourceDefinition, + methods = packageInfo.parseMethods(resourceDefinition), + isProvider = false + ) + case (coordinates, (_, true)) => + Overlay.readFiles( + packageInfo, + coordinates.token, + Vector( + coordinates.asResourceClass(asArgsType = false), + coordinates.asResourceClass(asArgsType = true) + ) + ) } .toSeq .flatten @@ -273,9 +299,9 @@ class CodeGen(implicit def sourceFilesForResource( typeCoordinates: PulumiDefinitionCoordinates, resourceDefinition: ResourceDefinition, - methods: Map[FunctionName, (PulumiDefinitionCoordinates, FunctionDefinition)], + methods: Map[FunctionName, (PulumiDefinitionCoordinates, (FunctionDefinition, Boolean))], isProvider: Boolean - ): Seq[SourceFile] = { + )(using Config.Provider): Seq[SourceFile] = { val token = typeCoordinates.token val baseClassCoordinates = typeCoordinates.asResourceClass(asArgsType = false) val argsClassCoordinates = typeCoordinates.asResourceClass(asArgsType = true) @@ -418,30 +444,33 @@ class CodeGen(implicit |) extends ${resourceBaseClass}""".stripMargin.parse[Stat].get val (methodFiles, baseClassMethods) = - methods.map { case (name, (functionCoordinates, functionDefinition)) => - val functionToken = functionCoordinates.token - val methodCoordinates = functionCoordinates.resourceMethod - if (!methodCoordinates.definitionName.contains(name)) { - logger.warn( - s"""|Resource definition method name '${name}' (used) does not match the name - |in method definition '${methodCoordinates.definitionName.getOrElse("")}' (ignored) - |for function '${functionToken.asString}' - this is a schema error""".stripMargin - ) - } - val (supportClassSourceFiles, argsClassRef, argsDefault, resultTypeRef) = - functionSupport(functionCoordinates, functionDefinition) - - val thisTypeRef = baseClassCoordinates.typeRef - val tokenLiteral = Lit.String(functionToken.asString) - val code = - m"""| def ${Term.Name(name)}(using ctx: besom.types.Context)( - | args: ${argsClassRef}${argsDefault}, - | opts: besom.InvokeOptions = besom.InvokeOptions() - | ): besom.types.Output[${resultTypeRef}] = - | ctx.call[$argsClassRef, $resultTypeRef, $thisTypeRef](${tokenLiteral}, args, this, opts) - |""".stripMargin.parse[Stat].get - - (supportClassSourceFiles, code) + methods.map { + case (name, (functionCoordinates, (functionDefinition, false))) => + val functionToken = functionCoordinates.token + val methodCoordinates = functionCoordinates.asFunctionClass + if (!methodCoordinates.definitionName.contains(name)) { + logger.warn( + s"""|Resource definition method name '${name}' (used) does not match the name + |in method definition '${methodCoordinates.definitionName.getOrElse("")}' (ignored) + |for function '${functionToken.asString}' - this is a schema error""".stripMargin + ) + } + val (supportClassSourceFiles, argsClassRef, argsDefault, resultTypeRef) = + functionSupport(functionCoordinates, functionDefinition) + + val thisTypeRef = baseClassCoordinates.typeRef + val tokenLiteral = Lit.String(functionToken.asString) + val code = + m"""| def ${Term.Name(name)}(using ctx: besom.types.Context)( + | args: ${argsClassRef}${argsDefault}, + | opts: besom.InvokeOptions = besom.InvokeOptions() + | ): besom.types.Output[${resultTypeRef}] = + | ctx.call[$argsClassRef, $resultTypeRef, $thisTypeRef](${tokenLiteral}, args, this, opts) + |""".stripMargin.parse[Stat].get + + (supportClassSourceFiles, code) + case (_, (_, (_, true))) => + throw GeneralCodegenException("Overlay files are not supported for methods, resource overlay was expected instead") }.unzip val hasDefaultArgsConstructor = requiredInputs.forall { propertyName => @@ -516,14 +545,26 @@ class CodeGen(implicit Seq(baseClassSourceFile, argsClassSourceFile) ++ methodFiles.flatten } - def sourceFilesForFunctions(pulumiPackage: PulumiPackage)(implicit logger: Logger): Seq[SourceFile] = { - pulumiPackage.parsedFunctions - .filterNot { case (_, f) => isMethod(f) } - .map { case (coordinates, functionDefinition) => - sourceFilesForFunction( - functionCoordinates = coordinates, - functionDefinition = functionDefinition - ) + def sourceFilesForFunctions(packageInfo: PulumiPackageInfo): Seq[SourceFile] = { + given Config.Provider = packageInfo.providerConfig + packageInfo.parsedFunctions + .filterNot { case (_, (f, _)) => isMethod(f) } + .map { + case (coordinates, (functionDefinition, false)) => + sourceFilesForFunction( + functionCoordinates = coordinates, + functionDefinition = functionDefinition + ) + case (coordinates, (_, true)) => + Overlay.readFiles( + packageInfo, + coordinates.token, + Vector( + coordinates.asFunctionClass, + coordinates.asFunctionArgsClass, + coordinates.asFunctionResultClass + ) + ) } .toSeq .flatten @@ -532,7 +573,7 @@ class CodeGen(implicit private def functionSupport( functionCoordinates: PulumiDefinitionCoordinates, functionDefinition: FunctionDefinition - ): (Seq[SourceFile], Type.Ref, String, Type) = { + )(using Config.Provider): (Seq[SourceFile], Type.Ref, String, Type) = { val requiredInputs = functionDefinition.inputs.required val inputProperties = functionDefinition.inputs.properties.toSeq @@ -549,7 +590,7 @@ class CodeGen(implicit ) } - val argsClassCoordinates = functionCoordinates.methodArgsClass + val argsClassCoordinates = functionCoordinates.asFunctionArgsClass val argsClassSourceFile = makeArgsClassSourceFile( classCoordinates = argsClassCoordinates, properties = inputProperties, @@ -557,7 +598,7 @@ class CodeGen(implicit isProvider = false ) - val resultClassCoordinates = functionCoordinates.methodResultClass + val resultClassCoordinates = functionCoordinates.asFunctionResultClass val resultClassSourceFileOpt = functionDefinition.outputs.objectTypeDefinition.map { outputTypeDefinition => val requiredOutputs = outputTypeDefinition.required val outputProperties = @@ -598,13 +639,13 @@ class CodeGen(implicit def sourceFilesForFunction( functionCoordinates: PulumiDefinitionCoordinates, functionDefinition: FunctionDefinition - ): Seq[SourceFile] = { + )(using Config.Provider): Seq[SourceFile] = { val (supportClassSourceFiles, argsClassRef, argsDefault, resultTypeRef) = functionSupport( functionCoordinates, functionDefinition ) val functionToken = functionCoordinates.token - val methodCoordinates = functionCoordinates.resourceMethod + val methodCoordinates = functionCoordinates.asFunctionClass val methodName = methodCoordinates.selectionName.getOrElse( throw GeneralCodegenException( @@ -631,16 +672,15 @@ class CodeGen(implicit } def sourceFilesForConfig( - pulumiPackage: PulumiPackage, packageInfo: PulumiPackageInfo ): (Seq[SourceFile], Seq[ConfigDependency]) = { val PulumiToken(_, _, providerName) = PulumiToken(packageInfo.providerTypeToken) val coordinates = PulumiToken(providerName, Utils.configModuleName, Utils.configTypeName) - .toCoordinates(pulumiPackage) + .toCoordinates(packageInfo) .asConfigClass - val configsWithDefault = pulumiPackage.config.defaults - val configVariables = pulumiPackage.config.variables.toSeq.sortBy(_._1) + val configsWithDefault = packageInfo.pulumiPackage.config.defaults + val configVariables = packageInfo.pulumiPackage.config.variables.toSeq.sortBy(_._1) val configs = configVariables.collect { case (configName, configDefinition) => val get = Name(s"get${configName.capitalize}") val propType = configDefinition.typeReference.asScalaType() @@ -695,7 +735,7 @@ class CodeGen(implicit |${configs.mkString("\n")} |""".stripMargin - if pulumiPackage.config.variables.isEmpty + if packageInfo.pulumiPackage.config.variables.isEmpty then (Seq.empty, Seq.empty) else { val _ = scalameta.parseSource(code) @@ -704,9 +744,9 @@ class CodeGen(implicit val sources = Seq(SourceFile(file, code)) val dependencies: Seq[ConfigDependency] = configVariables.flatMap { case (_, configDefinition) => configDefinition.typeReference.asTokenAndDependency.flatMap { - case (Some(token), None) => ConfigSourceDependency(token.toCoordinates(pulumiPackage)) :: Nil + case (Some(token), None) => ConfigSourceDependency(token.toCoordinates(packageInfo)) :: Nil case (Some(token), Some(metadata)) => - ConfigRuntimeDependency(token.toCoordinates(pulumiPackage), metadata) :: Nil + ConfigRuntimeDependency(token.toCoordinates(packageInfo), metadata) :: Nil case _ => Nil } } @@ -719,7 +759,7 @@ class CodeGen(implicit classCoordinates: ScalaDefinitionCoordinates, properties: Seq[PropertyInfo], requiresJsonFormat: Boolean - ): SourceFile = { + )(using Config.Provider): SourceFile = { val className = classCoordinates.definitionTypeName.getOrElse( throw GeneralCodegenException( s"Class name for ${classCoordinates.typeRef} could not be found" @@ -833,7 +873,7 @@ class CodeGen(implicit properties: Seq[PropertyInfo], isResource: Boolean, isProvider: Boolean - ): SourceFile = { + )(using Config.Provider): SourceFile = { val argsClass = makeArgsClass( classCoordinates = classCoordinates, inputProperties = properties @@ -998,12 +1038,12 @@ class CodeGen(implicit } object CodeGen: - def packageDependency(name: SchemaName, version: SchemaVersion)(implicit codegenConfig: CodegenConfig): String = + def packageDependency(name: SchemaName, version: SchemaVersion)(using Config): String = packageDependencies(List((name, version))) - def packageDependencies(dependencies: List[(SchemaName, SchemaVersion)])(implicit codegenConfig: CodegenConfig): String = + def packageDependencies(dependencies: List[(SchemaName, SchemaVersion)])(using config: Config): String = dependencies .map { case (name, version) => - s"""|//> using dep "org.virtuslab::besom-${name}:${version}-core.${codegenConfig.coreShortVersion}" + s"""|//> using dep "org.virtuslab::besom-${name}:${version}-core.${config.coreShortVersion}" |""".stripMargin } .mkString("\n") diff --git a/codegen/src/CodeGen.test.scala b/codegen/src/CodeGen.test.scala index 61e6527b..88af46cd 100644 --- a/codegen/src/CodeGen.test.scala +++ b/codegen/src/CodeGen.test.scala @@ -1,7 +1,8 @@ package besom.codegen.metaschema import besom.codegen.* -import besom.codegen.Config.{CodegenConfig, ProviderConfig} +import besom.codegen.Config +import besom.codegen.Utils.PulumiPackageOps import scala.meta.* import scala.meta.dialects.Scala33 @@ -809,23 +810,23 @@ class CodeGenTest extends munit.FunSuite { ) ).foreach(data => test(data.name.withTags(data.tags)) { - implicit val config: Config.CodegenConfig = CodegenConfig() - implicit val logger: Logger = new Logger(config.logLevel) + given config: Config = Config() + given logger: Logger = Logger() + given schemaProvider: SchemaProvider = DownloadingSchemaProvider() - implicit val schemaProvider: SchemaProvider = - new DownloadingSchemaProvider(schemaCacheDirPath = Config.DefaultSchemasDir) - val (pulumiPackage, packageInfo) = schemaProvider.packageInfo( - PackageMetadata(defaultTestSchemaName, "0.0.0"), - PulumiPackage.fromString(data.json) + val pulumiPackage = PulumiPackage.fromString(data.json) + val packageInfo = schemaProvider.packageInfo( + pulumiPackage.toPackageMetadata(), + pulumiPackage ) - implicit val providerConfig: ProviderConfig = Config.providersConfigs(packageInfo.name) - implicit val tm: TypeMapper = new TypeMapper(packageInfo, schemaProvider) + + given TypeMapper = TypeMapper(packageInfo) val codegen = new CodeGen if (data.expectedError.isDefined) - interceptMessage[Exception](data.expectedError.get)(codegen.scalaFiles(pulumiPackage, packageInfo)) + interceptMessage[Exception](data.expectedError.get)(codegen.scalaFiles(packageInfo)) else - codegen.scalaFiles(pulumiPackage, packageInfo).foreach { + codegen.scalaFiles(packageInfo).foreach { case SourceFile(FilePath(f: String), code: String) if data.expected.contains(f) => assertNoDiff(code, data.expected(f)) code.parse[Source].get diff --git a/codegen/src/Config.scala b/codegen/src/Config.scala index 7f49c5d1..c695f5c6 100644 --- a/codegen/src/Config.scala +++ b/codegen/src/Config.scala @@ -2,6 +2,25 @@ package besom.codegen import besom.model.SemanticVersion +case class Config( + logLevel: Logger.Level = Logger.Level.Info, + besomVersion: String = Config.DefaultBesomVersion, + javaVersion: String = Config.DefaultJavaVersion, + scalaVersion: String = Config.DefaultScalaVersion, + schemasDir: os.Path = Config.DefaultSchemasDir, + codegenDir: os.Path = Config.DefaultCodegenDir, + overlaysDir: os.Path = Config.DefaultOverlaysDir, + outputDir: Option[os.RelPath] = None, + providers: String => Config.Provider = Config.DefaultProvidersConfigs +): + val coreShortVersion: String = SemanticVersion + .parseTolerant(besomVersion) + .fold( + e => throw GeneralCodegenException(s"Invalid besom version: ${besomVersion}", e), + _.copy(patch = 0).toShortString + ) +end Config + // noinspection ScalaWeakerAccess object Config { @@ -14,33 +33,19 @@ object Config { } catch { case ex: java.nio.file.NoSuchFileException => throw GeneralCodegenException( - "Expected './version.txt' file or explicit 'besom.codegen.Config.CodegenConfig(besomVersion = \"1.2.3\")", + "Expected './version.txt' file or explicit 'besom.codegen.Config(besomVersion = \"1.2.3\")", ex ) } } - val DefaultSchemasDir: os.Path = os.pwd / ".out" / "schemas" - val DefaultCodegenDir: os.Path = os.pwd / ".out" / "codegen" - - case class CodegenConfig( - besomVersion: String = DefaultBesomVersion, - schemasDir: os.Path = DefaultSchemasDir, - codegenDir: os.Path = DefaultCodegenDir, - outputDir: Option[os.RelPath] = None, - scalaVersion: String = DefaultScalaVersion, - javaVersion: String = DefaultJavaVersion, - logLevel: Logger.Level = Logger.Level.Info - ): - val coreShortVersion: String = SemanticVersion - .parseTolerant(besomVersion) - .fold( - e => throw GeneralCodegenException(s"Invalid besom version: ${besomVersion}", e), - _.copy(patch = 0).toShortString - ) + val DefaultSchemasDir: os.Path = os.pwd / ".out" / "schemas" + val DefaultCodegenDir: os.Path = os.pwd / ".out" / "codegen" + val DefaultOverlaysDir: os.Path = os.pwd / "codegen" / "resources" / "overlays" - case class ProviderConfig( - noncompiledModules: Seq[String] = Seq.empty + case class Provider( + nonCompiledModules: Seq[String] = Seq.empty, + moduleToPackages: Map[String, String] = Map.empty ) - val providersConfigs: Map[String, ProviderConfig] = Map().withDefaultValue(ProviderConfig()) + val DefaultProvidersConfigs: Map[String, Provider] = Map().withDefault(_ => Config.Provider()) } diff --git a/codegen/src/DefinitionCoordinates.test.scala b/codegen/src/DefinitionCoordinates.test.scala index 0ef7423c..d76db9d8 100644 --- a/codegen/src/DefinitionCoordinates.test.scala +++ b/codegen/src/DefinitionCoordinates.test.scala @@ -1,14 +1,13 @@ package besom.codegen import besom.codegen.metaschema.PulumiPackage +import besom.codegen.Utils.PulumiPackageOps import scala.meta.* import scala.meta.dialects.Scala33 //noinspection ScalaFileName,TypeAnnotation class DefinitionCoordinatesTest extends munit.FunSuite { - implicit val logger: Logger = new Logger - implicit val providerConfig: Config.ProviderConfig = Config.ProviderConfig() case class Data( token: PulumiToken, @@ -60,8 +59,16 @@ class DefinitionCoordinatesTest extends munit.FunSuite { tests.foreach(data => { test(s"Type: ${data.token.asString}".withTags(data.tags.toSet)) { + given config: Config = Config() + given logger: Logger = Logger() + given schemaProvider: SchemaProvider = DownloadingSchemaProvider() + val pulumiPackage = PulumiPackage("test") - val coords = data.token.toCoordinates(pulumiPackage) + val packageInfo = schemaProvider.packageInfo(pulumiPackage.toPackageMetadata(), pulumiPackage) + + val coords = data.token.toCoordinates(packageInfo) + + given Config.Provider = packageInfo.providerConfig data.expected.foreach { case ResourceClassExpectations(fullPackageName, fullyQualifiedTypeRef, filePath, asArgsType) => diff --git a/codegen/src/Logger.scala b/codegen/src/Logger.scala index ecdf9a2d..e3e0c1bf 100644 --- a/codegen/src/Logger.scala +++ b/codegen/src/Logger.scala @@ -2,7 +2,7 @@ package besom.codegen import scala.collection.mutable.ListBuffer -class Logger(val printLevel: Logger.Level = Logger.Level.Info) { +class Logger(val printLevel: Logger.Level) { import Logger.Level.* private val buffer = ListBuffer.empty[String] @@ -40,6 +40,8 @@ class Logger(val printLevel: Logger.Level = Logger.Level.Info) { } object Logger { + def apply()(using config: Config) = new Logger(config.logLevel) + sealed abstract class Level(val level: Int) extends Ordered[Level] { override def compare(that: Level): Int = level.compare(that.level) } diff --git a/codegen/src/Main.scala b/codegen/src/Main.scala index bdbd17ce..1b4b30bc 100644 --- a/codegen/src/Main.scala +++ b/codegen/src/Main.scala @@ -1,27 +1,25 @@ package besom.codegen -import besom.codegen.Config.{CodegenConfig, ProviderConfig} import besom.codegen.UpickleApi.* -import besom.codegen.metaschema.PulumiPackage import besom.codegen.{PackageVersion, SchemaFile} object Main { def main(args: Array[String]): Unit = { val result = args.toList match { case "named" :: name :: version :: Nil => - implicit val codegenConfig: CodegenConfig = CodegenConfig() + given Config = Config() generator.generatePackageSources(metadata = PackageMetadata(name, version)) case "named" :: name :: version :: outputDir :: Nil => - implicit val codegenConfig: CodegenConfig = CodegenConfig(outputDir = Some(os.rel / outputDir)) + given Config = Config(outputDir = Some(os.rel / outputDir)) generator.generatePackageSources(metadata = PackageMetadata(name, version)) case "metadata" :: metadataPath :: Nil => - implicit val codegenConfig: CodegenConfig = CodegenConfig() + given Config = Config() generator.generatePackageSources(metadata = PackageMetadata.fromJsonFile(os.Path(metadataPath))) case "metadata" :: metadataPath :: outputDir :: Nil => - implicit val codegenConfig: CodegenConfig = CodegenConfig(outputDir = Some(os.rel / outputDir)) + given Config = Config(outputDir = Some(os.rel / outputDir)) generator.generatePackageSources(metadata = PackageMetadata.fromJsonFile(os.Path(metadataPath))) case "schema" :: name :: version :: schemaPath :: Nil => - implicit val codegenConfig: CodegenConfig = CodegenConfig() + given Config = Config() generator.generatePackageSources( metadata = PackageMetadata(name, version), schema = Some(os.Path(schemaPath)) @@ -70,26 +68,24 @@ object generator { def generatePackageSources( metadata: PackageMetadata, schema: Option[SchemaFile] = None - )(implicit config: CodegenConfig): Result = { - implicit val logger: Logger = new Logger(config.logLevel) - implicit val schemaProvider: DownloadingSchemaProvider = new DownloadingSchemaProvider( - schemaCacheDirPath = config.schemasDir - ) + )(using config: Config): Result = { + given logger: Logger = Logger() + given schemaProvider: DownloadingSchemaProvider = DownloadingSchemaProvider() // detect possible problems with GH API throttling // noinspection ScalaUnusedSymbol if !sys.env.contains("GITHUB_TOKEN") then logger.warn("Setting GITHUB_TOKEN environment variable might solve some problems") - val (pulumiPackage, packageInfo) = schemaProvider.packageInfo(metadata, schema) - val packageName = packageInfo.name - val packageVersion = packageInfo.version + val packageInfo = schemaProvider.packageInfo(metadata, schema) + val packageName = packageInfo.name + val packageVersion = packageInfo.version - implicit val providerConfig: ProviderConfig = Config.providersConfigs(packageName) + given typeMapper: TypeMapper = TypeMapper(packageInfo) val outputDir: os.Path = config.outputDir.getOrElse(os.rel / packageName / packageVersion.asString).resolveFrom(config.codegenDir) - val total = generatePackageSources(pulumiPackage, packageInfo, outputDir) + val total = generatePackageSources(packageInfo, outputDir) logger.info(s"Finished generating package '$packageName:$packageVersion' codebase (${total} files)") val dependencies = schemaProvider.dependencies(packageName, packageVersion).map { case (name, version) => @@ -107,39 +103,36 @@ object generator { } private def generatePackageSources( - pulumiPackage: PulumiPackage, packageInfo: PulumiPackageInfo, outputDir: os.Path - )(implicit + )(using logger: Logger, - codegenConfig: CodegenConfig, - providerConfig: ProviderConfig, - schemaProvider: SchemaProvider + config: Config, + schemaProvider: SchemaProvider, + typeMapper: TypeMapper ): Int = { // Print diagnostic information logger.info { val relOutputDir = outputDir.relativeTo(os.pwd) s"""|Generating package '${packageInfo.name}:${packageInfo.version}' into '$relOutputDir' - | - Besom version : ${codegenConfig.besomVersion} - | - Scala version : ${codegenConfig.scalaVersion} - | - Java version : ${codegenConfig.javaVersion} + | - Besom version : ${config.besomVersion} + | - Scala version : ${config.scalaVersion} + | - Java version : ${config.javaVersion} | - | - Resources: ${pulumiPackage.resources.size} - | - Types : ${pulumiPackage.types.size} - | - Functions: ${pulumiPackage.functions.size} - | - Config : ${pulumiPackage.config.variables.size} + | - Resources: ${packageInfo.pulumiPackage.resources.size} + | - Types : ${packageInfo.pulumiPackage.types.size} + | - Functions: ${packageInfo.pulumiPackage.functions.size} + | - Config : ${packageInfo.pulumiPackage.config.variables.size} |""".stripMargin } - implicit val typeMapper: TypeMapper = new TypeMapper(packageInfo, schemaProvider) - // make sure we don't have a dirty state os.remove.all(outputDir) os.makeDir.all(outputDir) val codeGen = new CodeGen try { - val sources = codeGen.sourcesFromPulumiPackage(pulumiPackage, packageInfo) + val sources = codeGen.sourcesFromPulumiPackage(packageInfo) sources .foreach { sourceFile => val filePath = outputDir / sourceFile.filePath.osSubPath diff --git a/codegen/src/Overlay.scala b/codegen/src/Overlay.scala new file mode 100644 index 00000000..46fb237c --- /dev/null +++ b/codegen/src/Overlay.scala @@ -0,0 +1,20 @@ +package besom.codegen + +object Overlay: + def readFiles( + packageInfo: PulumiPackageInfo, + token: PulumiToken, + scalaDefinitions: Vector[ScalaDefinitionCoordinates] + )(using config: Config, logger: Logger): Vector[SourceFile] = + given Config.Provider = packageInfo.providerConfig + + scalaDefinitions.flatMap { scalaDefinition => + val relativeFilePath = scalaDefinition.filePath.copy(scalaDefinition.filePath.pathParts.tail) // drop "src" from the path + val path = config.overlaysDir / packageInfo.name / relativeFilePath.osSubPath + if os.exists(path) then + val content = os.read(path) + Vector(SourceFile(scalaDefinition.filePath, content)) + else + logger.info(s"Token '${token.asString}' was not generated because it was marked as overlay and not found at '${path}'") + Vector.empty + } diff --git a/codegen/src/PackageMetadata.test.scala b/codegen/src/PackageMetadata.test.scala index 436989a2..3e2808c3 100644 --- a/codegen/src/PackageMetadata.test.scala +++ b/codegen/src/PackageMetadata.test.scala @@ -45,8 +45,7 @@ class PackageMetadataTest extends munit.FunSuite { } class PackageVersionTest extends munit.FunSuite { - - implicit val logger: Logger = new Logger + given Logger = new Logger(Logger.Level.Info) test("parse") { assertEquals(PackageVersion("v6.7.0").map(_.asString), Some("6.7.0")) diff --git a/codegen/src/PropertyInfo.test.scala b/codegen/src/PropertyInfo.test.scala index 8f8c3c25..6f88cb45 100644 --- a/codegen/src/PropertyInfo.test.scala +++ b/codegen/src/PropertyInfo.test.scala @@ -1,6 +1,6 @@ package besom.codegen.metaschema -import besom.codegen.Config.CodegenConfig +import besom.codegen.Config import besom.codegen._ import scala.meta._ @@ -110,16 +110,15 @@ class PropertyInfoTest extends munit.FunSuite { ) ).foreach(data => test(data.name.withTags(data.tags)) { - implicit val config: Config.CodegenConfig = CodegenConfig() - implicit val logger: Logger = new Logger(config.logLevel) + given Config = Config() + given Logger = Logger() + given schemaProvider: SchemaProvider = DownloadingSchemaProvider() - implicit val schemaProvider: SchemaProvider = - new DownloadingSchemaProvider(schemaCacheDirPath = Config.DefaultSchemasDir) - val (_, packageInfo) = data.metadata match { + val packageInfo = data.metadata match { case m @ TestPackageMetadata => schemaProvider.packageInfo(m, PulumiPackage(name = m.name)) case _ => schemaProvider.packageInfo(data.metadata) } - implicit val tm: TypeMapper = new TypeMapper(packageInfo, schemaProvider) + given TypeMapper = TypeMapper(packageInfo) val (name, definition) = UpickleApi.read[Map[String, PropertyDefinition]](data.json).head val property = PropertyInfo.from(name, definition, isPropertyRequired = false) diff --git a/codegen/src/PulumiDefinitionCoordinates.scala b/codegen/src/PulumiDefinitionCoordinates.scala index c9a11b8c..9787ac11 100644 --- a/codegen/src/PulumiDefinitionCoordinates.scala +++ b/codegen/src/PulumiDefinitionCoordinates.scala @@ -47,7 +47,7 @@ case class PulumiDefinitionCoordinates private ( case _ => throw PulumiDefinitionCoordinatesError(s"Invalid definition name '$definitionName'") } - def resourceMethod(implicit logger: Logger): ScalaDefinitionCoordinates = { + def asFunctionClass(using Logger): ScalaDefinitionCoordinates = { val (methodPrefix, methodName) = splitMethodName(definitionName) ScalaDefinitionCoordinates( providerPackageParts = providerPackageParts, @@ -56,7 +56,7 @@ case class PulumiDefinitionCoordinates private ( selectionName = Some(mangleMethodName(methodName)) ) } - def methodArgsClass(implicit logger: Logger): ScalaDefinitionCoordinates = { + def asFunctionArgsClass(using Logger): ScalaDefinitionCoordinates = { val (methodPrefix, methodName) = splitMethodName(definitionName) ScalaDefinitionCoordinates( providerPackageParts = providerPackageParts, @@ -64,7 +64,7 @@ case class PulumiDefinitionCoordinates private ( definitionName = Some((methodPrefix.toSeq :+ mangleTypeName(methodName)).mkString("") + "Args") ) } - def methodResultClass(implicit logger: Logger): ScalaDefinitionCoordinates = { + def asFunctionResultClass(using Logger): ScalaDefinitionCoordinates = { val (methodPrefix, methodName) = splitMethodName(definitionName) ScalaDefinitionCoordinates( providerPackageParts = providerPackageParts, @@ -73,7 +73,7 @@ case class PulumiDefinitionCoordinates private ( ) } - def asConfigClass(implicit logger: Logger): ScalaDefinitionCoordinates = { + def asConfigClass(using Logger): ScalaDefinitionCoordinates = { ScalaDefinitionCoordinates( providerPackageParts = providerPackageParts, modulePackageParts = modulePackageParts, diff --git a/codegen/src/PulumiDefinitionCoordinates.test.scala b/codegen/src/PulumiDefinitionCoordinates.test.scala index d4958312..a9421072 100644 --- a/codegen/src/PulumiDefinitionCoordinates.test.scala +++ b/codegen/src/PulumiDefinitionCoordinates.test.scala @@ -1,7 +1,7 @@ package besom.codegen -import besom.codegen.Config.ProviderConfig import besom.codegen.SchemaName +import besom.codegen.Utils.PulumiPackageOps import scala.meta.* import scala.meta.dialects.Scala33 @@ -11,8 +11,6 @@ class PulumiDefinitionCoordinatesTest extends munit.FunSuite { import besom.codegen.metaschema.{Java, Language, Meta, PulumiPackage} - implicit val logger: Logger = new Logger - val schemaDir = os.pwd / ".out" / "schemas" case class Data( @@ -216,8 +214,13 @@ class PulumiDefinitionCoordinatesTest extends munit.FunSuite { tests.foreach { data => test(s"Type: ${data.typeToken}".withTags(data.tags)) { - implicit val providerConfig: ProviderConfig = Config.providersConfigs(data.schemaName) - val coords = PulumiToken(data.typeToken).toCoordinates(data.pulumiPackage) + given config: Config = Config() + given Logger = Logger() + given Config.Provider = config.providers(data.schemaName) + given schemaProvider: SchemaProvider = DownloadingSchemaProvider() + + val packageInfo = schemaProvider.packageInfo(data.pulumiPackage.toPackageMetadata(), data.pulumiPackage) + val coords = PulumiToken(data.typeToken).toCoordinates(packageInfo) data.expected.foreach { case ResourceClassExpectations(fullPackageName, fullyQualifiedTypeRef, filePath, asArgsType) => @@ -236,15 +239,15 @@ class PulumiDefinitionCoordinatesTest extends munit.FunSuite { assertEquals(ec.typeRef.syntax, fullyQualifiedTypeRef) assertEquals(ec.filePath.osSubPath.toString(), filePath) case FunctionClassExpectations(fullPackageName, fullyQualifiedTypeRef, filePath, args, result) => - val m = coords.resourceMethod + val m = coords.asFunctionClass assertEquals(m.packageRef.syntax, fullPackageName) assertEquals(m.typeRef.syntax, fullyQualifiedTypeRef) assertEquals(m.filePath.osSubPath.toString(), filePath) - val ac = coords.methodArgsClass + val ac = coords.asFunctionArgsClass assertEquals(ac.packageRef.syntax, args.fullPackageName) assertEquals(ac.typeRef.syntax, args.fullyQualifiedTypeRef) assertEquals(ac.filePath.osSubPath.toString(), args.filePath) - val rc = coords.methodResultClass + val rc = coords.asFunctionResultClass assertEquals(rc.packageRef.syntax, result.fullPackageName) assertEquals(rc.typeRef.syntax, result.fullyQualifiedTypeRef) assertEquals(rc.filePath.osSubPath.toString(), result.filePath) diff --git a/codegen/src/PulumiPackageInfo.scala b/codegen/src/PulumiPackageInfo.scala index 9c7fd856..120e849b 100644 --- a/codegen/src/PulumiPackageInfo.scala +++ b/codegen/src/PulumiPackageInfo.scala @@ -1,20 +1,255 @@ package besom.codegen -import besom.codegen.metaschema.ConstValue +import besom.codegen.Utils.{FunctionName, PulumiPackageOps, isMethod} +import besom.codegen.metaschema.{FunctionDefinition, *} import besom.codegen.{PackageVersion, SchemaName} +import scala.collection.mutable.ListBuffer +import scala.util.matching.Regex + type EnumInstanceName = String case class PulumiPackageInfo( name: SchemaName, version: PackageVersion, - enumTypeTokens: Set[String], - objectTypeTokens: Set[String], providerTypeToken: String, - resourceTypeTokens: Set[String], moduleToPackageParts: String => Seq[String], providerToPackageParts: String => Seq[String], - enumValueToInstances: Map[PulumiToken, Map[ConstValue, EnumInstanceName]] + enumTypeTokens: Set[String], + objectTypeTokens: Set[String], + resourceTypeTokens: Set[String], + enumValueToInstances: Map[PulumiToken, Map[ConstValue, EnumInstanceName]], + parsedTypes: Map[PulumiDefinitionCoordinates, (TypeDefinition, Boolean)], + parsedResources: Map[PulumiDefinitionCoordinates, (ResourceDefinition, Boolean)], + parsedFunctions: Map[PulumiDefinitionCoordinates, (FunctionDefinition, Boolean)] +)( + private[codegen] val pulumiPackage: PulumiPackage, + private[codegen] val providerConfig: Config.Provider ) { + import PulumiPackageInfo.* + def asPackageMetadata: PackageMetadata = PackageMetadata(name, Some(version)) + + def parseMethods( + resourceDefinition: ResourceDefinition + )(using logger: Logger): Map[FunctionName, (PulumiDefinitionCoordinates, (FunctionDefinition, Boolean))] = { + val (methods, notMethods) = resourceDefinition.methods.toSeq + .sortBy { case (name, _) => name } + .map { case (name, token) => + ( + name, + PulumiDefinitionCoordinates.fromRawToken( + typeToken = token, + moduleToPackageParts = moduleToPackageParts, + providerToPackageParts = providerToPackageParts + ) + ) + } + .map { case (name, definition) => + ( + name, + ( + definition, + parsedFunctions.getOrElse( + definition, + throw TypeMapperError( + s"Function '${definition.token.asString}' defined as a resource method not found in package '${this.name}'" + ) + ) + ) + ) + } + .partition { case (_, (_, (d, _))) => isMethod(d) } + + notMethods.foreach { case (token, _) => + logger.info(s"Method '${token}' was not generated because it was not marked as method") + } + methods.toMap + } +} + +object PulumiPackageInfo { + def from( + pulumiPackage: PulumiPackage, + packageMetadata: PackageMetadata + )(using Logger, Config): PulumiPackageInfo = PreProcessed.from(pulumiPackage, packageMetadata).process + + private[PulumiPackageInfo] case class PreProcessed( + name: SchemaName, + version: PackageVersion, + moduleToPackageParts: String => Seq[String], + providerToPackageParts: String => Seq[String] + )( + private[codegen] val pulumiPackage: PulumiPackage, + private[codegen] val providerConfig: Config.Provider + ): + def parseResources: Map[PulumiDefinitionCoordinates, (ResourceDefinition, Boolean)] = + pulumiPackage.resources.map { case (token, resource) => + val coordinates = PulumiDefinitionCoordinates.fromRawToken( + typeToken = token, + moduleToPackageParts = moduleToPackageParts, + providerToPackageParts = providerToPackageParts + ) + (coordinates, (resource, resource.isOverlay)) + } + + def parseTypes(using Logger): Map[PulumiDefinitionCoordinates, (TypeDefinition, Boolean)] = + pulumiPackage.types.map { case (token, typeRef) => + val coordinates = PulumiDefinitionCoordinates.fromRawToken( + typeToken = token, + moduleToPackageParts = moduleToPackageParts, + providerToPackageParts = providerToPackageParts + ) + (coordinates, (typeRef, typeRef.isOverlay)) + } + + def parseFunctions(using Logger): Map[PulumiDefinitionCoordinates, (FunctionDefinition, Boolean)] = + pulumiPackage.functions.map { case (token, function) => + val coordinates = PulumiDefinitionCoordinates.fromRawToken( + typeToken = token, + moduleToPackageParts = moduleToPackageParts, + providerToPackageParts = providerToPackageParts + ) + (coordinates, (function, function.isOverlay)) + } + + def process(using logger: Logger): PulumiPackageInfo = + // pre-process the package to gather information about types, that are used later during various parts of codegen + // most notable place is TypeMapper.scalaTypeFromTypeUri + val enumTypeTokensBuffer = ListBuffer.empty[String] + val objectTypeTokensBuffer = ListBuffer.empty[String] + val resourceTypeTokensBuffer = ListBuffer.empty[String] + + val enumValueToInstancesBuffer = ListBuffer.empty[(PulumiToken, Map[ConstValue, EnumInstanceName])] + + def valueToInstances(enumDefinition: EnumTypeDefinition): Map[ConstValue, EnumInstanceName] = + enumDefinition.`enum`.map { (valueDefinition: EnumValueDefinition) => + val caseRawName: EnumInstanceName = valueDefinition.name.getOrElse { + valueDefinition.value match { + case StringConstValue(value) => value + case const => throw GeneralCodegenException(s"The name of enum cannot be derived from value ${const}") + } + } + valueDefinition.value -> caseRawName + }.toMap + + // Post-process the tokens to unify them to lower case to circumvent inconsistencies in low quality schemas (e.g. aws) + // This allows us to use case-insensitive matching when looking up tokens + val parsedTypes = parseTypes + parsedTypes.foreach { + case (coordinates, (definition: EnumTypeDefinition, _)) => + enumValueToInstancesBuffer += coordinates.token -> valueToInstances(definition) + + if (enumTypeTokensBuffer.contains(coordinates.token.asLookupKey)) + logger.warn(s"Duplicate enum type token '${coordinates.token.asLookupKey}' in package '${name}'") + enumTypeTokensBuffer += coordinates.token.asLookupKey + case (coordinates, (_: ObjectTypeDefinition, _)) => + if (objectTypeTokensBuffer.contains(coordinates.token.asLookupKey)) + logger.warn(s"Duplicate object type token '${coordinates.token.asLookupKey}' in package '${name}'") + objectTypeTokensBuffer += coordinates.token.asLookupKey + } + + val parsedResources = parseResources + parsedResources.foreach { case (coordinates, (_: ResourceDefinition, _)) => + if (resourceTypeTokensBuffer.contains(coordinates.token.asLookupKey)) + logger.warn(s"Duplicate resource type token '${coordinates.token.asLookupKey}' in package '${name}'") + resourceTypeTokensBuffer += coordinates.token.asLookupKey + } + + PulumiPackageInfo( + name = name, + version = version, + providerTypeToken = s"pulumi:providers:${name}", + moduleToPackageParts = moduleToPackageParts, + providerToPackageParts = providerToPackageParts, + enumTypeTokens = enumTypeTokensBuffer.toSet, + objectTypeTokens = objectTypeTokensBuffer.toSet, + resourceTypeTokens = resourceTypeTokensBuffer.toSet, + enumValueToInstances = enumValueToInstancesBuffer.toMap, + parsedTypes = parsedTypes, + parsedResources = parsedResources, + parsedFunctions = parseFunctions + )(pulumiPackage, providerConfig) + end process + end PreProcessed + private[PulumiPackageInfo] object PreProcessed: + private[PulumiPackageInfo] def from( + pulumiPackage: PulumiPackage, + packageMetadata: PackageMetadata + )(using logger: Logger, config: Config): PreProcessed = { + val originalName = pulumiPackage.name + val overrideName = packageMetadata.name + if (originalName != overrideName) { + logger.warn( + s"Package name mismatch for '$overrideName' != '$originalName', " + + s"will be reconciled - this is fine in tests" + ) + } + val originalVersion = PackageVersion(pulumiPackage.version).orDefault + val overrideVersion = packageMetadata.version.orDefault + if (originalVersion != overrideVersion) { + logger.warn( + s"Package version mismatch for '$overrideName:$overrideVersion' != '${originalVersion}', " + + s"will be reconciled - this is fine in tests" + ) + } + + val reconciledMetadata = pulumiPackage.toPackageMetadata(packageMetadata) + val providerConfig = config.providers(reconciledMetadata.name) + val overrideModuleToPackages = providerConfig.moduleToPackages.view.mapValues { pkg => + pkg.split("\\.").filter(_.nonEmpty).toSeq + }.toMap + + PreProcessed( + name = reconciledMetadata.name, + version = reconciledMetadata.version.orDefault, + moduleToPackageParts = moduleToPackageParts(pulumiPackage, overrideModuleToPackages), + providerToPackageParts = providerToPackageParts + )(pulumiPackage, providerConfig) + } + + // to get all of the package parts, first use the regexp provided by the schema + // then use a language specific mapping, and if everything fails, fallback to slash mapping + private def moduleToPackageParts( + pulumiPackage: PulumiPackage, + overrideModuleToPackages: Map[String, Seq[String]] + ): String => Seq[String] = { (module: String) => + val moduleFormat: Regex = pulumiPackage.meta.moduleFormat.r + module match { + case _ if module.isEmpty => + throw TypeMapperError("Module cannot be empty") + case _ if module == Utils.indexModuleName => Seq(Utils.indexModuleName) + case _ if module == Utils.configModuleName => Seq(Utils.configModuleName) + case moduleFormat(name) => overrideModuleToPackages.withDefault(languageModuleToPackageParts(pulumiPackage))(name) + case _ => + throw TypeMapperError( + s"Cannot parse module portion '$module' with moduleFormat: $moduleFormat" + ) + } + } + + def providerToPackageParts: String => Seq[String] = module => Seq(module) + + private def languageModuleToPackageParts(pulumiPackage: PulumiPackage): String => Seq[String] = { + if (pulumiPackage.language.java.packages.view.nonEmpty) { + pulumiPackage.language.java.packages.view + .mapValues { pkg => + pkg.split("\\.").filter(_.nonEmpty).toSeq + } + .toMap + .withDefault(slashModuleToPackageParts) // fallback to slash mapping + } else { + // use nodejs mapping as a fallback + pulumiPackage.language.nodejs.moduleToPackage.view + .mapValues { pkg => + pkg.split("/").filter(_.nonEmpty).toSeq + } + .toMap + .withDefault(slashModuleToPackageParts) // fallback to slash mapping + } + } + + private def slashModuleToPackageParts: String => Seq[String] = + pkg => pkg.split("/").filter(_.nonEmpty).toSeq + end PreProcessed } diff --git a/codegen/src/PulumiToken.scala b/codegen/src/PulumiToken.scala index eec31e2c..20052c3f 100644 --- a/codegen/src/PulumiToken.scala +++ b/codegen/src/PulumiToken.scala @@ -1,7 +1,5 @@ package besom.codegen -import besom.codegen.metaschema.PulumiPackage - import scala.util.matching.Regex /** The parsed Pulumi type token used in Pulumi schema in a clean, canonical form, that enforces all three parts present @@ -18,18 +16,16 @@ case class PulumiToken private (provider: String, module: String, name: String, def asString: String = PulumiToken.concat(provider, module, name) /** Transform the Pulumi type token to a Pulumi definition coordinates - * @param pulumiPackage + * @param packageInfo * the Pulumi package containing the schema information * @return * the Pulumi definition coordinates for the given Pulumi type token */ - def toCoordinates(pulumiPackage: PulumiPackage): PulumiDefinitionCoordinates = { - import besom.codegen.Utils.PulumiPackageOps - + def toCoordinates(packageInfo: PulumiPackageInfo): PulumiDefinitionCoordinates = { PulumiDefinitionCoordinates.fromToken( typeToken = this, - moduleToPackageParts = pulumiPackage.moduleToPackageParts, - providerToPackageParts = pulumiPackage.providerToPackageParts + moduleToPackageParts = packageInfo.moduleToPackageParts, + providerToPackageParts = packageInfo.providerToPackageParts ) } } diff --git a/codegen/src/ScalaDefinitionCoordinates.scala b/codegen/src/ScalaDefinitionCoordinates.scala index cd497452..57eb44fd 100644 --- a/codegen/src/ScalaDefinitionCoordinates.scala +++ b/codegen/src/ScalaDefinitionCoordinates.scala @@ -64,12 +64,12 @@ case class ScalaDefinitionCoordinates private ( ) } - def filePath(implicit providerConfig: Config.ProviderConfig): FilePath = { + def filePath(using providerConfig: Config.Provider): FilePath = { // we DO NOT remove index from the file path, we add it if necessary val moduleParts = sanitizeParts(modulePackageParts) match { case moduleName :: tail => // HACK: we need to exclude a module it does not compile - if (providerConfig.noncompiledModules.contains(moduleName)) { + if (providerConfig.nonCompiledModules.contains(moduleName)) { println(s"Excluding module: $moduleName") s".${moduleName}" +: tail // A leading dot excludes a directory from scala-cli sources } else { diff --git a/codegen/src/ScalaDefinitionCoordinates.test.scala b/codegen/src/ScalaDefinitionCoordinates.test.scala index 7c912227..66f8a193 100644 --- a/codegen/src/ScalaDefinitionCoordinates.test.scala +++ b/codegen/src/ScalaDefinitionCoordinates.test.scala @@ -4,7 +4,6 @@ import scala.meta.* //noinspection ScalaFileName,TypeAnnotation class ScalaDefinitionCoordinatesTest extends munit.FunSuite { - implicit val providerConfig: Config.ProviderConfig = Config.ProviderConfig() case class Data( providerPackageParts: Seq[String], @@ -56,6 +55,8 @@ class ScalaDefinitionCoordinatesTest extends munit.FunSuite { tests.foreach { data => test(s"Type: ${data.definitionName}".withTags(data.tags.toSet)) { + given Config.Provider = Config().providers(data.providerPackageParts.head) + val coords: ScalaDefinitionCoordinates = ScalaDefinitionCoordinates( providerPackageParts = data.providerPackageParts, modulePackageParts = data.modulePackageParts, diff --git a/codegen/src/SchemaProvider.scala b/codegen/src/SchemaProvider.scala index ff1e965f..54975c32 100644 --- a/codegen/src/SchemaProvider.scala +++ b/codegen/src/SchemaProvider.scala @@ -5,24 +5,21 @@ import besom.codegen.Utils.PulumiPackageOps import besom.codegen.metaschema.* import besom.codegen.{PackageVersion, SchemaFile, SchemaName} -import scala.collection.mutable.ListBuffer - type SchemaFile = os.Path trait SchemaProvider { - def packageInfo(metadata: PackageMetadata, schema: Option[SchemaFile] = None): (PulumiPackage, PulumiPackageInfo) - def packageInfo(metadata: PackageMetadata, pulumiPackage: PulumiPackage): (PulumiPackage, PulumiPackageInfo) + def packageInfo(metadata: PackageMetadata, schema: Option[SchemaFile] = None): PulumiPackageInfo + def packageInfo(metadata: PackageMetadata, pulumiPackage: PulumiPackage): PulumiPackageInfo def dependencies(schemaName: SchemaName, packageVersion: PackageVersion): List[(SchemaName, PackageVersion)] } -class DownloadingSchemaProvider(schemaCacheDirPath: os.Path)(implicit logger: Logger) extends SchemaProvider { - +class DownloadingSchemaProvider(using logger: Logger, config: Config) extends SchemaProvider { private val schemaFileName = "schema.json" private val packageInfos: collection.mutable.Map[(SchemaName, PackageVersion), PulumiPackageInfo] = collection.mutable.Map.empty private def pulumiPackage(metadata: PackageMetadata): (PulumiPackage, PackageMetadata) = { - val schemaFilePath = schemaCacheDirPath / metadata.name / metadata.version.orDefault.asString / schemaFileName + val schemaFilePath = config.schemasDir / metadata.name / metadata.version.orDefault.asString / schemaFileName if (!os.exists(schemaFilePath)) { logger.debug( @@ -69,7 +66,7 @@ class DownloadingSchemaProvider(schemaCacheDirPath: os.Path)(implicit logger: Lo // parse and save the schema using path corrected for the actual name and version for the package val pulumiPackage = PulumiPackage.fromString(schema) val reconciled = pulumiPackage.toPackageMetadata(metadata) - val correctSchemaFilePath = schemaCacheDirPath / reconciled.name / reconciled.version.orDefault.asString / schemaFileName + val correctSchemaFilePath = config.schemasDir / reconciled.name / reconciled.version.orDefault.asString / schemaFileName os.write.over(correctSchemaFilePath, schema, createFolders = true) (pulumiPackage, reconciled) } else { @@ -84,7 +81,7 @@ class DownloadingSchemaProvider(schemaCacheDirPath: os.Path)(implicit logger: Lo // parse and save the schema using path corrected for the actual name and version for the package val pulumiPackage = PulumiPackage.fromFile(schema) val reconciled = pulumiPackage.toPackageMetadata(metadata) - val correctSchemaFilePath = schemaCacheDirPath / reconciled.name / reconciled.version.orDefault.asString / schemaFileName + val correctSchemaFilePath = config.schemasDir / reconciled.name / reconciled.version.orDefault.asString / schemaFileName os.copy.over(schema, correctSchemaFilePath, replaceExisting = true, createFolders = true) (pulumiPackage, metadata) } @@ -92,7 +89,7 @@ class DownloadingSchemaProvider(schemaCacheDirPath: os.Path)(implicit logger: Lo def dependencies(schemaName: SchemaName, packageVersion: PackageVersion): List[(SchemaName, PackageVersion)] = packageInfos.keys.filterNot { case (name, _) => name == schemaName }.toList - def packageInfo(metadata: PackageMetadata, schema: Option[SchemaFile] = None): (PulumiPackage, PulumiPackageInfo) = { + def packageInfo(metadata: PackageMetadata, schema: Option[SchemaFile] = None): PulumiPackageInfo = { val (pulumiPackage, packageMetadata) = (metadata, schema) match { case (m, Some(schema)) => this.pulumiPackage(m, schema) case (m: PackageMetadata, None) => this.pulumiPackage(m) @@ -103,75 +100,15 @@ class DownloadingSchemaProvider(schemaCacheDirPath: os.Path)(implicit logger: Lo def packageInfo( metadata: PackageMetadata, pulumiPackage: PulumiPackage - ): (PulumiPackage, PulumiPackageInfo) = { - val info = packageInfos.getOrElseUpdate( + ): PulumiPackageInfo = { + packageInfos.getOrElseUpdate( (metadata.name, metadata.version.orDefault), reconcilePackageInfo(pulumiPackage, metadata) ) - (pulumiPackage, info) } private def reconcilePackageInfo( pulumiPackage: PulumiPackage, packageMetadata: PackageMetadata - ): PulumiPackageInfo = { - if (pulumiPackage.name != packageMetadata.name) { - logger.warn( - s"Package name mismatch for '${packageMetadata.name}' != '${pulumiPackage.name}', " + - s"will be reconciled - this is fine in tests" - ) - } - - // pre-process the package to gather information about types, that are used later during various parts of codegen - // most notable place is TypeMapper.scalaTypeFromTypeUri - val enumTypeTokensBuffer = ListBuffer.empty[String] - val objectTypeTokensBuffer = ListBuffer.empty[String] - val resourceTypeTokensBuffer = ListBuffer.empty[String] - - val enumValueToInstancesBuffer = ListBuffer.empty[(PulumiToken, Map[ConstValue, EnumInstanceName])] - - def valueToInstances(enumDefinition: EnumTypeDefinition): Map[ConstValue, EnumInstanceName] = - enumDefinition.`enum`.map { (valueDefinition: EnumValueDefinition) => - val caseRawName: EnumInstanceName = valueDefinition.name.getOrElse { - valueDefinition.value match { - case StringConstValue(value) => value - case const => throw GeneralCodegenException(s"The name of enum cannot be derived from value ${const}") - } - } - valueDefinition.value -> caseRawName - }.toMap - - // Post-process the tokens to unify them to lower case to circumvent inconsistencies in low quality schemas (e.g. aws) - // This allows us to use case-insensitive matching when looking up tokens - pulumiPackage.parsedTypes.foreach { - case (coordinates, definition: EnumTypeDefinition) => - enumValueToInstancesBuffer += coordinates.token -> valueToInstances(definition) - - if (enumTypeTokensBuffer.contains(coordinates.token.asLookupKey)) - logger.warn(s"Duplicate enum type token '${coordinates.token.asLookupKey}' in package '${packageMetadata.name}'") - enumTypeTokensBuffer += coordinates.token.asLookupKey - case (coordinates, _: ObjectTypeDefinition) => - if (objectTypeTokensBuffer.contains(coordinates.token.asLookupKey)) - logger.warn(s"Duplicate object type token '${coordinates.token.asLookupKey}' in package '${packageMetadata.name}'") - objectTypeTokensBuffer += coordinates.token.asLookupKey - } - pulumiPackage.parsedResources.foreach { case (coordinates, _: ResourceDefinition) => - if (resourceTypeTokensBuffer.contains(coordinates.token.asLookupKey)) - logger.warn(s"Duplicate resource type token '${coordinates.token.asLookupKey}' in package '${packageMetadata.name}'") - resourceTypeTokensBuffer += coordinates.token.asLookupKey - } - - val reconciledMetadata = pulumiPackage.toPackageMetadata(packageMetadata) - PulumiPackageInfo( - name = reconciledMetadata.name, - version = reconciledMetadata.version.orDefault, - enumTypeTokens = enumTypeTokensBuffer.toSet, - objectTypeTokens = objectTypeTokensBuffer.toSet, - providerTypeToken = pulumiPackage.providerTypeToken, - resourceTypeTokens = resourceTypeTokensBuffer.toSet, - moduleToPackageParts = pulumiPackage.moduleToPackageParts, - providerToPackageParts = pulumiPackage.providerToPackageParts, - enumValueToInstances = enumValueToInstancesBuffer.toMap - ) - } + ): PulumiPackageInfo = PulumiPackageInfo.from(pulumiPackage, packageMetadata) } diff --git a/codegen/src/TypeMapper.scala b/codegen/src/TypeMapper.scala index 86739bef..1a2bba7f 100644 --- a/codegen/src/TypeMapper.scala +++ b/codegen/src/TypeMapper.scala @@ -7,9 +7,8 @@ import scala.meta.* import scala.meta.dialects.Scala33 class TypeMapper( - val defaultPackageInfo: PulumiPackageInfo, - schemaProvider: SchemaProvider -)(implicit logger: Logger) { + val defaultPackageInfo: PulumiPackageInfo +)(using logger: Logger, schemaProvider: SchemaProvider) { import TypeMapper.* private def scalaTypeFromTypeUri( @@ -34,14 +33,13 @@ class TypeMapper( case "" => defaultPackageInfo case s"/${providerName}/v${schemaVersion}/schema.json" => - schemaProvider.packageInfo(PackageMetadata(providerName, schemaVersion))._2 + schemaProvider.packageInfo(PackageMetadata(providerName, schemaVersion)) case s"${protocol}://${host}/${providerName}/v${schemaVersion}/schema.json" => schemaProvider .packageInfo( PackageMetadata(providerName, schemaVersion) .withUrl(s"${protocol}://${host}/${providerName}") ) - ._2 case _ => throw TypeMapperError(s"Unexpected file URI format: ${fileUri}") } diff --git a/codegen/src/TypeMapper.test.scala b/codegen/src/TypeMapper.test.scala index 302beb89..255d6db7 100644 --- a/codegen/src/TypeMapper.test.scala +++ b/codegen/src/TypeMapper.test.scala @@ -9,8 +9,6 @@ import scala.meta.dialects.Scala33 class TypeMapperTest extends munit.FunSuite { import besom.codegen.metaschema.* - implicit val logger: Logger = new Logger - object TestPackageMetadata extends PackageMetadata("test-as-scala-type", Some(PackageVersion.default)) case class Data( @@ -124,12 +122,14 @@ class TypeMapperTest extends munit.FunSuite { tests.foreach { data => test(s"${data.`type`.getClass.getSimpleName}".withTags(data.tags)) { - val schemaProvider = new DownloadingSchemaProvider(schemaCacheDirPath = Config.DefaultSchemasDir) - val (_, packageInfo) = data.metadata match { + given Config = Config() + given Logger = Logger() + given schemaProvider: SchemaProvider = DownloadingSchemaProvider() + val packageInfo = data.metadata match { case m @ TestPackageMetadata => schemaProvider.packageInfo(m, PulumiPackage(name = m.name)) case _ => schemaProvider.packageInfo(data.metadata) } - implicit val tm: TypeMapper = new TypeMapper(packageInfo, schemaProvider) + given TypeMapper = TypeMapper(packageInfo) data.expected.foreach { e => assertEquals(data.`type`.asScalaType(data.asArgsType).syntax, e.scalaType) diff --git a/codegen/src/Utils.scala b/codegen/src/Utils.scala index 660fab35..4ddf3921 100644 --- a/codegen/src/Utils.scala +++ b/codegen/src/Utils.scala @@ -3,7 +3,6 @@ package besom.codegen import besom.codegen.metaschema.* import scala.meta.{Lit, Type} -import scala.util.matching.Regex object Utils { // "index" is a placeholder module for classes that should be in @@ -21,7 +20,8 @@ object Utils { // TODO: Find some workaround to enable passing the remaining arguments val jvmMaxParamsCount = 253 // https://github.com/scala/bug/issues/7324 - type FunctionName = String + type FunctionName = String + type FunctionToken = String implicit class ConstValueOps(constValue: ConstValue) { def asScala: Lit = constValue match { @@ -33,10 +33,10 @@ object Utils { } implicit class TypeReferenceOps(typeRef: TypeReference) { - def asTokenAndDependency(implicit typeMapper: TypeMapper): Vector[(Option[PulumiToken], Option[PackageMetadata])] = + def asTokenAndDependency(using typeMapper: TypeMapper): Vector[(Option[PulumiToken], Option[PackageMetadata])] = typeMapper.findTokenAndDependencies(typeRef) - def asScalaType(asArgsType: Boolean = false)(implicit typeMapper: TypeMapper): Type = + def asScalaType(asArgsType: Boolean = false)(using typeMapper: TypeMapper): Type = try { typeMapper.asScalaType(typeRef, asArgsType) } catch { @@ -46,139 +46,14 @@ object Utils { } implicit class PulumiPackageOps(pulumiPackage: PulumiPackage) { - private def slashModuleToPackageParts: String => Seq[String] = - pkg => pkg.split("/").filter(_.nonEmpty).toSeq - - private def languageModuleToPackageParts: String => Seq[String] = { - if (pulumiPackage.language.java.packages.view.nonEmpty) { - pulumiPackage.language.java.packages.view - .mapValues { pkg => - pkg.split("\\.").filter(_.nonEmpty).toSeq - } - .toMap - .withDefault(slashModuleToPackageParts) // fallback to slash mapping - } else { - // use nodejs mapping as a fallback - pulumiPackage.language.nodejs.moduleToPackage.view - .mapValues { pkg => - pkg.split("/").filter(_.nonEmpty).toSeq - } - .toMap - .withDefault(slashModuleToPackageParts) // fallback to slash mapping - } - } - - private def packageFormatModuleToPackageParts: String => Seq[String] = { (module: String) => - val moduleFormat: Regex = pulumiPackage.meta.moduleFormat.r - module match { - case _ if module.isEmpty => - throw TypeMapperError("Module cannot be empty") - case _ if module == indexModuleName => Seq(indexModuleName) - case _ if module == configModuleName => Seq(configModuleName) - case moduleFormat(name) => languageModuleToPackageParts(name) - case _ => - throw TypeMapperError( - s"Cannot parse module portion '$module' with moduleFormat: $moduleFormat" - ) - } - } - - // to get all of the package parts, first use the regexp provided by the schema - // then use a language specific mapping, and if everything fails, fallback to slash mapping - def moduleToPackageParts: String => Seq[String] = packageFormatModuleToPackageParts - def providerToPackageParts: String => Seq[String] = module => Seq(module) - - def providerTypeToken: String = s"pulumi:providers:${pulumiPackage.name}" - def toPackageMetadata(overrideMetadata: PackageMetadata): PackageMetadata = toPackageMetadata(Some(overrideMetadata)) def toPackageMetadata(overrideMetadata: Option[PackageMetadata] = None): PackageMetadata = { - import PackageVersion.* overrideMetadata match { case Some(d) => PackageMetadata(d.name, PackageVersion(pulumiPackage.version).reconcile(d.version)) case None => PackageMetadata(pulumiPackage.name, PackageVersion(pulumiPackage.version)) } } - - type FunctionToken = String - - private[Utils] def nonOverlayFunctions(implicit logger: Logger): Map[FunctionToken, FunctionDefinition] = { - val (overlays, functions) = pulumiPackage.functions.partition { case (_, d) => d.isOverlay } - overlays.foreach { case (token, _) => - logger.info(s"Function '${token}' was not generated because it was marked as overlay") - } - functions - } - - def parsedFunctions(implicit logger: Logger): Map[PulumiDefinitionCoordinates, FunctionDefinition] = { - nonOverlayFunctions.map { case (token, function) => - val coordinates = PulumiDefinitionCoordinates.fromRawToken( - typeToken = token, - moduleToPackageParts = moduleToPackageParts, - providerToPackageParts = providerToPackageParts - ) - (coordinates, function) - } - } - - def parsedTypes(implicit logger: Logger): Map[PulumiDefinitionCoordinates, TypeDefinition] = { - val (overlays, types) = pulumiPackage.types.partition { case (_, d) => d.isOverlay } - overlays.foreach { case (token, _) => - logger.info(s"Type '${token}' was not generated because it was marked as overlay") - } - types.collect { case (token, typeRef) => - val coordinates = PulumiDefinitionCoordinates.fromRawToken( - typeToken = token, - moduleToPackageParts = moduleToPackageParts, - providerToPackageParts = providerToPackageParts - ) - (coordinates, typeRef) - } - } - - def parsedResources(implicit logger: Logger): Map[PulumiDefinitionCoordinates, ResourceDefinition] = { - val (overlays, resources) = pulumiPackage.resources.partition { case (_, d) => d.isOverlay } - overlays.foreach { case (token, _) => - logger.info(s"Resource '${token}' was not generated because it was marked as overlay") - } - resources.collect { case (token, resource) => - val coordinates = PulumiDefinitionCoordinates.fromRawToken( - typeToken = token, - moduleToPackageParts = moduleToPackageParts, - providerToPackageParts = providerToPackageParts - ) - (coordinates, resource) - } - } - - def parsedMethods( - resourceDefinition: ResourceDefinition - )(implicit logger: Logger): Map[FunctionName, (PulumiDefinitionCoordinates, FunctionDefinition)] = { - val (methods, notMethods) = resourceDefinition.methods.toSeq - .sortBy { case (name, _) => name } - .map { case (name, token) => - ( - name, - ( - PulumiDefinitionCoordinates.fromRawToken( - typeToken = token, - moduleToPackageParts = pulumiPackage.moduleToPackageParts, - providerToPackageParts = pulumiPackage.providerToPackageParts - ), - pulumiPackage.nonOverlayFunctions.getOrElse( - token, - throw TypeMapperError(s"Function '${token}' not found in package '${pulumiPackage.name}'") - ) - ) - ) - } - .partition { case (_, (_, d)) => isMethod(d) } - - notMethods.foreach { case (token, _) => - logger.info(s"Method '${token}' was not generated because it was not marked as method") - } - methods.toMap - } } def isMethod(functionDefinition: FunctionDefinition): Boolean = diff --git a/integration-tests/CoreTests.test.scala b/integration-tests/CoreTests.test.scala index ef7f1d50..ea247157 100644 --- a/integration-tests/CoreTests.test.scala +++ b/integration-tests/CoreTests.test.scala @@ -11,7 +11,7 @@ import scala.concurrent.duration.* class CoreTests extends munit.FunSuite { override val munitTimeout = 5.minutes - implicit val codegenConfig: Config.CodegenConfig = Config.CodegenConfig() + given config: Config = Config() val wd = os.pwd / "integration-tests" diff --git a/integration-tests/integration.scala b/integration-tests/integration.scala index 0d462f10..5e241f8b 100644 --- a/integration-tests/integration.scala +++ b/integration-tests/integration.scala @@ -1,6 +1,5 @@ package besom.integration.common -import besom.codegen.Config.CodegenConfig import besom.codegen.generator.Result import besom.codegen.{Config, PackageMetadata} import munit.{Tag, Test} @@ -303,8 +302,7 @@ object codegen { metadata: PackageMetadata, outputDir: Option[os.RelPath] = None ): generator.Result = { - // noinspection TypeAnnotation - implicit val config = CodegenConfig(outputDir = outputDir) + given Config = Config(outputDir = outputDir) generator.generatePackageSources(metadata) } @@ -313,8 +311,7 @@ object codegen { schema: os.Path, outputDir: Option[os.RelPath] = None ): generator.Result = { - // noinspection TypeAnnotation - implicit val config = CodegenConfig(outputDir = outputDir) + given Config = Config(outputDir = outputDir) generator.generatePackageSources(metadata, Some(schema)) } } diff --git a/scripts/Packages.scala b/scripts/Packages.scala index 36f21aae..fdcfffab 100644 --- a/scripts/Packages.scala +++ b/scripts/Packages.scala @@ -1,7 +1,6 @@ package besom.scripts import besom.codegen.* -import besom.codegen.Config.CodegenConfig import besom.model.SemanticVersion import coursier.error.ResolutionError.CantDownloadModule import org.virtuslab.yaml.* @@ -295,7 +294,7 @@ object Packages: val versionOrLatest = m.version.getOrElse("latest") Progress.report(label = s"${m.name}:${versionOrLatest}") try - implicit val codegenConfig: CodegenConfig = CodegenConfig( + given Config = Config( schemasDir = schemasDir, codegenDir = codegenDir )