diff --git a/Justfile b/Justfile index be4947ac..1e7c330b 100644 --- a/Justfile +++ b/Justfile @@ -261,9 +261,6 @@ clean-test-integration: clean-test-integration-codegen # Runs integration tests for core test-integration-core: publish-local-codegen publish-local-core install-language-plugin publish-local-compiler-plugin - just generate-provider-sdk random 4.13.2 - just publish-local-provider-sdk random 4.13.2 - PULUMI_SCALA_PLUGIN_LOCAL_PATH={{language-plugin-output-dir}} \ scala-cli --power test integration-tests --test-only 'besom.integration.core*' # Runs integration tests for compiler plugin @@ -272,7 +269,6 @@ test-integration-compiler-plugin: publish-local-codegen publish-local-core insta # Runs integration tests for language plugin test-integration-language-plugin: publish-local-codegen publish-local-core install-language-plugin publish-local-compiler-plugin - PULUMI_SCALA_PLUGIN_LOCAL_PATH={{language-plugin-output-dir}} \ scala-cli --power test integration-tests --test-only 'besom.integration.languageplugin*' # Runs integration tests for codegen diff --git a/codegen/src/CodeGen.scala b/codegen/src/CodeGen.scala index cb9eff7e..59724d09 100644 --- a/codegen/src/CodeGen.scala +++ b/codegen/src/CodeGen.scala @@ -8,20 +8,28 @@ import scala.meta.dialects.Scala33 import besom.codegen.metaschema._ import besom.codegen.Utils._ +//noinspection ScalaWeakerAccess +object CodeGen { + val javaVersion = "11" + val scalaVersion = "3.3.1" +} + +//noinspection ScalaWeakerAccess,TypeAnnotation class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMapper, logger: Logger) { + + import CodeGen._ + val commonImportedIdentifiers = Seq( "besom.types.Output", "besom.types.Context" ) - def sourcesFromPulumiPackage(pulumiPackage: PulumiPackage, schemaVersion: String, besomVersion: String): Seq[SourceFile] = { - val scalaSources = ( - sourceFilesForProviderResource(pulumiPackage) ++ - sourceFilesForNonResourceTypes(pulumiPackage) ++ - sourceFilesForCustomResources(pulumiPackage) - ) - - scalaSources ++ Seq( + def sourcesFromPulumiPackage( + pulumiPackage: PulumiPackage, + schemaVersion: String, + besomVersion: String + ): Seq[SourceFile] = + scalaFiles(pulumiPackage) ++ Seq( projectConfigFile( pulumiPackageName = pulumiPackage.name, schemaVersion = schemaVersion, @@ -31,16 +39,18 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa pluginName = pulumiPackage.name, pluginVersion = schemaVersion, pluginDownloadUrl = pulumiPackage.pluginDownloadURL - ), + ) ) - } - def projectConfigFile(pulumiPackageName: String, schemaVersion: String, besomVersion: String): SourceFile = { - // TODO use original package version from the schema as publish.version? + def scalaFiles(pulumiPackage: PulumiPackage): Seq[SourceFile] = + sourceFilesForProviderResource(pulumiPackage) ++ + sourceFilesForNonResourceTypes(pulumiPackage) ++ + sourceFilesForResources(pulumiPackage) + def projectConfigFile(pulumiPackageName: String, schemaVersion: String, besomVersion: String): SourceFile = { val fileContent = - s"""|//> using scala "3.3.0" - |//> using options "-java-output-version:11" + s"""|//> using scala $scalaVersion + |//> using options "-java-output-version:$javaVersion" |//> using options "-skip-by-regex:.*" | |//> using dep "org.virtuslab::besom-core:${besomVersion}" @@ -64,10 +74,14 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa SourceFile(filePath = filePath, sourceCode = fileContent) } - def resourcePluginMetadataFile(pluginName: String, pluginVersion: String, pluginDownloadUrl: Option[String]): SourceFile = { + def resourcePluginMetadataFile( + pluginName: String, + pluginVersion: String, + pluginDownloadUrl: Option[String] + ): SourceFile = { val pluginDownloadUrlJsonValue = pluginDownloadUrl match { case Some(url) => s"\"${url}\"" - case None => "null" + case None => "null" } val fileContent = s"""|{ @@ -80,10 +94,10 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa val filePath = FilePath(Seq("resources", "besom", "api", pluginName, "plugin.json")) SourceFile(filePath = filePath, sourceCode = fileContent) - } + } def sourceFilesForProviderResource(pulumiPackage: PulumiPackage): Seq[SourceFile] = { - val providerName = pulumiPackage.name + val providerName = pulumiPackage.name val providerPackageParts = typeMapper.defaultPackageInfo.moduleToPackageParts(providerName) val typeCoordinates = PulumiTypeCoordinates( providerPackageParts = typeMapper.defaultPackageInfo.moduleToPackageParts(providerName), @@ -98,50 +112,57 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa ) } - def sourceFilesForNonResourceTypes(pulumiPackage: PulumiPackage): Seq[SourceFile] = { val moduleToPackageParts = pulumiPackage.moduleToPackageParts - + pulumiPackage.types.flatMap { case (typeToken, typeDefinition) => val typeCoordinates = typeMapper.parseTypeToken(typeToken, moduleToPackageParts) typeDefinition match { - case enumDef: EnumTypeDefinition => sourceFilesForEnum(typeCoordinates = typeCoordinates, enumDefinition = enumDef) - case objectDef: ObjectTypeDefinition => sourceFilesForObjectType(typeCoordinates = typeCoordinates, objectTypeDefinition = objectDef, typeToken = typeToken) + case enumDef: EnumTypeDefinition => + sourceFilesForEnum(typeCoordinates = typeCoordinates, enumDefinition = enumDef) + case objectDef: ObjectTypeDefinition => + sourceFilesForObjectType( + typeCoordinates = typeCoordinates, + objectTypeDefinition = objectDef, + typeToken = typeToken + ) } }.toSeq } - def sourceFilesForEnum(typeCoordinates: PulumiTypeCoordinates, enumDefinition: EnumTypeDefinition): Seq[SourceFile] = { + def sourceFilesForEnum( + typeCoordinates: PulumiTypeCoordinates, + enumDefinition: EnumTypeDefinition + ): Seq[SourceFile] = { val classCoordinates = typeCoordinates.asEnumClass - val enumClassName = Type.Name(classCoordinates.className).syntax + val enumClassName = Type.Name(classCoordinates.className).syntax val enumClassStringName = Lit.String(classCoordinates.className).syntax val (superclass, valueType) = enumDefinition.`type` match { case BooleanType => ("besom.types.BooleanEnum", "Boolean") case IntegerType => ("besom.types.IntegerEnum", "Int") - case NumberType => ("besom.types.NumberEnum", "Double") - case StringType => ("besom.types.StringEnum", "String") + case NumberType => ("besom.types.NumberEnum", "Double") + case StringType => ("besom.types.StringEnum", "String") } val instances = enumDefinition.`enum`.map { valueDefinition => val caseRawName = valueDefinition.name.getOrElse { valueDefinition.value match { case StringConstValue(value) => value - case const => throw new Exception(s"The name of enum cannot be derived from value ${const}") + case const => throw new Exception(s"The name of enum cannot be derived from value ${const}") } } - val caseName = Term.Name(caseRawName).syntax + val caseName = Term.Name(caseRawName).syntax val caseStringName = Lit.String(caseRawName).syntax - val caseValue = constValueAsCode(valueDefinition.value).syntax + val caseValue = constValueAsCode(valueDefinition.value).syntax val definition = s"""object ${caseName} extends ${enumClassName}(${caseStringName}, ${caseValue})""" - val reference = caseName + val reference = caseName (definition, reference) } - val fileContent = s"""|package ${classCoordinates.fullPackageName} | @@ -155,59 +176,68 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa | ) |""".stripMargin - Seq(SourceFile( - classCoordinates.filePath, - fileContent - )) + Seq( + SourceFile( + classCoordinates.filePath, + fileContent + ) + ) } - def sourceFilesForObjectType(typeCoordinates: PulumiTypeCoordinates, objectTypeDefinition: ObjectTypeDefinition, typeToken: String): Seq[SourceFile] = { + def sourceFilesForObjectType( + typeCoordinates: PulumiTypeCoordinates, + objectTypeDefinition: ObjectTypeDefinition, + typeToken: String + ): Seq[SourceFile] = { val baseClassCoordinates = typeCoordinates.asObjectClass(asArgsType = false) val argsClassCoordinates = typeCoordinates.asObjectClass(asArgsType = true) - + val baseClassName = Type.Name(baseClassCoordinates.className).syntax val argsClassName = Type.Name(argsClassCoordinates.className).syntax - - val baseFileImports = makeImportStatements(commonImportedIdentifiers ++ Seq( - "besom.types.Decoder" - )) - val argsFileImports = makeImportStatements(commonImportedIdentifiers ++ Seq( - "besom.types.Input", - "besom.types.Encoder", - "besom.types.ArgsEncoder" - )) + val baseFileImports = makeImportStatements( + commonImportedIdentifiers ++ Seq( + "besom.types.Decoder" + ) + ) + + val argsFileImports = makeImportStatements( + commonImportedIdentifiers ++ Seq( + "besom.types.Input", + "besom.types.Encoder", + "besom.types.ArgsEncoder" + ) + ) val objectProperties = { val allProperties = objectTypeDefinition.properties.toSeq.sortBy(_._1) val truncatedProperties = - if (allProperties.size <= jvmMaxParamsCount) - allProperties + if (allProperties.size <= jvmMaxParamsCount) allProperties else { logger.warn(s"Object type ${typeToken} has too many properties. Only first ${jvmMaxParamsCount} will be kept") allProperties.take(jvmMaxParamsCount) } - - truncatedProperties.map { - case (propertyName, propertyDefinition) => - val isPropertyRequired = objectTypeDefinition.required.contains(propertyName) - makePropertyInfo( - propertyName = propertyName, - propertyDefinition = propertyDefinition, - isPropertyRequired = objectTypeDefinition.required.contains(propertyName) - ) + + truncatedProperties.map { case (propertyName, propertyDefinition) => + makePropertyInfo( + propertyName = propertyName, + propertyDefinition = propertyDefinition, + isPropertyRequired = objectTypeDefinition.required.contains(propertyName) + ) } } val baseClassParams = objectProperties.map { propertyInfo => - val fieldType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.baseType}]""" else propertyInfo.baseType - Term.Param( - mods = List.empty, + val fieldType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.baseType}]""" else propertyInfo.baseType + Term + .Param( + mods = List.empty, name = propertyInfo.name, decltpe = Some(fieldType), default = None - ).syntax - } + ) + .syntax + } val baseOutputExtensionMethods = objectProperties.map { propertyInfo => val innerType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.baseType}]""" else propertyInfo.baseType q"""def ${propertyInfo.name}: Output[$innerType] = output.map(_.${propertyInfo.name})""".syntax @@ -230,7 +260,8 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa val baseCompanion = if (hasOutputExtensions) { - val classNameTerminator = if (baseClassName.endsWith("_")) " " else "" // colon after underscore would be treated as a part of the name + val classNameTerminator = + if (baseClassName.endsWith("_")) " " else "" // colon after underscore would be treated as a part of the name s"""|object ${baseClassName}${classNameTerminator}: | given outputOps: {} with @@ -244,9 +275,15 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa } else { s"""object $baseClassName""" } - - val argsClass = makeArgsClass(argsClassName = argsClassName, inputProperties = objectProperties, isResource = false, isProvider = false) - val argsCompanion = makeArgsCompanion(argsClassName = argsClassName, inputProperties = objectProperties, isResource = false) + + val argsClass = makeArgsClass( + argsClassName = argsClassName, + inputProperties = objectProperties, + isResource = false, + isProvider = false + ) + val argsCompanion = + makeArgsCompanion(argsClassName = argsClassName, inputProperties = objectProperties, isResource = false) val baseClassFileContent = s"""|package ${baseClassCoordinates.fullPackageName} @@ -261,7 +298,7 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa |""".stripMargin val argsClassFileContent = - s"""|package ${argsClassCoordinates.fullPackageName} + s"""|package ${argsClassCoordinates.fullPackageName} | |${argsFileImports} | @@ -282,24 +319,33 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa ) } - def sourceFilesForCustomResources(pulumiPackage: PulumiPackage): Seq[SourceFile] = { + def sourceFilesForResources(pulumiPackage: PulumiPackage): Seq[SourceFile] = { val moduleToPackageParts = pulumiPackage.moduleToPackageParts - pulumiPackage.resources.collect { case (typeToken, resourceDefinition) if !resourceDefinition.isOverlay => - sourceFilesForResource( - typeCoordinates = typeMapper.parseTypeToken(typeToken, moduleToPackageParts), - resourceDefinition = resourceDefinition, - typeToken = typeToken, - isProvider = false, - ) - }.toSeq.flatten + pulumiPackage.resources + .collect { + case (typeToken, resourceDefinition) if !resourceDefinition.isOverlay => + sourceFilesForResource( + typeCoordinates = typeMapper.parseTypeToken(typeToken, moduleToPackageParts), + resourceDefinition = resourceDefinition, + typeToken = typeToken, + isProvider = false + ) + } + .toSeq + .flatten } - def sourceFilesForResource(typeCoordinates: PulumiTypeCoordinates, resourceDefinition: ResourceDefinition, typeToken: String, isProvider: Boolean): Seq[SourceFile] = { + def sourceFilesForResource( + typeCoordinates: PulumiTypeCoordinates, + resourceDefinition: ResourceDefinition, + typeToken: String, + isProvider: Boolean + ): Seq[SourceFile] = { val baseClassCoordinates = typeCoordinates.asResourceClass(asArgsType = false) val argsClassCoordinates = typeCoordinates.asResourceClass(asArgsType = true) - val baseClassName = Type.Name(baseClassCoordinates.className).syntax - val argsClassName = Type.Name(argsClassCoordinates.className).syntax + val baseClassName = Type.Name(baseClassCoordinates.className).syntax + val argsClassName = Type.Name(argsClassCoordinates.className).syntax val baseFileImports = { val conditionalIdentifiers = @@ -310,7 +356,7 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa val unconditionalIdentifiers = Seq( "besom.ResourceDecoder", "besom.CustomResourceOptions", - "besom.util.NonEmptyString", + "besom.util.NonEmptyString" ) makeImportStatements(commonImportedIdentifiers ++ conditionalIdentifiers ++ unconditionalIdentifiers) } @@ -338,19 +384,19 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa val resourceProperties = { val allProperties = (resourceBaseProperties ++ resourceDefinition.properties.toSeq.sortBy(_._1)) val truncatedProperties = - if (allProperties.size <= jvmMaxParamsCount) - allProperties + if (allProperties.size <= jvmMaxParamsCount) allProperties else { - logger.warn(s"Resource type ${typeToken} has too many properties. Only first ${jvmMaxParamsCount} will be kept") + logger.warn( + s"Resource type ${typeToken} has too many properties. Only first ${jvmMaxParamsCount} will be kept" + ) allProperties.take(jvmMaxParamsCount) } - truncatedProperties.map { - case (propertyName, propertyDefinition) => - makePropertyInfo( - propertyName = propertyName, - propertyDefinition = propertyDefinition, - isPropertyRequired = requiredOutputs.contains(propertyName) - ) + truncatedProperties.map { case (propertyName, propertyDefinition) => + makePropertyInfo( + propertyName = propertyName, + propertyDefinition = propertyDefinition, + isPropertyRequired = requiredOutputs.contains(propertyName) + ) } } @@ -358,36 +404,38 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa val baseClassParams = resourceProperties.map { propertyInfo => val innerType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.baseType}]""" else propertyInfo.baseType - Term.Param( - mods = List.empty, - name = propertyInfo.name, - decltpe = Some(t"""Output[${innerType}]"""), - default = None - ).syntax + Term + .Param( + mods = List.empty, + name = propertyInfo.name, + decltpe = Some(t"""Output[${innerType}]"""), + default = None + ) + .syntax } val baseOutputExtensionMethods = resourceProperties.map { propertyInfo => - val innerType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.baseType}]""" else propertyInfo.baseType + val innerType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.baseType}]""" else propertyInfo.baseType val resultType = t"Output[${innerType}]" q"""def ${propertyInfo.name}: $resultType = output.flatMap(_.${propertyInfo.name})""".syntax } val inputProperties = { val allProperties = resourceDefinition.inputProperties.toSeq.sortBy(_._1) - val truncatedProperties = - if (allProperties.size <= jvmMaxParamsCount) - allProperties + val truncatedProperties = + if (allProperties.size <= jvmMaxParamsCount) allProperties else { - logger.warn(s"Resource type ${typeToken} has too many input properties. Only first ${jvmMaxParamsCount} will be kept") + logger.warn( + s"Resource type ${typeToken} has too many input properties. Only first ${jvmMaxParamsCount} will be kept" + ) allProperties.take(jvmMaxParamsCount) } - truncatedProperties.map { - case (propertyName, propertyDefinition) => - makePropertyInfo( - propertyName = propertyName, - propertyDefinition = propertyDefinition, - isPropertyRequired = requiredInputs.contains(propertyName) - ) + truncatedProperties.map { case (propertyName, propertyDefinition) => + makePropertyInfo( + propertyName = propertyName, + propertyDefinition = propertyDefinition, + isPropertyRequired = requiredInputs.contains(propertyName) + ) } } @@ -411,7 +459,8 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa val argsDefault = if (hasDefaultArgsConstructor) s""" = ${argsClassName}()""" else "" - // the type has to match pulumi's resource type schema, ie kubernetes:core/v1:Pod + // the type has to match Pulumi's resource type schema, e.g. kubernetes:core/v1:Pod + // please make sure, it contains 'index' instead of empty module part if needed val typ = Lit.String(typeToken) val baseCompanion = @@ -421,7 +470,7 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa | name: NonEmptyString, | args: ${argsClassName}${argsDefault}, | opts: CustomResourceOptions = CustomResourceOptions() - | ): Output[$baseClassName] = + | ): Output[$baseClassName] = | ctx.registerResource[$baseClassName, $argsClassName](${typ}, name, args, opts) | | given outputOps: {} with @@ -431,9 +480,15 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa s"""object $baseClassName""" } - val argsClass = makeArgsClass(argsClassName = argsClassName, inputProperties = inputProperties, isResource = true, isProvider = isProvider) + val argsClass = makeArgsClass( + argsClassName = argsClassName, + inputProperties = inputProperties, + isResource = true, + isProvider = isProvider + ) - val argsCompanion = makeArgsCompanion(argsClassName = argsClassName, inputProperties = inputProperties, isResource = true) + val argsCompanion = + makeArgsCompanion(argsClassName = argsClassName, inputProperties = inputProperties, isResource = true) val baseClassFileContent = s"""|package ${baseClassCoordinates.fullPackageName} @@ -468,13 +523,17 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa ) } - private def makePropertyInfo(propertyName: String, propertyDefinition: PropertyDefinition, isPropertyRequired: Boolean): PropertyInfo = { + private def makePropertyInfo( + propertyName: String, + propertyDefinition: PropertyDefinition, + isPropertyRequired: Boolean + ): PropertyInfo = { val isRequired = - isPropertyRequired || + isPropertyRequired || propertyDefinition.default.nonEmpty || propertyDefinition.const.nonEmpty val baseType = propertyDefinition.typeReference.asScalaType() - val argType = propertyDefinition.typeReference.asScalaType(asArgsType = true) + val argType = propertyDefinition.typeReference.asScalaType(asArgsType = true) val inputArgType = propertyDefinition.typeReference match { case ArrayType(innerType) => t"""scala.List[Input[${innerType.asScalaType(asArgsType = true)}]]""" @@ -484,10 +543,12 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa tp.asScalaType(asArgsType = true) } val defaultValue = - propertyDefinition.default.orElse(propertyDefinition.const) + propertyDefinition.default + .orElse(propertyDefinition.const) .map { value => constValueAsCode(value) - }.orElse { + } + .orElse { if (isPropertyRequired) None else Some(q"""None""") } val constValue = propertyDefinition.const.map(constValueAsCode) @@ -500,23 +561,30 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa inputArgType = inputArgType, defaultValue = defaultValue, constValue = constValue, - isSecret = propertyDefinition.secret, + isSecret = propertyDefinition.secret ) } - private def makeArgsClass(argsClassName: String, inputProperties: Seq[PropertyInfo], isResource: Boolean, isProvider: Boolean) = { + private def makeArgsClass( + argsClassName: String, + inputProperties: Seq[PropertyInfo], + isResource: Boolean, + isProvider: Boolean + ): String = { val derivedTypeclasses = if (isProvider) "ProviderArgsEncoder" - else if (isResource) "ArgsEncoder" + else if (isResource) "ArgsEncoder" else "Encoder, ArgsEncoder" val argsClassParams = inputProperties.map { propertyInfo => val fieldType = if (propertyInfo.isOptional) t"""Option[${propertyInfo.argType}]""" else propertyInfo.argType - Term.Param( - mods = List.empty, - name = propertyInfo.name, - decltpe = Some(t"""Output[${fieldType}]"""), - default = None - ).syntax + Term + .Param( + mods = List.empty, + name = propertyInfo.name, + decltpe = Some(t"""Output[${fieldType}]"""), + default = None + ) + .syntax } s"""|final case class $argsClassName private( @@ -526,15 +594,19 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa private def makeArgsCompanion(argsClassName: String, inputProperties: Seq[PropertyInfo], isResource: Boolean) = { val argsCompanionApplyParams = inputProperties.filter(_.constValue.isEmpty).map { propertyInfo => - val paramType = if (propertyInfo.isOptional) t"""Input.Optional[${propertyInfo.inputArgType}]""" else t"""Input[${propertyInfo.inputArgType}]""" - Term.Param( - mods = List.empty, - name = propertyInfo.name, - decltpe = Some(paramType), - default = propertyInfo.defaultValue, - ).syntax + val paramType = + if (propertyInfo.isOptional) t"""Input.Optional[${propertyInfo.inputArgType}]""" + else t"""Input[${propertyInfo.inputArgType}]""" + Term + .Param( + mods = List.empty, + name = propertyInfo.name, + decltpe = Some(paramType), + default = propertyInfo.defaultValue + ) + .syntax } - + val argsCompanionApplyBodyArgs = inputProperties.map { propertyInfo => val isSecret = Lit.Boolean(propertyInfo.isSecret) val argValue = propertyInfo.constValue match { @@ -584,12 +656,12 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa "getClass", "hashCode", "isInstanceOf", - "toString", + "toString" ) // This logic must be undone the same way in codecs // Keep in sync with `unmanglePropertyName` in codecs.scala - private def manglePropertyName(name: String)(implicit logger: Logger): String = + private def manglePropertyName(name: String)(implicit logger: Logger): String = if (anyRefMethodNames.contains(name)) { val mangledName = name + "_" logger.warn(s"Mangled property name '$name' as '$mangledName'") diff --git a/codegen/src/Main.scala b/codegen/src/Main.scala index 0ce33358..92770144 100644 --- a/codegen/src/Main.scala +++ b/codegen/src/Main.scala @@ -1,52 +1,87 @@ package besom.codegen +import besom.codegen.SchemaProvider.{ProviderName, SchemaVersion} + object Main { def main(args: Array[String]): Unit = { - if (args(0) == "test") - args.drop(1).toList match { - case schemaPath :: outputPath :: providerName :: schemaVersion :: besomVersion :: Nil => - generateTestPackageSources( - schemaPath = os.Path(schemaPath), - outputPath = os.Path(outputPath), - providerName = providerName, - schemaVersion = schemaVersion, - besomVersion = besomVersion - ) - sys.exit(0) - case _ => - System.err.println("Codegen's expected test arguments: ") - sys.exit(1) - } - args.toList match { case schemasDirPath :: outputDirBasePath :: providerName :: schemaVersion :: besomVersion :: Nil => generatePackageSources( - schemasDirPath = os.Path(schemasDirPath), - outputDirBasePath = os.Path(outputDirBasePath), + schemasDir = os.Path(schemasDirPath), + codegenDir = os.Path(outputDirBasePath), providerName = providerName, schemaVersion = schemaVersion, besomVersion = besomVersion ) case _ => - System.err.println("Codegen's expected arguments: ") + System.err.println( + "Codegen's expected arguments: " + ) sys.exit(1) } } - def generatePackageSources(schemasDirPath: os.Path, outputDirBasePath: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = { - val schemaProvider = new DownloadingSchemaProvider(schemaCacheDirPath = schemasDirPath) - val destinationDir = outputDirBasePath / providerName / schemaVersion + // noinspection ScalaWeakerAccess + def generatePackageSources( + schemasDir: os.Path, + codegenDir: os.Path, + providerName: String, + schemaVersion: String, + besomVersion: String, + preLoadSchemas: Map[(ProviderName, SchemaVersion), os.Path] = Map() + ): os.Path = { + implicit val providerConfig: Config.ProviderConfig = Config.providersConfigs(providerName) + implicit val logger: Logger = new Logger + + val schemaProvider = new DownloadingSchemaProvider(schemaCacheDirPath = schemasDir) + val outputDir = codegenDir / providerName / schemaVersion + + // Print diagnostic information + val preLoadInfo = if (preLoadSchemas.nonEmpty) { + val preloadList = preLoadSchemas + .map { case ((name, version), path) => + s" - $name:$version -> ${path.relativeTo(os.pwd)}" + } + .mkString("\n") + s"""| - Pre-load schemas: + |$preloadList + |""".stripMargin + } else { + "" + } + println( + s"""|Generating package '$providerName:$schemaVersion' into '${outputDir.relativeTo(os.pwd)}' + | - Besom version : $besomVersion + | - Scala version : ${CodeGen.scalaVersion} + | - Java version : ${CodeGen.javaVersion} + |""".stripMargin + preLoadInfo + ) + + // Pre-load schemas from files if needed + preLoadSchemas.foreach { case ((name, version), path) => + schemaProvider.addSchemaFile(name, version, path) + } + + generatePackageSources(schemaProvider, outputDir, providerName, schemaVersion, besomVersion) + println(s"Finished generating provider '$providerName' codebase") - generatePackageSources(schemaProvider, destinationDir, providerName, schemaVersion, besomVersion) + outputDir } - def generatePackageSources(schemaProvider: SchemaProvider, destinationDir: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = { - println(s"Generating provider SDK for $providerName") - + private def generatePackageSources( + schemaProvider: SchemaProvider, + outputDir: os.Path, + providerName: String, + schemaVersion: String, + besomVersion: String + )(implicit logger: Logger, providerConfig: Config.ProviderConfig): Unit = { val pulumiPackage = schemaProvider.pulumiPackage(providerName = providerName, schemaVersion = schemaVersion) - - implicit val providerConfig: Config.ProviderConfig = Config.providersConfigs(providerName) - implicit val logger: Logger = new Logger + println( + s"""|Loaded package: ${pulumiPackage.name} ${pulumiPackage.version.getOrElse("")} + | - Resources: ${pulumiPackage.resources.size} + | - Types : ${pulumiPackage.types.size} + |""".stripMargin + ) implicit val typeMapper: TypeMapper = new TypeMapper( defaultProviderName = providerName, @@ -56,32 +91,29 @@ object Main { ) // make sure we don't have a dirty state - os.remove.all(destinationDir) - os.makeDir.all(destinationDir) + os.remove.all(outputDir) + os.makeDir.all(outputDir) val codeGen = new CodeGen try { - codeGen.sourcesFromPulumiPackage( - pulumiPackage, - schemaVersion = schemaVersion, - besomVersion = besomVersion - ).foreach { sourceFile => - val filePath = destinationDir / sourceFile.filePath.osSubPath - os.makeDir.all(filePath / os.up) - os.write(filePath, sourceFile.sourceCode) - } + codeGen + .sourcesFromPulumiPackage( + pulumiPackage, + schemaVersion = schemaVersion, + besomVersion = besomVersion + ) + .foreach { sourceFile => + val filePath = outputDir / sourceFile.filePath.osSubPath + os.makeDir.all(filePath / os.up) + os.write(filePath, sourceFile.sourceCode, createFolders = true) + } println("Finished generating SDK codebase") } finally { if (logger.nonEmpty) { - val logFile = destinationDir / ".codegen-log.txt" + val logFile = outputDir / ".codegen-log.txt" println(s"Some problems were encountered during the code generation. See ${logFile}") logger.writeToFile(logFile) } } } - - def generateTestPackageSources(schemaPath: os.Path, outputPath: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = { - val schemaProvider = new TestSchemaProvider(schemaPath) - generatePackageSources(schemaProvider, outputPath, providerName, schemaVersion, besomVersion) - } } diff --git a/codegen/src/SchemaProvider.scala b/codegen/src/SchemaProvider.scala index c36acf20..298fc6b8 100644 --- a/codegen/src/SchemaProvider.scala +++ b/codegen/src/SchemaProvider.scala @@ -1,7 +1,5 @@ package besom.codegen -import besom.codegen.SchemaProvider.{ProviderName, SchemaVersion} - import scala.collection.mutable.ListBuffer import besom.codegen.metaschema._ import besom.codegen.Utils.PulumiPackageOps @@ -25,10 +23,12 @@ trait SchemaProvider { val enumTypeTokensBuffer = ListBuffer.empty[String] val objectTypeTokensBuffer = ListBuffer.empty[String] + // post-process the package to improve its quality pulumiPackage.types.foreach { case (typeToken, _: EnumTypeDefinition) => enumTypeTokensBuffer += typeToken.toLowerCase // Unifying to lower case to circumvent inconsistencies in low quality schemas (e.g. aws) - case (typeToken, _: ObjectTypeDefinition) => objectTypeTokensBuffer += typeToken.toLowerCase + case (typeToken, _: ObjectTypeDefinition) => + objectTypeTokensBuffer += typeToken.toLowerCase } PulumiPackageInfo( @@ -41,14 +41,8 @@ trait SchemaProvider { } } -class TestSchemaProvider(schemaPath: os.Path) extends SchemaProvider { - override def pulumiPackage(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackage = - pulumiPackage(schemaPath) - override def packageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo = - loadPackageInfo(pulumiPackage(schemaPath)) -} - class DownloadingSchemaProvider(schemaCacheDirPath: os.Path) extends SchemaProvider { + import SchemaProvider._ private val packageInfos: collection.mutable.Map[(ProviderName, SchemaVersion), PulumiPackageInfo] = @@ -58,25 +52,36 @@ class DownloadingSchemaProvider(schemaCacheDirPath: os.Path) extends SchemaProvi val schemaFilePath = schemaCacheDirPath / providerName / schemaVersion / "schema.json" if (!os.exists(schemaFilePath)) { - val schemaSource = s"${providerName}@${schemaVersion}" + val schemaSource = s"$providerName@$schemaVersion" os.makeDir.all(schemaFilePath / os.up) - os.proc("pulumi", "plugin", "install", "resource", providerName, schemaVersion).call() os.proc("pulumi", "package", "get-schema", schemaSource).call(stdout = schemaFilePath) } schemaFilePath } + def addSchemaFile(providerName: ProviderName, schemaVersion: SchemaVersion, content: os.Path): os.Path = { + val schemaFilePath = schemaCacheDirPath / providerName / schemaVersion / "schema.json" + os.copy.over(content, schemaFilePath, replaceExisting = true, createFolders = true) + schemaFilePath + } + + def addSchemaString(providerName: ProviderName, schemaVersion: SchemaVersion, content: String): os.Path = { + val schemaFilePath = schemaCacheDirPath / providerName / schemaVersion / "schema.json" + os.write.over(schemaFilePath, content, createFolders = true) + schemaFilePath + } + def pulumiPackage(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackage = pulumiPackage(downloadedSchemaFilePath(providerName, schemaVersion)) - private def loadPackageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo = - loadPackageInfo(pulumiPackage(providerName, schemaVersion)) - def packageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo = { packageInfos.getOrElseUpdate( (providerName, schemaVersion), loadPackageInfo(providerName, schemaVersion) ) } + + private def loadPackageInfo(providerName: ProviderName, schemaVersion: SchemaVersion): PulumiPackageInfo = + loadPackageInfo(pulumiPackage(providerName, schemaVersion)) } diff --git a/integration-tests/CodegenTests.test.scala b/integration-tests/CodegenTests.test.scala index e8b912d0..91a58ea1 100644 --- a/integration-tests/CodegenTests.test.scala +++ b/integration-tests/CodegenTests.test.scala @@ -21,28 +21,6 @@ class CodegenTests extends munit.FunSuite { val testdata = os.pwd / "integration-tests" / "resources" / "testdata" - def codegen( - schemaPath: os.Path, - codegenOutputDir: os.Path, - schemaName: String, - schemaVersion: String, - besomVersion: String - ): os.proc = - pproc( - "scala-cli", - "run", - "codegen", - "--suppress-experimental-feature-warning", - "--suppress-directives-in-multiple-files-warning", - "--", - "test", - schemaPath, - codegenOutputDir, - schemaName, - schemaVersion, - besomVersion - ) - val slowFileList = List( "docker" ) @@ -119,13 +97,8 @@ class CodegenTests extends munit.FunSuite { case _ => name } test(options) { - val outputDir = codegenDir / data.name / "0.0.0" - val result = codegen(data.schema, outputDir, data.name, "0.0.0", coreVersion).call(check = false) - val output = result.out.text() - assert(output.contains("Finished generating SDK codebase"), s"Output:\n$output\n") - assert(result.exitCode == 0) - - val compiled = scalaCli.compile(outputDir).call(check = false) + val outputDir = codegen.generatePackageFromSchema(data.schema, data.name, "0.0.0") + val compiled = scalaCli.compile(outputDir).call(check = false) assert { clue(data) compiled.exitCode == 0 diff --git a/integration-tests/CoreTests.test.scala b/integration-tests/CoreTests.test.scala index 77056d0d..76a243ed 100644 --- a/integration-tests/CoreTests.test.scala +++ b/integration-tests/CoreTests.test.scala @@ -21,11 +21,14 @@ class CoreTests extends munit.FunSuite { FunFixture[pulumi.FixtureContext]( setup = { + val schemaName = "random" + val providerSource = codegen.generatePackage(schemaName, providerRandomSchemaVersion) + scalaCli.publishLocal(providerSource) pulumi.fixture.setup( wd / "resources" / "random-example", projectFiles = Map( "project.scala" -> - (defaultProjectFile + s"""//> using dep org.virtuslab::besom-random:$providerRandomVersion""") + (defaultProjectFile + s"""//> using dep org.virtuslab::besom-$schemaName:$providerRandomVersion""") ) ) }, diff --git a/integration-tests/LanguagePluginTest.test.scala b/integration-tests/LanguagePluginTest.test.scala index 4d8cc054..95a08b60 100644 --- a/integration-tests/LanguagePluginTest.test.scala +++ b/integration-tests/LanguagePluginTest.test.scala @@ -15,7 +15,7 @@ class LanguagePluginTest extends munit.FunSuite { val wd = os.pwd / "integration-tests" val resourcesDir = wd / "resources" val executorsDir = resourcesDir / "executors" - val bootstrapLibJarPath = scalaPluginLocalPath.get / "bootstrap.jar" + val bootstrapLibJarPath = languagePluginDir / "bootstrap.jar" val projectFile = s"""|//> using scala $scalaVersion diff --git a/integration-tests/integration.scala b/integration-tests/integration.scala index b207aa2c..208dee07 100644 --- a/integration-tests/integration.scala +++ b/integration-tests/integration.scala @@ -11,11 +11,12 @@ val javaVersion = "11" val scalaVersion = "3.3.1" val coreVersion = os.read(os.pwd / "version.txt").trim val scalaPluginVersion = coreVersion -val scalaPluginLocalPath = envVar("PULUMI_SCALA_PLUGIN_LOCAL_PATH").map(os.Path(_)) val providerRandomSchemaVersion = "4.13.2" val providerRandomVersion = s"$providerRandomSchemaVersion-core.$coreVersion" +val schemaDir = os.pwd / ".out" / "schemas" val codegenDir = os.pwd / ".out" / "codegen" +val languagePluginDir = os.pwd / ".out" / "language-plugin" val defaultProjectFile = s"""|//> using scala $scalaVersion @@ -72,7 +73,7 @@ object pulumi { "scala", scalaPluginVersion, "--file", - scalaPluginLocalPath.get, + languagePluginDir, "--reinstall" ) @@ -122,6 +123,7 @@ object pulumi { object scalaCli { def compile(additional: os.Shellable*) = pproc( "scala-cli", + "--power", "compile", "--suppress-experimental-feature-warning", "--suppress-directives-in-multiple-files-warning", @@ -139,6 +141,38 @@ object scalaCli { ) } +object codegen { + import besom.codegen.Main.generatePackageSources + + def generatePackage( + providerName: String, + schemaVersion: String, + schemas: os.Path = schemaDir, + codegen: os.Path = codegenDir, + version: String = coreVersion + ): os.Path = + println(s"Generating package '$providerName' form schema") + generatePackageSources(schemas, codegen, providerName, schemaVersion, version) + + def generatePackageFromSchema( + schema: os.Path, + providerName: String, + schemaVersion: String, + schemas: os.Path = schemaDir, + codegen: os.Path = codegenDir, + version: String = coreVersion + ): os.Path = + println(s"Generating test package '$providerName' form schema: ${schema.relativeTo(os.pwd)}") + generatePackageSources( + schemas, + codegen, + providerName, + schemaVersion, + version, + Map((providerName, schemaVersion) -> schema) + ) +} + def pproc(command: Shellable*) = { val cmd = os.proc(command) println(cmd.commandChunks.mkString(" ")) diff --git a/integration-tests/project.scala b/integration-tests/project.scala index 2c666679..ea5f124d 100644 --- a/integration-tests/project.scala +++ b/integration-tests/project.scala @@ -3,6 +3,6 @@ //> using exclude "*/resources/*" -//> using toolkit latest +//> using toolkit 0.2.1 //> using dep org.virtuslab:besom-codegen_2.13:0.1.1-SNAPSHOT //> using dep org.scalameta::munit::1.0.0-M10