perf: improve preprocess action perf by capturing context instead of partially resolving template strings

core/.gitignore

@@ -4,7 +4,6 @@ dist/
 src/**/*.js
 src/**/*.map
 src/**/*.d.ts
-!src/template-string/parser.d.ts
 support/**/*.js
 support/**/*.map
 support/**/*.d.ts

core/package.json

@@ -39,10 +39,10 @@
     "@opentelemetry/sdk-node": "^0.57.0",
     "@opentelemetry/sdk-trace-base": "^1.27.0",
     "@opentelemetry/semantic-conventions": "^1.21.0",
-    "@trpc/client": "^11.0.0-next-beta.308",
-    "@trpc/server": "^11.0.0-next-beta.308",
     "@scg82/exit-hook": "^3.4.1",
     "@segment/analytics-node": "^2.2.0",
+    "@trpc/client": "^11.0.0-next-beta.308",
+    "@trpc/server": "^11.0.0-next-beta.308",
     "@types/ws": "^8.5.10",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
@@ -52,7 +52,6 @@
     "chalk": "^5.3.0",
     "chokidar": "^3.6.0",
     "ci-info": "^4.1.0",
-    "cli-highlight": "^2.1.11",
     "cli-table3": "^0.6.5",
     "cli-truncate": "^4.0.0",
     "cpy": "^11.1.0",
@@ -251,11 +250,11 @@
     "utility-types": "^3.11.0"
   },
   "scripts": {
-    "build": "mkdir -p build/src/template-string && peggy src/template-string/parser.pegjs --output build/src/template-string/parser.js --format es",
+    "build": "mkdir -p build/src/template && peggy src/template/parser.pegjs --output build/src/template/parser.js --format es",
     "check-package-lock": "git diff-index --quiet HEAD -- package-lock.json || (echo 'package-lock.json is dirty!' && exit 1)",
     "check-types": "tsc -p . --noEmit",
     "clean": "shx rm -rf build",
-    "dev": "nodemon --watch src/template-string/*.pegjs --exec npm run build",
+    "dev": "nodemon --watch src/template/*.pegjs --exec npm run build",
     "fix-format": "npm run lint -- --fix --quiet",
     "lint": "eslint --ignore-pattern 'src/lib/**' --ext .ts,.tsx src/ test/",
     "migration:generate": "typeorm migration:generate --config ormconfig.js -n",

core/src/actions/base.ts

@@ -8,7 +8,7 @@
 
 import titleize from "titleize"
 import type { ConfigGraph, GetActionOpts, PickTypeByKind, ResolvedConfigGraph } from "../graph/config-graph.js"
-import type { ActionReference, DeepPrimitiveMap } from "../config/common.js"
+import type { ActionReference } from "../config/common.js"
 import {
   createSchema,
   includeGuideLink,
@@ -34,7 +34,6 @@ import { actionOutputsSchema } from "../plugin/handlers/base/base.js"
 import type { GraphResult, GraphResults } from "../graph/results.js"
 import type { RunResult } from "../plugin/base.js"
 import { Memoize } from "typescript-memoize"
-import cloneDeep from "fast-copy"
 import { flatten, fromPairs, isString, memoize, omit, sortBy } from "lodash-es"
 import { ActionConfigContext, ActionSpecContext } from "../config/template-contexts/actions.js"
 import { relative } from "path"
@@ -67,6 +66,10 @@ import type { LinkedSource } from "../config-store/local.js"
 import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js"
 import { styles } from "../logger/styles.js"
 import { dirname } from "node:path"
+import type { ConfigContext } from "../config/template-contexts/base.js"
+import type { ResolvedTemplate } from "../template/types.js"
+import type { WorkflowConfig } from "../config/workflow.js"
+import type { VariablesContext } from "../config/template-contexts/variables.js"
 
 // TODO: split this file
 
@@ -370,7 +373,7 @@ export abstract class BaseAction<
   protected readonly projectRoot: string
   protected readonly _supportedModes: ActionModes
   protected readonly _treeVersion: TreeVersion
-  protected readonly variables: DeepPrimitiveMap
+  protected readonly variables: VariablesContext
 
   constructor(protected readonly params: ActionWrapperParams<C>) {
     this.kind = params.config.kind
@@ -583,7 +586,7 @@ export abstract class BaseAction<
     }
   }
 
-  getVariables(): DeepPrimitiveMap {
+  getVariablesContext(): VariablesContext {
     return this.variables
   }
 
@@ -598,7 +601,7 @@ export abstract class BaseAction<
   getConfig(): C
   getConfig<K extends keyof C>(key: K): C[K]
   getConfig(key?: keyof C["spec"]) {
-    return cloneDeep(key ? this._config[key] : this._config)
+    return key ? this._config[key] : this._config
   }
 
   /**
@@ -735,7 +738,9 @@ export interface ResolvedActionExtension<
 
   getOutputs(): StaticOutputs
 
-  getVariables(): DeepPrimitiveMap
+  getVariablesContext(): VariablesContext
+
+  getResolvedVariables(): Record<string, ResolvedTemplate>
 }
 
 // TODO: see if we can avoid the duplication here with ResolvedBuildAction
@@ -806,7 +811,7 @@ export abstract class ResolvedRuntimeAction<
   getSpec(): Config["spec"]
   getSpec<K extends keyof Config["spec"]>(key: K): Config["spec"][K]
   getSpec(key?: keyof Config["spec"]) {
-    return cloneDeep(key ? this._config.spec[key] : this._config.spec)
+    return key ? this._config.spec[key] : this._config.spec
   }
 
   getOutput<K extends keyof StaticOutputs>(key: K): GetOutputValueType<K, StaticOutputs, RuntimeOutputs> {
@@ -816,6 +821,10 @@ export abstract class ResolvedRuntimeAction<
   getOutputs() {
     return this._staticOutputs
   }
+
+  getResolvedVariables(): Record<string, ResolvedTemplate> {
+    return this.params.resolvedVariables
+  }
 }
 
 export interface ExecutedActionExtension<
@@ -898,7 +907,10 @@ export function getSourceAbsPath(basePath: string, sourceRelPath: string) {
   return joinWithPosix(basePath, sourceRelPath)
 }
 
-export function describeActionConfig(config: ActionConfig) {
+export function describeActionConfig(config: ActionConfig | WorkflowConfig) {
+  if (config.kind === "Workflow") {
+    return `${config.kind} ${config.name}`
+  }
   const d = `${config.type} ${config.kind} ${config.name}`
   if (config.internal?.moduleName) {
     return d + ` (from module ${config.internal?.moduleName})`
@@ -944,8 +956,18 @@ export function actionIsDisabled(config: ActionConfig, environmentName: string):
  *   see {@link VcsHandler.getTreeVersion} and {@link VcsHandler.getFiles}.
  * - The description field is just informational, shouldn't affect execution.
  * - The disabled flag is not relevant to the config version, since it only affects execution.
+ * - The variables and varfiles are only relevant if they have an effect on a relevant piece of configuration and thus can be omitted.
  */
-const nonVersionedActionConfigKeys = ["internal", "source", "include", "exclude", "description", "disabled"] as const
+const nonVersionedActionConfigKeys = [
+  "internal",
+  "source",
+  "include",
+  "exclude",
+  "description",
+  "disabled",
+  "variables",
+  "varfiles",
+] as const
 export type NonVersionedActionConfigKey = keyof Pick<BaseActionConfig, (typeof nonVersionedActionConfigKeys)[number]>
 
 export function getActionConfigVersion<C extends BaseActionConfig>(config: C) {

core/src/actions/build.ts

@@ -31,6 +31,7 @@ import { DEFAULT_BUILD_TIMEOUT_SEC } from "../constants.js"
 import { createBuildTask } from "../tasks/build.js"
 import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js"
 import { ResolveActionTask } from "../tasks/resolve-action.js"
+import type { ResolvedTemplate } from "../template/types.js"
 
 export interface BuildCopyFrom {
   build: string
@@ -247,6 +248,10 @@ export class ResolvedBuildAction<
   getOutputs() {
     return this._staticOutputs
   }
+
+  getResolvedVariables(): Record<string, ResolvedTemplate> {
+    return this.params.resolvedVariables
+  }
 }
 
 export class ExecutedBuildAction<

core/src/actions/types.ts

@@ -21,6 +21,8 @@ import type { ValidResultType } from "../tasks/base.js"
 import type { BaseGardenResource, GardenResourceInternalFields } from "../config/base.js"
 import type { LinkedSource } from "../config-store/local.js"
 import type { GardenApiVersion } from "../constants.js"
+import type { ResolvedTemplate } from "../template/types.js"
+import type { VariablesContext } from "../config/template-contexts/variables.js"
 
 // TODO: split this file
 
@@ -63,7 +65,6 @@ export interface BaseActionConfig<K extends ActionKind = ActionKind, T = string,
   // -> No templating is allowed on these.
   internal: GardenResourceInternalFields & {
     groupName?: string
-    resolved?: boolean // Set to true if no resolution is required, e.g. set for actions converted from modules
     treeVersion?: TreeVersion // Set during module resolution to avoid duplicate scanning for Build actions
     // For forwards-compatibility, applied on actions returned from module conversion handlers
     remoteClonePath?: string
@@ -170,7 +171,7 @@ export interface ActionWrapperParams<C extends BaseActionConfig> {
   remoteSourcePath: string | null
   supportedModes: ActionModes
   treeVersion: TreeVersion
-  variables: DeepPrimitiveMap
+  variables: VariablesContext
 }
 
 export interface ResolveActionParams<C extends BaseActionConfig, StaticOutputs extends Record<string, unknown> = any> {
@@ -181,7 +182,8 @@ export interface ResolveActionParams<C extends BaseActionConfig, StaticOutputs e
   spec: C["spec"]
   staticOutputs: StaticOutputs
   inputs: DeepPrimitiveMap
-  variables: DeepPrimitiveMap
+  variables: VariablesContext
+  resolvedVariables: Record<string, ResolvedTemplate>
 }
 
 export type ResolvedActionWrapperParams<

core/src/commands/custom.ts

@@ -7,8 +7,6 @@
  */
 
 import { execa } from "execa"
-import { apply as jsonMerge } from "json-merge-patch"
-import cloneDeep from "fast-copy"
 import { keyBy, mapValues, flatten } from "lodash-es"
 import { parseCliArgs, prepareMinimistOpts } from "../cli/helpers.js"
 import type { Parameter, ParameterObject } from "../cli/params.js"
@@ -21,7 +19,6 @@ import { CustomCommandContext } from "../config/template-contexts/custom-command
 import { validateWithPath } from "../config/validation.js"
 import type { GardenError } from "../exceptions.js"
 import { ConfigurationError, RuntimeError, InternalError, toGardenError } from "../exceptions.js"
-import { resolveTemplateStrings } from "../template-string/template-string.js"
 import { listDirectory, isConfigFilename } from "../util/fs.js"
 import type { CommandParams, CommandResult, PrintHeaderParams } from "./base.js"
 import { Command } from "./base.js"
@@ -32,6 +29,8 @@ import { getBuiltinCommands } from "./commands.js"
 import type { Log } from "../logger/log-entry.js"
 import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation.js"
 import { styles } from "../logger/styles.js"
+import { deepEvaluate } from "../template/evaluate.js"
+import { VariablesContext } from "../config/template-contexts/variables.js"
 
 function convertArgSpec(spec: CustomCommandOption) {
   const params = {
@@ -112,19 +111,18 @@ export class CustomCommandWrapper extends Command {
     // Strip the command name and any specified arguments off the $rest variable
     const rest = removeSlice(parsed._unknown, this.getPath()).slice(Object.keys(this.arguments || {}).length)
 
-    const yamlDoc = this.spec.internal.yamlDoc
-
     // Render the command variables
-    const variablesContext = new CustomCommandContext({ ...garden, args, opts, rest })
-    const commandVariables = resolveTemplateStrings({
-      value: this.spec.variables,
-      context: variablesContext,
-      source: { yamlDoc, path: ["variables"] },
-    })
-    const variables: any = jsonMerge(cloneDeep(garden.variables), commandVariables)
+
+    const variableContext = new CustomCommandContext({ ...garden, args, opts, variables: garden.variables, rest })
 
     // Make a new template context with the resolved variables
-    const commandContext = new CustomCommandContext({ ...garden, args, opts, variables, rest })
+    const commandContext = new CustomCommandContext({
+      ...garden,
+      args,
+      opts,
+      variables: VariablesContext.forCustomCommand(garden, this.spec, variableContext),
+      rest,
+    })
 
     const result: CustomCommandResult = {}
     const errors: GardenError[] = []
@@ -134,10 +132,9 @@ export class CustomCommandWrapper extends Command {
       const startedAt = new Date()
 
       const exec = validateWithPath<CommandResource["exec"]>({
-        config: resolveTemplateStrings({
-          value: this.spec.exec,
+        config: deepEvaluate(this.spec.exec, {
           context: commandContext,
-          source: { yamlDoc, path: ["exec"] },
+          opts: {},
         }),
         schema: customCommandExecSchema(),
         path: this.spec.internal.basePath,
@@ -186,10 +183,9 @@ export class CustomCommandWrapper extends Command {
       const startedAt = new Date()
 
       let gardenCommand = validateWithPath<CommandResource["gardenCommand"]>({
-        config: resolveTemplateStrings({
-          value: this.spec.gardenCommand,
+        config: deepEvaluate(this.spec.gardenCommand, {
           context: commandContext,
-          source: { yamlDoc, path: ["gardenCommand"] },
+          opts: {},
         }),
         schema: customCommandGardenCommandSchema(),
         path: this.spec.internal.basePath,

core/src/commands/get/get-config.ts

@@ -42,6 +42,9 @@ export class GetConfigCommand extends Command<{}, Opts, ConfigDump> {
   override outputsSchema = () =>
     joi.object().keys({
       allEnvironmentNames: joiArray(environmentNameSchema()).required(),
+      allAvailablePlugins: joiArray(joi.string())
+        .description("A list of all plugins available to be used in the provider configuration.")
+        .required(),
       environmentName: environmentNameSchema().required(),
       namespace: joiIdentifier().description("The namespace of the current environment (if applicable)."),
       providers: joiArray(joi.alternatives(providerSchema(), providerConfigBaseSchema())).description(

core/src/commands/get/get-modules.ts

@@ -20,7 +20,7 @@ import { renderTable, dedent, deline } from "../../util/string.js"
 import { relative, sep } from "path"
 import type { Garden } from "../../index.js"
 import type { Log } from "../../logger/log-entry.js"
-import { highlightYaml, safeDumpYaml } from "../../util/serialization.js"
+import { safeDumpYaml } from "../../util/serialization.js"
 import { deepMap } from "../../util/objects.js"
 import { styles } from "../../logger/styles.js"
 
@@ -137,7 +137,7 @@ function logFull(garden: Garden, modules: GardenModule[], log: Log) {
       ${printEmoji("🌱", log)}  Module: ${styles.success(module.name)}
       ${divider}\n
     `)
-    log.info(highlightYaml(yaml))
+    log.info(yaml)
   }
 }
 

core/src/commands/plugins.ts

@@ -7,7 +7,7 @@
  */
 
 import { max, fromPairs, zip } from "lodash-es"
-import { findByName, getNames } from "../util/util.js"
+import { findByName } from "../util/util.js"
 import { dedent, naturalList, renderTable, tablePresets } from "../util/string.js"
 import { ParameterError, toGardenError } from "../exceptions.js"
 import type { Log } from "../logger/log-entry.js"
@@ -25,7 +25,7 @@ const pluginArgs = {
     help: "The name of the plugin, whose command you wish to run.",
     required: false,
     getSuggestions: ({ configDump }) => {
-      return getNames(configDump.providers)
+      return configDump.allAvailablePlugins
     },
   }),
   command: new StringOption({
@@ -65,7 +65,7 @@ export class PluginsCommand extends Command<Args> {
   }
 
   async action({ garden, log, args }: CommandParams<Args>): Promise<CommandResult> {
-    const providerConfigs = garden.getRawProviderConfigs()
+    const providerConfigs = garden.getUnresolvedProviderConfigs()
     const configuredPlugins = providerConfigs.map((p) => p.name)
 
     if (!args.command || !args.plugin) {

core/src/commands/util/fetch-tools.ts

@@ -17,7 +17,11 @@ import { PluginTool } from "../../util/ext-tools.js"
 import { fromPairs, omit, uniqBy } from "lodash-es"
 import { printHeader, printFooter } from "../../logger/util.js"
 import { BooleanParameter } from "../../cli/params.js"
+import type { BaseProviderConfig, Provider } from "../../config/provider.js"
 
+interface PluginConfigWithToolVersion extends BaseProviderConfig {
+  version?: string
+}
 const fetchToolsOpts = {
   "all": new BooleanParameter({
     help: "Fetch all tools for registered plugins, instead of just ones in the current env/project.",
@@ -93,12 +97,21 @@ export class FetchToolsCommand extends Command<{}, FetchToolsOpts> {
     // No need to fetch the same tools multiple times, if they're used in multiple providers
     const deduplicated = uniqBy(tools, ({ tool }) => tool["versionPath"])
 
-    const versionedConfigs = garden.getRawProviderConfigs({ names: ["pulumi", "terraform"], allowMissing: true })
+    const configuredProviders = garden
+      .getUnresolvedProviderConfigs({ names: ["pulumi", "terraform"], allowMissing: true })
+      .map((c) => c.name)
+
+    const resolvedProviderConfigs: Provider<PluginConfigWithToolVersion>[] = []
+    for (const providerName of configuredProviders) {
+      resolvedProviderConfigs.push(
+        await garden.resolveProvider<PluginConfigWithToolVersion>({ name: providerName, log })
+      )
+    }
 
     // If the version of the tool is configured on the provider,
     // download only that version of the tool.
     const toolsNeeded = deduplicated.filter((tool) => {
-      const pluginToolVersion = versionedConfigs.find((p) => p.name === tool.plugin.name)?.version
+      const pluginToolVersion = resolvedProviderConfigs.find((p) => p.name === tool.plugin.name)?.config.version
       const pluginHasVersionConfigured = !!pluginToolVersion
       if (!pluginHasVersionConfigured) {
         return true